it.unimi.dsi.fastutil.doubles.DoubleList Java Examples

public PLMMAlgorithm(final PLInferenceProblem input, final DoubleList skillVector, final IOwnerBasedAlgorithmConfig config) {
	super(config, input);
	this.numRankings = this.getInput().getRankings().size();
	this.numObjects = this.getInput().getNumObjects();
	if (this.numObjects < 2) {
		throw new IllegalArgumentException("Cannot create PL-Algorithm for choice problems with only one option.");
	}
	this.rankings = input.getRankings();
	for (ShortList ranking : this.rankings) {
		if (ranking.size() != this.numObjects) {
			throw new UnsupportedOperationException("This MM implementation only supports full rankings!");
		}
	}
	this.skillVector = skillVector != null ? skillVector : getDefaultSkillVector(this.numObjects);
	this.winVector = this.getWinVector();
}
 

@Test
public void testInnerSegmentGroupBy() {
  // For inner segment case, percentile does not affect the intermediate result
  AggregationGroupByOperator groupByOperator = getOperatorForQuery(getGroupByQuery(0));
  IntermediateResultsBlock resultsBlock = groupByOperator.nextBlock();
  AggregationGroupByResult groupByResult = resultsBlock.getAggregationGroupByResult();
  Assert.assertNotNull(groupByResult);
  Iterator<GroupKeyGenerator.GroupKey> groupKeyIterator = groupByResult.getGroupKeyIterator();
  while (groupKeyIterator.hasNext()) {
    GroupKeyGenerator.GroupKey groupKey = groupKeyIterator.next();
    DoubleList doubleList = (DoubleList) groupByResult.getResultForKey(groupKey, 0);
    Collections.sort(doubleList);
    assertTDigest((TDigest) groupByResult.getResultForKey(groupKey, 1), doubleList);
    assertTDigest((TDigest) groupByResult.getResultForKey(groupKey, 2), doubleList);
  }
}
 

@Override
public IAlgorithmEvent nextWithException() throws InterruptedException, AlgorithmExecutionCanceledException, AlgorithmTimeoutedException, AlgorithmException {
	DoubleList lastSkillVector = null;
	double epsilon = 0.00001;
	double diffToLast = 0;
	do {
		this.skillVector = this.normalizeSkillVector(this.getUpdatedSkillVectorImproved(this.skillVector));
		if (lastSkillVector != null) {
			diffToLast = 0;
			for (int i = 0; i < this.numObjects; i++) {
				diffToLast += Math.abs(this.skillVector.getDouble(i) - lastSkillVector.getDouble(i));
			}
		}
		else {
			diffToLast = Double.MAX_VALUE;
		}
		lastSkillVector = this.skillVector;
	}
	while (diffToLast > epsilon);
	return null;
}
 

final public void ensureScalarVar(String name, int size, double initial, boolean randomize) {
    writeLock.lock();
    try {
        int curSize = scalarVars.get(name).size();
        if (curSize < size) {
            DoubleList toAdd = new DoubleArrayList(size - curSize);
            for (int i=0; i<size - curSize; i++) {
                toAdd.add(0.0);
            }
            initializeDoubleList(toAdd, initial, randomize);
            scalarVars.get(name).addAll(toAdd);
        }
        curSize = readLocks.size();
        SpaceUtilities.fillReadWriteLocks(readLocks, writeLocks, curSize, size);
    } finally {
        writeLock.unlock();
    }
}
 

public void randInitDoubleList(DoubleList doubleList, boolean normalize) {
    int size = doubleList.size();
    double sum = 0.0;
    for (int i=0; i<size; i++) {
        double val;
        if (normalize) {
            val = rand.nextDouble();
        } else {
            val = (rand.nextDouble() - subtract) * multi;
        }
        doubleList.set(i, val);
        if (normalize) {
            sum += val;
        }
    }
    if (normalize) {
        for (int i=0; i<size; i++) {
            doubleList.set(i, doubleList.getDouble(i) / sum);
        }
    }
}
 

public MarketDataList(Symbol symbol, List<LocalDateTime> dateTimeList, DoubleList bidList, DoubleList askList, DoubleList midList) {
    super();
    this.symbol = symbol;
    this.dateTimeList = dateTimeList;
    this.bidList = bidList;
    this.askList = askList;
    this.midList = midList;
}
 

@Test
public void testNextDouble() {
	DoubleList list = DoubleLists.singleton(1.0);
	OfDouble it = list.iterator();
	assertThat(IteratorUtils.next(it).boxed()).hasValue(1.0);
	assertThat(IteratorUtils.next(it).boxed()).isEmpty();
}
 

public GradientBoostingData(LearningData learningData, DoubleList preds,
                            IntList subset, ObjectiveFunction objectiveFunction) {
    this.learningData = learningData;
    this.preds = preds;
    this.objectiveFunction = objectiveFunction;
    this.subset = subset;
}
 

public double boost(BoostedPredictiveModel model, LearningData learnData, LearningData validData) {
    ObjectiveFunction objFunc = model.getObjectiveFunction();
    GradientBoostingMachine gbm = new GradientBoostingMachine(objFunc);
    DoubleList preds = null;
    DoubleList valids = null;
    double objVal = Double.MAX_VALUE;
    int bestIter = 0;
    for (int i=0; i<maxIter; i++) {
        logger.info("Iteration {} learning.", i + 1);
        PredictiveModel component = model.getPredictiveModel();
        LearningMethod method = model.getLearningMethod();
        gbm.boostModel(preds, valids, null, null, component, method, learnData, validData);
        if (validData != null) {
            double obj = gbm.evaluate(valids, component, validData, null);
            if (obj < objVal) {
                objVal = obj;
                bestIter = i;
            }
            logger.info("Iteration {}: {}", i + 1, objVal);
            valids = gbm.boostPrediction(valids, component, validData, null);
        }
        preds = gbm.boostPrediction(preds, component, learnData, null);
        model.addPredictiveModel(component);
    }
    if (objVal != Double.MAX_VALUE) {
        logger.info("The best iteration is {} with objVal {}.", bestIter + 1, objVal);
        model.setBestIteration(bestIter);
    }
    return objVal;
}
 

public static void setStochasticOracles(List<LearningInstance> instances, List<StochasticOracle> oracles,
                                        SVDFeature svdfeaModel, DoubleList preds) {
    for (LearningInstance ins : instances) {
        GBCentLearningInstance centIns = (GBCentLearningInstance) ins;
        SVDFeatureInstance svdfeaIns = centIns.getSvdfeaIns();
        double pred = svdfeaModel.predict(svdfeaIns)[0];
        preds.add(pred);
        StochasticOracle oracle = new StochasticOracle(pred, svdfeaIns.getLabel(),
                svdfeaIns.getWeight());
        oracles.add(oracle);
    }
}
 

public double evaluate(DoubleList preds, PredictiveModel boostModel,
                       LearningData validData, IntList subset) {
    double objVal = 0.0;
    int idx = 0;
    validData.startNewIteration();
    List<LearningInstance> instances;
    while ((instances = validData.getLearningInstance()).size() > 0) {
        List<StochasticOracle> oracles = new ArrayList<>(instances.size());
        for (LearningInstance ins : instances) {
            double modelOutput = 0.0;
            if (preds != null) {
                int subidx = idx;
                if (subset != null) {
                    subidx = subset.getInt(idx);
                }
                modelOutput += (preds.getDouble(subidx));
                idx++;
            }
            modelOutput += boostModel.predict(ins)[0];
            oracles.add(new StochasticOracle(modelOutput, ins.getLabel(), ins.getWeight()));
        }
        oracles = objectiveFunction.wrapOracle(oracles);
        for (StochasticOracle oracle : oracles) {
            objVal += oracle.getObjectiveValue();
        }
    }
    return objVal;
}
 

/**
 * クエリの結果を MarketDataList に変換します
 * ※Symbolが1つの場合のみ
 *
 * @param table
 * @return
 */
private MarketDataList convertToMarketDataList(QTable table) {
    int size = table.getRowsCount();
    Object[] data = table.getData();
    // col0: QDateTime
    QDateTime[] qDateTimes = (QDateTime[]) data[0];
    List<LocalDateTime> dateTimeList = new ArrayList<>(size);
    for (QDateTime qDateTime : qDateTimes) {
        dateTimeList.add(KdbUtility.convertToLocalDateTime(qDateTime));
    }
    // col1: sym
    String ccypair = ((String[]) data[1])[0];
    Symbol symbol = Symbol.valueOf(ccypair);
    // col2: bid
    DoubleList bidList = new DoubleArrayList((double[]) data[2]);
    // col3: ask
    DoubleList askList = new DoubleArrayList((double[]) data[3]);
    // mid
    double[] midArray = new double[size];
    for (int i = 0; i < size; i++) {
        midArray[i] = (bidList.getDouble(i) + askList.getDouble(i)) / 2;
    }
    DoubleList midList = new DoubleArrayList(midArray);

    // create MarketDataList
    MarketDataList marketDataList = new MarketDataList(symbol, dateTimeList, bidList, askList, midList);
    return marketDataList;
}
 

static public MetricResult getTopNResults(String metricName,
                                          List<Integer> N, Double threshold, double minValue,
                                          DoubleList metrics, int cnt) {
    List<ObjectNode> results = new ArrayList<>(N.size());
    boolean pass = true;
    for (int i=0; i<N.size(); i++) {
        ObjectNode result = Json.newObject();
        result.put(ConfigKey.EVALUATOR_METRIC_NAME.get(), metricName);
        ObjectNode metricPara = Json.newObject();
        if (threshold != null) {
            metricPara.put("threshold", threshold);
        }
        metricPara.put("minValue", minValue);
        metricPara.put("N", N.get(i));
        result.set(ConfigKey.EVALUATOR_METRIC_PARA.get(), metricPara);
        double value = 0.0;
        if (cnt > 0) {
            value = metrics.getDouble(i) / cnt;
        }
        result.put(ConfigKey.EVALUATOR_METRIC_VALUE.get(), value);
        result.put(ConfigKey.EVALUATOR_METRIC_SUPPORT.get(), cnt);
        results.add(result);
        if (value < minValue) {
            pass = false;
        }
    }
    return new MetricResult(results, pass);
}
 

public DoubleList getPriceList() {
    switch (priceDataType) {
        case MID_PRICE:
            return midList;
        case BID_PRICE:
            return bidList;
        case ASK_PRICE:
            return askList;
        default:
            throw new ATSRuntimeException("PriceDataType is not set.");
    }
}
 

private static double[] filterDoubles(DoubleSet doubleSet, double[] source) {
  DoubleList doubleList = new DoubleArrayList();
  for (double value : source) {
    if (doubleSet.contains(value)) {
      doubleList.add(value);
    }
  }
  if (doubleList.size() == source.length) {
    return source;
  } else {
    return doubleList.toDoubleArray();
  }
}
 

@Test
public void testInnerSegmentAggregation() {
  // For inner segment case, percentile does not affect the intermediate result
  AggregationOperator aggregationOperator = getOperatorForQuery(getAggregationQuery(0));
  IntermediateResultsBlock resultsBlock = aggregationOperator.nextBlock();
  List<Object> aggregationResult = resultsBlock.getAggregationResult();
  Assert.assertNotNull(aggregationResult);
  Assert.assertEquals(aggregationResult.size(), 3);
  DoubleList doubleList = (DoubleList) aggregationResult.get(0);
  Collections.sort(doubleList);
  assertTDigest((TDigest) aggregationResult.get(1), doubleList);
  assertTDigest((TDigest) aggregationResult.get(2), doubleList);
}
 

private void assertTDigest(TDigest tDigest, DoubleList doubleList) {
  for (int percentile = 0; percentile <= 100; percentile++) {
    double expected;
    if (percentile == 100) {
      expected = doubleList.getDouble(doubleList.size() - 1);
    } else {
      expected = doubleList.getDouble(doubleList.size() * percentile / 100);
    }
    Assert.assertEquals(tDigest.quantile(percentile / 100.0), expected, DELTA, ERROR_MESSAGE);
  }
}
 

final public void requestScalarVar(String name, int size, double initial, boolean randomize) {
    DoubleList var = new DoubleArrayList(size);
    for (int i=0; i<size; i++) {
        var.add(0.0);
    }
    initializeDoubleList(var, initial, randomize);
    writeLock.lock();
    try {
        scalarVars.put(name, var);
    } finally {
        writeLock.unlock();
    }
}
 

@Override
public void signalNewScore(final ILabeledPath<N, A> path, final double newScore) {

	/* add the observation to all stats on the path */
	List<N> nodes = path.getNodes();
	int l = nodes.size();
	for (int i = l - 1; i >= 0; i --) {
		N node = nodes.get(i);
		DoubleList list = this.observations.computeIfAbsent(node, n -> new DoubleArrayList());
		list.add(newScore);
		if (list.size() > this.maxNumSamplesInHistory) {
			list.removeDouble(0);
		}
	}
}
 

public static DoubleList getDefaultSkillVector(final int n) {
	DoubleList skillVector = new DoubleArrayList();
	double p = 1.0 / n;
	for (int i = 0; i < n; i++) {
		skillVector.add(p);
	}
	return skillVector;
}
 

@Override
public DoubleList call() throws InterruptedException, AlgorithmExecutionCanceledException, AlgorithmTimeoutedException, AlgorithmException {
	this.next();
	if (this.skillVector.size() != this.numObjects) {
		throw new IllegalStateException("Have " + this.skillVector.size() + " skills (" + this.skillVector + ") for " + this.numObjects + " objects.");
	}
	for (double d : this.skillVector) {
		if (Double.isNaN(d)) {
			throw new IllegalStateException("Illegal skill return value: " + this.skillVector);
		}
	}
	return this.skillVector;
}
 

private void initializeDoubleList(DoubleList var, double initial, boolean randomize) {
    if (randomize) {
        RandomInitializer randInit = new RandomInitializer();
        randInit.randInitDoubleList(var, false);
    } else {
        if (initial != 0.0) {
            setDoubleList(var, initial);
        }
    }
}
 

final public RealVector getScalarVarByName(String name) {
    readLock.lock();
    try {
        DoubleList var = scalarVars.get(name);
        double[] newVar = new double[var.size()];
        setDoubleList(newVar, var);
        return MatrixUtils.createRealVector(newVar);
    } finally {
        readLock.unlock();
    }
}
 

private void setDoubleList(double[] newVar, DoubleList var) {
    int size = var.size();
    for (int i=0; i<size; i++) {
        newVar[i] = var.getDouble(i);
    }
}
 

private void initializeValidObjs(DoubleList validObjs, int size) {
    for (int i=0; i<size; i++) {
        validObjs.add(0.0);
    }
}
 

private void initializeLearnObjs(DoubleList learnObjs, int size) {
    for (int i=0; i<size; i++) {
        learnObjs.add(0.0);
    }
}
 

private void setDoubleList(DoubleList doubleList, RealVector vars) {
    int size = doubleList.size();
    for (int i=0; i<size; i++) {
        doubleList.set(i, vars.getEntry(i));
    }
}
 

private void setDoubleList(DoubleList doubleList, double value) {
    int size = doubleList.size();
    for (int i=0; i<size; i++) {
        doubleList.set(i, value);
    }
}
 

public LearningInstance featurize(JsonNode entity, boolean update) {
    Map<String, List<Feature>> feaMap = FeaturizerUtilities.getFeatureMap(entity, true,
            featureExtractors, indexSpace);
    if (equalSizeChecks != null) {
        for (List<String> features : equalSizeChecks) {
            int size = -1;
            for (String fea : features) {
                if (size < 0) {
                    if (feaMap.containsKey(fea)) {
                        size = feaMap.get(fea).size();
                    } else {
                        throw new BadRequestException(
                                "Feature " + fea + " is not present in the extracted feature map with keys " +
                                        feaMap.keySet().toString());
                    }
                } else if (size != feaMap.get(fea).size()) {
                    throw new ConfigurationException(
                            "Equal size checks with " + features.toString() +
                                    " failed for " + entity.toString());
                }
            }
        }
    }
    String group = null;
    if (groupKeys != null && groupKeys.size() > 0) {
        group = FeatureExtractorUtilities.composeConcatenatedKey(entity, groupKeys);
    }
    TensorFlowInstance instance = new TensorFlowInstance(group);
    for (Map.Entry<String, List<Feature>> entry : feaMap.entrySet()) {
        DoubleList doubles = new DoubleArrayList();
        IntList ints = new IntArrayList();
        for (Feature feature : entry.getValue()) {
            doubles.add(feature.getValue());
            ints.add(feature.getIndex());
        }
        double[] darr = doubles.toDoubleArray();
        instance.putValues(entry.getKey(), darr);
        int[] iarr = ints.toIntArray();
        instance.putIndices(entry.getKey(), iarr);
    }
    return instance;
}
 

@Override
public DoubleList getValues(Direction.Axis axis) {
    return shape.getValues(axis);
}