org.eclipse.collections.impl.map.mutable.primitive.IntLongHashMap Java Examples

@Override
public Optional<TotalCurrencyBalanceReportResult> process(final MatchingEngineRouter matchingEngine) {

    final IntLongHashMap currencyBalance = new IntLongHashMap();

    matchingEngine.getOrderBooks().stream()
            .filter(ob -> ob.getSymbolSpec().type == SymbolType.CURRENCY_EXCHANGE_PAIR)
            .forEach(ob -> {
                final CoreSymbolSpecification spec = ob.getSymbolSpec();

                currencyBalance.addToValue(
                        spec.getBaseCurrency(),
                        ob.askOrdersStream(false).mapToLong(ord -> CoreArithmeticUtils.calculateAmountAsk(ord.getSize() - ord.getFilled(), spec)).sum());

                currencyBalance.addToValue(
                        spec.getQuoteCurrency(),
                        ob.bidOrdersStream(false).mapToLong(ord -> CoreArithmeticUtils.calculateAmountBidTakerFee(ord.getSize() - ord.getFilled(), ord.getReserveBidPrice(), spec)).sum());
            });

    return Optional.of(TotalCurrencyBalanceReportResult.ofOrderBalances(currencyBalance));
}
 

/**
 * @param trainPointData data to run down trees
 * @param model random decision forest model to count on
 * @return map of predictor index to the number of training examples that reached a
 *  node whose decision is based on that feature. The index is among predictors, not all
 *  features, since there are fewer predictors than features. That is, the index will
 *  match the one used in the {@link RandomForestModel}.
 */
private static IntLongHashMap predictorExampleCounts(JavaRDD<? extends LabeledPoint> trainPointData,
                                                     RandomForestModel model) {
  return trainPointData.mapPartitions(data -> {
      IntLongHashMap featureIndexCount = new IntLongHashMap();
      data.forEachRemaining(datum -> {
        double[] featureVector = datum.features().toArray();
        for (DecisionTreeModel tree : model.trees()) {
          org.apache.spark.mllib.tree.model.Node node = tree.topNode();
          // This logic cloned from Node.predict:
          while (!node.isLeaf()) {
            Split split = node.split().get();
            int featureIndex = split.feature();
            // Count feature
            featureIndexCount.addToValue(featureIndex, 1);
            node = nextNode(featureVector, node, split, featureIndex);
          }
        }
      });
      return Collections.singleton(featureIndexCount).iterator();
  }).reduce(RDFUpdate::merge);
}
 

private void copyPagesAfterCurrent(MasterSyncResult syncResult, int maxPage, long pageSize, long newBase,
        IntLongHashMap pageLocationMap, FastList<FastUnsafeOffHeapPageBuffer> buffers, int currentPage, long newSize)
{
    for(int page=currentPage+1;page<=maxPage;page++)
    {
        long pageFromSyncResult = pageLocationMap.get(page);
        if (pageFromSyncResult == 0)
        {
            throw new RuntimeException("missing page after current page "+page);
        }
        else
        {
            copyBufferPage(syncResult, pageSize, newBase, buffers, page, pageFromSyncResult, newSize);
        }
    }

    int newCurrent = ((maxPage + 1) << PAGE_POWER_OF_TWO) - 1;
    if (newCurrent > this.current)
    {
        this.current = newCurrent;
    }
}
 

private void createUserAccountsRegular(List<BitSet> userCurrencies, IntLongHashMap amountPerAccount) {
    final int numUsers = userCurrencies.size() - 1;

    IntStream.rangeClosed(1, numUsers).forEach(uid -> {
        api.submitCommand(ApiAddUser.builder().uid(uid).build());
        userCurrencies.get(uid).stream().forEach(currency ->
                api.submitCommand(ApiAdjustUserBalance.builder()
                        .uid(uid)
                        .transactionId(getRandomTransactionId())
                        .amount(amountPerAccount.get(currency))
                        .currency(currency)
                        .build()));
    });

    api.submitCommandAsync(ApiNop.builder().build()).join();
}
 

public TotalCurrencyBalanceReportResult totalBalanceReport() {
    final TotalCurrencyBalanceReportResult res = api.processReport(new TotalCurrencyBalanceReportQuery(), getRandomTransferId()).join();
    final IntLongHashMap openInterestLong = res.getOpenInterestLong();
    final IntLongHashMap openInterestShort = res.getOpenInterestShort();
    final IntLongHashMap openInterestDiff = new IntLongHashMap(openInterestLong);
    openInterestShort.forEachKeyValue((k, v) -> openInterestDiff.addToValue(k, -v));
    if (openInterestDiff.anySatisfy(vol -> vol != 0)) {
        throw new IllegalStateException("Open Interest balance check failed");
    }

    return res;
}
 

private static IntLongHashMap merge(IntLongHashMap a, IntLongHashMap b) {
  if (b.size() > a.size()) {
    return merge(b, a);
  }
  b.forEachKeyValue(a::addToValue);
  return a;
}
 

/**
 * @param trainPointData data to run down trees
 * @param model random decision forest model to count on
 * @return maps of node IDs to the count of training examples that reached that node, one
 *  per tree in the model
 * @see #predictorExampleCounts(JavaRDD,RandomForestModel)
 */
private static List<IntLongHashMap> treeNodeExampleCounts(JavaRDD<? extends LabeledPoint> trainPointData,
                                                          RandomForestModel model) {
  return trainPointData.mapPartitions(data -> {
      DecisionTreeModel[] trees = model.trees();
      List<IntLongHashMap> treeNodeIDCounts = IntStream.range(0, trees.length).
          mapToObj(i -> new IntLongHashMap()).collect(Collectors.toList());
      data.forEachRemaining(datum -> {
        double[] featureVector = datum.features().toArray();
        for (int i = 0; i < trees.length; i++) {
          DecisionTreeModel tree = trees[i];
          IntLongHashMap nodeIDCount = treeNodeIDCounts.get(i);
          org.apache.spark.mllib.tree.model.Node node = tree.topNode();
          // This logic cloned from Node.predict:
          while (!node.isLeaf()) {
            // Count node ID
            nodeIDCount.addToValue(node.id(), 1);
            Split split = node.split().get();
            int featureIndex = split.feature();
            node = nextNode(featureVector, node, split, featureIndex);
          }
          nodeIDCount.addToValue(node.id(), 1);
        }
      });
      return Collections.singleton(treeNodeIDCounts).iterator();
    }
  ).reduce((a, b) -> {
      Preconditions.checkArgument(a.size() == b.size());
      for (int i = 0; i < a.size(); i++) {
        merge(a.get(i), b.get(i));
      }
      return a;
    });
}
 

private long copyExistingAndIncomingPagesIntoNew(MasterSyncResult syncResult, int maxPage)
{
    long pageSize = getPageSize();
    long newSize = (int)((maxPage+1)*1.1) * pageSize;
    long newBase = UNSAFE.allocateMemory(newSize);

    IntLongHashMap pageLocationMap = syncResult.getPageLocationMap();
    FastList<FastUnsafeOffHeapPageBuffer> buffers = syncResult.getBuffers();
    int currentPage = this.current >> PAGE_POWER_OF_TWO;
    for(int page=0;page<=currentPage;page++)
    {
        long pageFromSyncResult = pageLocationMap.get(page);
        if (pageFromSyncResult == 0)
        {
            assertedCopyMemory(this.baseAddress + page * pageSize, newBase + page*pageSize, pageSize, this.baseAddress, this.totalAllocated, newBase, newSize);
        }
        else
        {
            copyBufferPage(syncResult, pageSize, newBase, buffers, page, pageFromSyncResult, newSize);
        }
    }
    copyPagesAfterCurrent(syncResult, maxPage, pageSize, newBase, pageLocationMap, buffers, currentPage, newSize);
    long remainder = newSize - (maxPage + 1) * pageSize;
    if (remainder > 0)
    {
        UNSAFE.setMemory(newBase + (maxPage+1)*pageSize, remainder, (byte) 0);
    }
    totalAllocated = newSize;
    return newBase;
}
 

private void copyDataFromBuffers(MasterSyncResult syncResult, FastList<FastUnsafeOffHeapPageBuffer> buffers, IntLongHashMap pageLocationMap, int toCopyDataIndex)
{
    int page = toCopyDataIndex >> PAGE_POWER_OF_TWO;
    int pageDataIndex = toCopyDataIndex & ((1 << PAGE_POWER_OF_TWO) - 1);
    long pageFromSyncResult = pageLocationMap.get(page);
    int pageBufferLocation = syncResult.getPageBufferLocation(pageFromSyncResult);
    int bufferPageIndex = syncResult.getBufferPageIndex(pageFromSyncResult);
    FastUnsafeOffHeapPageBuffer buffer = buffers.get(pageBufferLocation);
    long src = buffer.getPageStartLocation(bufferPageIndex) + pageDataIndex * dataSize;
    long dest = computeAddress(toCopyDataIndex, 0);
    assertedCopyMemory(src + 1, dest + 1, dataSize - 1, buffer.getBaseAddress(), buffer.getAllocatedLength(), this.baseAddress, this.totalAllocated);
    fence++;
    UNSAFE.putByte(dest, UNSAFE.getByte(src)); // we copy the dataVersion field last
}
 

private void checkBufferPages(FastList<FastUnsafeOffHeapPageBuffer> buffers, int pages, IntLongHashMap pageLocationMap)
{
    for(int page=0;page<pages;page++)
    {
        long pageFromSyncResult = pageLocationMap.get(page);
        if (pageFromSyncResult == 0)
        {
            logger.error("Missing page "+page+" in initial sync");
            String msg = "Buffers "+buffers.size()+"\n";
            for(int i=0;i<buffers.size();i++)
            {
                String bufMsg = "buffer "+i+": ";
                FastUnsafeOffHeapIntList pageIndicies = buffers.get(i).getMasterPageIndicies();
                for(int j=0;j<pageIndicies.size();j++)
                {
                    bufMsg+= ", "+pageIndicies.get(j);
                }
                msg += bufMsg+"\n";
            }
            logger.error(msg);
            throw new RuntimeException("Unexpected buffer pages. see above");
        }
    }
    int copied = 0;
    for(int i=0;i<buffers.size();i++)
    {
        FastUnsafeOffHeapPageBuffer buffer = buffers.get(i);
        int pagesToCopy = buffer.getUsedPages();
        assert copied == buffer.getMasterPageIndicies().get(0);
        copied += pagesToCopy;
    }
}
 

private void processIncomingSyncResult(MasterSyncResult syncResult, OffHeapSyncableCache cache, SyncLog syncLog)
{
    if (syncResult.getBuffers().isEmpty())
    {
        return;
    }
    int currentCopy = this.current;
    if (currentCopy == 2)
    {
        processInitialSync(syncResult, cache, syncLog);
    }
    else
    {
        IntLongHashMap pageLocationMap = syncResult.getPageLocationMap();
        FastList<FastUnsafeOffHeapPageBuffer> buffers = syncResult.getBuffers();
        int currentPage = this.current >> PAGE_POWER_OF_TWO;
        int maxPage = currentPage;

        for (IntIterator intIterator = pageLocationMap.keySet().intIterator(); intIterator.hasNext(); )
        {
            int pageIndex = intIterator.next();
            if (pageIndex > maxPage)
            {
                maxPage = pageIndex;
            }
        }
        if (maxPage >= (max >> PAGE_POWER_OF_TWO))
        {
            processDataViaCopy(syncResult, maxPage, cache, syncLog);
        }
        else
        {
            processDataInPlace(syncResult, maxPage, cache, syncLog);
        }
    }
    this.maxReplicatedPageVersion = syncResult.getMaxReplicatedVersion();
    syncLog.setFinalMaxPageVersion(this.maxReplicatedPageVersion);
}
 

public final void userAccountsInit(List<BitSet> userCurrencies) {

        // calculate max amount can transfer to each account so that it is not possible to get long overflow
        final IntLongHashMap accountsNumPerCurrency = new IntLongHashMap();
        userCurrencies.forEach(accounts -> accounts.stream().forEach(currency -> accountsNumPerCurrency.addToValue(currency, 1)));
        final IntLongHashMap amountPerAccount = new IntLongHashMap();
        accountsNumPerCurrency.forEachKeyValue((currency, numAcc) -> amountPerAccount.put(currency, Long.MAX_VALUE / (numAcc + 1)));
        // amountPerAccount.forEachKeyValue((k, v) -> log.debug("{}={}", k, v));

        createUserAccountsRegular(userCurrencies, amountPerAccount);
    }
 

public UserProfile(long uid, UserStatus userStatus) {
    //log.debug("New {}", uid);
    this.uid = uid;
    this.positions = new IntObjectHashMap<>();
    this.adjustmentsCounter = 0L;
    this.accounts = new IntLongHashMap();
    this.userStatus = userStatus;
}
 

@Override
public Optional<TotalCurrencyBalanceReportResult> process(final RiskEngine riskEngine) {

    // prepare fast price cache for profit estimation with some price (exact value is not important, except ask==bid condition)
    final IntObjectHashMap<RiskEngine.LastPriceCacheRecord> dummyLastPriceCache = new IntObjectHashMap<>();
    riskEngine.getLastPriceCache().forEachKeyValue((s, r) -> dummyLastPriceCache.put(s, r.averagingRecord()));

    final IntLongHashMap currencyBalance = new IntLongHashMap();

    final IntLongHashMap symbolOpenInterestLong = new IntLongHashMap();
    final IntLongHashMap symbolOpenInterestShort = new IntLongHashMap();

    final SymbolSpecificationProvider symbolSpecificationProvider = riskEngine.getSymbolSpecificationProvider();

    riskEngine.getUserProfileService().getUserProfiles().forEach(userProfile -> {
        userProfile.accounts.forEachKeyValue(currencyBalance::addToValue);
        userProfile.positions.forEachKeyValue((symbolId, positionRecord) -> {
            final CoreSymbolSpecification spec = symbolSpecificationProvider.getSymbolSpecification(symbolId);
            final RiskEngine.LastPriceCacheRecord avgPrice = dummyLastPriceCache.getIfAbsentPut(symbolId, RiskEngine.LastPriceCacheRecord.dummy);
            currencyBalance.addToValue(positionRecord.currency, positionRecord.estimateProfit(spec, avgPrice));

            if (positionRecord.direction == PositionDirection.LONG) {
                symbolOpenInterestLong.addToValue(symbolId, positionRecord.openVolume);
            } else if (positionRecord.direction == PositionDirection.SHORT) {
                symbolOpenInterestShort.addToValue(symbolId, positionRecord.openVolume);
            }
        });
    });

    return Optional.of(
            new TotalCurrencyBalanceReportResult(
                    currencyBalance,
                    new IntLongHashMap(riskEngine.getFees()),
                    new IntLongHashMap(riskEngine.getAdjustments()),
                    new IntLongHashMap(riskEngine.getSuspends()),
                    null,
                    symbolOpenInterestLong,
                    symbolOpenInterestShort));
}
 

public static IntLongHashMap readIntLongHashMap(final BytesIn bytes) {
    int length = bytes.readInt();
    final IntLongHashMap hashMap = new IntLongHashMap(length);
    // TODO shuffle (? performance can be reduced if populating linearly)
    for (int i = 0; i < length; i++) {
        int k = bytes.readInt();
        long v = bytes.readLong();
        hashMap.put(k, v);
    }
    return hashMap;
}
 

public RiskEngine(final int shardId,
                  final long numShards,
                  final ISerializationProcessor serializationProcessor,
                  final SharedPool sharedPool,
                  final ExchangeConfiguration exchangeConfiguration) {
    if (Long.bitCount(numShards) != 1) {
        throw new IllegalArgumentException("Invalid number of shards " + numShards + " - must be power of 2");
    }
    this.shardId = shardId;
    this.shardMask = numShards - 1;
    this.serializationProcessor = serializationProcessor;

    // initialize object pools // TODO move to perf config
    final HashMap<Integer, Integer> objectsPoolConfig = new HashMap<>();
    objectsPoolConfig.put(ObjectsPool.SYMBOL_POSITION_RECORD, 1024 * 256);
    this.objectsPool = new ObjectsPool(objectsPoolConfig);

    this.logDebug = exchangeConfiguration.getLoggingCfg().getLoggingLevels().contains(LoggingConfiguration.LoggingLevel.LOGGING_RISK_DEBUG);

    if (exchangeConfiguration.getInitStateCfg().fromSnapshot()) {

        // TODO refactor, change to creator (simpler init)`
        final State state = serializationProcessor.loadData(
                exchangeConfiguration.getInitStateCfg().getSnapshotId(),
                ISerializationProcessor.SerializedModuleType.RISK_ENGINE,
                shardId,
                bytesIn -> {
                    if (shardId != bytesIn.readInt()) {
                        throw new IllegalStateException("wrong shardId");
                    }
                    if (shardMask != bytesIn.readLong()) {
                        throw new IllegalStateException("wrong shardMask");
                    }
                    final SymbolSpecificationProvider symbolSpecificationProvider = new SymbolSpecificationProvider(bytesIn);
                    final UserProfileService userProfileService = new UserProfileService(bytesIn);
                    final BinaryCommandsProcessor binaryCommandsProcessor = new BinaryCommandsProcessor(
                            this::handleBinaryMessage,
                            this::handleReportQuery,
                            sharedPool,
                            exchangeConfiguration.getReportsQueriesCfg(),
                            bytesIn,
                            shardId);
                    final IntObjectHashMap<LastPriceCacheRecord> lastPriceCache = SerializationUtils.readIntHashMap(bytesIn, LastPriceCacheRecord::new);
                    final IntLongHashMap fees = SerializationUtils.readIntLongHashMap(bytesIn);
                    final IntLongHashMap adjustments = SerializationUtils.readIntLongHashMap(bytesIn);
                    final IntLongHashMap suspends = SerializationUtils.readIntLongHashMap(bytesIn);

                    return new State(
                            symbolSpecificationProvider,
                            userProfileService,
                            binaryCommandsProcessor,
                            lastPriceCache,
                            fees,
                            adjustments,
                            suspends);
                });

        this.symbolSpecificationProvider = state.symbolSpecificationProvider;
        this.userProfileService = state.userProfileService;
        this.binaryCommandsProcessor = state.binaryCommandsProcessor;
        this.lastPriceCache = state.lastPriceCache;
        this.fees = state.fees;
        this.adjustments = state.adjustments;
        this.suspends = state.suspends;

    } else {
        this.symbolSpecificationProvider = new SymbolSpecificationProvider();
        this.userProfileService = new UserProfileService();
        this.binaryCommandsProcessor = new BinaryCommandsProcessor(
                this::handleBinaryMessage,
                this::handleReportQuery,
                sharedPool,
                exchangeConfiguration.getReportsQueriesCfg(),
                shardId);
        this.lastPriceCache = new IntObjectHashMap<>();
        this.fees = new IntLongHashMap();
        this.adjustments = new IntLongHashMap();
        this.suspends = new IntLongHashMap();
    }

    final OrdersProcessingConfiguration ordersProcCfg = exchangeConfiguration.getOrdersProcessingCfg();
    this.cfgIgnoreRiskProcessing = ordersProcCfg.getRiskProcessingMode() == OrdersProcessingConfiguration.RiskProcessingMode.NO_RISK_PROCESSING;
    this.cfgMarginTradingEnabled = ordersProcCfg.getMarginTradingMode() == OrdersProcessingConfiguration.MarginTradingMode.MARGIN_TRADING_ENABLED;
}
 

public IntLongHashMap getPageLocationMap()
{
    return pageLocationMap;
}
 

public IntLongHashMap getClientsBalancesSum() {
    return SerializationUtils.mergeSum(accountBalances, ordersBalances, suspends);
}
 

public IntLongHashMap getGlobalBalancesSum() {
    return SerializationUtils.mergeSum(accountBalances, ordersBalances, fees, adjustments, suspends);
}
 

public static TotalCurrencyBalanceReportResult ofOrderBalances(final IntLongHashMap currencyBalance) {
    return new TotalCurrencyBalanceReportResult(
            null, null, null, null, currencyBalance, null, null);
}
 

@Override
public PMML buildModel(JavaSparkContext sparkContext,
                       JavaRDD<String> trainData,
                       List<?> hyperParameters,
                       Path candidatePath) {

  int maxSplitCandidates = (Integer) hyperParameters.get(0);
  int maxDepth = (Integer) hyperParameters.get(1);
  String impurity = (String) hyperParameters.get(2);
  Preconditions.checkArgument(maxSplitCandidates >= 2,
                              "max-split-candidates must be at least 2");
  Preconditions.checkArgument(maxDepth > 0,
                              "max-depth must be at least 1");

  JavaRDD<String[]> parsedRDD = trainData.map(MLFunctions.PARSE_FN);
  CategoricalValueEncodings categoricalValueEncodings =
      new CategoricalValueEncodings(getDistinctValues(parsedRDD));
  JavaRDD<LabeledPoint> trainPointData =
      parseToLabeledPointRDD(parsedRDD, categoricalValueEncodings);

  Map<Integer,Integer> categoryInfo = categoricalValueEncodings.getCategoryCounts();
  categoryInfo.remove(inputSchema.getTargetFeatureIndex()); // Don't specify target count
  // Need to translate indices to predictor indices
  Map<Integer,Integer> categoryInfoByPredictor = new HashMap<>(categoryInfo.size());
  categoryInfo.forEach((k, v) -> categoryInfoByPredictor.put(inputSchema.featureToPredictorIndex(k), v));

  int seed = RandomManager.getRandom().nextInt();

  RandomForestModel model;
  if (inputSchema.isClassification()) {
    int numTargetClasses =
        categoricalValueEncodings.getValueCount(inputSchema.getTargetFeatureIndex());
    model = RandomForest.trainClassifier(trainPointData,
                                         numTargetClasses,
                                         categoryInfoByPredictor,
                                         numTrees,
                                         "auto",
                                         impurity,
                                         maxDepth,
                                         maxSplitCandidates,
                                         seed);
  } else {
    model = RandomForest.trainRegressor(trainPointData,
                                        categoryInfoByPredictor,
                                        numTrees,
                                        "auto",
                                        impurity,
                                        maxDepth,
                                        maxSplitCandidates,
                                        seed);
  }

  List<IntLongHashMap> treeNodeIDCounts = treeNodeExampleCounts(trainPointData, model);
  IntLongHashMap predictorIndexCounts = predictorExampleCounts(trainPointData, model);

  return rdfModelToPMML(model,
                        categoricalValueEncodings,
                        maxDepth,
                        maxSplitCandidates,
                        impurity,
                        treeNodeIDCounts,
                        predictorIndexCounts);
}
 

public static SingleUserReportResult createFromRiskEngineFound(long uid, UserStatus userStatus, IntLongHashMap accounts, IntObjectHashMap<Position> positions) {
    return new SingleUserReportResult(uid, userStatus, accounts, positions, null, QueryExecutionStatus.OK);
}