Java Code Examples for it.unimi.dsi.fastutil.longs.LongArrayList#add()

/**
 * Runs PageRank, either until the max number of iterations has been reached or the L1 norm of
 * the difference between PageRank vectors drops below the tolerance.
 *
 * @return number of iterations that was actually run
 */
public int run() {
  LongArrayList noOuts = new LongArrayList();
  LongIterator iter = nodes.iterator();
  while (iter.hasNext()) {
    long v = iter.nextLong();
    if (graph.getOutDegree(v) == 0) {
      noOuts.add(v);
    }
  }

  double dampingAmount = (1.0 - dampingFactor) / nodeCount;
  prVector = new double[(int) (maxNodeId + 1)];
  nodes.forEach(v -> prVector[(int) (long) v] = 1.0 / nodeCount);

  int i = 0;
  while (i < this.maxIterations && normL1 > tolerance) {
    iterate(dampingAmount, noOuts);
    i++;
  }

  return i;
}
 

@Override
public LongArrayList recommendInternal(ClickData clickData) {
	//filter out items with low overall click counts
	//first, retrieve the recommendation results of the underlying algorithm
	LongArrayList rec = mainStrategy.recommendInternal(clickData);
	
	//create lists of filtered items and retained items
	LongArrayList filteredRec = new LongArrayList();
	LongArrayList filteredRecNotMatch = new LongArrayList();
	//iterate over the recommendation list of the underlying strategy
	for (int j = 0; j < rec.size(); j++) {
		long i = rec.getLong(j);
		//filter items if they do not have enough clicks
		if ((itemClickCount.containsKey(i)) && (itemClickCount.get(i) >= minClickCount)) {
			filteredRec.add(i);
		} else if (fallback) {
			//if we have a fallback, add the filtered item to the fallback list
			filteredRecNotMatch.add(i);
		}
	}
	//merge the filtered list with the fallback list (empty in case fallback==false)
	filteredRec.addAll(filteredRecNotMatch);
	//return the filtered list
	return filteredRec;
}
 

@Override
public LongArrayList recommendInternal(ClickData clickData) {
	//step down the pattern tree to match the right pattern
	SequenceTreeNode currentNode = patternTree;
	for (Transaction click : clickData.session) {
		if (!currentNode.children.containsKey(getTreeNodeKey(click))) {
			return new LongArrayList();
		}
		currentNode = currentNode.children.get(getTreeNodeKey(click));
	}
	
	//if we found the right pattern, sort the possible completions of this pattern 
	//by their support values and create a recommendation list
	Map<String, SequenceTreeNode> sortMap = Util.sortByValue(currentNode.children, false);
	LongArrayList returnList = new LongArrayList();
	for (String i : sortMap.keySet()){
		returnList.add(Long.parseLong(i));
	}
	return returnList;
}
 

@Override
public long[] getAllUpdateTimes(String sessionID, String typeID, String workerID) {
    String sql = "SELECT Timestamp FROM " + TABLE_NAME_UPDATES + " WHERE SessionID = '" + sessionID + "'  "
            + "AND TypeID = '" + typeID + "' AND workerID = '" + workerID + "';";
    try (Statement statement = connection.createStatement()) {
        ResultSet rs = statement.executeQuery(sql);
        LongArrayList list = new LongArrayList();
        while (rs.next()) {
            list.add(rs.getLong(1));
        }
        return list.toLongArray();
    } catch (SQLException e) {
        throw new RuntimeException(e);
    }
}
 

@Override
public Collection<Long> read(final DataInput in) throws IOException {
    final int length = in.readInt();
    LongArrayList ret = new LongArrayList(length);
    long previous = 0;
    for (int i = 0; i < length; i++) {
        final long delta = VIntUtils.readVInt64(in);
        final long id = previous+delta;
        previous = id;
        ret.add(id);
    }
    return ret;
}
 

@Override
public Transaction checkpoint(Transaction tx) {
  long newWritePointer = getWritePointer();
  LongArrayList newCheckpointPointers = new LongArrayList(tx.getCheckpointWritePointers());
  newCheckpointPointers.add(newWritePointer);
  return new Transaction(tx, newWritePointer, newCheckpointPointers.toLongArray());
}
 

@Override
protected NavigableMap<Long, TransactionManager.InProgressTx> decodeInProgress(BinaryDecoder decoder)
    throws IOException {

  int size = decoder.readInt();
  NavigableMap<Long, TransactionManager.InProgressTx> inProgress = Maps.newTreeMap();
  while (size != 0) { // zero denotes end of list as per AVRO spec
    for (int remaining = size; remaining > 0; --remaining) {
      long txId = decoder.readLong();
      long expiration = decoder.readLong();
      long visibilityUpperBound = decoder.readLong();
      int txTypeIdx = decoder.readInt();
      TransactionManager.InProgressType txType;
      try {
        txType = TransactionManager.InProgressType.values()[txTypeIdx];
      } catch (ArrayIndexOutOfBoundsException e) {
        throw new IOException("Type enum ordinal value is out of range: " + txTypeIdx);
      }
      // read checkpoint tx IDs
      int checkpointPointerSize = decoder.readInt();
      LongArrayList checkpointPointers = new LongArrayList(checkpointPointerSize);
      while (checkpointPointerSize != 0) {
        for (int checkpointRemaining = checkpointPointerSize; checkpointRemaining > 0; --checkpointRemaining) {
          checkpointPointers.add(decoder.readLong());
        }
        checkpointPointerSize = decoder.readInt();
      }
      inProgress.put(txId,
          new TransactionManager.InProgressTx(visibilityUpperBound, expiration, txType, checkpointPointers));
    }
    size = decoder.readInt();
  }
  return inProgress;
}
 

/**
 * Creates a new Transaction. This method only get called from start transaction, which is already
 * synchronized.
 */
private Transaction createTransaction(long writePointer, TransactionType type) {
  // For holding the first in progress short transaction Id (with timeout >= 0).
  long firstShortTx = Transaction.NO_TX_IN_PROGRESS;
  LongArrayList inProgressIds = new LongArrayList(inProgress.size());
  for (Map.Entry<Long, InProgressTx> entry : inProgress.entrySet()) {
    long txId = entry.getKey();
    inProgressIds.add(txId);
    if (firstShortTx == Transaction.NO_TX_IN_PROGRESS && !entry.getValue().isLongRunning()) {
      firstShortTx = txId;
    }
  }
  return new Transaction(readPointer, writePointer, invalidTxList.toSortedArray(),
                         inProgressIds.toLongArray(), firstShortTx, type);
}
 

@Override
public LongArrayList recommendInternal(ClickData clickData) {
	//filter out items that have been release too long ago
	//first, retrieve the recommendation results of the underlying algorithm
	LongArrayList rec = mainStrategy.recommendInternal(clickData);
	
	//create lists of filtered items and retained items
	LongArrayList filteredRec = new LongArrayList();
	LongArrayList filteredRecNotMatch = new LongArrayList();
	//iterate over the recommendation list of the underlying strategy
	for (int j = 0; j < rec.size(); j++) {
		long i = rec.getLong(j);
		// filter item based on the difference between the current (simulation) time and
		// the time of publication
		if ((clickData.click.timestamp.getTime() - timestampMap.get(i)) <= filterTime
				&& (clickData.click.timestamp.getTime() - timestampMap.get(i)) > 0) {
			filteredRec.add(i);
		} else if (fallback) {
			//if we have a fallback, add the filtered item to the fallback list
			filteredRecNotMatch.add(i);
		}
	}
	//merge the filtered list with the fallback list (empty in case fallback==false)
	filteredRec.addAll(filteredRecNotMatch);
	//return the filtered list
	return filteredRec;

}
 

@Override
public LongList successors(final long node) {
	final int lid = GID2LID.get(node);
	if (lid < 0) throw new IllegalArgumentException("GID " + node + " does not exist");
	final int outdegree = graph.outdegree(lid);
	final LongArrayList gidList = new LongArrayList(outdegree);
	for (final int s: graph.successorArray(lid)) gidList.add(LID2GID[s]);
	return gidList;
}
 

/**
 * Generate a result list from a map of summed up co-occurence counts.
 * In this case, we are just parsing the string back to a long (item id).
 * In future implementations, this method can be overridden and do more intersting stuff.
 * @param sortedKeys -
 * @param clickData -
 * @return a sorted recommendation list
 */
protected LongArrayList generateResultList(Map<String, Double> sortedKeys, ClickData clickData) {
	// remap all item ids back to the actual Item objects
	LongArrayList sortedItems = new LongArrayList();
	for (String itemId : sortedKeys.keySet()) {
		sortedItems.add(Long.parseLong(itemId));
	}
	return sortedItems;
}
 

public List<LongArrayList> calculateConditions(PositionListIndex partialUnique,
                                               PositionListIndex PLICondition,
                                               int frequency,
                                               List<LongArrayList> unsatisfiedClusters) {
  List<LongArrayList> result = new LinkedList<>();
  Long2LongOpenHashMap uniqueHashMap = partialUnique.asHashMap();
  LongArrayList touchedClusters = new LongArrayList();
  nextCluster:
  for (LongArrayList cluster : PLICondition.getClusters()) {
    if (cluster.size() < frequency) {
      continue;
    }
    int unsatisfactionCount = 0;
    touchedClusters.clear();
    for (long rowNumber : cluster) {
      if (uniqueHashMap.containsKey(rowNumber)) {
        if (touchedClusters.contains(uniqueHashMap.get(rowNumber))) {
          unsatisfactionCount++;
        } else {
          touchedClusters.add(uniqueHashMap.get(rowNumber));
        }
      }
    }
    if (unsatisfactionCount == 0) {
      result.add(cluster);
    } else {
      if ((cluster.size() - unsatisfactionCount) >= frequency) {
        unsatisfiedClusters.add(cluster);
      }
    }
  }
  return result;
}
 

public List<LongArrayList> calculateConditions(PositionListIndex partialUnique,
                                               PositionListIndex PLICondition,
                                               int frequency,
                                               List<LongArrayList> unsatisfiedClusters) {
  List<LongArrayList> result = new LinkedList<>();
  Long2LongOpenHashMap uniqueHashMap = partialUnique.asHashMap();
  LongArrayList touchedClusters = new LongArrayList();
  nextCluster:
  for (LongArrayList cluster : PLICondition.getClusters()) {
    int unsatisfactionCount = 0;
    touchedClusters.clear();
    for (long rowNumber : cluster) {
      if (uniqueHashMap.containsKey(rowNumber)) {
        if (touchedClusters.contains(uniqueHashMap.get(rowNumber))) {
          unsatisfactionCount++;
        } else {
          touchedClusters.add(uniqueHashMap.get(rowNumber));
        }
      }
    }
    if (unsatisfactionCount == 0) {
      result.add(cluster);
    } else {
      //if ((cluster.size() - unsatisfactionCount) >= frequency) {
      unsatisfiedClusters.add(cluster);
      //}
    }
  }
  return result;
}
 

protected LongArrayList getLongSet(long... longs) {
  LongArrayList longSet = new LongArrayList();
  for (long longNumber : longs) {
    longSet.add(longNumber);
  }
  return longSet;
}
 

@Override
public LongList predecessors(final long node) {
	final int lid = (int)GID2LID.getLong(node);
	if (lid < 0 || lid >= numNodes() || LID2GID[lid] != node) throw new IllegalArgumentException("GID " + node + " does not exist");
	final int indegree = transpose.outdegree(lid);
	final LongArrayList gidList = new LongArrayList(indegree);
	for (final int s: transpose.successorArray(lid)) gidList.add(LID2GID[s]);
	return gidList;
}
 

@Override
public LongList successors(final long node) {
	final int lid = (int)GID2LID.getLong(node);
	if (lid < 0 || lid >= numNodes() || LID2GID[lid] != node) throw new IllegalArgumentException("GID " + node + " does not exist");
	final int outdegree = graph.outdegree(lid);
	final LongArrayList gidList = new LongArrayList(outdegree);
	for (final int s: graph.successorArray(lid)) gidList.add(LID2GID[s]);
	return gidList;
}
 

@Override
public LongList predecessors(final long node) {
	final int lid = GID2LID.get(node);
	if (lid < 0) throw new IllegalArgumentException("GID " + node + " does not exist");
	final int indegree = transpose.outdegree(lid);
	final LongArrayList gidList = new LongArrayList(indegree);
	for (final int s: transpose.successorArray(lid)) gidList.add(LID2GID[s]);
	return gidList;
}
 

@Override
public LongArrayList recommendInternal(ClickData clickData) {
	//create a result list
	LongArrayList results = new LongArrayList();
	try {
		//determine the input query, which can either be based on the current item 
		//or all items from the current session depending on the configuration
		String input;
		if (!wholeSession){
			//extract the content from the current item
			input = extractContent(clickData.click.item);
		}else{
			//iteratively append the content of every item from the current user session
			input="";
			for (int i = 0 ; i<clickData.session.size(); i++ ){
				input += " "+ extractContent(clickData.session.get(i).item);
			}
		}
		//avoid an exception that happens for too large queries
           BooleanQuery.setMaxClauseCount(Integer.MAX_VALUE);
           //create a query
		Query q = new QueryParser("text", analyzer)
				.parse(QueryParserUtil.escape(input));
		//set an unreasonably high retrieval amount, because we want a long recommendation list
		int hitsPerPage = 100000;
		//instantiate the retrieval objects
		IndexReader reader = DirectoryReader.open(index);
		IndexSearcher searcher = new IndexSearcher(reader);
		TopScoreDocCollector collector = TopScoreDocCollector.create(hitsPerPage);
		//execute the query
		searcher.search(q, collector);
		//iterate the hits and extract the item ids
		ScoreDoc[] hits = collector.topDocs().scoreDocs;
		for (int i = 1; i < hits.length; ++i) {
			if (hits[i].score < minScore) {
				//stop retrieving, if the lucene score is too low
				break;
			}
			int docId = hits[i].doc;
			Document d = searcher.doc(docId);
			results.add(Long.parseLong(d.get("id")));
		}
		reader.close();
	} catch (ParseException | IOException e) {
		e.printStackTrace();
	}
	//return the results
	return results;
}
 

protected void combineClusterIntoResult(ColumnCombinationBitset partialUnique)
    throws AlgorithmExecutionException {
  LongArrayList touchedCluster = new LongArrayList();
  Long2LongOpenHashMap partialUniqueHash = this.algorithm.getPLI(partialUnique).asHashMap();
  Set<ColumnCombinationBitset> startPoints = this.getConditionStartPoints();
  for (ColumnCombinationBitset minimalConditionStartPoint : startPoints) {
    if (minimalConditionStartPoint.getSetBits().size() != 1) {
      minimalConditionStartPoint =
          minimalConditionStartPoint.getContainedOneColumnCombinations().get(0);
    }

    List<ConditionEntry> satisfiedCluster = new ArrayList<>();
    Long2ObjectOpenHashMap<LongArrayList> intersectingCluster = new Long2ObjectOpenHashMap<>();
    int clusterNumber = 0;
    //build intersecting cluster
    for (ConditionEntry singleCluster : this.singleConditions.get(minimalConditionStartPoint)) {
      satisfiedCluster.add(singleCluster.setClusterNumber(clusterNumber));
      touchedCluster.clear();
      for (long rowNumber : singleCluster.cluster) {
        if (partialUniqueHash.containsKey(rowNumber)) {
          touchedCluster.add(partialUniqueHash.get(rowNumber));
        }
      }
      for (long partialUniqueClusterNumber : touchedCluster) {
        if (intersectingCluster.containsKey(partialUniqueClusterNumber)) {
          intersectingCluster.get(partialUniqueClusterNumber).add(clusterNumber);
        } else {
          LongArrayList newConditionClusterNumbers = new LongArrayList();
          newConditionClusterNumbers.add(clusterNumber);
          intersectingCluster.put(partialUniqueClusterNumber, newConditionClusterNumbers);
        }
      }
      clusterNumber++;
    }
    intersectingCluster = purgeIntersectingClusterEntries(intersectingCluster);
    //convert into list
    List<LongArrayList> intersectingClusterList = new ArrayList<>();
    for (long partialUniqueCluster : intersectingCluster.keySet()) {
      intersectingClusterList.add(intersectingCluster.get(partialUniqueCluster));
    }

    Object2FloatArrayMap<List<ConditionEntry>>
        clustergroups =
        this.combineClusters(this.algorithm.frequency, satisfiedCluster,
                             intersectingClusterList);

    for (List<ConditionEntry> singleCondition : clustergroups.keySet()) {
      ResultSingleton.getInstance().addConditionToResult(partialUnique, singleCondition,
                                                         clustergroups.get(singleCondition));
    }
  }
}
 

/**
 * Generates a new snapshot object with semi-randomly populated values.  This does not necessarily accurately
 * represent a typical snapshot's distribution of values, as we only set an upper bound on pointer values.
 *
 * We generate a new snapshot with the contents:
 * <ul>
 *   <li>readPointer = 1M + (random % 1M)</li>
 *   <li>writePointer = readPointer + 1000</li>
 *   <li>waterMark = writePointer + 1000</li>
 *   <li>inProgress = one each for (writePointer - 500)..writePointer, ~ 5% "long" transaction</li>
 *   <li>invalid = 100 randomly distributed, 0..1M</li>
 *   <li>committing = one each, (readPointer + 1)..(readPointer + 100)</li>
 *   <li>committed = one each, (readPointer - 1000)..readPointer</li>
 * </ul>
 * @return a new snapshot of transaction state.
 */
private TransactionSnapshot createRandomSnapshot() {
  // limit readPointer to a reasonable range, but make it > 1M so we can assign enough keys below
  long readPointer = (Math.abs(random.nextLong()) % 1000000L) + 1000000L;
  long writePointer = readPointer + 1000L;

  // generate in progress -- assume last 500 write pointer values
  NavigableMap<Long, TransactionManager.InProgressTx> inProgress = Maps.newTreeMap();
  long startPointer = writePointer - 500L;
  for (int i = 0; i < 500; i++) {
    long currentTime = System.currentTimeMillis();
    // make some "long" transactions
    if (i % 20 == 0) {
      inProgress.put(startPointer + i,
                     new TransactionManager.InProgressTx(startPointer - 1, currentTime + TimeUnit.DAYS.toSeconds(1),
                                                         TransactionManager.InProgressType.LONG));
    } else {
      inProgress.put(startPointer + i,
                     new TransactionManager.InProgressTx(startPointer - 1, currentTime + 300000L,
                                                         TransactionManager.InProgressType.SHORT));
    }
  }

  // make 100 random invalid IDs
  LongArrayList invalid = new LongArrayList();
  for (int i = 0; i < 100; i++) {
    invalid.add(Math.abs(random.nextLong()) % 1000000L);
  }

  // make 100 committing entries, 10 keys each
  Map<Long, Set<ChangeId>> committing = Maps.newHashMap();
  for (int i = 0; i < 100; i++) {
    committing.put(readPointer + i, generateChangeSet(10));
  }

  // make 1000 committed entries, 10 keys each
  long startCommitted = readPointer - 1000L;
  NavigableMap<Long, Set<ChangeId>> committed = Maps.newTreeMap();
  for (int i = 0; i < 1000; i++) {
    committed.put(startCommitted + i, generateChangeSet(10));
  }

  return new TransactionSnapshot(System.currentTimeMillis(), readPointer, writePointer,
                                 invalid, inProgress, committing, committed);
}