org.apache.mahout.cf.taste.impl.common.LongPrimitiveIterator Java Examples

/**
 * @param n approximate number of items to choose
 * @param stream stream to choose from randomly
 * @param streamSize (approximate) stream size
 * @param random random number generator
 * @return up to n elements chosen uninformly at random from the stream
 */
public static long[] chooseAboutNFromStream(int n, 
                                            LongPrimitiveIterator stream,
                                            int streamSize, 
                                            RandomGenerator random) {
  LongPrimitiveIterator it;
  if (n < streamSize) {
    it = new SamplingLongPrimitiveIterator(random, stream, (double) n / streamSize);      
  } else {
    it = stream;
  }
  FastIDSet chosen = new FastIDSet(n);    
  while (it.hasNext()) {
    chosen.add(it.nextLong());
  }
  return chosen.toArray();
}
 

private static void removeNotUpdated(LongPrimitiveIterator it,
                                     FastIDSet updated,
                                     FastIDSet recentlyActive,
                                     Lock writeLock) {
  writeLock.lock();
  try {
    while (it.hasNext()) {
      long id = it.nextLong();
      if (!updated.contains(id) && !recentlyActive.contains(id)) {
        it.remove();
      }
    }
  } finally {
    writeLock.unlock();
  }
}
 

private static void writeKnownIDs(ObjectOutputStream out, FastByIDMap<FastIDSet> knownItemIDs) throws IOException {
  if (knownItemIDs == null) {
    out.writeInt(NULL_COUNT);
  } else {
    out.writeInt(knownItemIDs.size());
    for (FastByIDMap.MapEntry<FastIDSet> entry : knownItemIDs.entrySet()) {
      out.writeLong(entry.getKey());
      FastIDSet itemIDs = entry.getValue();
      out.writeInt(itemIDs.size());
      LongPrimitiveIterator it = itemIDs.iterator();
      while (it.hasNext()) {
        out.writeLong(it.nextLong());
      }
    }
  }
}
 

private static void writeClusters(Collection<IDCluster> clusters, ObjectOutputStream out) throws IOException {
  if (clusters == null) {
    out.writeInt(0);
  } else {
    out.writeInt(clusters.size());
    for (IDCluster cluster : clusters) {
      FastIDSet members = cluster.getMembers();
      out.writeInt(members.size());
      LongPrimitiveIterator it = members.iterator();
      while (it.hasNext()) {
        out.writeLong(it.nextLong());
      }
      float[] centroid = cluster.getCentroid();
      out.writeInt(centroid.length);
      for (float f : centroid) {
        out.writeFloat(f);
      }
    }
  }
}
 

/**
 * Like {@link MatrixUtils#transposeTimesSelf(FastByIDMap)}, but instead of computing MT * M, 
 * it computes MT * C * M, where C is a diagonal matrix of 1s and 0s. This is like pretending some
 * rows of M are 0.
 * 
 * @see MatrixUtils#transposeTimesSelf(FastByIDMap) 
 * @see #LOSS_IGNORES_UNSPECIFIED
 */
private static RealMatrix partialTransposeTimesSelf(FastByIDMap<float[]> M, 
                                                    int dimension, 
                                                    LongPrimitiveIterator keys) {
  RealMatrix result = new Array2DRowRealMatrix(dimension, dimension);
  while (keys.hasNext()) {
    long key = keys.next();
    float[] vector = M.get(key);
    for (int row = 0; row < dimension; row++) {
      float rowValue = vector[row];
      for (int col = 0; col < dimension; col++) {
        result.addToEntry(row, col, rowValue * vector[col]);
      }
    }
  }
  return result;
}
 

private static FastIDSet getIDsFromKeys(FastByIDMap<float[]> map, Lock readLock, FastIDSet tagIDs) {
  readLock.lock();
  try {
    FastIDSet ids = new FastIDSet(map.size());
    LongPrimitiveIterator it = map.keySetIterator();
    while (it.hasNext()) {
      long id = it.nextLong();
      if (!tagIDs.contains(id)) {
        ids.add(id);
      }
    }
    return ids;
  } finally {
    readLock.unlock();
  }
}
 

@Test
public void testIterator() {
  FastIDSet set = buildTestFastSet();
  Collection<Long> expected = new HashSet<Long>(3);
  expected.add(1L);
  expected.add(2L);
  expected.add(3L);
  LongPrimitiveIterator it = set.iterator();
  while (it.hasNext()) {
    expected.remove(it.nextLong());
  }
  assertTrue(expected.isEmpty());
}
 

/**
 * @param set to choose from
 * @param random random number generator
 * @return element of the set chosen uniformly at random
 */
public static int randomFrom(FastIDSet set, RandomGenerator random) {
  int size = set.size();
  Preconditions.checkArgument(size > 0, "Empty set");
  LongPrimitiveIterator it = set.iterator();
  it.skip(random.nextInt(size));
  return (int) it.nextLong();
}
 

public SamplingLongPrimitiveIterator(RandomGenerator random, LongPrimitiveIterator delegate, double samplingRate) {
  Preconditions.checkNotNull(random);
  Preconditions.checkNotNull(delegate);
  Preconditions.checkArgument(samplingRate > 0.0 && samplingRate <= 1.0);
  // Geometric distribution is special case of negative binomial (aka Pascal) with r=1:
  geometricDistribution = new PascalDistribution(random, 1, samplingRate);
  this.delegate = delegate;
  this.hasNext = true;
  doNext();
}
 

private static long[] unionColumnKeysInOrder(FastByIDMap<FastByIDFloatMap> M) {
  FastIDSet keys = new FastIDSet(1000);
  for (FastByIDMap.MapEntry<FastByIDFloatMap> entry : M.entrySet()) {
    LongPrimitiveIterator it = entry.getValue().keySetIterator();
    while (it.hasNext()) {
      keys.add(it.nextLong());
    }
  }
  long[] keysArray = keys.toArray();
  Arrays.sort(keysArray);
  return keysArray;
}
 

private static long[] keysInOrder(FastByIDMap<?> map) {
  FastIDSet keys = new FastIDSet(map.size());
  LongPrimitiveIterator it = map.keySetIterator();
  while (it.hasNext()) {
    keys.add(it.nextLong());
  }
  long[] keysArray = keys.toArray();
  Arrays.sort(keysArray);
  return keysArray;
}
 

private Collection<String> translate(FastIDSet itemIDs) {
  Collection<String> result = Lists.newArrayListWithCapacity(itemIDs.size());
  LongPrimitiveIterator it = itemIDs.iterator();
  while (it.hasNext()) {
    result.add(untranslateItem(it.nextLong()));
  }
  return result;
}
 

@Override
public Collection<String> getUserCluster(int n) throws TasteException {
  FastIDSet userIDs = delegate.getUserCluster(n);
  Collection<String> translated = Lists.newArrayListWithCapacity(userIDs.size());
  LongPrimitiveIterator it = userIDs.iterator();
  while (it.hasNext()) {
    translated.add(Long.toString(it.nextLong()));
  }
  return translated;
}
 

/**
 * Outputs IDs in CSV or JSON format. When outputting CSV, one ID is written per line. When outputting
 * JSON, the output is an array of IDs.
 */
final void outputIDs(HttpServletRequest request, ServletResponse response, FastIDSet ids) throws IOException {
  Writer writer = response.getWriter();
  LongPrimitiveIterator it = ids.iterator();
  switch (determineResponseType(request)) {
    case JSON:
      writer.write('[');
      boolean first = true;
      while (it.hasNext()) {
        if (first) {
          first = false;
        } else {
          writer.write(',');
        }
        writer.write(Long.toString(it.nextLong()));
      }
      writer.write(']');
      break;
    case CSV:
      while (it.hasNext()) {
        writer.write(Long.toString(it.nextLong()));
        writer.write('\n');
      }
      break;
    default:
      throw new IllegalStateException("Unknown response type");
  }
}
 

private static void writeIDSet(FastIDSet ids, ObjectOutputStream out) throws IOException {
  if (ids == null) {
    out.writeInt(0);
  } else {
    out.writeInt(ids.size());
    LongPrimitiveIterator it = ids.iterator();
    while (it.hasNext()) {
      out.writeLong(it.nextLong());
    }
  }
}
 

private static FastIDSet keysToSet(FastByIDMap<?> map) {
  FastIDSet result = new FastIDSet(map.size());
  LongPrimitiveIterator it = map.keySetIterator();
  while (it.hasNext()) {
    result.add(it.nextLong());
  }
  return result;
}
 

public static void merge(File model1File, File model2File, File mergedModelFile) throws IOException {

    Generation model1 = GenerationSerializer.readGeneration(model1File);
    Generation model2 = GenerationSerializer.readGeneration(model2File);

    FastByIDMap<float[]> x1 = model1.getX();
    FastByIDMap<float[]> y1 = model1.getY();
    FastByIDMap<float[]> x2 = model2.getX();
    FastByIDMap<float[]> y2 = model2.getY();

    RealMatrix translation = multiply(y1, x2);

    FastByIDMap<float[]> xMerged = MatrixUtils.multiply(translation.transpose(), x1);

    FastIDSet emptySet = new FastIDSet();
    FastByIDMap<FastIDSet> knownItems = new FastByIDMap<FastIDSet>();
    LongPrimitiveIterator it = xMerged.keySetIterator();
    while (it.hasNext()) {
      knownItems.put(it.nextLong(), emptySet);
    }

    FastIDSet x1ItemTagIDs = model1.getItemTagIDs();
    FastIDSet y2UserTagIDs = model2.getUserTagIDs();

    Generation merged = new Generation(knownItems, xMerged, y2, x1ItemTagIDs, y2UserTagIDs);
    GenerationSerializer.writeGeneration(merged, mergedModelFile);
  }
 

@Override
public Iterable<Long> getUsers() {
    if (model == null) {
        generateDatamodel();
    }
    List<Long> users = new ArrayList<>();
    LongPrimitiveIterator lpi = model.getUserIDs();
    while (lpi.hasNext()) {
        users.add(lpi.nextLong());
    }
    return users;
}
 

@Override
public Iterable<Long> getItems() {
    if (model == null) {
        generateDatamodel();
    }
    List<Long> items = new ArrayList<>();
    LongPrimitiveIterator lpi = model.getItemIDs();
    while (lpi.hasNext()) {
        items.add(lpi.nextLong());
    }
    return items;
}
 

/**
 * <p>Lists the items that were most influential in recommending a given item to a given user. Exactly how this
 * is determined is left to the implementation, but, generally this will return items that the user prefers
 * and that are similar to the given item.</p>
 *
 * <p>These values by which the results are ordered are opaque values and have no interpretation
 * other than that larger means stronger.</p>
 *
 * @param userID ID of user who was recommended the item
 * @param itemID ID of item that was recommended
 * @param howMany maximum number of items to return
 * @return {@link List} of {@link RecommendedItem}, ordered from most influential in recommended the given
 *  item to least
 * @throws NoSuchUserException if the user is not known in the model
 * @throws NoSuchItemException if the item is not known in the model
 * @throws NotReadyException if the recommender has no model available yet
 */
@Override
public List<RecommendedItem> recommendedBecause(long userID, long itemID, int howMany)
    throws NoSuchUserException, NoSuchItemException, NotReadyException {

  Preconditions.checkArgument(howMany > 0, "howMany must be positive");

  Generation generation = getCurrentGeneration();
  FastByIDMap<FastIDSet> knownItemIDs = generation.getKnownItemIDs();
  if (knownItemIDs == null) {
    throw new UnsupportedOperationException("No known item IDs available");
  }

  Lock knownItemLock = generation.getKnownItemLock().readLock();
  FastIDSet userKnownItemIDs;
  knownItemLock.lock();
  try {
    userKnownItemIDs = knownItemIDs.get(userID);
  } finally {
    knownItemLock.unlock();
  }
  if (userKnownItemIDs == null) {
    throw new NoSuchUserException(userID);
  }

  FastByIDMap<float[]> Y = generation.getY();

  Lock yLock = generation.getYLock().readLock();
  yLock.lock();
  try {

    float[] features = Y.get(itemID);
    if (features == null) {
      throw new NoSuchItemException(itemID);
    }
    FastByIDMap<float[]> toFeatures;
    synchronized (userKnownItemIDs) {
      toFeatures = new FastByIDMap<float[]>(userKnownItemIDs.size());
      LongPrimitiveIterator it = userKnownItemIDs.iterator();
      while (it.hasNext()) {
        long fromItemID = it.nextLong();
        float[] fromFeatures = Y.get(fromItemID);
        toFeatures.put(fromItemID, fromFeatures);
      }
    }

    return TopN.selectTopN(new RecommendedBecauseIterator(toFeatures.entrySet().iterator(), 
                                                          generation.getUserTagIDs(), 
                                                          features), 
                           howMany);
  } finally {
    yLock.unlock();
  }
}
 

private IDToEntryIterator(LongPrimitiveIterator input) {
  this.input = input;
  this.delegate = new MutableMapEntry();
}
 

private static void printTagIDs(FastIDSet ids, Appendable out) throws IOException {
  LongPrimitiveIterator it = ids.iterator();
  while (it.hasNext()) {
    out.append(Long.toString(it.nextLong())).append('\n');
  }
}
 

public LongPrimitiveIterator keySetIterator() {
  return new KeyIterator();
}
 

public LongPrimitiveIterator keySetIterator() {
  return new KeyIterator();
}
 

@Override
public LongPrimitiveIterator iterator() {
  return new KeyIterator();
}
 

public SamplingLongPrimitiveIterator(LongPrimitiveIterator delegate, double samplingRate) {
  this(RandomManager.getRandom(), delegate, samplingRate);
}
 

public static LongPrimitiveIterator maybeWrapIterator(LongPrimitiveIterator delegate, double samplingRate) {
  return samplingRate >= 1.0 ? delegate : new SamplingLongPrimitiveIterator(delegate, samplingRate);
}
 

/**
 * {@inheritDoc}
 */
@Override
public LongPrimitiveIterator getItemIDs() throws TasteException {
    return wrapper.getItemIDs();
}
 

/**
 * {@inheritDoc}
 */
@Override
public LongPrimitiveIterator getUserIDs() throws TasteException {
    return wrapper.getUserIDs();
}
 

/**
 * Runs a Mahout recommender using the provided datamodels and the
 * previously provided properties.
 *
 * @param opts see
 * {@link net.recommenders.rival.recommend.frameworks.AbstractRunner.RUN_OPTIONS}
 * @param trainingModel model to be used to train the recommender.
 * @param testModel model to be used to test the recommender.
 * @return nothing when opts is
 * {@link net.recommenders.rival.recommend.frameworks.AbstractRunner.RUN_OPTIONS#OUTPUT_RECS},
 * otherwise, when opts is
 * {@link net.recommenders.rival.recommend.frameworks.AbstractRunner.RUN_OPTIONS#RETURN_RECS}
 * or
 * {@link net.recommenders.rival.recommend.frameworks.AbstractRunner.RUN_OPTIONS#RETURN_AND_OUTPUT_RECS}
 * it returns the predictions
 * @throws TasteException when there is a problem with the Mahout
 * recommender
 * @throws RecommenderException when recommender cannot be instantiated
 * properly
 */
public TemporalDataModelIF<Long, Long> runMahoutRecommender(final RUN_OPTIONS opts, final DataModel trainingModel, final DataModel testModel)
        throws RecommenderException, TasteException {
    if (isAlreadyRecommended()) {
        return null;
    }

    GenericRecommenderBuilder grb = new GenericRecommenderBuilder();

    if (getProperties().containsKey(RecommendationRunner.NEIGHBORHOOD) && getProperties().getProperty(RecommendationRunner.NEIGHBORHOOD).equals("-1")) {
        getProperties().setProperty(RecommendationRunner.NEIGHBORHOOD, Math.round(Math.sqrt(trainingModel.getNumItems())) + "");
    }
    if (getProperties().containsKey(RecommendationRunner.FACTORS) && getProperties().getProperty(RecommendationRunner.FACTORS).equals("-1")) {
        getProperties().setProperty(RecommendationRunner.FACTORS, Math.round(Math.sqrt(trainingModel.getNumItems())) + "");
    }

    Recommender recommender = null;
    if (getProperties().getProperty(RecommendationRunner.FACTORS) == null) {
        recommender = grb.buildRecommender(
                trainingModel,
                getProperties().getProperty(RecommendationRunner.RECOMMENDER),
                getProperties().getProperty(RecommendationRunner.SIMILARITY),
                Integer.parseInt(getProperties().getProperty(RecommendationRunner.NEIGHBORHOOD)));
    }
    if (getProperties().getProperty(RecommendationRunner.FACTORS) != null) {
        recommender = grb.buildRecommender(
                trainingModel,
                getProperties().getProperty(RecommendationRunner.RECOMMENDER),
                getProperties().getProperty(RecommendationRunner.FACTORIZER),
                DEFAULT_ITERATIONS,
                Integer.parseInt(getProperties().getProperty(RecommendationRunner.FACTORS)));
    }

    LongPrimitiveIterator users = testModel.getUserIDs();

    TemporalDataModelIF<Long, Long> model = null;
    switch (opts) {
        case RETURN_AND_OUTPUT_RECS:
        case RETURN_RECS:
            model = new TemporalDataModel<>();
            break;
        default:
            model = null;
    }
    String name = null;
    switch (opts) {
        case RETURN_AND_OUTPUT_RECS:
        case OUTPUT_RECS:
            name = getFileName();
            break;
        default:
            name = null;
    }
    boolean createFile = true;
    while (users.hasNext()) {
        long u = users.nextLong();
        try {
            List<RecommendedItem> items = recommender.recommend(u, trainingModel.getNumItems());
            //
            List<RecommenderIO.Preference<Long, Long>> prefs = new ArrayList<>();
            for (RecommendedItem i : items) {
                prefs.add(new RecommenderIO.Preference<>(u, i.getItemID(), i.getValue()));
            }
            //
            RecommenderIO.writeData(u, prefs, getPath(), name, !createFile, model);
            createFile = false;
        } catch (TasteException e) {
            e.printStackTrace();
        }
    }
    return model;
}