it.unimi.dsi.fastutil.ints.Int2FloatMap Java Examples

@Nonnull
private static Int2FloatMap int2floatMap(final int item, @Nonnull final Map<?, ?> map,
        @Nonnull final PrimitiveObjectInspector keyOI,
        @Nonnull final PrimitiveObjectInspector valueOI, @Nullable final FloatMatrix dataMatrix,
        @Nullable Int2FloatMap dst) {
    if (dst == null) {
        dst = new Int2FloatOpenHashMap(map.size());
        dst.defaultReturnValue(0.f);
    } else {
        dst.clear();
    }

    for (Map.Entry<?, ?> entry : map.entrySet()) {
        float rating = PrimitiveObjectInspectorUtils.getFloat(entry.getValue(), valueOI);
        if (rating == 0.f) {
            continue;
        }
        int user = PrimitiveObjectInspectorUtils.getInt(entry.getKey(), keyOI);
        dst.put(user, rating);
        if (dataMatrix != null) {
            dataMatrix.set(item, user, rating);
        }
    }

    return dst;
}
 

@Nonnull
private static Int2FloatMap int2floatMap(@Nonnull final Map<?, ?> map,
        @Nonnull final PrimitiveObjectInspector keyOI,
        @Nonnull final PrimitiveObjectInspector valueOI) {
    final Int2FloatMap result = new Int2FloatOpenHashMap(map.size());
    result.defaultReturnValue(0.f);

    for (Map.Entry<?, ?> entry : map.entrySet()) {
        float v = PrimitiveObjectInspectorUtils.getFloat(entry.getValue(), valueOI);
        if (v == 0.f) {
            continue;
        }
        int k = PrimitiveObjectInspectorUtils.getInt(entry.getKey(), keyOI);
        result.put(k, v);
    }

    return result;
}
 

@Nonnull
private static Int2ObjectMap<Int2FloatMap> kNNentries(@Nonnull final Object kNNiObj,
        @Nonnull final MapObjectInspector knnItemsOI,
        @Nonnull final PrimitiveObjectInspector knnItemsKeyOI,
        @Nonnull final MapObjectInspector knnItemsValueOI,
        @Nonnull final PrimitiveObjectInspector knnItemsValueKeyOI,
        @Nonnull final PrimitiveObjectInspector knnItemsValueValueOI,
        @Nullable Int2ObjectMap<Int2FloatMap> knnItems, @Nonnull final MutableInt nnzKNNi) {
    if (knnItems == null) {
        knnItems = new Int2ObjectOpenHashMap<>(1024);
    } else {
        knnItems.clear();
    }

    int numElementOfKNNItems = 0;
    for (Map.Entry<?, ?> entry : knnItemsOI.getMap(kNNiObj).entrySet()) {
        int user = PrimitiveObjectInspectorUtils.getInt(entry.getKey(), knnItemsKeyOI);
        Int2FloatMap ru = int2floatMap(knnItemsValueOI.getMap(entry.getValue()),
            knnItemsValueKeyOI, knnItemsValueValueOI);
        knnItems.put(user, ru);
        numElementOfKNNItems += ru.size();
    }

    nnzKNNi.setValue(numElementOfKNNItems);
    return knnItems;
}
 

private static double predict(final int user, final int itemI,
        @Nonnull final Int2ObjectMap<Int2FloatMap> knnItems, final int excludeIndex,
        @Nonnull final FloatMatrix weightMatrix) {
    final Int2FloatMap kNNu = knnItems.get(user);
    if (kNNu == null) {
        return 0.d;
    }

    double pred = 0.d;
    for (Int2FloatMap.Entry e : Fastutil.fastIterable(kNNu)) {
        final int itemK = e.getIntKey();
        if (itemK == excludeIndex) {
            continue;
        }
        float ruk = e.getFloatValue();
        pred += ruk * weightMatrix.get(itemI, itemK, 0.d);
    }
    return pred;
}
 

private void replayTrain(@Nonnull final ByteBuffer buf) {
    final int itemI = buf.getInt();
    final int knnSize = buf.getInt();

    final Int2ObjectMap<Int2FloatMap> knnItems = new Int2ObjectOpenHashMap<>(1024);
    final IntSet pairItems = new IntOpenHashSet();
    for (int i = 0; i < knnSize; i++) {
        int user = buf.getInt();
        int ruSize = buf.getInt();
        Int2FloatMap ru = new Int2FloatOpenHashMap(ruSize);
        ru.defaultReturnValue(0.f);

        for (int j = 0; j < ruSize; j++) {
            int itemK = buf.getInt();
            pairItems.add(itemK);
            float ruk = buf.getFloat();
            ru.put(itemK, ruk);
        }
        knnItems.put(user, ru);
    }

    for (int itemJ : pairItems) {
        train(itemI, knnItems, itemJ);
    }
}
 

@Override
protected Int2FloatMap calculateTermFrequency(Int2FloatMap keyValues) {
    for (Int2FloatMap.Entry term : keyValues.int2FloatEntrySet()) {
        term.setValue(1F);
    }
    return keyValues;
}
 

private void train(final int itemI, @Nonnull final Int2FloatMap ri,
        @Nonnull final Int2ObjectMap<Int2FloatMap> kNNi, final int itemJ,
        @Nonnull final Int2FloatMap rj) {
    final FloatMatrix W = _weightMatrix;

    final int N = rj.size();
    if (N == 0) {
        return;
    }

    double gradSum = 0.d;
    double rateSum = 0.d;
    double lossSum = 0.d;

    for (Int2FloatMap.Entry e : Fastutil.fastIterable(rj)) {
        int user = e.getIntKey();
        double ruj = e.getFloatValue();
        double rui = ri.get(user); // ri.getOrDefault(user, 0.f);

        double eui = rui - predict(user, itemI, kNNi, itemJ, W);
        gradSum += ruj * eui;
        rateSum += ruj * ruj;
        lossSum += eui * eui;
    }

    gradSum /= N;
    rateSum /= N;

    double wij = W.get(itemI, itemJ, 0.d);
    double loss = lossSum / N + 0.5d * l2 * wij * wij + l1 * wij;
    _cvState.incrLoss(loss);

    W.set(itemI, itemJ, getUpdateTerm(gradSum, rateSum, l1, l2));
}
 

@Override
public void process(@Nonnull Object[] args) throws HiveException {
    if (_weightMatrix == null) {// initialize variables
        this._weightMatrix = new DoKFloatMatrix();
        if (numIterations >= 2) {
            this._dataMatrix = new DoKFloatMatrix();
        }
        this._nnzKNNi = new MutableInt();
    }

    final int itemI = PrimitiveObjectInspectorUtils.getInt(args[0], itemIOI);

    if (itemI != _previousItemId || _ri == null) {
        // cache Ri and kNNi
        this._ri =
                int2floatMap(itemI, riOI.getMap(args[1]), riKeyOI, riValueOI, _dataMatrix, _ri);
        this._kNNi = kNNentries(args[2], knnItemsOI, knnItemsKeyOI, knnItemsValueOI,
            knnItemsValueKeyOI, knnItemsValueValueOI, _kNNi, _nnzKNNi);

        final int numKNNItems = _nnzKNNi.getValue();
        if (numIterations >= 2 && numKNNItems >= 1) {
            recordTrainingInput(itemI, _kNNi, numKNNItems);
        }
        this._previousItemId = itemI;
    }

    int itemJ = PrimitiveObjectInspectorUtils.getInt(args[3], itemJOI);
    Int2FloatMap rj =
            int2floatMap(itemJ, rjOI.getMap(args[4]), rjKeyOI, rjValueOI, _dataMatrix);

    train(itemI, _ri, _kNNi, itemJ, rj);
    _observedTrainingExamples++;
}
 

@Override
public HashInstance iterateQuantityFeatures(QuantityAccessor accessor) {
    for (Int2FloatMap.Entry term : this.quantityFeatures.int2FloatEntrySet()) {
        accessor.accessorFeature(term.getIntKey(), term.getFloatValue());
    }
    return this;
}
 

private void train(final int itemI, @Nonnull final Int2ObjectMap<Int2FloatMap> knnItems,
        final int itemJ) {
    final FloatMatrix A = _dataMatrix;
    final FloatMatrix W = _weightMatrix;

    final int N = A.numColumns(itemJ);
    if (N == 0) {
        return;
    }

    final MutableDouble mutableGradSum = new MutableDouble(0.d);
    final MutableDouble mutableRateSum = new MutableDouble(0.d);
    final MutableDouble mutableLossSum = new MutableDouble(0.d);

    A.eachNonZeroInRow(itemJ, new VectorProcedure() {
        @Override
        public void apply(int user, double ruj) {
            double rui = A.get(itemI, user, 0.d);
            double eui = rui - predict(user, itemI, knnItems, itemJ, W);

            mutableGradSum.addValue(ruj * eui);
            mutableRateSum.addValue(ruj * ruj);
            mutableLossSum.addValue(eui * eui);
        }
    });

    double gradSum = mutableGradSum.getValue() / N;
    double rateSum = mutableRateSum.getValue() / N;

    double wij = W.get(itemI, itemJ, 0.d);
    double loss = mutableLossSum.getValue() / N + 0.5 * l2 * wij * wij + l1 * wij;
    _cvState.incrLoss(loss);

    W.set(itemI, itemJ, getUpdateTerm(gradSum, rateSum, l1, l2));
}
 

@Override
protected Int2FloatMap calculateInverseDocumentFrequency(Int2FloatMap keyValues) {
    for (Int2FloatMap.Entry term : keyValues.int2FloatEntrySet()) {
        term.setValue((float) FastMath.log((2F - term.getFloatValue()) / term.getFloatValue()));
    }
    return keyValues;
}
 

@Override
protected Int2FloatMap calculateInverseDocumentFrequency(Int2FloatMap keyValues) {
    for (Int2FloatMap.Entry term : keyValues.int2FloatEntrySet()) {
        term.setValue((float) FastMath.log(2F / term.getFloatValue()));
    }
    return keyValues;
}
 

@Override
protected Int2FloatMap calculateInverseDocumentFrequency(Int2FloatMap keyValues) {
    for (Int2FloatMap.Entry term : keyValues.int2FloatEntrySet()) {
        term.setValue(1F);
    }
    return keyValues;
}
 

@Override
protected Int2FloatMap calculateInverseDocumentFrequency(Int2FloatMap keyValues) {
    for (Int2FloatMap.Entry term : keyValues.int2FloatEntrySet()) {
        term.setValue((float) FastMath.log(1F + 2F / term.getFloatValue()));
    }
    return keyValues;
}
 

@Override
protected Int2FloatMap calculateInverseDocumentFrequency(Int2FloatMap keyValues) {
    for (Int2FloatMap.Entry term : keyValues.int2FloatEntrySet()) {
        term.setValue((float) FastMath.log(2F / (1F + term.getFloatValue())));
    }
    return keyValues;
}
 

@Override
protected Int2FloatMap calculateTermFrequency(Int2FloatMap keyValues) {
    for (Int2FloatMap.Entry term : keyValues.int2FloatEntrySet()) {
        term.setValue((float) FastMath.log(1F + term.getFloatValue()));
    }
    return keyValues;
}
 

public NormalizationTermFrequency(Int2FloatMap keyValues, float k, Collection<Integer> document) {
	super(keyValues, document.size());
	assert k >= 0 && k < 1;
	float denominator = 0F;
	for (int term : document) {
		float numerator = keyValues.getOrDefault(term, 0F) + 1F;
		keyValues.put(term, numerator);
		if (numerator > denominator) {
			denominator = numerator;
		}
	}
	for (int term : keyValues.keySet()) {
		keyValues.put(term, k + (1 - k) * keyValues.get(term) / denominator);
	}
}
 

public NormalizationTermFrequency(Int2FloatMap keyValues, float k, int... document) {
	super(keyValues, document.length);
	assert k >= 0 && k < 1;
	float denominator = 0F;
	for (int term : document) {
		float numerator = keyValues.getOrDefault(term, 0F) + 1F;
		keyValues.put(term, numerator);
		if (numerator > denominator) {
			denominator = numerator;
		}
	}
	for (int term : keyValues.keySet()) {
		keyValues.put(term, k + (1 - k) * keyValues.get(term) / denominator);
	}
}
 

public UnaryInverseDocumentFrequency(Int2FloatMap keyValues, Iterator<TermFrequency> documents) {
	super(keyValues);
	while (documents.hasNext()) {
		TermFrequency document = documents.next();
		IntIterator iterator = document.getKeys().iterator();
		while (iterator.hasNext()) {
			int term = iterator.nextInt();
			keyValues.put(term, 1F);
		}
	}
}
 

public UnaryInverseDocumentFrequency(Int2FloatMap keyValues, Collection<TermFrequency> documents) {
	super(keyValues);
	for (TermFrequency document : documents) {
		IntIterator iterator = document.getKeys().iterator();
		while (iterator.hasNext()) {
			int term = iterator.nextInt();
			keyValues.put(term, 1F);
		}
	}
}
 

public UnaryInverseDocumentFrequency(Int2FloatMap keyValues, TermFrequency... documents) {
	super(keyValues);
	for (TermFrequency document : documents) {
		IntIterator iterator = document.getKeys().iterator();
		while (iterator.hasNext()) {
			int term = iterator.nextInt();
			keyValues.put(term, 1F);
		}
	}
}
 

@Nonnull
private static Int2FloatMap int2floatMap(final int item, @Nonnull final Map<?, ?> map,
        @Nonnull final PrimitiveObjectInspector keyOI,
        @Nonnull final PrimitiveObjectInspector valueOI,
        @Nullable final FloatMatrix dataMatrix) {
    return int2floatMap(item, map, keyOI, valueOI, dataMatrix, null);
}
 

@Nonnull
public static ObjectIterable<Int2FloatMap.Entry> fastIterable(@Nonnull final Int2FloatMap map) {
    final ObjectSet<Int2FloatMap.Entry> entries = map.int2FloatEntrySet();
    return entries instanceof Int2FloatMap.FastEntrySet
            ? new ObjectIterable<Int2FloatMap.Entry>() {
                public ObjectIterator<Int2FloatMap.Entry> iterator() {
                    return ((Int2FloatMap.FastEntrySet) entries).fastIterator();
                }
            }
            : entries;
}
 

@Override
protected Int2FloatMap calculateTermFrequency(Int2FloatMap keyValues) {
    return keyValues;
}
 

private void recordTrainingInput(final int itemI,
        @Nonnull final Int2ObjectMap<Int2FloatMap> knnItems, final int numKNNItems)
        throws HiveException {
    ByteBuffer buf = this._inputBuf;
    NioStatefulSegment dst = this._fileIO;

    if (buf == null) {
        // invoke only at task node (initialize is also invoked in compilation)
        final File file;
        try {
            file = File.createTempFile("hivemall_slim", ".sgmt"); // to save KNN data
            file.deleteOnExit();
            if (!file.canWrite()) {
                throw new UDFArgumentException(
                    "Cannot write a temporary file: " + file.getAbsolutePath());
            }
        } catch (IOException ioe) {
            throw new UDFArgumentException(ioe);
        }

        this._inputBuf = buf = ByteBuffer.allocateDirect(8 * 1024 * 1024); // 8MB
        this._fileIO = dst = new NioStatefulSegment(file, false);
    }

    int recordBytes = SizeOf.INT + SizeOf.INT + SizeOf.INT * 2 * knnItems.size()
            + (SizeOf.INT + SizeOf.FLOAT) * numKNNItems;
    int requiredBytes = SizeOf.INT + recordBytes; // need to allocate space for "recordBytes" itself

    int remain = buf.remaining();
    if (remain < requiredBytes) {
        writeBuffer(buf, dst);
    }

    buf.putInt(recordBytes);
    buf.putInt(itemI);
    buf.putInt(knnItems.size());

    for (Int2ObjectMap.Entry<Int2FloatMap> e1 : Fastutil.fastIterable(knnItems)) {
        int user = e1.getIntKey();
        buf.putInt(user);

        Int2FloatMap ru = e1.getValue();
        buf.putInt(ru.size());
        for (Int2FloatMap.Entry e2 : Fastutil.fastIterable(ru)) {
            buf.putInt(e2.getIntKey());
            buf.putFloat(e2.getFloatValue());
        }
    }
}
 

protected AbstractTermFrequency(Int2FloatMap keyValues, int length) {
	this.keyValues = keyValues;
	this.length = length;
}
 

public BinaryTermFrequency(Int2FloatMap keyValues, Collection<Integer> document) {
	super(keyValues, document.size());
	for (int term : document) {
		keyValues.put(term, 1F);
	}
}
 

public BinaryTermFrequency(Int2FloatMap keyValues, int... document) {
	super(keyValues, document.length);
	for (int term : document) {
		keyValues.put(term, 1F);
	}
}
 

public CountTermFrequency(Int2FloatMap keyValues, Collection<Integer> document) {
	super(keyValues, document.size());
	for (int term : document) {
		keyValues.put(term, keyValues.getOrDefault(term, 0F) + 1F);
	}
}
 

public CountTermFrequency(Int2FloatMap keyValues, int... document) {
	super(keyValues, document.length);
	for (int term : document) {
		keyValues.put(term, keyValues.getOrDefault(term, 0F) + 1F);
	}
}