Java Code Examples for com.jstarcraft.core.utility.KeyValue#getKey()

/**
 * 个性化搜索
 * 
 * @param userIndex
 * @param searchKey
 * @return
 * @throws Exception
 */
@LockableMethod(strategy = HashLockableStrategy.class)
public Object2FloatMap<MovieItem> getSearchItems(@LockableParameter int userIndex, String searchKey) throws Exception {
    // 标识-得分映射
    Object2FloatMap<MovieItem> item2ScoreMap = new Object2FloatOpenHashMap<>();

    long current = System.currentTimeMillis();
    Query query = queryParser.parse(searchKey, MovieItem.TITLE);
    KeyValue<List<Document>, FloatList> search = engine.retrieveDocuments(query, null, 0, 1000);
    List<Document> documents = search.getKey();
    FloatList scores = search.getValue();
    for (int index = 0, size = documents.size(); index < size; index++) {
        Document document = documents.get(index);
        MovieItem item = items.get(document.getField(MovieItem.INDEX).numericValue().intValue());
        float score = scores.getFloat(index);
        item2ScoreMap.put(item, score);
    }
    String message = StringUtility.format("搜索数量:{},搜索耗时:{}", documents.size(), System.currentTimeMillis() - current);
    logger.info(message);

    return item2ScoreMap;
}
 

@Override
public <K extends Comparable, T extends IdentityObject<K>> List<T> queryUnion(Class<T> clazz, Map<String, Object> condition, StoragePagination pagination) {
	LuceneMetadata metadata = metadatas.get(clazz);
	Query query = null;
	BooleanQuery.Builder buffer = new BooleanQuery.Builder();
	for (Entry<String, Object> term : condition.entrySet()) {
		KeyValue<Field, IndexConverter> keyValue = metadata.getIndexKeyValue(term.getKey());
		Field key = keyValue.getKey();
		IndexConverter value = keyValue.getValue();
		query = value.query(context, term.getKey(), key, key.getAnnotation(LuceneIndex.class), key.getGenericType(), ConditionType.Equal, term.getValue());
		buffer.add(query, Occur.SHOULD);
	}
	query = buffer.build();
	int offset = pagination == null ? 0 : pagination.getFirst();
	int size = pagination == null ? Integer.MAX_VALUE : pagination.getSize();
	KeyValue<List<Document>, FloatList> retrieve = engine.retrieveDocuments(query, null, offset, size);
	List<Document> documents = retrieve.getKey();
	List<T> list = new ArrayList<>(BATCH_SIZE);
	for (Document document : documents) {
		list.add((T) metadata.decodeDocument(document));
	}
	return list;
}
 

@Override
public <K extends Comparable, T extends IdentityObject<K>> void iterateUnion(StorageIterator<T> iterator, Class<T> clazz, Map<String, Object> condition, StoragePagination pagination) {
	LuceneMetadata metadata = metadatas.get(clazz);
	Query query = null;
	BooleanQuery.Builder buffer = new BooleanQuery.Builder();
	for (Entry<String, Object> term : condition.entrySet()) {
		KeyValue<Field, IndexConverter> keyValue = metadata.getIndexKeyValue(term.getKey());
		Field key = keyValue.getKey();
		IndexConverter value = keyValue.getValue();
		query = value.query(context, term.getKey(), key, key.getAnnotation(LuceneIndex.class), key.getGenericType(), ConditionType.Equal, term.getValue());
		buffer.add(query, Occur.SHOULD);
	}
	query = buffer.build();
	int offset = pagination == null ? 0 : pagination.getFirst();
	int size = pagination == null ? Integer.MAX_VALUE : pagination.getSize();
	engine.iterateDocuments((document) -> {
		iterator.iterate((T) metadata.decodeDocument(document));
	}, query, null, offset, size);
}
 

/**
 * Compute group-to-item AR and store them into map itemCorrsGAR
 */
private void computeAssociationRuleByGroup(int groupIndex, LinkedList<KeyValue<Integer, Integer>> itemList) {
    List<KeyValue<KeyValue<Integer, Integer>, Float>> coefficientList = new LinkedList<>();

    for (KeyValue<Integer, Integer> keyValue : itemList) {
        int leftIndex = keyValue.getKey();
        int rightIndex = keyValue.getValue();
        SparseVector groupVector = scoreMatrix.getColumnVector(groupIndex);
        int count = 0;
        for (VectorScalar term : groupVector) {
            int userIndex = term.getIndex();
            if (dataTable.contains(userIndex, leftIndex) && dataTable.contains(userIndex, rightIndex)) {
                count++;
            }
        }
        if (count > 0) {
            float shrink = count / (count + reliability);
            int co_bc = itemCount.get(leftIndex, rightIndex);
            float coefficient = shrink * (count + 0F) / co_bc;
            coefficientList.add(new KeyValue<>(keyValue, coefficient));
        }
    }
    itemCorrsGAR.put(groupIndex, new ArrayList<>(coefficientList));
}
 

/**
 * 预测
 * 
 * @param inputs
 * @param outputs
 */
public void predict(MathMatrix[] samples, MathMatrix[] marks) {
    doCache(samples, marks);
    for (int index = 0, size = marks.length; index < size; index++) {
        // 检查数量
        if (marks[index].getRowSize() != numberOfSamples) {
            throw new IllegalArgumentException();
        }
    }

    doForward();
    for (int index = 0, size = outputVertices.length; index < size; index++) {
        Vertex vertex = outputVertices[index];
        KeyValue<MathMatrix, MathMatrix> keyValue = vertex.getOutputKeyValue();
        MathMatrix outputData = keyValue.getKey();
        marks[index].iterateElement(MathCalculator.PARALLEL, (scalar) -> {
            scalar.setValue(outputData.getValue(scalar.getRow(), scalar.getColumn()));
        });
    }
}
 

@Override
public <K extends Comparable, I, T extends IdentityObject<K>> List<T> queryInstances(Class<T> clazz, String name, StorageCondition<I> condition) {
	LuceneMetadata metadata = metadatas.get(clazz);
	Query query;
	{
		KeyValue<Field, IndexConverter> keyValue = metadata.getIndexKeyValue(name);
		Field key = keyValue.getKey();
		IndexConverter value = keyValue.getValue();
		query = value.query(context, metadata.getPrimaryName(), key, key.getAnnotation(LuceneIndex.class), key.getGenericType(), condition.getType(), condition.getValues());
	}
	KeyValue<List<Document>, FloatList> retrieve = engine.retrieveDocuments(query, null, 0, Integer.MAX_VALUE);
	List<Document> documents = retrieve.getKey();
	List<T> list = new ArrayList<>(BATCH_SIZE);
	for (Document document : documents) {
		list.add((T) metadata.decodeDocument(document));
	}
	return list;
}
 

@Override
public <K extends Comparable, T extends IdentityObject<K>> void iterateIntersection(StorageIterator<T> iterator, Class<T> clazz, Map<String, Object> condition, StoragePagination pagination) {
	LuceneMetadata metadata = metadatas.get(clazz);
	Query query = null;
	BooleanQuery.Builder buffer = new BooleanQuery.Builder();
	for (Entry<String, Object> term : condition.entrySet()) {
		KeyValue<Field, IndexConverter> keyValue = metadata.getIndexKeyValue(term.getKey());
		Field key = keyValue.getKey();
		IndexConverter value = keyValue.getValue();
		query = value.query(context, term.getKey(), key, key.getAnnotation(LuceneIndex.class), key.getGenericType(), ConditionType.Equal, term.getValue());
		buffer.add(query, Occur.MUST);
	}
	query = buffer.build();
	int offset = pagination == null ? 0 : pagination.getFirst();
	int size = pagination == null ? Integer.MAX_VALUE : pagination.getSize();
	engine.iterateDocuments((document) -> {
		iterator.iterate((T) metadata.decodeDocument(document));
	}, query, null, offset, size);
}
 

public void connect(Vertex vertex, String... dependencies) {
    String name = vertex.getVertexName();
    int index = vertices.size();
    KeyValue<Integer, Vertex> vertexKeyValue = new KeyValue<>(index, vertex);
    if (vertices.putIfAbsent(name, vertexKeyValue) != null) {
        throw new IllegalArgumentException("节点冲突");
    }

    for (String dependency : dependencies) {
        vertexKeyValue = vertices.get(dependency);
        if (vertexKeyValue == null) {
            throw new IllegalArgumentException("节点缺失");
        }
        Integer2IntegerKeyValue edgeKeyValue = new Integer2IntegerKeyValue(vertexKeyValue.getKey(), index);
        if (!edges.add(edgeKeyValue)) {
            throw new IllegalArgumentException("边冲突");
        }
    }
}
 

/**
 * 
 * @param userIndex
 * @param modelKey
 * @param queryKey
 * @param filterClicked
 * @return
 * @throws Exception
 */
@LockableMethod(strategy = HashLockableStrategy.class)
public Object2FloatMap<MovieItem> getItems(@LockableParameter int userIndex, String modelKey, String queryKey, boolean filterClicked) throws Exception {
    // 标识-得分映射
    Object2FloatMap<MovieItem> item2ScoreMap = new Object2FloatOpenHashMap<>();

    long current = System.currentTimeMillis();
    Model model = models.get(modelKey);
    ArrayInstance instance = new ArrayInstance(qualityOrder, quantityOrder);
    MovieUser user = users.get(userIndex);
    Query query = StringUtility.isBlank(queryKey) ? new MatchAllDocsQuery() : queryParser.parse(queryKey, MovieItem.TITLE);
    KeyValue<List<Document>, FloatList> retrieve = engine.retrieveDocuments(query, null, 0, 1000);
    List<Document> documents = retrieve.getKey();
    for (int index = 0, size = documents.size(); index < size; index++) {
        Document document = documents.get(index);
        MovieItem item = items.get(document.getField(MovieItem.INDEX).numericValue().intValue());
        int itemIndex = item.getIndex();
        // 过滤条目
        if (filterClicked && user.isClicked(itemIndex)) {
            continue;
        }
        instance.setQualityFeature(userDimension, userIndex);
        instance.setQualityFeature(itemDimension, itemIndex);
        model.predict(instance);
        float score = instance.getQuantityMark();
        item2ScoreMap.put(item, score);
    }
    String message = StringUtility.format("预测数量:{},预测耗时:{}", modelKey, documents.size(), System.currentTimeMillis() - current);
    logger.info(message);

    return item2ScoreMap;
}
 

@Override
public <K extends Comparable, T extends IdentityObject<K>> long countUnion(Class<T> clazz, Map<String, Object> condition) {
	LuceneMetadata metadata = metadatas.get(clazz);
	Query query = null;
	BooleanQuery.Builder buffer = new BooleanQuery.Builder();
	for (Entry<String, Object> term : condition.entrySet()) {
		KeyValue<Field, IndexConverter> keyValue = metadata.getIndexKeyValue(term.getKey());
		Field key = keyValue.getKey();
		IndexConverter value = keyValue.getValue();
		query = value.query(context, term.getKey(), key, key.getAnnotation(LuceneIndex.class), key.getGenericType(), ConditionType.Equal, term.getValue());
		buffer.add(query, Occur.SHOULD);
	}
	query = buffer.build();
	return engine.countDocuments(query);
}
 

@Override
public <K extends Comparable, T extends IdentityObject<K>> long countIntersection(Class<T> clazz, Map<String, Object> condition) {
	LuceneMetadata metadata = metadatas.get(clazz);
	Query query = null;
	BooleanQuery.Builder buffer = new BooleanQuery.Builder();
	for (Entry<String, Object> term : condition.entrySet()) {
		KeyValue<Field, IndexConverter> keyValue = metadata.getIndexKeyValue(term.getKey());
		Field key = keyValue.getKey();
		IndexConverter value = keyValue.getValue();
		query = value.query(context, term.getKey(), key, key.getAnnotation(LuceneIndex.class), key.getGenericType(), ConditionType.Equal, term.getValue());
		buffer.add(query, Occur.MUST);
	}
	query = buffer.build();
	return engine.countDocuments(query);
}
 

@Override
public <K extends Comparable, T extends IdentityObject<K>> List<T> queryInstances(Class<T> clazz, StoragePagination pagination) {
	LuceneMetadata metadata = metadatas.get(clazz);
	Query query = new MatchAllDocsQuery();
	int offset = pagination == null ? 0 : pagination.getFirst();
	int size = pagination == null ? Integer.MAX_VALUE : pagination.getSize();
	KeyValue<List<Document>, FloatList> retrieve = engine.retrieveDocuments(query, null, offset, size);
	List<Document> documents = retrieve.getKey();
	List<T> list = new ArrayList<>(BATCH_SIZE);
	for (Document document : documents) {
		list.add((T) metadata.decodeDocument(document));
	}
	return list;
}
 

@Override
public <K extends Comparable, T extends IdentityObject<K>> T getInstance(Class<T> clazz, K id) {
	LuceneMetadata metadata = metadatas.get(clazz);
	KeyValue<Field, IndexConverter> keyValue = metadata.getIndexKeyValue(metadata.getPrimaryName());
	Field key = keyValue.getKey();
	IndexConverter value = keyValue.getValue();
	Query query = value.query(context, metadata.getPrimaryName(), key, key.getAnnotation(LuceneIndex.class), key.getGenericType(), ConditionType.Equal, id);
	KeyValue<List<Document>, FloatList> retrieve = engine.retrieveDocuments(query, null, 0, 100);
	List<Document> documents = retrieve.getKey();
	if (documents.size() > 0) {
		return (T) metadata.decodeDocument(documents.get(0));
	} else {
		return null;
	}
}
 

@Override
public void doForward() {
    MathMatrix outputData = outputKeyValue.getKey();
    outputData.setValues(0F);
    for (KeyValue<MathMatrix, MathMatrix> keyValue : inputKeyValues) {
        MathMatrix inputData = keyValue.getKey();
        outputData.addMatrix(inputData, false);
    }
    MathMatrix innerError = outputKeyValue.getValue();
    innerError.setValues(0F);
}
 

@Override
public float computeScore(MathMatrix tests, MathMatrix trains, MathMatrix masks) {
    KeyValue<Float, Float> keyValue = computeNumeratorWithDenominator(tests, trains, masks);
    float numerator = keyValue.getKey();
    float denominator = keyValue.getValue();
    if (numerator == 0F && denominator == 0F) {
        return 0F;
    }
    return 1F - numerator / denominator;
}
 

@Override
public void prepare(Configurator configuration, DataModule model, DataSpace space) {
    userField = configuration.getString("data.model.fields.user", "user");
    itemField = configuration.getString("data.model.fields.item", "item");

    userDimension = model.getQualityInner(userField);
    itemDimension = model.getQualityInner(itemField);
    userSize = space.getQualityAttribute(userField).getSize();
    itemSize = space.getQualityAttribute(itemField).getSize();

    DataSplitter splitter = new QualityFeatureDataSplitter(userDimension);
    DataModule[] models = splitter.split(model, userSize);
    DataSorter sorter = new AllFeatureDataSorter();
    for (int index = 0; index < userSize; index++) {
        models[index] = sorter.sort(models[index]);
    }

    HashMatrix dataTable = new HashMatrix(true, userSize, itemSize, new Long2FloatRBTreeMap());
    for (DataInstance instance : model) {
        int rowIndex = instance.getQualityFeature(userDimension);
        int columnIndex = instance.getQualityFeature(itemDimension);
        dataTable.setValue(rowIndex, columnIndex, instance.getQuantityMark());
    }
    scoreMatrix = SparseMatrix.valueOf(userSize, itemSize, dataTable);
    actionSize = scoreMatrix.getElementSize();
    KeyValue<Float, Float> attribute = scoreMatrix.getBoundary(false);
    minimumScore = attribute.getKey();
    maximumScore = attribute.getValue();
    meanScore = scoreMatrix.getSum(false);
    meanScore /= actionSize;
}
 

@Override
protected NlpTokenizer<? extends NlpToken> getNlpTokenizer(Map<String, String> configurations) {
    KeyValue<JiebaSegmenter, SegMode> keyValue = build(configurations);
    JiebaSegmenter segmenter = keyValue.getKey();
    SegMode mode = keyValue.getValue();

    JiebaTokenizer tokenizer = new JiebaTokenizer(segmenter, mode);
    return tokenizer;
}
 

@Override
protected NlpTokenizer<? extends NlpToken> getNlpTokenizer(Map<String, String> configurations) {
    KeyValue<Analysis, List<Recognition>> keyValue = build(configurations);
    Analysis analysis = keyValue.getKey();
    List<Recognition> recognitions = keyValue.getValue();

    AnsjTokenizer tokenizer = new AnsjTokenizer(analysis, recognitions);
    return tokenizer;
}
 

@Override
public void doCache(KeyValue<MathMatrix, MathMatrix>... samples) {
    // 检查样本
    if (samples.length == 0) {
        throw new IllegalArgumentException();
    }

    this.inputKeyValues = samples;
    this.outputKeyValue = new KeyValue<>(null, null);
    this.middleKeyValue = new KeyValue<>(null, null);

    inputLocalDatas = new MathMatrix[numberOfShares];
    middleLocalDatas = new MathMatrix[numberOfShares];
    outputLocalDatas = new MathMatrix[numberOfShares];
    outterLocalErrors = new MathMatrix[numberOfShares];
    middleLocalErrors = new MathMatrix[numberOfShares];
    innerLocalErrors = new MathMatrix[numberOfShares];

    for (int shareIndex = 0; shareIndex < numberOfShares; shareIndex++) {
        // 输入部分
        MathMatrix key = inputKeyValues[0].getKey();
        MathMatrix value = inputKeyValues[0].getValue();
        int from = shareIndex * key.getColumnSize() / numberOfShares;
        int to = from + key.getColumnSize() / numberOfShares;

        if (key instanceof ColumnGlobalMatrix) {
            key = ColumnGlobalMatrix.detachOf(ColumnGlobalMatrix.class.cast(key), from, to);
            if (value != null) {
                value = ColumnGlobalMatrix.detachOf(ColumnGlobalMatrix.class.cast(value), from, to);
            }
        } else {
            key = new LocalMatrix(key, from, to, 0, key.getRowSize());
            if (value != null) {
                value = new LocalMatrix(value, from, to, 0, key.getRowSize());
            }
        }

        KeyValue<MathMatrix, MathMatrix> keyValue = new KeyValue<>(key, value);
        layer.doCache(factory, keyValue);

        keyValue = layer.getInputKeyValue();
        inputLocalDatas[shareIndex] = keyValue.getKey();
        outterLocalErrors[shareIndex] = keyValue.getValue();
        keyValue = layer.getMiddleKeyValue();
        middleLocalDatas[shareIndex] = keyValue.getKey();
        middleLocalErrors[shareIndex] = keyValue.getValue();
        keyValue = layer.getOutputKeyValue();
        outputLocalDatas[shareIndex] = keyValue.getKey();
        innerLocalErrors[shareIndex] = keyValue.getValue();
    }

    inputGlobalData = ColumnGlobalMatrix.attachOf(inputLocalDatas);
    middleGlobalData = ColumnGlobalMatrix.attachOf(middleLocalDatas);
    outputGlobalData = ColumnGlobalMatrix.attachOf(outputLocalDatas);
    if (inputKeyValues[0].getValue() != null) {
        outterGlobalError = ColumnGlobalMatrix.attachOf(outterLocalErrors);
    }
    middleGlobalError = ColumnGlobalMatrix.attachOf(middleLocalErrors);
    innerGlobalError = ColumnGlobalMatrix.attachOf(innerLocalErrors);

    // 中间部分
    middleKeyValue.setKey(middleGlobalData);
    middleKeyValue.setValue(middleGlobalError);

    // 输出部分
    outputKeyValue.setKey(outputGlobalData);
    outputKeyValue.setValue(innerGlobalError);

    learner.doCache(layer.getGradients());
    epoch++;
    iteration = 0;
}
 

@Override
protected void doPractice() {
    for (int epocheIndex = 0; epocheIndex < epocheSize; epocheIndex++) {
        totalError = 0F;
        for (int sampleIndex = 0, sampleTimes = userSize * 100; sampleIndex < sampleTimes; sampleIndex++) {
            // randomly draw (userIdx, posItemIdx, negItemIdx)
            int userIndex, positiveItemIndex, negativeItemIndex = 0;
            List<KeyValue<Integer, Double>> probabilities;
            while (true) {
                userIndex = RandomUtility.randomInteger(userSize);
                SparseVector userVector = scoreMatrix.getRowVector(userIndex);
                if (userVector.getElementSize() == 0) {
                    continue;
                }
                positiveItemIndex = userVector.getIndex(RandomUtility.randomInteger(userVector.getElementSize()));
                // sample j by popularity (probability)
                probabilities = itemProbabilities[userIndex];
                double random = RandomUtility.randomDouble(1D);
                for (KeyValue<Integer, Double> term : probabilities) {
                    if ((random -= term.getValue()) <= 0D) {
                        negativeItemIndex = term.getKey();
                        break;
                    }
                }
                break;
            }

            // update parameters
            float positiveScore = predict(userIndex, positiveItemIndex);
            float negativeScore = predict(userIndex, negativeItemIndex);
            float error = positiveScore - negativeScore;
            float value = (float) -Math.log(LogisticUtility.getValue(error));
            totalError += value;
            value = LogisticUtility.getValue(-error);

            // update bias
            float positiveBias = itemBiases.getValue(positiveItemIndex), negativeBias = itemBiases.getValue(negativeItemIndex);
            itemBiases.shiftValue(positiveItemIndex, learnRatio * (value - biasRegularization * positiveBias));
            itemBiases.shiftValue(negativeItemIndex, learnRatio * (-value - biasRegularization * negativeBias));
            totalError += biasRegularization * (positiveBias * positiveBias + negativeBias * negativeBias);

            // update user/item vectors
            for (int factorIndex = 0; factorIndex < factorSize; factorIndex++) {
                float userFactor = userFactors.getValue(userIndex, factorIndex);
                float positiveItemFactor = itemFactors.getValue(positiveItemIndex, factorIndex);
                float negativeItemFactor = itemFactors.getValue(negativeItemIndex, factorIndex);
                userFactors.shiftValue(userIndex, factorIndex, learnRatio * (value * (positiveItemFactor - negativeItemFactor) - userRegularization * userFactor));
                itemFactors.shiftValue(positiveItemIndex, factorIndex, learnRatio * (value * userFactor - itemRegularization * positiveItemFactor));
                itemFactors.shiftValue(negativeItemIndex, factorIndex, learnRatio * (value * (-userFactor) - itemRegularization * negativeItemFactor));
                totalError += userRegularization * userFactor * userFactor + itemRegularization * positiveItemFactor * positiveItemFactor + itemRegularization * negativeItemFactor * negativeItemFactor;
            }
        }
        if (isConverged(epocheIndex) && isConverged) {
            break;
        }
        isLearned(epocheIndex);
        currentError = totalError;
    }
}