com.jstarcraft.ai.data.DataModule Java Examples

@Override
public void prepare(Configurator configuration, DataModule model, DataSpace space) {
    super.prepare(configuration, model, space);
    inputDimension = getInputDimension();
    hiddenDimension = configuration.getInteger("recommender.hidden.dimension");
    hiddenActivation = configuration.getString("recommender.hidden.activation");
    outputActivation = configuration.getString("recommender.output.activation");
    learnRatio = configuration.getFloat("recommender.iterator.learnrate");
    momentum = configuration.getFloat("recommender.iterator.momentum");
    weightRegularization = configuration.getFloat("recommender.weight.regularization");

    // transform the sparse matrix to INDArray
    int[] matrixShape = new int[] { itemSize, userSize };
    inputData = new Nd4jMatrix(Nd4j.zeros(matrixShape));
    maskData = new Nd4jMatrix(Nd4j.zeros(matrixShape));
    outputData = new Nd4jMatrix(Nd4j.zeros(matrixShape));
    for (MatrixScalar term : scoreMatrix) {
        if (term.getValue() > 0D) {
            inputData.setValue(term.getColumn(), term.getRow(), term.getValue());
            maskData.setValue(term.getColumn(), term.getRow(), 1F);
        }
    }
}
 

@Override
public void prepare(Configurator configuration, DataModule model, DataSpace space) {
    super.prepare(configuration, model, space);

    // Matrix for p * (factor * filed)
    // TODO 此处应该还是稀疏
    featureFactors = DenseMatrix.valueOf(featureSize, factorSize * marker.getQualityOrder());
    featureFactors.iterateElement(MathCalculator.SERIAL, (scalar) -> {
        scalar.setValue(distribution.sample().floatValue());
    });

    // init the map for feature of filed
    featureOrders = new int[featureSize];
    int count = 0;
    for (int orderIndex = 0, orderSize = dimensionSizes.length; orderIndex < orderSize; orderIndex++) {
        int size = dimensionSizes[orderIndex];
        for (int index = 0; index < size; index++) {
            featureOrders[count + index] = orderIndex;
        }
        count += size;
    }
}
 

@Override
public void prepare(Configurator configuration, DataModule model, DataSpace space) {
    super.prepare(configuration, model, space);
    // init Q
    // TODO 此处为rateFactors
    actionFactors = DenseMatrix.valueOf(actionSize, factorSize);

    // construct training appender matrix
    HashMatrix table = new HashMatrix(true, actionSize, featureSize, new Long2FloatRBTreeMap());
    int index = 0;
    int order = marker.getQualityOrder();
    for (DataInstance sample : model) {
        int count = 0;
        for (int orderIndex = 0; orderIndex < order; orderIndex++) {
            table.setValue(index, count + sample.getQualityFeature(orderIndex), 1F);
            count += dimensionSizes[orderIndex];
        }
        index++;
    }
    // TODO 考虑重构(.此处似乎就是FactorizationMachineRecommender.getFeatureVector);
    featureMatrix = SparseMatrix.valueOf(actionSize, featureSize, table);
}
 

@Override
public ReferenceModule sort(DataModule module) {
    int size = module.getSize();
    IntegerArray reference = new IntegerArray(size, size);
    for (int index = 0; index < size; index++) {
        reference.associateData(index);
    }
    int from = 0;
    int to = size;
    for (int index = from; index < to; index++) {
        int random = RandomUtility.randomInteger(from, to);
        int data = reference.getData(index);
        reference.setData(index, reference.getData(random));
        reference.setData(random, data);
    }
    return new ReferenceModule(reference, module);
}
 

/**
 * 划分
 * 
 * @param module
 * @param number
 * @return
 */
default ReferenceModule[] split(DataModule module, int number) {
    int size = module.getSize();
    int maximum = size;
    int minimum = 1000;
    if (maximum < minimum) {
        maximum = minimum;
    }
    IntegerArray[] references = new IntegerArray[number];
    for (int index = 0; index < number; index++) {
        references[index] = new IntegerArray(minimum, maximum);
    }
    int cursor = 0;
    DataInstance instance = module.getInstance(cursor);
    while (cursor < size) {
        instance.setCursor(cursor);
        references[split(instance) % number].associateData(cursor);
        cursor++;
    }
    ReferenceModule[] modules = new ReferenceModule[number];
    for (int index = 0; index < number; index++) {
        modules[index] = new ReferenceModule(references[index], module);
    }
    return modules;
}
 

/**
 * setup init member method
 *
 * @throws ModelException if error occurs during setting up
 */
@Override
public void prepare(Configurator configuration, DataModule model, DataSpace space) {
    super.prepare(configuration, model, space);
    factorSize = configuration.getInteger("recommender.topic.number", 10);
    burnIn = configuration.getInteger("recommender.pgm.burnin", 100);
    sampleSize = configuration.getInteger("recommender.pgm.samplelag", 100);

    // TODO 此处会与scoreIndexes一起重构,本质为连续特征离散化.
    FloatSet scores = new FloatRBTreeSet();
    for (MatrixScalar term : scoreMatrix) {
        scores.add(term.getValue());
    }
    scores.remove(0F);
    scoreIndexes = new Float2IntLinkedOpenHashMap();
    int index = 0;
    for (float score : scores) {
        scoreIndexes.put(score, index++);
    }
    scoreSize = scoreIndexes.size();
}
 

@Override
public void prepare(Configurator configuration, DataModule model, DataSpace space) {
    super.prepare(configuration, model, space);
    // 注意:FISM使用itemFactors来组成userFactors
    userFactors = DenseMatrix.valueOf(itemSize, factorSize);
    userFactors.iterateElement(MathCalculator.SERIAL, (scalar) -> {
        scalar.setValue(distribution.sample().floatValue());
    });
    itemFactors = DenseMatrix.valueOf(itemSize, factorSize);
    itemFactors.iterateElement(MathCalculator.SERIAL, (scalar) -> {
        scalar.setValue(distribution.sample().floatValue());
    });
    // TODO
    itemBiases = DenseVector.valueOf(itemSize);
    itemBiases.iterateElement(MathCalculator.SERIAL, (scalar) -> {
        scalar.setValue(distribution.sample().floatValue());
    });
    rho = configuration.getFloat("recommender.fismauc.rho");// 3-15
    alpha = configuration.getFloat("recommender.fismauc.alpha", 0.5F);
    beta = configuration.getFloat("recommender.fismauc.beta", 0.6F);
    gamma = configuration.getFloat("recommender.fismauc.gamma", 0.1F);
    biasRegularization = configuration.getFloat("recommender.iteration.learnrate", 0.0001F);
    // cacheSpec = conf.get("guava.cache.spec",
    // "maximumSize=200,expireAfterAccess=2m");
}
 

@Override
public void prepare(Configurator configuration, DataModule model, DataSpace space) {
    super.prepare(configuration, model, space);

    lossType = configuration.getInteger("losstype", 3);
    epsilon = configuration.getFloat("epsilon");
    orderLosses = new float[itemSize - 1];
    float orderLoss = 0F;
    for (int orderIndex = 1; orderIndex < itemSize; orderIndex++) {
        orderLoss += 1D / orderIndex;
        orderLosses[orderIndex - 1] = orderLoss;
    }
    for (int rankIndex = 1; rankIndex < itemSize; rankIndex++) {
        orderLosses[rankIndex - 1] /= orderLoss;
    }
}
 

private DataModule getDataModule() {
    List<KeyValue<KeyValue<String, Boolean>, Integer>> moduleDefinition = new LinkedList<>();
    for (int index = 0; index < order; index++) {
        moduleDefinition.add(new KeyValue<>(new KeyValue<>("quality", true), 1));
        moduleDefinition.add(new KeyValue<>(new KeyValue<>("ontinuous", false), 1));
    }
    DataModule module = new SparseModule(moduleName, moduleDefinition, instanceCapacity);
    Int2IntSortedMap qualityFeatures = new Int2IntAVLTreeMap();
    Int2FloatSortedMap quantityFeatures = new Int2FloatAVLTreeMap();
    for (int index = 0; index < instanceCapacity; index++) {
        qualityFeatures.clear();
        qualityFeatures.put(index, index);
        module.associateInstance(qualityFeatures, quantityFeatures);
    }
    return module;
}
 

@Override
public void practice(DataModule module, DataModule... contexts) {
    // 每个属性的概率估计器
    int size = attribute.getSize();
    priorDistribution = new DiscreteEstimator(size, true);
    posteriorQualityDistributions = new Estimator[module.getQualityOrder()][size];
    posteriorQuantityDistributions = new Estimator[module.getQuantityOrder()][size];
    for (int mark = 0; mark < size; mark++) {
        for (int dimension = 0, order = module.getQualityOrder(); dimension < order; dimension++) {
            posteriorQualityDistributions[dimension][mark] = new DiscreteEstimator(size, true);
        }
        for (int dimension = 0, order = module.getQuantityOrder(); dimension < order; dimension++) {
            // TODO 处理连续属性
        }
    }

    // 计算条件概率
    for (DataInstance instance : module) {
        update(instance);
    }
}
 

@Override
public void prepare(Configurator configuration, DataModule model, DataSpace space) {
    super.prepare(configuration, model, space);
    mode = configuration.getString("recommender.social.model", "T");
    // algoName = "TrustMF (" + model + ")";
    switch (mode) {
    case "Tr":
        prepareByTruster();
        break;
    case "Te":
        prepareByTrustee();
        break;
    case "T":
    default:
        prepareByTruster();
        prepareByTrustee();
    }
}
 

private DataModule getDataModule() {
    List<KeyValue<KeyValue<String, Boolean>, Integer>> moduleDefinition = new LinkedList<>();
    for (int index = 0; index < order; index++) {
        moduleDefinition.add(new KeyValue<>(new KeyValue<>("quality", true), 1));
        moduleDefinition.add(new KeyValue<>(new KeyValue<>("ontinuous", false), 1));
    }
    DataModule module = new SparseModule(moduleName, moduleDefinition, instanceCapacity);
    Int2IntSortedMap qualityFeatures = new Int2IntAVLTreeMap();
    Int2FloatSortedMap quantityFeatures = new Int2FloatAVLTreeMap();
    for (int index = 0; index < instanceCapacity; index++) {
        qualityFeatures.clear();
        qualityFeatures.put(0, index);
        module.associateInstance(qualityFeatures, quantityFeatures);
    }
    return module;
}
 

@Override
public void prepare(Configurator configuration, DataModule model, DataSpace space) {
    super.prepare(configuration, model, space);
    // compute item sampling probability
    DefaultScalar sum = DefaultScalar.getInstance();
    sum.setValue(0F);
    itemProbabilities = DenseVector.valueOf(itemSize);
    itemProbabilities.iterateElement(MathCalculator.SERIAL, (scalar) -> {
        int index = scalar.getIndex();
        float userSize = scoreMatrix.getColumnScope(index);
        // sample items based on popularity
        float value = (userSize + 0F) / actionSize;
        sum.shiftValue(value);
        scalar.setValue(sum.getValue());
    });
}
 

private DataModule getDataModule() {
    List<KeyValue<KeyValue<String, Boolean>, Integer>> moduleDefinition = new LinkedList<>();
    for (int index = 0; index < order; index++) {
        moduleDefinition.add(new KeyValue<>(new KeyValue<>("quality", true), 1));
        moduleDefinition.add(new KeyValue<>(new KeyValue<>("ontinuous", false), 1));
    }
    DataModule module = new SparseModule(moduleName, moduleDefinition, instanceCapacity);
    Int2IntSortedMap qualityFeatures = new Int2IntAVLTreeMap();
    Int2FloatSortedMap quantityFeatures = new Int2FloatAVLTreeMap();
    for (int index = 0; index < instanceCapacity; index++) {
        qualityFeatures.clear();
        qualityFeatures.put(0, index);
        module.associateInstance(qualityFeatures, quantityFeatures);
    }
    return module;
}
 

@Test
public void testSort() {
    DataModule module = getDataModule();
    DataSorter sorter = new DataSorter() {

        @Override
        public int sort(DataInstance left, DataInstance right) {
            return right.getQualityFeature(0) - left.getQualityFeature(0);
        }

    };
    module = sorter.sort(module);
    for (int index = 0; index < instanceCapacity; index++) {
        Assert.assertEquals(instanceCapacity - 1 - index, module.getInstance(index).getQualityFeature(0));
    }
}
 

public RandomSeparator(DataSpace space, DataModule dataModule, String matchField, float random) {
    this.dataModule = dataModule;
    ReferenceModule[] modules;
    if (matchField == null) {
        modules = new ReferenceModule[] { new ReferenceModule(dataModule) };
    } else {
        int matchDimension = dataModule.getQualityInner(matchField);
        DataSplitter splitter = new QualityFeatureDataSplitter(matchDimension);
        int size = space.getQualityAttribute(matchField).getSize();
        modules = splitter.split(dataModule, size);
    }
    this.trainReference = new IntegerArray();
    this.testReference = new IntegerArray();
    for (ReferenceModule module : modules) {
        IntegerArray reference = module.getReference();
        for (int cursor = 0, length = reference.getSize(); cursor < length; cursor++) {
            if (RandomUtility.randomFloat(1F) < random) {
                this.trainReference.associateData(reference.getData(cursor));
            } else {
                this.testReference.associateData(reference.getData(cursor));
            }
        }
    }
}
 

@Override
public void prepare(Configurator configuration, DataModule model, DataSpace space) {
    super.prepare(configuration, model, space);
    inputDimension = getInputDimension();
    hiddenDimension = configuration.getInteger("recommender.hidden.dimension");
    hiddenActivation = configuration.getString("recommender.hidden.activation");
    outputActivation = configuration.getString("recommender.output.activation");
    learnRatio = configuration.getFloat("recommender.iterator.learnrate");
    momentum = configuration.getFloat("recommender.iterator.momentum");
    weightRegularization = configuration.getFloat("recommender.weight.regularization");
    binarie = configuration.getFloat("recommender.binarize.threshold");
    // transform the sparse matrix to INDArray
    // the sparse training matrix has been binarized

    INDArray array = Nd4j.create(userSize, itemSize);
    inputData = new Nd4jMatrix(array);

    array = Nd4j.create(userSize, itemSize);
    labelData = new Nd4jMatrix(array);
    for (MatrixScalar term : scoreMatrix) {
        labelData.setValue(term.getRow(), term.getColumn(), 1F);
    }

    array = Nd4j.create(userSize, itemSize);
    outputData = new Nd4jMatrix(array);
}
 

@Override
public void prepare(Configurator configuration, DataModule model, DataSpace space) {
    super.prepare(configuration, model, space);
    userExponentials = DenseVector.valueOf(userSize);
    for (MatrixScalar matrixentry : scoreMatrix) {
        int userIndex = matrixentry.getRow();
        float score = matrixentry.getValue();
        userExponentials.shiftValue(userIndex, (float) Math.exp(score));
    }
}
 

@Override
public void prepare(Configurator configuration, DataModule model, DataSpace space) {
    super.prepare(configuration, model, space);
    userFactors = DenseMatrix.valueOf(userSize, factorSize);
    userFactors.iterateElement(MathCalculator.SERIAL, (scalar) -> {
        scalar.setValue(RandomUtility.randomFloat(0.01F));
    });
    itemFactors = DenseMatrix.valueOf(itemSize, factorSize);
    itemFactors.iterateElement(MathCalculator.SERIAL, (scalar) -> {
        scalar.setValue(RandomUtility.randomFloat(0.01F));
    });
}
 

@Override
public void prepare(Configurator configuration, DataModule model, DataSpace space) {
    super.prepare(configuration, model, space);
    numberOfGlobalFactors = configuration.getInteger("recommender.global.factors.num", 20);
    numberOfLocalFactors = factorSize;

    globalEpocheSize = configuration.getInteger("recommender.global.iteration.maximum", 100);
    localEpocheSize = epocheSize;

    globalUserRegularization = configuration.getFloat("recommender.global.user.regularization", 0.01F);
    globalItemRegularization = configuration.getFloat("recommender.global.item.regularization", 0.01F);
    localUserRegularization = userRegularization;
    localItemRegularization = itemRegularization;

    globalLearnRatio = configuration.getFloat("recommender.global.iteration.learnrate", 0.01F);
    localLearnRatio = configuration.getFloat("recommender.iteration.learnrate", 0.01F);

    numberOfThreads = configuration.getInteger("recommender.thread.count", 4);
    numberOfModels = configuration.getInteger("recommender.model.num", 50);

    numberOfThreads = numberOfThreads > numberOfModels ? numberOfModels : numberOfThreads;

    // global svd P Q to calculate the kernel value between users (or items)
    globalUserFactors = DenseMatrix.valueOf(userSize, numberOfGlobalFactors);
    globalUserFactors.iterateElement(MathCalculator.SERIAL, (scalar) -> {
        scalar.setValue(distribution.sample().floatValue());
    });
    globalItemFactors = DenseMatrix.valueOf(itemSize, numberOfGlobalFactors);
    globalItemFactors.iterateElement(MathCalculator.SERIAL, (scalar) -> {
        scalar.setValue(distribution.sample().floatValue());
    });
}
 

@Override
public int convert(DataModule module, InputStream iterator) {
    try {
        InputSource xmlSource = new InputSource(iterator);
        SAXParserFactory saxFactory = SAXParserFactory.newInstance();
        SAXParser saxParser = saxFactory.newSAXParser();
        XMLReader sheetParser = saxParser.getXMLReader();
        YongfengZhangInstanceHandler handler = new YongfengZhangInstanceHandler(module);
        sheetParser.setContentHandler(handler);
        sheetParser.parse(xmlSource);
        return handler.getCount();
    } catch (Exception exception) {
        throw new RuntimeException(exception);
    }
}
 

@Override
public void prepare(Configurator configuration, DataModule model, DataSpace space) {
    super.prepare(configuration, model, space);
    regBias = configuration.getFloat("recommender.bias.regularization", 0.01F);

    // initialize the userBiased and itemBiased
    userBiases = DenseVector.valueOf(userSize);
    userBiases.iterateElement(MathCalculator.SERIAL, (scalar) -> {
        scalar.setValue(distribution.sample().floatValue());
    });
    itemBiases = DenseVector.valueOf(itemSize);
    itemBiases.iterateElement(MathCalculator.SERIAL, (scalar) -> {
        scalar.setValue(distribution.sample().floatValue());
    });
}
 

@Test
public void testSelect() {
    DataModule module = getDataModule();
    DataSelector selector = new DataSelector() {

        @Override
        public boolean select(DataInstance instance) {
            return false;
        }

    };
    module = selector.select(module);
    Assert.assertEquals(0, module.getSize());
}
 

@Override
public void prepare(Configurator configuration, DataModule model, DataSpace space) {
    super.prepare(configuration, model, space);
    regImpItem = configuration.getFloat("recommender.impItem.regularization", 0.015F);
    factorMatrix = DenseMatrix.valueOf(itemSize, factorSize);
    factorMatrix.iterateElement(MathCalculator.SERIAL, (element) -> {
        element.setValue(distribution.sample().floatValue());
    });
}
 

@Override
public void prepare(Configurator configuration, DataModule model, DataSpace space) {
    super.prepare(configuration, model, space);
    // 注意:FISM使用itemFactors来组成userFactors
    userFactors = DenseMatrix.valueOf(itemSize, factorSize);
    userFactors.iterateElement(MathCalculator.SERIAL, (scalar) -> {
        scalar.setValue(distribution.sample().floatValue());
    });
    itemFactors = DenseMatrix.valueOf(itemSize, factorSize);
    itemFactors.iterateElement(MathCalculator.SERIAL, (scalar) -> {
        scalar.setValue(distribution.sample().floatValue());
    });
    userBiases = DenseVector.valueOf(userSize);
    userBiases.iterateElement(MathCalculator.SERIAL, (scalar) -> {
        scalar.setValue(distribution.sample().floatValue());
    });
    itemBiases = DenseVector.valueOf(itemSize);
    itemBiases.iterateElement(MathCalculator.SERIAL, (scalar) -> {
        scalar.setValue(distribution.sample().floatValue());
    });

    numNeighbors = scoreMatrix.getElementSize();
    rho = configuration.getFloat("recommender.fismrmse.rho");// 3-15
    alpha = configuration.getFloat("recommender.fismrmse.alpha", 0.5F);
    beta = configuration.getFloat("recommender.fismrmse.beta", 0.6F);
    itemRegularization = configuration.getFloat("recommender.fismrmse.gamma", 0.1F);
    userRegularization = configuration.getFloat("recommender.fismrmse.gamma", 0.1F);
    learnRatio = configuration.getFloat("recommender.fismrmse.lrate", 0.0001F);
}
 

@Override
public void practice(DataModule module, DataModule... contexts) {
    float markSum = 0F;
    float weightSum = 0F;
    for (DataInstance instance : module) {
        markSum += instance.getQuantityMark() * instance.getWeight();
        weightSum += instance.getWeight();
    }
    quantity = markSum / weightSum;
}
 

@Override
public void prepare(Configurator configuration, DataModule model, DataSpace space) {
    super.prepare(configuration, model, space);
    // set for this alg
    lambdaItem = (int) (configuration.getFloat("recommender.item.distribution.parameter") * itemSize);
    // lamda_Item=500;
    loopNumber = (int) (itemSize * Math.log(itemSize));

    factorVariances = new float[factorSize];
    factorRanks = new int[factorSize][itemSize];
}
 

@Override
public void prepare(Configurator configuration, DataModule model, DataSpace space) {
    super.prepare(configuration, model, space);

    userRegularization = configuration.getFloat("recommender.user.regularization", 0.01f);
    itemRegularization = configuration.getFloat("recommender.item.regularization", 0.01f);

    factorSize = configuration.getInteger("recommender.factor.number", 10);

    isLearned = configuration.getBoolean("recommender.learnrate.bolddriver", false);
    learnDecay = configuration.getFloat("recommender.learnrate.decay", 1.0f);
    learnRatio = configuration.getFloat("recommender.iterator.learnrate", 0.01f);
    learnLimit = configuration.getFloat("recommender.iterator.learnrate.maximum", 1000.0f);

    // TODO 此处需要重构
    initMean = configuration.getFloat("recommender.init.mean", 0F);
    initStd = configuration.getFloat("recommender.init.std", 0.1F);

    distribution = new QuantityProbability(JDKRandomGenerator.class, 0, NormalDistribution.class, initMean, initStd);
    userFactors = DenseMatrix.valueOf(userSize, factorSize);
    userFactors.iterateElement(MathCalculator.SERIAL, (scalar) -> {
        scalar.setValue(distribution.sample().floatValue());
    });
    itemFactors = DenseMatrix.valueOf(itemSize, factorSize);
    itemFactors.iterateElement(MathCalculator.SERIAL, (scalar) -> {
        scalar.setValue(distribution.sample().floatValue());
    });
}
 

private Model getModel(Class<? extends Model> clazz, DataSpace dataSpace, DataModule dataModule) throws Exception {
    Model model = ReflectionUtility.getInstance(clazz);
    EnvironmentContext context = EnvironmentFactory.getContext();
    Future<?> task = context.doTask(() -> {
        model.prepare(configuration, dataModule, dataSpace);
        model.practice();
    });
    task.get();
    return model;
}
 

@Override
public void prepare(Configurator configuration, DataModule model, DataSpace space) {
    super.prepare(configuration, model, space);
    weightCoefficient = configuration.getFloat("recommender.wrmf.weight.coefficient", 4.0f);
    ratio = configuration.getFloat("recommender.eals.ratio", 0.4f);
    overallWeight = configuration.getFloat("recommender.eals.overall", 128.0f);
    type = configuration.getInteger("recommender.eals.wrmf.judge", 1);

    confidences = new float[itemSize];

    // get ci
    if (type == 0 || type == 2) {
        float sumPopularity = 0F;
        for (int itemIndex = 0; itemIndex < itemSize; itemIndex++) {
            float alphaPopularity = (float) Math.pow(scoreMatrix.getColumnScope(itemIndex) * 1.0 / actionSize, ratio);
            confidences[itemIndex] = overallWeight * alphaPopularity;
            sumPopularity += alphaPopularity;
        }
        for (int itemIndex = 0; itemIndex < itemSize; itemIndex++) {
            confidences[itemIndex] = confidences[itemIndex] / sumPopularity;
        }
    } else {
        for (int itemIndex = 0; itemIndex < itemSize; itemIndex++) {
            confidences[itemIndex] = 1;
        }
    }

    weights = SparseMatrix.copyOf(scoreMatrix, false);
    weights.iterateElement(MathCalculator.SERIAL, (scalar) -> {
        if (type == 1 || type == 2) {
            scalar.setValue(1F + weightCoefficient * scalar.getValue());
        } else {
            scalar.setValue(1F);
        }
    });
}