Java Code Examples for com.jstarcraft.ai.data.DataSpace#getQualityAttribute()
The following examples show how to use
com.jstarcraft.ai.data.DataSpace#getQualityAttribute() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: MovieDataConfigurer.java From jstarcraft-example with Apache License 2.0 | 5 votes |
|
@Bean("movieItems")
List<MovieItem> getItems(DataSpace movieDataSpace) throws Exception {
File movieItemFile = new File("data/ml-100k/u.item");
List<MovieItem> items = new LinkedList<>();
QualityAttribute<Integer> itemAttribute = movieDataSpace.getQualityAttribute("item");
try (InputStream stream = new FileInputStream(movieItemFile); InputStreamReader reader = new InputStreamReader(stream, StringUtility.CHARSET); BufferedReader buffer = new BufferedReader(reader)) {
try (CSVParser parser = new CSVParser(buffer, CSVFormat.newFormat('|'))) {
Iterator<CSVRecord> iterator = parser.iterator();
while (iterator.hasNext()) {
CSVRecord datas = iterator.next();
// 物品标识
int id = Integer.parseInt(datas.get(0));
// 物品索引
int index = itemAttribute.convertData(id);
// 物品标题
String title = datas.get(1);
// 物品日期
LocalDate date = StringUtility.isEmpty(datas.get(2)) ? LocalDate.of(1970, 1, 1) : LocalDate.parse(datas.get(2), formatter);
MovieItem item = new MovieItem(index, title, date);
items.add(item);
}
}
}
items = new ArrayList<>(items);
return items;
}
Example 2
| Source File: YongfengZhangAttributeHandler.java From jstarcraft-rns with Apache License 2.0 | 5 votes |
|
YongfengZhangAttributeHandler(DataSpace space) {
this.space = space;
this.userAttribute = space.getQualityAttribute("user");
this.itemAttribute = space.getQualityAttribute("item");
this.wordAttribute = space.getQualityAttribute("word");
this.scoreAttribute = space.getQuantityAttribute("score");
this.sentimentAttribute = space.getQuantityAttribute("sentiment");
}
Example 3
| Source File: TopicMFMTModel.java From jstarcraft-rns with Apache License 2.0 | 4 votes |
|
@Override
public void prepare(Configurator configuration, DataModule model, DataSpace space) {
super.prepare(configuration, model, space);
commentField = configuration.getString("data.model.fields.comment");
commentDimension = model.getQualityInner(commentField);
MemoryQualityAttribute attribute = (MemoryQualityAttribute) space.getQualityAttribute(commentField);
Object[] documentValues = attribute.getDatas();
// init hyper-parameters
lambda = configuration.getFloat("recommender.regularization.lambda", 0.001F);
lambdaU = configuration.getFloat("recommender.regularization.lambdaU", 0.001F);
lambdaV = configuration.getFloat("recommender.regularization.lambdaV", 0.001F);
lambdaB = configuration.getFloat("recommender.regularization.lambdaB", 0.001F);
numberOfTopics = configuration.getInteger("recommender.topic.number", 10);
learnRatio = configuration.getFloat("recommender.iterator.learnrate", 0.01F);
epocheSize = configuration.getInteger("recommender.iterator.maximum", 10);
numberOfDocuments = scoreMatrix.getElementSize();
// count the number of words, build the word dictionary and
// userItemToDoc dictionary
Map<String, Integer> wordDictionaries = new HashMap<>();
Table<Integer, Integer, Float> documentTable = HashBasedTable.create();
int rowCount = 0;
userItemToDocument = HashBasedTable.create();
for (DataInstance sample : model) {
int userIndex = sample.getQualityFeature(userDimension);
int itemIndex = sample.getQualityFeature(itemDimension);
int documentIndex = sample.getQualityFeature(commentDimension);
userItemToDocument.put(userIndex, itemIndex, rowCount);
// convert wordIds to wordIndices
String data = (String) documentValues[documentIndex];
String[] words = data.isEmpty() ? new String[0] : data.split(":");
for (String word : words) {
Integer wordIndex = wordDictionaries.get(word);
if (wordIndex == null) {
wordIndex = numberOfWords++;
wordDictionaries.put(word, wordIndex);
}
Float oldValue = documentTable.get(rowCount, wordIndex);
if (oldValue == null) {
oldValue = 0F;
}
float newValue = oldValue + 1F / words.length;
documentTable.put(rowCount, wordIndex, newValue);
}
rowCount++;
}
// build W
W = SparseMatrix.valueOf(numberOfDocuments, numberOfWords, documentTable);
userBiases = DenseVector.valueOf(userSize);
userBiases.iterateElement(MathCalculator.SERIAL, (scalar) -> {
scalar.setValue(distribution.sample().floatValue());
});
itemBiases = DenseVector.valueOf(itemSize);
itemBiases.iterateElement(MathCalculator.SERIAL, (scalar) -> {
scalar.setValue(distribution.sample().floatValue());
});
userFactors = DenseMatrix.valueOf(userSize, numberOfTopics);
userFactors.iterateElement(MathCalculator.SERIAL, (scalar) -> {
scalar.setValue(distribution.sample().floatValue());
});
itemFactors = DenseMatrix.valueOf(itemSize, numberOfTopics);
itemFactors.iterateElement(MathCalculator.SERIAL, (scalar) -> {
scalar.setValue(distribution.sample().floatValue());
});
K = initStd;
topicVector = DenseVector.valueOf(numberOfTopics);
function = new SoftMaxActivationFunction();
// init theta and phi
// TODO theta实际是documentFactors
documentFactors = DenseMatrix.valueOf(numberOfDocuments, numberOfTopics);
calculateTheta();
// TODO phi实际是wordFactors
wordFactors = DenseMatrix.valueOf(numberOfTopics, numberOfWords);
wordFactors.iterateElement(MathCalculator.SERIAL, (scalar) -> {
scalar.setValue(RandomUtility.randomFloat(0.01F));
});
logger.info("number of users : " + userSize);
logger.info("number of Items : " + itemSize);
logger.info("number of words : " + wordDictionaries.size());
}
Example 4
| Source File: TopicMFATModel.java From jstarcraft-rns with Apache License 2.0 | 4 votes |
|
@Override
public void prepare(Configurator configuration, DataModule model, DataSpace space) {
super.prepare(configuration, model, space);
commentField = configuration.getString("data.model.fields.comment");
commentDimension = model.getQualityInner(commentField);
MemoryQualityAttribute attribute = (MemoryQualityAttribute) space.getQualityAttribute(commentField);
Object[] documentValues = attribute.getDatas();
// init hyper-parameters
lambda = configuration.getFloat("recommender.regularization.lambda", 0.001F);
lambdaU = configuration.getFloat("recommender.regularization.lambdaU", 0.001F);
lambdaV = configuration.getFloat("recommender.regularization.lambdaV", 0.001F);
lambdaB = configuration.getFloat("recommender.regularization.lambdaB", 0.001F);
numberOfTopics = configuration.getInteger("recommender.topic.number", 10);
learnRatio = configuration.getFloat("recommender.iterator.learnrate", 0.01F);
epocheSize = configuration.getInteger("recommender.iterator.maximum", 10);
numberOfDocuments = scoreMatrix.getElementSize();
// count the number of words, build the word dictionary and
// userItemToDoc dictionary
Map<String, Integer> wordDictionaries = new HashMap<>();
Table<Integer, Integer, Float> documentTable = HashBasedTable.create();
// TODO rowCount改为documentIndex?
int rowCount = 0;
userItemToDocument = HashBasedTable.create();
for (DataInstance sample : model) {
int userIndex = sample.getQualityFeature(userDimension);
int itemIndex = sample.getQualityFeature(itemDimension);
int documentIndex = sample.getQualityFeature(commentDimension);
userItemToDocument.put(userIndex, itemIndex, rowCount);
// convert wordIds to wordIndices
String data = (String) documentValues[documentIndex];
String[] words = data.isEmpty() ? new String[0] : data.split(":");
for (String word : words) {
Integer wordIndex = wordDictionaries.get(word);
if (wordIndex == null) {
wordIndex = numberOfWords++;
wordDictionaries.put(word, wordIndex);
}
Float oldValue = documentTable.get(rowCount, wordIndex);
if (oldValue == null) {
oldValue = 0F;
}
float newValue = oldValue + 1F / words.length;
documentTable.put(rowCount, wordIndex, newValue);
}
rowCount++;
}
// build W
W = SparseMatrix.valueOf(numberOfDocuments, numberOfWords, documentTable);
userBiases = DenseVector.valueOf(userSize);
userBiases.iterateElement(MathCalculator.SERIAL, (scalar) -> {
scalar.setValue(distribution.sample().floatValue());
});
itemBiases = DenseVector.valueOf(itemSize);
itemBiases.iterateElement(MathCalculator.SERIAL, (scalar) -> {
scalar.setValue(distribution.sample().floatValue());
});
userFactors = DenseMatrix.valueOf(userSize, numberOfTopics);
userFactors.iterateElement(MathCalculator.SERIAL, (scalar) -> {
scalar.setValue(distribution.sample().floatValue());
});
itemFactors = DenseMatrix.valueOf(itemSize, numberOfTopics);
itemFactors.iterateElement(MathCalculator.SERIAL, (scalar) -> {
scalar.setValue(distribution.sample().floatValue());
});
K1 = initStd;
K2 = initStd;
topicVector = DenseVector.valueOf(numberOfTopics);
function = new SoftMaxActivationFunction();
// init theta and phi
// TODO theta实际是documentFactors
documentFactors = DenseMatrix.valueOf(numberOfDocuments, numberOfTopics);
calculateTheta();
// TODO phi实际是wordFactors
wordFactors = DenseMatrix.valueOf(numberOfTopics, numberOfWords);
wordFactors.iterateElement(MathCalculator.SERIAL, (scalar) -> {
scalar.setValue(RandomUtility.randomFloat(0.01F));
});
logger.info("number of users : " + userSize);
logger.info("number of Items : " + itemSize);
logger.info("number of words : " + wordDictionaries.size());
}
Example 5
| Source File: YongfengZhangDatasetTestCase.java From jstarcraft-rns with Apache License 2.0 | 4 votes |
|
@Test
public void testDataset() throws Exception {
File file = new File("data/labeled_DC/DC_feature_opinion/DC.txt");
// 定义数据空间
Map<String, Class<?>> qualityDifinitions = new HashMap<>();
qualityDifinitions.put("user", String.class);
qualityDifinitions.put("item", String.class);
qualityDifinitions.put("word", String.class);
Map<String, Class<?>> quantityDifinitions = new HashMap<>();
quantityDifinitions.put("score", Float.class);
quantityDifinitions.put("sentiment", Float.class);
DataSpace dataSpace = new DataSpace(qualityDifinitions, quantityDifinitions);
// 处理数据属性
try (InputStream stream = new FileInputStream(file)) {
InputSource xmlSource = new InputSource(stream);
SAXParserFactory saxFactory = SAXParserFactory.newInstance();
SAXParser saxParser = saxFactory.newSAXParser();
XMLReader sheetParser = saxParser.getXMLReader();
YongfengZhangAttributeHandler handler = new YongfengZhangAttributeHandler(dataSpace);
sheetParser.setContentHandler(handler);
sheetParser.parse(xmlSource);
}
QualityAttribute<String> userAttribute = dataSpace.getQualityAttribute("user");
QualityAttribute<String> itemAttribute = dataSpace.getQualityAttribute("item");
QualityAttribute<String> wordAttribute = dataSpace.getQualityAttribute("word");
Assert.assertEquals(89373, userAttribute.getSize());
Assert.assertEquals(2397, itemAttribute.getSize());
Assert.assertEquals(333, wordAttribute.getSize());
// 定义数据模块
// 使用word属性大小作为sentiment特征维度
TreeMap<Integer, String> configuration = new TreeMap<>();
configuration.put(1, "user");
configuration.put(2, "item");
configuration.put(3, "score");
configuration.put(3 + wordAttribute.getSize(), "sentiment");
DataModule dataModule = dataSpace.makeSparseModule("score", configuration, 1000000);
// 处理数据实例
DataConverter<InputStream> convertor = new YongfengZhangDatasetConverter(wordAttribute, dataSpace.getQualityAttributes(), dataSpace.getQuantityAttributes());
try (InputStream stream = new FileInputStream(file)) {
convertor.convert(dataModule, stream);
}
Assert.assertEquals(123732, dataModule.getSize());
}
