com.jstarcraft.ai.data.attribute.QualityAttribute Java Examples

@Bean("movieItems")
List<MovieItem> getItems(DataSpace movieDataSpace) throws Exception {
    File movieItemFile = new File("data/ml-100k/u.item");
    List<MovieItem> items = new LinkedList<>();

    QualityAttribute<Integer> itemAttribute = movieDataSpace.getQualityAttribute("item");
    try (InputStream stream = new FileInputStream(movieItemFile); InputStreamReader reader = new InputStreamReader(stream, StringUtility.CHARSET); BufferedReader buffer = new BufferedReader(reader)) {
        try (CSVParser parser = new CSVParser(buffer, CSVFormat.newFormat('|'))) {
            Iterator<CSVRecord> iterator = parser.iterator();
            while (iterator.hasNext()) {
                CSVRecord datas = iterator.next();
                // 物品标识
                int id = Integer.parseInt(datas.get(0));
                // 物品索引
                int index = itemAttribute.convertData(id);
                // 物品标题
                String title = datas.get(1);
                // 物品日期
                LocalDate date = StringUtility.isEmpty(datas.get(2)) ? LocalDate.of(1970, 1, 1) : LocalDate.parse(datas.get(2), formatter);
                MovieItem item = new MovieItem(index, title, date);
                items.add(item);
            }
        }
    }

    items = new ArrayList<>(items);
    return items;
}
 

@Test
public void testDataset() throws Exception {
    File file = new File("data/labeled_DC/DC_feature_opinion/DC.txt");

    // 定义数据空间
    Map<String, Class<?>> qualityDifinitions = new HashMap<>();
    qualityDifinitions.put("user", String.class);
    qualityDifinitions.put("item", String.class);
    qualityDifinitions.put("word", String.class);
    Map<String, Class<?>> quantityDifinitions = new HashMap<>();
    quantityDifinitions.put("score", Float.class);
    quantityDifinitions.put("sentiment", Float.class);
    DataSpace dataSpace = new DataSpace(qualityDifinitions, quantityDifinitions);

    // 处理数据属性
    try (InputStream stream = new FileInputStream(file)) {
        InputSource xmlSource = new InputSource(stream);
        SAXParserFactory saxFactory = SAXParserFactory.newInstance();
        SAXParser saxParser = saxFactory.newSAXParser();
        XMLReader sheetParser = saxParser.getXMLReader();
        YongfengZhangAttributeHandler handler = new YongfengZhangAttributeHandler(dataSpace);
        sheetParser.setContentHandler(handler);
        sheetParser.parse(xmlSource);
    }
    QualityAttribute<String> userAttribute = dataSpace.getQualityAttribute("user");
    QualityAttribute<String> itemAttribute = dataSpace.getQualityAttribute("item");
    QualityAttribute<String> wordAttribute = dataSpace.getQualityAttribute("word");
    Assert.assertEquals(89373, userAttribute.getSize());
    Assert.assertEquals(2397, itemAttribute.getSize());
    Assert.assertEquals(333, wordAttribute.getSize());

    // 定义数据模块
    // 使用word属性大小作为sentiment特征维度
    TreeMap<Integer, String> configuration = new TreeMap<>();
    configuration.put(1, "user");
    configuration.put(2, "item");
    configuration.put(3, "score");
    configuration.put(3 + wordAttribute.getSize(), "sentiment");
    DataModule dataModule = dataSpace.makeSparseModule("score", configuration, 1000000);

    // 处理数据实例
    DataConverter<InputStream> convertor = new YongfengZhangDatasetConverter(wordAttribute, dataSpace.getQualityAttributes(), dataSpace.getQuantityAttributes());
    try (InputStream stream = new FileInputStream(file)) {
        convertor.convert(dataModule, stream);
    }
    Assert.assertEquals(123732, dataModule.getSize());
}
 

public YongfengZhangDatasetConverter(QualityAttribute<String> wordAttribute, Collection<QualityAttribute> qualityAttributes, Collection<QuantityAttribute> quantityAttributes) {
    super(qualityAttributes, quantityAttributes);
    this.wordAttribute = wordAttribute;
}
 

protected StreamConverter(Collection<QualityAttribute> qualityAttributes, Collection<QuantityAttribute> quantityAttributes) {
    super(qualityAttributes, quantityAttributes);
}
 

public JsonConverter(Collection<QualityAttribute> qualityAttributes, Collection<QuantityAttribute> quantityAttributes) {
    super(qualityAttributes, quantityAttributes);
}
 

public CsvConverter(CSVFormat format, Collection<QualityAttribute> qualityAttributes, Collection<QuantityAttribute> quantityAttributes) {
    super(qualityAttributes, quantityAttributes);
    this.format = format;
}
 

public ParquetConverter(Collection<QualityAttribute> qualityAttributes, Collection<QuantityAttribute> quantityAttributes) {
    super(qualityAttributes, quantityAttributes);
}
 

protected QueryConverter(Collection<QualityAttribute> qualityAttributes, Collection<QuantityAttribute> quantityAttributes) {
    super(qualityAttributes, quantityAttributes);
}
 

public ArffConverter(Collection<QualityAttribute> qualityAttributes, Collection<QuantityAttribute> quantityAttributes) {
    super(CSVFormat.DEFAULT, qualityAttributes, quantityAttributes);
}
 

public QualityAttribute getQualityAttribute(String attributeName) {
    return qualityAttributes.get(attributeName);
}
 

public Collection<QualityAttribute> getQualityAttributes() {
    return qualityAttributes.values();
}
 

public NaiveBayesianClassifier(QualityAttribute attribute) {
    this.attribute = attribute;
}