org.datavec.api.records.metadata.RecordMetaDataIndex Java Examples

@Override
public List<SequenceRecord> loadSequenceFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
    Set<Integer> toLoad = new LinkedHashSet<>();
    for (RecordMetaData recordMetaData : recordMetaDatas) {
        if (!(recordMetaData instanceof RecordMetaDataIndex)) {
            throw new IllegalArgumentException("Expected RecordMetaDataIndex; got: " + recordMetaData);
        }
        long idx = ((RecordMetaDataIndex) recordMetaData).getIndex();
        if (idx >= original.size()) {
            throw new IllegalStateException(
                            "Cannot get index " + idx + " from collection: contains " + original + " elements");
        }
        toLoad.add((int) idx);
    }

    List<SequenceRecord> out = new ArrayList<>();
    Iterator<? extends Collection<? extends Collection<Writable>>> iter = original.iterator();
    int i = 0;
    while (iter.hasNext()) {
        Collection<? extends Collection<Writable>> c = iter.next();
        if (!toLoad.contains(i++)) {
            continue;
        }
        List<List<Writable>> record = toList(c);
        SequenceRecord r = new org.datavec.api.records.impl.SequenceRecord(record,
                        new RecordMetaDataIndex(i - 1, null, CollectionSequenceRecordReader.class));
        out.add(r);
    }
    return out;
}
 

@Override
public List<SequenceRecord> loadSequenceFromMetaData(List<RecordMetaData> recordMetaDatas) throws IOException {
    Set<Integer> toLoad = new LinkedHashSet<>();
    for (RecordMetaData recordMetaData : recordMetaDatas) {
        if (!(recordMetaData instanceof RecordMetaDataIndex)) {
            throw new IllegalArgumentException("Expected RecordMetaDataIndex; got: " + recordMetaData);
        }
        long idx = ((RecordMetaDataIndex) recordMetaData).getIndex();
        if (idx >= original.size()) {
            throw new IllegalStateException(
                            "Cannot get index " + idx + " from collection: contains " + original + " elements");
        }
        toLoad.add((int) idx);
    }

    List<SequenceRecord> out = new ArrayList<>();
    Iterator<? extends Collection<? extends Collection<Writable>>> iter = original.iterator();
    int i = 0;
    while (iter.hasNext()) {
        Collection<? extends Collection<Writable>> c = iter.next();
        if (!toLoad.contains(i++)) {
            continue;
        }
        List<List<Writable>> record = toList(c);
        SequenceRecord r = new org.datavec.api.records.impl.SequenceRecord(record,
                        new RecordMetaDataIndex(i - 1, null, CollectionSequenceRecordReader.class));
        out.add(r);
    }
    return out;
}
 

@Test
public void testRecordReaderMetaData() throws Exception {
    val recordsToWrite = recordToWrite();
    //send file
    File tmp =  tmpDataFile(recordsToWrite);
    RecordReader recordReader = new ArrowRecordReader();
    RecordMetaDataIndex recordMetaDataIndex = new RecordMetaDataIndex(0,tmp.toURI(),ArrowRecordReader.class);
    recordReader.loadFromMetaData(recordMetaDataIndex);

    Record record = recordReader.nextRecord();
    assertEquals(2,record.getRecord().size());
}
 

@Test
public void testRecordReaderMetaDataList() throws Exception {
    val recordsToWrite = recordToWrite();
    //send file
    File tmp =  tmpDataFile(recordsToWrite);
    RecordReader recordReader = new ArrowRecordReader();
    RecordMetaDataIndex recordMetaDataIndex = new RecordMetaDataIndex(0,tmp.toURI(),ArrowRecordReader.class);
    recordReader.loadFromMetaData(Arrays.<RecordMetaData>asList(recordMetaDataIndex));

    Record record = recordReader.nextRecord();
    assertEquals(2,record.getRecord().size());

}
 

@Override
public List<Record> loadFromMetaData(List<RecordMetaData> recordMetaDatas) {
    Map<String,List<RecordMetaData>> metaDataByUri = new HashMap<>();
    //gather all unique locations for the metadata
    //this will prevent initialization multiple times of the record
    for(RecordMetaData recordMetaData : recordMetaDatas) {
        if(!(recordMetaData instanceof RecordMetaDataIndex)) {
            throw new IllegalArgumentException("Unable to load from meta data. No index specified for record");
        }

        List<RecordMetaData> recordMetaData1 = metaDataByUri.get(recordMetaData.getURI().toString());
        if(recordMetaData1 == null) {
            recordMetaData1 = new ArrayList<>();
            metaDataByUri.put(recordMetaData.getURI().toString(),recordMetaData1);
        }

        recordMetaData1.add(recordMetaData);

    }

    List<Record> ret = new ArrayList<>();
    for(String uri : metaDataByUri.keySet()) {
        List<RecordMetaData> metaData = metaDataByUri.get(uri);
        InputSplit fileSplit = new FileSplit(new File(URI.create(uri)));
        initialize(fileSplit);
        for(RecordMetaData index : metaData) {
            RecordMetaDataIndex index2 = (RecordMetaDataIndex) index;
            this.currIdx = (int) index2.getIndex();
            ret.add(nextRecord());
        }

    }

    return ret;
}
 

@Override
public Record loadFromMetaData(RecordMetaData recordMetaData) {
    if(!(recordMetaData instanceof RecordMetaDataIndex)) {
        throw new IllegalArgumentException("Unable to load from meta data. No index specified for record");
    }

    RecordMetaDataIndex index = (RecordMetaDataIndex) recordMetaData;
    InputSplit fileSplit = new FileSplit(new File(index.getURI()));
    initialize(fileSplit);
    this.currIdx = (int) index.getIndex();
    return nextRecord();
}
 

@Test
public void testRecordReaderMetaData() throws Exception {
    val recordsToWrite = recordToWrite();
    //send file
    File tmp =  tmpDataFile(recordsToWrite);
    RecordReader recordReader = new ArrowRecordReader();
    RecordMetaDataIndex recordMetaDataIndex = new RecordMetaDataIndex(0,tmp.toURI(),ArrowRecordReader.class);
    recordReader.loadFromMetaData(recordMetaDataIndex);

    Record record = recordReader.nextRecord();
    assertEquals(2,record.getRecord().size());
}
 

@Test
public void testRecordReaderMetaDataList() throws Exception {
    val recordsToWrite = recordToWrite();
    //send file
    File tmp =  tmpDataFile(recordsToWrite);
    RecordReader recordReader = new ArrowRecordReader();
    RecordMetaDataIndex recordMetaDataIndex = new RecordMetaDataIndex(0,tmp.toURI(),ArrowRecordReader.class);
    recordReader.loadFromMetaData(Arrays.<RecordMetaData>asList(recordMetaDataIndex));

    Record record = recordReader.nextRecord();
    assertEquals(2,record.getRecord().size());

}
 

@Override
public List<Record> loadFromMetaData(List<RecordMetaData> recordMetaDatas) {
    Map<String,List<RecordMetaData>> metaDataByUri = new HashMap<>();
    //gather all unique locations for the metadata
    //this will prevent initialization multiple times of the record
    for(RecordMetaData recordMetaData : recordMetaDatas) {
        if(!(recordMetaData instanceof RecordMetaDataIndex)) {
            throw new IllegalArgumentException("Unable to load from meta data. No index specified for record");
        }

        List<RecordMetaData> recordMetaData1 = metaDataByUri.get(recordMetaData.getURI().toString());
        if(recordMetaData1 == null) {
            recordMetaData1 = new ArrayList<>();
            metaDataByUri.put(recordMetaData.getURI().toString(),recordMetaData1);
        }

        recordMetaData1.add(recordMetaData);

    }

    List<Record> ret = new ArrayList<>();
    for(String uri : metaDataByUri.keySet()) {
        List<RecordMetaData> metaData = metaDataByUri.get(uri);
        InputSplit fileSplit = new FileSplit(new File(URI.create(uri)));
        initialize(fileSplit);
        for(RecordMetaData index : metaData) {
            RecordMetaDataIndex index2 = (RecordMetaDataIndex) index;
            this.currIdx = (int) index2.getIndex();
            ret.add(nextRecord());
        }

    }

    return ret;
}
 

@Override
public Record loadFromMetaData(RecordMetaData recordMetaData) {
    if(!(recordMetaData instanceof RecordMetaDataIndex)) {
        throw new IllegalArgumentException("Unable to load from meta data. No index specified for record");
    }

    RecordMetaDataIndex index = (RecordMetaDataIndex) recordMetaData;
    InputSplit fileSplit = new FileSplit(new File(index.getURI()));
    initialize(fileSplit);
    this.currIdx = (int) index.getIndex();
    return nextRecord();
}
 

private Record processNextRecord()
  {
  	List<Writable> next = super.next();
  	//As we increase in next the listIndex, then we need to reduce in one the value of listIndex
  	RecordMetaData meta = new RecordMetaDataIndex(listIndex-1, null, VasttextExtraMemoryReader.class);
return new org.datavec.api.records.impl.Record(next, meta);
  }
 

@Override
public Record nextRecord() {
	List<Writable> next = next();
	//As we increase in next the listIndex, then we need to reduce in one the value of listIndex
	RecordMetaData meta = new RecordMetaDataIndex(listIndex-1, null, VasttextMemoryRecordReader.class);
	return new org.datavec.api.records.impl.Record(next, meta);
}
 

@Override
public Record nextRecord() {
    return new org.datavec.api.records.impl.Record(next(),
                    new RecordMetaDataIndex(count - 1, null, CollectionRecordReader.class));
}
 

@Override
public SequenceRecord nextSequence() {
    return new org.datavec.api.records.impl.SequenceRecord(sequenceRecord(),
                    new RecordMetaDataIndex(count - 1, null, CollectionSequenceRecordReader.class));
}
 

public Record processNextRecord() {
     List<Writable> nextProcessed = new ArrayList<>();
     List<Writable> next = next();
     int label;
     
     String text = next.get(0).toString();
     //Text part
     nextProcessed.add(new Text(text));
    
    //Label part 
    if(labelled)
    {
if (multilabel)
{
	String[] labels = next.get(1).toString().split(" ");
	// We need to create a vector of labels
	List<Double> transformedLabels = new ArrayList<Double>();
	boolean seenLabels = false;
	for (int i = 0; i < labels.length; i++)
	{
		label = vasttextTextVectorizer.getLabelIndex(labels[i]);
		if (label == -1 && vasttextTextVectorizer.hasfitFinished())
		{
			System.err.println("WARNING: The label \"" + next.get(1).toString()
					+ "\" wasn't seen during the training. It has been removed from the entry "
					+ (listIndex - 1));
			continue;
		}
		else if (label == -1)
			throw new UnsupportedOperationException(
					"Unknown error from the vectorizer. Returned a label of -1 during fitting the dictionary.");
		transformedLabels.add((double) label);
		seenLabels = true;
	}
	// This means that none of the labels was seen in the training
	if (!seenLabels)
		return null;
	INDArray transformed = Nd4j.create(transformedLabels.stream().mapToDouble(d -> d).toArray(),
			new int[] { transformedLabels.size(), 1 });
	nextProcessed.add(new NDArrayWritable(transformed));

}
else 
{
	label = vasttextTextVectorizer.getLabelIndex(next.get(1).toString());
	if (label == -1 && vasttextTextVectorizer.hasfitFinished())
	{
		System.err.println(
				"WARNING: The label \"" + next.get(1).toString() + "\" wasn't seen during the training.");
		return null;
	}
	else if (label == -1)
		throw new UnsupportedOperationException(
				"Unknown error from the vectorizer. Returned a label of -1 during fitting the dictionary.");
	nextProcessed.add(new IntWritable(label));
      }
    }
    //As we increase in next the listIndex, then we need to reduce in one the value of listIndex
    RecordMetaData meta = new RecordMetaDataIndex(listIndex-1, null, VasttextMemoryRecordReader.class); 
    return new org.datavec.api.records.impl.Record(nextProcessed, meta);
 }
 

@Override
public SequenceRecord loadSequenceFromMetaData(@NonNull RecordMetaData recordMetaData) throws IOException {
    long idx = ((RecordMetaDataIndex) recordMetaData).getIndex();
    return new org.datavec.api.records.impl.SequenceRecord(mapFileReader.getRecord(idx).getSequenceRecord(),
                    recordMetaData);
}
 

@Override
public SequenceRecord loadSequenceFromMetaData(@NonNull RecordMetaData recordMetaData) throws IOException {
    long idx = ((RecordMetaDataIndex) recordMetaData).getIndex();
    return new org.datavec.api.records.impl.SequenceRecord(mapFileReader.getRecord(idx).getSequenceRecord(),
                    recordMetaData);
}
 

@Override
public RecordMetaData getMetaData() {
    RecordMetaData ret = new RecordMetaDataIndex(index,recordUri,ArrowRecordReader.class);
    return ret;
}
 

@Override
public RecordMetaData getMetaData() {
    RecordMetaData ret = new RecordMetaDataIndex(index,recordUri,ArrowRecordReader.class);
    return ret;
}
 

@Override
public Record nextRecord() {
    return new org.datavec.api.records.impl.Record(next(),
                    new RecordMetaDataIndex(count - 1, null, CollectionRecordReader.class));
}
 

@Override
public SequenceRecord nextSequence() {
    return new org.datavec.api.records.impl.SequenceRecord(sequenceRecord(),
                    new RecordMetaDataIndex(count - 1, null, CollectionSequenceRecordReader.class));
}