Java Code Examples for org.apache.commons.csv.CSVRecord#iterator()
The following examples show how to use
org.apache.commons.csv.CSVRecord#iterator() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: FrameworkUtils.java From data-polygamy with BSD 3-Clause "New" or "Revised" License | 6 votes |
|
/**
* String Parsing
*/
public static String[] splitStr(String val, Integer len) throws IOException {
String[] input;
try {
CSVParser parser = new CSVParser(new StringReader(val), CSVFormat.DEFAULT);
CSVRecord record = parser.getRecords().get(0);
input = new String[len];
Iterator<String> valuesIt = record.iterator();
int i = 0;
while (valuesIt.hasNext()) {
input[i] = valuesIt.next().trim();
i++;
}
parser.close();
} catch (ArrayIndexOutOfBoundsException e) {
input = val.split(",", len);
for (int i = 0; i < input.length; i++)
input[i] = input[i].trim();
}
return input;
}
Example 2
| Source File: DLPTextToBigQueryStreaming.java From dlp-dataflow-deidentification with Apache License 2.0 | 5 votes |
|
private Table.Row convertCsvRowToTableRow(CSVRecord csvRow) {
/** convert from CSV row to DLP Table Row */
Iterator<String> valueIterator = csvRow.iterator();
Table.Row.Builder tableRowBuilder = Table.Row.newBuilder();
while (valueIterator.hasNext()) {
String value = valueIterator.next();
if (value != null) {
tableRowBuilder.addValues(Value.newBuilder().setStringValue(value.toString()).build());
} else {
tableRowBuilder.addValues(Value.newBuilder().setStringValue("").build());
}
}
return tableRowBuilder.build();
}
Example 3
| Source File: CsvReader.java From jstarcraft-core with Apache License 2.0 | 5 votes |
|
public CsvReader(InputStream inputStream, CodecDefinition definition) {
super(definition);
InputStreamReader buffer = new InputStreamReader(inputStream, StringUtility.CHARSET);
try (CSVParser input = new CSVParser(buffer, FORMAT)) {
Iterator<CSVRecord> iterator = input.iterator();
if (iterator.hasNext()) {
CSVRecord values = iterator.next();
this.inputStream = values.iterator();
}
} catch (Exception exception) {
throw new RuntimeException(exception);
}
}
Example 4
| Source File: DLPTextToBigQueryStreaming.java From DataflowTemplates with Apache License 2.0 | 5 votes |
|
private Table.Row convertCsvRowToTableRow(CSVRecord csvRow) {
/** convert from CSV row to DLP Table Row */
Iterator<String> valueIterator = csvRow.iterator();
Table.Row.Builder tableRowBuilder = Table.Row.newBuilder();
while (valueIterator.hasNext()) {
String value = valueIterator.next();
if (value != null) {
tableRowBuilder.addValues(Value.newBuilder().setStringValue(value.toString()).build());
} else {
tableRowBuilder.addValues(Value.newBuilder().setStringValue("").build());
}
}
return tableRowBuilder.build();
}
Example 5
| Source File: DelimitedTextReader.java From marklogic-contentpump with Apache License 2.0 | 5 votes |
|
protected String[] getLine(CSVRecord record)
throws IOException {
Iterator<String> recordIterator = record.iterator();
int recordSize = record.size();
String[] values = new String[recordSize];
for (int i = 0; i < recordSize; i++) {
if (recordIterator.hasNext()) {
values[i] = (String)recordIterator.next();
} else {
throw new IOException("Record size doesn't match the real size");
}
}
return values;
}
Example 6
| Source File: CommandLineInterfaceTest.java From utah-parser with Apache License 2.0 | 5 votes |
|
private void assertCsvContent(String content, List<List<String>> expected) throws IOException {
CSVParser csv = CSVParser.parse(content, CSVFormat.DEFAULT);
List<List<String>> observed = new ArrayList<List<String>>();
for (CSVRecord line : csv.getRecords()) {
List<String> lineValues = new ArrayList<>();
for (Iterator<String> seeker = line.iterator(); seeker.hasNext(); ) {
lineValues.add(seeker.next());
}
observed.add(lineValues);
}
assertEquals(expected, observed);
}
Example 7
| Source File: FrameworkUtils.java From data-polygamy with BSD 3-Clause "New" or "Revised" License | 5 votes |
|
public static String[] splitStr(String val) throws IOException {
CSVParser parser = new CSVParser(new StringReader(val), CSVFormat.DEFAULT);
CSVRecord record = parser.getRecords().get(0);
Iterator<String> valuesIt = record.iterator();
String[] input = new String[record.size()];
int i = 0;
while (valuesIt.hasNext()) {
input[i] = valuesIt.next();
i++;
}
parser.close();
return input;
}
Example 8
| Source File: DelimitedTextInputFormat.java From marklogic-contentpump with Apache License 2.0 | 4 votes |
|
public List<InputSplit> getSplits(JobContext job) throws IOException {
boolean delimSplit = isSplitInput(job.getConfiguration());
//if delimSplit is true, size of each split is determined by
//Math.max(minSize, Math.min(maxSize, blockSize)) in FileInputFormat
List<InputSplit> splits = super.getSplits(job);
if (!delimSplit) {
return splits;
}
if (splits.size()>= SPLIT_COUNT_LIMIT) {
//if #splits > 1 million, there is enough parallelism
//therefore no point to split
LOG.warn("Exceeding SPLIT_COUNT_LIMIT, input_split is off:"
+ SPLIT_COUNT_LIMIT);
DefaultStringifier.store(job.getConfiguration(), false, ConfigConstants.CONF_SPLIT_INPUT);
return splits;
}
// add header info into splits
List<InputSplit> populatedSplits = new ArrayList<InputSplit>();
LOG.info(splits.size() + " DelimitedSplits generated");
Configuration conf = job.getConfiguration();
char delimiter =0;
ArrayList<Text> hlist = new ArrayList<Text>();
for (InputSplit file: splits) {
FileSplit fsplit = ((FileSplit)file);
Path path = fsplit.getPath();
FileSystem fs = path.getFileSystem(conf);
if (fsplit.getStart() == 0) {
// parse the inSplit, get the header
FSDataInputStream fileIn = fs.open(path);
String delimStr = conf.get(ConfigConstants.CONF_DELIMITER,
ConfigConstants.DEFAULT_DELIMITER);
if (delimStr.length() == 1) {
delimiter = delimStr.charAt(0);
} else {
LOG.error("Incorrect delimitor: " + delimiter
+ ". Expects single character.");
}
String encoding = conf.get(
MarkLogicConstants.OUTPUT_CONTENT_ENCODING,
MarkLogicConstants.DEFAULT_OUTPUT_CONTENT_ENCODING);
InputStreamReader instream = new InputStreamReader(fileIn, encoding);
CSVParser parser = new CSVParser(instream, CSVParserFormatter.
getFormat(delimiter, DelimitedTextReader.encapsulator,
true, true));
Iterator<CSVRecord> it = parser.iterator();
String[] header = null;
if (it.hasNext()) {
CSVRecord record = (CSVRecord)it.next();
Iterator<String> recordIterator = record.iterator();
int recordSize = record.size();
header = new String[recordSize];
for (int i = 0; i < recordSize; i++) {
if (recordIterator.hasNext()) {
header[i] = (String)recordIterator.next();
} else {
throw new IOException("Record size doesn't match the real size");
}
}
EncodingUtil.handleBOMUTF8(header, 0);
hlist.clear();
for (String s : header) {
hlist.add(new Text(s));
}
}
instream.close();
}
DelimitedSplit ds = new DelimitedSplit(new TextArrayWritable(
hlist.toArray(new Text[hlist.size()])), path,
fsplit.getStart(), fsplit.getLength(),
fsplit.getLocations());
populatedSplits.add(ds);
}
return populatedSplits;
}
Example 9
| Source File: DefaultConverter.java From swagger-inflector with Apache License 2.0 | 4 votes |
|
public Object cast(List<String> arguments, Parameter parameter, JavaType javaType, Map<String, Schema> definitions) throws ConversionException {
if (arguments == null || arguments.size() == 0) {
return null;
}
Class<?> cls = javaType.getRawClass();
LOGGER.debug("converting array `" + arguments + "` to `" + cls + "`");
if (javaType.isArrayType()) {
if (parameter.getSchema() != null) {
List<Object> output = new ArrayList<>();
if (parameter.getSchema() instanceof ArraySchema) {
ArraySchema arraySchema = (ArraySchema) parameter.getSchema();
if (arraySchema.getItems() != null) {
Schema inner = arraySchema.getItems();
// TODO: this does not need to be done this way, update the helper method
Parameter innerParam = new QueryParameter().schema(inner);
JavaType innerClass = getTypeFromParameter(innerParam, definitions);
for (String obj : arguments) {
String[] parts = new String[0];
CSVFormat format = null;
if (Parameter.StyleEnum.FORM.equals(parameter.getStyle()) && !StringUtils.isEmpty(obj) && parameter.getExplode() == false) {
format = CSVFormat.DEFAULT;
} else if (Parameter.StyleEnum.PIPEDELIMITED.equals(parameter.getStyle()) && !StringUtils.isEmpty(obj)) {
format = CSVFormat.newFormat('|').withQuote('"');
} else if (Parameter.StyleEnum.SPACEDELIMITED.equals(parameter.getStyle()) && !StringUtils.isEmpty(obj)) {
format = CSVFormat.newFormat(' ').withQuote('"');
}
if (format != null) {
try {
for (CSVRecord record : CSVParser.parse(obj, format).getRecords()) {
List<String> it = new ArrayList<String>();
for (Iterator<String> x = record.iterator(); x.hasNext(); ) {
it.add(x.next());
}
parts = it.toArray(new String[it.size()]);
}
} catch (IOException e) {
}
} else {
parts = new String[1];
parts[0] = obj;
}
for (String p : parts) {
Object ob = cast(p, inner, innerClass);
if (ob != null) {
output.add(ob);
}
}
}
return output;
}
}
}
} else if (parameter != null) {
return cast(arguments.get(0), parameter.getSchema(), javaType);
}
return null;
}
Example 10
| Source File: DataFlowFromCsvMain.java From Decision with Apache License 2.0 | 4 votes |
|
public static void main(String[] args) throws IOException, NumberFormatException, InterruptedException {
if (args.length < 4) {
log.info("Usage: \n param 1 - path to file \n param 2 - stream name to send the data \n param 3 - time in ms to wait to send each data \n param 4 - broker list");
} else {
Producer<String, String> producer = new Producer<String, String>(createProducerConfig(args[3]));
Gson gson = new Gson();
Reader in = new FileReader(args[0]);
CSVParser parser = CSVFormat.DEFAULT.parse(in);
List<String> columnNames = new ArrayList<>();
for (CSVRecord csvRecord : parser.getRecords()) {
if (columnNames.size() == 0) {
Iterator<String> iterator = csvRecord.iterator();
while (iterator.hasNext()) {
columnNames.add(iterator.next());
}
} else {
StratioStreamingMessage message = new StratioStreamingMessage();
message.setOperation(STREAM_OPERATIONS.MANIPULATION.INSERT.toLowerCase());
message.setStreamName(args[1]);
message.setTimestamp(System.currentTimeMillis());
message.setSession_id(String.valueOf(System.currentTimeMillis()));
message.setRequest_id(String.valueOf(System.currentTimeMillis()));
message.setRequest("dummy request");
List<ColumnNameTypeValue> sensorData = new ArrayList<>();
for (int i = 0; i < columnNames.size(); i++) {
// Workaround
Object value = null;
try {
value = Double.valueOf(csvRecord.get(i));
} catch (NumberFormatException e) {
value = csvRecord.get(i);
}
sensorData.add(new ColumnNameTypeValue(columnNames.get(i), null, value));
}
message.setColumns(sensorData);
String json = gson.toJson(message);
log.info("Sending data: {}", json);
producer.send(new KeyedMessage<String, String>(InternalTopic.TOPIC_DATA.getTopicName(),
STREAM_OPERATIONS.MANIPULATION.INSERT, json));
log.info("Sleeping {} ms...", args[2]);
Thread.sleep(Long.valueOf(args[2]));
}
}
log.info("Program completed.");
}
}
