org.supercsv.io.CsvListReader Java Examples

static CsvParsingContext readCsvHeader(CsvListReader csvListReader, String separatorStr) throws IOException {
    String[] tokens = csvListReader.getHeader(true);
    if (tokens == null) {
        throw new TimeSeriesException("CSV header is missing");
    }

    if (tokens.length < 2 || !"Time".equals(tokens[0]) || !"Version".equals(tokens[1])) {
        throw new TimeSeriesException("Bad CSV header, should be \nTime" + separatorStr + "Version" + separatorStr + "...");
    }
    List<String> duplicates = new ArrayList<>();
    Set<String> namesWithoutDuplicates = new HashSet<>();
    for (String token : tokens) {
        if (!namesWithoutDuplicates.add(token)) {
            duplicates.add(token);
        }
    }
    if (!duplicates.isEmpty()) {
        throw new TimeSeriesException("Bad CSV header, there are duplicates in time series names " + duplicates);
    }
    List<String> names = Arrays.asList(tokens).subList(2, tokens.length);
    return new CsvParsingContext(names);
}
 

static Map<Integer, List<TimeSeries>> parseCsv(BufferedReader reader, char separator) {
    Objects.requireNonNull(reader);

    Stopwatch stopwatch = Stopwatch.createStarted();

    Map<Integer, List<TimeSeries>> timeSeriesPerVersion = new HashMap<>();
    String separatorStr = Character.toString(separator);

    try {
        CsvListReader csvListReader = new CsvListReader(reader, new CsvPreference.Builder('"', separator, System.lineSeparator()).build());
        CsvParsingContext context = readCsvHeader(csvListReader, separatorStr);
        readCsvValues(csvListReader, context, timeSeriesPerVersion);
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    }

    LoggerFactory.getLogger(TimeSeries.class)
            .info("{} time series loaded from CSV in {} ms",
            timeSeriesPerVersion.entrySet().stream().mapToInt(e -> e.getValue().size()).sum(),
            stopwatch.elapsed(TimeUnit.MILLISECONDS));

    return timeSeriesPerVersion;
}
 

static void readCsvValues(CsvListReader reader, CsvParsingContext context,
                          Map<Integer, List<TimeSeries>> timeSeriesPerVersion) throws IOException {
    List<String> tokens;
    int currentVersion = Integer.MIN_VALUE;
    while ((tokens = reader.read()) != null) {

        if (tokens.size() != context.names.size() + 2 && tokens.size() != context.names.size() + 1) {
            throw new TimeSeriesException("Columns of line " + context.times.size() + " are inconsistent with header");
        }

        int version = Integer.parseInt(tokens.get(1));
        if (currentVersion == Integer.MIN_VALUE) {
            currentVersion = version;
        } else if (version != currentVersion) {
            timeSeriesPerVersion.put(currentVersion, context.createTimeSeries());
            context.reInit();
            currentVersion = version;
        }

        context.parseLine(tokens);
    }
    timeSeriesPerVersion.put(currentVersion, context.createTimeSeries());
}
 

public static Table<String, String, Float> parseMatrix(Reader reader) throws IOException {
    Table<String, String, Float> table = HashBasedTable.create();
    try (ICsvListReader csvReader = new CsvListReader(reader, CsvPreference.STANDARD_PREFERENCE)) {
        List<String> columnHeaders = csvReader.read();
        List<String> row;
        while ((row = csvReader.read()) != null) {
            String rowHeader = row.get(0);
            for (int i = 1; i < row.size(); i++) {
                String columnHeader = columnHeaders.get(i);
                String value = row.get(i);
                table.put(rowHeader, columnHeader, value == null ? Float.NaN : Float.parseFloat(value));
            }
        }
    }
    return table;
}
 

String[] getColumns(Map<String, String> query) throws IOException, InterruptedException {
    Objects.requireNonNull(query, "query is null");

    Map<String, String> queryMap = new LinkedHashMap<>();
    queryMap.put("headers", "true");
    queryMap.put("count", "1");
    if (query != null) {
        queryMap.putAll(query);
    }

    try (InputStream is =  httpClient.getHttpRequest(new HistoDbUrl(config, "itesla/" + HistoQueryType.data + ".csv", queryMap));
         CsvListReader reader = new CsvListReader(new InputStreamReader(is), CsvPreference.STANDARD_PREFERENCE)) {
        return reader.getHeader(true);
    }

}
 

@Test
public void testSkipCommentLines() throws IOException {
	String csv = "<!--Sarah,Connor-->\r\nJohn,Connor\r\n<!--Kyle,Reese-->";
	
	CsvPreference customPreference = new Builder(STANDARD_PREFERENCE).skipComments(new CommentMatches("<!--.*-->"))
		.build();
	CsvListReader listReader = new CsvListReader(new StringReader(csv), customPreference);
	List<String> line = listReader.read();
	List<String> emptyLine = listReader.read();
	
	Assert.assertNotNull(line);
	Assert.assertEquals(2, line.size());
	Assert.assertEquals("John", line.get(0));
	Assert.assertEquals("Connor", line.get(1));
	Assert.assertNull(emptyLine);
}
 

static
private boolean checkFormat(BufferedReader reader, CsvPreference format) throws IOException {
	CsvListReader parser = new CsvListReader(reader, format){

		@Override
		public void close(){
		}
	};

	int columns = 0;

	// Check the header line and the first ten lines
	for(int line = 0; line < (1 + 10); line++){
		List<String> row = parser.read();

		if(row == null){
			break;
		}

		int rowColumns = row.size();
		if((rowColumns > 1) && (columns == 0 || columns == rowColumns)){
			columns = rowColumns;
		} else

		{
			return false;
		}
	}

	parser.close();

	return (columns > 1);
}
 

@Test
public void testIgnoreEmptyLines() throws IOException {
	String csv = "\r\n\n\nJohn,Connor\r\n\r\n";
	
	CsvListReader listReader = new CsvListReader(new StringReader(csv), STANDARD_PREFERENCE);
	List<String> line = listReader.read();
	List<String> emptyLine = listReader.read();
	
	Assert.assertNotNull(line);
	Assert.assertEquals(2, line.size());
	Assert.assertEquals("John", line.get(0));
	Assert.assertEquals("Connor", line.get(1));
	Assert.assertNull(emptyLine);
}
 

@Test
public void testReadSingleLineStreaming() throws IOException {
	String csv = "Sarah,Connor\r\nJohn,Connor\r\nKyle,Reese";
	
	CsvListReader listReader = new CsvListReader(new StringReader(csv), STANDARD_PREFERENCE);
	// skip first line
	listReader.read();
	// read second line
	List<String> line = listReader.read();
	
	Assert.assertNotNull(line);
	Assert.assertEquals(2, line.size());
	Assert.assertEquals("John", line.get(0));
	Assert.assertEquals("Connor", line.get(1));
}
 

@Test
public void testGetByColumnIndex() throws IOException {
	String csv = "John,Connor";
	
	CsvListReader listReader = new CsvListReader(new StringReader(csv), STANDARD_PREFERENCE);
	List<String> line = listReader.read();
	
	Assert.assertNotNull(line);
	Assert.assertEquals(2, line.size());
	Assert.assertEquals("John", line.get(0));
	Assert.assertEquals("Connor", line.get(1));
}
 

@Test
public void testDealWithLeadingTrailingWhitespace() throws IOException {
	String csv = "     John    ,     Connor   ";
	CellProcessor[] processors = { new Trim(), new Trim() };
	
	char customQuote = '"';
	CsvPreference customPreference = new Builder(customQuote, ',', "").build();
	CsvListReader listReader = new CsvListReader(new StringReader(csv), customPreference);
	List<Object> result = listReader.read(processors);
	
	Assert.assertNotNull(result);
	Assert.assertEquals(2, result.size());
	Assert.assertEquals("John", result.get(0));
	Assert.assertEquals("Connor", result.get(1));
}
 

@Test
public void testEscapedQuoteInQuotedField() throws IOException {
	String csv = "\"Joh\"\"n\",\"Con\"\"nor\"";
	CellProcessor[] processors = { new NotNull(), new NotNull() };
	
	CsvPreference customPreference = new Builder('"', ',', "").build();
	CsvListReader listReader = new CsvListReader(new StringReader(csv), customPreference);
	List<Object> result = listReader.read(processors);
	
	Assert.assertNotNull(result);
	Assert.assertEquals(2, result.size());
	Assert.assertEquals("Joh\"n", result.get(0));
	Assert.assertEquals("Con\"nor", result.get(1));
}
 

@Test
public void testNewLineInDelimitedField() throws IOException {
	String csv = "\"Jo\nhn\",\"Con\nnor\"\n";
	CellProcessor[] processors = { new NotNull(), new NotNull() };
	
	CsvPreference customPreference = new Builder('"', ',', "\n").build();
	CsvListReader listReader = new CsvListReader(new StringReader(csv), customPreference);
	List<Object> result = listReader.read(processors);
	
	Assert.assertNotNull(result);
	Assert.assertEquals(2, result.size());
	Assert.assertEquals("Jo\nhn", result.get(0));
	Assert.assertEquals("Con\nnor", result.get(1));
}
 

@Test
public void testCustomEOL() throws IOException {
	String csv = "John,Connor\r>\n";
	CellProcessor[] processors = { new NotNull(), new NotNull() };
	
	String customEndOfLine = "\r>\n";
	CsvPreference customPreference = new Builder('"', ',', customEndOfLine).build();
	CsvListReader listReader = new CsvListReader(new StringReader(csv), customPreference);
	List<Object> result = listReader.read(processors);
	
	Assert.assertNotNull(result);
	Assert.assertEquals(2, result.size());
	Assert.assertEquals("John", result.get(0));
	Assert.assertEquals("Connor", result.get(1));
}
 

@Test
public void testCustomQuote() throws IOException {
	String csv = "|John  Connor|";
	CellProcessor[] processors = { new NotNull() };
	
	char customQuote = '|';
	CsvPreference customPreference = new Builder(customQuote, ',', "").build();
	CsvListReader listReader = new CsvListReader(new StringReader(csv), customPreference);
	List<Object> result = listReader.read(processors);
	
	Assert.assertNotNull(result);
	Assert.assertEquals(1, result.size());
	Assert.assertEquals("John  Connor", result.get(0));
}
 

@Test
public void testCustomSeparator() throws IOException {
	String csv = "John+Connor";
	CellProcessor[] processors = { new NotNull(), new NotNull() };
	
	char customSeparator = '+';
	CsvPreference customPreference = new Builder('"', customSeparator, "").build();
	CsvListReader listReader = new CsvListReader(new StringReader(csv), customPreference);
	List<Object> result = listReader.read(processors);
	
	Assert.assertNotNull(result);
	Assert.assertEquals(2, result.size());
	Assert.assertEquals("John", result.get(0));
	Assert.assertEquals("Connor", result.get(1));
}
 

@Override
public void setup(OperatorContext context)
{
  super.setup(context);
  parseMapping();
  lineListReader = new CsvListReader(lineSr,
  CsvPreference.STANDARD_PREFERENCE);
}
 

@Override
public void setup(OperatorContext context)
{
  super.setup(context);
  parseMapping();
  lineListReader = new CsvListReader(lineSr,CsvPreference.STANDARD_PREFERENCE);
}
 

public void readCsv(BufferedReader reader, char separator) {
    CsvListReader csvReader = new CsvListReader(reader, createPreference(separator));
    List<String> nextLine;
    try {
        while ((nextLine = csvReader.read()) != null) {
            if (nextLine.size() != 2) {
                throw new PowsyblException("Invalid line '" + nextLine + "'");
            }
            mapping.put(nextLine.get(0), nextLine.get(1));
        }
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    }
}
 

public static List<String> parseHintedHandoffEnabledDCs(final String dcNames) throws IOException
{
    final CsvListReader csvListReader = new CsvListReader(new StringReader(dcNames), STANDARD_SURROUNDING_SPACES_NEED_QUOTES);
    return csvListReader.read();
}
 

public static DataFrame<Object> readCsv(final InputStream input, String separator, NumberDefault numDefault, String naString, boolean hasHeader)
throws IOException {
    CsvPreference csvPreference;
    switch (separator) {
        case "\\t":
            csvPreference = CsvPreference.TAB_PREFERENCE;
            break;
        case ",":
            csvPreference = CsvPreference.STANDARD_PREFERENCE;
            break;
        case ";":
            csvPreference = CsvPreference.EXCEL_NORTH_EUROPE_PREFERENCE;
            break;
        case "|":
        	csvPreference  = new CsvPreference.Builder('"', '|', "\n").build();
            break;
        default:
            throw new IllegalArgumentException("Separator: " + separator + " is not currently supported");
    }
    try (CsvListReader reader = new CsvListReader(new InputStreamReader(input), csvPreference)) {
    	final List<String> header;
    	final DataFrame<Object> df;
    	final CellProcessor[] procs;
    	if(hasHeader) {
    		header = Arrays.asList(reader.getHeader(true));
    		procs = new CellProcessor[header.size()];
            df = new DataFrame<>(header);
    	} else {
    		// Read the first row to figure out how many columns we have
    		reader.read();
    		header = new ArrayList<String>();
    		for (int i = 0; i < reader.length(); i++) {
	header.add("V"+i);
}
    		procs = new CellProcessor[header.size()];
    		df = new DataFrame<>(header);
    		// The following line executes the procs on the previously read row again
    		df.append(reader.executeProcessors(procs));
    	}
        for (List<Object> row = reader.read(procs); row != null; row = reader.read(procs)) {
            df.append(new ArrayList<>(row));
        }
        return df.convert(numDefault, naString);
    }
}
 

@Test
public void foo() throws IOException {

    char[] delimeters = new char[]{',', '|', '\t', ';', '!'};
    char[] quoteChars = new char[]{'"', '\''};


    List<ParseAnalysis> analysisResults = ListUtilities.newArrayList();


    for (char quoteChar : quoteChars) {
        for (char delimeter : delimeters) {

            InputStreamReader simpleReader = getReader();

            CsvPreference prefs = new CsvPreference.Builder(quoteChar, delimeter, "\n")
                    .ignoreEmptyLines(false)
                    .build();

            CsvListReader csvReader = new CsvListReader(simpleReader, prefs);

            List<String> cells = csvReader.read();

            ImmutableParseAnalysis.Builder parseAnalysisBuilder = ImmutableParseAnalysis.builder()
                    .quoteChar(quoteChar)
                    .delimiterChar(delimeter);


            while (cells != null) {
                parseAnalysisBuilder.addFieldCounts(cells.size());
                cells = csvReader.read();
            }

            ParseAnalysis parseAnalysis = parseAnalysisBuilder.build();
            analysisResults.add(parseAnalysis);

        }
    }

    analysisResults
            .forEach(r -> {
                System.out.println(r.quoteChar()
                        + " "
                        + r.delimiterChar()
                        + " => [ "
                        + r.fieldCounts().size()
                        + " ] "
                        + r.fieldCounts());
            });
}