Java Code Examples for org.apache.commons.csv.CSVParser#getHeaderMap()

/**
 * Send records to getToolFileDetails() and store the output in neo4j database
 *
 * @param csvParser
 * @param label
 * @param insightsTimeField
 * @param insightsTimeFormat
 * @return boolean
 * @throws InsightsCustomException
 */
private boolean parseCsvRecords(CSVParser csvParser, String label, String insightsTimeField,
		String insightsTimeFormat) throws InsightsCustomException {
	List<JsonObject> nodeProperties = new ArrayList<>();
	String query = "UNWIND {props} AS properties " + "CREATE (n:" + label.toUpperCase() + ") "
			+ "SET n = properties";
	Map<String, Integer> headerMap = csvParser.getHeaderMap();
	try {
		if (headerMap.containsKey("")) {
			throw new InsightsCustomException("Error in file.");
		} else if (headerMap.containsKey(insightsTimeField)) {
			for (CSVRecord csvRecord : csvParser.getRecords()) {
				JsonObject json = getCSVRecordDetails(csvRecord, headerMap, insightsTimeField, insightsTimeFormat);
				nodeProperties.add(json);
			}
		} else {
			throw new InsightsCustomException("Insights Time Field not present in csv file");
		}
		insertDataInDatabase(nodeProperties, query);
		return true;
	} catch (Exception ex) {
		log.error("Error while parsing the .CSV records. {} ", ex.getMessage());
		throw new InsightsCustomException(ex.getMessage());
	}
}
 

@Override
public List<CSVReportEntry> apply(final GetCredentialReportResult report) {
    Assert.state(Textcsv.toString().equals(report.getReportFormat()), "unknown credential report format: " + report.getReportFormat());

    try (final Reader r = new BufferedReader(new InputStreamReader(new ByteBufferBackedInputStream(report.getContent())))) {
        final CSVParser parser = new CSVParser(r, CSV_FORMAT);
        final Map<String, Integer> headers = parser.getHeaderMap();

        Assert.state(headers.containsKey("user"), "Header 'user' not found in CSV");
        Assert.state(headers.containsKey("arn"), "Header 'arn' not found in CSV");
        Assert.state(headers.containsKey("password_enabled"), "Header 'password_enabled' not found in CSV");
        Assert.state(headers.containsKey("mfa_active"), "Header 'mfa_active' not found in CSV");
        Assert.state(headers.containsKey("access_key_1_active"), "Header 'access_key_1_active' not found in CSV");
        Assert.state(headers.containsKey("access_key_2_active"), "Header 'access_key_2_active' not found in CSV");

        return stream(parser.spliterator(), false).map(this::toCSVReportEntry).filter(Objects::nonNull).collect(toList());
    } catch (final IOException e) {
        throw new RuntimeException("Could not read csv report", e);
    }
}
 

public static CSVHParser getCSVHParser(File file) {
    if (file.exists()) {
        try (Reader in = new FileReader(file)) {
            CSVParser parser = CSVFormat.EXCEL.withHeader().withSkipHeaderRecord().withIgnoreEmptyLines().parse(in);
            if (!parser.getHeaderMap().isEmpty()) {
                return new CSVHParser(parser.getHeaderMap(), parser.getRecords());
            }
        } catch (IOException ex) {
            LOGGER.log(Level.SEVERE, null, ex);
        }
    } else {
        LOGGER.log(Level.SEVERE, "File [{0}] doesn''t exist", file.getAbsolutePath());
    }
    return null;
}
 

@Override
public void execute(Query query) {
    String queryName = query.name;
    String queryValue = query.value.trim();
    log.info("Running {}", queryName);

    Map<String, String> metadata = query.getMetadata();

    String delimiter = Query.getKey(metadata, METADATA_DELIMITER_KEY).orElse(defaultDelimiter);
    char character = delimiter.charAt(0);
    log.info("Using delimiter as a character: {}", character);

    boolean isFile = isPath(queryValue);

    log.info("Running or loading data from \n{}", queryValue);

    try (InputStream stream = isFile ? new FileInputStream(queryValue) : new ByteArrayInputStream(getBytes(queryValue));
         Reader reader = new InputStreamReader(stream)) {

        CSVFormat format = CSVFormat.RFC4180.withDelimiter(character).withFirstRecordAsHeader().withIgnoreSurroundingSpaces();
        CSVParser parser = new CSVParser(reader, format);

        Map<String, Integer> headerIndices = parser.getHeaderMap();
        Map<String, Type> typeMap = getTypeMapping(headerIndices, metadata);
        Result result = query.createResults();
        parser.iterator().forEachRemaining(r -> addRow(r, result, typeMap));
    } catch (Exception e) {
        log.error("Error while parsing data for {} with {}", queryName, queryValue);
        log.error("Error", e);
        query.setFailure(e.toString());
    }
}
 

@Override
public RowSet getRows(String baseQuery,
                      Map<String, String> preds,
                      int limit,
                      int offset) throws Exception{

    filename = conf.getString(MacroBaseConf.CSV_INPUT_FILE);
    Compression compression = conf.getCsvCompression();

    if (compression == Compression.GZIP) {
        InputStream fileStream = new FileInputStream(filename);
        InputStream gzipStream = new GZIPInputStream(fileStream);
        Reader decoder = new InputStreamReader(gzipStream);
        csvParser = new CSVParser(decoder, CSVFormat.DEFAULT.withHeader());
    } else {
        File csvFile = new File(conf.getString(MacroBaseConf.CSV_INPUT_FILE));
        csvParser = CSVParser.parse(csvFile, Charset.defaultCharset(), CSVFormat.DEFAULT.withHeader());
    }
    schema = csvParser.getHeaderMap();
    Iterator<CSVRecord> rawIterator = csvParser.iterator();
    int rowCount = 0;

    List<RowSet.Row> rows = Lists.newArrayList();
    while (rawIterator.hasNext() && rowCount < limit) {
        CSVRecord record = rawIterator.next();
        List<ColumnValue> columnValues = Lists.newArrayList();

        if (includeRow(record, preds)) {
            for (Map.Entry<String, Integer> se : schema.entrySet()) {
                columnValues.add(new ColumnValue(se.getKey(),record.get(se.getValue())));
            }

            rows.add(new RowSet.Row(columnValues));
            rowCount++;
        }
    }
     return new RowSet(rows);
}
 

@Override
public MBStream<Datum> getStream() throws Exception {
    if(!loaded) {
        long st = System.currentTimeMillis();

        filename = conf.getString(MacroBaseConf.CSV_INPUT_FILE);
        Compression compression = conf.getCsvCompression();

        if (compression == Compression.GZIP) {
            InputStream fileStream = new FileInputStream(filename);
            InputStream gzipStream = new GZIPInputStream(fileStream);
            Reader decoder = new InputStreamReader(gzipStream);
            csvParser = new CSVParser(decoder, CSVFormat.DEFAULT.withHeader());
        } else {
            File csvFile = new File(conf.getString(MacroBaseConf.CSV_INPUT_FILE));
            csvParser = CSVParser.parse(csvFile, Charset.defaultCharset(), CSVFormat.DEFAULT.withHeader());
        }
        schema = csvParser.getHeaderMap(); //equal to resultSet.getmetadata or smt

        for (Map.Entry<String, Integer> se : schema.entrySet()) {
            conf.getEncoder().recordAttributeName(se.getValue() + 1, se.getKey()); //numbering off each column for encoding
        }

        // Load all records into memory to filter out rows with missing data
        Iterator<CSVRecord> rawIterator = csvParser.iterator();

        int numRows = 0;
        while (rawIterator.hasNext()) {
            try {
                CSVRecord record = rawIterator.next();
                Datum curRow = parseRecord(record);
                dataStream.add(curRow);
                numRows++;
            } catch (NumberFormatException e) {
                badRows++;
            }
        }
        log.info("{}/{} rows successfully parsed ({} malformed rows)", numRows, numRows + badRows, badRows);
    }

    return dataStream;
}