com.csvreader.CsvReader Java Examples

public static double[] readDoubleArrayFromCSVFile(Path inFilePath,  int nrows, char delimiter) throws NumberFormatException, IOException {
    double[] mlDouble = new double[nrows];
    CsvReader cvsReader = new CsvReader(inFilePath.toString());
    cvsReader.setDelimiter(delimiter);
    int i = 0;
    while (cvsReader.readRecord()) {
        String[] rows = cvsReader.getValues();
        for (String col : rows) {
            mlDouble[i] = Double.parseDouble(col);
        }
        i++;
        if (i >= nrows) {
            break;
        }
    }
    return mlDouble;
}
 

private List<OrganisationUnitGroup> orgUnitGroupsFromCsv( CsvReader reader )
    throws IOException
{
    List<OrganisationUnitGroup> list = new ArrayList<>();

    while ( reader.readRecord() )
    {
        String[] values = reader.getValues();

        if ( values != null && values.length > 0 )
        {
            OrganisationUnitGroup object = new OrganisationUnitGroup();
            setIdentifiableObject( object, values );
            object.setAutoFields();
            object.setShortName( getSafe( values, 3, object.getName(), 50 ) );
            list.add( object );
        }
    }

    return list;
}
 

private List<CategoryOptionGroup> categoryOptionGroupsFromCsv( CsvReader reader )
    throws IOException
{
    List<CategoryOptionGroup> list = new ArrayList<>();

    while ( reader.readRecord() )
    {
        String[] values = reader.getValues();

        if ( values != null && values.length > 0 )
        {
            CategoryOptionGroup object = new CategoryOptionGroup();
            setIdentifiableObject( object, values );
            object.setShortName( getSafe( values, 3, object.getName(), 50 ) );
            list.add( object );
        }
    }

    return list;
}
 

private List<CategoryCombo> categoryCombosFromCsv( CsvReader reader )
    throws IOException
{
    List<CategoryCombo> list = new ArrayList<>();

    while ( reader.readRecord() )
    {
        String[] values = reader.getValues();

        if ( values != null && values.length > 0 )
        {
            CategoryCombo object = new CategoryCombo();
            setIdentifiableObject( object, values );
            object.setDataDimensionType( DataDimensionType.valueOf( getSafe( values, 3, DataDimensionType.DISAGGREGATION.toString(), 40 ) ) );
            object.setSkipTotal( Boolean.valueOf( getSafe( values, 4, Boolean.FALSE.toString(), 40 ) ) );
            list.add( object );
        }
    }

    return list;
}
 

private List<Category> categoriesFromCsv( CsvReader reader )
    throws IOException
{
    List<Category> list = new ArrayList<>();

    while ( reader.readRecord() )
    {
        String[] values = reader.getValues();

        if ( values != null && values.length > 0 )
        {
            Category object = new Category();
            setIdentifiableObject( object, values );
            object.setDescription( getSafe( values, 3, null, 255 ) );
            object.setDataDimensionType( DataDimensionType.valueOf( getSafe( values, 4, DataDimensionType.DISAGGREGATION.toString(), 40 ) ) );
            object.setDataDimension( Boolean.valueOf( getSafe( values, 5, Boolean.FALSE.toString(), 40 ) ) );
            list.add( object );
        }
    }

    return list;
}
 

private List<CategoryOption> categoryOptionsFromCsv( CsvReader reader )
    throws IOException
{
    List<CategoryOption> list = new ArrayList<>();

    while ( reader.readRecord() )
    {
        String[] values = reader.getValues();

        if ( values != null && values.length > 0 )
        {
            CategoryOption object = new CategoryOption();
            setIdentifiableObject( object, values );
            object.setShortName( getSafe( values, 3, object.getName(), 50 ) );
            list.add( object );
        }
    }

    return list;
}
 

/**
 * @param inputLine
 *            输入待分隔字符串
 * @param delimiter
 *            字符串分割符
 * @return 分隔符分隔后的字符串数组，出现异常时返回为null 支持转义，即数据中可包含分隔符
 * */
public static String[] splitOneLine(String inputLine, char delimiter) {
	String[] splitedResult = null;
	if (null != inputLine) {
		try {
			CsvReader csvReader = new CsvReader(new StringReader(inputLine));
			csvReader.setDelimiter(delimiter);
			if (csvReader.readRecord()) {
				splitedResult = csvReader.getValues();
			}
		} catch (IOException e) {
			// nothing to do
		}
	}
	return splitedResult;
}
 

private List<DataElementGroup> dataElementGroupsFromCsv( CsvReader reader )
    throws IOException
{
    List<DataElementGroup> list = new ArrayList<>();

    while ( reader.readRecord() )
    {
        String[] values = reader.getValues();

        if ( values != null && values.length > 0 )
        {
            DataElementGroup object = new DataElementGroup();
            setIdentifiableObject( object, values );
            object.setShortName( getSafe( values, 3, object.getName(), 50 ) );
            object.setAutoFields();
            list.add( object );
        }
    }

    return list;
}
 

@Override
@Transactional
public ImportSummary saveDataValueSetCsv( InputStream in, ImportOptions importOptions, JobConfiguration id )
{
    try
    {
        in = StreamUtils.wrapAndCheckCompressionFormat( in );
        CsvReader csvReader = CsvUtils.getReader( in );

        if ( importOptions == null || importOptions.isFirstRowIsHeader() )
        {
            csvReader.readRecord(); // Ignore the first row
        }

        DataValueSet dataValueSet = new StreamingCsvDataValueSet( csvReader );
        return saveDataValueSet( importOptions, id, dataValueSet );
    }
    catch ( Exception ex )
    {
        log.error( DebugUtils.getStackTrace( ex ) );
        notifier.clear( id ).notify( id, ERROR, "Process failed: " + ex.getMessage(), true );
        return new ImportSummary( ImportStatus.ERROR, "The import process failed: " + ex.getMessage() );
    }
}
 

/**
 * Returns the CSV file represented by the given input stream as a
 * list of string arrays.
 *
 * @param in the {@link InputStream} representing the CSV file.
 * @param ignoreFirstRow whether to ignore the first row.
 * @return a list of string arrays.
 * @throws IOException
 */
public static List<String[]> readCsvAsList( InputStream in, boolean ignoreFirstRow )
    throws IOException
{
    CsvReader reader = getReader( in );

    if ( ignoreFirstRow )
    {
        reader.readRecord();
    }

    List<String[]> lines = new ArrayList<>();

    while ( reader.readRecord() )
    {
        lines.add( reader.getValues() );
    }

    return lines;
}
 

public static SensorObservation getAarhusWeatherObservation(CsvReader streamData, EventDeclaration ed) {
	try {
		// CsvReader streamData = (CsvReader) data;
		int hum = Integer.parseInt(streamData.get("hum"));
		double tempm = Double.parseDouble(streamData.get("tempm"));
		double wspdm = Double.parseDouble(streamData.get("wspdm"));
		Date obTime = sdf2.parse(streamData.get("TIMESTAMP"));
		WeatherObservation wo = new WeatherObservation(tempm, hum, wspdm, obTime);
		logger.debug(ed.getServiceId() + ": streaming record @" + wo.getObTimeStamp());
		wo.setObId("AarhusWeatherObservation-" + (int) Math.random() * 1000);
		// this.currentObservation = wo;
		return wo;
	} catch (NumberFormatException | IOException | ParseException e) {
		e.printStackTrace();
	}
	return null;

}
 

public static double[][] readDoubleMatrixFromCSVFile(Path inFilePath, int nrows, int ncols, char delimiter) throws NumberFormatException, IOException {
    double[][] mlDouble = new double[nrows][ncols];
    CsvReader cvsReader = new CsvReader(inFilePath.toString());
    cvsReader.setDelimiter(delimiter);
    int i = 0;
    while (cvsReader.readRecord()) {
        String[] rows = cvsReader.getValues();
        int j = 0;
        for (String col : rows) {
            mlDouble[i][j] = Double.parseDouble(col);
            j++;
            if (j >= ncols) {
                break;
            }
        }
        i++;
        if (i >= nrows) {
            break;
        }

    }
    return mlDouble;
}
 

public static MLDouble readMDoubleFromCSVFile(Path inFilePath, String mName, int nrows, int ncols, char delimiter) throws NumberFormatException, IOException {
    MLDouble mlDouble = new MLDouble(mName, new int[] {nrows, ncols});
    CsvReader cvsReader = new CsvReader(inFilePath.toString());
    cvsReader.setDelimiter(delimiter);
    int i = 0;
    while (cvsReader.readRecord()) {
        String[] rows = cvsReader.getValues();
        int j = 0;
        for (String col : rows) {
            mlDouble.set(new Double(col), i, j);
            j++;
        }
        i++;
    }
    return mlDouble;
}
 

@Test
public void get() {
    try {
        CsvReader e = new CsvReader("src/test/resources/aml/iomatrix/testcsv.csv");
        e.readRecord();
        String val = e.get(0);
        Assert.assertEquals("1494574053", val);
        String val2 = e.get(100500);
        Assert.assertEquals("", val2);
        String val3 = e.get("SOME_COLUMN");
        Assert.assertEquals("", val3);
        
        System.out.println("finished");
        
    } catch (Exception ex) {
        fail(ex.getMessage());
    }
}
 

private int prepareSafetySwitchTest(boolean safetySwitchParameter, int columnLength) throws IOException {
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    CsvWriter csvWriter = new CsvWriter(new BufferedOutputStream(out), ',', Charset.forName("UTF-8"));

    StringBuilder sb = new StringBuilder();
    for (int i = 0; i < columnLength; i++) {
        sb.append("a");
    }
    String[] data = new String[] { "fieldValueA", "fieldValueB", sb.toString() };
    csvWriter.writeRecord(data);
    csvWriter.close();

    CsvReader csvReader = new CsvReader(new BufferedInputStream(new ByteArrayInputStream(out.toByteArray())), ',',
            Charset.forName("UTF-8"));
    csvReader.setSafetySwitch(safetySwitchParameter);
    BulkResultSet resultSet = new BulkResultSet(csvReader, Arrays.asList("fieldA", "fieldB", "fieldC"));
    BulkResult result = resultSet.next();
    return ((String) result.getValue("fieldC")).length();
}
 

public synchronized static SensorObservation getAarhusPollutionObservation(CsvReader streamData, EventDeclaration ed) {

		try {
			// CsvReader streamData = (CsvReader) data;
			int ozone = Integer.parseInt(streamData.get("ozone")), particullate_matter = Integer.parseInt(streamData
					.get("particullate_matter")), carbon_monoxide = Integer.parseInt(streamData.get("carbon_monoxide")), sulfure_dioxide = Integer
					.parseInt(streamData.get("sulfure_dioxide")), nitrogen_dioxide = Integer.parseInt(streamData
					.get("nitrogen_dioxide"));
			Date obTime = sdf.parse(streamData.get("timestamp"));
			PollutionObservation po = new PollutionObservation(0.0, 0.0, 0.0, ozone, particullate_matter,
					carbon_monoxide, sulfure_dioxide, nitrogen_dioxide, obTime);
			// logger.debug(ed.getServiceId() + ": streaming record @" + po.getObTimeStamp());
			po.setObId("AarhusPollutionObservation-" + (int) Math.random() * 10000);
			return po;
		} catch (NumberFormatException | IOException | ParseException e) {
			e.printStackTrace();
		}
		return null;
	}
 

public static SensorObservation getAarhusTrafficObservation(CsvReader streamData, EventDeclaration ed) {
	try {
		// CsvReader streamData = (CsvReader) objData;
		AarhusTrafficObservation data;
		// if (!this.txtFile.contains("mean"))
		data = new AarhusTrafficObservation(Double.parseDouble(streamData.get("REPORT_ID")),
				Double.parseDouble(streamData.get("avgSpeed")), Double.parseDouble(streamData.get("vehicleCount")),
				Double.parseDouble(streamData.get("avgMeasuredTime")), 0, 0, null, null, 0.0, 0.0, null, null, 0.0,
				0.0, null, null, streamData.get("TIMESTAMP"));
		String obId = "AarhusTrafficObservation-" + streamData.get("_id");
		Double distance = Double.parseDouble(((TrafficReportService) ed).getDistance() + "");
		if (data.getAverageSpeed() != 0)
			data.setEstimatedTime(distance / data.getAverageSpeed());
		else
			data.setEstimatedTime(-1.0);
		if (distance != 0)
			data.setCongestionLevel(data.getVehicle_count() / distance);
		else
			data.setCongestionLevel(-1.0);
		data.setObId(obId);
		;
		// this.currentObservation = data;
		return data;
	} catch (NumberFormatException | IOException e) {
		e.printStackTrace();
	}
	return null;
}
 

public CSPARQLAarhusWeatherStream(String uri, String txtFile, EventDeclaration ed) throws IOException {
	super(uri);
	streamData = new CsvReader(String.valueOf(txtFile));
	this.ed = ed;
	streamData.setTrimWhitespace(false);
	streamData.setDelimiter(',');
	streamData.readHeaders();
}
 

public CSPARQLAarhusWeatherStream(String uri, String txtFile, EventDeclaration ed, Date start, Date end)
		throws IOException {
	super(uri);
	streamData = new CsvReader(String.valueOf(txtFile));
	this.ed = ed;
	streamData.setTrimWhitespace(false);
	streamData.setDelimiter(',');
	streamData.readHeaders();
	this.startDate = start;
	this.endDate = end;
}
 

public CQELSAarhusTrafficStream(ExecContext context, String uri, String txtFile, EventDeclaration ed, Date start,
		Date end) throws IOException {
	super(context, uri);
	this.startDate = start;
	this.endDate = end;
	messageCnt = 0;
	byteCnt = 0;
	this.txtFile = txtFile;
	this.ed = ed;
	streamData = new CsvReader(String.valueOf(txtFile));
	streamData.setTrimWhitespace(false);
	streamData.setDelimiter(',');
	streamData.readHeaders();
	metaData = new CsvReader("dataset/MetaData/trafficMetaData.csv");
	metaData.readHeaders();
	streamData.readRecord();
	while (metaData.readRecord()) {
		if (streamData.get("REPORT_ID").equals(metaData.get("REPORT_ID"))) {
			// p1Street = metaData.get("POINT_1_STREET");
			// p1City = metaData.get("POINT_1_CITY");
			// p1Lat = metaData.get("POINT_1_LAT");
			// p1Lon = metaData.get("POINT_1_LNG");
			// p1Country = metaData.get("POINT_2_COUNTRY");
			// p2Street = metaData.get("POINT_2_STREET");
			// p2City = metaData.get("POINT_2_CITY");
			// p2Lat = metaData.get("POINT_2_LAT");
			// p2Lon = metaData.get("POINT_2_LNG");
			// p2Country = metaData.get("POINT_2_COUNTRY");
			distance = metaData.get("DISTANCE_IN_METERS");
			// timestamp = metaData.get("TIMESTAMP");
			// id = metaData.get("extID");
			metaData.close();
			break;
		}
	}
}
 

public CSPARQLAarhusParkingStream(String uri, String txtFile, EventDeclaration ed) throws IOException {
	super(uri);
	streamData = new CsvReader(String.valueOf(txtFile));
	this.ed = ed;
	streamData.setTrimWhitespace(false);
	streamData.setDelimiter(',');
	streamData.readHeaders();
}
 

public CSPARQLAarhusParkingStream(String uri, String txtFile, EventDeclaration ed, Date start, Date end)
		throws IOException {
	super(uri);
	logger.info("init");
	streamData = new CsvReader(String.valueOf(txtFile));
	this.ed = ed;
	streamData.setTrimWhitespace(false);
	streamData.setDelimiter(',');
	streamData.readHeaders();
	this.startDate = start;
	this.endDate = end;
}
 

public CQELSAarhusParkingStream(ExecContext context, String uri, String txtFile, EventDeclaration ed)
		throws IOException {
	super(context, uri);
	streamData = new CsvReader(String.valueOf(txtFile));
	this.ed = ed;
	streamData.setTrimWhitespace(false);
	streamData.setDelimiter(',');
	streamData.readHeaders();
}
 

public CSPARQLAarhusPollutionStream(String uri, String txtFile, EventDeclaration ed) throws IOException {
	super(uri);
	streamData = new CsvReader(String.valueOf(txtFile));
	this.ed = ed;
	streamData.setTrimWhitespace(false);
	streamData.setDelimiter(',');
	streamData.readHeaders();
}
 

public CSPARQLAarhusPollutionStream(String uri, String txtFile, EventDeclaration ed, Date start, Date end)
		throws IOException {
	super(uri);

	streamData = new CsvReader(String.valueOf(txtFile));
	this.ed = ed;
	streamData.setTrimWhitespace(false);
	streamData.setDelimiter(',');
	streamData.readHeaders();
	this.startDate = start;
	this.endDate = end;
}
 

private List<OrganisationUnitGroup> orgUnitGroupMembersFromCsv( CsvReader reader )
    throws IOException
{
    CachingMap<String, OrganisationUnitGroup> uidMap = new CachingMap<>();

    while ( reader.readRecord() )
    {
        String[] values = reader.getValues();

        if ( values != null && values.length > 0 )
        {
            String groupUid = values[0];
            String memberUid = values[1];

            OrganisationUnitGroup persistedGroup = organisationUnitGroupService.getOrganisationUnitGroup( groupUid );

            if ( persistedGroup != null )
            {

                OrganisationUnitGroup group = uidMap.get( groupUid, () -> {
                    OrganisationUnitGroup nonPersistedGroup = new OrganisationUnitGroup();

                    nonPersistedGroup.setUid( persistedGroup.getUid() );
                    nonPersistedGroup.setName( persistedGroup.getName() );

                    return nonPersistedGroup;
                } );

                OrganisationUnit member = new OrganisationUnit();
                member.setUid( memberUid );
                group.addOrganisationUnit( member );
            }
        }
    }

    return new ArrayList<>( uidMap.values() );
}
 

public CSPARQLAarhusTrafficStream(String uri, String txtFile, EventDeclaration ed, Date start, Date end)
		throws IOException {
	super(uri);
	logger.info("IRI: " + this.getIRI().split("#")[1] + ed.getInternalQos());
	this.startDate = start;
	this.endDate = end;
	messageCnt = 0;
	byteCnt = 0;
	this.txtFile = txtFile;
	this.ed = ed;
	streamData = new CsvReader(String.valueOf(txtFile));
	streamData.setTrimWhitespace(false);
	streamData.setDelimiter(',');
	streamData.readHeaders();
	metaData = new CsvReader("dataset/MetaData/trafficMetaData.csv");
	metaData.readHeaders();
	streamData.readRecord();
	while (metaData.readRecord()) {
		if (streamData.get("REPORT_ID").equals(metaData.get("REPORT_ID"))) {
			// p1Street = metaData.get("POINT_1_STREET");
			// p1City = metaData.get("POINT_1_CITY");
			// p1Lat = metaData.get("POINT_1_LAT");
			// p1Lon = metaData.get("POINT_1_LNG");
			// p1Country = metaData.get("POINT_2_COUNTRY");
			// p2Street = metaData.get("POINT_2_STREET");
			// p2City = metaData.get("POINT_2_CITY");
			// p2Lat = metaData.get("POINT_2_LAT");
			// p2Lon = metaData.get("POINT_2_LNG");
			// p2Country = metaData.get("POINT_2_COUNTRY");
			distance = metaData.get("DISTANCE_IN_METERS");
			if (ed instanceof TrafficReportService)
				((TrafficReportService) ed).setDistance(Integer.parseInt(distance));

			// timestamp = metaData.get("TIMESTAMP");
			// id = metaData.get("extID");
			metaData.close();
			break;
		}
	}
}
 

public CQELSAarhusPollutionStream(ExecContext context, String uri, String txtFile, EventDeclaration ed)
		throws IOException {
	super(context, uri);
	streamData = new CsvReader(String.valueOf(txtFile));
	this.ed = ed;
	streamData.setTrimWhitespace(false);
	streamData.setDelimiter(',');
	streamData.readHeaders();
}
 

/**
 * Read csv file and store result in list of maps
 *
 * @param csvFile
 */
@When("^I read info from csv file '(.+?)' with separator '(.+?)'$")
public void readFromCSV(String csvFile, String separator) throws Exception {
    //By default separator is a coma
    char sep = ',';
    if (separator.length() > 1) {
        switch (separator) {
            case "\\t":
                sep = '\t';
                break;
            default:
                sep = ',';
                break;
        }
    } else {
        sep = separator.charAt(0);
    }

    CsvReader rows = new CsvReader(csvFile, sep);

    String[] columns = null;
    if (rows.readRecord()) {
        columns = rows.getValues();
        rows.setHeaders(columns);
    }

    List<Map<String, String>> results = new ArrayList<Map<String, String>>();
    while (rows.readRecord()) {
        Map<String, String> row = new HashMap<String, String>();
        for (String column : columns) {
            row.put(column, rows.get(rows.getIndex(column)));
        }
        results.add(row);
    }

    rows.close();

    commonspec.setResultsType("csv");
    commonspec.setCSVResults(results);
}
 

public CSVMatrixReader(InputStream inputStream, char delimiter, String encoding) throws IOException {

        Reader rdr = new UnicodeReader(inputStream, encoding);

        reader = new CsvReader(rdr);
        reader.setDelimiter(delimiter);
        reader.setSkipEmptyRecords(false);
        reader.setTrimWhitespace(true);
        readRecord();
    }