Java Code Examples for com.opencsv.CSVReader#readNext()

public void processFormattedFile(BufferedReader fr, FileProcessorState state) throws Exception {
    CSVReader csvr = new CSVReader(fr);
    String[] line = null;

    while (((line = csvr.readNext()) != null)) {
        state.setRecordCnt(state.getRecordCnt() + 1);

        boolean headerPresent = state.isHeaderRowPresent();

        if (state.getColumns() != line.length) {
            state.appendError("Wrong Number Columns Row:, " + state.getRecordCnt() + "Saw:" + line.length + ", Expecting: " + state.getColumns());
            state.setErrorCnt(state.getErrorCnt() + 1);
        } else if ((headerPresent && state.getRecordCnt() > 1) || !headerPresent) {
            try {
                line = trimLine(line);
                processRow(line, state);
                state.setProcessedCnt(state.getProcessedCnt() + 1);
            } catch (Exception e) {
                log.debug(e.getMessage(), e);
                state.appendError("Row " + state.getRecordCnt() + " " + e.getMessage());
                state.setErrorCnt(state.getErrorCnt() + 1);
            }
        }
    }
    fr.close();
}
 

private Response getPreviewFromCSV(CSVReader reader, final int numLines) {
    List<Map<String, String>> columns = new ArrayList<>();
    List<List<String>> rows = new ArrayList<>();
    try {
        for (final String columnName: reader.readNext()) {
            columns.add(new HashMap<String, String>(){{
                put("name", columnName);
            }});
        }
        int counter = 0;
        for (String[] line : reader) {
            counter++;
            rows.add(Arrays.asList(line));
            if (counter >= numLines) {
              break;
            }
        }
    } catch (IOException e) {
        log.error(e.getMessage());
        return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
    }
    return Response.ok(new PreviewResponse(columns, rows)).build();
}
 

public static List<String[]> oneByOne(Reader reader) {
    List<String[]> list = new ArrayList<>();
    try {
        CSVParser parser = new CSVParserBuilder()
                .withSeparator(',')
                .withIgnoreQuotations(true)
                .build();

        CSVReader csvReader = new CSVReaderBuilder(reader)
                .withSkipLines(0)
                .withCSVParser(parser)
                .build();

        String[] line;
        while ((line = csvReader.readNext()) != null) {
            list.add(line);
        }
        reader.close();
        csvReader.close();
    } catch (Exception ex) {
        Helpers.err(ex);
    }
    return list;
}
 

public static void main(String[] args) throws FileNotFoundException, IOException {
    // TODO code application logic here 
    File phase3File = new File("C:\\Users\\Sophie Mulc\\Documents\\DEPICT2\\phase3_corrected.psam");
    File traitFile = new File("C:\\Users\\Sophie Mulc\\Documents\\DEPICT2\\TraitFile.txt");
    File probeAnnotationFile = new File("C:\\Users\\Sophie Mulc\\Documents\\DEPICT2\\ProbeAnnotationFile.txt");
    File couplingFile = new File("C:\\Users\\Sophie Mulc\\Documents\\DEPICT2\\CouplingFile.txt");
    //FileReader(String phase3_corrected)
    final CSVParser gmtParser = new CSVParserBuilder().withSeparator('\t').withIgnoreQuotations(true).build();
    final CSVReader gmtReader = new CSVReaderBuilder(new BufferedReader(new FileReader(phase3File))).withSkipLines(1).withCSVParser(gmtParser).build();


    List<String> iids = new ArrayList<>();

    String[] inputLine;
    while ((inputLine = gmtReader.readNext()) != null) {

        String iid = inputLine[0];

        iids.add(iid);
    }

    trait(iids, traitFile);
    probeAnnotation(probeAnnotationFile);
    coupling(iids, couplingFile);
}
 

public static Map<String, Map<String, String>> readNestedMapFromFile(String fileName, boolean ignoreFirstLine) throws IOException {
	Map<String, Map<String, String>> map = new HashMap<>();

	CSVReader reader = new CSVReader(new FileReader(fileName));
	if(ignoreFirstLine) reader.readNext();
	String[] line = reader.readNext();
	while(line != null) {
		MapUtils.getMap(line[0], map).put(line[1], line[2]);
		line = reader.readNext();
	}
	reader.close();
	return map;
}
 

/**
 * @param fileName excluding .dat
 * @return
 * @throws FileNotFoundException
 * @throws IOException
 */
public static DoubleMatrixDataset<String, String> loadTransEqtlExpressionMatrix(String fileName) throws FileNotFoundException, IOException {

	File matrix = new File(fileName + ".dat");
	File probeFile = new File(fileName + "-ColNames.txt.gz");
	File snpFile = new File(fileName + "-RowNames.txt.gz");

	LinkedHashMap<String, Integer> rowMap = new LinkedHashMap<>();
	LinkedHashMap<String, Integer> colMap = new LinkedHashMap<>();

	final CSVParser parser = new CSVParserBuilder().withSeparator('\t').withIgnoreQuotations(true).build();
	final CSVReader probeReader = new CSVReaderBuilder(new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(probeFile))))).withSkipLines(0).withCSVParser(parser).build();
	final CSVReader snpReader = new CSVReaderBuilder(new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(snpFile))))).withSkipLines(1).withCSVParser(parser).build();

	String[] nextLine;
	int nrCols = 0;
	while ((nextLine = probeReader.readNext()) != null) {
		if (colMap.put(nextLine[0], nrCols++) != null) {
			throw new RuntimeException("Duplicate col names not allowed: " + nextLine[0]);
		}
	}

	int nrRows = 0;
	while ((nextLine = snpReader.readNext()) != null) {
		if (rowMap.put(nextLine[0], nrRows++) != null) {
			throw new RuntimeException("Duplicate row names not allowed: " + nextLine[0]);
		}
	}

	System.out.println("Number of cols in " + probeFile.getName() + ": " + nrCols);
	System.out.println("Number of rows in " + snpFile.getName() + ": " + nrRows);

	DoubleMatrixDataset<String, String> dataset = new DoubleMatrixDataset<>(rowMap, colMap);

	DataInputStream dis = new DataInputStream(new FileInputStream(matrix));
	int magicNumber = dis.readInt();
	if (magicNumber == 1) {
		throw new RuntimeException("Cannot read cis matrix");
	} else if (magicNumber > 1) {
		throw new RuntimeException("Invalid magic number");
	}

	for (int r = 0; r < nrRows; ++r) {

		for (int c = 0; c < nrCols; ++c) {

			dataset.setElementQuick(r, c, dis.readFloat());

		}

	}

	System.out.println("Done loading eQTL result matrix");

	return dataset;

}
 

private void processImportedUserFile(SessionState state, Context context, Reference file) {
	
	try{
		ContentResource resource = contentHostingService.getResource(file.getId());
		String contentType = resource.getContentType();
		
		//check mime type
		if(!StringUtils.equals(contentType, CSV_MIME_TYPE)) {
			addAlert(state, rb.getString("import.error"));
			return;
		}
		//SAK-21405 SAK-21884 original parse method, auto maps column headers to bean properties
		/*
		HeaderColumnNameTranslateMappingStrategy<ImportedUser> strat = new HeaderColumnNameTranslateMappingStrategy<ImportedUser>();
		strat.setType(ImportedUser.class);

		//map the column headers to the field names in the ImportedUser class
		Map<String, String> map = new HashMap<String, String>();
		map.put("user id", "eid");
		map.put("first name", "firstName");
		map.put("last name", "lastName");
		map.put("email", "email");
		map.put("password", "password");
		map.put("type", "type");
		map.put("properties", "rawProps"); //specially formatted string, see ImportedUser class.
		
		strat.setColumnMapping(map);

		CsvToBean<ImportedUser> csv = new CsvToBean<ImportedUser>();
		List<ImportedUser> list = new ArrayList<ImportedUser>();
		
		list = csv.parse(strat, new CSVReader(new InputStreamReader(resource.streamContent())));
		*/
		
		//SAK-21884 manual parse method so we can support arbitrary columns
		CSVReader reader = new CSVReader(new InputStreamReader(resource.streamContent()));
	    String [] nextLine;
	    int lineCount = 0;
	    List<ImportedUser> list = new ArrayList<ImportedUser>();
	    Map<Integer,String> mapping = null;
	    
	    while ((nextLine = reader.readNext()) != null) {
	        
	    	if(lineCount == 0) {
	        	//header row, capture it
	    		mapping = mapHeaderRow(nextLine);
	        } else {
	        	//map the fields into the object
	        	list.add(mapLine(nextLine, mapping));
	        }
	    	
	        lineCount++;
	    }
		
		state.setAttribute("importedUsers", list);
		context.put("importedUsers", list);
		
	} catch (Exception e) {
	 	log.error("Error reading imported file: {}", e.getMessage(), e);
		addAlert(state, rb.getString("import.error"));
		return;
	}
	
	return;

}
 

private static HashMap<String, String> loadNcbiToEnsgMap(File ncbiToEnsgMapFile) throws FileNotFoundException, IOException, Exception {

		final CSVParser parser = new CSVParserBuilder().withSeparator('\t').withIgnoreQuotations(true).build();
		final CSVReader reader = new CSVReaderBuilder(new BufferedReader(new FileReader(ncbiToEnsgMapFile))).withSkipLines(0).withCSVParser(parser).build();

		String[] nextLine = reader.readNext();

		if (!nextLine[0].equals("Gene stable ID") || !nextLine[1].equals("NCBI gene ID")) {
			throw new Exception("Header of ncbi to ensg map should be: \"Gene stable ID	NCBI gene ID\"");
		}

		HashMap<String, String> ncbiToEnsgMap = new HashMap<>(70000);

		while ((nextLine = reader.readNext()) != null) {
			ncbiToEnsgMap.put(nextLine[1], nextLine[0]);
		}

		return ncbiToEnsgMap;

	}
 

@Override
public Result loadDataFromNetwork() throws Exception {
    long entryTime = System.currentTimeMillis();
    int recCount = 0;

    final long size = getFileSize(uri, context);

    InputStream inputStream = context.getContentResolver().openInputStream(uri);

    if(inputStream == null) {
        throw new IOException();
    }

    CountingInputStream countingInputStream = new CountingInputStream(inputStream);
    SamplerDatabase database = SamplerDatabase.getInstance(context);

    try {
        CSVReader reader = new CSVReader(new InputStreamReader(countingInputStream, "UTF-8"));
        final String[] headerLine = reader.readNext();
        if (headerLine == null) {
            throw new IOException();
        }

        int bssidIndex = -1;
        int latIndex = -1;
        int lonIndex = -1;
        int ssidIndex = -1;
        int idx = 0;
        for (String s : headerLine) {
            if (s.equals("bssid"))
                bssidIndex = idx;
            else if (s.equals("lat"))
                latIndex = idx;
            else if (s.equals("lon"))
                lonIndex = idx;
            else if (s.equals("ssid"))
                ssidIndex = idx;
            idx++;
        }
        Log.i(TAG, "bssidIndex=" + bssidIndex +
                ", latIndex=" + latIndex +
                ", lonIndex=" + lonIndex +
                ", ssidIndex=" + ssidIndex);
        if ((bssidIndex < 0) || (latIndex < 0) || (lonIndex < 0)) {
            throw new IOException();
        }
        String[] nextLine;

        database.beginTransaction();
        while ((nextLine = reader.readNext()) != null) {
            String rfId = nextLine[bssidIndex];
            String latString = nextLine[latIndex];
            String lonString = nextLine[lonIndex];
            String ssid = "";
            if (ssidIndex >= 0)
                ssid = nextLine[ssidIndex];

            database.addSample(Database.TYPE_WIFI, ssid, rfId, SimpleLocation.fromLatLon(latString,lonString,false));
            recCount++;
            if ((recCount % 100) == 0) {
                // Log.i(TAG, "recCount="+recCount+", committing transaction.");
                database.commitTransaction();
                database.beginTransaction();
            }

            if (size != 0) {
                publishProgress(countingInputStream.getBytesRead() * MAX_PROGRESS / size);
            }
        }
    } catch (Exception e) {
        Log.i(TAG, e.toString());
        e.printStackTrace();
    } finally {
        inputStream.close();
        database.commitTransaction();
    }
    Log.i(TAG, "Total Records processed: " + recCount);
    Log.i(TAG, "Import data elapsed time: " + (System.currentTimeMillis() - entryTime) + " ms");

    return null;
}
 

private static ArrayList<String> readGenes(File geneOrderFile) throws IOException {

		final CSVParser parser = new CSVParserBuilder().withSeparator('\t').withIgnoreQuotations(true).build();
		final CSVReader reader = new CSVReaderBuilder(new BufferedReader(new FileReader(geneOrderFile))).withSkipLines(0).withCSVParser(parser).build();

		String[] nextLine;
		ArrayList<String> geneOrder = new ArrayList<>();

		while ((nextLine = reader.readNext()) != null) {

			geneOrder.add(nextLine[0]);

		}

		return geneOrder;

	}
 

private static Map<String, String> loadUpdatedIds(File updatedIdFile) throws IOException {

		final CSVParser parser = new CSVParserBuilder().withSeparator('\t').withIgnoreQuotations(true).build();
		final CSVReader reader = new CSVReaderBuilder(new BufferedReader(new FileReader(updatedIdFile))).withCSVParser(parser).build();

		HashMap<String, String> updates = new HashMap<>();

		String[] nextLine;
		while ((nextLine = reader.readNext()) != null) {

			updates.put(nextLine[0], nextLine[1]);

		}

		return Collections.unmodifiableMap(updates);

	}
 

public static SampleIdIncludeFilter readSampleFile(File sampleFile) throws IOException {

        final CSVParser parser = new CSVParserBuilder().withSeparator('\t').withIgnoreQuotations(true).build();
        final CSVReader reader = new CSVReaderBuilder(new BufferedReader(new FileReader(sampleFile))).withCSVParser(parser).withSkipLines(0).build();

        final HashSet<String> samples = new HashSet<>();

        String[] nextLine;
        while ((nextLine = reader.readNext()) != null) {

            samples.add(nextLine[0]);

        }

        return new SampleIdIncludeFilter(samples);

    }
 

private static HashMap<String, HashSet<String>> loadVariantGeneMapping(File variantGeneMappingFile) throws FileNotFoundException, IOException {

		final CSVParser parser = new CSVParserBuilder().withSeparator('\t').withIgnoreQuotations(true).build();
		final CSVReader reader = new CSVReaderBuilder(new BufferedReader(new FileReader(variantGeneMappingFile))).withCSVParser(parser).build();

		HashMap<String, HashSet<String>> geneVariantMapping = new HashMap<>();

		String[] nextLine;
		while ((nextLine = reader.readNext()) != null) {

			HashSet<String> geneVariant = geneVariantMapping.get(nextLine[1]);
			if (geneVariant == null) {
				geneVariant = new HashSet<>();
				geneVariantMapping.put(nextLine[1], geneVariant);
			}
			geneVariant.add(nextLine[0]);

		}

		return geneVariantMapping;

	}
 

private static ArrayList<String> readGenes(File geneOrderFile) throws IOException {

		final CSVParser parser = new CSVParserBuilder().withSeparator('\t').withIgnoreQuotations(true).build();
		final CSVReader reader = new CSVReaderBuilder(new BufferedReader(new FileReader(geneOrderFile))).withSkipLines(0).withCSVParser(parser).build();

		String[] nextLine;
		ArrayList<String> geneOrder = new ArrayList<>();

		while ((nextLine = reader.readNext()) != null) {

			geneOrder.add(nextLine[0]);

		}

		return geneOrder;

	}
 

private static Map<String, String> loadUpdatedIds(File updatedIdFile) throws IOException {

		final CSVParser parser = new CSVParserBuilder().withSeparator('\t').withIgnoreQuotations(true).build();
		final CSVReader reader = new CSVReaderBuilder(new BufferedReader(new FileReader(updatedIdFile))).withCSVParser(parser).build();

		HashMap<String, String> updates = new HashMap<>();

		String[] nextLine;
		while ((nextLine = reader.readNext()) != null) {

			updates.put(nextLine[0], nextLine[1]);

		}

		return Collections.unmodifiableMap(updates);

	}
 

/**
 * @param args the command line arguments
 * @throws java.io.IOException
 */
public static void main(String[] args) throws IOException {
    // TODO code application logic here

    //To compare the format of the files we created in the eQTL pipeline for running GWAS to the format needed to run in Lude's code.
    final File predictionMatrixFile = new File("/groups/umcg-wijmenga/scr02/umcg-smulcahy/eQTLResults_texttrue2/eQTL.binary");
    System.out.println(predictionMatrixFile);
    DoubleMatrixDataset<String, String> predictionMatrixFull = DoubleMatrixDataset.loadTransEqtlExpressionMatrix(predictionMatrixFile.getAbsolutePath());

    System.out.println(predictionMatrixFull.getElement("rs351365", "PH443"));
    File eQTLfile = new File("/groups/umcg-wijmenga/scr02/umcg-smulcahy/eQTLResults_texttrue2/eQTLs.txt.gz");
    final CSVParser eQTLparser = new CSVParserBuilder().withSeparator('\t').withIgnoreQuotations(true).build();
    final CSVReader eQTLreader = new CSVReaderBuilder((new InputStreamReader(new GZIPInputStream(new FileInputStream(eQTLfile))))).withSkipLines(1).withCSVParser(eQTLparser).build();

    int c = 0;
    String[] inputLine;
    while ((inputLine = eQTLreader.readNext()) != null) {

        String snp = inputLine[1];
        String pheno = inputLine[4];
        String zscore = inputLine[10];

        // Test with one site only need this line and to read in file: predictionMatrixFull.getElement("rs351365", "PH443"); i.e (rowName(snp), columnName(pheno))
    

        double zscore2 = (predictionMatrixFull.getElement(snp, pheno));

        //convert string to double, then look up how to compare two doubles - this is to compare the zscores
        double d_zscore = Double.parseDouble(zscore);

        double compare_zscores = d_zscore - zscore2;

        //count occurances of above 0 comparisons
        if (Math.abs(compare_zscores) > 0.00001) {
            c++;
        }
    }
    System.out.println("Number of occurrances where z-scores differ: " + c);
    //save new file 
    predictionMatrixFull.saveBinary("/groups/umcg-wijmenga/scr02/umcg-smulcahy/eQTLResults_texttrue2/eQTL2");
    
    System.out.println("Done saving");

}
 

public static Map<String, PredictionInfo> loadPredictionInfo(File hpoPredictionInfoFile) throws FileNotFoundException, IOException {

		final CSVParser parser = new CSVParserBuilder().withSeparator('\t').withIgnoreQuotations(true).build();
		final CSVReader reader = new CSVReaderBuilder(new BufferedReader(new FileReader(hpoPredictionInfoFile))).withSkipLines(1).withCSVParser(parser).build();

		HashMap<String, PredictionInfo> predictionInfo = new HashMap<>();

		String[] nextLine;
		while ((nextLine = reader.readNext()) != null) {

			PredictionInfo info = new PredictionInfo(nextLine[0], Double.parseDouble(nextLine[2]), Double.parseDouble(nextLine[3]), Double.parseDouble(nextLine[4]));
			predictionInfo.put(info.getHpo(), info);

		}

		return Collections.unmodifiableMap(predictionInfo);

	}
 

public static HashMap<String, ArrayList<String>> loadNcbiToEnsgMap(File ncbiToEnsgMapFile) throws FileNotFoundException, IOException, Exception {

		final CSVParser parser = new CSVParserBuilder().withSeparator('\t').withIgnoreQuotations(true).build();
		final CSVReader reader = new CSVReaderBuilder(new BufferedReader(new FileReader(ncbiToEnsgMapFile))).withSkipLines(0).withCSVParser(parser).build();

		String[] nextLine = reader.readNext();

		if (!nextLine[0].equals("Gene stable ID") || !nextLine[1].equals("NCBI gene ID")) {
			throw new Exception("Header of ncbi to ensg map should be: \"Gene stable ID	NCBI gene ID\"");
		}

		HashMap<String, ArrayList<String>> ncbiToEnsgMap = new HashMap<>(70000);

		while ((nextLine = reader.readNext()) != null) {

			String ncbiId = nextLine[1];

			ArrayList<String> ncbiEnsgIds = ncbiToEnsgMap.get(ncbiId);
			if (ncbiEnsgIds == null) {
				ncbiEnsgIds = new ArrayList<>();
				ncbiToEnsgMap.put(ncbiId, ncbiEnsgIds);
			}

			ncbiEnsgIds.add(nextLine[0]);

		}

		return ncbiToEnsgMap;

	}
 

public static HashMap<String, ArrayList<String>> loadNcbiToEnsgMap(File ncbiToEnsgMapFile) throws FileNotFoundException, IOException, Exception {

		final CSVParser parser = new CSVParserBuilder().withSeparator('\t').withIgnoreQuotations(true).build();
		final CSVReader reader = new CSVReaderBuilder(new BufferedReader(new FileReader(ncbiToEnsgMapFile))).withSkipLines(0).withCSVParser(parser).build();

		String[] nextLine = reader.readNext();

		if (!nextLine[0].equals("Gene stable ID") || !nextLine[1].equals("NCBI gene ID")) {
			throw new Exception("Header of ncbi to ensg map should be: \"Gene stable ID	NCBI gene ID\"");
		}

		HashMap<String, ArrayList<String>> ncbiToEnsgMap = new HashMap<>(70000);

		while ((nextLine = reader.readNext()) != null) {

			String ncbiId = nextLine[1];

			ArrayList<String> ncbiEnsgIds = ncbiToEnsgMap.get(ncbiId);
			if (ncbiEnsgIds == null) {
				ncbiEnsgIds = new ArrayList<>();
				ncbiToEnsgMap.put(ncbiId, ncbiEnsgIds);
			}

			ncbiEnsgIds.add(nextLine[0]);

		}

		return ncbiToEnsgMap;

	}