Java Code Examples for org.apache.commons.csv.CSVRecord#isSet()

/**
 * For reading values from a CSVRecord. If the column was not defined then
 * CSVRecord.get() throws an exception. Therefore for optional CSV columns
 * better to use this function so don't get exception. This way can continue
 * processing and all errors for the data will be logged. Better than just
 * logging first error and then quitting.
 * <p>
 * Also, if the value is empty string then it is converted to null for
 * consistency. Also trims the resulting string since some agencies leave in
 * spaces.
 * 
 * @param record
 *            The data for the row in the CSV file
 * @param name
 *            The name of the column in the CSV file
 * @param required
 *            Whether this value is required. If required and the value is
 *            not set then an error is logged and null is returned.
 * @return The value, or null if it was not defined
 */
private String getValue(CSVRecord record, String name, boolean required) {
	// If the column is not defined in the file then return null.
	// After all, the item is optional so it is fine for it to
	// not be in the file.
	if (!record.isSet(name)) {
		if (required) {
			logger.error("Column {} not defined in file \"{}\" yet it is required", 
					name, getFileName());	
		} 
		return null;
	}
	
	// Get the value. First trim whitespace so that
	// value will be consistent. Sometimes agencies will mistakenly have
	// some whitespace in the columns.
	String value = record.get(name).trim();
	
	// Return the value. But if the value is empty string
	// convert to null for consistency.
	if (value.isEmpty()) {
		if (required) {
			logger.error("For file \"{}\" line number {} for column {} value was not set " + 
					"yet it is required", 
					getFileName(), lineNumber, name);
		}
		return null;
	} else { 
		// Successfully got value so return intern() version of it. Using 
		// intern() because many strings are duplicates such as headsign
		// or shape info. By sharing the strings we can save a huge amount
		// of memory.
		return value.intern();
	}
}
 

private static String getValue(CSVRecord record, String name) {
	if (!record.isSet(name)) {
		logger.error("Column {} not defined", name);	
		return null;
	}
	
	// Get the value. First trim whitespace so that
	// value will be consistent. 
	String value = record.get(name).trim();
	return value;
}
 

private static String getValue(CSVRecord record, String name) {
	if (!record.isSet(name)) {
		logger.error("Column {} not defined", name);	
		return null;
	}
	
	// Get the value. First trim whitespace so that
	// value will be consistent. 
	String value = record.get(name).trim();
	return value;
}
 

private void load(String filename) throws IOException{
    try (InputStream conceptStream = Database.class.getResourceAsStream(filename)) {
        for (CSVRecord conceptRow : CSVFormat.RFC4180.withHeader().parse(new InputStreamReader(conceptStream))) {
            String omopTableName = conceptRow.get("omop_cdm_table");
            String omopFieldName = conceptRow.get("omop_cdm_field");

            Concept concept = new Concept();
            concept.setConceptId(conceptRow.get("concept_id"));
            concept.setConceptName(conceptRow.get("concept_name"));
            concept.setStandardConcept(conceptRow.get("standard_concept"));

            // Optional fields
            if (conceptRow.isSet("domain_id")) {
                concept.setDomainId(conceptRow.get("domain_id"));
            }

            if (conceptRow.isSet("vocabulary_id")) {
                concept.setVocabularyId(conceptRow.get("vocabulary_id"));
            }

            if (conceptRow.isSet("concept_class_id")) {
                concept.setConceptClassId(conceptRow.get("concept_class_id"));
            }

            this.put(omopTableName, omopFieldName, concept);
        }
    } catch (IOException e) {
        throw new IOException("Could not load concept_id hints: " + e.getMessage());
    }
}
 

@Override
public GenericRow extract(CSVRecord from, GenericRow to) {
  for (String fieldName : _fields) {
    String value = from.isSet(fieldName) ? from.get(fieldName) : null;
    if (value == null || StringUtils.isEmpty(value)) {
      to.putValue(fieldName, null);
      // NOTE about CSV behavior for empty string e.g. foo,bar,,zoo or foo,bar,"",zoo. These both are equivalent to a CSVParser
      // Empty string has to be treated as null, as this could be a column of any data type.
      // This could be incorrect for STRING dataType, as "" could be a legit entry, different than null.
    } else {
      String[] stringValues = StringUtils.split(value, _multiValueDelimiter);
      int numValues = stringValues.length;
      // NOTE about CSV behavior for multi value column - cannot distinguish between multi value column with just 1 entry vs single value
      // MV column with single value will be treated as single value until DataTypeTransformer.
      // Transform functions on such columns will have to handle the special case.
      if (numValues > 1) {
        Object[] array = new Object[numValues];
        int index = 0;
        for (String stringValue : stringValues) {
            array[index++] = stringValue;
        }
        to.putValue(fieldName, array);
      } else {
        to.putValue(fieldName, value);
      }
    }
  }
  return to;
}
 

public static Database generateModelFromCSV(InputStream stream, String dbName) {
	Database database = new Database();

	database.dbName = dbName.substring(0, dbName.lastIndexOf("."));

	Map<String, Table> nameToTable = new HashMap<>();
	try {
		ConceptsMap conceptsMap = new ConceptsMap(CONCEPT_ID_HINTS_FILE_NAME);

		for (CSVRecord row : CSVFormat.RFC4180.withHeader().parse(new InputStreamReader(stream))) {
			String tableNameColumn;
			String fieldNameColumn;
			String isNullableColumn;
			String nullableValue;
			String dataTypeColumn;
			String descriptionColumn;
			if (row.isSet("TABLE_NAME")) {
				tableNameColumn = "TABLE_NAME";
				fieldNameColumn = "COLUMN_NAME";
				isNullableColumn = "IS_NULLABLE";
				nullableValue = "YES";
				dataTypeColumn = "DATA_TYPE";
				descriptionColumn = "DESCRIPTION";
			} else {
				tableNameColumn = "table";
				fieldNameColumn = "field";
				isNullableColumn = "required";
				nullableValue = "No";
				dataTypeColumn = "type";
				descriptionColumn = "description";
			}
			Table table = nameToTable.get(row.get(tableNameColumn).toLowerCase());

			if (table == null) {
				table = new Table();
				table.setDb(database);
				table.setName(row.get(tableNameColumn).toLowerCase());
				nameToTable.put(row.get(tableNameColumn).toLowerCase(), table);
				database.tables.add(table);
			}
			Field field = new Field(row.get(fieldNameColumn).toLowerCase(), table);
			field.setNullable(row.get(isNullableColumn).equals(nullableValue));
			field.setType(row.get(dataTypeColumn));
			field.setDescription(row.get(descriptionColumn));
			field.setConceptIdHints(conceptsMap.get(table.getName(), field.getName()));

			table.getFields().add(field);
		}
	} catch (IOException e) {
		throw new RuntimeException(e.getMessage());
	}
	return database;
}