Java Code Examples for org.apache.flink.types.parser.FieldParser#getParserForType()

protected void setFieldTypesGeneric(Class<?> ... fieldTypes) {
	if (fieldTypes == null) {
		throw new IllegalArgumentException("Field types must not be null.");
	}
	
	this.fieldIncluded = new boolean[fieldTypes.length];
	ArrayList<Class<?>> types = new ArrayList<Class<?>>();
	
	// check if we support parsers for these types
	for (int i = 0; i < fieldTypes.length; i++) {
		Class<?> type = fieldTypes[i];
		
		if (type != null) {
			if (FieldParser.getParserForType(type) == null) {
				throw new IllegalArgumentException("The type '" + type.getName() + "' is not supported for the CSV input format.");
			}
			types.add(type);
			fieldIncluded[i] = true;
		}
	}

	this.fieldTypes = types.toArray(new Class<?>[types.size()]);
}
 

protected void setFieldTypesGeneric(Class<?> ... fieldTypes) {
	if (fieldTypes == null) {
		throw new IllegalArgumentException("Field types must not be null.");
	}
	
	this.fieldIncluded = new boolean[fieldTypes.length];
	ArrayList<Class<?>> types = new ArrayList<Class<?>>();
	
	// check if we support parsers for these types
	for (int i = 0; i < fieldTypes.length; i++) {
		Class<?> type = fieldTypes[i];
		
		if (type != null) {
			if (FieldParser.getParserForType(type) == null) {
				throw new IllegalArgumentException("The type '" + type.getName() + "' is not supported for the CSV input format.");
			}
			types.add(type);
			fieldIncluded[i] = true;
		}
	}

	this.fieldTypes = types.toArray(new Class<?>[types.size()]);
}
 

protected void setFieldTypesGeneric(Class<?> ... fieldTypes) {
	if (fieldTypes == null) {
		throw new IllegalArgumentException("Field types must not be null.");
	}
	
	this.fieldIncluded = new boolean[fieldTypes.length];
	ArrayList<Class<?>> types = new ArrayList<Class<?>>();
	
	// check if we support parsers for these types
	for (int i = 0; i < fieldTypes.length; i++) {
		Class<?> type = fieldTypes[i];
		
		if (type != null) {
			if (FieldParser.getParserForType(type) == null) {
				throw new IllegalArgumentException("The type '" + type.getName() + "' is not supported for the CSV input format.");
			}
			types.add(type);
			fieldIncluded[i] = true;
		}
	}

	this.fieldTypes = types.toArray(new Class<?>[types.size()]);
}
 

private void initializeParsers() {
    Class<?>[] fieldClasses = extractTypeClasses(fieldTypes);

    // instantiate the parsers
    FieldParser<?>[] parsers = new FieldParser<?>[fieldClasses.length];

    for (int i = 0; i < fieldClasses.length; i++) {
        if (fieldClasses[i] != null) {
            Class<? extends FieldParser<?>> parserType = FieldParser.getParserForType(fieldClasses[i]);
            if (parserType == null) {
                throw new RuntimeException("No parser available for type '" + fieldClasses[i].getName() + "'.");
            }

            FieldParser<?> p = InstantiationUtil.instantiate(parserType, FieldParser.class);
            p.setCharset(charset);
            parsers[i] = p;
        }
    }
    this.fieldParsers = parsers;
    this.holders = new Object[fieldTypes.length];
    for (int i = 0; i < fieldTypes.length; i++) {
        holders[i] = fieldParsers[i].createValue();
    }
}
 

/**
 * The Constructor.
 *
 * @param types      Column types.
 * @param fieldDelim Field delimiter in the text line.
 * @param quoteChar  Quoting character. Contents between a pair of quoting chars are treated as a field, even if
 *                   contains field delimiters. Two consecutive quoting chars represents a real quoting char.
 */
public CsvParser(TypeInformation[] types, String fieldDelim, @Nullable Character quoteChar) {
    this.fieldDelim = fieldDelim;
    this.lenFieldDelim = this.fieldDelim.length();
    this.quoteChar = quoteChar;
    this.enableQuote = quoteChar != null;
    this.parsers = new FieldParser[types.length];
    this.isString = new boolean[types.length];

    if (enableQuote) {
        this.quoteString = quoteChar.toString();
        this.escapedQuote = this.quoteString + this.quoteString;
    }

    for (int i = 0; i < types.length; i++) {
        Class typeClazz = types[i].getTypeClass();
        Class<? extends FieldParser<?>> parserType = FieldParser.getParserForType(typeClazz);
        if (parserType == null) {
            throw new RuntimeException("No parser available for type '" + typeClazz.getName() + "'.");
        }
        parsers[i] = InstantiationUtil.instantiate(parserType, FieldParser.class);
        isString[i] = types[i].equals(Types.STRING);
    }
}
 

@Override
public void open(FileInputSplit split) throws IOException {
	super.open(split);

	// instantiate the parsers
	FieldParser<?>[] parsers = new FieldParser<?>[fieldTypes.length];
	
	for (int i = 0; i < fieldTypes.length; i++) {
		if (fieldTypes[i] != null) {
			Class<? extends FieldParser<?>> parserType = FieldParser.getParserForType(fieldTypes[i]);
			if (parserType == null) {
				throw new RuntimeException("No parser available for type '" + fieldTypes[i].getName() + "'.");
			}

			FieldParser<?> p = InstantiationUtil.instantiate(parserType, FieldParser.class);

			p.setCharset(getCharset());
			if (this.quotedStringParsing) {
				if (p instanceof StringParser) {
					((StringParser)p).enableQuotedStringParsing(this.quoteCharacter);
				} else if (p instanceof StringValueParser) {
					((StringValueParser)p).enableQuotedStringParsing(this.quoteCharacter);
				}
			}

			parsers[i] = p;
		}
	}
	this.fieldParsers = parsers;
	
	// skip the first line, if we are at the beginning of a file and have the option set
	if (this.skipFirstLineAsHeader && this.splitStart == 0) {
		readLine(); // read and ignore
	}
}
 

@Override
public void open(FileInputSplit split) throws IOException {
	super.open(split);
	Class<? extends FieldParser<OT>> parserType = FieldParser.getParserForType(primitiveClass);
	if (parserType == null) {
		throw new IllegalArgumentException("The type '" + primitiveClass.getName() + "' is not supported for the primitive input format.");
	}
	parser = InstantiationUtil.instantiate(parserType, FieldParser.class);
}
 

protected void setFieldsGeneric(boolean[] includedMask, Class<?>[] fieldTypes) {
	checkNotNull(includedMask);
	checkNotNull(fieldTypes);

	ArrayList<Class<?>> types = new ArrayList<Class<?>>();

	// check if types are valid for included fields
	int typeIndex = 0;
	for (int i = 0; i < includedMask.length; i++) {

		if (includedMask[i]) {
			if (typeIndex > fieldTypes.length - 1) {
				throw new IllegalArgumentException("Missing type for included field " + i + ".");
			}
			Class<?> type = fieldTypes[typeIndex++];

			if (type == null) {
				throw new IllegalArgumentException("Type for included field " + i + " should not be null.");
			} else {
				// check if we support parsers for this type
				if (FieldParser.getParserForType(type) == null) {
					throw new IllegalArgumentException("The type '" + type.getName() + "' is not supported for the CSV input format.");
				}
				types.add(type);
			}
		}
	}

	this.fieldTypes = types.toArray(new Class<?>[types.size()]);
	this.fieldIncluded = includedMask;
}
 

protected void setFieldsGeneric(int[] sourceFieldIndices, Class<?>[] fieldTypes) {
	checkNotNull(sourceFieldIndices);
	checkNotNull(fieldTypes);
	checkArgument(sourceFieldIndices.length == fieldTypes.length,
		"Number of field indices and field types must match.");

	for (int i : sourceFieldIndices) {
		if (i < 0) {
			throw new IllegalArgumentException("Field indices must not be smaller than zero.");
		}
	}

	int largestFieldIndex = max(sourceFieldIndices);
	this.fieldIncluded = new boolean[largestFieldIndex + 1];
	ArrayList<Class<?>> types = new ArrayList<Class<?>>();

	// check if we support parsers for these types
	for (int i = 0; i < fieldTypes.length; i++) {
		Class<?> type = fieldTypes[i];

		if (type != null) {
			if (FieldParser.getParserForType(type) == null) {
				throw new IllegalArgumentException("The type '" + type.getName()
					+ "' is not supported for the CSV input format.");
			}
			types.add(type);
			fieldIncluded[sourceFieldIndices[i]] = true;
		}
	}

	this.fieldTypes = types.toArray(new Class<?>[types.size()]);
}
 

@Override
public void open(FileInputSplit split) throws IOException {
	super.open(split);
	Class<? extends FieldParser<OT>> parserType = FieldParser.getParserForType(primitiveClass);
	if (parserType == null) {
		throw new IllegalArgumentException("The type '" + primitiveClass.getName() + "' is not supported for the primitive input format.");
	}
	parser = InstantiationUtil.instantiate(parserType, FieldParser.class);
}
 

static FieldParser <?> getFieldParser(Class typeClazz) {
	Class <? extends FieldParser <?>> parserType = FieldParser.getParserForType(typeClazz);
	if (parserType == null) {
		throw new RuntimeException("No parser available for type '" + typeClazz.getName() + "'.");
	}
	return InstantiationUtil.instantiate(parserType, FieldParser.class);
}
 

static FieldParser<?> getFieldParser(Class typeClazz) {
    Class<? extends FieldParser<?>> parserType = FieldParser.getParserForType(typeClazz);
    if (parserType == null) {
        throw new RuntimeException("No parser available for type '" + typeClazz.getName() + "'.");
    }
    return InstantiationUtil.instantiate(parserType, FieldParser.class);
}
 

@Override
public void open(FileInputSplit split) throws IOException {
	super.open(split);

	// instantiate the parsers
	FieldParser<?>[] parsers = new FieldParser<?>[fieldTypes.length];
	
	for (int i = 0; i < fieldTypes.length; i++) {
		if (fieldTypes[i] != null) {
			Class<? extends FieldParser<?>> parserType = FieldParser.getParserForType(fieldTypes[i]);
			if (parserType == null) {
				throw new RuntimeException("No parser available for type '" + fieldTypes[i].getName() + "'.");
			}

			FieldParser<?> p = InstantiationUtil.instantiate(parserType, FieldParser.class);

			p.setCharset(getCharset());
			if (this.quotedStringParsing) {
				if (p instanceof StringParser) {
					((StringParser)p).enableQuotedStringParsing(this.quoteCharacter);
				} else if (p instanceof StringValueParser) {
					((StringValueParser)p).enableQuotedStringParsing(this.quoteCharacter);
				}
			}

			parsers[i] = p;
		}
	}
	this.fieldParsers = parsers;
	
	// skip the first line, if we are at the beginning of a file and have the option set
	if (this.skipFirstLineAsHeader && this.splitStart == 0) {
		readLine(); // read and ignore
	}
}
 

protected void setFieldsGeneric(boolean[] includedMask, Class<?>[] fieldTypes) {
	checkNotNull(includedMask);
	checkNotNull(fieldTypes);

	ArrayList<Class<?>> types = new ArrayList<Class<?>>();

	// check if types are valid for included fields
	int typeIndex = 0;
	for (int i = 0; i < includedMask.length; i++) {

		if (includedMask[i]) {
			if (typeIndex > fieldTypes.length - 1) {
				throw new IllegalArgumentException("Missing type for included field " + i + ".");
			}
			Class<?> type = fieldTypes[typeIndex++];

			if (type == null) {
				throw new IllegalArgumentException("Type for included field " + i + " should not be null.");
			} else {
				// check if we support parsers for this type
				if (FieldParser.getParserForType(type) == null) {
					throw new IllegalArgumentException("The type '" + type.getName() + "' is not supported for the CSV input format.");
				}
				types.add(type);
			}
		}
	}

	this.fieldTypes = types.toArray(new Class<?>[types.size()]);
	this.fieldIncluded = includedMask;
}
 

protected void setFieldsGeneric(int[] sourceFieldIndices, Class<?>[] fieldTypes) {
	checkNotNull(sourceFieldIndices);
	checkNotNull(fieldTypes);
	checkArgument(sourceFieldIndices.length == fieldTypes.length,
		"Number of field indices and field types must match.");

	for (int i : sourceFieldIndices) {
		if (i < 0) {
			throw new IllegalArgumentException("Field indices must not be smaller than zero.");
		}
	}

	int largestFieldIndex = max(sourceFieldIndices);
	this.fieldIncluded = new boolean[largestFieldIndex + 1];
	ArrayList<Class<?>> types = new ArrayList<Class<?>>();

	// check if we support parsers for these types
	for (int i = 0; i < fieldTypes.length; i++) {
		Class<?> type = fieldTypes[i];

		if (type != null) {
			if (FieldParser.getParserForType(type) == null) {
				throw new IllegalArgumentException("The type '" + type.getName()
					+ "' is not supported for the CSV input format.");
			}
			types.add(type);
			fieldIncluded[sourceFieldIndices[i]] = true;
		}
	}

	this.fieldTypes = types.toArray(new Class<?>[types.size()]);
}
 

@Override
public void open(FileInputSplit split) throws IOException {
	super.open(split);
	Class<? extends FieldParser<OT>> parserType = FieldParser.getParserForType(primitiveClass);
	if (parserType == null) {
		throw new IllegalArgumentException("The type '" + primitiveClass.getName() + "' is not supported for the primitive input format.");
	}
	parser = InstantiationUtil.instantiate(parserType, FieldParser.class);
}
 

@Override
public void open(FileInputSplit split) throws IOException {
	super.open(split);

	// instantiate the parsers
	FieldParser<?>[] parsers = new FieldParser<?>[fieldTypes.length];
	
	for (int i = 0; i < fieldTypes.length; i++) {
		if (fieldTypes[i] != null) {
			Class<? extends FieldParser<?>> parserType = FieldParser.getParserForType(fieldTypes[i]);
			if (parserType == null) {
				throw new RuntimeException("No parser available for type '" + fieldTypes[i].getName() + "'.");
			}

			FieldParser<?> p = InstantiationUtil.instantiate(parserType, FieldParser.class);

			p.setCharset(getCharset());
			if (this.quotedStringParsing) {
				if (p instanceof StringParser) {
					((StringParser)p).enableQuotedStringParsing(this.quoteCharacter);
				} else if (p instanceof StringValueParser) {
					((StringValueParser)p).enableQuotedStringParsing(this.quoteCharacter);
				}
			}

			parsers[i] = p;
		}
	}
	this.fieldParsers = parsers;
	
	// skip the first line, if we are at the beginning of a file and have the option set
	if (this.skipFirstLineAsHeader && this.splitStart == 0) {
		readLine(); // read and ignore
	}
}
 

protected void setFieldsGeneric(boolean[] includedMask, Class<?>[] fieldTypes) {
	checkNotNull(includedMask);
	checkNotNull(fieldTypes);

	ArrayList<Class<?>> types = new ArrayList<Class<?>>();

	// check if types are valid for included fields
	int typeIndex = 0;
	for (int i = 0; i < includedMask.length; i++) {

		if (includedMask[i]) {
			if (typeIndex > fieldTypes.length - 1) {
				throw new IllegalArgumentException("Missing type for included field " + i + ".");
			}
			Class<?> type = fieldTypes[typeIndex++];

			if (type == null) {
				throw new IllegalArgumentException("Type for included field " + i + " should not be null.");
			} else {
				// check if we support parsers for this type
				if (FieldParser.getParserForType(type) == null) {
					throw new IllegalArgumentException("The type '" + type.getName() + "' is not supported for the CSV input format.");
				}
				types.add(type);
			}
		}
	}

	this.fieldTypes = types.toArray(new Class<?>[types.size()]);
	this.fieldIncluded = includedMask;
}
 

protected void setFieldsGeneric(int[] sourceFieldIndices, Class<?>[] fieldTypes) {
	checkNotNull(sourceFieldIndices);
	checkNotNull(fieldTypes);
	checkArgument(sourceFieldIndices.length == fieldTypes.length,
		"Number of field indices and field types must match.");

	for (int i : sourceFieldIndices) {
		if (i < 0) {
			throw new IllegalArgumentException("Field indices must not be smaller than zero.");
		}
	}

	int largestFieldIndex = max(sourceFieldIndices);
	this.fieldIncluded = new boolean[largestFieldIndex + 1];
	ArrayList<Class<?>> types = new ArrayList<Class<?>>();

	// check if we support parsers for these types
	for (int i = 0; i < fieldTypes.length; i++) {
		Class<?> type = fieldTypes[i];

		if (type != null) {
			if (FieldParser.getParserForType(type) == null) {
				throw new IllegalArgumentException("The type '" + type.getName()
					+ "' is not supported for the CSV input format.");
			}
			types.add(type);
			fieldIncluded[sourceFieldIndices[i]] = true;
		}
	}

	this.fieldTypes = types.toArray(new Class<?>[types.size()]);
}