Java Code Examples for java.nio.charset.CharsetDecoder#averageCharsPerByte()
The following examples show how to use
java.nio.charset.CharsetDecoder#averageCharsPerByte() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: CompositeReadableBuffer.java From qpid-proton-j with Apache License 2.0 | 5 votes |
|
private CharBuffer readStringFromComponents(CharsetDecoder decoder) throws CharacterCodingException {
int size = (int)(remaining() * decoder.averageCharsPerByte());
CharBuffer decoded = CharBuffer.allocate(size);
int arrayIndex = currentArrayIndex;
final int viewSpan = limit() - position();
int processed = Math.min(currentArray.length - currentOffset, viewSpan);
ByteBuffer wrapper = ByteBuffer.wrap(currentArray, currentOffset, processed);
CoderResult step = CoderResult.OVERFLOW;
do {
boolean endOfInput = processed == viewSpan;
step = decoder.decode(wrapper, decoded, endOfInput);
if (step.isUnderflow() && endOfInput) {
step = decoder.flush(decoded);
break;
}
if (step.isOverflow()) {
size = 2 * size + 1;
CharBuffer upsized = CharBuffer.allocate(size);
decoded.flip();
upsized.put(decoded);
decoded = upsized;
continue;
}
byte[] next = contents.get(++arrayIndex);
int wrapSize = Math.min(next.length, viewSpan - processed);
wrapper = ByteBuffer.wrap(next, 0, wrapSize);
processed += wrapSize;
} while (!step.isError());
if (step.isError()) {
step.throwException();
}
return (CharBuffer) decoded.flip();
}
Example 2
| Source File: AbstractAdaptiveByteBuffer.java From craft-atom with MIT License | 4 votes |
|
/**
* Reads a string which has a length field before the actual
* encoded string, using the specified <code>decoder</code> and returns it.
*
* @param prefixLength the length of the length field (1, 2, or 4)
* @param decoder the decoder to use for decoding the string
* @return the prefixed string
* @throws CharacterCodingException when decoding fails
* @throws BufferUnderflowException when there is not enough data available
*/
@Override
public String getPrefixedString(int prefixLength, CharsetDecoder decoder) throws CharacterCodingException {
if (!prefixedDataAvailable(prefixLength)) {
throw new BufferUnderflowException();
}
int fieldSize = 0;
switch (prefixLength) {
case 1:
fieldSize = getUnsigned();
break;
case 2:
fieldSize = getUnsignedShort();
break;
case 4:
fieldSize = getInt();
break;
}
if (fieldSize == 0) {
return "";
}
boolean utf16 = decoder.charset().name().startsWith("UTF-16");
if (utf16 && (fieldSize & 1) != 0) {
throw new IllegalArgumentException("fieldSize is not even for a UTF-16 string.");
}
int oldLimit = limit();
int end = position() + fieldSize;
if (oldLimit < end) {
throw new BufferUnderflowException();
}
limit(end);
decoder.reset();
int expectedLength = (int) (remaining() * decoder.averageCharsPerByte()) + 1;
CharBuffer out = CharBuffer.allocate(expectedLength);
for (;;) {
CoderResult cr;
if (hasRemaining()) {
cr = decoder.decode(buf(), out, true);
} else {
cr = decoder.flush(out);
}
if (cr.isUnderflow()) {
break;
}
if (cr.isOverflow()) {
CharBuffer o = CharBuffer.allocate(out.capacity() + expectedLength);
out.flip();
o.put(out);
out = o;
continue;
}
cr.throwException();
}
limit(oldLimit);
position(end);
return out.flip().toString();
}
Example 3
| Source File: BaseFileManager.java From lua-for-android with BSD 3-Clause "New" or "Revised" License | 4 votes |
|
public CharBuffer decode(ByteBuffer inbuf, boolean ignoreEncodingErrors) {
String encName = getEncodingName();
CharsetDecoder decoder;
try {
decoder = getDecoder(encName, ignoreEncodingErrors);
} catch (IllegalCharsetNameException | UnsupportedCharsetException e) {
log.error(Errors.UnsupportedEncoding(encName));
return (CharBuffer) CharBuffer.allocate(1).flip();
}
// slightly overestimate the buffer size to avoid reallocation.
float factor =
decoder.averageCharsPerByte() * 0.8f +
decoder.maxCharsPerByte() * 0.2f;
CharBuffer dest = CharBuffer.
allocate(10 + (int)(inbuf.remaining()*factor));
while (true) {
CoderResult result = decoder.decode(inbuf, dest, true);
dest.flip();
if (result.isUnderflow()) { // done reading
// make sure there is at least one extra character
if (dest.limit() == dest.capacity()) {
dest = CharBuffer.allocate(dest.capacity()+1).put(dest);
dest.flip();
}
return dest;
} else if (result.isOverflow()) { // buffer too small; expand
int newCapacity =
10 + dest.capacity() +
(int)(inbuf.remaining()*decoder.maxCharsPerByte());
dest = CharBuffer.allocate(newCapacity).put(dest);
} else if (result.isMalformed() || result.isUnmappable()) {
// bad character in input
StringBuilder unmappable = new StringBuilder();
int len = result.length();
for (int i = 0; i < len; i++) {
unmappable.append(String.format("%02X", inbuf.get()));
}
String charsetName = charset == null ? encName : charset.name();
log.error(dest.limit(),
Errors.IllegalCharForEncoding(unmappable.toString(), charsetName));
// undo the flip() to prepare the output buffer
// for more translation
dest.position(dest.limit());
dest.limit(dest.capacity());
dest.put((char)0xfffd); // backward compatible
} else {
throw new AssertionError(result);
}
}
// unreached
}
Example 4
| Source File: AbstractAdaptiveByteBuffer.java From craft-atom with MIT License | 4 votes |
|
/**
* {@inheritDoc}
*/
@Override
public String getString(int fieldSize, CharsetDecoder decoder) throws CharacterCodingException {
checkFieldSize(fieldSize);
if (fieldSize == 0) {
return "";
}
if (!hasRemaining()) {
return "";
}
boolean utf16 = decoder.charset().name().startsWith("UTF-16");
if (utf16 && (fieldSize & 1) != 0) {
throw new IllegalArgumentException("fieldSize is not even.");
}
int oldPos = position();
int oldLimit = limit();
int end = oldPos + fieldSize;
if (oldLimit < end) {
throw new BufferUnderflowException();
}
int i;
if (!utf16) {
for (i = oldPos; i < end; i++) {
if (get(i) == 0) {
break;
}
}
if (i == end) {
limit(end);
} else {
limit(i);
}
} else {
for (i = oldPos; i < end; i += 2) {
if (get(i) == 0 && get(i + 1) == 0) {
break;
}
}
if (i == end) {
limit(end);
} else {
limit(i);
}
}
if (!hasRemaining()) {
limit(oldLimit);
position(end);
return "";
}
decoder.reset();
int expectedLength = (int) (remaining() * decoder.averageCharsPerByte()) + 1;
CharBuffer out = CharBuffer.allocate(expectedLength);
for (;;) {
CoderResult cr;
if (hasRemaining()) {
cr = decoder.decode(buf(), out, true);
} else {
cr = decoder.flush(out);
}
if (cr.isUnderflow()) {
break;
}
if (cr.isOverflow()) {
CharBuffer o = CharBuffer.allocate(out.capacity() + expectedLength);
out.flip();
o.put(out);
out = o;
continue;
}
if (cr.isError()) {
// Revert the buffer back to the previous state.
limit(oldLimit);
position(oldPos);
cr.throwException();
}
}
limit(oldLimit);
position(end);
return out.flip().toString();
}
Example 5
| Source File: AbstractAdaptiveByteBuffer.java From craft-atom with MIT License | 4 votes |
|
/**
* {@inheritDoc}
*/
@Override
public String getString(CharsetDecoder decoder) throws CharacterCodingException {
if (!hasRemaining()) {
return "";
}
boolean utf16 = decoder.charset().name().startsWith("UTF-16");
int oldPos = position();
int oldLimit = limit();
int end = -1;
int newPos;
if (!utf16) {
end = indexOf((byte) 0x00);
if (end < 0) {
newPos = end = oldLimit;
} else {
newPos = end + 1;
}
} else {
int i = oldPos;
for (;;) {
boolean wasZero = get(i) == 0;
i++;
if (i >= oldLimit) {
break;
}
if (get(i) != 0) {
i++;
if (i >= oldLimit) {
break;
}
continue;
}
if (wasZero) {
end = i - 1;
break;
}
}
if (end < 0) {
newPos = end = oldPos + (oldLimit - oldPos & 0xFFFFFFFE);
} else {
if (end + 2 <= oldLimit) {
newPos = end + 2;
} else {
newPos = end;
}
}
}
if (oldPos == end) {
position(newPos);
return "";
}
limit(end);
decoder.reset();
int expectedLength = (int) (remaining() * decoder.averageCharsPerByte()) + 1;
CharBuffer out = CharBuffer.allocate(expectedLength);
for (;;) {
CoderResult cr;
if (hasRemaining()) {
cr = decoder.decode(buf(), out, true);
} else {
cr = decoder.flush(out);
}
if (cr.isUnderflow()) {
break;
}
if (cr.isOverflow()) {
CharBuffer o = CharBuffer.allocate(out.capacity() + expectedLength);
out.flip();
o.put(out);
out = o;
continue;
}
if (cr.isError()) {
// Revert the buffer back to the previous state.
limit(oldLimit);
position(oldPos);
cr.throwException();
}
}
limit(oldLimit);
position(newPos);
return out.flip().toString();
}
Example 6
| Source File: PerforceShiftJISCharset.java From p4ic4idea with Apache License 2.0 | 4 votes |
|
/**
* Call the superclass constructor with the Charset object and pass
* along the chars/byte values from the decoder.
*/
Decoder(Charset cs, CharsetDecoder decoder) {
super(cs, decoder.averageCharsPerByte(), decoder.maxCharsPerByte());
this.decoder = decoder;
}
Example 7
| Source File: PerforceShiftJISCharset.java From p4ic4idea with Apache License 2.0 | 4 votes |
|
/**
* Call the superclass constructor with the Charset object and pass
* along the chars/byte values from the decoder.
*/
Decoder(Charset cs, CharsetDecoder decoder) {
super(cs, decoder.averageCharsPerByte(), decoder.maxCharsPerByte());
this.decoder = decoder;
}
Example 8
| Source File: PerforceShiftJISCharset.java From p4ic4idea with Apache License 2.0 | 4 votes |
|
/**
* Call the superclass constructor with the Charset object and pass
* along the chars/byte values from the decoder.
*/
Decoder(Charset cs, CharsetDecoder decoder) {
super(cs, decoder.averageCharsPerByte(), decoder.maxCharsPerByte());
this.decoder = decoder;
}
Example 9
| Source File: PerforceShiftJISCharset.java From p4ic4idea with Apache License 2.0 | 4 votes |
|
/**
* Call the superclass constructor with the Charset object and pass
* along the chars/byte values from the decoder.
*/
Decoder(Charset cs, CharsetDecoder decoder) {
super(cs, decoder.averageCharsPerByte(), decoder.maxCharsPerByte());
this.decoder = decoder;
}
Example 10
| Source File: PerforceShiftJISCharset.java From p4ic4idea with Apache License 2.0 | 4 votes |
|
/**
* Call the superclass constructor with the Charset object and pass
* along the chars/byte values from the decoder.
*/
Decoder(Charset cs, CharsetDecoder decoder) {
super(cs, decoder.averageCharsPerByte(), decoder.maxCharsPerByte());
this.decoder = decoder;
}
Example 11
| Source File: PerforceShiftJISCharset.java From p4ic4idea with Apache License 2.0 | 4 votes |
|
/**
* Call the superclass constructor with the Charset object and pass
* along the chars/byte values from the decoder.
*/
Decoder(Charset cs, CharsetDecoder decoder) {
super(cs, decoder.averageCharsPerByte(), decoder.maxCharsPerByte());
this.decoder = decoder;
}
Example 12
| Source File: BaseFileManager.java From openjdk-jdk9 with GNU General Public License v2.0 | 4 votes |
|
@SuppressWarnings("cast")
public CharBuffer decode(ByteBuffer inbuf, boolean ignoreEncodingErrors) {
String encName = getEncodingName();
CharsetDecoder decoder;
try {
decoder = getDecoder(encName, ignoreEncodingErrors);
} catch (IllegalCharsetNameException | UnsupportedCharsetException e) {
log.error("unsupported.encoding", encName);
return (CharBuffer)CharBuffer.allocate(1).flip();
}
// slightly overestimate the buffer size to avoid reallocation.
float factor =
decoder.averageCharsPerByte() * 0.8f +
decoder.maxCharsPerByte() * 0.2f;
CharBuffer dest = CharBuffer.
allocate(10 + (int)(inbuf.remaining()*factor));
while (true) {
CoderResult result = decoder.decode(inbuf, dest, true);
dest.flip();
if (result.isUnderflow()) { // done reading
// make sure there is at least one extra character
if (dest.limit() == dest.capacity()) {
dest = CharBuffer.allocate(dest.capacity()+1).put(dest);
dest.flip();
}
return dest;
} else if (result.isOverflow()) { // buffer too small; expand
int newCapacity =
10 + dest.capacity() +
(int)(inbuf.remaining()*decoder.maxCharsPerByte());
dest = CharBuffer.allocate(newCapacity).put(dest);
} else if (result.isMalformed() || result.isUnmappable()) {
// bad character in input
StringBuilder unmappable = new StringBuilder();
int len = result.length();
for (int i = 0; i < len; i++) {
unmappable.append(String.format("%02X", inbuf.get()));
}
String charsetName = charset == null ? encName : charset.name();
log.error(dest.limit(),
Errors.IllegalCharForEncoding(unmappable.toString(), charsetName));
// undo the flip() to prepare the output buffer
// for more translation
dest.position(dest.limit());
dest.limit(dest.capacity());
dest.put((char)0xfffd); // backward compatible
} else {
throw new AssertionError(result);
}
}
// unreached
}
Example 13
| Source File: AbstractIoBuffer.java From neoscada with Eclipse Public License 1.0 | 4 votes |
|
/**
* Reads a string which has a length field before the actual
* encoded string, using the specified <code>decoder</code> and returns it.
*
* @param prefixLength the length of the length field (1, 2, or 4)
* @param decoder the decoder to use for decoding the string
* @return the prefixed string
* @throws CharacterCodingException when decoding fails
* @throws BufferUnderflowException when there is not enough data available
*/
@Override
public String getPrefixedString(int prefixLength, CharsetDecoder decoder) throws CharacterCodingException {
if (!prefixedDataAvailable(prefixLength)) {
throw new BufferUnderflowException();
}
int fieldSize = 0;
switch (prefixLength) {
case 1:
fieldSize = getUnsigned();
break;
case 2:
fieldSize = getUnsignedShort();
break;
case 4:
fieldSize = getInt();
break;
}
if (fieldSize == 0) {
return "";
}
boolean utf16 = decoder.charset().name().startsWith("UTF-16");
if (utf16 && (fieldSize & 1) != 0) {
throw new BufferDataException("fieldSize is not even for a UTF-16 string.");
}
int oldLimit = limit();
int end = position() + fieldSize;
if (oldLimit < end) {
throw new BufferUnderflowException();
}
limit(end);
decoder.reset();
int expectedLength = (int) (remaining() * decoder.averageCharsPerByte()) + 1;
CharBuffer out = CharBuffer.allocate(expectedLength);
for (;;) {
CoderResult cr;
if (hasRemaining()) {
cr = decoder.decode(buf(), out, true);
} else {
cr = decoder.flush(out);
}
if (cr.isUnderflow()) {
break;
}
if (cr.isOverflow()) {
CharBuffer o = CharBuffer.allocate(out.capacity() + expectedLength);
out.flip();
o.put(out);
out = o;
continue;
}
cr.throwException();
}
limit(oldLimit);
position(end);
return out.flip().toString();
}
Example 14
| Source File: AbstractIoBuffer.java From neoscada with Eclipse Public License 1.0 | 4 votes |
|
/**
* {@inheritDoc}
*/
@Override
public String getString(int fieldSize, CharsetDecoder decoder) throws CharacterCodingException {
checkFieldSize(fieldSize);
if (fieldSize == 0) {
return "";
}
if (!hasRemaining()) {
return "";
}
boolean utf16 = decoder.charset().name().startsWith("UTF-16");
if (utf16 && (fieldSize & 1) != 0) {
throw new IllegalArgumentException("fieldSize is not even.");
}
int oldPos = position();
int oldLimit = limit();
int end = oldPos + fieldSize;
if (oldLimit < end) {
throw new BufferUnderflowException();
}
int i;
if (!utf16) {
for (i = oldPos; i < end; i++) {
if (get(i) == 0) {
break;
}
}
if (i == end) {
limit(end);
} else {
limit(i);
}
} else {
for (i = oldPos; i < end; i += 2) {
if (get(i) == 0 && get(i + 1) == 0) {
break;
}
}
if (i == end) {
limit(end);
} else {
limit(i);
}
}
if (!hasRemaining()) {
limit(oldLimit);
position(end);
return "";
}
decoder.reset();
int expectedLength = (int) (remaining() * decoder.averageCharsPerByte()) + 1;
CharBuffer out = CharBuffer.allocate(expectedLength);
for (;;) {
CoderResult cr;
if (hasRemaining()) {
cr = decoder.decode(buf(), out, true);
} else {
cr = decoder.flush(out);
}
if (cr.isUnderflow()) {
break;
}
if (cr.isOverflow()) {
CharBuffer o = CharBuffer.allocate(out.capacity() + expectedLength);
out.flip();
o.put(out);
out = o;
continue;
}
if (cr.isError()) {
// Revert the buffer back to the previous state.
limit(oldLimit);
position(oldPos);
cr.throwException();
}
}
limit(oldLimit);
position(end);
return out.flip().toString();
}
Example 15
| Source File: AbstractIoBuffer.java From neoscada with Eclipse Public License 1.0 | 4 votes |
|
/**
* {@inheritDoc}
*/
@Override
public String getString(CharsetDecoder decoder) throws CharacterCodingException {
if (!hasRemaining()) {
return "";
}
boolean utf16 = decoder.charset().name().startsWith("UTF-16");
int oldPos = position();
int oldLimit = limit();
int end = -1;
int newPos;
if (!utf16) {
end = indexOf((byte) 0x00);
if (end < 0) {
newPos = end = oldLimit;
} else {
newPos = end + 1;
}
} else {
int i = oldPos;
for (;;) {
boolean wasZero = get(i) == 0;
i++;
if (i >= oldLimit) {
break;
}
if (get(i) != 0) {
i++;
if (i >= oldLimit) {
break;
}
continue;
}
if (wasZero) {
end = i - 1;
break;
}
}
if (end < 0) {
newPos = end = oldPos + (oldLimit - oldPos & 0xFFFFFFFE);
} else {
if (end + 2 <= oldLimit) {
newPos = end + 2;
} else {
newPos = end;
}
}
}
if (oldPos == end) {
position(newPos);
return "";
}
limit(end);
decoder.reset();
int expectedLength = (int) (remaining() * decoder.averageCharsPerByte()) + 1;
CharBuffer out = CharBuffer.allocate(expectedLength);
for (;;) {
CoderResult cr;
if (hasRemaining()) {
cr = decoder.decode(buf(), out, true);
} else {
cr = decoder.flush(out);
}
if (cr.isUnderflow()) {
break;
}
if (cr.isOverflow()) {
CharBuffer o = CharBuffer.allocate(out.capacity() + expectedLength);
out.flip();
o.put(out);
out = o;
continue;
}
if (cr.isError()) {
// Revert the buffer back to the previous state.
limit(oldLimit);
position(oldPos);
cr.throwException();
}
}
limit(oldLimit);
position(newPos);
return out.flip().toString();
}
Example 16
| Source File: FileEncodingQuery.java From netbeans with Apache License 2.0 | 4 votes |
|
private ProxyDecoder (final CharsetDecoder defaultDecoder) {
super (ProxyCharset.this, defaultDecoder.averageCharsPerByte(), defaultDecoder.maxCharsPerByte());
this.currentDecoder = defaultDecoder;
initialized = true;
}
Example 17
| Source File: InputStreamReader.java From jtransc with Apache License 2.0 | 3 votes |
|
/**
* Constructs a new InputStreamReader on the InputStream {@code in} and
* CharsetDecoder {@code dec}.
*
* @param in
* the source InputStream from which to read characters.
* @param dec
* the CharsetDecoder used by the character conversion.
*/
public InputStreamReader(InputStream in, CharsetDecoder dec) {
super(in);
dec.averageCharsPerByte();
this.in = in;
decoder = dec;
bytes.limit(0);
}
Example 18
| Source File: InputStreamReader.java From TorrentEngine with GNU General Public License v3.0 | 3 votes |
|
/**
* Constructs a new InputStreamReader on the InputStream {@code in} and
* CharsetDecoder {@code dec}.
*
* @param in
* the source InputStream from which to read characters.
* @param dec
* the CharsetDecoder used by the character conversion.
*/
public InputStreamReader(InputStream in, CharsetDecoder dec) {
super(in);
dec.averageCharsPerByte();
this.in = in;
decoder = dec;
bytes.limit(0);
}
Example 19
| Source File: InputStreamReader.java From openjdk-jdk9 with GNU General Public License v2.0 | 3 votes |
|
/**
* Constructs a new InputStreamReader on the InputStream {@code in} and
* CharsetDecoder {@code dec}.
*
* @param in
* the source InputStream from which to read characters.
* @param dec
* the CharsetDecoder used by the character conversion.
*/
public InputStreamReader(InputStream in, CharsetDecoder dec) {
super(in);
dec.averageCharsPerByte();
this.in = in;
decoder = dec;
bytes.limit(0);
}
