java.nio.charset.CharsetDecoder Java Examples

/**
 * Utility method to find a CharsetDecoder in the
 * cache or create a new one if necessary.  Throws an
 * INTERNAL if the code set is unknown.
 */
protected CharsetDecoder getConverter(String javaCodeSetName) {

    CharsetDecoder result = null;
    try {
        result = cache.getByteToCharConverter(javaCodeSetName);

        if (result == null) {
            Charset tmpCharset = Charset.forName(javaCodeSetName);
            result = tmpCharset.newDecoder();
            cache.setConverter(javaCodeSetName, result);
        }

    } catch(IllegalCharsetNameException icne) {
        // This can only happen if one of our charset entries has
        // an illegal name.
        throw wrapper.invalidBtcConverterName( icne, javaCodeSetName ) ;
    }

    return result;
}
 

@Override
public void inform(ResourceLoader loader) throws IOException {
  if (userDictionaryPath != null) {
    try (InputStream stream = loader.openResource(userDictionaryPath)) {
      String encoding = userDictionaryEncoding;
      if (encoding == null) {
        encoding = IOUtils.UTF_8;
      }
      CharsetDecoder decoder = Charset.forName(encoding).newDecoder()
        .onMalformedInput(CodingErrorAction.REPORT)
        .onUnmappableCharacter(CodingErrorAction.REPORT);
      Reader reader = new InputStreamReader(stream, decoder);
      userDictionary = UserDictionary.open(reader);
    }
  } else {
    userDictionary = null;
  }
}
 

public static Object isCodepage( Context actualContext, Scriptable actualObject, Object[] ArgList,
  Function FunctionContext ) {
  boolean bRC = false;
  if ( ArgList.length == 2 ) {
    try {
      if ( isNull( ArgList, new int[] { 0, 1 } ) ) {
        return null;
      } else if ( isUndefined( ArgList, new int[] { 0, 1 } ) ) {
        return Context.getUndefinedValue();
      }
      String strValueToCheck = Context.toString( ArgList[0] );
      String strCodePage = Context.toString( ArgList[1] );
      byte[] bytearray = strValueToCheck.getBytes();
      CharsetDecoder d = Charset.forName( strCodePage ).newDecoder();
      CharBuffer r = d.decode( ByteBuffer.wrap( bytearray ) );
      r.toString();
      bRC = true;
    } catch ( Exception e ) {
      bRC = false;
    }
  } else {
    throw Context.reportRuntimeError( "The function call isCodepage requires 2 arguments." );
  }
  return Boolean.valueOf( bRC );
}
 

@Override
public String decode(DecoderImpl decoder, final ReadableBuffer buffer)
{
    CharsetDecoder charsetDecoder = decoder.getCharsetDecoder();
    try
    {
        return buffer.readString(charsetDecoder);
    }
    catch (CharacterCodingException e)
    {
        throw new IllegalArgumentException("Cannot parse String", e);
    }
    finally
    {
        charsetDecoder.reset();
    }
}
 

/**
 * Try to determine whether a byte buffer's character encoding is that of the
 * passed-in charset. Uses inefficient
 * heuristics that will be revisited when we're more familiar with likely
 * usage patterns.
 * 
 * Note this has been heavily changed since inception and will
 * almost certainly disappear in the 10.x timeframe -- HR.
 */
public static boolean inferCharset(byte[] bytes, int bytesRead, Charset clientCharset) {
	ByteBuffer byteBuf = ByteBuffer.wrap(bytes, 0, bytesRead);
	CharBuffer charBuf = CharBuffer.allocate(byteBuf.capacity() * 2);
	
	if (clientCharset != null) {
		CharsetDecoder decoder = clientCharset.newDecoder();
		decoder.onMalformedInput(CodingErrorAction.REPORT);
		decoder.onUnmappableCharacter(CodingErrorAction.REPORT);
		CoderResult coderResult = decoder.decode(byteBuf, charBuf, false);
		if (coderResult != null) {
			if (coderResult.isError()) {
				// Wasn't this one...
				return false;
			} else {
				return true;	// Still only *probably* true, dammit...
			}
		}
	}
	
	return true;
}
 

@SuppressWarnings("unchecked")
public static <T> T parseObject(byte[] input, //
                                int off, //
                                int len, //
                                CharsetDecoder charsetDecoder, //
                                Type clazz, //
                                Feature... features) {
    charsetDecoder.reset();

    int scaleLength = (int) (len * (double) charsetDecoder.maxCharsPerByte());
    char[] chars = allocateChars(scaleLength);

    ByteBuffer byteBuf = ByteBuffer.wrap(input, off, len);
    CharBuffer charByte = CharBuffer.wrap(chars);
    IOUtils.decode(charsetDecoder, byteBuf, charByte);

    int position = charByte.position();

    return (T) parseObject(chars, position, clazz, features);
}
 

String toString(byte[] ba, int length) {
    CharsetDecoder cd = decoder().reset();
    int len = (int)(length * cd.maxCharsPerByte());
    char[] ca = new char[len];
    if (len == 0)
        return new String(ca);
    // UTF-8 only for now. Other ArrayDeocder only handles
    // CodingErrorAction.REPLACE mode. ZipCoder uses
    // REPORT mode.
    if (isUTF8 && cd instanceof ArrayDecoder) {
        int clen = ((ArrayDecoder)cd).decode(ba, 0, length, ca);
        if (clen == -1)    // malformed
            throw new IllegalArgumentException("MALFORMED");
        return new String(ca, 0, clen);
    }
    ByteBuffer bb = ByteBuffer.wrap(ba, 0, length);
    CharBuffer cb = CharBuffer.wrap(ca);
    CoderResult cr = cd.decode(bb, cb, true);
    if (!cr.isUnderflow())
        throw new IllegalArgumentException(cr.toString());
    cr = cd.flush(cb);
    if (!cr.isUnderflow())
        throw new IllegalArgumentException(cr.toString());
    return new String(ca, 0, cb.position());
}
 

private CharsetDecoder decoder() {
    CharsetDecoder dec = decTL.get();
    if (dec == null) {
        dec = cs.newDecoder()
          .onMalformedInput(CodingErrorAction.REPORT)
          .onUnmappableCharacter(CodingErrorAction.REPORT);
        decTL.set(dec);
    }
    return dec;
}
 

@Override
public ConfigFrom propertyFile(CharsetDecoder decoder, String... filenames) {
  for (String filename : filenames) {
    if (filename != null) {
      propertyFile(decoder, new File(filename));
    }
  }
  return this;
}
 

public CharsetDecoder newDecoder() {
    return new DoubleByte.Decoder(this,
                                  IBM942.b2c,
                                  b2cSB,
                                  0x40,
                                  0xfc);
}
 

public String getPayloadTracingString() {
	if (null == payload || 0 == payload.length)
		return "no payload";
	boolean text = true;
	for (byte b:payload) {
		if (' ' > b) {
			switch(b) {
			case '\t':
			case '\n':
			case '\r':
				continue;
			}
			text = false;
			break;
		}
	}
	if (text) {
		CharsetDecoder decoder = CoAP.UTF8_CHARSET.newDecoder();
		decoder.onMalformedInput(CodingErrorAction.REPORT);
		decoder.onUnmappableCharacter(CodingErrorAction.REPORT);
		ByteBuffer in = ByteBuffer.wrap(payload);
		CharBuffer out = CharBuffer.allocate(24);
		CoderResult result = decoder.decode(in, out, true);
		decoder.flush(out);
		out.flip();
		if (CoderResult.OVERFLOW == result) {
			return "\"" + out +  "\".. " + payload.length + " bytes";
		} else if (!result.isError()){
			return "\"" + out + "\"" ;
		}
	}
	return Utils.toHexText(payload, 256);
}
 

private CharsetDecoder decoder() {
    if (dec == null) {
        dec = cs.newDecoder()
          .onMalformedInput(CodingErrorAction.REPORT)
          .onUnmappableCharacter(CodingErrorAction.REPORT);
    }
    return dec;
}
 

private CharsetDecoder decoder() {
    if (dec == null) {
        dec = cs.newDecoder()
          .onMalformedInput(CodingErrorAction.REPORT)
          .onUnmappableCharacter(CodingErrorAction.REPORT);
    }
    return dec;
}
 

public CharsetDecoder newDecoder() {
    return new DoubleByte.Decoder(this,
                                  IBM949.b2c,
                                  b2cSB,
                                  0xa1,
                                  0xfe);
}
 

/**
 * Returns a cached thread-local {@link CharsetDecoder} for the specified {@link Charset}.
 *
 * @param charset The specified charset
 * @return The decoder for the specified {@code charset}
 */
public static CharsetDecoder decoder(Charset charset) {
    checkNotNull(charset, "charset");

    Map<Charset, CharsetDecoder> map = new HashMap<>();
    CharsetDecoder d = map.get(charset);
    if (d != null) {
        d.reset().onMalformedInput(CodingErrorAction.REPLACE).onUnmappableCharacter(CodingErrorAction.REPLACE);
        return d;
    }

    d = decoder(charset, CodingErrorAction.REPLACE, CodingErrorAction.REPLACE);
    map.put(charset, d);
    return d;
}
 

public CharsetDecoder newDecoder() {
    return new DoubleByte.Decoder(this,
                                  IBM943.b2c,
                                  b2cSB,
                                  0x40,
                                  0xfc);
}
 

/**
 * Stores the given CharsetDecoder in the thread local cache,
 * and returns the same converter.
 */
CharsetDecoder setConverter(Object key, CharsetDecoder converter) {
    Map btcMap = ((Map[])converterCaches.get())[BTC_CACHE_MAP];

    btcMap.put(key, converter);

    return converter;
}
 

@Test
public void testPropagateErrorInTheMiddleOfMultibyte() {
    Observable<byte[]> src = Observable.just(new byte[] { (byte) 0xc2 });
    Observable<byte[]> err = Observable.error(new IOException());
    CharsetDecoder charsetDecoder = Charset.forName("UTF-8").newDecoder();
    try {
        decode(Observable.concat(src, err), charsetDecoder).toList().toBlocking().single();
        fail();
    } catch (RuntimeException e) {
        assertEquals(IOException.class, e.getCause().getClass());
    }
}
 

static void test(String expectedCharset, byte[] input) throws Exception {
    Charset cs = Charset.forName("x-JISAutoDetect");
    CharsetDecoder autoDetect = cs.newDecoder();

    Charset cs2 = Charset.forName(expectedCharset);
    CharsetDecoder decoder = cs2.newDecoder();

    ByteBuffer bb = ByteBuffer.allocate(128);
    CharBuffer charOutput = CharBuffer.allocate(128);
    CharBuffer charExpected = CharBuffer.allocate(128);

    bb.put(input);
    bb.flip();
    bb.mark();

    CoderResult result = autoDetect.decode(bb, charOutput, true);
    checkCoderResult(result);
    charOutput.flip();
    String actual = charOutput.toString();

    bb.reset();

    result = decoder.decode(bb, charExpected, true);
    checkCoderResult(result);
    charExpected.flip();
    String expected = charExpected.toString();

    check(actual.equals(expected),
          String.format("actual=%s expected=%s", actual, expected));
}
 

public static boolean isValidUTF8(byte[] input)
{

    CharsetDecoder cs = Charset.forName(UTF8).newDecoder();

    try
    {
        cs.decode(ByteBuffer.wrap(input));
        return true;
    }
    catch (CharacterCodingException e)
    {
        return false;
    }
}
 

/**
 * Stores the given CharsetDecoder in the thread local cache,
 * and returns the same converter.
 */
CharsetDecoder setConverter(Object key, CharsetDecoder converter) {
    Map btcMap = ((Map[])converterCaches.get())[BTC_CACHE_MAP];

    btcMap.put(key, converter);

    return converter;
}
 

public static TextDecoder newStrict(Charset charset)
{
    CharsetDecoder encoder = charset.newDecoder().
        onMalformedInput(CodingErrorAction.REPORT).
        onUnmappableCharacter(CodingErrorAction.REPORT);
    TextDecoder instance = new TextDecoder(encoder);
    return instance;
}
 

@Override
protected CharsetDecoder initialValue() {
  return asciiCharset.newDecoder();
}
 

public CharsetDecoder newDecoder() {
    return new Decoder(this);
}
 

public CharsetDecoder newDecoder() {
    return new Decoder(this);
}
 

public CharsetDecoder newDecoder() {
    return new Decoder(this);
}
 

public CharsetDecoder newDecoder() {
    return new Decoder(this);
}
 

public CharsetDecoder newDecoder() {
    return new Decoder(this);
}
 

@Override
protected Map<Charset, CharsetDecoder> initialValue() {
    return new IdentityHashMap<Charset, CharsetDecoder>();
}
 

public CharBuffer decode(ByteBuffer inbuf, boolean ignoreEncodingErrors) {
    String encName = getEncodingName();
    CharsetDecoder decoder;
    try {
        decoder = getDecoder(encName, ignoreEncodingErrors);
    } catch (IllegalCharsetNameException | UnsupportedCharsetException e) {
        log.error(Errors.UnsupportedEncoding(encName));
        return (CharBuffer) CharBuffer.allocate(1).flip();
    }

    // slightly overestimate the buffer size to avoid reallocation.
    float factor =
        decoder.averageCharsPerByte() * 0.8f +
        decoder.maxCharsPerByte() * 0.2f;
    CharBuffer dest = CharBuffer.
        allocate(10 + (int)(inbuf.remaining()*factor));

    while (true) {
        CoderResult result = decoder.decode(inbuf, dest, true);
        dest.flip();

        if (result.isUnderflow()) { // done reading
            // make sure there is at least one extra character
            if (dest.limit() == dest.capacity()) {
                dest = CharBuffer.allocate(dest.capacity()+1).put(dest);
                dest.flip();
            }
            return dest;
        } else if (result.isOverflow()) { // buffer too small; expand
            int newCapacity =
                10 + dest.capacity() +
                (int)(inbuf.remaining()*decoder.maxCharsPerByte());
            dest = CharBuffer.allocate(newCapacity).put(dest);
        } else if (result.isMalformed() || result.isUnmappable()) {
            // bad character in input
            StringBuilder unmappable = new StringBuilder();
            int len = result.length();

            for (int i = 0; i < len; i++) {
                unmappable.append(String.format("%02X", inbuf.get()));
            }

            String charsetName = charset == null ? encName : charset.name();

            log.error(dest.limit(),
                      Errors.IllegalCharForEncoding(unmappable.toString(), charsetName));

            // undo the flip() to prepare the output buffer
            // for more translation
            dest.position(dest.limit());
            dest.limit(dest.capacity());
            dest.put((char)0xfffd); // backward compatible
        } else {
            throw new AssertionError(result);
        }
    }
    // unreached
}