Java Code Examples for java.net.URLConnection#getContentEncoding()

private static Charset getCharset(URLConnection urlConnection) {
  Charset cs = StandardCharsets.UTF_8;
  String encoding = urlConnection.getContentEncoding();
  if (Objects.nonNull(encoding)) {
    cs = Charset.forName(encoding);
  } else {
    String contentType = urlConnection.getContentType();
    Stream.of(contentType.split(";"))
        .map(String::trim)
        .filter(s -> !s.isEmpty() && s.toLowerCase(Locale.ENGLISH).startsWith("charset="))
        .map(s -> s.substring("charset=".length()))
        .findFirst()
        .ifPresent(Charset::forName);
  }
  System.out.println(cs);
  return cs;
}
 

private String retrieveObjectValue(final String urlString) {
    try {
        final URL url = new URL(urlString);
        final URLConnection connection = url.openConnection();
        connection.connect();
        final String encodingName = connection.getContentEncoding();
        final Charset encoding = encodingName != null ?
                Charset.forName(encodingName) :
                defaultEncoding;
        try (InputStream inputStream = connection.getInputStream()) {
            return ResourceUtil.readAll(inputStream, encoding);
        }
    } catch (final IOException e) {
        throw new MetafactureException(e);
    }
}
 

/**
 * Initialise response
 * 
 * @param urlConnection  url connection
 */
protected void initialiseResponse(URLConnection urlConnection)
{
    String type = urlConnection.getContentType();
    if (type != null)
    {
        int encodingIdx = type.lastIndexOf("charset=");
        if (encodingIdx == -1)
        {
            String encoding = urlConnection.getContentEncoding();
            if (encoding != null && encoding.length() > 0)
            {
                type += ";charset=" + encoding;
            }
        }
        
        response.setContentType(type);
    }
}
 

@Override
public void process(final String urlStr) {
    try {
        final URL url = new URL(urlStr);
        final URLConnection con = url.openConnection();
        con.addRequestProperty("Accept", accept);
        con.addRequestProperty("Accept-Charset", encoding);
        String enc = con.getContentEncoding();
        if (enc == null) {
            enc = encoding;
        }
        getReceiver().process(new InputStreamReader(con.getInputStream(), enc));
    } catch (IOException e) {
        throw new MetafactureException(e);
    }
}
 

@Override
public void load() throws IOException, InterruptedException {
  String inputEncoding;
  InputStream inputStream = null;
  try {
    URLConnection connection = configureConnection(new URL(manifestUrl));
    inputStream = connection.getInputStream();
    inputEncoding = connection.getContentEncoding();
    result = parser.parse(inputStream, inputEncoding, contentId,
        Util.parseBaseUri(connection.getURL().toString()));
  } finally {
    if (inputStream != null) {
      inputStream.close();
    }
  }
}
 

private static TrackingInputStream getInputStream(URLConnection conn) throws IOException {
    String encoding = conn.getContentEncoding();
    if (encoding != null && encoding.equalsIgnoreCase("gzip")) {
      return new TrackingGZIPInputStream(new ExposedGZIPInputStream(conn.getInputStream()));
    }
    return new SimpleTrackingInputStream(conn.getInputStream());
}
 

private static TrackingInputStream getInputStream(URLConnection conn) throws IOException {
    String encoding = conn.getContentEncoding();
    if (encoding != null && encoding.equalsIgnoreCase("gzip")) {
      return new TrackingGZIPInputStream(new ExposedGZIPInputStream(conn.getInputStream()));
    }
    return new SimpleTrackingInputStream(conn.getInputStream());
}
 

private static TrackingInputStream getInputStream(URLConnection conn) throws IOException {
    String encoding = conn.getContentEncoding();
    if (encoding != null && encoding.equalsIgnoreCase("gzip")) {
      return new TrackingGZIPInputStream(new ExposedGZIPInputStream(conn.getInputStream()));
    }
    return new SimpleTrackingInputStream(conn.getInputStream());
}
 

/**
 * Get a BufferedReader on the crawler journal given.
 * 
 * @param source URL journal
 * @return journal buffered reader.
 * @throws IOException
 */
public static BufferedReader getBufferedReader(URL source) throws IOException {
    URLConnection conn = source.openConnection();
    boolean isGzipped = conn.getContentType() != null && conn.getContentType().equalsIgnoreCase("application/x-gzip")
            || conn.getContentEncoding() != null && conn.getContentEncoding().equalsIgnoreCase("gzip");
    InputStream uis = conn.getInputStream();
    return new BufferedReader(isGzipped?
        new InputStreamReader(new GZIPInputStream(uis)):
        new InputStreamReader(uis));    
}
 

private static TrackingInputStream getInputStream(URLConnection conn) throws IOException {
    String encoding = conn.getContentEncoding();
    if (encoding != null && encoding.equalsIgnoreCase("gzip")) {
    	return new TrackingGZIPInputStream(new ExposedGZIPInputStream(conn.getInputStream()));
    }
    return new TrackingHTTPInputStream(conn.getInputStream());
}
 

private static TrackingInputStream getInputStream(URLConnection conn) throws IOException {
    String encoding = conn.getContentEncoding();
    if (encoding != null && encoding.equalsIgnoreCase("gzip")) {
      return new TrackingGZIPInputStream(new ExposedGZIPInputStream(conn.getInputStream()));
    }
    return new SimpleTrackingInputStream(conn.getInputStream());
}
 

private static TrackingInputStream getInputStream(URLConnection conn) throws IOException {
    String encoding = conn.getContentEncoding();
    if (encoding != null && encoding.equalsIgnoreCase("gzip")) {
      return new TrackingGZIPInputStream(new ExposedGZIPInputStream(conn.getInputStream()));
    }
    return new SimpleTrackingInputStream(conn.getInputStream());
}
 

/**
 * Gets a string with the proper encoding (including using the charset specified in the MIME type
 * of the request) from a URLConnection.
 */
private static String readFromConnection(URLConnection connection) throws IOException {
  InputStream stream;
  if (connection instanceof HttpURLConnection) {
    HttpURLConnection httpConnection = (HttpURLConnection) connection;
    try {
      stream = connection.getInputStream();
    } catch (Exception e) {
      stream = httpConnection.getErrorStream();
    }
  } else {
    stream = connection.getInputStream();
  }
  try {
    ByteArrayOutputStream output = new ByteArrayOutputStream();
    byte[] buffer = new byte[1024];
    int read = 0;
    while ((read = stream.read(buffer)) != -1) {
      output.write(buffer, 0, read);
    }
    String charset = connection.getContentEncoding();
    if (charset == null) {
      String mimeType = connection.getContentType();
      String[] parts = mimeType.split(";");
      for (String part : parts) {
        part = part.trim();
        if (part.startsWith("charset=")) {
          charset = part.substring("charset=".length());
          break;
        }
      }
      if (charset == null) {
        charset = "UTF-8";
      }
    }
    return new String(output.toByteArray(), charset);
  } finally {
    stream.close();
  }
}
 

private static TrackingInputStream getInputStream(URLConnection conn) throws IOException {
    String encoding = conn.getContentEncoding();
    if (encoding != null && encoding.equalsIgnoreCase("gzip")) {
      return new TrackingGZIPInputStream(new ExposedGZIPInputStream(conn.getInputStream()));
    }
    return new SimpleTrackingInputStream(conn.getInputStream());
}
 

private static TrackingInputStream getInputStream(URLConnection conn) throws IOException {
    String encoding = conn.getContentEncoding();
    if (encoding != null && encoding.equalsIgnoreCase("gzip")) {
      return new TrackingGZIPInputStream(new ExposedGZIPInputStream(conn.getInputStream()));
    }
    return new SimpleTrackingInputStream(conn.getInputStream());
}
 

@Override
protected Data makeData () throws Exception
{
    final URLConnection connection = this.url.openConnection ();
    connection.connect ();

    final int len = connection.getContentLength ();

    final ByteArrayOutputStream bos = new ByteArrayOutputStream ( len > 0 ? len : 0 );

    final byte[] buffer = new byte[4096];
    try ( InputStream stream = connection.getInputStream () )
    {
        int rc;
        while ( ( rc = stream.read ( buffer ) ) > 0 )
        {
            bos.write ( buffer, 0, rc );
        }
        bos.close ();
    }

    final String encoding = connection.getContentEncoding ();
    final String type = connection.getContentType ();

    logger.debug ( "Content-Encoding: {}", encoding );
    logger.debug ( "Content-Type: {}", type );

    Charset charset = null;

    if ( this.charset != null )
    {
        charset = this.charset;
    }
    else if ( this.probeCharset )
    {
        charset = makeCharsetFromType ( type );
    }

    return new UrlConnectionData ( convert ( buffer, charset ), null );
}
 

@Override
public void download(URL src, File dest, CopyProgressListener l) throws IOException {
    // Install the IvyAuthenticator
    if ("http".equals(src.getProtocol()) || "https".equals(src.getProtocol())) {
        IvyAuthenticator.install();
    }

    URLConnection srcConn = null;
    try {
        src = normalizeToURL(src);
        srcConn = src.openConnection();
        srcConn.setRequestProperty("User-Agent", getUserAgent());
        srcConn.setRequestProperty("Accept-Encoding", "gzip,deflate");
        if (srcConn instanceof HttpURLConnection) {
            final HttpURLConnection httpCon = (HttpURLConnection) srcConn;
            final boolean redirect = checkRedirect(httpCon);
            if (redirect) {
                final String newUrl = httpCon.getHeaderField("Location");
                disconnect(srcConn);
                download(new URL(newUrl), dest, l);
                return;
            }
            if (!checkStatusCode(src, httpCon)) {
                throw new IOException("The HTTP response code for " + src
                        + " did not indicate a success." + " See log for more detail.");
            }
        }

        // do the download
        final InputStream inStream = getDecodingInputStream(srcConn.getContentEncoding(),
                srcConn.getInputStream());
        FileUtil.copy(inStream, dest, l);

        // check content length only if content was not encoded
        if (srcConn.getContentEncoding() == null) {
            final int contentLength = srcConn.getContentLength();
            if (contentLength != -1 && dest.length() != contentLength) {
                dest.delete();
                throw new IOException(
                        "Downloaded file size doesn't match expected Content Length for " + src
                                + ". Please retry.");
            }
        }

        // update modification date
        final long lastModified = srcConn.getLastModified();
        if (lastModified > 0) {
            dest.setLastModified(lastModified);
        }
    } finally {
        disconnect(srcConn);
    }
}
 

private InputStream start_download(String f) throws IOException {
    Qaop a = qaop;
    a.dl_length = a.dl_loaded = 0;
    a.dl_text = f;

    URL u = new URL(f);

    f = u.getFile();
    int i = f.lastIndexOf('/');
    if (i >= 0) {
        f = f.substring(i + 1);
    }

    a.dl_text = f;
    a.repaint(200);

    URLConnection c = u.openConnection();
    try {
        c.setConnectTimeout(15000);
        c.setReadTimeout(10000);
    } catch (Error e) {
    }

    if (c instanceof java.net.HttpURLConnection) {
        int x = ((java.net.HttpURLConnection) c).getResponseCode();
        if (x < 200 || x > 299) {
            throw new FileNotFoundException();
        }
    }

    check_type(c.getContentType(), f);

    String s = c.getContentEncoding();
    if (s != null && s.contains("gzip")) {
        gz = true;
    }

    int l = c.getContentLength();
    qaop.dl_length = l > 0 ? l : 1 << 16;

    file = u.getRef();
    return c.getInputStream();
}
 

@Override
public void download(final URL src, final File dest, final CopyProgressListener listener,
                     final TimeoutConstraint timeoutConstraint) throws IOException {

    // Install the IvyAuthenticator
    if ("http".equals(src.getProtocol()) || "https".equals(src.getProtocol())) {
        IvyAuthenticator.install();
    }
    final int connectionTimeout = (timeoutConstraint == null || timeoutConstraint.getConnectionTimeout() < 0) ? 0 : timeoutConstraint.getConnectionTimeout();
    final int readTimeout = (timeoutConstraint == null || timeoutConstraint.getReadTimeout() < 0) ? 0 : timeoutConstraint.getReadTimeout();

    URLConnection srcConn = null;
    try {
        final URL normalizedURL = normalizeToURL(src);
        srcConn = normalizedURL.openConnection();
        srcConn.setConnectTimeout(connectionTimeout);
        srcConn.setReadTimeout(readTimeout);
        srcConn.setRequestProperty("User-Agent", getUserAgent());
        srcConn.setRequestProperty("Accept-Encoding", "gzip,deflate");
        if (srcConn instanceof HttpURLConnection) {
            HttpURLConnection httpCon = (HttpURLConnection) srcConn;
            if (!checkStatusCode(normalizedURL, httpCon)) {
                throw new IOException("The HTTP response code for " + normalizedURL
                        + " did not indicate a success." + " See log for more detail.");
            }
        }

        // do the download
        InputStream inStream = getDecodingInputStream(srcConn.getContentEncoding(),
                srcConn.getInputStream());
        FileUtil.copy(inStream, dest, listener);

        // check content length only if content was not encoded
        if (srcConn.getContentEncoding() == null) {
            final int contentLength = srcConn.getContentLength();
            final long destFileSize = dest.length();
            if (contentLength != -1 && destFileSize != contentLength) {
                dest.delete();
                throw new IOException(
                        "Downloaded file size (" + destFileSize + ") doesn't match expected " +
                                "Content Length (" + contentLength + ") for " + normalizedURL + ". Please retry.");
            }
        }

        // update modification date
        long lastModified = srcConn.getLastModified();
        if (lastModified > 0) {
            dest.setLastModified(lastModified);
        }
    } finally {
        disconnect(srcConn);
    }
}
 

/**
 * TODO(jwagenleitner): remove or fix in future release
 *
 * According to the spec getContentEncoding() returns the Content-Encoding
 * HTTP Header which typically carries values such as 'gzip' or 'deflate'
 * and is not the character set encoding. For compatibility in 2.4.x,
 * this behavior is retained but should be removed or fixed (parse
 * charset from Content-Type header) in future releases.
 *
 * see GROOVY-8056 and https://github.com/apache/groovy/pull/500
 */
private static String getContentEncoding(URL url) throws IOException {
    URLConnection urlConnection = url.openConnection();
    String encoding = urlConnection.getContentEncoding();
    try {
        IOGroovyMethods.closeQuietly(urlConnection.getInputStream());
    } catch (IOException ignore) {
        // For compatibility, ignore exceptions from getInputStream() call
    }
    return encoding;
}