Java Code Examples for java.net.URLConnection#setRequestProperty()

/**
 * Open an input stream to a POST(-like) operation on an URL.
 *
 * @param url The URL
 * @param content Content to POST
 * @param contentType Content type
 * @return Input stream to the URL connection
 * @throws IOException If an I/O error occurs
 */
public static InputStream openPostStream(URL url, byte[] content, String contentType)
    throws IOException
{
	URLConnection conn = url.openConnection();
	conn.setDoOutput(true);

	conn.setConnectTimeout(CONNECT_TIMEOUT);
	conn.setReadTimeout(READ_TIMEOUT);

	// TODO: User-Agent?

	if (contentType != null)
	{
		conn.setRequestProperty("Content-Type", contentType);
	}

	conn.setRequestProperty("Content-Length", String.valueOf(content.length));

	try (OutputStream out = conn.getOutputStream())
	{
		out.write(content);
	}

	return conn.getInputStream();
}
 

/**
 * 向指定 URL 发送POST方法的请求
 *
 * @param url   发送请求的 URL
 * @param param 请求参数，请求参数应该是 name1=value1&name2=value2 的形式。
 * @return 所代表远程资源的响应结果
 */
public static String sendPost(String url, String param) throws IOException {
    StringBuilder result = new StringBuilder();

    String urlNameString = url + "?" + param;
    URL realUrl = new URL(urlNameString);
    URLConnection conn = realUrl.openConnection();
    conn.setDoInput(true);
    conn.setDoOutput(true);
    conn.setRequestProperty(CONTENTTYPE, UTF8);
    conn.setRequestProperty(ACCEPT_CHARSET, UTF8);
    conn.setRequestProperty(USER_AGENT, USER_AGENT_VALUE);
    conn.setRequestProperty(CONNECTION, CONNECTION_VALUE);
    conn.setRequestProperty(ACCEPT, "*/*");
    try (PrintWriter out = new PrintWriter(conn.getOutputStream()); BufferedReader in = new BufferedReader(new InputStreamReader(conn.getInputStream(), UTF8))) {
        String line;
        while ((line = in.readLine()) != null) {
            result.append(line);
        }
        out.flush();
        out.print(param);
    } catch (Exception e) {
        log.error("发送 POST 请求出现异常！", e);
    }
    return result.toString();
}
 

private static JsonValue urlGetJson(String getUrl) {
    try {
        String proxyAddress = Configuration.updateProxyAddress.get();
        URL url = new URL(getUrl);

        URLConnection uc;
        if (proxyAddress != null && !proxyAddress.isEmpty()) {
            int port = 8080;
            if (proxyAddress.contains(":")) {
                String[] parts = proxyAddress.split(":");
                port = Integer.parseInt(parts[1]);
                proxyAddress = parts[0];
            }

            uc = url.openConnection(new Proxy(Proxy.Type.HTTP, new InetSocketAddress(proxyAddress, port)));
        } else {
            uc = url.openConnection();
        }
        uc.setRequestProperty("User-Agent", ApplicationInfo.shortApplicationVerName);
        uc.connect();
        JsonValue value = Json.parse(new InputStreamReader(uc.getInputStream()));
        return value;
    } catch (IOException | NumberFormatException ex) {
        return null;
    }
}
 

public static String getAppVersion() {
    try {
        String str = "";
        URL url = new URL("https://play.google.com/store/apps/details?id=com.nianticlabs.pokemongo");
        URLConnection hc = url.openConnection();
        hc.setRequestProperty("User-Agent", "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.4; en-US; rv:1.9.2.2) Gecko/20100316 Firefox/3.6.2");
        BufferedReader in = new BufferedReader(new InputStreamReader(hc.getInputStream()));
        String temp;
        while ((temp = in.readLine()) != null) {
            str += temp;
        }
        in.close();
        int idx = str.lastIndexOf(KEY) + KEY.length();
        if (idx > 0)
            return str.substring(idx, str.indexOf("<", idx)).trim();
    } catch (IOException e) {
        e.printStackTrace();
    }
    return null;
}
 

private void setupWrapperJar() {
    Path path = Paths.get("wrapper/CloudNet-Wrapper.jar");
    if (!Files.exists(path)) {
        try {
            System.out.println("Downloading wrapper...");
            URLConnection urlConnection = new URL(WRAPPER_URL).openConnection();
            urlConnection.setRequestProperty("User-Agent",
                                             "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.95 Safari/537.11");
            urlConnection.connect();
            Files.copy(urlConnection.getInputStream(), path);
            System.out.println("Download completed!");
        } catch (Exception exception) {
            System.err.println("Error on setting up wrapper: " + exception.getMessage());
            return;
        }
    }
}
 

private static String getHtml(String address){  
    StringBuffer html = new StringBuffer();  
    String result = null;  
    try{  
        URL url = new URL(address);  
        URLConnection conn = url.openConnection();  
        conn.setRequestProperty("User-Agent","Mozilla/4.0 (compatible; MSIE 7.0; NT 5.1; GTB5; .NET CLR 2.0.50727; CIBA)");  
        BufferedInputStream in = new BufferedInputStream(conn.getInputStream());  
          
        try{  
            String inputLine;  
            byte[] buf = new byte[4096];  
            int bytesRead = 0;  
            while (bytesRead >= 0) {  
                inputLine = new String(buf, 0, bytesRead, "ISO-8859-1");  
                html.append(inputLine);  
                bytesRead = in.read(buf);  
                inputLine = null;  
            }  
            buf = null;  
        }finally{  
            in.close();  
            conn = null;  
            url = null;  
        }  
        result = new String(html.toString().trim().getBytes("ISO-8859-1"), "gb2312").toLowerCase();  
          
    }catch (Exception e) {  
        e.printStackTrace();  
        return null;  
    }finally{  
        html = null;              
    }  
    return result;  
}
 

protected String doUrl (URL url) throws IOException {
    StringBuilder sb = new StringBuilder(5000);
    
    if (url != null) {
        URLConnection con = url.openConnection();
        Wandora.initUrlConnection(con);
        con.setDoInput(true);
        con.setUseCaches(false);
        con.setRequestProperty("Content-type", "text/plain");
            
        try {
            BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream(), Charset.forName(defaultEncoding)));

            String s;
            while ((s = in.readLine()) != null) {
                sb.append(s);
                if(!(s.endsWith("\n") || s.endsWith("\r"))) sb.append("\n");
            }
            in.close();
        } 
        catch (Exception ex) {
            log(ex);
        }
    }
    
    return sb.toString();
}
 

private InputStream fetchUrl(String source, String accept) throws IOException {
  URL url = new URL(source);
  URLConnection c = url.openConnection();
  if (accept != null) {
    c.setRequestProperty("accept", accept);
  }
  return c.getInputStream();
}
 

private JarFile getJarFile(URL url) throws IOException {
    // Optimize case where url refers to a local jar file
    if (isOptimizable(url)) {
        FileURLMapper p = new FileURLMapper (url);
        if (!p.exists()) {
            throw new FileNotFoundException(p.getPath());
        }
        return checkJar(new JarFile(p.getPath()));
    }
    URLConnection uc = getBaseURL().openConnection();
    uc.setRequestProperty(USER_AGENT_JAVA_VERSION, JAVA_VERSION);
    JarFile jarFile = ((JarURLConnection)uc).getJarFile();
    return checkJar(jarFile);
}
 

@Test
public void testDfsClusterStart() throws Exception {
    
    // Write a file to HDFS containing the test string
    FileSystem hdfsFsHandle = dfsCluster.getHdfsFileSystemHandle();
    FSDataOutputStream writer = hdfsFsHandle.create(
            new Path(propertyParser.getProperty(ConfigVars.HDFS_TEST_FILE_KEY)));
    writer.writeUTF(propertyParser.getProperty(ConfigVars.HDFS_TEST_STRING_KEY));
    writer.close();

    // Read the file and compare to test string
    FSDataInputStream reader = hdfsFsHandle.open(
            new Path(propertyParser.getProperty(ConfigVars.HDFS_TEST_FILE_KEY)));
    assertEquals(reader.readUTF(), propertyParser.getProperty(ConfigVars.HDFS_TEST_STRING_KEY));
    reader.close();
    hdfsFsHandle.close();

    URL url = new URL(
            String.format( "http://localhost:%s/webhdfs/v1?op=GETHOMEDIRECTORY&user.name=guest",
                    propertyParser.getProperty( ConfigVars.HDFS_NAMENODE_HTTP_PORT_KEY ) ) );
    URLConnection connection = url.openConnection();
    connection.setRequestProperty( "Accept-Charset", "UTF-8" );
    BufferedReader response = new BufferedReader( new InputStreamReader( connection.getInputStream() ) );
    String line = response.readLine();
    response.close();
    assertEquals( "{\"Path\":\"/user/guest\"}", line );

}
 

/**
 * Downloads the file pointed to by the <code>url</code>
 * <p>
 * The downloaded file's name will be the last segment of the path of the URL.
 *
 * @param url location of the file to download, cannot be <code>null</code>
 * @return a path pointing to the downloaded file
 * @throws IOException if the URL cannot be opened, the output file cannot be written or the
 *         transfer of the remote file fails
 */
public IPath download(URL url, IProgressMonitor monitor) throws IOException {
  Preconditions.checkNotNull(url, "url is null");
  String lastSegment = new Path(url.getPath()).lastSegment();
  Preconditions.checkNotNull(lastSegment, "last segment is null");
  Preconditions.checkArgument(!lastSegment.isEmpty(), "last segment is empty string");

  File downloadedFile = downloadFolderPath.append(lastSegment).toFile();
  if (downloadedFile.exists()) {
    return new Path(downloadedFile.getAbsolutePath());
  }

  ensureDownloadFolderExists();
  URLConnection connection = url.openConnection();
  connection.setConnectTimeout(DEFAULT_CONNECT_TIMEOUT_MS);
  connection.setReadTimeout(DEFAULT_READ_TIMEOUT_MS);
  connection.setRequestProperty("User-Agent", CloudToolsInfo.USER_AGENT);
  try (InputStream inputStream = new BufferedInputStream(connection.getInputStream());
       OutputStream outputStream = 
           new BufferedOutputStream(Files.newOutputStream(downloadedFile.toPath()))) {
    byte[] buffer = new byte[4096];
    int read = 0;
    while ((read = inputStream.read(buffer)) != -1) {
      if (monitor.isCanceled()) {
        return null;
      }
      outputStream.write(buffer, 0, read);
    }
    return new Path(downloadedFile.getAbsolutePath());
  } finally {
    if (monitor.isCanceled()) {
      Files.delete(downloadedFile.toPath());
    }
  }
}
 

private void addAuthHeader(URLConnection connection) {
    if (username == null || password == null) {
        return;
    }
    String auth = String.format("%s:%s", username, password);
    String encoded = Base64.encodeBytes(auth.getBytes());
    connection.setRequestProperty("Authorization", "Basic " + encoded);
}
 

public static String sendRequest(URL url, String data, String ctype, String method) throws IOException {
        StringBuilder sb = new StringBuilder(1000);
        if (url != null) {
            URLConnection con = url.openConnection();
            Wandora.initUrlConnection(con);
            con.setDoInput(true);
            con.setUseCaches(false);

            if(method != null && con instanceof HttpURLConnection) {
                ((HttpURLConnection) con).setRequestMethod(method);
                //System.out.println("****** Setting HTTP request method to "+method);
            }

            if(ctype != null) {
                con.setRequestProperty("Content-type", ctype);
            }

            if(data != null && data.length() > 0) {
                con.setRequestProperty("Content-length", data.length() + "");
                con.setDoOutput(true);
                PrintWriter out = new PrintWriter(con.getOutputStream());
                out.print(data);
                out.flush();
                out.close();
            }
//            DataInputStream in = new DataInputStream(con.getInputStream());
            BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()));

            String s;
            while ((s = in.readLine()) != null) {
                sb.append(s);
            }
            in.close();
        }
        return sb.toString();
    }
 

public String doUrl (URL url) throws IOException {
    StringBuilder sb = new StringBuilder(5000);
    
    if (url != null) {
       
            URLConnection con = url.openConnection();
            Wandora.initUrlConnection(con);
            con.setDoInput(true);
            con.setUseCaches(false);
            con.setRequestProperty("Content-type", "text/plain");
            
        try {
            BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream(), Charset.forName(defaultEncoding)));

            String s;
            while ((s = in.readLine()) != null) {
                sb.append(s);
                if(!(s.endsWith("\n") || s.endsWith("\r"))) sb.append("\n");
            }
            in.close();
        } catch (Exception ex) {
            log("Authentication failed. Check API Key.");
        }
    }
    
    return sb.toString();
}
 

public String doUrl (URL url) throws IOException {
    StringBuilder sb = new StringBuilder(5000);
    
    if (url != null) {
        URLConnection con = url.openConnection();
        Wandora.initUrlConnection(con);
        con.setDoInput(true);
        con.setUseCaches(false);
        con.setRequestProperty("Content-type", "text/plain");
            
        try {
            BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream(), Charset.forName(defaultEncoding)));

            String s;
            while ((s = in.readLine()) != null) {
                sb.append(s);
                if(!(s.endsWith("\n") || s.endsWith("\r"))) sb.append("\n");
            }
            in.close();
        } 
        catch (Exception ex) {
            log(ex);
        }
    }
    
    return sb.toString();
}
 

private static void manageContentTypeHeaders( String contentType, String charset, URLConnection connection ) {
    connection.setRequestProperty( "Accept-Charset", charset == null ? StandardCharsets.UTF_8.displayName() : charset );
    if ( contentType != null && !contentType.isEmpty() ) {
        connection.setRequestProperty( "Content-Type", contentType );
    }
}
 

/**
 * Send an HTTP Post
 * @param targetURL
 *    String. The URL to transmit to.
 * @param data
 *            String. The payload String.
 * @param connTimeout
 * 			  int. The connection timeout in ms
 * @param readTimeout
 * 			  int. The read timeout in ms.
 * @return String. The contents of the POST return.
 * @throws Exception
 *             on network errors.
 */

public String sendPost(String targetURL, String data, int connTimeout, int readTimeout) throws Exception {
	URLConnection connection = new URL(targetURL).openConnection();
	connection.setRequestProperty("Content-Type", "application/json");
	connection.setDoInput(true);
	connection.setDoOutput(true);
	connection.setConnectTimeout(connTimeout);
	connection.setReadTimeout(readTimeout);
	OutputStream output = connection.getOutputStream();
	
	try {
		output.write(data.getBytes());
	} finally {
		try {
			output.close();
		} catch (IOException logOrIgnore) {
			logOrIgnore.printStackTrace();
		}
	}
	InputStream response = connection.getInputStream();
	
	http = (HttpURLConnection) connection;
	code = http.getResponseCode();
	
	String value = http.getHeaderField("Content-Encoding");
	
	if (value != null && value.equals("gzip")) {
		byte bytes [] = new byte[4096];
		ByteArrayOutputStream baos = new ByteArrayOutputStream();
		GZIPInputStream gis = new GZIPInputStream(http.getInputStream());
		while(true) {
			int n = gis.read(bytes);
			if (n < 0) break;
			baos.write(bytes,0,n);
		}
		return new String(baos.toByteArray());
	}
	
		

	byte [] b = getContents(response);
	if (b.length == 0)
		return null;
	return new String(b, 0, b.length);
}
 

/**
 * 向指定URL发送GET方法的请求
 *
 * @param url 发送请求的URL
 * @param param 请求参数
 * @return URL 所代表远程资源的响应结果
 */
public static String sendGet(String url, Map<String, String> param) {
    String result = "";
    BufferedReader in = null;
    try {
        StringBuffer query = new StringBuffer();

        for (Map.Entry<String, String> kv : param.entrySet()) {
            query.append(URLEncoder.encode(kv.getKey(), "UTF-8") + "=");
            query.append(URLEncoder.encode(kv.getValue(), "UTF-8") + "&");
        }
        if (query.lastIndexOf("&") > 0) {
            query.deleteCharAt(query.length() - 1);
        }

        String urlNameString = url + "?" + query.toString();
        URL realUrl = new URL(urlNameString);
        // 打开和URL之间的连接
        URLConnection connection = realUrl.openConnection();
        // 设置通用的请求属性
        connection.setRequestProperty("accept", "*/*");
        connection.setRequestProperty("connection", "Keep-Alive");
        connection.setRequestProperty("user-agent", "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1;SV1)");
        // 建立实际的连接
        connection.connect();
        // 获取所有响应头字段
        Map<String, List<String>> map = connection.getHeaderFields();
        // 遍历所有的响应头字段
        for (String key : map.keySet()) {
            System.out.println(key + "--->" + map.get(key));
        }
        // 定义 BufferedReader输入流来读取URL的响应
        in = new BufferedReader(new InputStreamReader(connection.getInputStream()));
        String line;
        while ((line = in.readLine()) != null) {
            result += line;
        }
    } catch (Exception e) {
        System.out.println("发送GET请求出现异常！" + e);
        e.printStackTrace();
    }
    // 使用finally块来关闭输入流
    finally {
        try {
            if (in != null) {
                in.close();
            }
        } catch (Exception e2) {
            e2.printStackTrace();
        }
    }
    return result;
}
 

@SuppressWarnings("deprecation")
@Override
protected Result[] doSearch()
{
    // Perform the search:

    String bingUrl = BING_API_URL +
        java.net.URLEncoder.encode(query.getQueryString()) +
        "%27&$format=JSON";

    byte[] accountKeyBytes = Base64.encodeBase64((BING_AZURE_ID + ":" + BING_AZURE_ID).getBytes());
    String accountKeyEnc = new String(accountKeyBytes);

    StringBuffer sb = new StringBuffer();
    try
    {
        URL url = new URL(bingUrl);
        URLConnection urlConnection = url.openConnection();
        String s1 = "Basic " + accountKeyEnc;
        urlConnection.setRequestProperty("Authorization", s1);
        urlConnection.setConnectTimeout(15000);
        urlConnection.setReadTimeout(25000);
        BufferedReader in = new BufferedReader(new InputStreamReader(
            urlConnection.getInputStream()));
        String inputLine;
        while ((inputLine = in.readLine()) != null)
        {
            sb.append(inputLine);
        }
        in.close();
    }
    catch (Exception ex)
    {
        MsgPrinter.printSearchError(ex);
    }

    // Collect the results:

    ArrayList<String> snippets = new ArrayList<String>();
    ArrayList<String> urls = new ArrayList<String>();

    // Deserialize the JSON string into a DataContainer object:
    Gson gson = new Gson();
    DataContainer myDataContainer = gson.fromJson(sb.toString(),
        DataContainer.class);
    
    if (myDataContainer != null)
    {
        // Fill the results collection with the description and URL
        // from the search results:
        SearchResult[] search_results = myDataContainer.d.results;
        for (SearchResult result : search_results)
        {
            snippets.add(result.Description);
            urls.add(result.Url);
        }
    }

    // Return results:
    return getResults(Collections.toStringArray(snippets),
        Collections.toStringArray(urls), true);
}
 

/**
 * Opens a basic URL connection for the given URL and performs basic
 * configurations. In particular, it will set the User-Agent. The current
 * proxy settings are also respected. For http(s) URLs, the result is a
 * {@link HttpURLConnection}.
 *
 * @param url
 *            the URL to open
 * @return the URL connection to access this URL
 * @throws IOException
 */
public static URLConnection getUrlConnection(URL url) throws IOException {
	URLConnection urlConnection;
	if (hasProxy()) {
		urlConnection = url.openConnection(proxy);
	} else {
		urlConnection = url.openConnection();
	}
	urlConnection.setRequestProperty("User-Agent", userAgent);
	return urlConnection;
}