【Java爬虫】005-HttpClient学习笔记（补充：网页内容获取相关）

訾博ZiBo

发布于 2025-01-06 15:12:26

29200

代码可运行

文章被收录于专栏：全栈开发工程师全栈开发工程师

运行总次数：0

代码可运行

最后更新时间：2020年8月31日11:42:37

一、创建HttpClient实例的6种方法

package com.zb.crawler.httpclient;

import org.apache.http.client.HttpClient;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.impl.client.HttpClients;

public class Main {
    public static void main(String[] args) {
        //6种实例化HttpClient的方式
        //第一种方法已过时，不建议使用
        HttpClient httpClient1 = new DefaultHttpClient();
        HttpClient httpClient2 = HttpClients.custom().build();
        HttpClient httpClient3 = HttpClientBuilder.create().build();
        HttpClient httpClient4 = HttpClients.createDefault();
        HttpClient httpClient5 = HttpClients.createSystem();
        HttpClient httpClient6 = HttpClients.createMinimal();
    }
}

二、设置头消息

1、方式一：一个一个设置

package com.zb.book.httpclient;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.IOException;

public class SetHeaderOne {
    public static void main(String[] args) throws IOException {
        //初始化httpClient
        HttpClient httpClient = HttpClients.custom().build();
        //创建get请求
        HttpGet httpGet = new HttpGet("http://www.baidu.com/");
        //请求头配置
        httpGet.setHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng.*/*;q=0.8");
        httpGet.setHeader("Accept-Encoding","gzip,deflate");
        httpGet.setHeader("Accept-Language","zh-CN,zh;q=0.9");
        httpGet.setHeader("Cache-Control","max-age=0");
        httpGet.setHeader("Host","http://www.baidu.com/");
        httpGet.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36");
        //发出Get请求
        HttpResponse response = httpClient.execute(httpGet);
        //获取响应状态码
        int statusCode = response.getStatusLine().getStatusCode();
        if(statusCode==200) {
            //获取网页内容流
            HttpEntity entity = response.getEntity();
            //转换为字符串形式，需要设置编码
            String content = EntityUtils.toString(response.getEntity(), "UTF-8");
            System.out.println(content);
            //关闭内容流
            EntityUtils.consume(entity);
        }
    }
}

2、方式二：封装到list集合统一设置

package com.zb.book.httpclient;

import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpResponse;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicHeader;
import org.apache.http.util.EntityUtils;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

public class SetHeaderList {
    public static void main(String[] args) throws IOException {
        //通过集合封装头信息
        List<Header> headerList = new ArrayList<>();
        headerList.add(new BasicHeader(HttpHeaders.ACCEPT,"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng.*/*;q=0.8"));
        headerList.add(new BasicHeader(HttpHeaders.ACCEPT_ENCODING,"gzip,deflate"));
        headerList.add(new BasicHeader(HttpHeaders.ACCEPT_LANGUAGE,"zh-CN,zh;q=0.9"));
        headerList.add(new BasicHeader(HttpHeaders.CACHE_CONTROL,"max-age=0"));
        headerList.add(new BasicHeader(HttpHeaders.HOST,"http://www.baidu.com/"));
        headerList.add(new BasicHeader(HttpHeaders.USER_AGENT,"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36"));
        //初始化httpClient
        HttpClient httpClient = HttpClients.custom().setDefaultHeaders(headerList).build();
        //创建get请求
        HttpGet httpGet = new HttpGet("http://www.baidu.com/");
        //发出Get请求
        HttpResponse response = httpClient.execute(httpGet);
        //获取响应状态码
        int statusCode = response.getStatusLine().getStatusCode();
        if(statusCode==200) {
            //获取网页内容流
            HttpEntity entity = response.getEntity();
            //转换为字符串形式，需要设置编码
            String content = EntityUtils.toString(response.getEntity(), "UTF-8");
            System.out.println(content);
            //关闭内容流
            EntityUtils.consume(entity);
        }
    }
}

3、其他方式

非常简单，可参考前两种方式实现；

三、POST提交表单

package com.zb.book.httpclient;

import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.message.BasicNameValuePair;

import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.List;

//Post提交表单
public class PostSubmitForm {
    public static void main(String[] args) throws UnsupportedEncodingException {
        //使用list集合存储欲传递参数
        List<NameValuePair> nvps = new ArrayList<>();
        nvps.add(new BasicNameValuePair("param1","value1"));
        nvps.add(new BasicNameValuePair("param2","value2"));
        //创建UrlEncodedFormEntity对象
        UrlEncodedFormEntity entity = new UrlEncodedFormEntity(nvps, "UTF-8");
        //创建HttpPost
        HttpPost httpPost = new HttpPost("http://xxx.xxx.xxx");
        httpPost.setEntity(entity);
        //执行该请求即可实现提交表单
    }
}

四、超时设置

1、概述

使用HttpClient可设置三种超时时间：RequestTimeout（获取连接超时时间）、ConnectTimeout（建立连接超时时间）、SocketTimeout（获取数据超时时间）。配置这三种超时时间，需要用到HttpClient的RequestConfig类中的方法custom()，该方法返回值为实例化的内部类Builder（配置器），其功能是配置先关请求的字段，还可以设置代理（proxy）、Cookie规范（cookieSpec）、是否允许HTTP相关认证等；

2、代码演示

package com.zb.book.httpclient;

import org.apache.http.client.config.RequestConfig;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;

//设置超时时间
public class SetTimeout {
    public static void main(String[] args) {
        //创建RequestConfig配置，全部设置为10秒
        RequestConfig requestConfig = RequestConfig.custom()
                .setSocketTimeout(10000)//SocketTimeout（获取数据超时时间）
                .setConnectTimeout(10000)//ConnectTimeout（建立连接超时时间）
                .setConnectionRequestTimeout(1000)//RequestTimeout（获取连接超时时间）
                .build();
        //配置到httpclient
        CloseableHttpClient httpClient = HttpClients.custom().setDefaultRequestConfig(requestConfig).build();
        //后面进行正常的请求及相关处理即可
        //另外可用请求方法设置配置，其他写法当做适当改变
        //httpGet.setConfig(requestConfig);
    }
}

五、代理服务器的设置

package com.zb.book.httpclient;

import org.apache.http.HttpHost;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;

//设置代理服务器
public class SetProxy {
    public static void main(String[] args) {
        //创建RequestConfig配置，全部设置为10秒
        RequestConfig requestConfig = RequestConfig.custom()
                .setProxy(new HttpHost("171.221.239.11",808,null))
                .build();
        //配置到httpclient
        CloseableHttpClient httpClient = HttpClients.custom().setDefaultRequestConfig(requestConfig).build();
        //后面进行正常的请求及相关处理即可
        //另外可用请求方法设置配置，其他写法当做适当改变
        //httpGet.setConfig(requestConfig);
    }
}

六、文件下载

1、概述

下载HTML、图片、PDF和压缩包等文件时，一种方法是使用HttpEntity类将响应实体转化为字节数组，再利用输出流的方式写入指定文件。另一种方法是使用HttpEntity类中的writeTo(OutputStream)方法，直接将响应实体写入指定的输出流中，这种方法简单切常用，代码演示如下。

2、代码演示

package com.zb.book.httpclient;

import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

import java.io.FileOutputStream;
import java.io.IOException;

//下载文件
public class DownloadFile {
    public static void main(String[] args) throws IOException {
        //创建HttpClient对象
        CloseableHttpClient httpClient = HttpClients.custom().build();
        //创建HttpGet对象
        HttpGet httpGet = new HttpGet("https://www.baidu.com/img/PCtm_d9c8750bed0b3c7d089fa7d55720d6cf.png");
        //获取结果
        CloseableHttpResponse response = httpClient.execute(httpGet);
        HttpEntity httpEntity = response.getEntity();
        //写出
        httpEntity.writeTo(new FileOutputStream("C:\\Users\\ZiBo\\Desktop\\1.png"));
        //消耗实体
        EntityUtils.consume(httpEntity);
    }
}

七、HTTPS请求认证

1、概述

与jsoup类似，具体做法见代码演示；

2、代码演示

SSLClient类：

package com.httpclient.ssl;

import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.X509Certificate;
import java.util.Arrays;
import javax.net.ssl.SSLContext;
import javax.net.ssl.X509TrustManager;
import org.apache.http.client.HttpClient;
import org.apache.http.client.config.AuthSchemes;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
public class SSLClient {
	/**
	 * 基于SSL配置httpClient
	 * @param  SSLProtocolVersion(SSL, SSLv3, TLS, TLSv1, TLSv1.1, TLSv1.2)
	 * @return httpClient
	 */
	public HttpClient initSSLClient(String SSLProtocolVersion){
		RequestConfig defaultConfig = null;  
		PoolingHttpClientConnectionManager pcm = null;
		try {
			X509TrustManager xtm = new SSL509TrustManager(); //创建信任管理
			//创建SSLContext对象,，并使用指定的信任管理器初始化
			SSLContext context = SSLContext.getInstance(SSLProtocolVersion);
			context.init(null, new X509TrustManager[]{xtm}, null);
			//从SSLContext对象中得到SSLConnectionSocketFactory对象
			SSLConnectionSocketFactory sslConnectionSocketFactory = new SSLConnectionSocketFactory(context, NoopHostnameVerifier.INSTANCE);
			/*从SSLContext对象中得到SSLConnectionSocketFactory对象
			*NoopHostnameVerifier.INSTANCE表示接受接受任何有效的和符合目标主机的SSL会话
			*/
			Registry<ConnectionSocketFactory> sfr = RegistryBuilder.<ConnectionSocketFactory>create()
					.register("http", PlainConnectionSocketFactory.INSTANCE)
					.register("https", sslConnectionSocketFactory).build();
			//基于配置创建连接池
			pcm = new PoolingHttpClientConnectionManager(sfr);
		}catch(NoSuchAlgorithmException | KeyManagementException e){
			e.printStackTrace();
		}
		//设置全局请求配置,包括Cookie规范,HTTP认证,超时
		defaultConfig = RequestConfig.custom().setCookieSpec(CookieSpecs.STANDARD_STRICT)
				.setExpectContinueEnabled(true)
				.setTargetPreferredAuthSchemes(Arrays.asList(AuthSchemes.NTLM, AuthSchemes.DIGEST))
				.setProxyPreferredAuthSchemes(Arrays.asList(AuthSchemes.BASIC))
				.setConnectionRequestTimeout(30*1000)
                .setConnectTimeout(30*1000)
                .setSocketTimeout(30*1000)
                .build();
		//初始化httpclient
		HttpClient httpClient = HttpClients.custom().setConnectionManager(pcm).setDefaultRequestConfig(defaultConfig)
				.build();
		return httpClient;
	}
	//实现X509TrustManager接口
	private static class SSL509TrustManager implements X509TrustManager {
		//检查客户端证书
		public void checkClientTrusted(X509Certificate[] x509Certificates, String s) {
			//do nothing 接受任意客户端证书
		}
		//检查服务器端证书  
		public void checkServerTrusted(X509Certificate[] x509Certificates, String s)  {
			//do nothing  接受任意服务端证书
		}
		//返回受信任的X509证书
		public X509Certificate[] getAcceptedIssuers() {
			return new X509Certificate[0];
		}
	};
}

test类：

package com.httpclient.ssl;

import java.io.IOException;

import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.ParseException;
import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.util.EntityUtils;
public class Test {
	public static void main(String[] args) throws ParseException, IOException {
		String url = "https://cn.kompass.com/a/hospitality-tourism-hotel-and-catering-industries/78/";
		SSLClient sslClient = new SSLClient();   //实例化
		HttpClient httpClientSSL = sslClient.initSSLClient("TLS");
		HttpGet httpGet = new HttpGet(url);
		//获取结果
		HttpResponse httpResponse = null;
		try {
			httpResponse = httpClientSSL.execute(httpGet);
		} catch (IOException e) {
			e.printStackTrace();
		}
		if(httpResponse .getStatusLine().getStatusCode() == HttpStatus.SC_OK){ //状态码200表示响应成功
			//获取实体内容
			String entity = EntityUtils.toString (httpResponse.getEntity(),"UTF-8");
			//输出实体内容
			System.out.println(entity);
			EntityUtils.consume(httpResponse.getEntity());       //消耗实体
		}else {
			//关闭HttpEntity的流实体
			EntityUtils.consume(httpResponse.getEntity());        //消耗实体
		}
	}
}

八、请求重试

1、概述

使用HtpClient请求URL时，有时会出现请求异常的情况。针对一些非致命的异常，可以通过请求重试解决。HttpClient提供了默认重试策略DefalutHttpRequestRetryHandler。DefalutHttpRequestRetryHandler类实现了HttpRequestRetryHandler接口，重写了retryRequest(方法。由源码可以发现DefalutHttpRequestRetryHandler类定义的默认重试次数为3次；幂等方法(如GET和HEAD是幂等的)可以重试：如果网页请求失败，可以重试。另外，针对4种异常不进行重试，这四种异常分别是InterruptedIOException (线程中断异常)、UnknownHostException (未知的Host异常)、ConnectException (连接异常,如连接拒绝异常)和SSLException ( HTTPS请求认证异常)。

2、代码演示

package com.zb.book.httpclient;

import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
import org.apache.http.impl.client.HttpClients;

//设置请求重试
public class SetRequestRetry {
    public static void main(String[] args) {
        //第一种：默认重试3次
        HttpClients.custom()
                .setRetryHandler(new DefaultHttpRequestRetryHandler())
                .build();
        //第二种：自定义重试5次
        HttpClients.custom()
                .setRetryHandler(new DefaultHttpRequestRetryHandler(5,true))
                .build();
    }
}

3、补充说明

值得注意的是，在进行数据爬取时经常遇到的两种超时时间: ConnectTimeout(建立连接的超时时间)和SocketTimeout(获取数据的超时时间)，这两种超时时间对应的异常( ConnectTimeoutException与SocketTimeoutException )都继承自InterruptedIOException类，即属于线程中断异常，不会进行重试。

九、多线程执行请求演示代码摘录

（可参考原始学习笔记的连接池）

package com.httpclient.thread;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.CodingErrorAction;
import java.util.Arrays;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.apache.http.Consts;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.config.AuthSchemes;
import org.apache.http.client.config.CookieSpecs;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.protocol.HttpClientContext;
import org.apache.http.config.ConnectionConfig;
import org.apache.http.config.SocketConfig;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.protocol.HttpContext;
import org.apache.http.util.EntityUtils;
public class Test {
	public static void main(String[] args) throws FileNotFoundException {
		//添加连接参数
		ConnectionConfig connectionConfig = ConnectionConfig.custom()
				.setMalformedInputAction(CodingErrorAction.IGNORE)
				.setUnmappableInputAction(CodingErrorAction.IGNORE)
				.setCharset(Consts.UTF_8)
				.build();
		//添加socket参数
		SocketConfig socketConfig = SocketConfig.custom()
				.setTcpNoDelay(true)
				.build();
		//配置连接池管理器
		PoolingHttpClientConnectionManager pcm = new PoolingHttpClientConnectionManager();
		// 设置最大连接数
		pcm.setMaxTotal(100);
		// 设置每个连接的路由数
		pcm.setDefaultMaxPerRoute(10);
		//设置连接信息
		pcm.setDefaultConnectionConfig(connectionConfig);
		//设置socket信息
		pcm.setDefaultSocketConfig(socketConfig);
		//设置全局请求配置,包括Cookie规范,HTTP认证,超时
		RequestConfig defaultConfig = RequestConfig.custom()
				.setCookieSpec(CookieSpecs.STANDARD_STRICT)
				.setExpectContinueEnabled(true)
				.setTargetPreferredAuthSchemes(Arrays
						.asList(AuthSchemes.NTLM, AuthSchemes.DIGEST))
				.setProxyPreferredAuthSchemes(Arrays.asList(AuthSchemes.BASIC))
				.setConnectionRequestTimeout(30*1000)
				.setConnectTimeout(30*1000)
				.setSocketTimeout(30*1000)
				.build();
		CloseableHttpClient httpClient = HttpClients.custom()
				.setConnectionManager(pcm)
				.setDefaultRequestConfig(defaultConfig)
				.build();
		// 请求的URL
		String[] urlArr = {
				"https://hbr.org/podcasts",
				"https://hbr.org/magazine",
				"https://hbr.org/most-popular",
				"https://hbr.org/big-ideas",
				"https://hbr.org/reading-lists"
		};
		//创建固定大小的线程池
		ExecutorService exec = Executors.newFixedThreadPool(3);
		for(int i = 0; i< urlArr.length;i++){
			String filename = urlArr[i].split("org/")[1]; //HTML需要输出的文件名
			//创建HTML文件输出目录
			OutputStream out = new FileOutputStream("file/" + filename);
			HttpGet httpget = new HttpGet(urlArr[i]);
			//启动线程执行请求
			exec.execute(new DownHtmlFileThread(httpClient, httpget, out));
		}
		//关闭线程
		exec.shutdown();
	}
	static class DownHtmlFileThread extends Thread {
		private final CloseableHttpClient httpClient;
		private final HttpContext context;
		private final HttpGet httpget;
		private final OutputStream out;
		//输入的参数
		public DownHtmlFileThread(CloseableHttpClient httpClient, 
				HttpGet httpget, OutputStream out) {
			this.httpClient = httpClient;
			this.context = HttpClientContext.create();
			this.httpget = httpget;
			this.out = out;
		}
		@Override
		public void run() {
			System.out.println(Thread.currentThread().getName() + 
					"线程请求的URL为:" + httpget.getURI());
			try {
				CloseableHttpResponse response = httpClient.execute(
						httpget, context);  //执行请求
				try {
					//HTML文件写入文档
					out.write(EntityUtils.toString(response.getEntity(),"gbk")
							.getBytes());
					out.close();
					//消耗实体
					EntityUtils.consume(response.getEntity());
				} finally{
					response.close(); //关闭响应
				}
			} catch (ClientProtocolException ex) {
				ex.printStackTrace(); // 处理 protocol错误
			} catch (IOException ex) {
				ex.printStackTrace(); // 处理I/O错误
			}
		}
	}
}

本文参与腾讯云自媒体同步曝光计划，分享自作者个人站点/博客。

原始发表：2025-01-06，如有侵权请联系 cloudcommunity@tencent.com 删除

学习笔记