1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 package org.htmlunit;
16
17 import static org.htmlunit.BrowserVersionFeatures.HTTP_HEADER_CH_UA;
18 import static org.htmlunit.BrowserVersionFeatures.HTTP_HEADER_PRIORITY;
19
20 import java.io.ByteArrayInputStream;
21 import java.io.ByteArrayOutputStream;
22 import java.io.EOFException;
23 import java.io.File;
24 import java.io.IOException;
25 import java.io.InputStream;
26 import java.io.OutputStream;
27 import java.net.InetAddress;
28 import java.net.URI;
29 import java.net.URISyntaxException;
30 import java.net.URL;
31 import java.nio.charset.Charset;
32 import java.nio.file.Files;
33 import java.util.ArrayList;
34 import java.util.HashMap;
35 import java.util.List;
36 import java.util.Map;
37 import java.util.WeakHashMap;
38 import java.util.concurrent.TimeUnit;
39
40 import javax.net.ssl.HostnameVerifier;
41 import javax.net.ssl.SSLContext;
42 import javax.net.ssl.SSLPeerUnverifiedException;
43 import javax.net.ssl.SSLSocketFactory;
44
45 import org.apache.commons.io.IOUtils;
46 import org.apache.commons.lang3.StringUtils;
47 import org.apache.commons.lang3.reflect.FieldUtils;
48 import org.apache.commons.logging.Log;
49 import org.apache.commons.logging.LogFactory;
50 import org.apache.http.ConnectionClosedException;
51 import org.apache.http.Header;
52 import org.apache.http.HttpEntity;
53 import org.apache.http.HttpEntityEnclosingRequest;
54 import org.apache.http.HttpException;
55 import org.apache.http.HttpHost;
56 import org.apache.http.HttpRequest;
57 import org.apache.http.HttpRequestInterceptor;
58 import org.apache.http.HttpResponse;
59 import org.apache.http.auth.AuthScheme;
60 import org.apache.http.auth.AuthScope;
61 import org.apache.http.auth.Credentials;
62 import org.apache.http.client.AuthCache;
63 import org.apache.http.client.CredentialsProvider;
64 import org.apache.http.client.config.RequestConfig;
65 import org.apache.http.client.methods.CloseableHttpResponse;
66 import org.apache.http.client.methods.HttpGet;
67 import org.apache.http.client.methods.HttpHead;
68 import org.apache.http.client.methods.HttpPatch;
69 import org.apache.http.client.methods.HttpPost;
70 import org.apache.http.client.methods.HttpPut;
71 import org.apache.http.client.methods.HttpRequestBase;
72 import org.apache.http.client.methods.HttpTrace;
73 import org.apache.http.client.methods.HttpUriRequest;
74 import org.apache.http.client.protocol.HttpClientContext;
75 import org.apache.http.client.protocol.RequestAcceptEncoding;
76 import org.apache.http.client.protocol.RequestAddCookies;
77 import org.apache.http.client.protocol.RequestAuthCache;
78 import org.apache.http.client.protocol.RequestDefaultHeaders;
79 import org.apache.http.client.protocol.RequestExpectContinue;
80 import org.apache.http.client.protocol.ResponseProcessCookies;
81 import org.apache.http.client.utils.URLEncodedUtils;
82 import org.apache.http.config.ConnectionConfig;
83 import org.apache.http.config.RegistryBuilder;
84 import org.apache.http.config.SocketConfig;
85 import org.apache.http.conn.DnsResolver;
86 import org.apache.http.conn.routing.RouteInfo;
87 import org.apache.http.conn.socket.ConnectionSocketFactory;
88 import org.apache.http.conn.socket.LayeredConnectionSocketFactory;
89 import org.apache.http.conn.ssl.DefaultHostnameVerifier;
90 import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
91 import org.apache.http.conn.util.PublicSuffixMatcher;
92 import org.apache.http.conn.util.PublicSuffixMatcherLoader;
93 import org.apache.http.cookie.CookieSpecProvider;
94 import org.apache.http.entity.ContentType;
95 import org.apache.http.entity.StringEntity;
96 import org.apache.http.entity.mime.MultipartEntityBuilder;
97 import org.apache.http.entity.mime.content.InputStreamBody;
98 import org.apache.http.impl.client.BasicAuthCache;
99 import org.apache.http.impl.client.CloseableHttpClient;
100 import org.apache.http.impl.client.HttpClientBuilder;
101 import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
102 import org.apache.http.protocol.HttpContext;
103 import org.apache.http.protocol.HttpProcessorBuilder;
104 import org.apache.http.protocol.RequestContent;
105 import org.apache.http.protocol.RequestTargetHost;
106 import org.apache.http.ssl.SSLContexts;
107 import org.apache.http.util.TextUtils;
108 import org.htmlunit.WebRequest.HttpHint;
109 import org.htmlunit.http.HttpUtils;
110 import org.htmlunit.httpclient.HtmlUnitCookieSpecProvider;
111 import org.htmlunit.httpclient.HtmlUnitCookieStore;
112 import org.htmlunit.httpclient.HtmlUnitRedirectStrategie;
113 import org.htmlunit.httpclient.HtmlUnitSSLConnectionSocketFactory;
114 import org.htmlunit.httpclient.HttpClientConverter;
115 import org.htmlunit.httpclient.SocksConnectionSocketFactory;
116 import org.htmlunit.util.KeyDataPair;
117 import org.htmlunit.util.MimeType;
118 import org.htmlunit.util.NameValuePair;
119 import org.htmlunit.util.UrlUtils;
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137 public class HttpWebConnection implements WebConnection {
138
139 private static final Log LOG = LogFactory.getLog(HttpWebConnection.class);
140
141 private static final String HACKED_COOKIE_POLICY = "mine";
142
143
144
145 private final Map<Thread, HttpClientBuilder> httpClientBuilder_ = new WeakHashMap<>();
146 private final WebClient webClient_;
147
148 private String virtualHost_;
149 private final HtmlUnitCookieSpecProvider htmlUnitCookieSpecProvider_;
150 private final WebClientOptions usedOptions_;
151 private PoolingHttpClientConnectionManager connectionManager_;
152
153
154 private final AuthCache sharedAuthCache_ = new SynchronizedAuthCache();
155
156
157 private final Map<Thread, HttpClientContext> httpClientContextByThread_ = new WeakHashMap<>();
158
159
160
161
162
163 public HttpWebConnection(final WebClient webClient) {
164 super();
165 webClient_ = webClient;
166 htmlUnitCookieSpecProvider_ = new HtmlUnitCookieSpecProvider(webClient.getBrowserVersion());
167 usedOptions_ = new WebClientOptions();
168 }
169
170
171
172
173 @Override
174 public WebResponse getResponse(final WebRequest webRequest) throws IOException {
175 final HttpClientBuilder builder = reconfigureHttpClientIfNeeded(getHttpClientBuilder(), webRequest);
176
177 HttpUriRequest httpMethod = null;
178 try {
179 try {
180 httpMethod = makeHttpMethod(webRequest, builder);
181 }
182 catch (final URISyntaxException e) {
183 throw new IOException("Unable to create URI from URL: " + webRequest.getUrl().toExternalForm()
184 + " (reason: " + e.getMessage() + ")", e);
185 }
186
187 final URL url = webRequest.getUrl();
188 final HttpHost httpHost = new HttpHost(url.getHost(), url.getPort(), url.getProtocol());
189 final long startTime = System.currentTimeMillis();
190
191 final HttpContext httpContext = getHttpContext();
192 try {
193 try (CloseableHttpClient closeableHttpClient = builder.build()) {
194 try (CloseableHttpResponse httpResponse =
195 closeableHttpClient.execute(httpHost, httpMethod, httpContext)) {
196 return downloadResponse(httpMethod, webRequest, httpResponse, startTime);
197 }
198 }
199 }
200 catch (final SSLPeerUnverifiedException ex) {
201
202 if (webClient_.getOptions().isUseInsecureSSL()) {
203 HtmlUnitSSLConnectionSocketFactory.setUseSSL3Only(httpContext, true);
204 try (CloseableHttpClient closeableHttpClient = builder.build()) {
205 try (CloseableHttpResponse httpResponse =
206 closeableHttpClient.execute(httpHost, httpMethod, httpContext)) {
207 return downloadResponse(httpMethod, webRequest, httpResponse, startTime);
208 }
209 }
210 }
211 throw ex;
212 }
213 catch (final Error e) {
214
215
216
217
218 synchronized (httpClientBuilder_) {
219 httpClientBuilder_.remove(Thread.currentThread());
220 }
221 throw e;
222 }
223 }
224 finally {
225 if (httpMethod != null) {
226 onResponseGenerated(httpMethod);
227 }
228 }
229 }
230
231
232
233
234
235
236 protected void onResponseGenerated(final HttpUriRequest httpMethod) {
237
238 }
239
240
241
242
243 private synchronized HttpContext getHttpContext() {
244 HttpClientContext httpClientContext = httpClientContextByThread_.get(Thread.currentThread());
245 if (httpClientContext == null) {
246 httpClientContext = new HttpClientContext();
247
248
249 httpClientContext.setAttribute(HttpClientContext.AUTH_CACHE, sharedAuthCache_);
250
251 httpClientContextByThread_.put(Thread.currentThread(), httpClientContext);
252 }
253 return httpClientContext;
254 }
255
256 private void setProxy(final HttpRequestBase httpRequest, final WebRequest webRequest) {
257 final InetAddress localAddress = webClient_.getOptions().getLocalAddress();
258 final RequestConfig.Builder requestBuilder = createRequestConfigBuilder(getTimeout(webRequest), localAddress);
259
260 if (webRequest.getProxyHost() == null) {
261 requestBuilder.setProxy(null);
262 httpRequest.setConfig(requestBuilder.build());
263 return;
264 }
265
266 final HttpHost proxy = new HttpHost(webRequest.getProxyHost(),
267 webRequest.getProxyPort(), webRequest.getProxyScheme());
268 if (webRequest.isSocksProxy()) {
269 SocksConnectionSocketFactory.setSocksProxy(getHttpContext(), proxy);
270 }
271 else {
272 requestBuilder.setProxy(proxy);
273 httpRequest.setConfig(requestBuilder.build());
274 }
275 }
276
277
278
279
280
281
282
283
284 private HttpUriRequest makeHttpMethod(final WebRequest webRequest, final HttpClientBuilder httpClientBuilder)
285 throws URISyntaxException {
286
287 final HttpContext httpContext = getHttpContext();
288 final Charset charset = webRequest.getCharset();
289
290
291
292
293 final URL url = UrlUtils.encodeUrl(webRequest.getUrl(), charset);
294
295 URI uri = UrlUtils.toURI(url, escapeQuery(url.getQuery()));
296 if (getVirtualHost() != null) {
297 uri = URI.create(getVirtualHost());
298 }
299 final HttpRequestBase httpMethod = buildHttpMethod(webRequest.getHttpMethod(), uri);
300 setProxy(httpMethod, webRequest);
301
302
303
304
305
306 if (httpMethod instanceof HttpPost
307 || httpMethod instanceof HttpPut
308 || httpMethod instanceof HttpPatch
309 || httpMethod instanceof org.htmlunit.httpclient.HttpDelete
310 || httpMethod instanceof org.htmlunit.httpclient.HttpOptions) {
311
312 final HttpEntityEnclosingRequest method = (HttpEntityEnclosingRequest) httpMethod;
313
314 if (FormEncodingType.URL_ENCODED == webRequest.getEncodingType()) {
315 if (webRequest.getRequestBody() == null) {
316 final List<NameValuePair> pairs = webRequest.getRequestParameters();
317 final String query = HttpUtils.toQueryFormFields(pairs, charset);
318
319 final StringEntity urlEncodedEntity;
320 if (webRequest.hasHint(HttpHint.IncludeCharsetInContentTypeHeader)) {
321 urlEncodedEntity = new StringEntity(query,
322 ContentType.create(URLEncodedUtils.CONTENT_TYPE, charset));
323
324 }
325 else {
326 urlEncodedEntity = new StringEntity(query, charset);
327 urlEncodedEntity.setContentType(URLEncodedUtils.CONTENT_TYPE);
328 }
329 method.setEntity(urlEncodedEntity);
330 }
331 else {
332 final String body = StringUtils.defaultString(webRequest.getRequestBody());
333 final StringEntity urlEncodedEntity = new StringEntity(body, charset);
334 urlEncodedEntity.setContentType(URLEncodedUtils.CONTENT_TYPE);
335 method.setEntity(urlEncodedEntity);
336 }
337 }
338 else if (FormEncodingType.TEXT_PLAIN == webRequest.getEncodingType()) {
339 if (webRequest.getRequestBody() == null) {
340 final StringBuilder body = new StringBuilder();
341 for (final NameValuePair pair : webRequest.getRequestParameters()) {
342 body.append(StringUtils.remove(StringUtils.remove(pair.getName(), '\r'), '\n'))
343 .append('=')
344 .append(StringUtils.remove(StringUtils.remove(pair.getValue(), '\r'), '\n'))
345 .append("\r\n");
346 }
347 final StringEntity bodyEntity = new StringEntity(body.toString(), charset);
348 bodyEntity.setContentType(MimeType.TEXT_PLAIN);
349 method.setEntity(bodyEntity);
350 }
351 else {
352 final String body = StringUtils.defaultString(webRequest.getRequestBody());
353 final StringEntity bodyEntity =
354 new StringEntity(body, ContentType.create(MimeType.TEXT_PLAIN, charset));
355 method.setEntity(bodyEntity);
356 }
357 }
358 else if (FormEncodingType.MULTIPART == webRequest.getEncodingType()) {
359 final Charset c = getCharset(charset, webRequest.getRequestParameters());
360 final MultipartEntityBuilder builder = MultipartEntityBuilder.create().setLaxMode();
361 builder.setCharset(c);
362
363 for (final NameValuePair pair : webRequest.getRequestParameters()) {
364 if (pair instanceof KeyDataPair dataPair) {
365 buildFilePart(dataPair, builder);
366 }
367 else {
368 builder.addTextBody(pair.getName(), pair.getValue(),
369 ContentType.create(MimeType.TEXT_PLAIN, charset));
370 }
371 }
372 method.setEntity(builder.build());
373 }
374 else {
375
376 final String body = webRequest.getRequestBody();
377 if (body != null) {
378 method.setEntity(new StringEntity(body, charset));
379 }
380 }
381 }
382 else {
383
384 final List<NameValuePair> pairs = webRequest.getRequestParameters();
385 if (!pairs.isEmpty()) {
386 final String query = HttpUtils.toQueryFormFields(pairs, charset);
387 uri = UrlUtils.toURI(url, query);
388 httpMethod.setURI(uri);
389 }
390 }
391
392 configureHttpProcessorBuilder(httpClientBuilder, webRequest);
393
394
395
396 final CredentialsProvider credentialsProvider = webClient_.getCredentialsProvider();
397
398
399 final Credentials requestUrlCredentials = webRequest.getUrlCredentials();
400 if (null != requestUrlCredentials) {
401 final URL requestUrl = webRequest.getUrl();
402 final AuthScope authScope = new AuthScope(requestUrl.getHost(), requestUrl.getPort());
403
404 credentialsProvider.setCredentials(authScope, requestUrlCredentials);
405 }
406
407
408 final Credentials requestCredentials = webRequest.getCredentials();
409 if (null != requestCredentials) {
410 final URL requestUrl = webRequest.getUrl();
411 final AuthScope authScope = new AuthScope(requestUrl.getHost(), requestUrl.getPort());
412
413 credentialsProvider.setCredentials(authScope, requestCredentials);
414 }
415 httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
416 httpContext.removeAttribute(HttpClientContext.CREDS_PROVIDER);
417 httpContext.removeAttribute(HttpClientContext.TARGET_AUTH_STATE);
418 return httpMethod;
419 }
420
421 private static String escapeQuery(final String query) {
422 if (query == null) {
423 return null;
424 }
425 return query.replace("%%", "%25%25");
426 }
427
428 private static Charset getCharset(final Charset charset, final List<NameValuePair> pairs) {
429 for (final NameValuePair pair : pairs) {
430 if (pair instanceof KeyDataPair pairWithFile) {
431 if (pairWithFile.getData() == null && pairWithFile.getFile() != null) {
432 final String fileName = pairWithFile.getFile().getName();
433 final int length = fileName.length();
434 for (int i = 0; i < length; i++) {
435 if (fileName.codePointAt(i) > 127) {
436 return charset;
437 }
438 }
439 }
440 }
441 }
442 return null;
443 }
444
445 void buildFilePart(final KeyDataPair pairWithFile, final MultipartEntityBuilder builder) {
446 String mimeType = pairWithFile.getMimeType();
447 if (mimeType == null) {
448 mimeType = MimeType.APPLICATION_OCTET_STREAM;
449 }
450
451 final ContentType contentType = ContentType.create(mimeType);
452
453 final File file = pairWithFile.getFile();
454 if (file != null) {
455 String filename = pairWithFile.getFileName();
456 if (filename == null) {
457 filename = pairWithFile.getFile().getName();
458 }
459 builder.addBinaryBody(pairWithFile.getName(), file, contentType, filename);
460 return;
461 }
462
463 final byte[] data = pairWithFile.getData();
464 if (data != null) {
465 String filename = pairWithFile.getFileName();
466 if (filename == null) {
467 filename = pairWithFile.getValue();
468 }
469
470 builder.addBinaryBody(pairWithFile.getName(), data, contentType, filename);
471 return;
472 }
473
474 builder.addPart(pairWithFile.getName(),
475
476 new InputStreamBody(new ByteArrayInputStream(new byte[0]), contentType, pairWithFile.getValue()) {
477 @Override
478 public long getContentLength() {
479 return 0;
480 }
481 });
482 }
483
484
485
486
487
488
489
490 private static HttpRequestBase buildHttpMethod(final HttpMethod submitMethod, final URI uri) {
491 final HttpRequestBase method = switch (submitMethod) {
492 case GET -> new HttpGet(uri);
493 case POST -> new HttpPost(uri);
494 case PUT -> new HttpPut(uri);
495 case DELETE -> new org.htmlunit.httpclient.HttpDelete(uri);
496 case OPTIONS -> new org.htmlunit.httpclient.HttpOptions(uri);
497 case HEAD -> new HttpHead(uri);
498 case TRACE -> new HttpTrace(uri);
499 case PATCH -> new HttpPatch(uri);
500 };
501 return method;
502 }
503
504
505
506
507
508
509 protected HttpClientBuilder getHttpClientBuilder() {
510 final Thread currentThread = Thread.currentThread();
511
512 synchronized (httpClientBuilder_) {
513 HttpClientBuilder builder = httpClientBuilder_.get(currentThread);
514 if (builder == null) {
515 builder = createHttpClientBuilder();
516
517
518
519 final RegistryBuilder<CookieSpecProvider> registeryBuilder
520 = RegistryBuilder.<CookieSpecProvider>create()
521 .register(HACKED_COOKIE_POLICY, htmlUnitCookieSpecProvider_);
522 builder.setDefaultCookieSpecRegistry(registeryBuilder.build());
523
524 builder.setDefaultCookieStore(new HtmlUnitCookieStore(webClient_.getCookieManager()));
525 builder.setUserAgent(webClient_.getBrowserVersion().getUserAgent());
526 httpClientBuilder_.put(currentThread, builder);
527 }
528
529 return builder;
530 }
531 }
532
533
534
535
536
537
538
539
540 protected int getTimeout(final WebRequest webRequest) {
541 if (webRequest == null || webRequest.getTimeout() < 0) {
542 return webClient_.getOptions().getTimeout();
543 }
544
545 return webRequest.getTimeout();
546 }
547
548
549
550
551
552
553
554
555
556 protected HttpClientBuilder createHttpClientBuilder() {
557 final HttpClientBuilder builder = HttpClientBuilder.create();
558 builder.setRedirectStrategy(new HtmlUnitRedirectStrategie());
559 configureTimeout(builder, getTimeout(null));
560 configureHttpsScheme(builder);
561 builder.setMaxConnPerRoute(6);
562
563 builder.setConnectionManagerShared(true);
564 return builder;
565 }
566
567 private void configureTimeout(final HttpClientBuilder builder, final int timeout) {
568 final InetAddress localAddress = webClient_.getOptions().getLocalAddress();
569 final RequestConfig.Builder requestBuilder = createRequestConfigBuilder(timeout, localAddress);
570 builder.setDefaultRequestConfig(requestBuilder.build());
571
572 builder.setDefaultSocketConfig(createSocketConfigBuilder(timeout).build());
573
574 getHttpContext().removeAttribute(HttpClientContext.REQUEST_CONFIG);
575 usedOptions_.setTimeout(timeout);
576 }
577
578 private static RequestConfig.Builder createRequestConfigBuilder(final int timeout, final InetAddress localAddress) {
579 return RequestConfig.custom()
580 .setCookieSpec(HACKED_COOKIE_POLICY)
581 .setRedirectsEnabled(false)
582 .setLocalAddress(localAddress)
583
584
585 .setConnectTimeout(timeout)
586 .setConnectionRequestTimeout(timeout)
587 .setSocketTimeout(timeout);
588 }
589
590 private static SocketConfig.Builder createSocketConfigBuilder(final int timeout) {
591 return SocketConfig.custom()
592
593 .setSoTimeout(timeout);
594 }
595
596
597
598
599
600 private HttpClientBuilder reconfigureHttpClientIfNeeded(final HttpClientBuilder httpClientBuilder,
601 final WebRequest webRequest) {
602 final WebClientOptions options = webClient_.getOptions();
603
604
605 if (options.isUseInsecureSSL() != usedOptions_.isUseInsecureSSL()
606 || options.getSSLClientCertificateStore() != usedOptions_.getSSLClientCertificateStore()
607 || options.getSSLTrustStore() != usedOptions_.getSSLTrustStore()
608 || options.getSSLClientCipherSuites() != usedOptions_.getSSLClientCipherSuites()
609 || options.getSSLClientProtocols() != usedOptions_.getSSLClientProtocols()
610 || options.getProxyConfig() != usedOptions_.getProxyConfig()) {
611 configureHttpsScheme(httpClientBuilder);
612
613 if (connectionManager_ != null) {
614 connectionManager_.shutdown();
615 connectionManager_ = null;
616 }
617 }
618
619 final int timeout = getTimeout(webRequest);
620 if (timeout != usedOptions_.getTimeout()) {
621 configureTimeout(httpClientBuilder, timeout);
622 }
623
624 final long connectionTimeToLive = webClient_.getOptions().getConnectionTimeToLive();
625 if (connectionTimeToLive != usedOptions_.getConnectionTimeToLive()) {
626 httpClientBuilder.setConnectionTimeToLive(connectionTimeToLive, TimeUnit.MILLISECONDS);
627 usedOptions_.setConnectionTimeToLive(connectionTimeToLive);
628 }
629
630 if (connectionManager_ == null) {
631 connectionManager_ = createConnectionManager(httpClientBuilder);
632 }
633 httpClientBuilder.setConnectionManager(connectionManager_);
634
635 return httpClientBuilder;
636 }
637
638 private void configureHttpsScheme(final HttpClientBuilder builder) {
639 final WebClientOptions options = webClient_.getOptions();
640
641 final SSLConnectionSocketFactory socketFactory =
642 HtmlUnitSSLConnectionSocketFactory.buildSSLSocketFactory(options);
643
644 builder.setSSLSocketFactory(socketFactory);
645
646 usedOptions_.setUseInsecureSSL(options.isUseInsecureSSL());
647 usedOptions_.setSSLClientCertificateKeyStore(options.getSSLClientCertificateStore(),
648 options.getSSLClientCertificatePassword());
649 usedOptions_.setSSLTrustStore(options.getSSLTrustStore());
650 usedOptions_.setSSLClientCipherSuites(options.getSSLClientCipherSuites());
651 usedOptions_.setSSLClientProtocols(options.getSSLClientProtocols());
652 usedOptions_.setProxyConfig(options.getProxyConfig());
653 }
654
655 private void configureHttpProcessorBuilder(final HttpClientBuilder builder, final WebRequest webRequest) {
656 final HttpProcessorBuilder b = HttpProcessorBuilder.create();
657 for (final HttpRequestInterceptor i : getHttpRequestInterceptors(webRequest)) {
658 b.add(i);
659 }
660
661
662
663 b.addAll(new RequestDefaultHeaders(null),
664 new RequestContent(),
665 new RequestTargetHost(),
666 new RequestExpectContinue());
667 b.add(new RequestAcceptEncoding());
668 b.add(new RequestAuthCache());
669
670 if (!webRequest.hasHint(HttpHint.BlockCookies)) {
671 b.add(new ResponseProcessCookies());
672 }
673 builder.setHttpProcessor(b.build());
674 }
675
676
677
678
679
680 public void setVirtualHost(final String virtualHost) {
681 virtualHost_ = virtualHost;
682 }
683
684
685
686
687
688 public String getVirtualHost() {
689 return virtualHost_;
690 }
691
692
693
694
695
696
697
698
699
700 protected WebResponse makeWebResponse(final HttpResponse httpResponse,
701 final WebRequest webRequest, final DownloadedContent responseBody, final long loadTime) {
702
703 String statusMessage = httpResponse.getStatusLine().getReasonPhrase();
704 if (statusMessage == null) {
705 statusMessage = "Unknown status message";
706 }
707 final int statusCode = httpResponse.getStatusLine().getStatusCode();
708 final List<NameValuePair> headers = new ArrayList<>();
709 for (final Header header : httpResponse.getAllHeaders()) {
710 headers.add(new NameValuePair(header.getName(), header.getValue()));
711 }
712 final WebResponseData responseData = new WebResponseData(responseBody, statusCode, statusMessage, headers);
713 return newWebResponseInstance(responseData, loadTime, webRequest);
714 }
715
716
717
718
719
720
721
722
723
724
725
726 protected WebResponse downloadResponse(final HttpUriRequest httpMethod,
727 final WebRequest webRequest, final HttpResponse httpResponse,
728 final long startTime) throws IOException {
729
730 final DownloadedContent downloadedBody = downloadResponseBody(httpResponse);
731 final long endTime = System.currentTimeMillis();
732
733 return makeWebResponse(httpResponse, webRequest, downloadedBody, endTime - startTime);
734 }
735
736
737
738
739
740
741
742 protected DownloadedContent downloadResponseBody(final HttpResponse httpResponse) throws IOException {
743 final HttpEntity httpEntity = httpResponse.getEntity();
744 if (httpEntity == null) {
745 return new DownloadedContent.InMemory(null);
746 }
747
748 try (InputStream is = httpEntity.getContent()) {
749 return downloadContent(is, webClient_.getOptions().getMaxInMemory(),
750 webClient_.getOptions().getTempFileDirectory());
751 }
752 }
753
754
755
756
757
758
759
760
761
762 public static DownloadedContent downloadContent(final InputStream is, final int maxInMemory,
763 final File tempFileDirectory) throws IOException {
764 if (is == null) {
765 return new DownloadedContent.InMemory(null);
766 }
767
768 try (ByteArrayOutputStream bos = new ByteArrayOutputStream()) {
769 final byte[] buffer = new byte[1024];
770 int nbRead;
771 try {
772 while ((nbRead = is.read(buffer)) != -1) {
773 bos.write(buffer, 0, nbRead);
774 if (maxInMemory > 0 && bos.size() > maxInMemory) {
775
776 final File file = File.createTempFile("htmlunit", ".tmp", tempFileDirectory);
777 file.deleteOnExit();
778 try (OutputStream fos = Files.newOutputStream(file.toPath())) {
779 bos.writeTo(fos);
780 IOUtils.copyLarge(is, fos);
781 }
782 return new DownloadedContent.OnFile(file, true);
783 }
784 }
785 }
786 catch (final ConnectionClosedException e) {
787 LOG.warn("Connection was closed while reading from stream.", e);
788 return new DownloadedContent.InMemory(bos.toByteArray());
789 }
790 catch (final EOFException e) {
791
792 LOG.warn("EOFException while reading from stream.", e);
793 return new DownloadedContent.InMemory(bos.toByteArray());
794 }
795
796 return new DownloadedContent.InMemory(bos.toByteArray());
797 }
798 }
799
800
801
802
803
804
805
806
807
808 protected WebResponse newWebResponseInstance(
809 final WebResponseData responseData,
810 final long loadTime,
811 final WebRequest webRequest) {
812 return new WebResponse(responseData, webRequest, loadTime);
813 }
814
815
816
817
818
819
820
821
822 private static String computeSecFetchMode(final WebRequest webRequest) {
823 final WebRequest.FetchMode override = webRequest.getFetchModeOverride();
824 if (override != null) {
825 return override.getValue();
826 }
827 return defaultFetchModeFor(webRequest.getFetchDestination()).getValue();
828 }
829
830
831
832
833
834
835
836 private static WebRequest.FetchMode defaultFetchModeFor(final WebRequest.FetchDestination destination) {
837 switch (destination) {
838 case DOCUMENT:
839 case IFRAME:
840 case FRAME:
841 return WebRequest.FetchMode.NAVIGATE;
842
843
844
845
846 case FONT:
847 case MANIFEST:
848 case EMPTY:
849 return WebRequest.FetchMode.CORS;
850
851
852 case WORKER:
853 case SHARED_WORKER:
854 case SERVICE_WORKER:
855 return WebRequest.FetchMode.SAME_ORIGIN;
856
857 case WEBSOCKET:
858 return WebRequest.FetchMode.WEBSOCKET;
859
860
861 default:
862 return WebRequest.FetchMode.NO_CORS;
863 }
864 }
865
866
867
868
869
870
871
872
873
874 private static String computeSecFetchSite(final WebRequest webRequest, final URL targetUrl) {
875 final URL requestingUrl = webRequest.getRequestingUrl();
876 if (requestingUrl == null) {
877
878 return "none";
879 }
880 if (isSameOrigin(requestingUrl, targetUrl)) {
881 return "same-origin";
882 }
883 if (isSameSite(requestingUrl, targetUrl)) {
884 return "same-site";
885 }
886 return "cross-site";
887 }
888
889 private static boolean isSameOrigin(final URL a, final URL b) {
890 return a.getProtocol().equals(b.getProtocol())
891 && a.getHost().equalsIgnoreCase(b.getHost())
892 && effectivePort(a) == effectivePort(b);
893 }
894
895 private static int effectivePort(final URL url) {
896 final int port = url.getPort();
897 return port == -1 ? url.getDefaultPort() : port;
898 }
899
900
901
902
903
904
905
906
907
908 private static boolean isSameSite(final URL a, final URL b) {
909 if (!a.getProtocol().equals(b.getProtocol())) {
910 return false;
911 }
912
913 final String registrableDomainA = HttpClientConverter.registrableDomain(a.getHost());
914 final String registrableDomainB = HttpClientConverter.registrableDomain(b.getHost());
915
916
917 return registrableDomainA.equals(registrableDomainB);
918 }
919
920
921
922
923
924
925
926
927
928 private static boolean isPotentiallyTrustworthy(final URL url) {
929 final String protocol = url.getProtocol();
930 if ("https".equals(protocol) || "wss".equals(protocol) || "file".equals(protocol)) {
931 return true;
932 }
933 if (!"http".equals(protocol) && !"ws".equals(protocol)) {
934
935
936 return false;
937 }
938
939 final String host = url.getHost();
940 return "localhost".equalsIgnoreCase(host)
941 || host.toLowerCase(java.util.Locale.ROOT).endsWith(".localhost")
942 || "127.0.0.1".equals(host)
943 || host.startsWith("127.")
944 || "::1".equals(host)
945 || "[::1]".equals(host);
946 }
947
948 private List<HttpRequestInterceptor> getHttpRequestInterceptors(final WebRequest webRequest) {
949 final List<HttpRequestInterceptor> list = new ArrayList<>();
950 final Map<String, String> requestHeaders = new HashMap<>(webRequest.getAdditionalHeaders());
951 final URL url = webRequest.getUrl();
952 final StringBuilder host = new StringBuilder(url.getHost());
953
954 final BrowserVersion browserVersion = webClient_.getBrowserVersion();
955
956 final int port = url.getPort();
957 if (port > 0 && port != url.getDefaultPort()) {
958 host.append(':').append(port);
959 }
960
961
962
963
964 final boolean isSecureContext = isPotentiallyTrustworthy(url);
965
966
967
968
969 final String secFetchMode = computeSecFetchMode(webRequest);
970
971
972 final String[] headerNames = browserVersion.getHeaderNamesOrdered();
973 for (final String header : headerNames) {
974 if (HttpHeader.HOST.equals(header)) {
975 list.add(new HostHeaderHttpRequestInterceptor(host.toString()));
976 }
977 else if (HttpHeader.USER_AGENT.equals(header)) {
978 String headerValue = webRequest.getAdditionalHeader(HttpHeader.USER_AGENT);
979 if (headerValue == null) {
980 headerValue = browserVersion.getUserAgent();
981 }
982 list.add(new UserAgentHeaderHttpRequestInterceptor(headerValue));
983 requestHeaders.remove(HttpHeader.USER_AGENT);
984 }
985 else if (HttpHeader.ACCEPT.equals(header)) {
986 final String headerValue = webRequest.getAdditionalHeader(HttpHeader.ACCEPT);
987 if (headerValue != null) {
988 list.add(new AcceptHeaderHttpRequestInterceptor(headerValue));
989 requestHeaders.remove(HttpHeader.ACCEPT);
990 }
991 }
992 else if (HttpHeader.ACCEPT_LANGUAGE.equals(header)) {
993 final String headerValue = webRequest.getAdditionalHeader(HttpHeader.ACCEPT_LANGUAGE);
994 if (headerValue != null) {
995 list.add(new AcceptLanguageHeaderHttpRequestInterceptor(headerValue));
996 requestHeaders.remove(HttpHeader.ACCEPT_LANGUAGE);
997 }
998 }
999 else if (HttpHeader.ACCEPT_ENCODING.equals(header)) {
1000 final String headerValue = webRequest.getAdditionalHeader(HttpHeader.ACCEPT_ENCODING);
1001 if (headerValue != null) {
1002 list.add(new AcceptEncodingHeaderHttpRequestInterceptor(headerValue));
1003 requestHeaders.remove(HttpHeader.ACCEPT_ENCODING);
1004 }
1005 }
1006 else if (HttpHeader.SEC_FETCH_DEST.equals(header)) {
1007 final String headerValue = webRequest.getAdditionalHeader(HttpHeader.SEC_FETCH_DEST);
1008 if (headerValue != null) {
1009 list.add(new SecFetchDestHeaderHttpRequestInterceptor(headerValue));
1010 requestHeaders.remove(HttpHeader.SEC_FETCH_DEST);
1011 }
1012 else if (isSecureContext) {
1013 list.add(new SecFetchDestHeaderHttpRequestInterceptor(webRequest.getFetchDestination().getValue()));
1014 }
1015 }
1016 else if (HttpHeader.SEC_FETCH_MODE.equals(header)) {
1017 final String headerValue = webRequest.getAdditionalHeader(HttpHeader.SEC_FETCH_MODE);
1018 if (headerValue != null) {
1019 list.add(new SecFetchModeHeaderHttpRequestInterceptor(headerValue));
1020 requestHeaders.remove(HttpHeader.SEC_FETCH_MODE);
1021 }
1022 else if (isSecureContext) {
1023 list.add(new SecFetchModeHeaderHttpRequestInterceptor(secFetchMode));
1024 }
1025 }
1026 else if (HttpHeader.SEC_FETCH_SITE.equals(header)) {
1027 final String headerValue = webRequest.getAdditionalHeader(HttpHeader.SEC_FETCH_SITE);
1028 if (headerValue != null) {
1029 list.add(new SecFetchSiteHeaderHttpRequestInterceptor(headerValue));
1030 requestHeaders.remove(HttpHeader.SEC_FETCH_SITE);
1031 }
1032 else if (isSecureContext) {
1033 list.add(new SecFetchSiteHeaderHttpRequestInterceptor(computeSecFetchSite(webRequest, url)));
1034 }
1035 }
1036 else if (HttpHeader.SEC_FETCH_USER.equals(header)) {
1037 final String headerValue = webRequest.getAdditionalHeader(HttpHeader.SEC_FETCH_USER);
1038 if (headerValue != null) {
1039 list.add(new SecFetchUserHeaderHttpRequestInterceptor(headerValue));
1040 requestHeaders.remove(HttpHeader.SEC_FETCH_USER);
1041 }
1042
1043
1044
1045 else if (isSecureContext
1046 && WebRequest.FetchMode.NAVIGATE.getValue().equals(secFetchMode)
1047 && webRequest.isUserActivation()) {
1048 list.add(new SecFetchUserHeaderHttpRequestInterceptor("?1"));
1049 }
1050 }
1051 else if (HttpHeader.SEC_CH_UA.equals(header)) {
1052
1053
1054 if (isSecureContext) {
1055 String headerValue = webRequest.getAdditionalHeader(HttpHeader.SEC_CH_UA);
1056 if (headerValue != null) {
1057 list.add(new SecClientHintUserAgentHeaderHttpRequestInterceptor(headerValue));
1058 requestHeaders.remove(HttpHeader.SEC_CH_UA);
1059 }
1060 else {
1061 if (browserVersion.hasFeature(HTTP_HEADER_CH_UA)) {
1062 headerValue = browserVersion.getSecClientHintUserAgentHeader();
1063 list.add(new SecClientHintUserAgentHeaderHttpRequestInterceptor(headerValue));
1064 }
1065 }
1066 }
1067 }
1068 else if (HttpHeader.SEC_CH_UA_MOBILE.equals(header)) {
1069 if (isSecureContext) {
1070 final String headerValue = webRequest.getAdditionalHeader(HttpHeader.SEC_CH_UA_MOBILE);
1071 if (headerValue != null) {
1072 list.add(new SecClientHintUserAgentMobileHeaderHttpRequestInterceptor(headerValue));
1073 requestHeaders.remove(HttpHeader.SEC_CH_UA_MOBILE);
1074 }
1075 else {
1076 if (browserVersion.hasFeature(HTTP_HEADER_CH_UA)) {
1077 list.add(new SecClientHintUserAgentMobileHeaderHttpRequestInterceptor("?0"));
1078 }
1079 }
1080 }
1081 }
1082 else if (HttpHeader.SEC_CH_UA_PLATFORM.equals(header)) {
1083 if (isSecureContext) {
1084 String headerValue = webRequest.getAdditionalHeader(HttpHeader.SEC_CH_UA_PLATFORM);
1085 if (headerValue != null) {
1086 list.add(new SecClientHintUserAgentPlatformHeaderHttpRequestInterceptor(headerValue));
1087 requestHeaders.remove(HttpHeader.SEC_CH_UA_PLATFORM);
1088 }
1089 else {
1090 if (browserVersion.hasFeature(HTTP_HEADER_CH_UA)) {
1091 headerValue = browserVersion.getSecClientHintUserAgentPlatformHeader();
1092 list.add(new SecClientHintUserAgentPlatformHeaderHttpRequestInterceptor(headerValue));
1093 }
1094 }
1095 }
1096 }
1097 else if (HttpHeader.PRIORITY.equals(header)) {
1098 final String headerValue = webRequest.getAdditionalHeader(HttpHeader.PRIORITY);
1099 if (headerValue != null) {
1100 list.add(new PriorityHeaderHttpRequestInterceptor(headerValue));
1101 requestHeaders.remove(HttpHeader.PRIORITY);
1102 }
1103 else {
1104 if (browserVersion.hasFeature(HTTP_HEADER_PRIORITY)) {
1105 list.add(new PriorityHeaderHttpRequestInterceptor("u=0, i"));
1106 }
1107 }
1108 }
1109 else if (HttpHeader.UPGRADE_INSECURE_REQUESTS.equals(header)) {
1110 final String headerValue = webRequest.getAdditionalHeader(HttpHeader.UPGRADE_INSECURE_REQUESTS);
1111 if (headerValue != null) {
1112 list.add(new UpgradeInsecureRequestHeaderHttpRequestInterceptor(headerValue));
1113 requestHeaders.remove(HttpHeader.UPGRADE_INSECURE_REQUESTS);
1114 }
1115
1116
1117
1118 else if (WebRequest.FetchMode.NAVIGATE.getValue().equals(secFetchMode)
1119 && HttpMethod.OPTIONS != webRequest.getHttpMethod()) {
1120 list.add(new UpgradeInsecureRequestHeaderHttpRequestInterceptor("1"));
1121 }
1122 }
1123 else if (HttpHeader.REFERER.equals(header)) {
1124 final String headerValue = webRequest.getAdditionalHeader(HttpHeader.REFERER);
1125 if (headerValue != null) {
1126 list.add(new RefererHeaderHttpRequestInterceptor(headerValue));
1127 requestHeaders.remove(HttpHeader.REFERER);
1128 }
1129 }
1130 else if (HttpHeader.CONNECTION.equals(header)) {
1131 list.add(new RequestClientConnControl());
1132 }
1133 else if (HttpHeader.COOKIE.equals(header)) {
1134 if (!webRequest.hasHint(HttpHint.BlockCookies)) {
1135 list.add(new RequestAddCookies());
1136 }
1137 }
1138 else if (HttpHeader.DNT.equals(header) && webClient_.getOptions().isDoNotTrackEnabled()) {
1139 list.add(new DntHeaderHttpRequestInterceptor("1"));
1140 }
1141 }
1142
1143
1144
1145 if (webClient_.getOptions().isDoNotTrackEnabled()) {
1146 list.add(new DntHeaderHttpRequestInterceptor("1"));
1147 }
1148
1149 synchronized (requestHeaders) {
1150 list.add(new MultiHttpRequestInterceptor(new HashMap<>(requestHeaders)));
1151 }
1152 return list;
1153 }
1154
1155
1156 private static final class HostHeaderHttpRequestInterceptor implements HttpRequestInterceptor {
1157 private final String value_;
1158
1159 HostHeaderHttpRequestInterceptor(final String value) {
1160 value_ = value;
1161 }
1162
1163 @Override
1164 public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException {
1165 request.setHeader(HttpHeader.HOST, value_);
1166 }
1167 }
1168
1169 private static final class UserAgentHeaderHttpRequestInterceptor implements HttpRequestInterceptor {
1170 private final String value_;
1171
1172 UserAgentHeaderHttpRequestInterceptor(final String value) {
1173 value_ = value;
1174 }
1175
1176 @Override
1177 public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException {
1178 request.setHeader(HttpHeader.USER_AGENT, value_);
1179 }
1180 }
1181
1182 private static final class AcceptHeaderHttpRequestInterceptor implements HttpRequestInterceptor {
1183 private final String value_;
1184
1185 AcceptHeaderHttpRequestInterceptor(final String value) {
1186 value_ = value;
1187 }
1188
1189 @Override
1190 public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException {
1191 request.setHeader(HttpHeader.ACCEPT, value_);
1192 }
1193 }
1194
1195 private static final class AcceptLanguageHeaderHttpRequestInterceptor implements HttpRequestInterceptor {
1196 private final String value_;
1197
1198 AcceptLanguageHeaderHttpRequestInterceptor(final String value) {
1199 value_ = value;
1200 }
1201
1202 @Override
1203 public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException {
1204 request.setHeader(HttpHeader.ACCEPT_LANGUAGE, value_);
1205 }
1206 }
1207
1208 private static final class UpgradeInsecureRequestHeaderHttpRequestInterceptor implements HttpRequestInterceptor {
1209 private final String value_;
1210
1211 UpgradeInsecureRequestHeaderHttpRequestInterceptor(final String value) {
1212 value_ = value;
1213 }
1214
1215 @Override
1216 public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException {
1217 request.setHeader(HttpHeader.UPGRADE_INSECURE_REQUESTS, value_);
1218 }
1219 }
1220
1221 private static final class AcceptEncodingHeaderHttpRequestInterceptor implements HttpRequestInterceptor {
1222 private final String value_;
1223
1224 AcceptEncodingHeaderHttpRequestInterceptor(final String value) {
1225 value_ = value;
1226 }
1227
1228 @Override
1229 public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException {
1230 request.setHeader("Accept-Encoding", value_);
1231 }
1232 }
1233
1234 private static final class RefererHeaderHttpRequestInterceptor implements HttpRequestInterceptor {
1235 private final String value_;
1236
1237 RefererHeaderHttpRequestInterceptor(final String value) {
1238 value_ = value;
1239 }
1240
1241 @Override
1242 public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException {
1243 request.setHeader(HttpHeader.REFERER, value_);
1244 }
1245 }
1246
1247 private static final class DntHeaderHttpRequestInterceptor implements HttpRequestInterceptor {
1248 private final String value_;
1249
1250 DntHeaderHttpRequestInterceptor(final String value) {
1251 value_ = value;
1252 }
1253
1254 @Override
1255 public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException {
1256 request.setHeader(HttpHeader.DNT, value_);
1257 }
1258 }
1259
1260 private static final class SecFetchModeHeaderHttpRequestInterceptor implements HttpRequestInterceptor {
1261 private final String value_;
1262
1263 SecFetchModeHeaderHttpRequestInterceptor(final String value) {
1264 value_ = value;
1265 }
1266
1267 @Override
1268 public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException {
1269 request.setHeader(HttpHeader.SEC_FETCH_MODE, value_);
1270 }
1271 }
1272
1273 private static final class SecFetchSiteHeaderHttpRequestInterceptor implements HttpRequestInterceptor {
1274 private final String value_;
1275
1276 SecFetchSiteHeaderHttpRequestInterceptor(final String value) {
1277 value_ = value;
1278 }
1279
1280 @Override
1281 public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException {
1282 request.setHeader(HttpHeader.SEC_FETCH_SITE, value_);
1283 }
1284 }
1285
1286 private static final class SecFetchUserHeaderHttpRequestInterceptor implements HttpRequestInterceptor {
1287 private final String value_;
1288
1289 SecFetchUserHeaderHttpRequestInterceptor(final String value) {
1290 value_ = value;
1291 }
1292
1293 @Override
1294 public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException {
1295 request.setHeader(HttpHeader.SEC_FETCH_USER, value_);
1296 }
1297 }
1298
1299 private static final class SecFetchDestHeaderHttpRequestInterceptor implements HttpRequestInterceptor {
1300 private final String value_;
1301
1302 SecFetchDestHeaderHttpRequestInterceptor(final String value) {
1303 value_ = value;
1304 }
1305
1306 @Override
1307 public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException {
1308 request.setHeader(HttpHeader.SEC_FETCH_DEST, value_);
1309 }
1310 }
1311
1312 private static final class SecClientHintUserAgentHeaderHttpRequestInterceptor implements HttpRequestInterceptor {
1313 private final String value_;
1314
1315 SecClientHintUserAgentHeaderHttpRequestInterceptor(final String value) {
1316 value_ = value;
1317 }
1318
1319 @Override
1320 public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException {
1321 request.setHeader(HttpHeader.SEC_CH_UA, value_);
1322 }
1323 }
1324
1325 private static final class SecClientHintUserAgentMobileHeaderHttpRequestInterceptor
1326 implements HttpRequestInterceptor {
1327 private final String value_;
1328
1329 SecClientHintUserAgentMobileHeaderHttpRequestInterceptor(final String value) {
1330 value_ = value;
1331 }
1332
1333 @Override
1334 public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException {
1335 request.setHeader(HttpHeader.SEC_CH_UA_MOBILE, value_);
1336 }
1337 }
1338
1339 private static final class SecClientHintUserAgentPlatformHeaderHttpRequestInterceptor
1340 implements HttpRequestInterceptor {
1341 private final String value_;
1342
1343 SecClientHintUserAgentPlatformHeaderHttpRequestInterceptor(final String value) {
1344 value_ = value;
1345 }
1346
1347 @Override
1348 public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException {
1349 request.setHeader(HttpHeader.SEC_CH_UA_PLATFORM, value_);
1350 }
1351 }
1352
1353 private static final class PriorityHeaderHttpRequestInterceptor
1354 implements HttpRequestInterceptor {
1355 private final String value_;
1356
1357 PriorityHeaderHttpRequestInterceptor(final String value) {
1358 value_ = value;
1359 }
1360
1361 @Override
1362 public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException {
1363 request.setHeader(HttpHeader.PRIORITY, value_);
1364 }
1365 }
1366
1367 private static class MultiHttpRequestInterceptor implements HttpRequestInterceptor {
1368 private final Map<String, String> map_;
1369
1370 MultiHttpRequestInterceptor(final Map<String, String> map) {
1371 map_ = map;
1372 }
1373
1374 @Override
1375 public void process(final HttpRequest request, final HttpContext context)
1376 throws HttpException, IOException {
1377 for (final Map.Entry<String, String> entry : map_.entrySet()) {
1378 request.setHeader(entry.getKey(), entry.getValue());
1379 }
1380 }
1381 }
1382
1383 private static class RequestClientConnControl implements HttpRequestInterceptor {
1384
1385 private static final String PROXY_CONN_DIRECTIVE = "Proxy-Connection";
1386 private static final String CONN_DIRECTIVE = "Connection";
1387 private static final String CONN_KEEP_ALIVE = "keep-alive";
1388
1389
1390
1391
1392 RequestClientConnControl() {
1393 super();
1394 }
1395
1396 @Override
1397 public void process(final HttpRequest request, final HttpContext context)
1398 throws HttpException, IOException {
1399 final String method = request.getRequestLine().getMethod();
1400 if ("CONNECT".equalsIgnoreCase(method)) {
1401 request.setHeader(PROXY_CONN_DIRECTIVE, CONN_KEEP_ALIVE);
1402 return;
1403 }
1404
1405 final HttpClientContext clientContext = HttpClientContext.adapt(context);
1406
1407
1408 final RouteInfo route = clientContext.getHttpRoute();
1409 if (route == null) {
1410 return;
1411 }
1412
1413 if ((route.getHopCount() == 1 || route.isTunnelled())
1414 && !request.containsHeader(CONN_DIRECTIVE)) {
1415 request.addHeader(CONN_DIRECTIVE, CONN_KEEP_ALIVE);
1416 }
1417 if (route.getHopCount() == 2
1418 && !route.isTunnelled()
1419 && !request.containsHeader(PROXY_CONN_DIRECTIVE)) {
1420 request.addHeader(PROXY_CONN_DIRECTIVE, CONN_KEEP_ALIVE);
1421 }
1422 }
1423 }
1424
1425
1426
1427
1428 private static final class SynchronizedAuthCache extends BasicAuthCache {
1429
1430
1431
1432
1433 SynchronizedAuthCache() {
1434 super();
1435 }
1436
1437
1438
1439
1440 @Override
1441 public synchronized void put(final HttpHost host, final AuthScheme authScheme) {
1442 super.put(host, authScheme);
1443 }
1444
1445
1446
1447
1448 @Override
1449 public synchronized AuthScheme get(final HttpHost host) {
1450 return super.get(host);
1451 }
1452
1453
1454
1455
1456 @Override
1457 public synchronized void remove(final HttpHost host) {
1458 super.remove(host);
1459 }
1460
1461
1462
1463
1464 @Override
1465 public synchronized void clear() {
1466 super.clear();
1467 }
1468
1469
1470
1471
1472 @Override
1473 public synchronized String toString() {
1474 return super.toString();
1475 }
1476 }
1477
1478
1479
1480
1481 @Override
1482 public void close() {
1483 synchronized (httpClientBuilder_) {
1484 httpClientBuilder_.clear();
1485 }
1486 sharedAuthCache_.clear();
1487 httpClientContextByThread_.clear();
1488
1489 if (connectionManager_ != null) {
1490 connectionManager_.shutdown();
1491 connectionManager_ = null;
1492 }
1493 }
1494
1495
1496
1497
1498
1499 private static PoolingHttpClientConnectionManager createConnectionManager(final HttpClientBuilder builder) {
1500 try {
1501 PublicSuffixMatcher publicSuffixMatcher = getField(builder, "publicSuffixMatcher");
1502 if (publicSuffixMatcher == null) {
1503 publicSuffixMatcher = PublicSuffixMatcherLoader.getDefault();
1504 }
1505
1506 LayeredConnectionSocketFactory sslSocketFactory = getField(builder, "sslSocketFactory");
1507 final SocketConfig defaultSocketConfig = getField(builder, "defaultSocketConfig");
1508 final ConnectionConfig defaultConnectionConfig = getField(builder, "defaultConnectionConfig");
1509 final boolean systemProperties = getField(builder, "systemProperties");
1510 final int maxConnTotal = getField(builder, "maxConnTotal");
1511 final int maxConnPerRoute = getField(builder, "maxConnPerRoute");
1512 HostnameVerifier hostnameVerifier = getField(builder, "hostnameVerifier");
1513 final SSLContext sslcontext = getField(builder, "sslContext");
1514 final DnsResolver dnsResolver = getField(builder, "dnsResolver");
1515 final long connTimeToLive = getField(builder, "connTimeToLive");
1516 final TimeUnit connTimeToLiveTimeUnit = getField(builder, "connTimeToLiveTimeUnit");
1517
1518 if (sslSocketFactory == null) {
1519 final String[] supportedProtocols = systemProperties
1520 ? split(System.getProperty("https.protocols")) : null;
1521 final String[] supportedCipherSuites = systemProperties
1522 ? split(System.getProperty("https.cipherSuites")) : null;
1523 if (hostnameVerifier == null) {
1524 hostnameVerifier = new DefaultHostnameVerifier(publicSuffixMatcher);
1525 }
1526 if (sslcontext == null) {
1527 if (systemProperties) {
1528 sslSocketFactory = new SSLConnectionSocketFactory(
1529 (SSLSocketFactory) SSLSocketFactory.getDefault(),
1530 supportedProtocols, supportedCipherSuites, hostnameVerifier);
1531 }
1532 else {
1533 sslSocketFactory = new SSLConnectionSocketFactory(
1534 SSLContexts.createDefault(),
1535 hostnameVerifier);
1536 }
1537 }
1538 else {
1539 sslSocketFactory = new SSLConnectionSocketFactory(
1540 sslcontext, supportedProtocols, supportedCipherSuites, hostnameVerifier);
1541 }
1542 }
1543
1544 final PoolingHttpClientConnectionManager poolingmgr = new PoolingHttpClientConnectionManager(
1545 RegistryBuilder.<ConnectionSocketFactory>create()
1546 .register("http", new SocksConnectionSocketFactory())
1547 .register("https", sslSocketFactory)
1548 .build(),
1549 null,
1550 null,
1551 dnsResolver,
1552 connTimeToLive,
1553 connTimeToLiveTimeUnit != null ? connTimeToLiveTimeUnit : TimeUnit.MILLISECONDS);
1554 if (defaultSocketConfig != null) {
1555 poolingmgr.setDefaultSocketConfig(defaultSocketConfig);
1556 }
1557 if (defaultConnectionConfig != null) {
1558 poolingmgr.setDefaultConnectionConfig(defaultConnectionConfig);
1559 }
1560 if (systemProperties) {
1561 String s = System.getProperty("http.keepAlive", "true");
1562 if ("true".equalsIgnoreCase(s)) {
1563 s = System.getProperty("http.maxConnections", "5");
1564 final int max = Integer.parseInt(s);
1565 poolingmgr.setDefaultMaxPerRoute(max);
1566 poolingmgr.setMaxTotal(2 * max);
1567 }
1568 }
1569 if (maxConnTotal > 0) {
1570 poolingmgr.setMaxTotal(maxConnTotal);
1571 }
1572 if (maxConnPerRoute > 0) {
1573 poolingmgr.setDefaultMaxPerRoute(maxConnPerRoute);
1574 }
1575 return poolingmgr;
1576 }
1577 catch (final IllegalAccessException e) {
1578 throw new RuntimeException(e);
1579 }
1580 }
1581
1582 private static String[] split(final String s) {
1583 if (TextUtils.isBlank(s)) {
1584 return null;
1585 }
1586 return s.split(" *, *");
1587 }
1588
1589 @SuppressWarnings("unchecked")
1590 private static <T> T getField(final Object target, final String fieldName) throws IllegalAccessException {
1591 return (T) FieldUtils.readDeclaredField(target, fieldName, true);
1592 }
1593 }