View Javadoc
1   /*
2    * Copyright (c) 2002-2026 Gargoyle Software Inc.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    * https://www.apache.org/licenses/LICENSE-2.0
8    *
9    * Unless required by applicable law or agreed to in writing, software
10   * distributed under the License is distributed on an "AS IS" BASIS,
11   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12   * See the License for the specific language governing permissions and
13   * limitations under the License.
14   */
15  package org.htmlunit;
16  
17  import static java.nio.charset.StandardCharsets.UTF_16BE;
18  import static java.nio.charset.StandardCharsets.UTF_16LE;
19  import static java.nio.charset.StandardCharsets.UTF_8;
20  
21  import java.io.IOException;
22  import java.io.InputStream;
23  import java.io.Serializable;
24  import java.net.URL;
25  import java.nio.charset.Charset;
26  import java.util.List;
27  
28  import org.apache.commons.io.ByteOrderMark;
29  import org.apache.commons.io.IOUtils;
30  import org.apache.commons.io.input.BOMInputStream;
31  import org.apache.commons.logging.Log;
32  import org.apache.commons.logging.LogFactory;
33  import org.htmlunit.http.HttpStatus;
34  import org.htmlunit.util.EncodingSniffer;
35  import org.htmlunit.util.MimeType;
36  import org.htmlunit.util.NameValuePair;
37  import org.htmlunit.util.StringUtils;
38  
39  /**
40   * A response from a web server.
41   *
42   * @author Mike Bowler
43   * @author Brad Clarke
44   * @author Noboru Sinohara
45   * @author Marc Guillemot
46   * @author Ahmed Ashour
47   * @author Ronald Brill
48   * @author Lai Quang Duong
49   */
50  public class WebResponse implements Serializable {
51  
52      private static final Log LOG = LogFactory.getLog(WebResponse.class);
53      private static final ByteOrderMark[] BOM_HEADERS = {
54          ByteOrderMark.UTF_8,
55          ByteOrderMark.UTF_16LE,
56          ByteOrderMark.UTF_16BE};
57  
58      private final long loadTime_;
59      private final WebResponseData responseData_;
60      private final WebRequest request_;
61      private boolean wasContentCharsetTentative_;
62      private boolean wasBlocked_;
63      private String blockReason_;
64  
65      /**
66       * Constructs a web response.
67       *
68       * @param responseData      the response data
69       * @param url               Where this response came from
70       * @param requestMethod     the method used to get this response
71       * @param loadTime          How long the response took to be sent
72       */
73      public WebResponse(final WebResponseData responseData, final URL url,
74              final HttpMethod requestMethod, final long loadTime) {
75          this(responseData, new WebRequest(url, requestMethod), loadTime);
76      }
77  
78      /**
79       * Constructs a web response.
80       *
81       * @param responseData      the response data
82       * @param request           the request used to get this response
83       * @param loadTime          How long the response took to be sent
84       */
85      public WebResponse(final WebResponseData responseData,
86              final WebRequest request, final long loadTime) {
87          responseData_ = responseData;
88          request_ = request;
89          loadTime_ = loadTime;
90      }
91  
92      /**
93       * Returns the request used to load this response.
94       *
95       * @return the associated {@link WebRequest}
96       */
97      public WebRequest getWebRequest() {
98          return request_;
99      }
100 
101     /**
102      * Returns the response headers.
103      * @return the response headers as a list of {@link NameValuePair}s
104      */
105     public List<NameValuePair> getResponseHeaders() {
106         return responseData_.getResponseHeaders();
107     }
108 
109     /**
110      * Returns the value of the specified response header.
111      * @param headerName the name of the header whose value is to be returned
112      * @return the header value, {@code null} if no response header exists with this name
113      */
114     public String getResponseHeaderValue(final String headerName) {
115         for (final NameValuePair pair : responseData_.getResponseHeaders()) {
116             if (pair.getName().equalsIgnoreCase(headerName)) {
117                 return pair.getValue();
118             }
119         }
120         return null;
121     }
122 
123     /**
124      * Returns the HTTP status code.
125      * @return the status code that was returned by the server
126      */
127     public int getStatusCode() {
128         return responseData_.getStatusCode();
129     }
130 
131     /**
132      * Returns the HTTP status message.
133      * @return the status message that was returned from the server
134      */
135     public String getStatusMessage() {
136         return responseData_.getStatusMessage();
137     }
138 
139     /**
140      * Returns the response content type.
141      * @return the content type, or an empty string if the {@code Content-Type}
142      *         header is absent
143      */
144     public String getContentType() {
145         final String contentTypeHeader = getResponseHeaderValue(HttpHeader.CONTENT_TYPE_LC);
146         if (contentTypeHeader == null) {
147             // Not technically legal but some servers don't return a content-type
148             return "";
149         }
150         final int index = contentTypeHeader.indexOf(';');
151         if (index == -1) {
152             return contentTypeHeader;
153         }
154         return contentTypeHeader.substring(0, index);
155     }
156 
157     /**
158      * Returns the content charset specified explicitly in the {@code Content-Type} header
159      * or {@code null} if none was specified.
160      * @return the content charset specified in the header or {@code null} if none was specified
161      */
162     public Charset getHeaderContentCharset() {
163         final String contentType = getResponseHeaderValue(HttpHeader.CONTENT_TYPE_LC);
164         if (contentType == null) {
165             return null;
166         }
167 
168         final int index = contentType.indexOf(';');
169         if (index == -1 || index == 0) {
170             return null;
171         }
172         if (StringUtils.isBlank(contentType.substring(0, index))) {
173             return null;
174         }
175 
176         return EncodingSniffer.extractEncodingFromContentType(contentType);
177     }
178 
179     /**
180      * Returns the content charset for this response, even if no charset was specified explicitly.
181      * <p>
182      * This method always returns a valid charset. This method first checks the {@code Content-Type}
183      * header or the content BOM for a viable charset. If not found, it attempts to determine the
184      * charset based on the type of the content. As a last resort, this method returns the
185      * value of {@link org.htmlunit.WebRequest#getDefaultResponseContentCharset()} which is
186      * {@link java.nio.charset.StandardCharsets#UTF_8} by default.
187      * </p>
188      * @return the content charset for this response
189      */
190     public Charset getContentCharset() {
191         wasContentCharsetTentative_ = false;
192 
193         try (InputStream is = getContentAsStreamWithBomIfApplicable()) {
194             if (is == null) {
195                 return getWebRequest().getDefaultResponseContentCharset();
196             }
197 
198             if (is instanceof BOMInputStream stream) {
199                 final String bomCharsetName = stream.getBOMCharsetName();
200                 if (bomCharsetName != null) {
201                     return Charset.forName(bomCharsetName);
202                 }
203             }
204 
205             Charset charset = getHeaderContentCharset();
206             if (charset != null) {
207                 return charset;
208             }
209 
210             final String contentType = getContentType();
211             switch (DefaultPageCreator.determinePageType(contentType)) {
212                 case HTML -> {
213                     charset = EncodingSniffer.sniffEncodingFromMetaTag(is);
214                     wasContentCharsetTentative_ = true;
215                 }
216                 case XML -> {
217                     charset = EncodingSniffer.sniffEncodingFromXmlDeclaration(is);
218                     if (charset == null) {
219                         charset = UTF_8;
220                     }
221                 }
222                 default -> {
223                     if (MimeType.TEXT_CSS.equals(contentType)) {
224                         charset = EncodingSniffer.sniffEncodingFromCssDeclaration(is);
225                     }
226                 }
227             }
228 
229             if (charset != null) {
230                 return charset;
231             }
232         }
233         catch (final IOException e) {
234             LOG.warn("Error trying to sniff encoding.", e);
235             wasContentCharsetTentative_ = true;
236         }
237         return getWebRequest().getDefaultResponseContentCharset();
238     }
239 
240     /**
241      * Returns whether the charset of the previous call to {@link #getContentCharset()} was "tentative".
242      * <p>
243      * A charset is classed as "tentative" if its detection is prone to false positive/negatives.
244      * </p>
245      * <p>
246      * For example, HTML meta-tag sniffing can be fooled by text that looks-like-a-meta-tag inside
247      * JavaScript code (false positive) or if the meta-tag is after the first 1024 bytes (false negative).
248      * </p>
249      * @return {@code true} if tentative; {@code false} if the charset was determined with confidence
250      *
251      * @see <a href="https://html.spec.whatwg.org/multipage/parsing.html#concept-encoding-confidence">
252      *     https://html.spec.whatwg.org/multipage/parsing.html#concept-encoding-confidence</a>
253      */
254     public boolean wasContentCharsetTentative() {
255         return wasContentCharsetTentative_;
256     }
257 
258     /**
259      * Returns the response content as a string, using the charset/encoding specified in the server response.
260      * @return the response content as a string, using the charset/encoding specified in the server response
261      *         or {@code null} if content retrieval failed
262      */
263     public String getContentAsString() {
264         return getContentAsString(getContentCharset());
265     }
266 
267     /**
268      * Returns the response content as a string, using the specified charset,
269      * rather than the charset/encoding specified in the server response.
270      * If there is a BOM header, the charset parameter will be overwritten by the BOM.
271      * @param encoding the charset/encoding to use to convert the response content into a string
272      * @return the response content as a string or {@code null} if content retrieval failed
273      */
274     public String getContentAsString(final Charset encoding) {
275         if (responseData_ != null) {
276             try (InputStream in = responseData_.getInputStreamWithBomIfApplicable(BOM_HEADERS)) {
277                 if (in instanceof BOMInputStream bomIn) {
278                     // there seems to be a bug in BOMInputStream
279                     // we have to call this before hasBOM(ByteOrderMark)
280                     if (bomIn.hasBOM()) {
281                         if (bomIn.hasBOM(ByteOrderMark.UTF_8)) {
282                             return IOUtils.toString(bomIn, UTF_8);
283                         }
284                         if (bomIn.hasBOM(ByteOrderMark.UTF_16BE)) {
285                             return IOUtils.toString(bomIn, UTF_16BE);
286                         }
287                         if (bomIn.hasBOM(ByteOrderMark.UTF_16LE)) {
288                             return IOUtils.toString(bomIn, UTF_16LE);
289                         }
290                     }
291                     return IOUtils.toString(bomIn, encoding);
292                 }
293 
294                 return IOUtils.toString(in, encoding);
295             }
296             catch (final IOException e) {
297                 LOG.warn(e.getMessage(), e);
298             }
299         }
300         return null;
301     }
302 
303     /**
304      * Returns the length of the content data.
305      * @return the length
306      */
307     public long getContentLength() {
308         if (responseData_ == null) {
309             return 0;
310         }
311         return responseData_.getContentLength();
312     }
313 
314     /**
315      * Returns the response content as an input stream.
316      * @return the response content as an input stream
317      * @throws IOException in case of I/O problems
318      */
319     public InputStream getContentAsStream() throws IOException {
320         return responseData_.getInputStream();
321     }
322 
323     /**
324      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
325      *
326      * Returns the response content as an {@link InputStream}, wrapped in a
327      * {@link BOMInputStream} so that any leading byte-order mark is detected
328      * and skipped automatically.
329      *
330      * @return the input stream, or {@code null} if no response data is available
331      * @throws IOException in case of I/O problems
332      */
333     public InputStream getContentAsStreamWithBomIfApplicable() throws IOException {
334         if (responseData_ != null) {
335             return responseData_.getInputStreamWithBomIfApplicable(BOM_HEADERS);
336         }
337         return null;
338     }
339 
340     /**
341      * Returns the time it took to load this web response, in milliseconds.
342      * @return the time it took to load this web response, in milliseconds
343      */
344     public long getLoadTime() {
345         return loadTime_;
346     }
347 
348     /**
349      * Clean up the response data.
350      */
351     public void cleanUp() {
352         if (responseData_ != null) {
353             responseData_.cleanUp();
354         }
355     }
356 
357     /**
358      * Returns whether the response has a successful HTTP status code.
359      *
360      * @return {@code true} if the status code is in the 2xx range
361      */
362     public boolean isSuccess() {
363         final int statusCode = getStatusCode();
364         return statusCode >= HttpStatus.OK_200 && statusCode < HttpStatus.MULTIPLE_CHOICES_300;
365     }
366 
367     /**
368      * Returns whether the response has a successful HTTP status code or a use-proxy redirection.
369      *
370      * @return {@code true} if the status code is in the 2xx range or 305
371      */
372     public boolean isSuccessOrUseProxy() {
373         final int statusCode = getStatusCode();
374         return (statusCode >= HttpStatus.OK_200 && statusCode < HttpStatus.MULTIPLE_CHOICES_300)
375                 || statusCode == HttpStatus.USE_PROXY_305;
376     }
377 
378     /**
379      * Returns whether the response has a successful HTTP status code, a proxy redirection, or a not-modified status.
380      *
381      * @return {@code true} if the status code is in the 2xx range, 304 (Not Modified), or 305 (Use Proxy)
382      */
383     public boolean isSuccessOrUseProxyOrNotModified() {
384         return isSuccessOrUseProxy() || getStatusCode() == HttpStatus.NOT_MODIFIED_304;
385     }
386 
387     /**
388      * Returns whether the request was blocked.
389      *
390      * @return {@code true} if the request was blocked
391      */
392     public boolean wasBlocked() {
393         return wasBlocked_;
394     }
395 
396     /**
397      * Returns the reason for blocking or null.
398      *
399      * @return the reason for blocking, or {@code null} if the request was not blocked
400      */
401     public String getBlockReason() {
402         return blockReason_;
403     }
404 
405     /**
406      * Sets the wasBlocked state to true.
407      *
408      * @param blockReason the reason
409      */
410     public void markAsBlocked(final String blockReason) {
411         wasBlocked_ = true;
412         blockReason_ = blockReason;
413     }
414 }