1 /*
2 * Copyright (c) 2002-2026 Gargoyle Software Inc.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 * https://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15 package org.htmlunit;
16
17 import static java.nio.charset.StandardCharsets.UTF_16BE;
18 import static java.nio.charset.StandardCharsets.UTF_16LE;
19 import static java.nio.charset.StandardCharsets.UTF_8;
20
21 import java.io.IOException;
22 import java.io.InputStream;
23 import java.io.Serializable;
24 import java.net.URL;
25 import java.nio.charset.Charset;
26 import java.util.List;
27
28 import org.apache.commons.io.ByteOrderMark;
29 import org.apache.commons.io.IOUtils;
30 import org.apache.commons.io.input.BOMInputStream;
31 import org.apache.commons.logging.Log;
32 import org.apache.commons.logging.LogFactory;
33 import org.htmlunit.http.HttpStatus;
34 import org.htmlunit.util.EncodingSniffer;
35 import org.htmlunit.util.MimeType;
36 import org.htmlunit.util.NameValuePair;
37 import org.htmlunit.util.StringUtils;
38
39 /**
40 * A response from a web server.
41 *
42 * @author Mike Bowler
43 * @author Brad Clarke
44 * @author Noboru Sinohara
45 * @author Marc Guillemot
46 * @author Ahmed Ashour
47 * @author Ronald Brill
48 * @author Lai Quang Duong
49 */
50 public class WebResponse implements Serializable {
51
52 private static final Log LOG = LogFactory.getLog(WebResponse.class);
53 private static final ByteOrderMark[] BOM_HEADERS = {
54 ByteOrderMark.UTF_8,
55 ByteOrderMark.UTF_16LE,
56 ByteOrderMark.UTF_16BE};
57
58 private final long loadTime_;
59 private final WebResponseData responseData_;
60 private final WebRequest request_;
61 private boolean wasContentCharsetTentative_;
62 private boolean wasBlocked_;
63 private String blockReason_;
64
65 /**
66 * Constructs a web response.
67 *
68 * @param responseData the response data
69 * @param url Where this response came from
70 * @param requestMethod the method used to get this response
71 * @param loadTime How long the response took to be sent
72 */
73 public WebResponse(final WebResponseData responseData, final URL url,
74 final HttpMethod requestMethod, final long loadTime) {
75 this(responseData, new WebRequest(url, requestMethod), loadTime);
76 }
77
78 /**
79 * Constructs a web response.
80 *
81 * @param responseData the response data
82 * @param request the request used to get this response
83 * @param loadTime How long the response took to be sent
84 */
85 public WebResponse(final WebResponseData responseData,
86 final WebRequest request, final long loadTime) {
87 responseData_ = responseData;
88 request_ = request;
89 loadTime_ = loadTime;
90 }
91
92 /**
93 * Returns the request used to load this response.
94 *
95 * @return the associated {@link WebRequest}
96 */
97 public WebRequest getWebRequest() {
98 return request_;
99 }
100
101 /**
102 * Returns the response headers.
103 * @return the response headers as a list of {@link NameValuePair}s
104 */
105 public List<NameValuePair> getResponseHeaders() {
106 return responseData_.getResponseHeaders();
107 }
108
109 /**
110 * Returns the value of the specified response header.
111 * @param headerName the name of the header whose value is to be returned
112 * @return the header value, {@code null} if no response header exists with this name
113 */
114 public String getResponseHeaderValue(final String headerName) {
115 for (final NameValuePair pair : responseData_.getResponseHeaders()) {
116 if (pair.getName().equalsIgnoreCase(headerName)) {
117 return pair.getValue();
118 }
119 }
120 return null;
121 }
122
123 /**
124 * Returns the HTTP status code.
125 * @return the status code that was returned by the server
126 */
127 public int getStatusCode() {
128 return responseData_.getStatusCode();
129 }
130
131 /**
132 * Returns the HTTP status message.
133 * @return the status message that was returned from the server
134 */
135 public String getStatusMessage() {
136 return responseData_.getStatusMessage();
137 }
138
139 /**
140 * Returns the response content type.
141 * @return the content type, or an empty string if the {@code Content-Type}
142 * header is absent
143 */
144 public String getContentType() {
145 final String contentTypeHeader = getResponseHeaderValue(HttpHeader.CONTENT_TYPE_LC);
146 if (contentTypeHeader == null) {
147 // Not technically legal but some servers don't return a content-type
148 return "";
149 }
150 final int index = contentTypeHeader.indexOf(';');
151 if (index == -1) {
152 return contentTypeHeader;
153 }
154 return contentTypeHeader.substring(0, index);
155 }
156
157 /**
158 * Returns the content charset specified explicitly in the {@code Content-Type} header
159 * or {@code null} if none was specified.
160 * @return the content charset specified in the header or {@code null} if none was specified
161 */
162 public Charset getHeaderContentCharset() {
163 final String contentType = getResponseHeaderValue(HttpHeader.CONTENT_TYPE_LC);
164 if (contentType == null) {
165 return null;
166 }
167
168 final int index = contentType.indexOf(';');
169 if (index == -1 || index == 0) {
170 return null;
171 }
172 if (StringUtils.isBlank(contentType.substring(0, index))) {
173 return null;
174 }
175
176 return EncodingSniffer.extractEncodingFromContentType(contentType);
177 }
178
179 /**
180 * Returns the content charset for this response, even if no charset was specified explicitly.
181 * <p>
182 * This method always returns a valid charset. This method first checks the {@code Content-Type}
183 * header or the content BOM for a viable charset. If not found, it attempts to determine the
184 * charset based on the type of the content. As a last resort, this method returns the
185 * value of {@link org.htmlunit.WebRequest#getDefaultResponseContentCharset()} which is
186 * {@link java.nio.charset.StandardCharsets#UTF_8} by default.
187 * </p>
188 * @return the content charset for this response
189 */
190 public Charset getContentCharset() {
191 wasContentCharsetTentative_ = false;
192
193 try (InputStream is = getContentAsStreamWithBomIfApplicable()) {
194 if (is == null) {
195 return getWebRequest().getDefaultResponseContentCharset();
196 }
197
198 if (is instanceof BOMInputStream stream) {
199 final String bomCharsetName = stream.getBOMCharsetName();
200 if (bomCharsetName != null) {
201 return Charset.forName(bomCharsetName);
202 }
203 }
204
205 Charset charset = getHeaderContentCharset();
206 if (charset != null) {
207 return charset;
208 }
209
210 final String contentType = getContentType();
211 switch (DefaultPageCreator.determinePageType(contentType)) {
212 case HTML -> {
213 charset = EncodingSniffer.sniffEncodingFromMetaTag(is);
214 wasContentCharsetTentative_ = true;
215 }
216 case XML -> {
217 charset = EncodingSniffer.sniffEncodingFromXmlDeclaration(is);
218 if (charset == null) {
219 charset = UTF_8;
220 }
221 }
222 default -> {
223 if (MimeType.TEXT_CSS.equals(contentType)) {
224 charset = EncodingSniffer.sniffEncodingFromCssDeclaration(is);
225 }
226 }
227 }
228
229 if (charset != null) {
230 return charset;
231 }
232 }
233 catch (final IOException e) {
234 LOG.warn("Error trying to sniff encoding.", e);
235 wasContentCharsetTentative_ = true;
236 }
237 return getWebRequest().getDefaultResponseContentCharset();
238 }
239
240 /**
241 * Returns whether the charset of the previous call to {@link #getContentCharset()} was "tentative".
242 * <p>
243 * A charset is classed as "tentative" if its detection is prone to false positive/negatives.
244 * </p>
245 * <p>
246 * For example, HTML meta-tag sniffing can be fooled by text that looks-like-a-meta-tag inside
247 * JavaScript code (false positive) or if the meta-tag is after the first 1024 bytes (false negative).
248 * </p>
249 * @return {@code true} if tentative; {@code false} if the charset was determined with confidence
250 *
251 * @see <a href="https://html.spec.whatwg.org/multipage/parsing.html#concept-encoding-confidence">
252 * https://html.spec.whatwg.org/multipage/parsing.html#concept-encoding-confidence</a>
253 */
254 public boolean wasContentCharsetTentative() {
255 return wasContentCharsetTentative_;
256 }
257
258 /**
259 * Returns the response content as a string, using the charset/encoding specified in the server response.
260 * @return the response content as a string, using the charset/encoding specified in the server response
261 * or {@code null} if content retrieval failed
262 */
263 public String getContentAsString() {
264 return getContentAsString(getContentCharset());
265 }
266
267 /**
268 * Returns the response content as a string, using the specified charset,
269 * rather than the charset/encoding specified in the server response.
270 * If there is a BOM header, the charset parameter will be overwritten by the BOM.
271 * @param encoding the charset/encoding to use to convert the response content into a string
272 * @return the response content as a string or {@code null} if content retrieval failed
273 */
274 public String getContentAsString(final Charset encoding) {
275 if (responseData_ != null) {
276 try (InputStream in = responseData_.getInputStreamWithBomIfApplicable(BOM_HEADERS)) {
277 if (in instanceof BOMInputStream bomIn) {
278 // there seems to be a bug in BOMInputStream
279 // we have to call this before hasBOM(ByteOrderMark)
280 if (bomIn.hasBOM()) {
281 if (bomIn.hasBOM(ByteOrderMark.UTF_8)) {
282 return IOUtils.toString(bomIn, UTF_8);
283 }
284 if (bomIn.hasBOM(ByteOrderMark.UTF_16BE)) {
285 return IOUtils.toString(bomIn, UTF_16BE);
286 }
287 if (bomIn.hasBOM(ByteOrderMark.UTF_16LE)) {
288 return IOUtils.toString(bomIn, UTF_16LE);
289 }
290 }
291 return IOUtils.toString(bomIn, encoding);
292 }
293
294 return IOUtils.toString(in, encoding);
295 }
296 catch (final IOException e) {
297 LOG.warn(e.getMessage(), e);
298 }
299 }
300 return null;
301 }
302
303 /**
304 * Returns the length of the content data.
305 * @return the length
306 */
307 public long getContentLength() {
308 if (responseData_ == null) {
309 return 0;
310 }
311 return responseData_.getContentLength();
312 }
313
314 /**
315 * Returns the response content as an input stream.
316 * @return the response content as an input stream
317 * @throws IOException in case of I/O problems
318 */
319 public InputStream getContentAsStream() throws IOException {
320 return responseData_.getInputStream();
321 }
322
323 /**
324 * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
325 *
326 * Returns the response content as an {@link InputStream}, wrapped in a
327 * {@link BOMInputStream} so that any leading byte-order mark is detected
328 * and skipped automatically.
329 *
330 * @return the input stream, or {@code null} if no response data is available
331 * @throws IOException in case of I/O problems
332 */
333 public InputStream getContentAsStreamWithBomIfApplicable() throws IOException {
334 if (responseData_ != null) {
335 return responseData_.getInputStreamWithBomIfApplicable(BOM_HEADERS);
336 }
337 return null;
338 }
339
340 /**
341 * Returns the time it took to load this web response, in milliseconds.
342 * @return the time it took to load this web response, in milliseconds
343 */
344 public long getLoadTime() {
345 return loadTime_;
346 }
347
348 /**
349 * Clean up the response data.
350 */
351 public void cleanUp() {
352 if (responseData_ != null) {
353 responseData_.cleanUp();
354 }
355 }
356
357 /**
358 * Returns whether the response has a successful HTTP status code.
359 *
360 * @return {@code true} if the status code is in the 2xx range
361 */
362 public boolean isSuccess() {
363 final int statusCode = getStatusCode();
364 return statusCode >= HttpStatus.OK_200 && statusCode < HttpStatus.MULTIPLE_CHOICES_300;
365 }
366
367 /**
368 * Returns whether the response has a successful HTTP status code or a use-proxy redirection.
369 *
370 * @return {@code true} if the status code is in the 2xx range or 305
371 */
372 public boolean isSuccessOrUseProxy() {
373 final int statusCode = getStatusCode();
374 return (statusCode >= HttpStatus.OK_200 && statusCode < HttpStatus.MULTIPLE_CHOICES_300)
375 || statusCode == HttpStatus.USE_PROXY_305;
376 }
377
378 /**
379 * Returns whether the response has a successful HTTP status code, a proxy redirection, or a not-modified status.
380 *
381 * @return {@code true} if the status code is in the 2xx range, 304 (Not Modified), or 305 (Use Proxy)
382 */
383 public boolean isSuccessOrUseProxyOrNotModified() {
384 return isSuccessOrUseProxy() || getStatusCode() == HttpStatus.NOT_MODIFIED_304;
385 }
386
387 /**
388 * Returns whether the request was blocked.
389 *
390 * @return {@code true} if the request was blocked
391 */
392 public boolean wasBlocked() {
393 return wasBlocked_;
394 }
395
396 /**
397 * Returns the reason for blocking or null.
398 *
399 * @return the reason for blocking, or {@code null} if the request was not blocked
400 */
401 public String getBlockReason() {
402 return blockReason_;
403 }
404
405 /**
406 * Sets the wasBlocked state to true.
407 *
408 * @param blockReason the reason
409 */
410 public void markAsBlocked(final String blockReason) {
411 wasBlocked_ = true;
412 blockReason_ = blockReason;
413 }
414 }