View Javadoc
1   /*
2    * Copyright (c) 2002-2025 Gargoyle Software Inc.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    * https://www.apache.org/licenses/LICENSE-2.0
8    *
9    * Unless required by applicable law or agreed to in writing, software
10   * distributed under the License is distributed on an "AS IS" BASIS,
11   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12   * See the License for the specific language governing permissions and
13   * limitations under the License.
14   */
15  package org.htmlunit;
16  
17  import static java.nio.charset.StandardCharsets.ISO_8859_1;
18  import static java.nio.charset.StandardCharsets.UTF_8;
19  
20  import java.io.File;
21  import java.io.FileInputStream;
22  import java.io.IOException;
23  import java.net.URISyntaxException;
24  import java.net.URL;
25  import java.nio.charset.Charset;
26  import java.time.Duration;
27  import java.util.ArrayList;
28  import java.util.Arrays;
29  import java.util.Collections;
30  import java.util.EnumSet;
31  import java.util.Enumeration;
32  import java.util.HashMap;
33  import java.util.HashSet;
34  import java.util.List;
35  import java.util.Locale;
36  import java.util.Map;
37  import java.util.Properties;
38  import java.util.Set;
39  import java.util.concurrent.Executor;
40  import java.util.concurrent.Executors;
41  
42  import javax.servlet.DispatcherType;
43  import javax.servlet.Filter;
44  import javax.servlet.FilterChain;
45  import javax.servlet.FilterConfig;
46  import javax.servlet.Servlet;
47  import javax.servlet.ServletException;
48  import javax.servlet.ServletRequest;
49  import javax.servlet.ServletResponse;
50  import javax.servlet.http.HttpServlet;
51  import javax.servlet.http.HttpServletRequest;
52  import javax.servlet.http.HttpServletResponse;
53  
54  import org.apache.commons.io.FileUtils;
55  import org.apache.commons.io.IOUtils;
56  import org.apache.commons.lang3.StringUtils;
57  import org.apache.commons.lang3.exception.ExceptionUtils;
58  import org.apache.commons.logging.Log;
59  import org.apache.commons.logging.LogFactory;
60  import org.eclipse.jetty.security.ConstraintMapping;
61  import org.eclipse.jetty.security.ConstraintSecurityHandler;
62  import org.eclipse.jetty.security.HashLoginService;
63  import org.eclipse.jetty.server.Request;
64  import org.eclipse.jetty.server.Server;
65  import org.eclipse.jetty.server.handler.HandlerWrapper;
66  import org.eclipse.jetty.util.security.Constraint;
67  import org.eclipse.jetty.webapp.WebAppContext;
68  import org.htmlunit.MockWebConnection.RawResponseData;
69  import org.htmlunit.html.HtmlElement;
70  import org.htmlunit.javascript.JavaScriptEngine;
71  import org.htmlunit.util.NameValuePair;
72  import org.junit.After;
73  import org.junit.AfterClass;
74  import org.junit.Assert;
75  import org.junit.Before;
76  import org.junit.ComparisonFailure;
77  import org.openqa.selenium.Alert;
78  import org.openqa.selenium.By;
79  import org.openqa.selenium.Dimension;
80  import org.openqa.selenium.JavascriptExecutor;
81  import org.openqa.selenium.NoAlertPresentException;
82  import org.openqa.selenium.NoSuchSessionException;
83  import org.openqa.selenium.NoSuchWindowException;
84  import org.openqa.selenium.UnhandledAlertException;
85  import org.openqa.selenium.WebDriver;
86  import org.openqa.selenium.WebDriverException;
87  import org.openqa.selenium.WebElement;
88  import org.openqa.selenium.chrome.ChromeDriver;
89  import org.openqa.selenium.chrome.ChromeDriverService;
90  import org.openqa.selenium.chrome.ChromeOptions;
91  import org.openqa.selenium.devtools.DevTools;
92  import org.openqa.selenium.devtools.v134.emulation.Emulation;
93  import org.openqa.selenium.edge.EdgeDriver;
94  import org.openqa.selenium.edge.EdgeDriverService;
95  import org.openqa.selenium.edge.EdgeOptions;
96  import org.openqa.selenium.firefox.FirefoxDriver;
97  import org.openqa.selenium.firefox.FirefoxDriverService;
98  import org.openqa.selenium.firefox.FirefoxOptions;
99  import org.openqa.selenium.firefox.FirefoxProfile;
100 import org.openqa.selenium.firefox.GeckoDriverService;
101 import org.openqa.selenium.htmlunit.HtmlUnitDriver;
102 import org.openqa.selenium.htmlunit.HtmlUnitWebElement;
103 import org.openqa.selenium.htmlunit.options.HtmlUnitDriverOptions;
104 import org.openqa.selenium.htmlunit.options.HtmlUnitOption;
105 import org.openqa.selenium.remote.UnreachableBrowserException;
106 
107 /**
108  * Base class for tests using WebDriver.
109  *
110  * By default, this test runs with HtmlUnit, but this behavior can be changed by having a property file named
111  * "{@code test.properties}" in the HtmlUnit root directory.
112  * Sample:
113  * <pre>
114    browsers=hu,ff,ie
115    chrome.bin=/path/to/chromedriver                     [Unix-like]
116    ff-esr.bin=/usr/bin/firefox                          [Unix-like]
117    ie.bin=C:\\path\\to\\32bit\\IEDriverServer.exe       [Windows]
118    edge.bin=C:\\path\\to\\msedgedriver.exe              [Windows]
119    autofix=true
120    </pre>
121  * The file could contain some properties:
122  * <ul>
123  *   <li>browsers: is a comma separated list contains any combination of "hu" (for HtmlUnit with all browser versions),
124  *   "hu-ff", "hu-ff-esr", "ff", "chrome", which will be used to drive real browsers</li>
125  *
126  *   <li>chrome.bin (mandatory if it does not exist in the <i>path</i>): is the location of the ChromeDriver binary (see
127  *   <a href="http://chromedriver.storage.googleapis.com/index.html">Chrome Driver downloads</a>)</li>
128  *   <li>geckodriver.bin (mandatory if it does not exist in the <i>path</i>): is the location of the GeckoDriver binary
129  *   (see <a href="https://firefox-source-docs.mozilla.org/testing/geckodriver/Usage.html">Gecko Driver Usage</a>)</li>
130  *   <li>ff.bin (optional): is the location of the FF binary, in Windows use double back-slashes</li>
131  *   <li>ff-esr.bin (optional): is the location of the FF binary, in Windows use double back-slashes</li>
132  *   <li>ie.bin (mandatory if it does not exist in the <i>path</i>): is the location of the IEDriverServer binary (see
133  *   <a href="http://selenium-release.storage.googleapis.com/index.html">IEDriverServer downloads</a>)</li>
134  *   <li>edge.bin (mandatory if it does not exist in the <i>path</i>): is the location of the MicrosoftWebDriver binary
135  *   (see <a href="http://go.microsoft.com/fwlink/?LinkId=619687">MicrosoftWebDriver downloads</a>)</li>
136  *   <li>autofix (optional): if {@code true}, try to automatically fix the real browser expectations,
137  *   or add/remove {@code @NotYetImplemented} annotations, use with caution!</li>
138  * </ul>
139  *
140  * @author Marc Guillemot
141  * @author Ahmed Ashour
142  * @author Ronald Brill
143  * @author Frank Danek
144  */
145 public abstract class WebDriverTestCase extends WebTestCase {
146 
147     private static final String LOG_EX_FUNCTION =
148             "  function logEx(e) {\n"
149             + "  let toStr = null;\n"
150             + "  if (toStr === null && e instanceof EvalError) { toStr = ''; }\n"
151             + "  if (toStr === null && e instanceof RangeError) { toStr = ''; }\n"
152             + "  if (toStr === null && e instanceof ReferenceError) { toStr = ''; }\n"
153             + "  if (toStr === null && e instanceof SyntaxError) { toStr = ''; }\n"
154             + "  if (toStr === null && e instanceof TypeError) { toStr = ''; }\n"
155             + "  if (toStr === null && e instanceof URIError) { toStr = ''; }\n"
156             + "  if (toStr === null && e instanceof AggregateError) { toStr = '/AggregateError'; }\n"
157             + "  if (toStr === null && typeof InternalError == 'function' "
158                         + "&& e instanceof InternalError) { toStr = '/InternalError'; }\n"
159             + "  if (toStr === null) {\n"
160             + "    let rx = /\\[object (.*)\\]/;\n"
161             + "    toStr = Object.prototype.toString.call(e);\n"
162             + "    let match = rx.exec(toStr);\n"
163             + "    if (match != null) { toStr = '/' + match[1]; }\n"
164             + "  }"
165             + "  log(e.name + toStr);\n"
166             + "}\n";
167 
168     /**
169      * Function used in many tests.
170      */
171     public static final String LOG_TITLE_FUNCTION =
172             "  function log(msg) { window.document.title += msg + '\\u00a7'; }\n"
173             + LOG_EX_FUNCTION;
174 
175     /**
176      * Function used in many tests.
177      */
178     public static final String LOG_TITLE_FUNCTION_NORMALIZE =
179             "  function log(msg) { "
180                     + "msg = '' + msg; "
181                     + "msg = msg.replace(/ /g, '\\\\s'); "
182                     + "msg = msg.replace(/\\n/g, '\\\\n'); "
183                     + "msg = msg.replace(/\\r/g, '\\\\r'); "
184                     + "msg = msg.replace(/\\t/g, '\\\\t'); "
185                     + "msg = msg.replace(/\\u001e/g, '\\\\u001e'); "
186                     + "window.document.title += msg + '\u00A7';}\n"
187 
188                     + LOG_EX_FUNCTION;
189 
190     /**
191      * Function used in many tests.
192      */
193     public static final String LOG_WINDOW_NAME_FUNCTION =
194             "  function log(msg) { window.top.name += msg + '\\u00a7'; }\n"
195             + "  window.top.name = '';"
196             + LOG_EX_FUNCTION;
197 
198     /**
199      * Function used in many tests.
200      */
201     public static final String LOG_SESSION_STORAGE_FUNCTION =
202             "  function log(msg) { "
203             + "var l = sessionStorage.getItem('Log');"
204             + "sessionStorage.setItem('Log', (null === l?'':l) + msg + '\\u00a7'); }\n";
205 
206     /**
207      * Function used in many tests.
208      */
209     public static final String LOG_TEXTAREA_FUNCTION = "  function log(msg) { "
210             + "document.getElementById('myLog').value += msg + '\u00A7';}\n"
211             + LOG_EX_FUNCTION;
212 
213     /**
214      * HtmlSniped to insert text area used for logging.
215      */
216     public static final String LOG_TEXTAREA = "  <textarea id='myLog' cols='80' rows='22'></textarea>\n";
217 
218     /**
219      * The system property for automatically fixing the test case expectations.
220      */
221     public static final String AUTOFIX_ = "htmlunit.autofix";
222 
223     /**
224      * All browsers supported.
225      */
226     private static List<BrowserVersion> ALL_BROWSERS_ = Collections.unmodifiableList(
227             Arrays.asList(BrowserVersion.CHROME,
228                     BrowserVersion.EDGE,
229                     BrowserVersion.FIREFOX,
230                     BrowserVersion.FIREFOX_ESR));
231 
232     /**
233      * Browsers which run by default.
234      */
235     private static BrowserVersion[] DEFAULT_RUNNING_BROWSERS_ =
236         {BrowserVersion.CHROME,
237             BrowserVersion.EDGE,
238             BrowserVersion.FIREFOX,
239             BrowserVersion.FIREFOX_ESR};
240 
241     private static final Log LOG = LogFactory.getLog(WebDriverTestCase.class);
242 
243     private static Set<String> BROWSERS_PROPERTIES_;
244     private static String CHROME_BIN_;
245     private static String EDGE_BIN_;
246     private static String GECKO_BIN_;
247     private static String FF_BIN_;
248     private static String FF_ESR_BIN_;
249 
250     /** The driver cache. */
251     protected static final Map<BrowserVersion, WebDriver> WEB_DRIVERS_ = new HashMap<>();
252 
253     /** The driver cache for real browsers. */
254     protected static final Map<BrowserVersion, WebDriver> WEB_DRIVERS_REAL_BROWSERS = new HashMap<>();
255     private static final Map<BrowserVersion, Integer> WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT = new HashMap<>();
256 
257     private static Server STATIC_SERVER_;
258     private static String STATIC_SERVER_STARTER_; // stack trace to save the server start code location
259     // second server for cross-origin tests.
260     private static Server STATIC_SERVER2_;
261     private static String STATIC_SERVER2_STARTER_; // stack trace to save the server start code location
262     // third server for multi-origin cross-origin tests.
263     private static Server STATIC_SERVER3_;
264     private static String STATIC_SERVER3_STARTER_; // stack trace to save the server start code location
265 
266     private static Boolean LAST_TEST_UsesMockWebConnection_;
267     private static final Executor EXECUTOR_POOL = Executors.newFixedThreadPool(4);
268 
269     private boolean useRealBrowser_;
270 
271     /**
272      * The HtmlUnitDriver.
273      */
274     private HtmlUnitDriver webDriver_;
275 
276     /**
277      * Override this function in a test class to ask for STATIC_SERVER2_ to be set up.
278      * @return true if two servers are needed.
279      */
280     protected boolean needThreeConnections() {
281         return false;
282     }
283 
284     /**
285      * @return the browser properties (and initializes them lazy)
286      */
287     public static Set<String> getBrowsersProperties() {
288         if (BROWSERS_PROPERTIES_ == null) {
289             try {
290                 final Properties properties = new Properties();
291                 final File file = new File("test.properties");
292                 if (file.exists()) {
293                     try (FileInputStream in = new FileInputStream(file)) {
294                         properties.load(in);
295                     }
296 
297                     String browsersValue = properties.getProperty("browsers");
298                     if (browsersValue == null || browsersValue.isEmpty()) {
299                         browsersValue = "hu";
300                     }
301                     BROWSERS_PROPERTIES_ = new HashSet<>(Arrays.asList(browsersValue.replaceAll(" ", "")
302                             .toLowerCase(Locale.ROOT).split(",")));
303                     CHROME_BIN_ = properties.getProperty("chrome.bin");
304                     EDGE_BIN_ = properties.getProperty("edge.bin");
305 
306                     GECKO_BIN_ = properties.getProperty("geckodriver.bin");
307                     FF_BIN_ = properties.getProperty("ff.bin");
308                     FF_ESR_BIN_ = properties.getProperty("ff-esr.bin");
309 
310                     final boolean autofix = Boolean.parseBoolean(properties.getProperty("autofix"));
311                     System.setProperty(AUTOFIX_, Boolean.toString(autofix));
312                 }
313             }
314             catch (final Exception e) {
315                 LOG.error("Error reading htmlunit.properties. Ignoring!", e);
316             }
317             if (BROWSERS_PROPERTIES_ == null) {
318                 BROWSERS_PROPERTIES_ = new HashSet<>(Arrays.asList("hu"));
319             }
320             if (BROWSERS_PROPERTIES_.contains("hu")) {
321                 for (final BrowserVersion browserVersion : DEFAULT_RUNNING_BROWSERS_) {
322                     BROWSERS_PROPERTIES_.add("hu-" + browserVersion.getNickname().toLowerCase());
323                 }
324             }
325         }
326         return BROWSERS_PROPERTIES_;
327     }
328 
329     /**
330      * @return the list of supported browsers
331      */
332     public static List<BrowserVersion> allBrowsers() {
333         return ALL_BROWSERS_;
334     }
335 
336     /**
337      * Configure the driver only once.
338      * @return the driver
339      */
340     protected WebDriver getWebDriver() {
341         final BrowserVersion browserVersion = getBrowserVersion();
342         WebDriver driver;
343         if (useRealBrowser()) {
344             synchronized (WEB_DRIVERS_REAL_BROWSERS) {
345                 driver = WEB_DRIVERS_REAL_BROWSERS.get(browserVersion);
346                 if (driver != null) {
347                     // there seems to be a memory leak at least with FF;
348                     // we have to restart sometimes
349                     Integer count = WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.get(browserVersion);
350                     if (null == count) {
351                         count = -1;
352                     }
353                     count += 1;
354                     if (count >= 1000) {
355                         shutDownReal(browserVersion);
356                         driver = null;
357                     }
358                     else {
359                         WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.put(browserVersion, count);
360                     }
361                 }
362 
363                 if (driver == null) {
364                     try {
365                         driver = buildWebDriver();
366                     }
367                     catch (final IOException e) {
368                         throw new RuntimeException(e);
369                     }
370 
371                     WEB_DRIVERS_REAL_BROWSERS.put(browserVersion, driver);
372                     WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.put(browserVersion, 0);
373                 }
374             }
375         }
376         else {
377             driver = WEB_DRIVERS_.get(browserVersion);
378             if (driver == null) {
379                 try {
380                     driver = buildWebDriver();
381                 }
382                 catch (final IOException e) {
383                     throw new RuntimeException(e);
384                 }
385 
386                 if (isWebClientCached()) {
387                     WEB_DRIVERS_.put(browserVersion, driver);
388                 }
389             }
390         }
391         return driver;
392     }
393 
394     /**
395      * Closes the drivers.
396      * @throws Exception If an error occurs
397      */
398     @AfterClass
399     public static void shutDownAll() throws Exception {
400         for (final WebDriver driver : WEB_DRIVERS_.values()) {
401             driver.quit();
402         }
403         WEB_DRIVERS_.clear();
404 
405         shutDownRealBrowsers();
406 
407         stopWebServers();
408     }
409 
410     /**
411      * Closes the real browser drivers.
412      */
413     private static void shutDownRealBrowsers() {
414         synchronized (WEB_DRIVERS_REAL_BROWSERS) {
415             for (final WebDriver driver : WEB_DRIVERS_REAL_BROWSERS.values()) {
416                 quit(driver);
417             }
418             WEB_DRIVERS_REAL_BROWSERS.clear();
419             WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.clear();
420         }
421     }
422 
423     /**
424      * Closes the real browser drivers.
425      * @param browser the real browser to close
426      */
427     private static void shutDownReal(final BrowserVersion browser) {
428         synchronized (WEB_DRIVERS_REAL_BROWSERS) {
429             final WebDriver driver = WEB_DRIVERS_REAL_BROWSERS.get(browser);
430             if (driver != null) {
431                 quit(driver);
432                 WEB_DRIVERS_REAL_BROWSERS.remove(browser);
433                 WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.remove(browser);
434             }
435         }
436     }
437 
438     private static void quit(final WebDriver driver) {
439         if (driver != null) {
440             try {
441                 driver.quit();
442             }
443             catch (final UnreachableBrowserException e) {
444                 LOG.error("Can't quit browser", e);
445                 // ignore, the browser is gone
446             }
447             catch (final NoClassDefFoundError e) {
448                 LOG.error("Can't quit browser", e);
449                 // ignore, the browser is gone
450             }
451             catch (final UnsatisfiedLinkError e) {
452                 LOG.error("Can't quit browser", e);
453                 // ignore, the browser is gone
454             }
455         }
456     }
457 
458     /**
459      * Asserts all static servers are null.
460      * @throws Exception if it fails
461      */
462     protected static void assertWebServersStopped() throws Exception {
463         Assert.assertNull(STATIC_SERVER_STARTER_, STATIC_SERVER_);
464         Assert.assertNull(STATIC_SERVER2_STARTER_, STATIC_SERVER2_);
465         Assert.assertNull(STATIC_SERVER3_STARTER_, STATIC_SERVER3_);
466     }
467 
468     /**
469      * Stops all WebServers.
470      * @throws Exception if it fails
471      */
472     protected static void stopWebServers() throws Exception {
473         if (STATIC_SERVER_ != null) {
474             STATIC_SERVER_.stop();
475             STATIC_SERVER_.destroy();
476             STATIC_SERVER_ = null;
477         }
478 
479         if (STATIC_SERVER2_ != null) {
480             STATIC_SERVER2_.stop();
481             STATIC_SERVER2_.destroy();
482             STATIC_SERVER2_ = null;
483         }
484 
485         if (STATIC_SERVER3_ != null) {
486             STATIC_SERVER3_.stop();
487             STATIC_SERVER3_.destroy();
488             STATIC_SERVER3_ = null;
489         }
490         LAST_TEST_UsesMockWebConnection_ = null;
491     }
492 
493     /**
494      * @return whether to use real browser or not.
495      */
496     protected boolean useRealBrowser() {
497         return useRealBrowser_;
498     }
499 
500     /**
501      * Sets whether to use real browser or not.
502      * @param useRealBrowser whether to use real browser or not
503      */
504     public void setUseRealBrowser(final boolean useRealBrowser) {
505         useRealBrowser_ = useRealBrowser;
506     }
507 
508     /**
509      * Builds a new WebDriver instance.
510      * @return the instance
511      * @throws IOException in case of exception
512      */
513     protected WebDriver buildWebDriver() throws IOException {
514         if (useRealBrowser()) {
515             if (BrowserVersion.EDGE.isSameBrowser(getBrowserVersion())) {
516                 final EdgeDriverService service = new EdgeDriverService.Builder()
517                         .withLogOutput(System.out)
518                         .usingDriverExecutable(new File(EDGE_BIN_))
519 
520                         .withAppendLog(true)
521                         .withReadableTimestamp(true)
522 
523                         .build();
524 
525                 final String locale = getBrowserVersion().getBrowserLocale().toLanguageTag();
526 
527                 final EdgeOptions options = new EdgeOptions();
528                 // BiDi
529                 // options.setCapability("webSocketUrl", true);
530 
531                 options.addArguments("--lang=" + locale);
532                 options.addArguments("--remote-allow-origins=*");
533 
534                 // seems to be not required for edge
535                 // options.addArguments("--disable-search-engine-choice-screen");
536                 // see https://www.selenium.dev/blog/2024/chrome-browser-woes/
537                 // options.addArguments("--disable-features=OptimizationGuideModelDownloading,"
538                 //         + "OptimizationHintsFetching,OptimizationTargetPrediction,OptimizationHints");
539 
540                 final EdgeDriver edge = new EdgeDriver(service, options);
541 
542                 final DevTools devTools = edge.getDevTools();
543                 devTools.createSession();
544 
545                 final String tz = getBrowserVersion().getSystemTimezone().getID();
546                 devTools.send(Emulation.setTimezoneOverride(tz));
547 
548                 return edge;
549             }
550 
551             if (BrowserVersion.CHROME.isSameBrowser(getBrowserVersion())) {
552                 final ChromeDriverService service = new ChromeDriverService.Builder()
553                         .withLogOutput(System.out)
554                         .usingDriverExecutable(new File(CHROME_BIN_))
555 
556                         .withAppendLog(true)
557                         .withReadableTimestamp(true)
558 
559                         .build();
560 
561                 final String locale = getBrowserVersion().getBrowserLocale().toLanguageTag();
562 
563                 final ChromeOptions options = new ChromeOptions();
564                 // BiDi
565                 // options.setCapability("webSocketUrl", true);
566 
567                 options.addArguments("--lang=" + locale);
568                 options.addArguments("--remote-allow-origins=*");
569                 options.addArguments("--disable-search-engine-choice-screen");
570                 // see https://www.selenium.dev/blog/2024/chrome-browser-woes/
571                 options.addArguments("--disable-features=OptimizationGuideModelDownloading,"
572                         + "OptimizationHintsFetching,OptimizationTargetPrediction,OptimizationHints");
573 
574                 final ChromeDriver chrome = new ChromeDriver(service, options);
575 
576                 final DevTools devTools = chrome.getDevTools();
577                 devTools.createSession();
578 
579                 final String tz = getBrowserVersion().getSystemTimezone().getID();
580                 devTools.send(Emulation.setTimezoneOverride(tz));
581 
582                 return chrome;
583             }
584 
585             if (BrowserVersion.FIREFOX.isSameBrowser(getBrowserVersion())) {
586                 return createFirefoxDriver(GECKO_BIN_, FF_BIN_);
587             }
588 
589             if (BrowserVersion.FIREFOX_ESR.isSameBrowser(getBrowserVersion())) {
590                 return createFirefoxDriver(GECKO_BIN_, FF_ESR_BIN_);
591             }
592 
593             throw new RuntimeException("Unexpected BrowserVersion: " + getBrowserVersion());
594         }
595 
596         if (webDriver_ == null) {
597             final HtmlUnitDriverOptions driverOptions = new HtmlUnitDriverOptions(getBrowserVersion());
598 
599             if (isWebClientCached()) {
600                 driverOptions.setCapability(HtmlUnitOption.optHistorySizeLimit, 0);
601             }
602 
603             if (getWebClientTimeout() != null) {
604                 driverOptions.setCapability(HtmlUnitOption.optTimeout, getWebClientTimeout());
605             }
606             webDriver_ = new HtmlUnitDriver(driverOptions);
607             webDriver_.setExecutor(EXECUTOR_POOL);
608         }
609         return webDriver_;
610     }
611 
612     private FirefoxDriver createFirefoxDriver(final String geckodriverBinary, final String binary) {
613         final FirefoxDriverService service = new GeckoDriverService.Builder()
614                 .withLogOutput(System.out)
615                 .usingDriverExecutable(new File(geckodriverBinary))
616                 .build();
617 
618         final FirefoxOptions options = new FirefoxOptions();
619         // BiDi
620         // options.setCapability("webSocketUrl", true);
621 
622         options.setBinary(binary);
623 
624         String locale = getBrowserVersion().getBrowserLocale().toLanguageTag();
625         locale = locale + "," + getBrowserVersion().getBrowserLocale().getLanguage();
626 
627         final FirefoxProfile profile = new FirefoxProfile();
628         profile.setPreference("intl.accept_languages", locale);
629         // no idea so far how to set this
630         // final String tz = getBrowserVersion().getSystemTimezone().getID();
631         // profile.setPreference("intl.tz", tz);
632         options.setProfile(profile);
633 
634         return new FirefoxDriver(service, options);
635     }
636 
637     /**
638      * Starts the web server delivering response from the provided connection.
639      * @param mockConnection the sources for responses
640      * @param serverCharset the {@link Charset} at the server side
641      * @throws Exception if a problem occurs
642      */
643     protected void startWebServer(final MockWebConnection mockConnection, final Charset serverCharset)
644             throws Exception {
645         if (Boolean.FALSE.equals(LAST_TEST_UsesMockWebConnection_)) {
646             stopWebServers();
647         }
648 
649         LAST_TEST_UsesMockWebConnection_ = Boolean.TRUE;
650         if (STATIC_SERVER_ == null) {
651             final Server server = new Server(PORT);
652 
653             final WebAppContext context = new WebAppContext();
654             context.setContextPath("/");
655             context.setResourceBase("./");
656 
657             if (isBasicAuthentication()) {
658                 final Constraint constraint = new Constraint();
659                 constraint.setName(Constraint.__BASIC_AUTH);
660                 constraint.setRoles(new String[]{"user"});
661                 constraint.setAuthenticate(true);
662 
663                 final ConstraintMapping constraintMapping = new ConstraintMapping();
664                 constraintMapping.setConstraint(constraint);
665                 constraintMapping.setPathSpec("/*");
666 
667                 final ConstraintSecurityHandler handler = (ConstraintSecurityHandler) context.getSecurityHandler();
668                 handler.setLoginService(new HashLoginService("MyRealm", "./src/test/resources/realm.properties"));
669                 handler.setConstraintMappings(new ConstraintMapping[]{constraintMapping});
670             }
671 
672             context.addServlet(MockWebConnectionServlet.class, "/*");
673             if (serverCharset != null) {
674                 AsciiEncodingFilter.CHARSET_ = serverCharset;
675                 context.addFilter(AsciiEncodingFilter.class, "/*",
676                         EnumSet.of(DispatcherType.INCLUDE, DispatcherType.REQUEST));
677             }
678             server.setHandler(context);
679             WebServerTestCase.tryStart(PORT, server);
680 
681             STATIC_SERVER_STARTER_ = ExceptionUtils.getStackTrace(new Throwable("StaticServerStarter"));
682             STATIC_SERVER_ = server;
683         }
684         MockWebConnectionServlet.MockConnection_ = mockConnection;
685 
686         if (STATIC_SERVER2_ == null && needThreeConnections()) {
687             final Server server2 = new Server(PORT2);
688             final WebAppContext context2 = new WebAppContext();
689             context2.setContextPath("/");
690             context2.setResourceBase("./");
691             context2.addServlet(MockWebConnectionServlet.class, "/*");
692             server2.setHandler(context2);
693             WebServerTestCase.tryStart(PORT2, server2);
694 
695             STATIC_SERVER2_STARTER_ = ExceptionUtils.getStackTrace(new Throwable("StaticServer2Starter"));
696             STATIC_SERVER2_ = server2;
697 
698             final Server server3 = new Server(PORT3);
699             final WebAppContext context3 = new WebAppContext();
700             context3.setContextPath("/");
701             context3.setResourceBase("./");
702             context3.addServlet(MockWebConnectionServlet.class, "/*");
703             server3.setHandler(context3);
704             WebServerTestCase.tryStart(PORT3, server3);
705 
706             STATIC_SERVER3_STARTER_ = ExceptionUtils.getStackTrace(new Throwable("StaticServer3Starter"));
707             STATIC_SERVER3_ = server3;
708             /*
709              * The mock connection servlet call sit under both servers, so long as tests
710              * keep the URLs distinct.
711              */
712         }
713     }
714 
715     /**
716      * Returns whether to use basic authentication for all resources or not.
717      * The default implementation returns false.
718      * @return whether to use basic authentication or not
719      */
720     protected boolean isBasicAuthentication() {
721         return false;
722     }
723 
724     /**
725      * Starts the web server on the default {@link #PORT}.
726      * The given resourceBase is used to be the ROOT directory that serves the default context.
727      * <p><b>Don't forget to stop the returned HttpServer after the test</b>
728      *
729      * @param resourceBase the base of resources for the default context
730      * @param classpath additional classpath entries to add (may be null)
731      * @param servlets map of {String, Class} pairs: String is the path spec, while class is the class
732      * @throws Exception if the test fails
733      */
734     protected static void startWebServer(final String resourceBase, final String[] classpath,
735             final Map<String, Class<? extends Servlet>> servlets) throws Exception {
736         startWebServer(resourceBase, classpath, servlets, null);
737     }
738 
739     /**
740      * Starts the <b>second</b> web server on the default {@link #PORT2}.
741      * The given resourceBase is used to be the ROOT directory that serves the default context.
742      * <p><b>Don't forget to stop the returned HttpServer after the test</b>
743      *
744      * @param resourceBase the base of resources for the default context
745      * @param classpath additional classpath entries to add (may be null)
746      * @param servlets map of {String, Class} pairs: String is the path spec, while class is the class
747      * @throws Exception if the test fails
748      */
749     protected static void startWebServer2(final String resourceBase, final String[] classpath,
750             final Map<String, Class<? extends Servlet>> servlets) throws Exception {
751 
752         if (STATIC_SERVER2_ != null) {
753             STATIC_SERVER2_.stop();
754         }
755         STATIC_SERVER2_STARTER_ = ExceptionUtils.getStackTrace(new Throwable("StaticServer2Starter"));
756         STATIC_SERVER2_ = WebServerTestCase.createWebServer(PORT2, resourceBase, classpath, servlets, null);
757     }
758 
759     /**
760      * Starts the web server on the default {@link #PORT}.
761      * The given resourceBase is used to be the ROOT directory that serves the default context.
762      * <p><b>Don't forget to stop the returned Server after the test</b>
763      *
764      * @param resourceBase the base of resources for the default context
765      * @param classpath additional classpath entries to add (may be null)
766      * @param servlets map of {String, Class} pairs: String is the path spec, while class is the class
767      * @param handler wrapper for handler (can be null)
768      * @throws Exception if the test fails
769      */
770     protected static void startWebServer(final String resourceBase, final String[] classpath,
771             final Map<String, Class<? extends Servlet>> servlets, final HandlerWrapper handler) throws Exception {
772         stopWebServers();
773         LAST_TEST_UsesMockWebConnection_ = Boolean.FALSE;
774 
775         STATIC_SERVER_STARTER_ = ExceptionUtils.getStackTrace(new Throwable("StaticServerStarter"));
776         STATIC_SERVER_ = WebServerTestCase.createWebServer(PORT, resourceBase, classpath, servlets, handler);
777     }
778 
779     /**
780      * Servlet delivering content from a MockWebConnection.
781      */
782     public static class MockWebConnectionServlet extends HttpServlet {
783         private static MockWebConnection MockConnection_;
784 
785         static void setMockconnection(final MockWebConnection connection) {
786             MockConnection_ = connection;
787         }
788 
789         /**
790          * {@inheritDoc}
791          */
792         @Override
793         protected void service(final HttpServletRequest request, final HttpServletResponse response)
794                 throws ServletException, IOException {
795 
796             try {
797                 doService(request, response);
798             }
799             catch (final ServletException e) {
800                 throw e;
801             }
802             catch (final IOException e) {
803                 throw e;
804             }
805             catch (final Exception e) {
806                 throw new ServletException(e);
807             }
808         }
809 
810         private static void doService(final HttpServletRequest request, final HttpServletResponse response)
811                 throws Exception {
812             String url = request.getRequestURL().toString();
813             if (LOG.isDebugEnabled()) {
814                 LOG.debug(request.getMethod() + " " + url);
815             }
816 
817             if (url.endsWith("/favicon.ico")) {
818                 response.setStatus(HttpServletResponse.SC_NOT_FOUND);
819                 return;
820             }
821 
822             if (url.contains("/delay")) {
823                 final String delay = StringUtils.substringBetween(url, "/delay", "/");
824                 final int ms = Integer.parseInt(delay);
825                 if (LOG.isDebugEnabled()) {
826                     LOG.debug("Sleeping for " + ms + " before to deliver " + url);
827                 }
828                 Thread.sleep(ms);
829             }
830 
831             // copy parameters
832             final List<NameValuePair> requestParameters = new ArrayList<>();
833             try {
834                 for (final Enumeration<String> paramNames = request.getParameterNames();
835                         paramNames.hasMoreElements();) {
836                     final String name = paramNames.nextElement();
837                     final String[] values = request.getParameterValues(name);
838                     for (final String value : values) {
839                         requestParameters.add(new NameValuePair(name, value));
840                     }
841                 }
842             }
843             catch (final IllegalArgumentException e) {
844                 // Jetty 8.1.7 throws it in getParameterNames for a query like "cb=%%RANDOM_NUMBER%%"
845                 // => we should use a more low level test server
846                 requestParameters.clear();
847                 final String query = request.getQueryString();
848                 if (query != null) {
849                     url += "?" + query;
850                 }
851             }
852 
853             final String queryString = request.getQueryString();
854             if (StringUtils.isNotBlank(queryString)) {
855                 url = url + "?" + queryString;
856             }
857             final URL requestedUrl = new URL(url);
858             final WebRequest webRequest = new WebRequest(requestedUrl);
859             webRequest.setHttpMethod(HttpMethod.valueOf(request.getMethod()));
860 
861             // copy headers
862             for (final Enumeration<String> en = request.getHeaderNames(); en.hasMoreElements();) {
863                 final String headerName = en.nextElement();
864                 final String headerValue = request.getHeader(headerName);
865                 webRequest.setAdditionalHeader(headerName, headerValue);
866             }
867 
868             if (requestParameters.isEmpty() && request.getContentLength() > 0) {
869                 final byte[] buffer = new byte[request.getContentLength()];
870                 IOUtils.read(request.getInputStream(), buffer, 0, buffer.length);
871 
872                 final String encoding = request.getCharacterEncoding();
873                 if (encoding == null) {
874                     webRequest.setRequestBody(new String(buffer, ISO_8859_1));
875                     webRequest.setCharset(null);
876                 }
877                 else {
878                     webRequest.setRequestBody(new String(buffer, encoding));
879                     webRequest.setCharset(Charset.forName(encoding));
880                 }
881             }
882             else {
883                 webRequest.setRequestParameters(requestParameters);
884             }
885 
886             // check content type if it is multipart enctype
887             if (request.getContentType() != null
888                         && request.getContentType().startsWith(FormEncodingType.MULTIPART.getName())) {
889                 webRequest.setEncodingType(FormEncodingType.MULTIPART);
890             }
891 
892             final RawResponseData resp = MockConnection_.getRawResponse(webRequest);
893 
894             // write WebResponse to HttpServletResponse
895             response.setStatus(resp.getStatusCode());
896 
897             boolean charsetInContentType = false;
898             for (final NameValuePair responseHeader : resp.getHeaders()) {
899                 final String headerName = responseHeader.getName();
900                 if (HttpHeader.CONTENT_TYPE.equals(headerName) && responseHeader.getValue().contains("charset=")) {
901                     charsetInContentType = true;
902                 }
903                 response.addHeader(headerName, responseHeader.getValue());
904             }
905 
906             if (resp.getByteContent() != null) {
907                 response.getOutputStream().write(resp.getByteContent());
908             }
909             else {
910                 if (!charsetInContentType) {
911                     response.setCharacterEncoding(resp.getCharset().name());
912                 }
913                 response.getWriter().print(resp.getStringContent());
914             }
915             response.flushBuffer();
916         }
917     }
918 
919     /**
920      * Same as {@link #loadPageWithAlerts2(String)}... but doesn't verify the alerts.
921      * @param html the HTML to use
922      * @return the web driver
923      * @throws Exception if something goes wrong
924      */
925     protected final WebDriver loadPage2(final String html) throws Exception {
926         return loadPage2(html, URL_FIRST);
927     }
928 
929     /**
930      * Same as {@link #loadPageWithAlerts2(String)}... but doesn't verify the alerts.
931      * @param html the HTML to use
932      * @param url the url to use to load the page
933      * @return the web driver
934      * @throws Exception if something goes wrong
935      */
936     protected final WebDriver loadPage2(final String html, final URL url) throws Exception {
937         return loadPage2(html, url, "text/html;charset=ISO-8859-1", ISO_8859_1, null);
938     }
939 
940     /**
941      * Same as {@link #loadPageWithAlerts2(String)}... but doesn't verify the alerts.
942      * @param html the HTML to use
943      * @param url the url to use to load the page
944      * @param contentType the content type to return
945      * @param charset the charset
946      * @return the web driver
947      * @throws Exception if something goes wrong
948      */
949     protected final WebDriver loadPage2(final String html, final URL url,
950             final String contentType, final Charset charset) throws Exception {
951         return loadPage2(html, url, contentType, charset, null);
952     }
953 
954     /**
955      * Same as {@link #loadPageWithAlerts2(String)}... but doesn't verify the alerts.
956      * @param html the HTML to use
957      * @param url the url to use to load the page
958      * @param contentType the content type to return
959      * @param charset the charset
960      * @param serverCharset the charset at the server side.
961      * @return the web driver
962      * @throws Exception if something goes wrong
963      */
964     protected final WebDriver loadPage2(final String html, final URL url,
965             final String contentType, final Charset charset, final Charset serverCharset) throws Exception {
966         getMockWebConnection().setResponse(url, html, contentType, charset);
967         return loadPage2(url, serverCharset);
968     }
969 
970     /**
971      * Load the page from the url.
972      * @param url the url to use to load the page
973      * @param serverCharset the charset at the server side.
974      * @return the web driver
975      * @throws Exception if something goes wrong
976      */
977     protected final WebDriver loadPage2(final URL url, final Charset serverCharset) throws Exception {
978         startWebServer(getMockWebConnection(), serverCharset);
979 
980         WebDriver driver = getWebDriver();
981         if (!(driver instanceof HtmlUnitDriver)) {
982             try {
983                 resizeIfNeeded(driver);
984             }
985             catch (final NoSuchSessionException e) {
986                 // maybe the driver was killed by the test before; setup a new one
987                 shutDownRealBrowsers();
988 
989                 driver = getWebDriver();
990                 resizeIfNeeded(driver);
991             }
992         }
993         driver.get(url.toExternalForm());
994 
995         return driver;
996     }
997 
998     /**
999      * Same as {@link #loadPage2(String)} with additional servlet configuration.
1000      * @param html the HTML to use for the default response
1001      * @param servlets the additional servlets to configure with their mapping
1002      * @return the web driver
1003      * @throws Exception if something goes wrong
1004      */
1005     protected final WebDriver loadPage2(final String html,
1006             final Map<String, Class<? extends Servlet>> servlets) throws Exception {
1007         return loadPage2(html, URL_FIRST, servlets, null);
1008     }
1009 
1010     /**
1011      * Same as {@link #loadPage2(String, URL)}, but with additional servlet configuration.
1012      * @param html the HTML to use for the default page
1013      * @param url the URL to use to load the page
1014      * @param servlets the additional servlets to configure with their mapping
1015      * @return the web driver
1016      * @throws Exception if something goes wrong
1017      */
1018     protected final WebDriver loadPage2(final String html, final URL url,
1019             final Map<String, Class<? extends Servlet>> servlets) throws Exception {
1020         return loadPage2(html, url, servlets, null);
1021     }
1022 
1023     /**
1024      * Same as {@link #loadPage2(String, URL)}, but with additional servlet configuration.
1025      * @param html the HTML to use for the default page
1026      * @param url the URL to use to load the page
1027      * @param servlets the additional servlets to configure with their mapping
1028      * @param servlets2 the additional servlets to configure with their mapping for a second server
1029      * @return the web driver
1030      * @throws Exception if something goes wrong
1031      */
1032     protected final WebDriver loadPage2(final String html, final URL url,
1033             final Map<String, Class<? extends Servlet>> servlets,
1034             final Map<String, Class<? extends Servlet>> servlets2) throws Exception {
1035         servlets.put("/*", MockWebConnectionServlet.class);
1036         getMockWebConnection().setResponse(url, html);
1037         MockWebConnectionServlet.MockConnection_ = getMockWebConnection();
1038 
1039         startWebServer("./", null, servlets);
1040         if (servlets2 != null) {
1041             startWebServer2("./", null, servlets2);
1042         }
1043 
1044         WebDriver driver = getWebDriver();
1045         if (!(driver instanceof HtmlUnitDriver)) {
1046             try {
1047                 resizeIfNeeded(driver);
1048             }
1049             catch (final NoSuchSessionException e) {
1050                 // maybe the driver was killed by the test before; setup a new one
1051                 shutDownRealBrowsers();
1052 
1053                 driver = getWebDriver();
1054                 resizeIfNeeded(driver);
1055             }
1056         }
1057         driver.get(url.toExternalForm());
1058 
1059         return driver;
1060     }
1061 
1062     protected void resizeIfNeeded(final WebDriver driver) {
1063         final Dimension size = driver.manage().window().getSize();
1064         if (size.getWidth() != 1272 || size.getHeight() != 768) {
1065             // only resize if needed because it may be quite expensive
1066             driver.manage().window().setSize(new Dimension(1272, 768));
1067         }
1068     }
1069 
1070     /**
1071      * Defines the provided HTML as the response for {@link WebTestCase#URL_FIRST}
1072      * and loads the page with this URL using the current WebDriver version; finally, asserts that the
1073      * alerts equal the expected alerts.
1074      * @param html the HTML to use
1075      * @return the web driver
1076      * @throws Exception if something goes wrong
1077      */
1078     protected final WebDriver loadPageWithAlerts2(final String html) throws Exception {
1079         return loadPageWithAlerts2(html, URL_FIRST, DEFAULT_WAIT_TIME);
1080     }
1081 
1082     /**
1083      * Defines the provided HTML as the response for {@link WebTestCase#URL_FIRST}
1084      * and loads the page with this URL using the current WebDriver version; finally, asserts that the
1085      * alerts equal the expected alerts.
1086      * @param html the HTML to use
1087      * @return the web driver
1088      * @throws Exception if something goes wrong
1089      */
1090     protected final WebDriver loadPageVerifyTitle2(final String html) throws Exception {
1091         return loadPageVerifyTitle2(html, getExpectedAlerts());
1092     }
1093 
1094     protected final WebDriver loadPageVerifyTitle2(final String html, final String... expectedAlerts) throws Exception {
1095         final WebDriver driver = loadPage2(html);
1096         return verifyTitle2(driver, expectedAlerts);
1097     }
1098 
1099     protected final WebDriver verifyTitle2(final Duration maxWaitTime, final WebDriver driver,
1100             final String... expectedAlerts) throws Exception {
1101 
1102         final StringBuilder expected = new StringBuilder();
1103         for (int i = 0; i < expectedAlerts.length; i++) {
1104             expected.append(expectedAlerts[i]).append('\u00A7');
1105         }
1106         final String expectedTitle = expected.toString();
1107 
1108         final long maxWait = System.currentTimeMillis() + maxWaitTime.toMillis();
1109 
1110         while (System.currentTimeMillis() < maxWait) {
1111             try {
1112                 final String title = driver.getTitle();
1113                 assertEquals(expectedTitle, title);
1114                 return driver;
1115             }
1116             catch (final AssertionError e) {
1117                 // ignore and wait
1118             }
1119         }
1120 
1121         assertEquals(expectedTitle, driver.getTitle());
1122         return driver;
1123     }
1124 
1125     protected final WebDriver verifyTitle2(final WebDriver driver,
1126             final String... expectedAlerts) throws Exception {
1127         if (expectedAlerts.length == 0) {
1128             assertEquals("", driver.getTitle());
1129             return driver;
1130         }
1131 
1132         final StringBuilder expected = new StringBuilder();
1133         for (int i = 0; i < expectedAlerts.length; i++) {
1134             expected.append(expectedAlerts[i]).append('\u00A7');
1135         }
1136 
1137         final String title = driver.getTitle();
1138         try {
1139             assertEquals(expected.toString(), title);
1140         }
1141         catch (final AssertionError e) {
1142             if (useRealBrowser() && StringUtils.isEmpty(title)) {
1143                 Thread.sleep(42);
1144                 assertEquals(expected.toString(), driver.getTitle());
1145                 return driver;
1146             }
1147             throw e;
1148         }
1149         return driver;
1150     }
1151 
1152     protected final WebDriver loadPageVerifyTextArea2(final String html) throws Exception {
1153         return loadPageTextArea2(html, getExpectedAlerts());
1154     }
1155 
1156     protected final WebDriver loadPageTextArea2(final String html, final String... expectedAlerts) throws Exception {
1157         final WebDriver driver = loadPage2(html);
1158         return verifyTextArea2(driver, expectedAlerts);
1159     }
1160 
1161     protected final WebDriver verifyTextArea2(final WebDriver driver,
1162             final String... expectedAlerts) throws Exception {
1163         final WebElement textArea = driver.findElement(By.id("myLog"));
1164 
1165         if (expectedAlerts.length == 0) {
1166             assertEquals("", textArea.getDomProperty("value"));
1167             return driver;
1168         }
1169 
1170         if (!useRealBrowser()
1171                 && expectedAlerts.length == 1
1172                 && expectedAlerts[0].startsWith("data:image/png;base64,")) {
1173             String value = textArea.getDomProperty("value");
1174             if (value.endsWith("\u00A7")) {
1175                 value = value.substring(0, value.length() - 1);
1176             }
1177             compareImages(expectedAlerts[0], value);
1178             return driver;
1179         }
1180 
1181         final StringBuilder expected = new StringBuilder();
1182         for (int i = 0; i < expectedAlerts.length; i++) {
1183             expected.append(expectedAlerts[i]).append('\u00A7');
1184         }
1185         verify(() -> textArea.getDomProperty("value"), expected.toString());
1186 
1187         return driver;
1188     }
1189 
1190     protected final String getJsVariableValue(final WebDriver driver, final String varName) throws Exception {
1191         final String script = "return String(" + varName + ")";
1192         final String result = (String) ((JavascriptExecutor) driver).executeScript(script);
1193 
1194         return result;
1195     }
1196 
1197     protected final WebDriver verifyJsVariable(final WebDriver driver, final String varName,
1198             final String expected) throws Exception {
1199         final String result = getJsVariableValue(driver, varName);
1200         assertEquals(expected, result);
1201 
1202         return driver;
1203     }
1204 
1205     protected final WebDriver verifyWindowName2(final Duration maxWaitTime, final WebDriver driver,
1206             final String... expectedAlerts) throws Exception {
1207         final long maxWait = System.currentTimeMillis() + maxWaitTime.toMillis();
1208 
1209         while (System.currentTimeMillis() < maxWait) {
1210             try {
1211                 return verifyWindowName2(driver, expectedAlerts);
1212             }
1213             catch (final AssertionError e) {
1214                 // ignore and wait
1215             }
1216         }
1217 
1218         return verifyWindowName2(driver, expectedAlerts);
1219     }
1220 
1221     protected final WebDriver verifyWindowName2(final WebDriver driver,
1222             final String... expectedAlerts) throws Exception {
1223         final StringBuilder expected = new StringBuilder();
1224         for (int i = 0; i < expectedAlerts.length; i++) {
1225             expected.append(expectedAlerts[i]).append('\u00A7');
1226         }
1227 
1228         return verifyJsVariable(driver, "window.top.name", expected.toString());
1229     }
1230 
1231     protected final WebDriver verifySessionStorage2(final WebDriver driver,
1232             final String... expectedAlerts) throws Exception {
1233         final StringBuilder expected = new StringBuilder();
1234         for (int i = 0; i < expectedAlerts.length; i++) {
1235             expected.append(expectedAlerts[i]).append('\u00A7');
1236         }
1237 
1238         return verifyJsVariable(driver, "sessionStorage.getItem('Log')", expected.toString());
1239     }
1240 
1241     /**
1242      * Defines the provided HTML as the response for {@link WebTestCase#URL_FIRST}
1243      * and loads the page with this URL using the current WebDriver version; finally, asserts that the
1244      * alerts equal the expected alerts.
1245      * @param html the HTML to use
1246      * @param maxWaitTime the maximum time to wait to get the alerts (in millis)
1247      * @return the web driver
1248      * @throws Exception if something goes wrong
1249      */
1250     protected final WebDriver loadPageWithAlerts2(final String html, final Duration maxWaitTime) throws Exception {
1251         return loadPageWithAlerts2(html, URL_FIRST, maxWaitTime);
1252     }
1253 
1254     /**
1255      * Same as {@link #loadPageWithAlerts2(String)}, but specifying the default URL.
1256      * @param html the HTML to use
1257      * @param url the URL to use to load the page
1258      * @return the web driver
1259      * @throws Exception if something goes wrong
1260      */
1261     protected final WebDriver loadPageWithAlerts2(final String html, final URL url) throws Exception {
1262         return loadPageWithAlerts2(html, url, DEFAULT_WAIT_TIME);
1263     }
1264 
1265     /**
1266      * Same as {@link #loadPageWithAlerts2(String, long)}, but specifying the default URL.
1267      * @param html the HTML to use
1268      * @param url the URL to use to load the page
1269      * @param maxWaitTime the maximum time to wait to get the alerts (in millis)
1270      * @return the web driver
1271      * @throws Exception if something goes wrong
1272      */
1273     protected final WebDriver loadPageWithAlerts2(final String html, final URL url, final Duration maxWaitTime)
1274             throws Exception {
1275         final WebDriver driver = loadPage2(html, url);
1276 
1277         verifyAlerts(maxWaitTime, driver, getExpectedAlerts());
1278         return driver;
1279     }
1280 
1281     /**
1282      * Verifies the captured alerts.
1283      * @param driver the driver instance
1284      * @param expectedAlerts the expected alerts
1285      * @throws Exception in case of failure
1286      */
1287     protected void verifyAlerts(final WebDriver driver, final String... expectedAlerts) throws Exception {
1288         verifyAlerts(DEFAULT_WAIT_TIME, driver, expectedAlerts);
1289     }
1290 
1291     /**
1292      * Verifies the captured alerts.
1293      *
1294      * @param maxWaitTime the maximum time to wait for the expected alert to be found
1295      * @param driver the driver instance
1296      * @param expected the expected alerts
1297      * @throws Exception in case of failure
1298      */
1299     protected void verifyAlerts(final Duration maxWaitTime, final WebDriver driver, final String... expected)
1300             throws Exception {
1301         final List<String> actualAlerts = getCollectedAlerts(maxWaitTime, driver, expected.length);
1302 
1303         assertEquals(expected.length, actualAlerts.size());
1304 
1305         if (!useRealBrowser()) {
1306             // check if we have data-image Url
1307             for (int i = 0; i < expected.length; i++) {
1308                 if (expected[i].startsWith("data:image/png;base64,")) {
1309                     // we have to compare element by element
1310                     for (int j = 0; j < expected.length; j++) {
1311                         if (expected[j].startsWith("data:image/png;base64,")) {
1312                             compareImages(expected[j], actualAlerts.get(j));
1313                         }
1314                         else {
1315                             assertEquals(expected[j], actualAlerts.get(j));
1316                         }
1317                     }
1318                     return;
1319                 }
1320             }
1321         }
1322 
1323         assertEquals(expected, actualAlerts);
1324     }
1325 
1326     /**
1327      * Same as {@link #loadPageWithAlerts2(String)} with additional servlet configuration.
1328      * @param html the HTML to use for the default response
1329      * @param servlets the additional servlets to configure with their mapping
1330      * @return the web driver
1331      * @throws Exception if something goes wrong
1332      */
1333     protected final WebDriver loadPageWithAlerts2(final String html,
1334             final Map<String, Class<? extends Servlet>> servlets) throws Exception {
1335         return loadPageWithAlerts2(html, URL_FIRST, DEFAULT_WAIT_TIME, servlets);
1336     }
1337 
1338     /**
1339      * Same as {@link #loadPageWithAlerts2(String, URL, long)}, but with additional servlet configuration.
1340      * @param html the HTML to use for the default page
1341      * @param url the URL to use to load the page
1342      * @param maxWaitTime the maximum time to wait to get the alerts (in millis)
1343      * @param servlets the additional servlets to configure with their mapping
1344      * @return the web driver
1345      * @throws Exception if something goes wrong
1346      */
1347     protected final WebDriver loadPageWithAlerts2(final String html, final URL url, final Duration maxWaitTime,
1348             final Map<String, Class<? extends Servlet>> servlets) throws Exception {
1349 
1350         expandExpectedAlertsVariables(URL_FIRST);
1351 
1352         final WebDriver driver = loadPage2(html, url, servlets);
1353         verifyAlerts(maxWaitTime, driver, getExpectedAlerts());
1354 
1355         return driver;
1356     }
1357 
1358     /**
1359      * Loads the provided URL serving responses from {@link #getMockWebConnection()}
1360      * and verifies that the captured alerts are correct.
1361      * @param url the URL to use to load the page
1362      * @return the web driver
1363      * @throws Exception if something goes wrong
1364      */
1365     protected final WebDriver loadPageWithAlerts2(final URL url) throws Exception {
1366         return loadPageWithAlerts2(url, DEFAULT_WAIT_TIME);
1367     }
1368 
1369     /**
1370      * Same as {@link #loadPageWithAlerts2(URL)}, but using with timeout.
1371      * @param url the URL to use to load the page
1372      * @param maxWaitTime the maximum time to wait to get the alerts (in millis)
1373      * @return the web driver
1374      * @throws Exception if something goes wrong
1375      */
1376     protected final WebDriver loadPageWithAlerts2(final URL url, final Duration maxWaitTime) throws Exception {
1377         startWebServer(getMockWebConnection(), null);
1378 
1379         final WebDriver driver = getWebDriver();
1380         driver.get(url.toExternalForm());
1381 
1382         verifyAlerts(maxWaitTime, driver, getExpectedAlerts());
1383         return driver;
1384     }
1385 
1386     /**
1387      * Gets the alerts collected by the driver.
1388      * Note: it currently works only if no new page has been loaded in the window
1389      * @param driver the driver
1390      * @return the collected alerts
1391      * @throws Exception in case of problem
1392      */
1393     protected List<String> getCollectedAlerts(final WebDriver driver) throws Exception {
1394         return getCollectedAlerts(driver, getExpectedAlerts().length);
1395     }
1396 
1397     /**
1398      * Gets the alerts collected by the driver.
1399      * Note: it currently works only if no new page has been loaded in the window
1400      * @param driver the driver
1401      * @param alertsLength the expected length of Alerts
1402      * @return the collected alerts
1403      * @throws Exception in case of problem
1404      */
1405     protected List<String> getCollectedAlerts(final WebDriver driver, final int alertsLength) throws Exception {
1406         return getCollectedAlerts(DEFAULT_WAIT_TIME, driver, alertsLength);
1407     }
1408 
1409     /**
1410      * Gets the alerts collected by the driver.
1411      * Note: it currently works only if no new page has been loaded in the window
1412      * @param maxWaitTime the maximum time to wait to get the alerts (in millis)
1413      * @param driver the driver
1414      * @param alertsLength the expected length of Alerts
1415      * @return the collected alerts
1416      * @throws Exception in case of problem
1417      */
1418     protected List<String> getCollectedAlerts(final Duration maxWaitTime,
1419             final WebDriver driver, final int alertsLength) throws Exception {
1420         final List<String> collectedAlerts = new ArrayList<>();
1421 
1422         long maxWait = System.currentTimeMillis() + maxWaitTime.toMillis();
1423 
1424         while (collectedAlerts.size() < alertsLength && System.currentTimeMillis() < maxWait) {
1425             try {
1426                 final Alert alert = driver.switchTo().alert();
1427                 final String text = alert.getText();
1428 
1429                 collectedAlerts.add(text);
1430                 alert.accept();
1431 
1432                 // handling of alerts requires some time
1433                 // at least for tests with many alerts we have to take this into account
1434                 maxWait += 100;
1435             }
1436             catch (final NoAlertPresentException e) {
1437                 Thread.sleep(10);
1438             }
1439         }
1440 
1441         return collectedAlerts;
1442     }
1443 
1444     /**
1445      * Returns the HtmlElement of the specified WebElement.
1446      * @param webElement the webElement
1447      * @return the HtmlElement
1448      * @throws Exception if an error occurs
1449      */
1450     protected HtmlElement toHtmlElement(final WebElement webElement) throws Exception {
1451         return (HtmlElement) ((HtmlUnitWebElement) webElement).getElement();
1452     }
1453 
1454     /**
1455      * Loads an expectation file for the specified browser search first for a browser specific resource
1456      * and falling back in a general resource.
1457      * @param resourcePrefix the start of the resource name
1458      * @param resourceSuffix the end of the resource name
1459      * @return the content of the file
1460      * @throws Exception in case of error
1461      */
1462     protected String loadExpectation(final String resourcePrefix, final String resourceSuffix) throws Exception {
1463         final Class<?> referenceClass = getClass();
1464         final BrowserVersion browserVersion = getBrowserVersion();
1465 
1466         String realBrowserNyiExpectation = null;
1467         String realNyiExpectation = null;
1468         final String browserExpectation;
1469         final String expectation;
1470         if (!useRealBrowser()) {
1471             // first try nyi
1472             final String browserSpecificNyiResource
1473                     = resourcePrefix + "." + browserVersion.getNickname() + "_NYI" + resourceSuffix;
1474             realBrowserNyiExpectation = loadContent(referenceClass.getResource(browserSpecificNyiResource));
1475 
1476             // next nyi without browser
1477             final String nyiResource = resourcePrefix + ".NYI" + resourceSuffix;
1478             realNyiExpectation = loadContent(referenceClass.getResource(nyiResource));
1479         }
1480 
1481         // implemented - browser specific
1482         final String browserSpecificResource = resourcePrefix + "." + browserVersion.getNickname() + resourceSuffix;
1483         browserExpectation = loadContent(referenceClass.getResource(browserSpecificResource));
1484 
1485         // implemented - all browsers
1486         final String resource = resourcePrefix + resourceSuffix;
1487         expectation = loadContent(referenceClass.getResource(resource));
1488 
1489         // check for duplicates
1490         if (realBrowserNyiExpectation != null) {
1491             if (realNyiExpectation != null) {
1492                 Assert.assertNotEquals("Duplicate NYI Expectation for Browser " + browserVersion.getNickname(),
1493                         realBrowserNyiExpectation, realNyiExpectation);
1494             }
1495 
1496             if (browserExpectation == null) {
1497                 if (expectation != null) {
1498                     Assert.assertNotEquals("NYI Expectation matches the expected "
1499                             + "result for Browser " + browserVersion.getNickname(),
1500                             realBrowserNyiExpectation, expectation);
1501                 }
1502             }
1503             else {
1504                 Assert.assertNotEquals("NYI Expectation matches the expected "
1505                         + "browser specific result for Browser " + browserVersion.getNickname(),
1506                         realBrowserNyiExpectation, browserExpectation);
1507             }
1508 
1509             return realBrowserNyiExpectation;
1510         }
1511 
1512         if (realNyiExpectation != null) {
1513             if (browserExpectation == null) {
1514                 if (expectation != null) {
1515                     Assert.assertNotEquals("NYI Expectation matches the expected "
1516                             + "result for Browser " + browserVersion.getNickname(),
1517                             realNyiExpectation, expectation);
1518                 }
1519             }
1520             else {
1521                 Assert.assertNotEquals("NYI Expectation matches the expected "
1522                         + "browser specific result for Browser " + browserVersion.getNickname(),
1523                         realNyiExpectation, browserExpectation);
1524             }
1525             return realNyiExpectation;
1526         }
1527 
1528         if (browserExpectation != null) {
1529             if (expectation != null) {
1530                 Assert.assertNotEquals("Browser specific NYI Expectation matches the expected "
1531                         + "result for Browser " + browserVersion.getNickname(),
1532                         browserExpectation, expectation);
1533             }
1534             return browserExpectation;
1535         }
1536         return expectation;
1537     }
1538 
1539     private static String loadContent(final URL url) throws URISyntaxException, IOException {
1540         if (url == null) {
1541             return null;
1542         }
1543 
1544         final File file = new File(url.toURI());
1545         String content = FileUtils.readFileToString(file, UTF_8);
1546         content = StringUtils.replace(content, "\r\n", "\n");
1547         return content;
1548     }
1549 
1550     /**
1551      * Reads the number of JS threads remaining from unit tests run before.
1552      * This should be always 0, if {@link #isWebClientCached()} is {@code false}.
1553      * @throws InterruptedException in case of problems
1554      */
1555     @Before
1556     public void beforeTest() throws InterruptedException {
1557         if (!isWebClientCached()) {
1558             if (!getJavaScriptThreads().isEmpty()) {
1559                 Thread.sleep(200);
1560             }
1561             assertTrue("There are already JS threads running before the test", getJavaScriptThreads().isEmpty());
1562         }
1563     }
1564 
1565     /**
1566      * Asserts the current title is equal to the expectation string.
1567      * @param webdriver the driver in use
1568      * @param expected the expected object
1569      * @throws Exception in case of failure
1570      */
1571     protected void assertTitle(final WebDriver webdriver, final String expected) throws Exception {
1572         final long maxWait = System.currentTimeMillis() + DEFAULT_WAIT_TIME.toMillis();
1573 
1574         while (true) {
1575             final String title = webdriver.getTitle();
1576             try {
1577                 assertEquals(expected, title);
1578                 return;
1579             }
1580             catch (final ComparisonFailure e) {
1581                 if (expected.length() <= title.length()
1582                         || System.currentTimeMillis() > maxWait) {
1583                     throw e;
1584                 }
1585                 Thread.sleep(10);
1586             }
1587         }
1588     }
1589 
1590     /**
1591      * Release resources but DON'T close the browser if we are running with a real browser.
1592      * Note that HtmlUnitDriver is not cached by default, but that can be configured by {@link #isWebClientCached()}.
1593      */
1594     @After
1595     @Override
1596     public void releaseResources() {
1597         final List<String> unhandledAlerts = new ArrayList<>();
1598         if (webDriver_ != null) {
1599             UnhandledAlertException ex = null;
1600             do {
1601                 ex = null;
1602                 try {
1603                     // getTitle will do an implicit check for open alerts
1604                     webDriver_.getTitle();
1605                 }
1606                 catch (final NoSuchWindowException e) {
1607                     // ignore
1608                 }
1609                 catch (final UnhandledAlertException e) {
1610                     ex = e;
1611                     unhandledAlerts.add(e.getMessage());
1612                 }
1613             }
1614             while (ex != null);
1615         }
1616 
1617         super.releaseResources();
1618 
1619         if (!isWebClientCached()) {
1620             boolean rhino = false;
1621             if (webDriver_ != null) {
1622                 try {
1623                     rhino = getWebClient().getJavaScriptEngine() instanceof JavaScriptEngine;
1624                 }
1625                 catch (final Exception e) {
1626                     throw new RuntimeException(e);
1627                 }
1628                 webDriver_.quit();
1629                 webDriver_ = null;
1630             }
1631             if (rhino) {
1632                 assertTrue("There are still JS threads running after the test", getJavaScriptThreads().isEmpty());
1633             }
1634         }
1635 
1636         if (useRealBrowser()) {
1637             synchronized (WEB_DRIVERS_REAL_BROWSERS) {
1638                 final WebDriver driver = WEB_DRIVERS_REAL_BROWSERS.get(getBrowserVersion());
1639                 if (driver != null) {
1640                     try {
1641                         final String currentWindow = driver.getWindowHandle();
1642 
1643                         final Set<String> handles = driver.getWindowHandles();
1644                         // close all windows except the current one
1645                         handles.remove(currentWindow);
1646 
1647                         if (handles.size() > 0) {
1648                             for (final String handle : handles) {
1649                                 try {
1650                                     driver.switchTo().window(handle);
1651                                     driver.close();
1652                                 }
1653                                 catch (final NoSuchWindowException e) {
1654                                     LOG.error("Error switching to browser window; quit browser.", e);
1655                                     WEB_DRIVERS_REAL_BROWSERS.remove(getBrowserVersion());
1656                                     WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.remove(getBrowserVersion());
1657                                     driver.quit();
1658                                     return;
1659                                 }
1660                             }
1661 
1662                             // we have to force WebDriver to treat the remaining window
1663                             // as the one we like to work with from now on
1664                             // looks like a web driver issue to me (version 2.47.2)
1665                             driver.switchTo().window(currentWindow);
1666                         }
1667 
1668                         driver.manage().deleteAllCookies();
1669 
1670                         // in the remaining window, load a blank page
1671                         driver.get("about:blank");
1672                     }
1673                     catch (final NoSuchSessionException e) {
1674                         LOG.error("Error browser session no longer available.", e);
1675                         WEB_DRIVERS_REAL_BROWSERS.remove(getBrowserVersion());
1676                         WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.remove(getBrowserVersion());
1677                         return;
1678                     }
1679                     catch (final WebDriverException e) {
1680                         shutDownRealBrowsers();
1681                     }
1682                 }
1683             }
1684         }
1685         assertTrue("There are still unhandled alerts: " + String.join("; ", unhandledAlerts),
1686                         unhandledAlerts.isEmpty());
1687     }
1688 
1689     /**
1690      * Returns the underlying WebWindow of the specified driver.
1691      *
1692      * <b>Your test shouldn't depend primarily on WebClient</b>
1693      *
1694      * @return the current web window
1695      * @see #toHtmlElement(WebElement)
1696      */
1697     protected WebWindow getWebWindow() {
1698         return webDriver_.getCurrentWindow().getWebWindow();
1699     }
1700 
1701     /**
1702      * Whether {@link WebClient} is cached or not, defaults to {@code false}.
1703      *
1704      * <p>This is needed to be {@code true} for huge test class, as we could run out of sockets.
1705      *
1706      * @return whether {@link WebClient} is cached or not
1707      */
1708     protected boolean isWebClientCached() {
1709         return false;
1710     }
1711 
1712     /**
1713      * Configure {@link WebClientOptions#getTimeout()}.
1714      *
1715      * @return null if unchanged otherwise the timeout as int
1716      */
1717     protected Integer getWebClientTimeout() {
1718         return null;
1719     }
1720 
1721     protected Page getEnclosedPage() {
1722         return getWebWindow().getEnclosedPage();
1723     }
1724 
1725     protected WebClient getWebClient() {
1726         return webDriver_.getWebClient();
1727     }
1728 
1729     /**
1730      * Needed as Jetty starting from 9.4.4 expects UTF-8 encoding by default.
1731      */
1732     public static class AsciiEncodingFilter implements Filter {
1733 
1734         private static Charset CHARSET_;
1735 
1736         /**
1737          * {@inheritDoc}
1738          */
1739         @Override
1740         public void init(final FilterConfig filterConfig) throws ServletException {
1741         }
1742 
1743         /**
1744          * {@inheritDoc}
1745          */
1746         @Override
1747         public void doFilter(final ServletRequest request, final ServletResponse response, final FilterChain chain)
1748                 throws IOException, ServletException {
1749             if (request instanceof Request) {
1750                 ((Request) request).setQueryEncoding(CHARSET_.name());
1751             }
1752             chain.doFilter(request, response);
1753         }
1754 
1755         /**
1756          * {@inheritDoc}
1757          */
1758         @Override
1759         public void destroy() {
1760         }
1761     }
1762 }