View Javadoc
1   /*
2    * Copyright (c) 2002-2026 Gargoyle Software Inc.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    * https://www.apache.org/licenses/LICENSE-2.0
8    *
9    * Unless required by applicable law or agreed to in writing, software
10   * distributed under the License is distributed on an "AS IS" BASIS,
11   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12   * See the License for the specific language governing permissions and
13   * limitations under the License.
14   */
15  package org.htmlunit;
16  
17  import static java.nio.charset.StandardCharsets.ISO_8859_1;
18  import static java.nio.charset.StandardCharsets.UTF_8;
19  
20  import java.io.File;
21  import java.io.FileInputStream;
22  import java.io.IOException;
23  import java.net.URISyntaxException;
24  import java.net.URL;
25  import java.nio.charset.Charset;
26  import java.time.Duration;
27  import java.util.ArrayList;
28  import java.util.Arrays;
29  import java.util.Enumeration;
30  import java.util.HashMap;
31  import java.util.HashSet;
32  import java.util.List;
33  import java.util.Locale;
34  import java.util.Map;
35  import java.util.Properties;
36  import java.util.Set;
37  import java.util.concurrent.Executor;
38  import java.util.concurrent.Executors;
39  
40  import org.apache.commons.io.FileUtils;
41  import org.apache.commons.io.IOUtils;
42  import org.apache.commons.lang3.StringUtils;
43  import org.apache.commons.lang3.exception.ExceptionUtils;
44  import org.apache.commons.logging.Log;
45  import org.apache.commons.logging.LogFactory;
46  import org.eclipse.jetty.server.Server;
47  import org.htmlunit.MockWebConnection.RawResponseData;
48  import org.htmlunit.WebServerTestCase.SSLVariant;
49  import org.htmlunit.html.HtmlElement;
50  import org.htmlunit.javascript.JavaScriptEngine;
51  import org.htmlunit.junit.TestCaseCorrector;
52  import org.htmlunit.util.JettyServerUtils;
53  import org.htmlunit.util.NameValuePair;
54  import org.junit.jupiter.api.AfterAll;
55  import org.junit.jupiter.api.AfterEach;
56  import org.junit.jupiter.api.Assertions;
57  import org.junit.jupiter.api.BeforeEach;
58  import org.junit.jupiter.api.extension.ExtendWith;
59  import org.openqa.selenium.Alert;
60  import org.openqa.selenium.By;
61  import org.openqa.selenium.Dimension;
62  import org.openqa.selenium.JavascriptExecutor;
63  import org.openqa.selenium.NoAlertPresentException;
64  import org.openqa.selenium.NoSuchSessionException;
65  import org.openqa.selenium.NoSuchWindowException;
66  import org.openqa.selenium.UnhandledAlertException;
67  import org.openqa.selenium.WebDriver;
68  import org.openqa.selenium.WebDriverException;
69  import org.openqa.selenium.WebElement;
70  import org.openqa.selenium.chrome.ChromeDriver;
71  import org.openqa.selenium.chrome.ChromeDriverService;
72  import org.openqa.selenium.chrome.ChromeOptions;
73  import org.openqa.selenium.devtools.DevTools;
74  import org.openqa.selenium.devtools.v155.emulation.Emulation;
75  import org.openqa.selenium.edge.EdgeDriver;
76  import org.openqa.selenium.edge.EdgeDriverService;
77  import org.openqa.selenium.edge.EdgeOptions;
78  import org.openqa.selenium.firefox.FirefoxDriver;
79  import org.openqa.selenium.firefox.FirefoxDriverService;
80  import org.openqa.selenium.firefox.FirefoxOptions;
81  import org.openqa.selenium.firefox.FirefoxProfile;
82  import org.openqa.selenium.firefox.GeckoDriverService;
83  import org.openqa.selenium.htmlunit.HtmlUnitDriver;
84  import org.openqa.selenium.htmlunit.HtmlUnitWebElement;
85  import org.openqa.selenium.htmlunit.options.HtmlUnitDriverOptions;
86  import org.openqa.selenium.htmlunit.options.HtmlUnitOption;
87  import org.openqa.selenium.remote.UnreachableBrowserException;
88  import org.opentest4j.AssertionFailedError;
89  
90  import jakarta.servlet.Servlet;
91  import jakarta.servlet.ServletException;
92  import jakarta.servlet.http.HttpServlet;
93  import jakarta.servlet.http.HttpServletRequest;
94  import jakarta.servlet.http.HttpServletResponse;
95  
96  /**
97   * Base class for tests using WebDriver.
98   * <p>
99   * By default, this test runs with HtmlUnit, but this behavior can be changed by having a property file named
100  * "{@code test.properties}" in the HtmlUnit root directory.
101  * Sample (remove the part not matching your os):
102  * <pre>
103    browsers=hu,ff,chrome
104 
105    ff.bin=/usr/bin/firefox                              [Unix]
106    ff-esr.bin=/usr/bin/firefox-esr                      [Unix]
107    geckodriver.bin=/usr/bin/driver/geckodriver          [Unix]
108    chrome.bin=/path/to/chromedriver                     [Unix]
109    edge.bin=/path/to/chromedriver                       [Unix]
110 
111    geckodriver.bin=C:\\path\\to\\geckodriver.exe              [Windows]
112    ff.bin=C:\\path\\to\\Mozilla Firefox\\firefox.exe          [Windows]
113    ff-esr.bin=C:\\path\\to\\Mozilla Firefox ESR\\firefox.exe  [Windows]
114    chrome.bin=C:\\path\\to\\chromedriver.exe                  [Windows]
115    edge.bin=C:\\path\\to\\msedgedriver.exe                    [Windows]
116 
117    autofix=false
118    </pre>
119 
120  * The file could contain some properties:
121  * <ul>
122  *   <li>browsers: is a comma separated list contains any combination of
123  *     <ul>
124  *       <li>hu (for HtmlUnit with all browser versions),</li>
125  *       <li>hu-ff,</li>
126  *       <li>hu-ff-esr,</li>
127  *       <li>hu-chrome,</li>
128  *       <li>hu-edge,</li>
129  *       <li>ff, (running test using real Firefox),</li>
130  *       <li>ff-esr, (running test using real Firefox ESR),</li>
131  *       <li>chrome (running test using real Chrome),</li>
132  *       <li>edge (running test using real Edge),</li>
133  *     </ul>
134  *   </li>
135  *
136  *   <li>chrome.bin (mandatory if it does not exist in the <i>path</i>): is the location of the ChromeDriver binary (see
137  *   <a href="https://googlechromelabs.github.io/chrome-for-testing/last-known-good-versions-with-downloads.json">Chrome Driver downloads</a>)</li>
138  *   <li>geckodriver.bin (mandatory if it does not exist in the <i>path</i>): is the location of the GeckoDriver binary
139  *   (see <a href="https://github.com/mozilla/geckodriver/releases">Gecko Driver Releases</a>)</li>
140  *   <li>ff.bin (optional): is the location of the FF binary, in Windows use double back-slashes</li>
141  *   <li>ff-esr.bin (optional): is the location of the FF binary, in Windows use double back-slashes</li>
142  *   <li>edge.bin (mandatory if it does not exist in the <i>path</i>): is the location of the MicrosoftWebDriver binary
143  *   (see <a href="https://developer.microsoft.com/en-us/microsoft-edge/tools/webdriver/">Microsoft Edge WebDriver downloads</a>)</li>
144  *   <li>autofix (optional): if {@code true}, try to automatically fix the real browser expectations,
145  *   or add/remove {@code @NotYetImplemented} annotations, use with caution!</li>
146  * </ul>
147  *
148  * @author Marc Guillemot
149  * @author Ahmed Ashour
150  * @author Ronald Brill
151  * @author Frank Danek
152  */
153 @ExtendWith(TestCaseCorrector.class)
154 public abstract class WebDriverTestCase extends WebTestCase {
155 
156     private static final String LOG_EX_FUNCTION =
157             "  function logEx(e) {\n"
158             + "    let toStr = null;\n"
159             + "    if (toStr === null && e instanceof EvalError) { toStr = ''; }\n"
160             + "    if (toStr === null && e instanceof RangeError) { toStr = ''; }\n"
161             + "    if (toStr === null && e instanceof ReferenceError) { toStr = ''; }\n"
162             + "    if (toStr === null && e instanceof SyntaxError) { toStr = ''; }\n"
163             + "    if (toStr === null && e instanceof TypeError) { toStr = ''; }\n"
164             + "    if (toStr === null && e instanceof URIError) { toStr = ''; }\n"
165             + "    if (toStr === null && e instanceof AggregateError) { toStr = '/AggregateError'; }\n"
166             + "    if (toStr === null && typeof InternalError == 'function' "
167                           + "&& e instanceof InternalError) { toStr = '/InternalError'; }\n"
168             + "    if (toStr === null) {\n"
169             + "      let rx = /\\[object (.*)\\]/;\n"
170             + "      toStr = Object.prototype.toString.call(e);\n"
171             + "      let match = rx.exec(toStr);\n"
172             + "      if (match != null) { toStr = '/' + match[1]; }\n"
173             + "    }"
174             + "    log(e.name + toStr);\n"
175             + "  }\n";
176 
177     /**
178      * Function used in many tests.
179      */
180     public static final String LOG_TITLE_FUNCTION =
181             "  function log(msg) { window.document.title += msg + '\\u00a7'; }\n"
182             + LOG_EX_FUNCTION;
183 
184     /**
185      * Function used in many tests.
186      */
187     public static final String LOG_TITLE_FUNCTION_NORMALIZE =
188             "  function log(msg) { "
189                     + "msg = '' + msg; "
190                     + "msg = msg.replace(/ /g, '\\\\s'); "
191                     + "msg = msg.replace(/\\n/g, '\\\\n'); "
192                     + "msg = msg.replace(/\\r/g, '\\\\r'); "
193                     + "msg = msg.replace(/\\t/g, '\\\\t'); "
194                     + "msg = msg.replace(/\\u00a0/g, '\\\\u00a0'); "
195                     + "msg = msg.replace(/\\u001e/g, '\\\\u001e'); "
196                     + "window.document.title += msg + '\u00A7';}\n"
197 
198                     + LOG_EX_FUNCTION;
199 
200     /**
201      * Function used in many tests.
202      */
203     public static final String LOG_WINDOW_NAME_FUNCTION =
204             "  function log(msg) { window.top.name += msg + '\\u00a7'; }\n"
205             + "  window.top.name = '';"
206             + LOG_EX_FUNCTION;
207 
208     /**
209      * Function used in many tests.
210      */
211     public static final String LOG_SESSION_STORAGE_FUNCTION =
212             "  function log(msg) { "
213             + "var l = sessionStorage.getItem('Log');"
214             + "sessionStorage.setItem('Log', (null === l?'':l) + msg + '\\u00a7'); }\n";
215 
216     /**
217      * Function used in many tests.
218      */
219     public static final String LOG_TEXTAREA_FUNCTION = "  function log(msg) { "
220             + "document.getElementById('myLog').value += msg + '\u00A7';}\n"
221             + LOG_EX_FUNCTION;
222 
223     /**
224      * HtmlSniped to insert text area used for logging.
225      */
226     public static final String LOG_TEXTAREA = "  <textarea id='myLog' cols='80' rows='22'></textarea>\n";
227 
228     /**
229      * The system property for automatically fixing the test case expectations.
230      */
231     public static final String AUTOFIX_ = "htmlunit.autofix";
232 
233     /**
234      * All browsers supported.
235      */
236     private static final List<BrowserVersion> ALL_BROWSERS_ = List.of(BrowserVersion.CHROME, BrowserVersion.EDGE, BrowserVersion.FIREFOX, BrowserVersion.FIREFOX_ESR);
237 
238     /**
239      * Browsers which run by default.
240      */
241     private static final BrowserVersion[] DEFAULT_RUNNING_BROWSERS_ =
242         {BrowserVersion.CHROME,
243             BrowserVersion.EDGE,
244             BrowserVersion.FIREFOX,
245             BrowserVersion.FIREFOX_ESR};
246 
247     private static final Log LOG = LogFactory.getLog(WebDriverTestCase.class);
248 
249     private static Set<String> BROWSERS_PROPERTIES_;
250     private static String CHROME_BIN_;
251     private static String EDGE_BIN_;
252     private static String GECKO_BIN_;
253     private static String FF_BIN_;
254     private static String FF_ESR_BIN_;
255 
256     /** The driver cache. */
257     protected static final Map<BrowserVersion, WebDriver> WEB_DRIVERS_ = new HashMap<>();
258 
259     /** The driver cache for real browsers. */
260     protected static final Map<BrowserVersion, WebDriver> WEB_DRIVERS_REAL_BROWSERS = new HashMap<>();
261     private static final Map<BrowserVersion, Integer> WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT = new HashMap<>();
262 
263     private static Server STATIC_SERVER_;
264     private static String STATIC_SERVER_STARTER_; // stack trace to save the server start code location
265     // second server for cross-origin tests.
266     private static Server STATIC_SERVER2_;
267     private static String STATIC_SERVER2_STARTER_; // stack trace to save the server start code location
268     // third server for multi-origin cross-origin tests.
269     private static Server STATIC_SERVER3_;
270     private static String STATIC_SERVER3_STARTER_; // stack trace to save the server start code location
271 
272     private static Boolean LAST_TEST_UsesMockWebConnection_;
273     private static final Executor EXECUTOR_POOL = Executors.newFixedThreadPool(4);
274 
275     private boolean useRealBrowser_;
276 
277     /**
278      * The HtmlUnitDriver.
279      */
280     private HtmlUnitDriver webDriver_;
281 
282     /**
283      * Override this function in a test class to ask for STATIC_SERVER2_ to be set up.
284      * @return true if two servers are needed.
285      */
286     protected boolean needThreeConnections() {
287         return false;
288     }
289 
290     /**
291      * @return the browser properties (and initializes them lazy)
292      */
293     public static Set<String> getBrowsersProperties() {
294         if (BROWSERS_PROPERTIES_ == null) {
295             try {
296                 final Properties properties = new Properties();
297                 final File file = new File("test.properties");
298                 if (file.exists()) {
299                     try (FileInputStream in = new FileInputStream(file)) {
300                         properties.load(in);
301                     }
302 
303                     String browsersValue = properties.getProperty("browsers");
304                     if (browsersValue == null || browsersValue.isEmpty()) {
305                         browsersValue = "hu";
306                     }
307                     BROWSERS_PROPERTIES_ = new HashSet<>(Arrays.asList(browsersValue.replaceAll(" ", "")
308                             .toLowerCase(Locale.ROOT).split(",")));
309                     CHROME_BIN_ = properties.getProperty("chrome.bin");
310                     EDGE_BIN_ = properties.getProperty("edge.bin");
311 
312                     GECKO_BIN_ = properties.getProperty("geckodriver.bin");
313                     FF_BIN_ = properties.getProperty("ff.bin");
314                     FF_ESR_BIN_ = properties.getProperty("ff-esr.bin");
315 
316                     final boolean autofix = Boolean.parseBoolean(properties.getProperty("autofix"));
317                     System.setProperty(AUTOFIX_, Boolean.toString(autofix));
318                 }
319             }
320             catch (final Exception e) {
321                 LOG.error("Error reading htmlunit.properties. Ignoring!", e);
322             }
323             if (BROWSERS_PROPERTIES_ == null) {
324                 BROWSERS_PROPERTIES_ = new HashSet<>(Arrays.asList("hu"));
325             }
326             if (BROWSERS_PROPERTIES_.contains("hu")) {
327                 for (final BrowserVersion browserVersion : DEFAULT_RUNNING_BROWSERS_) {
328                     BROWSERS_PROPERTIES_.add("hu-" + browserVersion.getNickname().toLowerCase());
329                 }
330             }
331         }
332         return BROWSERS_PROPERTIES_;
333     }
334 
335     /**
336      * @return the list of supported browsers
337      */
338     public static List<BrowserVersion> allBrowsers() {
339         return ALL_BROWSERS_;
340     }
341 
342     /**
343      * Configure the driver only once.
344      * @return the driver
345      */
346     protected WebDriver getWebDriver() {
347         final BrowserVersion browserVersion = getBrowserVersion();
348         WebDriver driver;
349         if (useRealBrowser()) {
350             synchronized (WEB_DRIVERS_REAL_BROWSERS) {
351                 driver = WEB_DRIVERS_REAL_BROWSERS.get(browserVersion);
352                 if (driver != null) {
353                     // there seems to be a memory leak at least with FF;
354                     // we have to restart sometimes
355                     Integer count = WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.get(browserVersion);
356                     if (null == count) {
357                         count = -1;
358                     }
359                     count += 1;
360                     if (count >= 1000) {
361                         shutDownReal(browserVersion);
362                         driver = null;
363                     }
364                     else {
365                         WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.put(browserVersion, count);
366                     }
367                 }
368 
369                 if (driver == null) {
370                     try {
371                         driver = buildWebDriver();
372                     }
373                     catch (final IOException e) {
374                         throw new RuntimeException(e);
375                     }
376 
377                     WEB_DRIVERS_REAL_BROWSERS.put(browserVersion, driver);
378                     WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.put(browserVersion, 0);
379                 }
380             }
381         }
382         else {
383             driver = WEB_DRIVERS_.get(browserVersion);
384             if (driver == null) {
385                 try {
386                     driver = buildWebDriver();
387                 }
388                 catch (final IOException e) {
389                     throw new RuntimeException(e);
390                 }
391 
392                 if (isWebClientCached()) {
393                     WEB_DRIVERS_.put(browserVersion, driver);
394                 }
395             }
396         }
397         return driver;
398     }
399 
400     /**
401      * Closes the drivers.
402      * @throws Exception If an error occurs
403      */
404     @AfterAll
405     public static void shutDownAll() throws Exception {
406         for (final WebDriver driver : WEB_DRIVERS_.values()) {
407             driver.quit();
408         }
409         WEB_DRIVERS_.clear();
410 
411         shutDownRealBrowsers();
412 
413         stopWebServers();
414     }
415 
416     /**
417      * Closes the real browser drivers.
418      */
419     private static void shutDownRealBrowsers() {
420         synchronized (WEB_DRIVERS_REAL_BROWSERS) {
421             for (final WebDriver driver : WEB_DRIVERS_REAL_BROWSERS.values()) {
422                 quit(driver);
423             }
424             WEB_DRIVERS_REAL_BROWSERS.clear();
425             WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.clear();
426         }
427     }
428 
429     /**
430      * Closes the real browser drivers.
431      * @param browser the real browser to close
432      */
433     private static void shutDownReal(final BrowserVersion browser) {
434         synchronized (WEB_DRIVERS_REAL_BROWSERS) {
435             final WebDriver driver = WEB_DRIVERS_REAL_BROWSERS.get(browser);
436             if (driver != null) {
437                 quit(driver);
438                 WEB_DRIVERS_REAL_BROWSERS.remove(browser);
439                 WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.remove(browser);
440             }
441         }
442     }
443 
444     private static void quit(final WebDriver driver) {
445         if (driver != null) {
446             try {
447                 driver.quit();
448             }
449             catch (final UnreachableBrowserException e) {
450                 LOG.error("Can't quit browser", e);
451                 // ignore, the browser is gone
452             }
453             catch (final NoClassDefFoundError e) {
454                 LOG.error("Can't quit browser", e);
455                 // ignore, the browser is gone
456             }
457             catch (final UnsatisfiedLinkError e) {
458                 LOG.error("Can't quit browser", e);
459                 // ignore, the browser is gone
460             }
461         }
462     }
463 
464     /**
465      * Asserts all static servers are null.
466      * @throws Exception if it fails
467      */
468     protected static void assertWebServersStopped() throws Exception {
469         Assertions.assertNull(STATIC_SERVER_, STATIC_SERVER_STARTER_);
470         Assertions.assertNull(STATIC_SERVER2_, STATIC_SERVER2_STARTER_);
471         Assertions.assertNull(STATIC_SERVER3_, STATIC_SERVER3_STARTER_);
472     }
473 
474     /**
475      * Stops all WebServers.
476      * @throws Exception if it fails
477      */
478     protected static void stopWebServers() throws Exception {
479         JettyServerUtils.stopServer(STATIC_SERVER_);
480         STATIC_SERVER_ = null;
481 
482         JettyServerUtils.stopServer(STATIC_SERVER2_);
483         STATIC_SERVER2_ = null;
484 
485         JettyServerUtils.stopServer(STATIC_SERVER3_);
486         STATIC_SERVER3_ = null;
487 
488         LAST_TEST_UsesMockWebConnection_ = null;
489     }
490 
491     /**
492      * @return whether to use real browser or not.
493      */
494     public boolean useRealBrowser() {
495         return useRealBrowser_;
496     }
497 
498     /**
499      * Sets whether to use real browser or not.
500      * @param useRealBrowser whether to use real browser or not
501      */
502     public void setUseRealBrowser(final boolean useRealBrowser) {
503         useRealBrowser_ = useRealBrowser;
504     }
505 
506     /**
507      * When using a real browser seep for some timem.
508      *
509      * @param  millis the length of time to sleep in milliseconds
510      * @throws InterruptedException in case of an interrupt
511      */
512     public void sleepRealBrowser(final long millis) throws InterruptedException {
513         if (useRealBrowser()) {
514             Thread.sleep(millis);
515         }
516     }
517 
518     /**
519      * Builds a new WebDriver instance.
520      * @return the instance
521      * @throws IOException in case of exception
522      */
523     protected WebDriver buildWebDriver() throws IOException {
524         if (useRealBrowser()) {
525             if (BrowserVersion.EDGE.isSameBrowser(getBrowserVersion())) {
526                 final EdgeDriverService service = new EdgeDriverService.Builder()
527                         .withLogOutput(System.out)
528                         .usingDriverExecutable(new File(EDGE_BIN_))
529 
530                         .withAppendLog(true)
531                         .withReadableTimestamp(true)
532 
533                         .build();
534 
535                 final String locale = getBrowserVersion().getBrowserLocale().toLanguageTag();
536 
537                 final EdgeOptions options = new EdgeOptions();
538                 // BiDi
539                 // options.setCapability("webSocketUrl", true);
540 
541                 options.addArguments("--lang=" + locale);
542                 // https://stackoverflow.com/questions/11289597/webdriver-how-to-specify-preferred-languages-for-chrome
543                 options.setExperimentalOption("prefs", Map.of("intl.accept_languages", locale));
544                 options.addArguments("--remote-allow-origins=*");
545 
546                 // seems to be not required for edge
547                 // options.addArguments("--disable-search-engine-choice-screen");
548                 // see https://www.selenium.dev/blog/2024/chrome-browser-woes/
549                 // options.addArguments("--disable-features=OptimizationGuideModelDownloading,"
550                 //         + "OptimizationHintsFetching,OptimizationTargetPrediction,OptimizationHints");
551 
552                 final EdgeDriver edge = new EdgeDriver(service, options);
553 
554                 final DevTools devTools = edge.getDevTools();
555                 devTools.createSession();
556 
557                 final String tz = getBrowserVersion().getSystemTimezone().getID();
558                 devTools.send(Emulation.setTimezoneOverride(tz));
559 
560                 return edge;
561             }
562 
563             if (BrowserVersion.CHROME.isSameBrowser(getBrowserVersion())) {
564                 final ChromeDriverService service = new ChromeDriverService.Builder()
565                         .withLogOutput(System.out)
566                         .usingDriverExecutable(new File(CHROME_BIN_))
567 
568                         .withAppendLog(true)
569                         .withReadableTimestamp(true)
570 
571                         .build();
572 
573                 final String locale = getBrowserVersion().getBrowserLocale().toLanguageTag();
574 
575                 final ChromeOptions options = new ChromeOptions();
576                 // BiDi
577                 // options.setCapability("webSocketUrl", true);
578 
579                 options.addArguments("--lang=" + locale);
580                 // https://stackoverflow.com/questions/11289597/webdriver-how-to-specify-preferred-languages-for-chrome
581                 options.setExperimentalOption("prefs", Map.of("intl.accept_languages", locale));
582                 options.addArguments("--remote-allow-origins=*");
583                 options.addArguments("--disable-search-engine-choice-screen");
584                 // see https://www.selenium.dev/blog/2024/chrome-browser-woes/
585                 options.addArguments("--disable-features=OptimizationGuideModelDownloading,"
586                         + "OptimizationHintsFetching,OptimizationTargetPrediction,OptimizationHints");
587 
588                 final ChromeDriver chrome = new ChromeDriver(service, options);
589 
590                 final DevTools devTools = chrome.getDevTools();
591                 devTools.createSession();
592 
593                 final String tz = getBrowserVersion().getSystemTimezone().getID();
594                 devTools.send(Emulation.setTimezoneOverride(tz));
595 
596                 return chrome;
597             }
598 
599             if (BrowserVersion.FIREFOX.isSameBrowser(getBrowserVersion())) {
600                 return createFirefoxDriver(GECKO_BIN_, FF_BIN_);
601             }
602 
603             if (BrowserVersion.FIREFOX_ESR.isSameBrowser(getBrowserVersion())) {
604                 return createFirefoxDriver(GECKO_BIN_, FF_ESR_BIN_);
605             }
606 
607             throw new RuntimeException("Unexpected BrowserVersion: " + getBrowserVersion());
608         }
609 
610         if (webDriver_ == null) {
611             final HtmlUnitDriverOptions driverOptions = new HtmlUnitDriverOptions(getBrowserVersion());
612 
613             if (isWebClientCached()) {
614                 driverOptions.setCapability(HtmlUnitOption.optHistorySizeLimit, 0);
615             }
616 
617             if (getWebClientTimeout() != null) {
618                 driverOptions.setCapability(HtmlUnitOption.optTimeout, getWebClientTimeout());
619             }
620             webDriver_ = new HtmlUnitDriver(driverOptions);
621             webDriver_.setExecutor(EXECUTOR_POOL);
622         }
623         return webDriver_;
624     }
625 
626     private FirefoxDriver createFirefoxDriver(final String geckodriverBinary, final String binary) {
627         final FirefoxDriverService service = new GeckoDriverService.Builder()
628                 .withLogOutput(System.out)
629                 .usingDriverExecutable(new File(geckodriverBinary))
630                 .build();
631 
632         final FirefoxOptions options = new FirefoxOptions();
633         // BiDi
634         // options.setCapability("webSocketUrl", true);
635 
636         options.setBinary(binary);
637 
638         String locale = getBrowserVersion().getBrowserLocale().toLanguageTag();
639         locale = locale + "," + getBrowserVersion().getBrowserLocale().getLanguage();
640 
641         final FirefoxProfile profile = new FirefoxProfile();
642         profile.setPreference("intl.accept_languages", locale);
643         // no idea so far how to set this
644         // final String tz = getBrowserVersion().getSystemTimezone().getID();
645         // profile.setPreference("intl.tz", tz);
646         options.setProfile(profile);
647 
648         return new FirefoxDriver(service, options);
649     }
650 
651     /**
652      * Starts the web server delivering response from the provided connection.
653      * @param mockConnection the sources for responses
654      * @param serverCharset the {@link Charset} at the server side
655      * @throws Exception if a problem occurs
656      */
657     protected void startWebServer(final MockWebConnection mockConnection, final Charset serverCharset)
658             throws Exception {
659         if (Boolean.FALSE.equals(LAST_TEST_UsesMockWebConnection_)) {
660             stopWebServers();
661         }
662 
663         // The mock connection servlet call site under both servers, so long as tests
664         // keep the URLs distinct.
665         final Map<String, Class<? extends Servlet>> servlets = new HashMap<>();
666         servlets.put("/*", MockWebConnectionServlet.class);
667 
668         LAST_TEST_UsesMockWebConnection_ = Boolean.TRUE;
669 
670         if (STATIC_SERVER_ == null) {
671 
672             final Server server = JettyServerUtils.startWebServer(PORT, "./", servlets, serverCharset, isBasicAuthentication(), SSLVariant.NONE);
673             STATIC_SERVER_STARTER_ = ExceptionUtils.getStackTrace(new Throwable("StaticServerStarter"));
674             STATIC_SERVER_ = server;
675         }
676         MockWebConnectionServlet.MockConnection_ = mockConnection;
677 
678         if (STATIC_SERVER2_ == null && needThreeConnections()) {
679             final Server server2 = JettyServerUtils.startWebServer(PORT2, "./", servlets, null, false, SSLVariant.NONE);
680             STATIC_SERVER2_STARTER_ = ExceptionUtils.getStackTrace(new Throwable("StaticServer2Starter"));
681             STATIC_SERVER2_ = server2;
682 
683             final Server server3 = JettyServerUtils.startWebServer(PORT3, "./", servlets, null, false, SSLVariant.NONE);
684             STATIC_SERVER3_STARTER_ = ExceptionUtils.getStackTrace(new Throwable("StaticServer3Starter"));
685             STATIC_SERVER3_ = server3;
686         }
687     }
688 
689     /**
690      * Returns whether to use basic authentication for all resources or not.
691      * The default implementation returns false.
692      * @return whether to use basic authentication or not
693      */
694     protected boolean isBasicAuthentication() {
695         return false;
696     }
697 
698     /**
699      * Starts the web server on the default {@link #PORT}.
700      * The given resourceBase is used to be the ROOT directory that serves the default context.
701      * <p><b>Don't forget to stop the returned HttpServer after the test</b>
702      *
703      * @param resourceBase the base of resources for the default context
704      * @param servlets map of {String, Class} pairs: String is the path spec, while class is the class
705      * @throws Exception if the test fails
706      */
707     protected static void startWebServer(final String resourceBase, final Map<String, Class<? extends Servlet>> servlets) throws Exception {
708         stopWebServers();
709         LAST_TEST_UsesMockWebConnection_ = Boolean.FALSE;
710 
711         STATIC_SERVER_STARTER_ = ExceptionUtils.getStackTrace(new Throwable("StaticServerStarter"));
712         STATIC_SERVER_ = JettyServerUtils.startWebServer(PORT, resourceBase, servlets, null, false, SSLVariant.NONE);
713     }
714 
715     /**
716      * Starts the <b>second</b> web server on the default {@link #PORT2}.
717      * The given resourceBase is used to be the ROOT directory that serves the default context.
718      * <p><b>Don't forget to stop the returned HttpServer after the test</b>
719      *
720      * @param resourceBase the base of resources for the default context
721      * @param servlets map of {String, Class} pairs: String is the path spec, while class is the class
722      * @throws Exception if the test fails
723      */
724     protected static void startWebServer2(final String resourceBase, final Map<String, Class<? extends Servlet>> servlets) throws Exception {
725         if (STATIC_SERVER2_ != null) {
726             JettyServerUtils.stopServer(STATIC_SERVER2_);
727         }
728         STATIC_SERVER2_STARTER_ = ExceptionUtils.getStackTrace(new Throwable("StaticServer2Starter"));
729         STATIC_SERVER2_ = JettyServerUtils.startWebServer(PORT2, resourceBase, servlets, null, false, SSLVariant.NONE);
730     }
731 
732     /**
733      * Servlet delivering content from a MockWebConnection.
734      */
735     public static class MockWebConnectionServlet extends HttpServlet {
736         private static MockWebConnection MockConnection_;
737 
738         public static void setMockconnection(final MockWebConnection connection) {
739             MockConnection_ = connection;
740         }
741 
742         /**
743          * {@inheritDoc}
744          */
745         @Override
746         protected void service(final HttpServletRequest request, final HttpServletResponse response)
747                 throws ServletException, IOException {
748 
749             try {
750                 doService(request, response);
751             }
752             catch (final ServletException e) {
753                 throw e;
754             }
755             catch (final IOException e) {
756                 throw e;
757             }
758             catch (final Exception e) {
759                 throw new ServletException(e);
760             }
761         }
762 
763         private static void doService(final HttpServletRequest request, final HttpServletResponse response)
764                 throws Exception {
765             String url = request.getRequestURL().toString();
766             if (LOG.isDebugEnabled()) {
767                 LOG.debug(request.getMethod() + " " + url);
768             }
769 
770             if (url.endsWith("/favicon.ico")) {
771                 response.setStatus(HttpServletResponse.SC_NOT_FOUND);
772                 return;
773             }
774 
775             if (url.contains("/delay")) {
776                 final String delay = StringUtils.substringBetween(url, "/delay", "/");
777                 final int ms = Integer.parseInt(delay);
778                 if (LOG.isDebugEnabled()) {
779                     LOG.debug("Sleeping for " + ms + " before to deliver " + url);
780                 }
781                 Thread.sleep(ms);
782             }
783 
784             // copy parameters
785             final List<NameValuePair> requestParameters = new ArrayList<>();
786             try {
787                 for (final Enumeration<String> paramNames = request.getParameterNames();
788                         paramNames.hasMoreElements();) {
789                     final String name = paramNames.nextElement();
790                     final String[] values = request.getParameterValues(name);
791                     for (final String value : values) {
792                         requestParameters.add(new NameValuePair(name, value));
793                     }
794                 }
795             }
796             catch (final IllegalArgumentException e) {
797                 // Jetty 8.1.7 throws it in getParameterNames for a query like "cb=%%RANDOM_NUMBER%%"
798                 // => we should use a more low level test server
799                 requestParameters.clear();
800                 final String query = request.getQueryString();
801                 if (query != null) {
802                     url += "?" + query;
803                 }
804             }
805 
806             final String queryString = request.getQueryString();
807             if (StringUtils.isNotBlank(queryString)) {
808                 url = url + "?" + queryString;
809             }
810             final URL requestedUrl = new URL(url);
811             final WebRequest webRequest = new WebRequest(requestedUrl);
812 
813             final String method = request.getMethod().toUpperCase(Locale.ROOT);
814             webRequest.setHttpMethod(HttpMethod.valueOf(method));
815 
816             // copy headers
817             for (final Enumeration<String> en = request.getHeaderNames(); en.hasMoreElements();) {
818                 final String headerName = en.nextElement();
819                 final String headerValue = request.getHeader(headerName);
820                 webRequest.setAdditionalHeader(headerName, headerValue);
821             }
822 
823             if (requestParameters.isEmpty() && request.getContentLength() > 0) {
824                 final byte[] buffer = new byte[request.getContentLength()];
825                 IOUtils.read(request.getInputStream(), buffer, 0, buffer.length);
826 
827                 final String encoding = request.getCharacterEncoding();
828                 if (encoding == null) {
829                     webRequest.setRequestBody(new String(buffer, ISO_8859_1));
830                     webRequest.setCharset(null);
831                 }
832                 else {
833                     webRequest.setRequestBody(new String(buffer, encoding));
834                     webRequest.setCharset(Charset.forName(encoding));
835                 }
836             }
837             else {
838                 webRequest.setRequestParameters(requestParameters);
839             }
840 
841             // check content type if it is multipart enctype
842             if (request.getContentType() != null
843                         && request.getContentType().startsWith(FormEncodingType.MULTIPART.getName())) {
844                 webRequest.setEncodingType(FormEncodingType.MULTIPART);
845             }
846 
847             final RawResponseData resp = MockConnection_.getRawResponse(webRequest);
848 
849             // write WebResponse to HttpServletResponse
850             response.setStatus(resp.getStatusCode());
851 
852             boolean charsetInContentType = false;
853             for (final NameValuePair responseHeader : resp.getHeaders()) {
854                 final String headerName = responseHeader.getName();
855                 if (HttpHeader.CONTENT_TYPE.equals(headerName) && responseHeader.getValue().contains("charset=")) {
856                     charsetInContentType = true;
857                 }
858                 response.addHeader(headerName, responseHeader.getValue());
859             }
860 
861             if (resp.getByteContent() != null) {
862                 response.getOutputStream().write(resp.getByteContent());
863             }
864             else {
865                 if (!charsetInContentType) {
866                     response.setCharacterEncoding(resp.getCharset().name());
867                 }
868                 response.getWriter().print(resp.getStringContent());
869             }
870             response.flushBuffer();
871         }
872     }
873 
874     /**
875      * Same as {@link #loadPageWithAlerts2(String)}... but doesn't verify the alerts.
876      * @param html the HTML to use
877      * @return the web driver
878      * @throws Exception if something goes wrong
879      */
880     protected final WebDriver loadPage2(final String html) throws Exception {
881         return loadPage2(html, URL_FIRST);
882     }
883 
884     /**
885      * Same as {@link #loadPageWithAlerts2(String)}... but doesn't verify the alerts.
886      * @param html the HTML to use
887      * @param url the url to use to load the page
888      * @return the web driver
889      * @throws Exception if something goes wrong
890      */
891     protected final WebDriver loadPage2(final String html, final URL url) throws Exception {
892         return loadPage2(html, url, "text/html;charset=ISO-8859-1", ISO_8859_1, null);
893     }
894 
895     /**
896      * Same as {@link #loadPageWithAlerts2(String)}... but doesn't verify the alerts.
897      * @param html the HTML to use
898      * @param url the url to use to load the page
899      * @param contentType the content type to return
900      * @param charset the charset
901      * @return the web driver
902      * @throws Exception if something goes wrong
903      */
904     protected final WebDriver loadPage2(final String html, final URL url,
905             final String contentType, final Charset charset) throws Exception {
906         return loadPage2(html, url, contentType, charset, null);
907     }
908 
909     /**
910      * Same as {@link #loadPageWithAlerts2(String)}... but doesn't verify the alerts.
911      * @param html the HTML to use
912      * @param url the url to use to load the page
913      * @param contentType the content type to return
914      * @param charset the charset
915      * @param serverCharset the charset at the server side.
916      * @return the web driver
917      * @throws Exception if something goes wrong
918      */
919     protected final WebDriver loadPage2(final String html, final URL url,
920             final String contentType, final Charset charset, final Charset serverCharset) throws Exception {
921         getMockWebConnection().setResponse(url, html, contentType, charset);
922         return loadPage2(url, serverCharset);
923     }
924 
925     /**
926      * Load the page from the url.
927      * @param url the url to use to load the page
928      * @param serverCharset the charset at the server side.
929      * @return the web driver
930      * @throws Exception if something goes wrong
931      */
932     protected final WebDriver loadPage2(final URL url, final Charset serverCharset) throws Exception {
933         startWebServer(getMockWebConnection(), serverCharset);
934 
935         WebDriver driver = getWebDriver();
936         if (!(driver instanceof HtmlUnitDriver)) {
937             try {
938                 resizeIfNeeded(driver);
939             }
940             catch (final NoSuchSessionException e) {
941                 // maybe the driver was killed by the test before; setup a new one
942                 shutDownRealBrowsers();
943 
944                 driver = getWebDriver();
945                 resizeIfNeeded(driver);
946             }
947         }
948         driver.get(url.toExternalForm());
949 
950         return driver;
951     }
952 
953     /**
954      * Same as {@link #loadPage2(String)} with additional servlet configuration.
955      * @param html the HTML to use for the default response
956      * @param servlets the additional servlets to configure with their mapping
957      * @return the web driver
958      * @throws Exception if something goes wrong
959      */
960     protected final WebDriver loadPage2(final String html,
961             final Map<String, Class<? extends Servlet>> servlets) throws Exception {
962         return loadPage2(html, URL_FIRST, servlets, null);
963     }
964 
965     /**
966      * Same as {@link #loadPage2(String, URL)}, but with additional servlet configuration.
967      * @param html the HTML to use for the default page
968      * @param url the URL to use to load the page
969      * @param servlets the additional servlets to configure with their mapping
970      * @return the web driver
971      * @throws Exception if something goes wrong
972      */
973     protected final WebDriver loadPage2(final String html, final URL url,
974             final Map<String, Class<? extends Servlet>> servlets) throws Exception {
975         return loadPage2(html, url, servlets, null);
976     }
977 
978     /**
979      * Same as {@link #loadPage2(String, URL)}, but with additional servlet configuration.
980      * @param html the HTML to use for the default page
981      * @param url the URL to use to load the page
982      * @param servlets the additional servlets to configure with their mapping
983      * @param servlets2 the additional servlets to configure with their mapping for a second server
984      * @return the web driver
985      * @throws Exception if something goes wrong
986      */
987     protected final WebDriver loadPage2(final String html, final URL url,
988             final Map<String, Class<? extends Servlet>> servlets,
989             final Map<String, Class<? extends Servlet>> servlets2) throws Exception {
990         servlets.put("/*", MockWebConnectionServlet.class);
991         getMockWebConnection().setResponse(url, html);
992         MockWebConnectionServlet.MockConnection_ = getMockWebConnection();
993 
994         startWebServer("./", servlets);
995         if (servlets2 != null) {
996             startWebServer2("./", servlets2);
997         }
998 
999         WebDriver driver = getWebDriver();
1000         if (!(driver instanceof HtmlUnitDriver)) {
1001             try {
1002                 resizeIfNeeded(driver);
1003             }
1004             catch (final NoSuchSessionException e) {
1005                 // maybe the driver was killed by the test before; setup a new one
1006                 shutDownRealBrowsers();
1007 
1008                 driver = getWebDriver();
1009                 resizeIfNeeded(driver);
1010             }
1011         }
1012         driver.get(url.toExternalForm());
1013 
1014         return driver;
1015     }
1016 
1017     protected void resizeIfNeeded(final WebDriver driver) {
1018         final Dimension size = driver.manage().window().getSize();
1019         if (size.getWidth() != 1272 || size.getHeight() != 768) {
1020             // only resize if needed because it may be quite expensive
1021             driver.manage().window().setSize(new Dimension(1272, 768));
1022         }
1023     }
1024 
1025     /**
1026      * Defines the provided HTML as the response for {@link WebTestCase#URL_FIRST}
1027      * and loads the page with this URL using the current WebDriver version; finally, asserts that the
1028      * alerts equal the expected alerts.
1029      * @param html the HTML to use
1030      * @return the web driver
1031      * @throws Exception if something goes wrong
1032      */
1033     protected final WebDriver loadPageWithAlerts2(final String html) throws Exception {
1034         return loadPageWithAlerts2(html, URL_FIRST, DEFAULT_WAIT_TIME);
1035     }
1036 
1037     /**
1038      * Defines the provided HTML as the response for {@link WebTestCase#URL_FIRST}
1039      * and loads the page with this URL using the current WebDriver version; finally, asserts that the
1040      * alerts equal the expected alerts.
1041      * @param html the HTML to use
1042      * @return the web driver
1043      * @throws Exception if something goes wrong
1044      */
1045     protected final WebDriver loadPageVerifyTitle2(final String html) throws Exception {
1046         return loadPageVerifyTitle2(html, getExpectedAlerts());
1047     }
1048 
1049     protected final WebDriver loadPageVerifyTitle2(final String html, final String... expectedAlerts) throws Exception {
1050         final WebDriver driver = loadPage2(html);
1051         return verifyTitle2(driver, expectedAlerts);
1052     }
1053 
1054     protected final WebDriver verifyTitle2(final Duration maxWaitTime, final WebDriver driver,
1055             final String... expectedAlerts) throws Exception {
1056 
1057         final StringBuilder expected = new StringBuilder();
1058         for (String expectedAlert : expectedAlerts) {
1059             expected.append(expectedAlert).append('\u00A7');
1060         }
1061         final String expectedTitle = expected.toString();
1062 
1063         final long maxWait = System.currentTimeMillis() + maxWaitTime.toMillis();
1064 
1065         while (System.currentTimeMillis() < maxWait) {
1066             try {
1067                 final String title = driver.getTitle();
1068                 assertEquals(expectedTitle, title);
1069                 return driver;
1070             }
1071             catch (final AssertionError e) {
1072                 // ignore and wait
1073             }
1074         }
1075 
1076         assertEquals(expectedTitle, driver.getTitle());
1077         return driver;
1078     }
1079 
1080     protected final WebDriver verifyTitle2(final WebDriver driver,
1081             final String... expectedAlerts) throws Exception {
1082         if (expectedAlerts.length == 0) {
1083             assertEquals("", driver.getTitle());
1084             return driver;
1085         }
1086 
1087         final StringBuilder expected = new StringBuilder();
1088         for (String expectedAlert : expectedAlerts) {
1089             expected.append(expectedAlert).append('\u00A7');
1090         }
1091 
1092         final String title = driver.getTitle();
1093         try {
1094             assertEquals(expected.toString(), title);
1095         }
1096         catch (final AssertionError e) {
1097             if (useRealBrowser() && StringUtils.isEmpty(title)) {
1098                 Thread.sleep(42);
1099                 assertEquals(expected.toString(), driver.getTitle());
1100                 return driver;
1101             }
1102             throw e;
1103         }
1104         return driver;
1105     }
1106 
1107     protected final WebDriver loadPageVerifyTextArea2(final String html) throws Exception {
1108         return loadPageTextArea2(html, getExpectedAlerts());
1109     }
1110 
1111     protected final WebDriver loadPageTextArea2(final String html, final String... expectedAlerts) throws Exception {
1112         final WebDriver driver = loadPage2(html);
1113         return verifyTextArea2(driver, expectedAlerts);
1114     }
1115 
1116     protected final WebDriver verifyTextArea2(final WebDriver driver,
1117             final String... expectedAlerts) throws Exception {
1118         final WebElement textArea = driver.findElement(By.id("myLog"));
1119         String value = textArea.getDomProperty("value");
1120 
1121         if (expectedAlerts.length == 0) {
1122             assertEquals("", value);
1123             return driver;
1124         }
1125 
1126         final StringBuilder expected = new StringBuilder();
1127         for (String expectedAlert : expectedAlerts) {
1128             expected.append(expectedAlert).append('\u00A7');
1129         }
1130 
1131         if (expectedAlerts.length == 1 && expectedAlerts[0].startsWith("data:image/png;base64,")) {
1132             try {
1133                 // fastpath
1134                 assertEquals(expected.toString(), value);
1135             } catch (final AssertionFailedError e) {
1136                 // starting with FF151 we have canvas fingerprinting protection
1137                 // therefore we have to compare the pictures
1138                 if (value.endsWith("\u00A7")) {
1139                     value = value.substring(0, value.length() - 1);
1140                 }
1141 
1142                 compareImages(expectedAlerts[0], value);
1143                 return driver;
1144             }
1145         }
1146 
1147         verify(() -> textArea.getDomProperty("value"), expected.toString());
1148         return driver;
1149     }
1150 
1151     protected final String getJsVariableValue(final WebDriver driver, final String varName) throws Exception {
1152         final String script = "return String(" + varName + ")";
1153         final String result = (String) ((JavascriptExecutor) driver).executeScript(script);
1154 
1155         return result;
1156     }
1157 
1158     protected final WebDriver verifyJsVariable(final WebDriver driver, final String varName,
1159             final String expected) throws Exception {
1160         final String result = getJsVariableValue(driver, varName);
1161         assertEquals(expected, result);
1162 
1163         return driver;
1164     }
1165 
1166     protected final WebDriver verifyWindowName2(final Duration maxWaitTime, final WebDriver driver,
1167             final String... expectedAlerts) throws Exception {
1168         final long maxWait = System.currentTimeMillis() + maxWaitTime.toMillis();
1169 
1170         while (System.currentTimeMillis() < maxWait) {
1171             try {
1172                 return verifyWindowName2(driver, expectedAlerts);
1173             }
1174             catch (final AssertionError e) {
1175                 // ignore and wait
1176             }
1177         }
1178 
1179         return verifyWindowName2(driver, expectedAlerts);
1180     }
1181 
1182     protected final WebDriver verifyWindowName2(final WebDriver driver,
1183             final String... expectedAlerts) throws Exception {
1184         final StringBuilder expected = new StringBuilder();
1185         for (String expectedAlert : expectedAlerts) {
1186             expected.append(expectedAlert).append('\u00A7');
1187         }
1188 
1189         return verifyJsVariable(driver, "window.top.name", expected.toString());
1190     }
1191 
1192     protected final WebDriver verifySessionStorage2(final WebDriver driver,
1193             final String... expectedAlerts) throws Exception {
1194         final StringBuilder expected = new StringBuilder();
1195         for (String expectedAlert : expectedAlerts) {
1196             expected.append(expectedAlert).append('\u00A7');
1197         }
1198 
1199         return verifyJsVariable(driver, "sessionStorage.getItem('Log')", expected.toString());
1200     }
1201 
1202     /**
1203      * Defines the provided HTML as the response for {@link WebTestCase#URL_FIRST}
1204      * and loads the page with this URL using the current WebDriver version; finally, asserts that the
1205      * alerts equal the expected alerts.
1206      * @param html the HTML to use
1207      * @param maxWaitTime the maximum time to wait to get the alerts (in millis)
1208      * @return the web driver
1209      * @throws Exception if something goes wrong
1210      */
1211     protected final WebDriver loadPageWithAlerts2(final String html, final Duration maxWaitTime) throws Exception {
1212         return loadPageWithAlerts2(html, URL_FIRST, maxWaitTime);
1213     }
1214 
1215     /**
1216      * Same as {@link #loadPageWithAlerts2(String)}, but specifying the default URL.
1217      * @param html the HTML to use
1218      * @param url the URL to use to load the page
1219      * @return the web driver
1220      * @throws Exception if something goes wrong
1221      */
1222     protected final WebDriver loadPageWithAlerts2(final String html, final URL url) throws Exception {
1223         return loadPageWithAlerts2(html, url, DEFAULT_WAIT_TIME);
1224     }
1225 
1226     /**
1227      * Same as {@link #loadPageWithAlerts2(String, long)}, but specifying the default URL.
1228      * @param html the HTML to use
1229      * @param url the URL to use to load the page
1230      * @param maxWaitTime the maximum time to wait to get the alerts (in millis)
1231      * @return the web driver
1232      * @throws Exception if something goes wrong
1233      */
1234     protected final WebDriver loadPageWithAlerts2(final String html, final URL url, final Duration maxWaitTime)
1235             throws Exception {
1236         final WebDriver driver = loadPage2(html, url);
1237 
1238         verifyAlerts(maxWaitTime, driver, getExpectedAlerts());
1239         return driver;
1240     }
1241 
1242     /**
1243      * Verifies the captured alerts.
1244      * @param driver the driver instance
1245      * @param expectedAlerts the expected alerts
1246      * @throws Exception in case of failure
1247      */
1248     protected void verifyAlerts(final WebDriver driver, final String... expectedAlerts) throws Exception {
1249         verifyAlerts(DEFAULT_WAIT_TIME, driver, expectedAlerts);
1250     }
1251 
1252     /**
1253      * Verifies the captured alerts.
1254      *
1255      * @param maxWaitTime the maximum time to wait for the expected alert to be found
1256      * @param driver the driver instance
1257      * @param expected the expected alerts
1258      * @throws Exception in case of failure
1259      */
1260     protected void verifyAlerts(final Duration maxWaitTime, final WebDriver driver, final String... expected)
1261             throws Exception {
1262         final List<String> actualAlerts = getCollectedAlerts(maxWaitTime, driver, expected.length);
1263 
1264         assertEquals(expected.length, actualAlerts.size());
1265 
1266         if (!useRealBrowser()) {
1267             // check if we have data-image Url
1268             for (String s : expected) {
1269                 if (s.startsWith("data:image/png;base64,")) {
1270                     // we have to compare element by element
1271                     for (int j = 0; j < expected.length; j++) {
1272                         if (expected[j].startsWith("data:image/png;base64,")) {
1273                             compareImages(expected[j], actualAlerts.get(j));
1274                         } else {
1275                             assertEquals(expected[j], actualAlerts.get(j));
1276                         }
1277                     }
1278                     return;
1279                 }
1280             }
1281         }
1282 
1283         assertEquals(expected, actualAlerts);
1284     }
1285 
1286     /**
1287      * Same as {@link #loadPageWithAlerts2(String)} with additional servlet configuration.
1288      * @param html the HTML to use for the default response
1289      * @param servlets the additional servlets to configure with their mapping
1290      * @return the web driver
1291      * @throws Exception if something goes wrong
1292      */
1293     protected final WebDriver loadPageWithAlerts2(final String html,
1294             final Map<String, Class<? extends Servlet>> servlets) throws Exception {
1295         return loadPageWithAlerts2(html, URL_FIRST, DEFAULT_WAIT_TIME, servlets);
1296     }
1297 
1298     /**
1299      * Same as {@link #loadPageWithAlerts2(String, URL, long)}, but with additional servlet configuration.
1300      * @param html the HTML to use for the default page
1301      * @param url the URL to use to load the page
1302      * @param maxWaitTime the maximum time to wait to get the alerts (in millis)
1303      * @param servlets the additional servlets to configure with their mapping
1304      * @return the web driver
1305      * @throws Exception if something goes wrong
1306      */
1307     protected final WebDriver loadPageWithAlerts2(final String html, final URL url, final Duration maxWaitTime,
1308             final Map<String, Class<? extends Servlet>> servlets) throws Exception {
1309 
1310         expandExpectedAlertsVariables(URL_FIRST);
1311 
1312         final WebDriver driver = loadPage2(html, url, servlets);
1313         verifyAlerts(maxWaitTime, driver, getExpectedAlerts());
1314 
1315         return driver;
1316     }
1317 
1318     /**
1319      * Loads the provided URL serving responses from {@link #getMockWebConnection()}
1320      * and verifies that the captured alerts are correct.
1321      * @param url the URL to use to load the page
1322      * @return the web driver
1323      * @throws Exception if something goes wrong
1324      */
1325     protected final WebDriver loadPageWithAlerts2(final URL url) throws Exception {
1326         return loadPageWithAlerts2(url, DEFAULT_WAIT_TIME);
1327     }
1328 
1329     /**
1330      * Same as {@link #loadPageWithAlerts2(URL)}, but using with timeout.
1331      * @param url the URL to use to load the page
1332      * @param maxWaitTime the maximum time to wait to get the alerts (in millis)
1333      * @return the web driver
1334      * @throws Exception if something goes wrong
1335      */
1336     protected final WebDriver loadPageWithAlerts2(final URL url, final Duration maxWaitTime) throws Exception {
1337         startWebServer(getMockWebConnection(), null);
1338 
1339         final WebDriver driver = getWebDriver();
1340         driver.get(url.toExternalForm());
1341 
1342         verifyAlerts(maxWaitTime, driver, getExpectedAlerts());
1343         return driver;
1344     }
1345 
1346     /**
1347      * Gets the alerts collected by the driver.
1348      * Note: it currently works only if no new page has been loaded in the window
1349      * @param driver the driver
1350      * @return the collected alerts
1351      * @throws Exception in case of problem
1352      */
1353     protected List<String> getCollectedAlerts(final WebDriver driver) throws Exception {
1354         return getCollectedAlerts(driver, getExpectedAlerts().length);
1355     }
1356 
1357     /**
1358      * Gets the alerts collected by the driver.
1359      * Note: it currently works only if no new page has been loaded in the window
1360      * @param driver the driver
1361      * @param alertsLength the expected length of Alerts
1362      * @return the collected alerts
1363      * @throws Exception in case of problem
1364      */
1365     protected List<String> getCollectedAlerts(final WebDriver driver, final int alertsLength) throws Exception {
1366         return getCollectedAlerts(DEFAULT_WAIT_TIME, driver, alertsLength);
1367     }
1368 
1369     /**
1370      * Gets the alerts collected by the driver.
1371      * Note: it currently works only if no new page has been loaded in the window
1372      * @param maxWaitTime the maximum time to wait to get the alerts (in millis)
1373      * @param driver the driver
1374      * @param alertsLength the expected length of Alerts
1375      * @return the collected alerts
1376      * @throws Exception in case of problem
1377      */
1378     protected List<String> getCollectedAlerts(final Duration maxWaitTime,
1379             final WebDriver driver, final int alertsLength) throws Exception {
1380         final List<String> collectedAlerts = new ArrayList<>();
1381 
1382         long maxWait = System.currentTimeMillis() + maxWaitTime.toMillis();
1383 
1384         while (collectedAlerts.size() < alertsLength && System.currentTimeMillis() < maxWait) {
1385             try {
1386                 final Alert alert = driver.switchTo().alert();
1387                 final String text = alert.getText();
1388 
1389                 collectedAlerts.add(text);
1390                 alert.accept();
1391 
1392                 // handling of alerts requires some time
1393                 // at least for tests with many alerts we have to take this into account
1394                 maxWait += 100;
1395             }
1396             catch (final NoAlertPresentException e) {
1397                 Thread.sleep(10);
1398             }
1399         }
1400 
1401         return collectedAlerts;
1402     }
1403 
1404     /**
1405      * Returns the HtmlElement of the specified WebElement.
1406      * @param webElement the webElement
1407      * @return the HtmlElement
1408      * @throws Exception if an error occurs
1409      */
1410     protected HtmlElement toHtmlElement(final WebElement webElement) throws Exception {
1411         return (HtmlElement) ((HtmlUnitWebElement) webElement).getElement();
1412     }
1413 
1414     /**
1415      * Loads an expectation file for the specified browser search first for a browser specific resource
1416      * and falling back in a general resource.
1417      * @param resourcePrefix the start of the resource name
1418      * @param resourceSuffix the end of the resource name
1419      * @return the content of the file
1420      * @throws Exception in case of error
1421      */
1422     protected String loadExpectation(final String resourcePrefix, final String resourceSuffix) throws Exception {
1423         final Class<?> referenceClass = getClass();
1424         final BrowserVersion browserVersion = getBrowserVersion();
1425 
1426         String realBrowserNyiExpectation = null;
1427         String realNyiExpectation = null;
1428         final String browserExpectation;
1429         final String expectation;
1430         if (!useRealBrowser()) {
1431             // first try nyi
1432             final String browserSpecificNyiResource
1433                     = resourcePrefix + "." + browserVersion.getNickname() + "_NYI" + resourceSuffix;
1434             realBrowserNyiExpectation = loadContent(referenceClass.getResource(browserSpecificNyiResource));
1435 
1436             // next nyi without browser
1437             final String nyiResource = resourcePrefix + ".NYI" + resourceSuffix;
1438             realNyiExpectation = loadContent(referenceClass.getResource(nyiResource));
1439         }
1440 
1441         // implemented - browser specific
1442         final String browserSpecificResource = resourcePrefix + "." + browserVersion.getNickname() + resourceSuffix;
1443         browserExpectation = loadContent(referenceClass.getResource(browserSpecificResource));
1444 
1445         // implemented - all browsers
1446         final String resource = resourcePrefix + resourceSuffix;
1447         expectation = loadContent(referenceClass.getResource(resource));
1448 
1449         // check for duplicates
1450         if (realBrowserNyiExpectation != null) {
1451             if (realNyiExpectation != null) {
1452                 Assertions.assertNotEquals(realBrowserNyiExpectation, realNyiExpectation,
1453                         "Duplicate NYI Expectation for Browser " + browserVersion.getNickname());
1454             }
1455 
1456             if (browserExpectation == null) {
1457                 if (expectation != null) {
1458                     Assertions.assertNotEquals(realBrowserNyiExpectation, expectation,
1459                             "NYI Expectation matches the expected "
1460                                     + "result for Browser " + browserVersion.getNickname());
1461                 }
1462             }
1463             else {
1464                 Assertions.assertNotEquals(realBrowserNyiExpectation, browserExpectation,
1465                         "NYI Expectation matches the expected "
1466                                 + "browser specific result for Browser "
1467                                 + browserVersion.getNickname());
1468             }
1469 
1470             return realBrowserNyiExpectation;
1471         }
1472 
1473         if (realNyiExpectation != null) {
1474             if (browserExpectation == null) {
1475                 if (expectation != null) {
1476                     Assertions.assertNotEquals(realNyiExpectation, expectation,
1477                             "NYI Expectation matches the expected "
1478                                     + "result for Browser " + browserVersion.getNickname());
1479                 }
1480             }
1481             else {
1482                 Assertions.assertNotEquals(realNyiExpectation, browserExpectation,
1483                             "NYI Expectation matches the expected "
1484                                     + "browser specific result for Browser "
1485                                     + browserVersion.getNickname());
1486             }
1487             return realNyiExpectation;
1488         }
1489 
1490         if (browserExpectation != null) {
1491             if (expectation != null) {
1492                 Assertions.assertNotEquals(browserExpectation, expectation,
1493                             "Browser specific NYI Expectation matches the expected "
1494                                     + "result for Browser " + browserVersion.getNickname());
1495             }
1496             return browserExpectation;
1497         }
1498         return expectation;
1499     }
1500 
1501     private static String loadContent(final URL url) throws URISyntaxException, IOException {
1502         if (url == null) {
1503             return null;
1504         }
1505 
1506         final File file = new File(url.toURI());
1507         final String content = FileUtils.readFileToString(file, UTF_8)
1508                                         .replace("\r\n", "\n");
1509         return content;
1510     }
1511 
1512     /**
1513      * Reads the number of JS threads remaining from unit tests run before.
1514      * This should be always 0, if {@link #isWebClientCached()} is {@code false}.
1515      * @throws InterruptedException in case of problems
1516      */
1517     @BeforeEach
1518     public void beforeTest() throws InterruptedException {
1519         if (!isWebClientCached()) {
1520             if (!getJavaScriptThreads().isEmpty()) {
1521                 Thread.sleep(200);
1522             }
1523             assertTrue("There are already JS threads running before the test", getJavaScriptThreads().isEmpty());
1524         }
1525     }
1526 
1527     /**
1528      * Asserts the current title is equal to the expectation string.
1529      * @param webdriver the driver in use
1530      * @param expected the expected object
1531      * @throws Exception in case of failure
1532      */
1533     protected void assertTitle(final WebDriver webdriver, final String expected) throws Exception {
1534         final long maxWait = System.currentTimeMillis() + DEFAULT_WAIT_TIME.toMillis();
1535 
1536         while (true) {
1537             final String title = webdriver.getTitle();
1538             try {
1539                 assertEquals(expected, title);
1540                 return;
1541             }
1542             catch (final org.opentest4j.AssertionFailedError e) {
1543                 if (expected.length() <= title.length()
1544                         || System.currentTimeMillis() > maxWait) {
1545                     throw e;
1546                 }
1547                 Thread.sleep(10);
1548             }
1549         }
1550     }
1551 
1552     /**
1553      * Release resources but DON'T close the browser if we are running with a real browser.
1554      * Note that HtmlUnitDriver is not cached by default, but that can be configured by {@link #isWebClientCached()}.
1555      */
1556     @AfterEach
1557     @Override
1558     public void releaseResources() {
1559         final List<String> unhandledAlerts = new ArrayList<>();
1560         if (webDriver_ != null) {
1561             UnhandledAlertException ex = null;
1562             do {
1563                 ex = null;
1564                 try {
1565                     // getTitle will do an implicit check for open alerts
1566                     webDriver_.getTitle();
1567                 }
1568                 catch (final NoSuchWindowException e) {
1569                     // ignore
1570                 }
1571                 catch (final UnhandledAlertException e) {
1572                     ex = e;
1573                     unhandledAlerts.add(e.getMessage());
1574                 }
1575             }
1576             while (ex != null);
1577         }
1578 
1579         super.releaseResources();
1580 
1581         if (!isWebClientCached()) {
1582             boolean rhino = false;
1583             if (webDriver_ != null) {
1584                 try {
1585                     rhino = getWebClient().getJavaScriptEngine() instanceof JavaScriptEngine;
1586                 }
1587                 catch (final Exception e) {
1588                     throw new RuntimeException(e);
1589                 }
1590                 webDriver_.quit();
1591                 webDriver_ = null;
1592             }
1593             if (rhino) {
1594                 assertTrue("There are still JS threads running after the test", getJavaScriptThreads().isEmpty());
1595             }
1596         }
1597 
1598         if (useRealBrowser()) {
1599             synchronized (WEB_DRIVERS_REAL_BROWSERS) {
1600                 final WebDriver driver = WEB_DRIVERS_REAL_BROWSERS.get(getBrowserVersion());
1601                 if (driver != null) {
1602                     try {
1603                         final String currentWindow = driver.getWindowHandle();
1604 
1605                         final Set<String> handles = driver.getWindowHandles();
1606                         // close all windows except the current one
1607                         handles.remove(currentWindow);
1608 
1609                         if (!handles.isEmpty()) {
1610                             for (final String handle : handles) {
1611                                 try {
1612                                     driver.switchTo().window(handle);
1613                                     driver.close();
1614                                 }
1615                                 catch (final NoSuchWindowException e) {
1616                                     LOG.error("Error switching to browser window; quit browser.", e);
1617                                     WEB_DRIVERS_REAL_BROWSERS.remove(getBrowserVersion());
1618                                     WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.remove(getBrowserVersion());
1619                                     driver.quit();
1620                                     return;
1621                                 }
1622                             }
1623 
1624                             // we have to force WebDriver to treat the remaining window
1625                             // as the one we like to work with from now on
1626                             // looks like a web driver issue to me (version 2.47.2)
1627                             driver.switchTo().window(currentWindow);
1628                         }
1629 
1630                         driver.manage().deleteAllCookies();
1631 
1632                         // in the remaining window, load a blank page
1633                         driver.get("about:blank");
1634                     }
1635                     catch (final NoSuchSessionException e) {
1636                         LOG.error("Error browser session no longer available.", e);
1637                         WEB_DRIVERS_REAL_BROWSERS.remove(getBrowserVersion());
1638                         WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.remove(getBrowserVersion());
1639                         return;
1640                     }
1641                     catch (final WebDriverException e) {
1642                         shutDownRealBrowsers();
1643                     }
1644                 }
1645             }
1646         }
1647         assertTrue("There are still unhandled alerts: " + String.join("; ", unhandledAlerts),
1648                         unhandledAlerts.isEmpty());
1649     }
1650 
1651     /**
1652      * Returns the underlying WebWindow of the specified driver.
1653      *
1654      * <b>Your test shouldn't depend primarily on WebClient</b>
1655      *
1656      * @return the current web window
1657      * @see #toHtmlElement(WebElement)
1658      */
1659     protected WebWindow getWebWindow() {
1660         return webDriver_.getCurrentWindow().getWebWindow();
1661     }
1662 
1663     /**
1664      * Whether {@link WebClient} is cached or not, defaults to {@code false}.
1665      *
1666      * <p>This is needed to be {@code true} for huge test class, as we could run out of sockets.
1667      *
1668      * @return whether {@link WebClient} is cached or not
1669      */
1670     protected boolean isWebClientCached() {
1671         return false;
1672     }
1673 
1674     /**
1675      * Configure {@link WebClientOptions#getTimeout()}.
1676      *
1677      * @return null if unchanged otherwise the timeout as int
1678      */
1679     protected Integer getWebClientTimeout() {
1680         return null;
1681     }
1682 
1683     protected Page getEnclosedPage() {
1684         return getWebWindow().getEnclosedPage();
1685     }
1686 
1687     protected WebClient getWebClient() {
1688         return webDriver_.getWebClient();
1689     }
1690 }