View Javadoc
1   /*
2    * Copyright (c) 2002-2025 Gargoyle Software Inc.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    * https://www.apache.org/licenses/LICENSE-2.0
8    *
9    * Unless required by applicable law or agreed to in writing, software
10   * distributed under the License is distributed on an "AS IS" BASIS,
11   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12   * See the License for the specific language governing permissions and
13   * limitations under the License.
14   */
15  package org.htmlunit;
16  
17  import static java.nio.charset.StandardCharsets.ISO_8859_1;
18  import static java.nio.charset.StandardCharsets.UTF_8;
19  
20  import java.io.File;
21  import java.io.FileInputStream;
22  import java.io.IOException;
23  import java.net.URISyntaxException;
24  import java.net.URL;
25  import java.nio.charset.Charset;
26  import java.time.Duration;
27  import java.util.ArrayList;
28  import java.util.Arrays;
29  import java.util.Collections;
30  import java.util.EnumSet;
31  import java.util.Enumeration;
32  import java.util.HashMap;
33  import java.util.HashSet;
34  import java.util.List;
35  import java.util.Locale;
36  import java.util.Map;
37  import java.util.Properties;
38  import java.util.Set;
39  import java.util.concurrent.Executor;
40  import java.util.concurrent.Executors;
41  
42  import javax.servlet.DispatcherType;
43  import javax.servlet.Filter;
44  import javax.servlet.FilterChain;
45  import javax.servlet.FilterConfig;
46  import javax.servlet.Servlet;
47  import javax.servlet.ServletException;
48  import javax.servlet.ServletRequest;
49  import javax.servlet.ServletResponse;
50  import javax.servlet.http.HttpServlet;
51  import javax.servlet.http.HttpServletRequest;
52  import javax.servlet.http.HttpServletResponse;
53  
54  import org.apache.commons.io.FileUtils;
55  import org.apache.commons.io.IOUtils;
56  import org.apache.commons.lang3.StringUtils;
57  import org.apache.commons.lang3.exception.ExceptionUtils;
58  import org.apache.commons.logging.Log;
59  import org.apache.commons.logging.LogFactory;
60  import org.eclipse.jetty.security.ConstraintMapping;
61  import org.eclipse.jetty.security.ConstraintSecurityHandler;
62  import org.eclipse.jetty.security.HashLoginService;
63  import org.eclipse.jetty.server.Request;
64  import org.eclipse.jetty.server.Server;
65  import org.eclipse.jetty.server.handler.HandlerWrapper;
66  import org.eclipse.jetty.util.security.Constraint;
67  import org.eclipse.jetty.webapp.WebAppContext;
68  import org.htmlunit.MockWebConnection.RawResponseData;
69  import org.htmlunit.html.HtmlElement;
70  import org.htmlunit.javascript.JavaScriptEngine;
71  import org.htmlunit.util.NameValuePair;
72  import org.junit.jupiter.api.AfterAll;
73  import org.junit.jupiter.api.AfterEach;
74  import org.junit.jupiter.api.Assertions;
75  import org.junit.jupiter.api.BeforeEach;
76  import org.openqa.selenium.Alert;
77  import org.openqa.selenium.By;
78  import org.openqa.selenium.Dimension;
79  import org.openqa.selenium.JavascriptExecutor;
80  import org.openqa.selenium.NoAlertPresentException;
81  import org.openqa.selenium.NoSuchSessionException;
82  import org.openqa.selenium.NoSuchWindowException;
83  import org.openqa.selenium.UnhandledAlertException;
84  import org.openqa.selenium.WebDriver;
85  import org.openqa.selenium.WebDriverException;
86  import org.openqa.selenium.WebElement;
87  import org.openqa.selenium.chrome.ChromeDriver;
88  import org.openqa.selenium.chrome.ChromeDriverService;
89  import org.openqa.selenium.chrome.ChromeOptions;
90  import org.openqa.selenium.devtools.DevTools;
91  import org.openqa.selenium.devtools.v138.emulation.Emulation;
92  import org.openqa.selenium.edge.EdgeDriver;
93  import org.openqa.selenium.edge.EdgeDriverService;
94  import org.openqa.selenium.edge.EdgeOptions;
95  import org.openqa.selenium.firefox.FirefoxDriver;
96  import org.openqa.selenium.firefox.FirefoxDriverService;
97  import org.openqa.selenium.firefox.FirefoxOptions;
98  import org.openqa.selenium.firefox.FirefoxProfile;
99  import org.openqa.selenium.firefox.GeckoDriverService;
100 import org.openqa.selenium.htmlunit.HtmlUnitDriver;
101 import org.openqa.selenium.htmlunit.HtmlUnitWebElement;
102 import org.openqa.selenium.htmlunit.options.HtmlUnitDriverOptions;
103 import org.openqa.selenium.htmlunit.options.HtmlUnitOption;
104 import org.openqa.selenium.remote.UnreachableBrowserException;
105 
106 /**
107  * Base class for tests using WebDriver.
108  *
109  * By default, this test runs with HtmlUnit, but this behavior can be changed by having a property file named
110  * "{@code test.properties}" in the HtmlUnit root directory.
111  * Sample (remove the part not matching your os):
112  * <pre>
113    browsers=hu,ff,chrome
114 
115    ff.bin=/usr/bin/firefox                              [Unix]
116    ff-esr.bin=/usr/bin/firefox-esr                      [Unix]
117    geckodriver.bin=/usr/bin/driver/geckodriver          [Unix]
118    chrome.bin=/path/to/chromedriver                     [Unix]
119    edge.bin=/path/to/chromedriver                       [Unix]
120 
121    geckodriver.bin=C:\\path\\to\\geckodriver.exe              [Windows]
122    ff.bin=C:\\path\\to\\Mozilla Firefox\\firefox.exe          [Windows]
123    ff-esr.bin=C:\\path\\to\\Mozilla Firefox ESR\\firefox.exe  [Windows]
124    chrome.bin=C:\\path\\to\\chromedriver.exe                  [Windows]
125    edge.bin=C:\\path\\to\\msedgedriver.exe                    [Windows]
126 
127    autofix=false
128    </pre>
129 
130  * The file could contain some properties:
131  * <ul>
132  *   <li>browsers: is a comma separated list contains any combination of
133  *     <ul>
134  *       <li>hu (for HtmlUnit with all browser versions),</li>
135  *       <li>hu-ff,</li>
136  *       <li>hu-ff-esr,</li>
137  *       <li>hu-chrome,</li>
138  *       <li>hu-edge,</li>
139  *       <li>ff, (running test using real Firefox),</li>
140  *       <li>ff-esr, (running test using real Firefox ESR),</li>
141  *       <li>chrome (running test using real Chrome),</li>
142  *       <li>edge (running test using real Edge),</li>
143  *     </ul>
144  *   </li>
145  *
146  *   <li>chrome.bin (mandatory if it does not exist in the <i>path</i>): is the location of the ChromeDriver binary (see
147  *   <a href="https://googlechromelabs.github.io/chrome-for-testing/last-known-good-versions-with-downloads.json">Chrome Driver downloads</a>)</li>
148  *   <li>geckodriver.bin (mandatory if it does not exist in the <i>path</i>): is the location of the GeckoDriver binary
149  *   (see <a href="https://github.com/mozilla/geckodriver/releases">Gecko Driver Releases</a>)</li>
150  *   <li>ff.bin (optional): is the location of the FF binary, in Windows use double back-slashes</li>
151  *   <li>ff-esr.bin (optional): is the location of the FF binary, in Windows use double back-slashes</li>
152  *   <li>edge.bin (mandatory if it does not exist in the <i>path</i>): is the location of the MicrosoftWebDriver binary
153  *   (see <a href="https://developer.microsoft.com/en-us/microsoft-edge/tools/webdriver/">Microsoft Edge WebDriver downloads</a>)</li>
154  *   <li>autofix (optional): if {@code true}, try to automatically fix the real browser expectations,
155  *   or add/remove {@code @NotYetImplemented} annotations, use with caution!</li>
156  * </ul>
157  *
158  * @author Marc Guillemot
159  * @author Ahmed Ashour
160  * @author Ronald Brill
161  * @author Frank Danek
162  */
163 public abstract class WebDriverTestCase extends WebTestCase {
164 
165     private static final String LOG_EX_FUNCTION =
166             "  function logEx(e) {\n"
167             + "  let toStr = null;\n"
168             + "  if (toStr === null && e instanceof EvalError) { toStr = ''; }\n"
169             + "  if (toStr === null && e instanceof RangeError) { toStr = ''; }\n"
170             + "  if (toStr === null && e instanceof ReferenceError) { toStr = ''; }\n"
171             + "  if (toStr === null && e instanceof SyntaxError) { toStr = ''; }\n"
172             + "  if (toStr === null && e instanceof TypeError) { toStr = ''; }\n"
173             + "  if (toStr === null && e instanceof URIError) { toStr = ''; }\n"
174             + "  if (toStr === null && e instanceof AggregateError) { toStr = '/AggregateError'; }\n"
175             + "  if (toStr === null && typeof InternalError == 'function' "
176                         + "&& e instanceof InternalError) { toStr = '/InternalError'; }\n"
177             + "  if (toStr === null) {\n"
178             + "    let rx = /\\[object (.*)\\]/;\n"
179             + "    toStr = Object.prototype.toString.call(e);\n"
180             + "    let match = rx.exec(toStr);\n"
181             + "    if (match != null) { toStr = '/' + match[1]; }\n"
182             + "  }"
183             + "  log(e.name + toStr);\n"
184             + "}\n";
185 
186     /**
187      * Function used in many tests.
188      */
189     public static final String LOG_TITLE_FUNCTION =
190             "  function log(msg) { window.document.title += msg + '\\u00a7'; }\n"
191             + LOG_EX_FUNCTION;
192 
193     /**
194      * Function used in many tests.
195      */
196     public static final String LOG_TITLE_FUNCTION_NORMALIZE =
197             "  function log(msg) { "
198                     + "msg = '' + msg; "
199                     + "msg = msg.replace(/ /g, '\\\\s'); "
200                     + "msg = msg.replace(/\\n/g, '\\\\n'); "
201                     + "msg = msg.replace(/\\r/g, '\\\\r'); "
202                     + "msg = msg.replace(/\\t/g, '\\\\t'); "
203                     + "msg = msg.replace(/\\u001e/g, '\\\\u001e'); "
204                     + "window.document.title += msg + '\u00A7';}\n"
205 
206                     + LOG_EX_FUNCTION;
207 
208     /**
209      * Function used in many tests.
210      */
211     public static final String LOG_WINDOW_NAME_FUNCTION =
212             "  function log(msg) { window.top.name += msg + '\\u00a7'; }\n"
213             + "  window.top.name = '';"
214             + LOG_EX_FUNCTION;
215 
216     /**
217      * Function used in many tests.
218      */
219     public static final String LOG_SESSION_STORAGE_FUNCTION =
220             "  function log(msg) { "
221             + "var l = sessionStorage.getItem('Log');"
222             + "sessionStorage.setItem('Log', (null === l?'':l) + msg + '\\u00a7'); }\n";
223 
224     /**
225      * Function used in many tests.
226      */
227     public static final String LOG_TEXTAREA_FUNCTION = "  function log(msg) { "
228             + "document.getElementById('myLog').value += msg + '\u00A7';}\n"
229             + LOG_EX_FUNCTION;
230 
231     /**
232      * HtmlSniped to insert text area used for logging.
233      */
234     public static final String LOG_TEXTAREA = "  <textarea id='myLog' cols='80' rows='22'></textarea>\n";
235 
236     /**
237      * The system property for automatically fixing the test case expectations.
238      */
239     public static final String AUTOFIX_ = "htmlunit.autofix";
240 
241     /**
242      * All browsers supported.
243      */
244     private static List<BrowserVersion> ALL_BROWSERS_ = Collections.unmodifiableList(
245             Arrays.asList(BrowserVersion.CHROME,
246                     BrowserVersion.EDGE,
247                     BrowserVersion.FIREFOX,
248                     BrowserVersion.FIREFOX_ESR));
249 
250     /**
251      * Browsers which run by default.
252      */
253     private static BrowserVersion[] DEFAULT_RUNNING_BROWSERS_ =
254         {BrowserVersion.CHROME,
255             BrowserVersion.EDGE,
256             BrowserVersion.FIREFOX,
257             BrowserVersion.FIREFOX_ESR};
258 
259     private static final Log LOG = LogFactory.getLog(WebDriverTestCase.class);
260 
261     private static Set<String> BROWSERS_PROPERTIES_;
262     private static String CHROME_BIN_;
263     private static String EDGE_BIN_;
264     private static String GECKO_BIN_;
265     private static String FF_BIN_;
266     private static String FF_ESR_BIN_;
267 
268     /** The driver cache. */
269     protected static final Map<BrowserVersion, WebDriver> WEB_DRIVERS_ = new HashMap<>();
270 
271     /** The driver cache for real browsers. */
272     protected static final Map<BrowserVersion, WebDriver> WEB_DRIVERS_REAL_BROWSERS = new HashMap<>();
273     private static final Map<BrowserVersion, Integer> WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT = new HashMap<>();
274 
275     private static Server STATIC_SERVER_;
276     private static String STATIC_SERVER_STARTER_; // stack trace to save the server start code location
277     // second server for cross-origin tests.
278     private static Server STATIC_SERVER2_;
279     private static String STATIC_SERVER2_STARTER_; // stack trace to save the server start code location
280     // third server for multi-origin cross-origin tests.
281     private static Server STATIC_SERVER3_;
282     private static String STATIC_SERVER3_STARTER_; // stack trace to save the server start code location
283 
284     private static Boolean LAST_TEST_UsesMockWebConnection_;
285     private static final Executor EXECUTOR_POOL = Executors.newFixedThreadPool(4);
286 
287     private boolean useRealBrowser_;
288 
289     /**
290      * The HtmlUnitDriver.
291      */
292     private HtmlUnitDriver webDriver_;
293 
294     /**
295      * Override this function in a test class to ask for STATIC_SERVER2_ to be set up.
296      * @return true if two servers are needed.
297      */
298     protected boolean needThreeConnections() {
299         return false;
300     }
301 
302     /**
303      * @return the browser properties (and initializes them lazy)
304      */
305     public static Set<String> getBrowsersProperties() {
306         if (BROWSERS_PROPERTIES_ == null) {
307             try {
308                 final Properties properties = new Properties();
309                 final File file = new File("test.properties");
310                 if (file.exists()) {
311                     try (FileInputStream in = new FileInputStream(file)) {
312                         properties.load(in);
313                     }
314 
315                     String browsersValue = properties.getProperty("browsers");
316                     if (browsersValue == null || browsersValue.isEmpty()) {
317                         browsersValue = "hu";
318                     }
319                     BROWSERS_PROPERTIES_ = new HashSet<>(Arrays.asList(browsersValue.replaceAll(" ", "")
320                             .toLowerCase(Locale.ROOT).split(",")));
321                     CHROME_BIN_ = properties.getProperty("chrome.bin");
322                     EDGE_BIN_ = properties.getProperty("edge.bin");
323 
324                     GECKO_BIN_ = properties.getProperty("geckodriver.bin");
325                     FF_BIN_ = properties.getProperty("ff.bin");
326                     FF_ESR_BIN_ = properties.getProperty("ff-esr.bin");
327 
328                     final boolean autofix = Boolean.parseBoolean(properties.getProperty("autofix"));
329                     System.setProperty(AUTOFIX_, Boolean.toString(autofix));
330                 }
331             }
332             catch (final Exception e) {
333                 LOG.error("Error reading htmlunit.properties. Ignoring!", e);
334             }
335             if (BROWSERS_PROPERTIES_ == null) {
336                 BROWSERS_PROPERTIES_ = new HashSet<>(Arrays.asList("hu"));
337             }
338             if (BROWSERS_PROPERTIES_.contains("hu")) {
339                 for (final BrowserVersion browserVersion : DEFAULT_RUNNING_BROWSERS_) {
340                     BROWSERS_PROPERTIES_.add("hu-" + browserVersion.getNickname().toLowerCase());
341                 }
342             }
343         }
344         return BROWSERS_PROPERTIES_;
345     }
346 
347     /**
348      * @return the list of supported browsers
349      */
350     public static List<BrowserVersion> allBrowsers() {
351         return ALL_BROWSERS_;
352     }
353 
354     /**
355      * Configure the driver only once.
356      * @return the driver
357      */
358     protected WebDriver getWebDriver() {
359         final BrowserVersion browserVersion = getBrowserVersion();
360         WebDriver driver;
361         if (useRealBrowser()) {
362             synchronized (WEB_DRIVERS_REAL_BROWSERS) {
363                 driver = WEB_DRIVERS_REAL_BROWSERS.get(browserVersion);
364                 if (driver != null) {
365                     // there seems to be a memory leak at least with FF;
366                     // we have to restart sometimes
367                     Integer count = WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.get(browserVersion);
368                     if (null == count) {
369                         count = -1;
370                     }
371                     count += 1;
372                     if (count >= 1000) {
373                         shutDownReal(browserVersion);
374                         driver = null;
375                     }
376                     else {
377                         WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.put(browserVersion, count);
378                     }
379                 }
380 
381                 if (driver == null) {
382                     try {
383                         driver = buildWebDriver();
384                     }
385                     catch (final IOException e) {
386                         throw new RuntimeException(e);
387                     }
388 
389                     WEB_DRIVERS_REAL_BROWSERS.put(browserVersion, driver);
390                     WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.put(browserVersion, 0);
391                 }
392             }
393         }
394         else {
395             driver = WEB_DRIVERS_.get(browserVersion);
396             if (driver == null) {
397                 try {
398                     driver = buildWebDriver();
399                 }
400                 catch (final IOException e) {
401                     throw new RuntimeException(e);
402                 }
403 
404                 if (isWebClientCached()) {
405                     WEB_DRIVERS_.put(browserVersion, driver);
406                 }
407             }
408         }
409         return driver;
410     }
411 
412     /**
413      * Closes the drivers.
414      * @throws Exception If an error occurs
415      */
416     @AfterAll
417     public static void shutDownAll() throws Exception {
418         for (final WebDriver driver : WEB_DRIVERS_.values()) {
419             driver.quit();
420         }
421         WEB_DRIVERS_.clear();
422 
423         shutDownRealBrowsers();
424 
425         stopWebServers();
426     }
427 
428     /**
429      * Closes the real browser drivers.
430      */
431     private static void shutDownRealBrowsers() {
432         synchronized (WEB_DRIVERS_REAL_BROWSERS) {
433             for (final WebDriver driver : WEB_DRIVERS_REAL_BROWSERS.values()) {
434                 quit(driver);
435             }
436             WEB_DRIVERS_REAL_BROWSERS.clear();
437             WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.clear();
438         }
439     }
440 
441     /**
442      * Closes the real browser drivers.
443      * @param browser the real browser to close
444      */
445     private static void shutDownReal(final BrowserVersion browser) {
446         synchronized (WEB_DRIVERS_REAL_BROWSERS) {
447             final WebDriver driver = WEB_DRIVERS_REAL_BROWSERS.get(browser);
448             if (driver != null) {
449                 quit(driver);
450                 WEB_DRIVERS_REAL_BROWSERS.remove(browser);
451                 WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.remove(browser);
452             }
453         }
454     }
455 
456     private static void quit(final WebDriver driver) {
457         if (driver != null) {
458             try {
459                 driver.quit();
460             }
461             catch (final UnreachableBrowserException e) {
462                 LOG.error("Can't quit browser", e);
463                 // ignore, the browser is gone
464             }
465             catch (final NoClassDefFoundError e) {
466                 LOG.error("Can't quit browser", e);
467                 // ignore, the browser is gone
468             }
469             catch (final UnsatisfiedLinkError e) {
470                 LOG.error("Can't quit browser", e);
471                 // ignore, the browser is gone
472             }
473         }
474     }
475 
476     /**
477      * Asserts all static servers are null.
478      * @throws Exception if it fails
479      */
480     protected static void assertWebServersStopped() throws Exception {
481         Assertions.assertNull(STATIC_SERVER_, STATIC_SERVER_STARTER_);
482         Assertions.assertNull(STATIC_SERVER2_, STATIC_SERVER2_STARTER_);
483         Assertions.assertNull(STATIC_SERVER3_, STATIC_SERVER3_STARTER_);
484     }
485 
486     /**
487      * Stops all WebServers.
488      * @throws Exception if it fails
489      */
490     protected static void stopWebServers() throws Exception {
491         if (STATIC_SERVER_ != null) {
492             STATIC_SERVER_.stop();
493             STATIC_SERVER_.destroy();
494             STATIC_SERVER_ = null;
495         }
496 
497         if (STATIC_SERVER2_ != null) {
498             STATIC_SERVER2_.stop();
499             STATIC_SERVER2_.destroy();
500             STATIC_SERVER2_ = null;
501         }
502 
503         if (STATIC_SERVER3_ != null) {
504             STATIC_SERVER3_.stop();
505             STATIC_SERVER3_.destroy();
506             STATIC_SERVER3_ = null;
507         }
508         LAST_TEST_UsesMockWebConnection_ = null;
509     }
510 
511     /**
512      * @return whether to use real browser or not.
513      */
514     public boolean useRealBrowser() {
515         return useRealBrowser_;
516     }
517 
518     /**
519      * Sets whether to use real browser or not.
520      * @param useRealBrowser whether to use real browser or not
521      */
522     public void setUseRealBrowser(final boolean useRealBrowser) {
523         useRealBrowser_ = useRealBrowser;
524     }
525 
526     /**
527      * Builds a new WebDriver instance.
528      * @return the instance
529      * @throws IOException in case of exception
530      */
531     protected WebDriver buildWebDriver() throws IOException {
532         if (useRealBrowser()) {
533             if (BrowserVersion.EDGE.isSameBrowser(getBrowserVersion())) {
534                 final EdgeDriverService service = new EdgeDriverService.Builder()
535                         .withLogOutput(System.out)
536                         .usingDriverExecutable(new File(EDGE_BIN_))
537 
538                         .withAppendLog(true)
539                         .withReadableTimestamp(true)
540 
541                         .build();
542 
543                 final String locale = getBrowserVersion().getBrowserLocale().toLanguageTag();
544 
545                 final EdgeOptions options = new EdgeOptions();
546                 // BiDi
547                 // options.setCapability("webSocketUrl", true);
548 
549                 options.addArguments("--lang=" + locale);
550                 // https://stackoverflow.com/questions/11289597/webdriver-how-to-specify-preferred-languages-for-chrome
551                 options.setExperimentalOption("prefs", Map.of("intl.accept_languages", locale));
552                 options.addArguments("--remote-allow-origins=*");
553 
554                 // seems to be not required for edge
555                 // options.addArguments("--disable-search-engine-choice-screen");
556                 // see https://www.selenium.dev/blog/2024/chrome-browser-woes/
557                 // options.addArguments("--disable-features=OptimizationGuideModelDownloading,"
558                 //         + "OptimizationHintsFetching,OptimizationTargetPrediction,OptimizationHints");
559 
560                 final EdgeDriver edge = new EdgeDriver(service, options);
561 
562                 final DevTools devTools = edge.getDevTools();
563                 devTools.createSession();
564 
565                 final String tz = getBrowserVersion().getSystemTimezone().getID();
566                 devTools.send(Emulation.setTimezoneOverride(tz));
567 
568                 return edge;
569             }
570 
571             if (BrowserVersion.CHROME.isSameBrowser(getBrowserVersion())) {
572                 final ChromeDriverService service = new ChromeDriverService.Builder()
573                         .withLogOutput(System.out)
574                         .usingDriverExecutable(new File(CHROME_BIN_))
575 
576                         .withAppendLog(true)
577                         .withReadableTimestamp(true)
578 
579                         .build();
580 
581                 final String locale = getBrowserVersion().getBrowserLocale().toLanguageTag();
582 
583                 final ChromeOptions options = new ChromeOptions();
584                 // BiDi
585                 // options.setCapability("webSocketUrl", true);
586 
587                 options.addArguments("--lang=" + locale);
588                 // https://stackoverflow.com/questions/11289597/webdriver-how-to-specify-preferred-languages-for-chrome
589                 options.setExperimentalOption("prefs", Map.of("intl.accept_languages", locale));
590                 options.addArguments("--remote-allow-origins=*");
591                 options.addArguments("--disable-search-engine-choice-screen");
592                 // see https://www.selenium.dev/blog/2024/chrome-browser-woes/
593                 options.addArguments("--disable-features=OptimizationGuideModelDownloading,"
594                         + "OptimizationHintsFetching,OptimizationTargetPrediction,OptimizationHints");
595 
596                 final ChromeDriver chrome = new ChromeDriver(service, options);
597 
598                 final DevTools devTools = chrome.getDevTools();
599                 devTools.createSession();
600 
601                 final String tz = getBrowserVersion().getSystemTimezone().getID();
602                 devTools.send(Emulation.setTimezoneOverride(tz));
603 
604                 return chrome;
605             }
606 
607             if (BrowserVersion.FIREFOX.isSameBrowser(getBrowserVersion())) {
608                 return createFirefoxDriver(GECKO_BIN_, FF_BIN_);
609             }
610 
611             if (BrowserVersion.FIREFOX_ESR.isSameBrowser(getBrowserVersion())) {
612                 return createFirefoxDriver(GECKO_BIN_, FF_ESR_BIN_);
613             }
614 
615             throw new RuntimeException("Unexpected BrowserVersion: " + getBrowserVersion());
616         }
617 
618         if (webDriver_ == null) {
619             final HtmlUnitDriverOptions driverOptions = new HtmlUnitDriverOptions(getBrowserVersion());
620 
621             if (isWebClientCached()) {
622                 driverOptions.setCapability(HtmlUnitOption.optHistorySizeLimit, 0);
623             }
624 
625             if (getWebClientTimeout() != null) {
626                 driverOptions.setCapability(HtmlUnitOption.optTimeout, getWebClientTimeout());
627             }
628             webDriver_ = new HtmlUnitDriver(driverOptions);
629             webDriver_.setExecutor(EXECUTOR_POOL);
630         }
631         return webDriver_;
632     }
633 
634     private FirefoxDriver createFirefoxDriver(final String geckodriverBinary, final String binary) {
635         final FirefoxDriverService service = new GeckoDriverService.Builder()
636                 .withLogOutput(System.out)
637                 .usingDriverExecutable(new File(geckodriverBinary))
638                 .build();
639 
640         final FirefoxOptions options = new FirefoxOptions();
641         // BiDi
642         // options.setCapability("webSocketUrl", true);
643 
644         options.setBinary(binary);
645 
646         String locale = getBrowserVersion().getBrowserLocale().toLanguageTag();
647         locale = locale + "," + getBrowserVersion().getBrowserLocale().getLanguage();
648 
649         final FirefoxProfile profile = new FirefoxProfile();
650         profile.setPreference("intl.accept_languages", locale);
651         // no idea so far how to set this
652         // final String tz = getBrowserVersion().getSystemTimezone().getID();
653         // profile.setPreference("intl.tz", tz);
654         options.setProfile(profile);
655 
656         return new FirefoxDriver(service, options);
657     }
658 
659     /**
660      * Starts the web server delivering response from the provided connection.
661      * @param mockConnection the sources for responses
662      * @param serverCharset the {@link Charset} at the server side
663      * @throws Exception if a problem occurs
664      */
665     protected void startWebServer(final MockWebConnection mockConnection, final Charset serverCharset)
666             throws Exception {
667         if (Boolean.FALSE.equals(LAST_TEST_UsesMockWebConnection_)) {
668             stopWebServers();
669         }
670 
671         LAST_TEST_UsesMockWebConnection_ = Boolean.TRUE;
672         if (STATIC_SERVER_ == null) {
673             final Server server = new Server(PORT);
674 
675             final WebAppContext context = new WebAppContext();
676             context.setContextPath("/");
677             context.setResourceBase("./");
678 
679             if (isBasicAuthentication()) {
680                 final Constraint constraint = new Constraint();
681                 constraint.setName(Constraint.__BASIC_AUTH);
682                 constraint.setRoles(new String[]{"user"});
683                 constraint.setAuthenticate(true);
684 
685                 final ConstraintMapping constraintMapping = new ConstraintMapping();
686                 constraintMapping.setConstraint(constraint);
687                 constraintMapping.setPathSpec("/*");
688 
689                 final ConstraintSecurityHandler handler = (ConstraintSecurityHandler) context.getSecurityHandler();
690                 handler.setLoginService(new HashLoginService("MyRealm", "./src/test/resources/realm.properties"));
691                 handler.setConstraintMappings(new ConstraintMapping[]{constraintMapping});
692             }
693 
694             context.addServlet(MockWebConnectionServlet.class, "/*");
695             if (serverCharset != null) {
696                 AsciiEncodingFilter.CHARSET_ = serverCharset;
697                 context.addFilter(AsciiEncodingFilter.class, "/*",
698                         EnumSet.of(DispatcherType.INCLUDE, DispatcherType.REQUEST));
699             }
700             server.setHandler(context);
701             WebServerTestCase.tryStart(PORT, server);
702 
703             STATIC_SERVER_STARTER_ = ExceptionUtils.getStackTrace(new Throwable("StaticServerStarter"));
704             STATIC_SERVER_ = server;
705         }
706         MockWebConnectionServlet.MockConnection_ = mockConnection;
707 
708         if (STATIC_SERVER2_ == null && needThreeConnections()) {
709             final Server server2 = new Server(PORT2);
710             final WebAppContext context2 = new WebAppContext();
711             context2.setContextPath("/");
712             context2.setResourceBase("./");
713             context2.addServlet(MockWebConnectionServlet.class, "/*");
714             server2.setHandler(context2);
715             WebServerTestCase.tryStart(PORT2, server2);
716 
717             STATIC_SERVER2_STARTER_ = ExceptionUtils.getStackTrace(new Throwable("StaticServer2Starter"));
718             STATIC_SERVER2_ = server2;
719 
720             final Server server3 = new Server(PORT3);
721             final WebAppContext context3 = new WebAppContext();
722             context3.setContextPath("/");
723             context3.setResourceBase("./");
724             context3.addServlet(MockWebConnectionServlet.class, "/*");
725             server3.setHandler(context3);
726             WebServerTestCase.tryStart(PORT3, server3);
727 
728             STATIC_SERVER3_STARTER_ = ExceptionUtils.getStackTrace(new Throwable("StaticServer3Starter"));
729             STATIC_SERVER3_ = server3;
730             /*
731              * The mock connection servlet call sit under both servers, so long as tests
732              * keep the URLs distinct.
733              */
734         }
735     }
736 
737     /**
738      * Returns whether to use basic authentication for all resources or not.
739      * The default implementation returns false.
740      * @return whether to use basic authentication or not
741      */
742     protected boolean isBasicAuthentication() {
743         return false;
744     }
745 
746     /**
747      * Starts the web server on the default {@link #PORT}.
748      * The given resourceBase is used to be the ROOT directory that serves the default context.
749      * <p><b>Don't forget to stop the returned HttpServer after the test</b>
750      *
751      * @param resourceBase the base of resources for the default context
752      * @param classpath additional classpath entries to add (may be null)
753      * @param servlets map of {String, Class} pairs: String is the path spec, while class is the class
754      * @throws Exception if the test fails
755      */
756     protected static void startWebServer(final String resourceBase, final String[] classpath,
757             final Map<String, Class<? extends Servlet>> servlets) throws Exception {
758         startWebServer(resourceBase, classpath, servlets, null);
759     }
760 
761     /**
762      * Starts the <b>second</b> web server on the default {@link #PORT2}.
763      * The given resourceBase is used to be the ROOT directory that serves the default context.
764      * <p><b>Don't forget to stop the returned HttpServer after the test</b>
765      *
766      * @param resourceBase the base of resources for the default context
767      * @param classpath additional classpath entries to add (may be null)
768      * @param servlets map of {String, Class} pairs: String is the path spec, while class is the class
769      * @throws Exception if the test fails
770      */
771     protected static void startWebServer2(final String resourceBase, final String[] classpath,
772             final Map<String, Class<? extends Servlet>> servlets) throws Exception {
773 
774         if (STATIC_SERVER2_ != null) {
775             STATIC_SERVER2_.stop();
776         }
777         STATIC_SERVER2_STARTER_ = ExceptionUtils.getStackTrace(new Throwable("StaticServer2Starter"));
778         STATIC_SERVER2_ = WebServerTestCase.createWebServer(PORT2, resourceBase, classpath, servlets, null);
779     }
780 
781     /**
782      * Starts the web server on the default {@link #PORT}.
783      * The given resourceBase is used to be the ROOT directory that serves the default context.
784      * <p><b>Don't forget to stop the returned Server after the test</b>
785      *
786      * @param resourceBase the base of resources for the default context
787      * @param classpath additional classpath entries to add (may be null)
788      * @param servlets map of {String, Class} pairs: String is the path spec, while class is the class
789      * @param handler wrapper for handler (can be null)
790      * @throws Exception if the test fails
791      */
792     protected static void startWebServer(final String resourceBase, final String[] classpath,
793             final Map<String, Class<? extends Servlet>> servlets, final HandlerWrapper handler) throws Exception {
794         stopWebServers();
795         LAST_TEST_UsesMockWebConnection_ = Boolean.FALSE;
796 
797         STATIC_SERVER_STARTER_ = ExceptionUtils.getStackTrace(new Throwable("StaticServerStarter"));
798         STATIC_SERVER_ = WebServerTestCase.createWebServer(PORT, resourceBase, classpath, servlets, handler);
799     }
800 
801     /**
802      * Servlet delivering content from a MockWebConnection.
803      */
804     public static class MockWebConnectionServlet extends HttpServlet {
805         private static MockWebConnection MockConnection_;
806 
807         static void setMockconnection(final MockWebConnection connection) {
808             MockConnection_ = connection;
809         }
810 
811         /**
812          * {@inheritDoc}
813          */
814         @Override
815         protected void service(final HttpServletRequest request, final HttpServletResponse response)
816                 throws ServletException, IOException {
817 
818             try {
819                 doService(request, response);
820             }
821             catch (final ServletException e) {
822                 throw e;
823             }
824             catch (final IOException e) {
825                 throw e;
826             }
827             catch (final Exception e) {
828                 throw new ServletException(e);
829             }
830         }
831 
832         private static void doService(final HttpServletRequest request, final HttpServletResponse response)
833                 throws Exception {
834             String url = request.getRequestURL().toString();
835             if (LOG.isDebugEnabled()) {
836                 LOG.debug(request.getMethod() + " " + url);
837             }
838 
839             if (url.endsWith("/favicon.ico")) {
840                 response.setStatus(HttpServletResponse.SC_NOT_FOUND);
841                 return;
842             }
843 
844             if (url.contains("/delay")) {
845                 final String delay = StringUtils.substringBetween(url, "/delay", "/");
846                 final int ms = Integer.parseInt(delay);
847                 if (LOG.isDebugEnabled()) {
848                     LOG.debug("Sleeping for " + ms + " before to deliver " + url);
849                 }
850                 Thread.sleep(ms);
851             }
852 
853             // copy parameters
854             final List<NameValuePair> requestParameters = new ArrayList<>();
855             try {
856                 for (final Enumeration<String> paramNames = request.getParameterNames();
857                         paramNames.hasMoreElements();) {
858                     final String name = paramNames.nextElement();
859                     final String[] values = request.getParameterValues(name);
860                     for (final String value : values) {
861                         requestParameters.add(new NameValuePair(name, value));
862                     }
863                 }
864             }
865             catch (final IllegalArgumentException e) {
866                 // Jetty 8.1.7 throws it in getParameterNames for a query like "cb=%%RANDOM_NUMBER%%"
867                 // => we should use a more low level test server
868                 requestParameters.clear();
869                 final String query = request.getQueryString();
870                 if (query != null) {
871                     url += "?" + query;
872                 }
873             }
874 
875             final String queryString = request.getQueryString();
876             if (StringUtils.isNotBlank(queryString)) {
877                 url = url + "?" + queryString;
878             }
879             final URL requestedUrl = new URL(url);
880             final WebRequest webRequest = new WebRequest(requestedUrl);
881             webRequest.setHttpMethod(HttpMethod.valueOf(request.getMethod()));
882 
883             // copy headers
884             for (final Enumeration<String> en = request.getHeaderNames(); en.hasMoreElements();) {
885                 final String headerName = en.nextElement();
886                 final String headerValue = request.getHeader(headerName);
887                 webRequest.setAdditionalHeader(headerName, headerValue);
888             }
889 
890             if (requestParameters.isEmpty() && request.getContentLength() > 0) {
891                 final byte[] buffer = new byte[request.getContentLength()];
892                 IOUtils.read(request.getInputStream(), buffer, 0, buffer.length);
893 
894                 final String encoding = request.getCharacterEncoding();
895                 if (encoding == null) {
896                     webRequest.setRequestBody(new String(buffer, ISO_8859_1));
897                     webRequest.setCharset(null);
898                 }
899                 else {
900                     webRequest.setRequestBody(new String(buffer, encoding));
901                     webRequest.setCharset(Charset.forName(encoding));
902                 }
903             }
904             else {
905                 webRequest.setRequestParameters(requestParameters);
906             }
907 
908             // check content type if it is multipart enctype
909             if (request.getContentType() != null
910                         && request.getContentType().startsWith(FormEncodingType.MULTIPART.getName())) {
911                 webRequest.setEncodingType(FormEncodingType.MULTIPART);
912             }
913 
914             final RawResponseData resp = MockConnection_.getRawResponse(webRequest);
915 
916             // write WebResponse to HttpServletResponse
917             response.setStatus(resp.getStatusCode());
918 
919             boolean charsetInContentType = false;
920             for (final NameValuePair responseHeader : resp.getHeaders()) {
921                 final String headerName = responseHeader.getName();
922                 if (HttpHeader.CONTENT_TYPE.equals(headerName) && responseHeader.getValue().contains("charset=")) {
923                     charsetInContentType = true;
924                 }
925                 response.addHeader(headerName, responseHeader.getValue());
926             }
927 
928             if (resp.getByteContent() != null) {
929                 response.getOutputStream().write(resp.getByteContent());
930             }
931             else {
932                 if (!charsetInContentType) {
933                     response.setCharacterEncoding(resp.getCharset().name());
934                 }
935                 response.getWriter().print(resp.getStringContent());
936             }
937             response.flushBuffer();
938         }
939     }
940 
941     /**
942      * Same as {@link #loadPageWithAlerts2(String)}... but doesn't verify the alerts.
943      * @param html the HTML to use
944      * @return the web driver
945      * @throws Exception if something goes wrong
946      */
947     protected final WebDriver loadPage2(final String html) throws Exception {
948         return loadPage2(html, URL_FIRST);
949     }
950 
951     /**
952      * Same as {@link #loadPageWithAlerts2(String)}... but doesn't verify the alerts.
953      * @param html the HTML to use
954      * @param url the url to use to load the page
955      * @return the web driver
956      * @throws Exception if something goes wrong
957      */
958     protected final WebDriver loadPage2(final String html, final URL url) throws Exception {
959         return loadPage2(html, url, "text/html;charset=ISO-8859-1", ISO_8859_1, null);
960     }
961 
962     /**
963      * Same as {@link #loadPageWithAlerts2(String)}... but doesn't verify the alerts.
964      * @param html the HTML to use
965      * @param url the url to use to load the page
966      * @param contentType the content type to return
967      * @param charset the charset
968      * @return the web driver
969      * @throws Exception if something goes wrong
970      */
971     protected final WebDriver loadPage2(final String html, final URL url,
972             final String contentType, final Charset charset) throws Exception {
973         return loadPage2(html, url, contentType, charset, null);
974     }
975 
976     /**
977      * Same as {@link #loadPageWithAlerts2(String)}... but doesn't verify the alerts.
978      * @param html the HTML to use
979      * @param url the url to use to load the page
980      * @param contentType the content type to return
981      * @param charset the charset
982      * @param serverCharset the charset at the server side.
983      * @return the web driver
984      * @throws Exception if something goes wrong
985      */
986     protected final WebDriver loadPage2(final String html, final URL url,
987             final String contentType, final Charset charset, final Charset serverCharset) throws Exception {
988         getMockWebConnection().setResponse(url, html, contentType, charset);
989         return loadPage2(url, serverCharset);
990     }
991 
992     /**
993      * Load the page from the url.
994      * @param url the url to use to load the page
995      * @param serverCharset the charset at the server side.
996      * @return the web driver
997      * @throws Exception if something goes wrong
998      */
999     protected final WebDriver loadPage2(final URL url, final Charset serverCharset) throws Exception {
1000         startWebServer(getMockWebConnection(), serverCharset);
1001 
1002         WebDriver driver = getWebDriver();
1003         if (!(driver instanceof HtmlUnitDriver)) {
1004             try {
1005                 resizeIfNeeded(driver);
1006             }
1007             catch (final NoSuchSessionException e) {
1008                 // maybe the driver was killed by the test before; setup a new one
1009                 shutDownRealBrowsers();
1010 
1011                 driver = getWebDriver();
1012                 resizeIfNeeded(driver);
1013             }
1014         }
1015         driver.get(url.toExternalForm());
1016 
1017         return driver;
1018     }
1019 
1020     /**
1021      * Same as {@link #loadPage2(String)} with additional servlet configuration.
1022      * @param html the HTML to use for the default response
1023      * @param servlets the additional servlets to configure with their mapping
1024      * @return the web driver
1025      * @throws Exception if something goes wrong
1026      */
1027     protected final WebDriver loadPage2(final String html,
1028             final Map<String, Class<? extends Servlet>> servlets) throws Exception {
1029         return loadPage2(html, URL_FIRST, servlets, null);
1030     }
1031 
1032     /**
1033      * Same as {@link #loadPage2(String, URL)}, but with additional servlet configuration.
1034      * @param html the HTML to use for the default page
1035      * @param url the URL to use to load the page
1036      * @param servlets the additional servlets to configure with their mapping
1037      * @return the web driver
1038      * @throws Exception if something goes wrong
1039      */
1040     protected final WebDriver loadPage2(final String html, final URL url,
1041             final Map<String, Class<? extends Servlet>> servlets) throws Exception {
1042         return loadPage2(html, url, servlets, null);
1043     }
1044 
1045     /**
1046      * Same as {@link #loadPage2(String, URL)}, but with additional servlet configuration.
1047      * @param html the HTML to use for the default page
1048      * @param url the URL to use to load the page
1049      * @param servlets the additional servlets to configure with their mapping
1050      * @param servlets2 the additional servlets to configure with their mapping for a second server
1051      * @return the web driver
1052      * @throws Exception if something goes wrong
1053      */
1054     protected final WebDriver loadPage2(final String html, final URL url,
1055             final Map<String, Class<? extends Servlet>> servlets,
1056             final Map<String, Class<? extends Servlet>> servlets2) throws Exception {
1057         servlets.put("/*", MockWebConnectionServlet.class);
1058         getMockWebConnection().setResponse(url, html);
1059         MockWebConnectionServlet.MockConnection_ = getMockWebConnection();
1060 
1061         startWebServer("./", null, servlets);
1062         if (servlets2 != null) {
1063             startWebServer2("./", null, servlets2);
1064         }
1065 
1066         WebDriver driver = getWebDriver();
1067         if (!(driver instanceof HtmlUnitDriver)) {
1068             try {
1069                 resizeIfNeeded(driver);
1070             }
1071             catch (final NoSuchSessionException e) {
1072                 // maybe the driver was killed by the test before; setup a new one
1073                 shutDownRealBrowsers();
1074 
1075                 driver = getWebDriver();
1076                 resizeIfNeeded(driver);
1077             }
1078         }
1079         driver.get(url.toExternalForm());
1080 
1081         return driver;
1082     }
1083 
1084     protected void resizeIfNeeded(final WebDriver driver) {
1085         final Dimension size = driver.manage().window().getSize();
1086         if (size.getWidth() != 1272 || size.getHeight() != 768) {
1087             // only resize if needed because it may be quite expensive
1088             driver.manage().window().setSize(new Dimension(1272, 768));
1089         }
1090     }
1091 
1092     /**
1093      * Defines the provided HTML as the response for {@link WebTestCase#URL_FIRST}
1094      * and loads the page with this URL using the current WebDriver version; finally, asserts that the
1095      * alerts equal the expected alerts.
1096      * @param html the HTML to use
1097      * @return the web driver
1098      * @throws Exception if something goes wrong
1099      */
1100     protected final WebDriver loadPageWithAlerts2(final String html) throws Exception {
1101         return loadPageWithAlerts2(html, URL_FIRST, DEFAULT_WAIT_TIME);
1102     }
1103 
1104     /**
1105      * Defines the provided HTML as the response for {@link WebTestCase#URL_FIRST}
1106      * and loads the page with this URL using the current WebDriver version; finally, asserts that the
1107      * alerts equal the expected alerts.
1108      * @param html the HTML to use
1109      * @return the web driver
1110      * @throws Exception if something goes wrong
1111      */
1112     protected final WebDriver loadPageVerifyTitle2(final String html) throws Exception {
1113         return loadPageVerifyTitle2(html, getExpectedAlerts());
1114     }
1115 
1116     protected final WebDriver loadPageVerifyTitle2(final String html, final String... expectedAlerts) throws Exception {
1117         final WebDriver driver = loadPage2(html);
1118         return verifyTitle2(driver, expectedAlerts);
1119     }
1120 
1121     protected final WebDriver verifyTitle2(final Duration maxWaitTime, final WebDriver driver,
1122             final String... expectedAlerts) throws Exception {
1123 
1124         final StringBuilder expected = new StringBuilder();
1125         for (int i = 0; i < expectedAlerts.length; i++) {
1126             expected.append(expectedAlerts[i]).append('\u00A7');
1127         }
1128         final String expectedTitle = expected.toString();
1129 
1130         final long maxWait = System.currentTimeMillis() + maxWaitTime.toMillis();
1131 
1132         while (System.currentTimeMillis() < maxWait) {
1133             try {
1134                 final String title = driver.getTitle();
1135                 assertEquals(expectedTitle, title);
1136                 return driver;
1137             }
1138             catch (final AssertionError e) {
1139                 // ignore and wait
1140             }
1141         }
1142 
1143         assertEquals(expectedTitle, driver.getTitle());
1144         return driver;
1145     }
1146 
1147     protected final WebDriver verifyTitle2(final WebDriver driver,
1148             final String... expectedAlerts) throws Exception {
1149         if (expectedAlerts.length == 0) {
1150             assertEquals("", driver.getTitle());
1151             return driver;
1152         }
1153 
1154         final StringBuilder expected = new StringBuilder();
1155         for (int i = 0; i < expectedAlerts.length; i++) {
1156             expected.append(expectedAlerts[i]).append('\u00A7');
1157         }
1158 
1159         final String title = driver.getTitle();
1160         try {
1161             assertEquals(expected.toString(), title);
1162         }
1163         catch (final AssertionError e) {
1164             if (useRealBrowser() && StringUtils.isEmpty(title)) {
1165                 Thread.sleep(42);
1166                 assertEquals(expected.toString(), driver.getTitle());
1167                 return driver;
1168             }
1169             throw e;
1170         }
1171         return driver;
1172     }
1173 
1174     protected final WebDriver loadPageVerifyTextArea2(final String html) throws Exception {
1175         return loadPageTextArea2(html, getExpectedAlerts());
1176     }
1177 
1178     protected final WebDriver loadPageTextArea2(final String html, final String... expectedAlerts) throws Exception {
1179         final WebDriver driver = loadPage2(html);
1180         return verifyTextArea2(driver, expectedAlerts);
1181     }
1182 
1183     protected final WebDriver verifyTextArea2(final WebDriver driver,
1184             final String... expectedAlerts) throws Exception {
1185         final WebElement textArea = driver.findElement(By.id("myLog"));
1186 
1187         if (expectedAlerts.length == 0) {
1188             assertEquals("", textArea.getDomProperty("value"));
1189             return driver;
1190         }
1191 
1192         if (!useRealBrowser()
1193                 && expectedAlerts.length == 1
1194                 && expectedAlerts[0].startsWith("data:image/png;base64,")) {
1195             String value = textArea.getDomProperty("value");
1196             if (value.endsWith("\u00A7")) {
1197                 value = value.substring(0, value.length() - 1);
1198             }
1199             compareImages(expectedAlerts[0], value);
1200             return driver;
1201         }
1202 
1203         final StringBuilder expected = new StringBuilder();
1204         for (int i = 0; i < expectedAlerts.length; i++) {
1205             expected.append(expectedAlerts[i]).append('\u00A7');
1206         }
1207         verify(() -> textArea.getDomProperty("value"), expected.toString());
1208 
1209         return driver;
1210     }
1211 
1212     protected final String getJsVariableValue(final WebDriver driver, final String varName) throws Exception {
1213         final String script = "return String(" + varName + ")";
1214         final String result = (String) ((JavascriptExecutor) driver).executeScript(script);
1215 
1216         return result;
1217     }
1218 
1219     protected final WebDriver verifyJsVariable(final WebDriver driver, final String varName,
1220             final String expected) throws Exception {
1221         final String result = getJsVariableValue(driver, varName);
1222         assertEquals(expected, result);
1223 
1224         return driver;
1225     }
1226 
1227     protected final WebDriver verifyWindowName2(final Duration maxWaitTime, final WebDriver driver,
1228             final String... expectedAlerts) throws Exception {
1229         final long maxWait = System.currentTimeMillis() + maxWaitTime.toMillis();
1230 
1231         while (System.currentTimeMillis() < maxWait) {
1232             try {
1233                 return verifyWindowName2(driver, expectedAlerts);
1234             }
1235             catch (final AssertionError e) {
1236                 // ignore and wait
1237             }
1238         }
1239 
1240         return verifyWindowName2(driver, expectedAlerts);
1241     }
1242 
1243     protected final WebDriver verifyWindowName2(final WebDriver driver,
1244             final String... expectedAlerts) throws Exception {
1245         final StringBuilder expected = new StringBuilder();
1246         for (int i = 0; i < expectedAlerts.length; i++) {
1247             expected.append(expectedAlerts[i]).append('\u00A7');
1248         }
1249 
1250         return verifyJsVariable(driver, "window.top.name", expected.toString());
1251     }
1252 
1253     protected final WebDriver verifySessionStorage2(final WebDriver driver,
1254             final String... expectedAlerts) throws Exception {
1255         final StringBuilder expected = new StringBuilder();
1256         for (int i = 0; i < expectedAlerts.length; i++) {
1257             expected.append(expectedAlerts[i]).append('\u00A7');
1258         }
1259 
1260         return verifyJsVariable(driver, "sessionStorage.getItem('Log')", expected.toString());
1261     }
1262 
1263     /**
1264      * Defines the provided HTML as the response for {@link WebTestCase#URL_FIRST}
1265      * and loads the page with this URL using the current WebDriver version; finally, asserts that the
1266      * alerts equal the expected alerts.
1267      * @param html the HTML to use
1268      * @param maxWaitTime the maximum time to wait to get the alerts (in millis)
1269      * @return the web driver
1270      * @throws Exception if something goes wrong
1271      */
1272     protected final WebDriver loadPageWithAlerts2(final String html, final Duration maxWaitTime) throws Exception {
1273         return loadPageWithAlerts2(html, URL_FIRST, maxWaitTime);
1274     }
1275 
1276     /**
1277      * Same as {@link #loadPageWithAlerts2(String)}, but specifying the default URL.
1278      * @param html the HTML to use
1279      * @param url the URL to use to load the page
1280      * @return the web driver
1281      * @throws Exception if something goes wrong
1282      */
1283     protected final WebDriver loadPageWithAlerts2(final String html, final URL url) throws Exception {
1284         return loadPageWithAlerts2(html, url, DEFAULT_WAIT_TIME);
1285     }
1286 
1287     /**
1288      * Same as {@link #loadPageWithAlerts2(String, long)}, but specifying the default URL.
1289      * @param html the HTML to use
1290      * @param url the URL to use to load the page
1291      * @param maxWaitTime the maximum time to wait to get the alerts (in millis)
1292      * @return the web driver
1293      * @throws Exception if something goes wrong
1294      */
1295     protected final WebDriver loadPageWithAlerts2(final String html, final URL url, final Duration maxWaitTime)
1296             throws Exception {
1297         final WebDriver driver = loadPage2(html, url);
1298 
1299         verifyAlerts(maxWaitTime, driver, getExpectedAlerts());
1300         return driver;
1301     }
1302 
1303     /**
1304      * Verifies the captured alerts.
1305      * @param driver the driver instance
1306      * @param expectedAlerts the expected alerts
1307      * @throws Exception in case of failure
1308      */
1309     protected void verifyAlerts(final WebDriver driver, final String... expectedAlerts) throws Exception {
1310         verifyAlerts(DEFAULT_WAIT_TIME, driver, expectedAlerts);
1311     }
1312 
1313     /**
1314      * Verifies the captured alerts.
1315      *
1316      * @param maxWaitTime the maximum time to wait for the expected alert to be found
1317      * @param driver the driver instance
1318      * @param expected the expected alerts
1319      * @throws Exception in case of failure
1320      */
1321     protected void verifyAlerts(final Duration maxWaitTime, final WebDriver driver, final String... expected)
1322             throws Exception {
1323         final List<String> actualAlerts = getCollectedAlerts(maxWaitTime, driver, expected.length);
1324 
1325         assertEquals(expected.length, actualAlerts.size());
1326 
1327         if (!useRealBrowser()) {
1328             // check if we have data-image Url
1329             for (int i = 0; i < expected.length; i++) {
1330                 if (expected[i].startsWith("data:image/png;base64,")) {
1331                     // we have to compare element by element
1332                     for (int j = 0; j < expected.length; j++) {
1333                         if (expected[j].startsWith("data:image/png;base64,")) {
1334                             compareImages(expected[j], actualAlerts.get(j));
1335                         }
1336                         else {
1337                             assertEquals(expected[j], actualAlerts.get(j));
1338                         }
1339                     }
1340                     return;
1341                 }
1342             }
1343         }
1344 
1345         assertEquals(expected, actualAlerts);
1346     }
1347 
1348     /**
1349      * Same as {@link #loadPageWithAlerts2(String)} with additional servlet configuration.
1350      * @param html the HTML to use for the default response
1351      * @param servlets the additional servlets to configure with their mapping
1352      * @return the web driver
1353      * @throws Exception if something goes wrong
1354      */
1355     protected final WebDriver loadPageWithAlerts2(final String html,
1356             final Map<String, Class<? extends Servlet>> servlets) throws Exception {
1357         return loadPageWithAlerts2(html, URL_FIRST, DEFAULT_WAIT_TIME, servlets);
1358     }
1359 
1360     /**
1361      * Same as {@link #loadPageWithAlerts2(String, URL, long)}, but with additional servlet configuration.
1362      * @param html the HTML to use for the default page
1363      * @param url the URL to use to load the page
1364      * @param maxWaitTime the maximum time to wait to get the alerts (in millis)
1365      * @param servlets the additional servlets to configure with their mapping
1366      * @return the web driver
1367      * @throws Exception if something goes wrong
1368      */
1369     protected final WebDriver loadPageWithAlerts2(final String html, final URL url, final Duration maxWaitTime,
1370             final Map<String, Class<? extends Servlet>> servlets) throws Exception {
1371 
1372         expandExpectedAlertsVariables(URL_FIRST);
1373 
1374         final WebDriver driver = loadPage2(html, url, servlets);
1375         verifyAlerts(maxWaitTime, driver, getExpectedAlerts());
1376 
1377         return driver;
1378     }
1379 
1380     /**
1381      * Loads the provided URL serving responses from {@link #getMockWebConnection()}
1382      * and verifies that the captured alerts are correct.
1383      * @param url the URL to use to load the page
1384      * @return the web driver
1385      * @throws Exception if something goes wrong
1386      */
1387     protected final WebDriver loadPageWithAlerts2(final URL url) throws Exception {
1388         return loadPageWithAlerts2(url, DEFAULT_WAIT_TIME);
1389     }
1390 
1391     /**
1392      * Same as {@link #loadPageWithAlerts2(URL)}, but using with timeout.
1393      * @param url the URL to use to load the page
1394      * @param maxWaitTime the maximum time to wait to get the alerts (in millis)
1395      * @return the web driver
1396      * @throws Exception if something goes wrong
1397      */
1398     protected final WebDriver loadPageWithAlerts2(final URL url, final Duration maxWaitTime) throws Exception {
1399         startWebServer(getMockWebConnection(), null);
1400 
1401         final WebDriver driver = getWebDriver();
1402         driver.get(url.toExternalForm());
1403 
1404         verifyAlerts(maxWaitTime, driver, getExpectedAlerts());
1405         return driver;
1406     }
1407 
1408     /**
1409      * Gets the alerts collected by the driver.
1410      * Note: it currently works only if no new page has been loaded in the window
1411      * @param driver the driver
1412      * @return the collected alerts
1413      * @throws Exception in case of problem
1414      */
1415     protected List<String> getCollectedAlerts(final WebDriver driver) throws Exception {
1416         return getCollectedAlerts(driver, getExpectedAlerts().length);
1417     }
1418 
1419     /**
1420      * Gets the alerts collected by the driver.
1421      * Note: it currently works only if no new page has been loaded in the window
1422      * @param driver the driver
1423      * @param alertsLength the expected length of Alerts
1424      * @return the collected alerts
1425      * @throws Exception in case of problem
1426      */
1427     protected List<String> getCollectedAlerts(final WebDriver driver, final int alertsLength) throws Exception {
1428         return getCollectedAlerts(DEFAULT_WAIT_TIME, driver, alertsLength);
1429     }
1430 
1431     /**
1432      * Gets the alerts collected by the driver.
1433      * Note: it currently works only if no new page has been loaded in the window
1434      * @param maxWaitTime the maximum time to wait to get the alerts (in millis)
1435      * @param driver the driver
1436      * @param alertsLength the expected length of Alerts
1437      * @return the collected alerts
1438      * @throws Exception in case of problem
1439      */
1440     protected List<String> getCollectedAlerts(final Duration maxWaitTime,
1441             final WebDriver driver, final int alertsLength) throws Exception {
1442         final List<String> collectedAlerts = new ArrayList<>();
1443 
1444         long maxWait = System.currentTimeMillis() + maxWaitTime.toMillis();
1445 
1446         while (collectedAlerts.size() < alertsLength && System.currentTimeMillis() < maxWait) {
1447             try {
1448                 final Alert alert = driver.switchTo().alert();
1449                 final String text = alert.getText();
1450 
1451                 collectedAlerts.add(text);
1452                 alert.accept();
1453 
1454                 // handling of alerts requires some time
1455                 // at least for tests with many alerts we have to take this into account
1456                 maxWait += 100;
1457             }
1458             catch (final NoAlertPresentException e) {
1459                 Thread.sleep(10);
1460             }
1461         }
1462 
1463         return collectedAlerts;
1464     }
1465 
1466     /**
1467      * Returns the HtmlElement of the specified WebElement.
1468      * @param webElement the webElement
1469      * @return the HtmlElement
1470      * @throws Exception if an error occurs
1471      */
1472     protected HtmlElement toHtmlElement(final WebElement webElement) throws Exception {
1473         return (HtmlElement) ((HtmlUnitWebElement) webElement).getElement();
1474     }
1475 
1476     /**
1477      * Loads an expectation file for the specified browser search first for a browser specific resource
1478      * and falling back in a general resource.
1479      * @param resourcePrefix the start of the resource name
1480      * @param resourceSuffix the end of the resource name
1481      * @return the content of the file
1482      * @throws Exception in case of error
1483      */
1484     protected String loadExpectation(final String resourcePrefix, final String resourceSuffix) throws Exception {
1485         final Class<?> referenceClass = getClass();
1486         final BrowserVersion browserVersion = getBrowserVersion();
1487 
1488         String realBrowserNyiExpectation = null;
1489         String realNyiExpectation = null;
1490         final String browserExpectation;
1491         final String expectation;
1492         if (!useRealBrowser()) {
1493             // first try nyi
1494             final String browserSpecificNyiResource
1495                     = resourcePrefix + "." + browserVersion.getNickname() + "_NYI" + resourceSuffix;
1496             realBrowserNyiExpectation = loadContent(referenceClass.getResource(browserSpecificNyiResource));
1497 
1498             // next nyi without browser
1499             final String nyiResource = resourcePrefix + ".NYI" + resourceSuffix;
1500             realNyiExpectation = loadContent(referenceClass.getResource(nyiResource));
1501         }
1502 
1503         // implemented - browser specific
1504         final String browserSpecificResource = resourcePrefix + "." + browserVersion.getNickname() + resourceSuffix;
1505         browserExpectation = loadContent(referenceClass.getResource(browserSpecificResource));
1506 
1507         // implemented - all browsers
1508         final String resource = resourcePrefix + resourceSuffix;
1509         expectation = loadContent(referenceClass.getResource(resource));
1510 
1511         // check for duplicates
1512         if (realBrowserNyiExpectation != null) {
1513             if (realNyiExpectation != null) {
1514                 Assertions.assertNotEquals(realBrowserNyiExpectation, realNyiExpectation,
1515                         "Duplicate NYI Expectation for Browser " + browserVersion.getNickname());
1516             }
1517 
1518             if (browserExpectation == null) {
1519                 if (expectation != null) {
1520                     Assertions.assertNotEquals(realBrowserNyiExpectation, expectation,
1521                             "NYI Expectation matches the expected "
1522                                     + "result for Browser " + browserVersion.getNickname());
1523                 }
1524             }
1525             else {
1526                 Assertions.assertNotEquals(realBrowserNyiExpectation, browserExpectation,
1527                         "NYI Expectation matches the expected "
1528                                 + "browser specific result for Browser "
1529                                 + browserVersion.getNickname());
1530             }
1531 
1532             return realBrowserNyiExpectation;
1533         }
1534 
1535         if (realNyiExpectation != null) {
1536             if (browserExpectation == null) {
1537                 if (expectation != null) {
1538                     Assertions.assertNotEquals(realNyiExpectation, expectation,
1539                             "NYI Expectation matches the expected "
1540                                     + "result for Browser " + browserVersion.getNickname());
1541                 }
1542             }
1543             else {
1544                 Assertions.assertNotEquals(realNyiExpectation, browserExpectation,
1545                             "NYI Expectation matches the expected "
1546                                     + "browser specific result for Browser "
1547                                     + browserVersion.getNickname());
1548             }
1549             return realNyiExpectation;
1550         }
1551 
1552         if (browserExpectation != null) {
1553             if (expectation != null) {
1554                 Assertions.assertNotEquals(browserExpectation, expectation,
1555                             "Browser specific NYI Expectation matches the expected "
1556                                     + "result for Browser " + browserVersion.getNickname());
1557             }
1558             return browserExpectation;
1559         }
1560         return expectation;
1561     }
1562 
1563     private static String loadContent(final URL url) throws URISyntaxException, IOException {
1564         if (url == null) {
1565             return null;
1566         }
1567 
1568         final File file = new File(url.toURI());
1569         String content = FileUtils.readFileToString(file, UTF_8);
1570         content = StringUtils.replace(content, "\r\n", "\n");
1571         return content;
1572     }
1573 
1574     /**
1575      * Reads the number of JS threads remaining from unit tests run before.
1576      * This should be always 0, if {@link #isWebClientCached()} is {@code false}.
1577      * @throws InterruptedException in case of problems
1578      */
1579     @BeforeEach
1580     public void beforeTest() throws InterruptedException {
1581         if (!isWebClientCached()) {
1582             if (!getJavaScriptThreads().isEmpty()) {
1583                 Thread.sleep(200);
1584             }
1585             assertTrue("There are already JS threads running before the test", getJavaScriptThreads().isEmpty());
1586         }
1587     }
1588 
1589     /**
1590      * Asserts the current title is equal to the expectation string.
1591      * @param webdriver the driver in use
1592      * @param expected the expected object
1593      * @throws Exception in case of failure
1594      */
1595     protected void assertTitle(final WebDriver webdriver, final String expected) throws Exception {
1596         final long maxWait = System.currentTimeMillis() + DEFAULT_WAIT_TIME.toMillis();
1597 
1598         while (true) {
1599             final String title = webdriver.getTitle();
1600             try {
1601                 assertEquals(expected, title);
1602                 return;
1603             }
1604             catch (final org.opentest4j.AssertionFailedError e) {
1605                 if (expected.length() <= title.length()
1606                         || System.currentTimeMillis() > maxWait) {
1607                     throw e;
1608                 }
1609                 Thread.sleep(10);
1610             }
1611         }
1612     }
1613 
1614     /**
1615      * Release resources but DON'T close the browser if we are running with a real browser.
1616      * Note that HtmlUnitDriver is not cached by default, but that can be configured by {@link #isWebClientCached()}.
1617      */
1618     @AfterEach
1619     @Override
1620     public void releaseResources() {
1621         final List<String> unhandledAlerts = new ArrayList<>();
1622         if (webDriver_ != null) {
1623             UnhandledAlertException ex = null;
1624             do {
1625                 ex = null;
1626                 try {
1627                     // getTitle will do an implicit check for open alerts
1628                     webDriver_.getTitle();
1629                 }
1630                 catch (final NoSuchWindowException e) {
1631                     // ignore
1632                 }
1633                 catch (final UnhandledAlertException e) {
1634                     ex = e;
1635                     unhandledAlerts.add(e.getMessage());
1636                 }
1637             }
1638             while (ex != null);
1639         }
1640 
1641         super.releaseResources();
1642 
1643         if (!isWebClientCached()) {
1644             boolean rhino = false;
1645             if (webDriver_ != null) {
1646                 try {
1647                     rhino = getWebClient().getJavaScriptEngine() instanceof JavaScriptEngine;
1648                 }
1649                 catch (final Exception e) {
1650                     throw new RuntimeException(e);
1651                 }
1652                 webDriver_.quit();
1653                 webDriver_ = null;
1654             }
1655             if (rhino) {
1656                 assertTrue("There are still JS threads running after the test", getJavaScriptThreads().isEmpty());
1657             }
1658         }
1659 
1660         if (useRealBrowser()) {
1661             synchronized (WEB_DRIVERS_REAL_BROWSERS) {
1662                 final WebDriver driver = WEB_DRIVERS_REAL_BROWSERS.get(getBrowserVersion());
1663                 if (driver != null) {
1664                     try {
1665                         final String currentWindow = driver.getWindowHandle();
1666 
1667                         final Set<String> handles = driver.getWindowHandles();
1668                         // close all windows except the current one
1669                         handles.remove(currentWindow);
1670 
1671                         if (handles.size() > 0) {
1672                             for (final String handle : handles) {
1673                                 try {
1674                                     driver.switchTo().window(handle);
1675                                     driver.close();
1676                                 }
1677                                 catch (final NoSuchWindowException e) {
1678                                     LOG.error("Error switching to browser window; quit browser.", e);
1679                                     WEB_DRIVERS_REAL_BROWSERS.remove(getBrowserVersion());
1680                                     WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.remove(getBrowserVersion());
1681                                     driver.quit();
1682                                     return;
1683                                 }
1684                             }
1685 
1686                             // we have to force WebDriver to treat the remaining window
1687                             // as the one we like to work with from now on
1688                             // looks like a web driver issue to me (version 2.47.2)
1689                             driver.switchTo().window(currentWindow);
1690                         }
1691 
1692                         driver.manage().deleteAllCookies();
1693 
1694                         // in the remaining window, load a blank page
1695                         driver.get("about:blank");
1696                     }
1697                     catch (final NoSuchSessionException e) {
1698                         LOG.error("Error browser session no longer available.", e);
1699                         WEB_DRIVERS_REAL_BROWSERS.remove(getBrowserVersion());
1700                         WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.remove(getBrowserVersion());
1701                         return;
1702                     }
1703                     catch (final WebDriverException e) {
1704                         shutDownRealBrowsers();
1705                     }
1706                 }
1707             }
1708         }
1709         assertTrue("There are still unhandled alerts: " + String.join("; ", unhandledAlerts),
1710                         unhandledAlerts.isEmpty());
1711     }
1712 
1713     /**
1714      * Returns the underlying WebWindow of the specified driver.
1715      *
1716      * <b>Your test shouldn't depend primarily on WebClient</b>
1717      *
1718      * @return the current web window
1719      * @see #toHtmlElement(WebElement)
1720      */
1721     protected WebWindow getWebWindow() {
1722         return webDriver_.getCurrentWindow().getWebWindow();
1723     }
1724 
1725     /**
1726      * Whether {@link WebClient} is cached or not, defaults to {@code false}.
1727      *
1728      * <p>This is needed to be {@code true} for huge test class, as we could run out of sockets.
1729      *
1730      * @return whether {@link WebClient} is cached or not
1731      */
1732     protected boolean isWebClientCached() {
1733         return false;
1734     }
1735 
1736     /**
1737      * Configure {@link WebClientOptions#getTimeout()}.
1738      *
1739      * @return null if unchanged otherwise the timeout as int
1740      */
1741     protected Integer getWebClientTimeout() {
1742         return null;
1743     }
1744 
1745     protected Page getEnclosedPage() {
1746         return getWebWindow().getEnclosedPage();
1747     }
1748 
1749     protected WebClient getWebClient() {
1750         return webDriver_.getWebClient();
1751     }
1752 
1753     /**
1754      * Needed as Jetty starting from 9.4.4 expects UTF-8 encoding by default.
1755      */
1756     public static class AsciiEncodingFilter implements Filter {
1757 
1758         private static Charset CHARSET_;
1759 
1760         /**
1761          * {@inheritDoc}
1762          */
1763         @Override
1764         public void init(final FilterConfig filterConfig) throws ServletException {
1765         }
1766 
1767         /**
1768          * {@inheritDoc}
1769          */
1770         @Override
1771         public void doFilter(final ServletRequest request, final ServletResponse response, final FilterChain chain)
1772                 throws IOException, ServletException {
1773             if (request instanceof Request) {
1774                 ((Request) request).setQueryEncoding(CHARSET_.name());
1775             }
1776             chain.doFilter(request, response);
1777         }
1778 
1779         /**
1780          * {@inheritDoc}
1781          */
1782         @Override
1783         public void destroy() {
1784         }
1785     }
1786 }