View Javadoc
1   /*
2    * Copyright (c) 2002-2025 Gargoyle Software Inc.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    * https://www.apache.org/licenses/LICENSE-2.0
8    *
9    * Unless required by applicable law or agreed to in writing, software
10   * distributed under the License is distributed on an "AS IS" BASIS,
11   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12   * See the License for the specific language governing permissions and
13   * limitations under the License.
14   */
15  package org.htmlunit;
16  
17  import static java.nio.charset.StandardCharsets.ISO_8859_1;
18  import static java.nio.charset.StandardCharsets.UTF_8;
19  
20  import java.io.File;
21  import java.io.FileInputStream;
22  import java.io.IOException;
23  import java.net.URISyntaxException;
24  import java.net.URL;
25  import java.nio.charset.Charset;
26  import java.time.Duration;
27  import java.util.ArrayList;
28  import java.util.Arrays;
29  import java.util.Collections;
30  import java.util.EnumSet;
31  import java.util.Enumeration;
32  import java.util.HashMap;
33  import java.util.HashSet;
34  import java.util.List;
35  import java.util.Locale;
36  import java.util.Map;
37  import java.util.Properties;
38  import java.util.Set;
39  import java.util.concurrent.Executor;
40  import java.util.concurrent.Executors;
41  
42  import javax.servlet.DispatcherType;
43  import javax.servlet.Filter;
44  import javax.servlet.FilterChain;
45  import javax.servlet.FilterConfig;
46  import javax.servlet.Servlet;
47  import javax.servlet.ServletException;
48  import javax.servlet.ServletRequest;
49  import javax.servlet.ServletResponse;
50  import javax.servlet.http.HttpServlet;
51  import javax.servlet.http.HttpServletRequest;
52  import javax.servlet.http.HttpServletResponse;
53  
54  import org.apache.commons.io.FileUtils;
55  import org.apache.commons.io.IOUtils;
56  import org.apache.commons.lang3.StringUtils;
57  import org.apache.commons.lang3.exception.ExceptionUtils;
58  import org.apache.commons.logging.Log;
59  import org.apache.commons.logging.LogFactory;
60  import org.eclipse.jetty.security.ConstraintMapping;
61  import org.eclipse.jetty.security.ConstraintSecurityHandler;
62  import org.eclipse.jetty.security.HashLoginService;
63  import org.eclipse.jetty.server.Request;
64  import org.eclipse.jetty.server.Server;
65  import org.eclipse.jetty.server.handler.HandlerWrapper;
66  import org.eclipse.jetty.util.security.Constraint;
67  import org.eclipse.jetty.webapp.WebAppContext;
68  import org.htmlunit.MockWebConnection.RawResponseData;
69  import org.htmlunit.html.HtmlElement;
70  import org.htmlunit.javascript.JavaScriptEngine;
71  import org.htmlunit.junit.TestCaseCorrector;
72  import org.htmlunit.util.NameValuePair;
73  import org.junit.jupiter.api.AfterAll;
74  import org.junit.jupiter.api.AfterEach;
75  import org.junit.jupiter.api.Assertions;
76  import org.junit.jupiter.api.BeforeEach;
77  import org.junit.jupiter.api.extension.ExtendWith;
78  import org.openqa.selenium.Alert;
79  import org.openqa.selenium.By;
80  import org.openqa.selenium.Dimension;
81  import org.openqa.selenium.JavascriptExecutor;
82  import org.openqa.selenium.NoAlertPresentException;
83  import org.openqa.selenium.NoSuchSessionException;
84  import org.openqa.selenium.NoSuchWindowException;
85  import org.openqa.selenium.UnhandledAlertException;
86  import org.openqa.selenium.WebDriver;
87  import org.openqa.selenium.WebDriverException;
88  import org.openqa.selenium.WebElement;
89  import org.openqa.selenium.chrome.ChromeDriver;
90  import org.openqa.selenium.chrome.ChromeDriverService;
91  import org.openqa.selenium.chrome.ChromeOptions;
92  import org.openqa.selenium.devtools.DevTools;
93  import org.openqa.selenium.devtools.v141.emulation.Emulation;
94  import org.openqa.selenium.edge.EdgeDriver;
95  import org.openqa.selenium.edge.EdgeDriverService;
96  import org.openqa.selenium.edge.EdgeOptions;
97  import org.openqa.selenium.firefox.FirefoxDriver;
98  import org.openqa.selenium.firefox.FirefoxDriverService;
99  import org.openqa.selenium.firefox.FirefoxOptions;
100 import org.openqa.selenium.firefox.FirefoxProfile;
101 import org.openqa.selenium.firefox.GeckoDriverService;
102 import org.openqa.selenium.htmlunit.HtmlUnitDriver;
103 import org.openqa.selenium.htmlunit.HtmlUnitWebElement;
104 import org.openqa.selenium.htmlunit.options.HtmlUnitDriverOptions;
105 import org.openqa.selenium.htmlunit.options.HtmlUnitOption;
106 import org.openqa.selenium.remote.UnreachableBrowserException;
107 
108 /**
109  * Base class for tests using WebDriver.
110  *
111  * By default, this test runs with HtmlUnit, but this behavior can be changed by having a property file named
112  * "{@code test.properties}" in the HtmlUnit root directory.
113  * Sample (remove the part not matching your os):
114  * <pre>
115    browsers=hu,ff,chrome
116 
117    ff.bin=/usr/bin/firefox                              [Unix]
118    ff-esr.bin=/usr/bin/firefox-esr                      [Unix]
119    geckodriver.bin=/usr/bin/driver/geckodriver          [Unix]
120    chrome.bin=/path/to/chromedriver                     [Unix]
121    edge.bin=/path/to/chromedriver                       [Unix]
122 
123    geckodriver.bin=C:\\path\\to\\geckodriver.exe              [Windows]
124    ff.bin=C:\\path\\to\\Mozilla Firefox\\firefox.exe          [Windows]
125    ff-esr.bin=C:\\path\\to\\Mozilla Firefox ESR\\firefox.exe  [Windows]
126    chrome.bin=C:\\path\\to\\chromedriver.exe                  [Windows]
127    edge.bin=C:\\path\\to\\msedgedriver.exe                    [Windows]
128 
129    autofix=false
130    </pre>
131 
132  * The file could contain some properties:
133  * <ul>
134  *   <li>browsers: is a comma separated list contains any combination of
135  *     <ul>
136  *       <li>hu (for HtmlUnit with all browser versions),</li>
137  *       <li>hu-ff,</li>
138  *       <li>hu-ff-esr,</li>
139  *       <li>hu-chrome,</li>
140  *       <li>hu-edge,</li>
141  *       <li>ff, (running test using real Firefox),</li>
142  *       <li>ff-esr, (running test using real Firefox ESR),</li>
143  *       <li>chrome (running test using real Chrome),</li>
144  *       <li>edge (running test using real Edge),</li>
145  *     </ul>
146  *   </li>
147  *
148  *   <li>chrome.bin (mandatory if it does not exist in the <i>path</i>): is the location of the ChromeDriver binary (see
149  *   <a href="https://googlechromelabs.github.io/chrome-for-testing/last-known-good-versions-with-downloads.json">Chrome Driver downloads</a>)</li>
150  *   <li>geckodriver.bin (mandatory if it does not exist in the <i>path</i>): is the location of the GeckoDriver binary
151  *   (see <a href="https://github.com/mozilla/geckodriver/releases">Gecko Driver Releases</a>)</li>
152  *   <li>ff.bin (optional): is the location of the FF binary, in Windows use double back-slashes</li>
153  *   <li>ff-esr.bin (optional): is the location of the FF binary, in Windows use double back-slashes</li>
154  *   <li>edge.bin (mandatory if it does not exist in the <i>path</i>): is the location of the MicrosoftWebDriver binary
155  *   (see <a href="https://developer.microsoft.com/en-us/microsoft-edge/tools/webdriver/">Microsoft Edge WebDriver downloads</a>)</li>
156  *   <li>autofix (optional): if {@code true}, try to automatically fix the real browser expectations,
157  *   or add/remove {@code @NotYetImplemented} annotations, use with caution!</li>
158  * </ul>
159  *
160  * @author Marc Guillemot
161  * @author Ahmed Ashour
162  * @author Ronald Brill
163  * @author Frank Danek
164  */
165 @ExtendWith(TestCaseCorrector.class)
166 public abstract class WebDriverTestCase extends WebTestCase {
167 
168     private static final String LOG_EX_FUNCTION =
169             "  function logEx(e) {\n"
170             + "  let toStr = null;\n"
171             + "  if (toStr === null && e instanceof EvalError) { toStr = ''; }\n"
172             + "  if (toStr === null && e instanceof RangeError) { toStr = ''; }\n"
173             + "  if (toStr === null && e instanceof ReferenceError) { toStr = ''; }\n"
174             + "  if (toStr === null && e instanceof SyntaxError) { toStr = ''; }\n"
175             + "  if (toStr === null && e instanceof TypeError) { toStr = ''; }\n"
176             + "  if (toStr === null && e instanceof URIError) { toStr = ''; }\n"
177             + "  if (toStr === null && e instanceof AggregateError) { toStr = '/AggregateError'; }\n"
178             + "  if (toStr === null && typeof InternalError == 'function' "
179                         + "&& e instanceof InternalError) { toStr = '/InternalError'; }\n"
180             + "  if (toStr === null) {\n"
181             + "    let rx = /\\[object (.*)\\]/;\n"
182             + "    toStr = Object.prototype.toString.call(e);\n"
183             + "    let match = rx.exec(toStr);\n"
184             + "    if (match != null) { toStr = '/' + match[1]; }\n"
185             + "  }"
186             + "  log(e.name + toStr);\n"
187             + "}\n";
188 
189     /**
190      * Function used in many tests.
191      */
192     public static final String LOG_TITLE_FUNCTION =
193             "  function log(msg) { window.document.title += msg + '\\u00a7'; }\n"
194             + LOG_EX_FUNCTION;
195 
196     /**
197      * Function used in many tests.
198      */
199     public static final String LOG_TITLE_FUNCTION_NORMALIZE =
200             "  function log(msg) { "
201                     + "msg = '' + msg; "
202                     + "msg = msg.replace(/ /g, '\\\\s'); "
203                     + "msg = msg.replace(/\\n/g, '\\\\n'); "
204                     + "msg = msg.replace(/\\r/g, '\\\\r'); "
205                     + "msg = msg.replace(/\\t/g, '\\\\t'); "
206                     + "msg = msg.replace(/\\u001e/g, '\\\\u001e'); "
207                     + "window.document.title += msg + '\u00A7';}\n"
208 
209                     + LOG_EX_FUNCTION;
210 
211     /**
212      * Function used in many tests.
213      */
214     public static final String LOG_WINDOW_NAME_FUNCTION =
215             "  function log(msg) { window.top.name += msg + '\\u00a7'; }\n"
216             + "  window.top.name = '';"
217             + LOG_EX_FUNCTION;
218 
219     /**
220      * Function used in many tests.
221      */
222     public static final String LOG_SESSION_STORAGE_FUNCTION =
223             "  function log(msg) { "
224             + "var l = sessionStorage.getItem('Log');"
225             + "sessionStorage.setItem('Log', (null === l?'':l) + msg + '\\u00a7'); }\n";
226 
227     /**
228      * Function used in many tests.
229      */
230     public static final String LOG_TEXTAREA_FUNCTION = "  function log(msg) { "
231             + "document.getElementById('myLog').value += msg + '\u00A7';}\n"
232             + LOG_EX_FUNCTION;
233 
234     /**
235      * HtmlSniped to insert text area used for logging.
236      */
237     public static final String LOG_TEXTAREA = "  <textarea id='myLog' cols='80' rows='22'></textarea>\n";
238 
239     /**
240      * The system property for automatically fixing the test case expectations.
241      */
242     public static final String AUTOFIX_ = "htmlunit.autofix";
243 
244     /**
245      * All browsers supported.
246      */
247     private static List<BrowserVersion> ALL_BROWSERS_ = Collections.unmodifiableList(
248             Arrays.asList(BrowserVersion.CHROME,
249                     BrowserVersion.EDGE,
250                     BrowserVersion.FIREFOX,
251                     BrowserVersion.FIREFOX_ESR));
252 
253     /**
254      * Browsers which run by default.
255      */
256     private static BrowserVersion[] DEFAULT_RUNNING_BROWSERS_ =
257         {BrowserVersion.CHROME,
258             BrowserVersion.EDGE,
259             BrowserVersion.FIREFOX,
260             BrowserVersion.FIREFOX_ESR};
261 
262     private static final Log LOG = LogFactory.getLog(WebDriverTestCase.class);
263 
264     private static Set<String> BROWSERS_PROPERTIES_;
265     private static String CHROME_BIN_;
266     private static String EDGE_BIN_;
267     private static String GECKO_BIN_;
268     private static String FF_BIN_;
269     private static String FF_ESR_BIN_;
270 
271     /** The driver cache. */
272     protected static final Map<BrowserVersion, WebDriver> WEB_DRIVERS_ = new HashMap<>();
273 
274     /** The driver cache for real browsers. */
275     protected static final Map<BrowserVersion, WebDriver> WEB_DRIVERS_REAL_BROWSERS = new HashMap<>();
276     private static final Map<BrowserVersion, Integer> WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT = new HashMap<>();
277 
278     private static Server STATIC_SERVER_;
279     private static String STATIC_SERVER_STARTER_; // stack trace to save the server start code location
280     // second server for cross-origin tests.
281     private static Server STATIC_SERVER2_;
282     private static String STATIC_SERVER2_STARTER_; // stack trace to save the server start code location
283     // third server for multi-origin cross-origin tests.
284     private static Server STATIC_SERVER3_;
285     private static String STATIC_SERVER3_STARTER_; // stack trace to save the server start code location
286 
287     private static Boolean LAST_TEST_UsesMockWebConnection_;
288     private static final Executor EXECUTOR_POOL = Executors.newFixedThreadPool(4);
289 
290     private boolean useRealBrowser_;
291 
292     /**
293      * The HtmlUnitDriver.
294      */
295     private HtmlUnitDriver webDriver_;
296 
297     /**
298      * Override this function in a test class to ask for STATIC_SERVER2_ to be set up.
299      * @return true if two servers are needed.
300      */
301     protected boolean needThreeConnections() {
302         return false;
303     }
304 
305     /**
306      * @return the browser properties (and initializes them lazy)
307      */
308     public static Set<String> getBrowsersProperties() {
309         if (BROWSERS_PROPERTIES_ == null) {
310             try {
311                 final Properties properties = new Properties();
312                 final File file = new File("test.properties");
313                 if (file.exists()) {
314                     try (FileInputStream in = new FileInputStream(file)) {
315                         properties.load(in);
316                     }
317 
318                     String browsersValue = properties.getProperty("browsers");
319                     if (browsersValue == null || browsersValue.isEmpty()) {
320                         browsersValue = "hu";
321                     }
322                     BROWSERS_PROPERTIES_ = new HashSet<>(Arrays.asList(browsersValue.replaceAll(" ", "")
323                             .toLowerCase(Locale.ROOT).split(",")));
324                     CHROME_BIN_ = properties.getProperty("chrome.bin");
325                     EDGE_BIN_ = properties.getProperty("edge.bin");
326 
327                     GECKO_BIN_ = properties.getProperty("geckodriver.bin");
328                     FF_BIN_ = properties.getProperty("ff.bin");
329                     FF_ESR_BIN_ = properties.getProperty("ff-esr.bin");
330 
331                     final boolean autofix = Boolean.parseBoolean(properties.getProperty("autofix"));
332                     System.setProperty(AUTOFIX_, Boolean.toString(autofix));
333                 }
334             }
335             catch (final Exception e) {
336                 LOG.error("Error reading htmlunit.properties. Ignoring!", e);
337             }
338             if (BROWSERS_PROPERTIES_ == null) {
339                 BROWSERS_PROPERTIES_ = new HashSet<>(Arrays.asList("hu"));
340             }
341             if (BROWSERS_PROPERTIES_.contains("hu")) {
342                 for (final BrowserVersion browserVersion : DEFAULT_RUNNING_BROWSERS_) {
343                     BROWSERS_PROPERTIES_.add("hu-" + browserVersion.getNickname().toLowerCase());
344                 }
345             }
346         }
347         return BROWSERS_PROPERTIES_;
348     }
349 
350     /**
351      * @return the list of supported browsers
352      */
353     public static List<BrowserVersion> allBrowsers() {
354         return ALL_BROWSERS_;
355     }
356 
357     /**
358      * Configure the driver only once.
359      * @return the driver
360      */
361     protected WebDriver getWebDriver() {
362         final BrowserVersion browserVersion = getBrowserVersion();
363         WebDriver driver;
364         if (useRealBrowser()) {
365             synchronized (WEB_DRIVERS_REAL_BROWSERS) {
366                 driver = WEB_DRIVERS_REAL_BROWSERS.get(browserVersion);
367                 if (driver != null) {
368                     // there seems to be a memory leak at least with FF;
369                     // we have to restart sometimes
370                     Integer count = WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.get(browserVersion);
371                     if (null == count) {
372                         count = -1;
373                     }
374                     count += 1;
375                     if (count >= 1000) {
376                         shutDownReal(browserVersion);
377                         driver = null;
378                     }
379                     else {
380                         WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.put(browserVersion, count);
381                     }
382                 }
383 
384                 if (driver == null) {
385                     try {
386                         driver = buildWebDriver();
387                     }
388                     catch (final IOException e) {
389                         throw new RuntimeException(e);
390                     }
391 
392                     WEB_DRIVERS_REAL_BROWSERS.put(browserVersion, driver);
393                     WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.put(browserVersion, 0);
394                 }
395             }
396         }
397         else {
398             driver = WEB_DRIVERS_.get(browserVersion);
399             if (driver == null) {
400                 try {
401                     driver = buildWebDriver();
402                 }
403                 catch (final IOException e) {
404                     throw new RuntimeException(e);
405                 }
406 
407                 if (isWebClientCached()) {
408                     WEB_DRIVERS_.put(browserVersion, driver);
409                 }
410             }
411         }
412         return driver;
413     }
414 
415     /**
416      * Closes the drivers.
417      * @throws Exception If an error occurs
418      */
419     @AfterAll
420     public static void shutDownAll() throws Exception {
421         for (final WebDriver driver : WEB_DRIVERS_.values()) {
422             driver.quit();
423         }
424         WEB_DRIVERS_.clear();
425 
426         shutDownRealBrowsers();
427 
428         stopWebServers();
429     }
430 
431     /**
432      * Closes the real browser drivers.
433      */
434     private static void shutDownRealBrowsers() {
435         synchronized (WEB_DRIVERS_REAL_BROWSERS) {
436             for (final WebDriver driver : WEB_DRIVERS_REAL_BROWSERS.values()) {
437                 quit(driver);
438             }
439             WEB_DRIVERS_REAL_BROWSERS.clear();
440             WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.clear();
441         }
442     }
443 
444     /**
445      * Closes the real browser drivers.
446      * @param browser the real browser to close
447      */
448     private static void shutDownReal(final BrowserVersion browser) {
449         synchronized (WEB_DRIVERS_REAL_BROWSERS) {
450             final WebDriver driver = WEB_DRIVERS_REAL_BROWSERS.get(browser);
451             if (driver != null) {
452                 quit(driver);
453                 WEB_DRIVERS_REAL_BROWSERS.remove(browser);
454                 WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.remove(browser);
455             }
456         }
457     }
458 
459     private static void quit(final WebDriver driver) {
460         if (driver != null) {
461             try {
462                 driver.quit();
463             }
464             catch (final UnreachableBrowserException e) {
465                 LOG.error("Can't quit browser", e);
466                 // ignore, the browser is gone
467             }
468             catch (final NoClassDefFoundError e) {
469                 LOG.error("Can't quit browser", e);
470                 // ignore, the browser is gone
471             }
472             catch (final UnsatisfiedLinkError e) {
473                 LOG.error("Can't quit browser", e);
474                 // ignore, the browser is gone
475             }
476         }
477     }
478 
479     /**
480      * Asserts all static servers are null.
481      * @throws Exception if it fails
482      */
483     protected static void assertWebServersStopped() throws Exception {
484         Assertions.assertNull(STATIC_SERVER_, STATIC_SERVER_STARTER_);
485         Assertions.assertNull(STATIC_SERVER2_, STATIC_SERVER2_STARTER_);
486         Assertions.assertNull(STATIC_SERVER3_, STATIC_SERVER3_STARTER_);
487     }
488 
489     /**
490      * Stops all WebServers.
491      * @throws Exception if it fails
492      */
493     protected static void stopWebServers() throws Exception {
494         if (STATIC_SERVER_ != null) {
495             STATIC_SERVER_.stop();
496             STATIC_SERVER_.destroy();
497             STATIC_SERVER_ = null;
498         }
499 
500         if (STATIC_SERVER2_ != null) {
501             STATIC_SERVER2_.stop();
502             STATIC_SERVER2_.destroy();
503             STATIC_SERVER2_ = null;
504         }
505 
506         if (STATIC_SERVER3_ != null) {
507             STATIC_SERVER3_.stop();
508             STATIC_SERVER3_.destroy();
509             STATIC_SERVER3_ = null;
510         }
511         LAST_TEST_UsesMockWebConnection_ = null;
512     }
513 
514     /**
515      * @return whether to use real browser or not.
516      */
517     public boolean useRealBrowser() {
518         return useRealBrowser_;
519     }
520 
521     /**
522      * Sets whether to use real browser or not.
523      * @param useRealBrowser whether to use real browser or not
524      */
525     public void setUseRealBrowser(final boolean useRealBrowser) {
526         useRealBrowser_ = useRealBrowser;
527     }
528 
529     /**
530      * Builds a new WebDriver instance.
531      * @return the instance
532      * @throws IOException in case of exception
533      */
534     protected WebDriver buildWebDriver() throws IOException {
535         if (useRealBrowser()) {
536             if (BrowserVersion.EDGE.isSameBrowser(getBrowserVersion())) {
537                 final EdgeDriverService service = new EdgeDriverService.Builder()
538                         .withLogOutput(System.out)
539                         .usingDriverExecutable(new File(EDGE_BIN_))
540 
541                         .withAppendLog(true)
542                         .withReadableTimestamp(true)
543 
544                         .build();
545 
546                 final String locale = getBrowserVersion().getBrowserLocale().toLanguageTag();
547 
548                 final EdgeOptions options = new EdgeOptions();
549                 // BiDi
550                 // options.setCapability("webSocketUrl", true);
551 
552                 options.addArguments("--lang=" + locale);
553                 // https://stackoverflow.com/questions/11289597/webdriver-how-to-specify-preferred-languages-for-chrome
554                 options.setExperimentalOption("prefs", Map.of("intl.accept_languages", locale));
555                 options.addArguments("--remote-allow-origins=*");
556 
557                 // seems to be not required for edge
558                 // options.addArguments("--disable-search-engine-choice-screen");
559                 // see https://www.selenium.dev/blog/2024/chrome-browser-woes/
560                 // options.addArguments("--disable-features=OptimizationGuideModelDownloading,"
561                 //         + "OptimizationHintsFetching,OptimizationTargetPrediction,OptimizationHints");
562 
563                 final EdgeDriver edge = new EdgeDriver(service, options);
564 
565                 final DevTools devTools = edge.getDevTools();
566                 devTools.createSession();
567 
568                 final String tz = getBrowserVersion().getSystemTimezone().getID();
569                 devTools.send(Emulation.setTimezoneOverride(tz));
570 
571                 return edge;
572             }
573 
574             if (BrowserVersion.CHROME.isSameBrowser(getBrowserVersion())) {
575                 final ChromeDriverService service = new ChromeDriverService.Builder()
576                         .withLogOutput(System.out)
577                         .usingDriverExecutable(new File(CHROME_BIN_))
578 
579                         .withAppendLog(true)
580                         .withReadableTimestamp(true)
581 
582                         .build();
583 
584                 final String locale = getBrowserVersion().getBrowserLocale().toLanguageTag();
585 
586                 final ChromeOptions options = new ChromeOptions();
587                 // BiDi
588                 // options.setCapability("webSocketUrl", true);
589 
590                 options.addArguments("--lang=" + locale);
591                 // https://stackoverflow.com/questions/11289597/webdriver-how-to-specify-preferred-languages-for-chrome
592                 options.setExperimentalOption("prefs", Map.of("intl.accept_languages", locale));
593                 options.addArguments("--remote-allow-origins=*");
594                 options.addArguments("--disable-search-engine-choice-screen");
595                 // see https://www.selenium.dev/blog/2024/chrome-browser-woes/
596                 options.addArguments("--disable-features=OptimizationGuideModelDownloading,"
597                         + "OptimizationHintsFetching,OptimizationTargetPrediction,OptimizationHints");
598 
599                 final ChromeDriver chrome = new ChromeDriver(service, options);
600 
601                 final DevTools devTools = chrome.getDevTools();
602                 devTools.createSession();
603 
604                 final String tz = getBrowserVersion().getSystemTimezone().getID();
605                 devTools.send(Emulation.setTimezoneOverride(tz));
606 
607                 return chrome;
608             }
609 
610             if (BrowserVersion.FIREFOX.isSameBrowser(getBrowserVersion())) {
611                 return createFirefoxDriver(GECKO_BIN_, FF_BIN_);
612             }
613 
614             if (BrowserVersion.FIREFOX_ESR.isSameBrowser(getBrowserVersion())) {
615                 return createFirefoxDriver(GECKO_BIN_, FF_ESR_BIN_);
616             }
617 
618             throw new RuntimeException("Unexpected BrowserVersion: " + getBrowserVersion());
619         }
620 
621         if (webDriver_ == null) {
622             final HtmlUnitDriverOptions driverOptions = new HtmlUnitDriverOptions(getBrowserVersion());
623 
624             if (isWebClientCached()) {
625                 driverOptions.setCapability(HtmlUnitOption.optHistorySizeLimit, 0);
626             }
627 
628             if (getWebClientTimeout() != null) {
629                 driverOptions.setCapability(HtmlUnitOption.optTimeout, getWebClientTimeout());
630             }
631             webDriver_ = new HtmlUnitDriver(driverOptions);
632             webDriver_.setExecutor(EXECUTOR_POOL);
633         }
634         return webDriver_;
635     }
636 
637     private FirefoxDriver createFirefoxDriver(final String geckodriverBinary, final String binary) {
638         final FirefoxDriverService service = new GeckoDriverService.Builder()
639                 .withLogOutput(System.out)
640                 .usingDriverExecutable(new File(geckodriverBinary))
641                 .build();
642 
643         final FirefoxOptions options = new FirefoxOptions();
644         // BiDi
645         // options.setCapability("webSocketUrl", true);
646 
647         options.setBinary(binary);
648 
649         String locale = getBrowserVersion().getBrowserLocale().toLanguageTag();
650         locale = locale + "," + getBrowserVersion().getBrowserLocale().getLanguage();
651 
652         final FirefoxProfile profile = new FirefoxProfile();
653         profile.setPreference("intl.accept_languages", locale);
654         // no idea so far how to set this
655         // final String tz = getBrowserVersion().getSystemTimezone().getID();
656         // profile.setPreference("intl.tz", tz);
657         options.setProfile(profile);
658 
659         return new FirefoxDriver(service, options);
660     }
661 
662     /**
663      * Starts the web server delivering response from the provided connection.
664      * @param mockConnection the sources for responses
665      * @param serverCharset the {@link Charset} at the server side
666      * @throws Exception if a problem occurs
667      */
668     protected void startWebServer(final MockWebConnection mockConnection, final Charset serverCharset)
669             throws Exception {
670         if (Boolean.FALSE.equals(LAST_TEST_UsesMockWebConnection_)) {
671             stopWebServers();
672         }
673 
674         LAST_TEST_UsesMockWebConnection_ = Boolean.TRUE;
675         if (STATIC_SERVER_ == null) {
676             final Server server = new Server(PORT);
677 
678             final WebAppContext context = new WebAppContext();
679             context.setContextPath("/");
680             context.setResourceBase("./");
681 
682             if (isBasicAuthentication()) {
683                 final Constraint constraint = new Constraint();
684                 constraint.setName(Constraint.__BASIC_AUTH);
685                 constraint.setRoles(new String[]{"user"});
686                 constraint.setAuthenticate(true);
687 
688                 final ConstraintMapping constraintMapping = new ConstraintMapping();
689                 constraintMapping.setConstraint(constraint);
690                 constraintMapping.setPathSpec("/*");
691 
692                 final ConstraintSecurityHandler handler = (ConstraintSecurityHandler) context.getSecurityHandler();
693                 handler.setLoginService(new HashLoginService("MyRealm", "./src/test/resources/realm.properties"));
694                 handler.setConstraintMappings(new ConstraintMapping[]{constraintMapping});
695             }
696 
697             context.addServlet(MockWebConnectionServlet.class, "/*");
698             if (serverCharset != null) {
699                 AsciiEncodingFilter.CHARSET_ = serverCharset;
700                 context.addFilter(AsciiEncodingFilter.class, "/*",
701                         EnumSet.of(DispatcherType.INCLUDE, DispatcherType.REQUEST));
702             }
703             server.setHandler(context);
704             WebServerTestCase.tryStart(PORT, server);
705 
706             STATIC_SERVER_STARTER_ = ExceptionUtils.getStackTrace(new Throwable("StaticServerStarter"));
707             STATIC_SERVER_ = server;
708         }
709         MockWebConnectionServlet.MockConnection_ = mockConnection;
710 
711         if (STATIC_SERVER2_ == null && needThreeConnections()) {
712             final Server server2 = new Server(PORT2);
713             final WebAppContext context2 = new WebAppContext();
714             context2.setContextPath("/");
715             context2.setResourceBase("./");
716             context2.addServlet(MockWebConnectionServlet.class, "/*");
717             server2.setHandler(context2);
718             WebServerTestCase.tryStart(PORT2, server2);
719 
720             STATIC_SERVER2_STARTER_ = ExceptionUtils.getStackTrace(new Throwable("StaticServer2Starter"));
721             STATIC_SERVER2_ = server2;
722 
723             final Server server3 = new Server(PORT3);
724             final WebAppContext context3 = new WebAppContext();
725             context3.setContextPath("/");
726             context3.setResourceBase("./");
727             context3.addServlet(MockWebConnectionServlet.class, "/*");
728             server3.setHandler(context3);
729             WebServerTestCase.tryStart(PORT3, server3);
730 
731             STATIC_SERVER3_STARTER_ = ExceptionUtils.getStackTrace(new Throwable("StaticServer3Starter"));
732             STATIC_SERVER3_ = server3;
733             /*
734              * The mock connection servlet call sit under both servers, so long as tests
735              * keep the URLs distinct.
736              */
737         }
738     }
739 
740     /**
741      * Returns whether to use basic authentication for all resources or not.
742      * The default implementation returns false.
743      * @return whether to use basic authentication or not
744      */
745     protected boolean isBasicAuthentication() {
746         return false;
747     }
748 
749     /**
750      * Starts the web server on the default {@link #PORT}.
751      * The given resourceBase is used to be the ROOT directory that serves the default context.
752      * <p><b>Don't forget to stop the returned HttpServer after the test</b>
753      *
754      * @param resourceBase the base of resources for the default context
755      * @param classpath additional classpath entries to add (may be null)
756      * @param servlets map of {String, Class} pairs: String is the path spec, while class is the class
757      * @throws Exception if the test fails
758      */
759     protected static void startWebServer(final String resourceBase, final String[] classpath,
760             final Map<String, Class<? extends Servlet>> servlets) throws Exception {
761         startWebServer(resourceBase, classpath, servlets, null);
762     }
763 
764     /**
765      * Starts the <b>second</b> web server on the default {@link #PORT2}.
766      * The given resourceBase is used to be the ROOT directory that serves the default context.
767      * <p><b>Don't forget to stop the returned HttpServer after the test</b>
768      *
769      * @param resourceBase the base of resources for the default context
770      * @param classpath additional classpath entries to add (may be null)
771      * @param servlets map of {String, Class} pairs: String is the path spec, while class is the class
772      * @throws Exception if the test fails
773      */
774     protected static void startWebServer2(final String resourceBase, final String[] classpath,
775             final Map<String, Class<? extends Servlet>> servlets) throws Exception {
776 
777         if (STATIC_SERVER2_ != null) {
778             STATIC_SERVER2_.stop();
779         }
780         STATIC_SERVER2_STARTER_ = ExceptionUtils.getStackTrace(new Throwable("StaticServer2Starter"));
781         STATIC_SERVER2_ = WebServerTestCase.createWebServer(PORT2, resourceBase, classpath, servlets, null);
782     }
783 
784     /**
785      * Starts the web server on the default {@link #PORT}.
786      * The given resourceBase is used to be the ROOT directory that serves the default context.
787      * <p><b>Don't forget to stop the returned Server after the test</b>
788      *
789      * @param resourceBase the base of resources for the default context
790      * @param classpath additional classpath entries to add (may be null)
791      * @param servlets map of {String, Class} pairs: String is the path spec, while class is the class
792      * @param handler wrapper for handler (can be null)
793      * @throws Exception if the test fails
794      */
795     protected static void startWebServer(final String resourceBase, final String[] classpath,
796             final Map<String, Class<? extends Servlet>> servlets, final HandlerWrapper handler) throws Exception {
797         stopWebServers();
798         LAST_TEST_UsesMockWebConnection_ = Boolean.FALSE;
799 
800         STATIC_SERVER_STARTER_ = ExceptionUtils.getStackTrace(new Throwable("StaticServerStarter"));
801         STATIC_SERVER_ = WebServerTestCase.createWebServer(PORT, resourceBase, classpath, servlets, handler);
802     }
803 
804     /**
805      * Servlet delivering content from a MockWebConnection.
806      */
807     public static class MockWebConnectionServlet extends HttpServlet {
808         private static MockWebConnection MockConnection_;
809 
810         static void setMockconnection(final MockWebConnection connection) {
811             MockConnection_ = connection;
812         }
813 
814         /**
815          * {@inheritDoc}
816          */
817         @Override
818         protected void service(final HttpServletRequest request, final HttpServletResponse response)
819                 throws ServletException, IOException {
820 
821             try {
822                 doService(request, response);
823             }
824             catch (final ServletException e) {
825                 throw e;
826             }
827             catch (final IOException e) {
828                 throw e;
829             }
830             catch (final Exception e) {
831                 throw new ServletException(e);
832             }
833         }
834 
835         private static void doService(final HttpServletRequest request, final HttpServletResponse response)
836                 throws Exception {
837             String url = request.getRequestURL().toString();
838             if (LOG.isDebugEnabled()) {
839                 LOG.debug(request.getMethod() + " " + url);
840             }
841 
842             if (url.endsWith("/favicon.ico")) {
843                 response.setStatus(HttpServletResponse.SC_NOT_FOUND);
844                 return;
845             }
846 
847             if (url.contains("/delay")) {
848                 final String delay = StringUtils.substringBetween(url, "/delay", "/");
849                 final int ms = Integer.parseInt(delay);
850                 if (LOG.isDebugEnabled()) {
851                     LOG.debug("Sleeping for " + ms + " before to deliver " + url);
852                 }
853                 Thread.sleep(ms);
854             }
855 
856             // copy parameters
857             final List<NameValuePair> requestParameters = new ArrayList<>();
858             try {
859                 for (final Enumeration<String> paramNames = request.getParameterNames();
860                         paramNames.hasMoreElements();) {
861                     final String name = paramNames.nextElement();
862                     final String[] values = request.getParameterValues(name);
863                     for (final String value : values) {
864                         requestParameters.add(new NameValuePair(name, value));
865                     }
866                 }
867             }
868             catch (final IllegalArgumentException e) {
869                 // Jetty 8.1.7 throws it in getParameterNames for a query like "cb=%%RANDOM_NUMBER%%"
870                 // => we should use a more low level test server
871                 requestParameters.clear();
872                 final String query = request.getQueryString();
873                 if (query != null) {
874                     url += "?" + query;
875                 }
876             }
877 
878             final String queryString = request.getQueryString();
879             if (StringUtils.isNotBlank(queryString)) {
880                 url = url + "?" + queryString;
881             }
882             final URL requestedUrl = new URL(url);
883             final WebRequest webRequest = new WebRequest(requestedUrl);
884             webRequest.setHttpMethod(HttpMethod.valueOf(request.getMethod()));
885 
886             // copy headers
887             for (final Enumeration<String> en = request.getHeaderNames(); en.hasMoreElements();) {
888                 final String headerName = en.nextElement();
889                 final String headerValue = request.getHeader(headerName);
890                 webRequest.setAdditionalHeader(headerName, headerValue);
891             }
892 
893             if (requestParameters.isEmpty() && request.getContentLength() > 0) {
894                 final byte[] buffer = new byte[request.getContentLength()];
895                 IOUtils.read(request.getInputStream(), buffer, 0, buffer.length);
896 
897                 final String encoding = request.getCharacterEncoding();
898                 if (encoding == null) {
899                     webRequest.setRequestBody(new String(buffer, ISO_8859_1));
900                     webRequest.setCharset(null);
901                 }
902                 else {
903                     webRequest.setRequestBody(new String(buffer, encoding));
904                     webRequest.setCharset(Charset.forName(encoding));
905                 }
906             }
907             else {
908                 webRequest.setRequestParameters(requestParameters);
909             }
910 
911             // check content type if it is multipart enctype
912             if (request.getContentType() != null
913                         && request.getContentType().startsWith(FormEncodingType.MULTIPART.getName())) {
914                 webRequest.setEncodingType(FormEncodingType.MULTIPART);
915             }
916 
917             final RawResponseData resp = MockConnection_.getRawResponse(webRequest);
918 
919             // write WebResponse to HttpServletResponse
920             response.setStatus(resp.getStatusCode());
921 
922             boolean charsetInContentType = false;
923             for (final NameValuePair responseHeader : resp.getHeaders()) {
924                 final String headerName = responseHeader.getName();
925                 if (HttpHeader.CONTENT_TYPE.equals(headerName) && responseHeader.getValue().contains("charset=")) {
926                     charsetInContentType = true;
927                 }
928                 response.addHeader(headerName, responseHeader.getValue());
929             }
930 
931             if (resp.getByteContent() != null) {
932                 response.getOutputStream().write(resp.getByteContent());
933             }
934             else {
935                 if (!charsetInContentType) {
936                     response.setCharacterEncoding(resp.getCharset().name());
937                 }
938                 response.getWriter().print(resp.getStringContent());
939             }
940             response.flushBuffer();
941         }
942     }
943 
944     /**
945      * Same as {@link #loadPageWithAlerts2(String)}... but doesn't verify the alerts.
946      * @param html the HTML to use
947      * @return the web driver
948      * @throws Exception if something goes wrong
949      */
950     protected final WebDriver loadPage2(final String html) throws Exception {
951         return loadPage2(html, URL_FIRST);
952     }
953 
954     /**
955      * Same as {@link #loadPageWithAlerts2(String)}... but doesn't verify the alerts.
956      * @param html the HTML to use
957      * @param url the url to use to load the page
958      * @return the web driver
959      * @throws Exception if something goes wrong
960      */
961     protected final WebDriver loadPage2(final String html, final URL url) throws Exception {
962         return loadPage2(html, url, "text/html;charset=ISO-8859-1", ISO_8859_1, null);
963     }
964 
965     /**
966      * Same as {@link #loadPageWithAlerts2(String)}... but doesn't verify the alerts.
967      * @param html the HTML to use
968      * @param url the url to use to load the page
969      * @param contentType the content type to return
970      * @param charset the charset
971      * @return the web driver
972      * @throws Exception if something goes wrong
973      */
974     protected final WebDriver loadPage2(final String html, final URL url,
975             final String contentType, final Charset charset) throws Exception {
976         return loadPage2(html, url, contentType, charset, null);
977     }
978 
979     /**
980      * Same as {@link #loadPageWithAlerts2(String)}... but doesn't verify the alerts.
981      * @param html the HTML to use
982      * @param url the url to use to load the page
983      * @param contentType the content type to return
984      * @param charset the charset
985      * @param serverCharset the charset at the server side.
986      * @return the web driver
987      * @throws Exception if something goes wrong
988      */
989     protected final WebDriver loadPage2(final String html, final URL url,
990             final String contentType, final Charset charset, final Charset serverCharset) throws Exception {
991         getMockWebConnection().setResponse(url, html, contentType, charset);
992         return loadPage2(url, serverCharset);
993     }
994 
995     /**
996      * Load the page from the url.
997      * @param url the url to use to load the page
998      * @param serverCharset the charset at the server side.
999      * @return the web driver
1000      * @throws Exception if something goes wrong
1001      */
1002     protected final WebDriver loadPage2(final URL url, final Charset serverCharset) throws Exception {
1003         startWebServer(getMockWebConnection(), serverCharset);
1004 
1005         WebDriver driver = getWebDriver();
1006         if (!(driver instanceof HtmlUnitDriver)) {
1007             try {
1008                 resizeIfNeeded(driver);
1009             }
1010             catch (final NoSuchSessionException e) {
1011                 // maybe the driver was killed by the test before; setup a new one
1012                 shutDownRealBrowsers();
1013 
1014                 driver = getWebDriver();
1015                 resizeIfNeeded(driver);
1016             }
1017         }
1018         driver.get(url.toExternalForm());
1019 
1020         return driver;
1021     }
1022 
1023     /**
1024      * Same as {@link #loadPage2(String)} with additional servlet configuration.
1025      * @param html the HTML to use for the default response
1026      * @param servlets the additional servlets to configure with their mapping
1027      * @return the web driver
1028      * @throws Exception if something goes wrong
1029      */
1030     protected final WebDriver loadPage2(final String html,
1031             final Map<String, Class<? extends Servlet>> servlets) throws Exception {
1032         return loadPage2(html, URL_FIRST, servlets, null);
1033     }
1034 
1035     /**
1036      * Same as {@link #loadPage2(String, URL)}, but with additional servlet configuration.
1037      * @param html the HTML to use for the default page
1038      * @param url the URL to use to load the page
1039      * @param servlets the additional servlets to configure with their mapping
1040      * @return the web driver
1041      * @throws Exception if something goes wrong
1042      */
1043     protected final WebDriver loadPage2(final String html, final URL url,
1044             final Map<String, Class<? extends Servlet>> servlets) throws Exception {
1045         return loadPage2(html, url, servlets, null);
1046     }
1047 
1048     /**
1049      * Same as {@link #loadPage2(String, URL)}, but with additional servlet configuration.
1050      * @param html the HTML to use for the default page
1051      * @param url the URL to use to load the page
1052      * @param servlets the additional servlets to configure with their mapping
1053      * @param servlets2 the additional servlets to configure with their mapping for a second server
1054      * @return the web driver
1055      * @throws Exception if something goes wrong
1056      */
1057     protected final WebDriver loadPage2(final String html, final URL url,
1058             final Map<String, Class<? extends Servlet>> servlets,
1059             final Map<String, Class<? extends Servlet>> servlets2) throws Exception {
1060         servlets.put("/*", MockWebConnectionServlet.class);
1061         getMockWebConnection().setResponse(url, html);
1062         MockWebConnectionServlet.MockConnection_ = getMockWebConnection();
1063 
1064         startWebServer("./", null, servlets);
1065         if (servlets2 != null) {
1066             startWebServer2("./", null, servlets2);
1067         }
1068 
1069         WebDriver driver = getWebDriver();
1070         if (!(driver instanceof HtmlUnitDriver)) {
1071             try {
1072                 resizeIfNeeded(driver);
1073             }
1074             catch (final NoSuchSessionException e) {
1075                 // maybe the driver was killed by the test before; setup a new one
1076                 shutDownRealBrowsers();
1077 
1078                 driver = getWebDriver();
1079                 resizeIfNeeded(driver);
1080             }
1081         }
1082         driver.get(url.toExternalForm());
1083 
1084         return driver;
1085     }
1086 
1087     protected void resizeIfNeeded(final WebDriver driver) {
1088         final Dimension size = driver.manage().window().getSize();
1089         if (size.getWidth() != 1272 || size.getHeight() != 768) {
1090             // only resize if needed because it may be quite expensive
1091             driver.manage().window().setSize(new Dimension(1272, 768));
1092         }
1093     }
1094 
1095     /**
1096      * Defines the provided HTML as the response for {@link WebTestCase#URL_FIRST}
1097      * and loads the page with this URL using the current WebDriver version; finally, asserts that the
1098      * alerts equal the expected alerts.
1099      * @param html the HTML to use
1100      * @return the web driver
1101      * @throws Exception if something goes wrong
1102      */
1103     protected final WebDriver loadPageWithAlerts2(final String html) throws Exception {
1104         return loadPageWithAlerts2(html, URL_FIRST, DEFAULT_WAIT_TIME);
1105     }
1106 
1107     /**
1108      * Defines the provided HTML as the response for {@link WebTestCase#URL_FIRST}
1109      * and loads the page with this URL using the current WebDriver version; finally, asserts that the
1110      * alerts equal the expected alerts.
1111      * @param html the HTML to use
1112      * @return the web driver
1113      * @throws Exception if something goes wrong
1114      */
1115     protected final WebDriver loadPageVerifyTitle2(final String html) throws Exception {
1116         return loadPageVerifyTitle2(html, getExpectedAlerts());
1117     }
1118 
1119     protected final WebDriver loadPageVerifyTitle2(final String html, final String... expectedAlerts) throws Exception {
1120         final WebDriver driver = loadPage2(html);
1121         return verifyTitle2(driver, expectedAlerts);
1122     }
1123 
1124     protected final WebDriver verifyTitle2(final Duration maxWaitTime, final WebDriver driver,
1125             final String... expectedAlerts) throws Exception {
1126 
1127         final StringBuilder expected = new StringBuilder();
1128         for (int i = 0; i < expectedAlerts.length; i++) {
1129             expected.append(expectedAlerts[i]).append('\u00A7');
1130         }
1131         final String expectedTitle = expected.toString();
1132 
1133         final long maxWait = System.currentTimeMillis() + maxWaitTime.toMillis();
1134 
1135         while (System.currentTimeMillis() < maxWait) {
1136             try {
1137                 final String title = driver.getTitle();
1138                 assertEquals(expectedTitle, title);
1139                 return driver;
1140             }
1141             catch (final AssertionError e) {
1142                 // ignore and wait
1143             }
1144         }
1145 
1146         assertEquals(expectedTitle, driver.getTitle());
1147         return driver;
1148     }
1149 
1150     protected final WebDriver verifyTitle2(final WebDriver driver,
1151             final String... expectedAlerts) throws Exception {
1152         if (expectedAlerts.length == 0) {
1153             assertEquals("", driver.getTitle());
1154             return driver;
1155         }
1156 
1157         final StringBuilder expected = new StringBuilder();
1158         for (int i = 0; i < expectedAlerts.length; i++) {
1159             expected.append(expectedAlerts[i]).append('\u00A7');
1160         }
1161 
1162         final String title = driver.getTitle();
1163         try {
1164             assertEquals(expected.toString(), title);
1165         }
1166         catch (final AssertionError e) {
1167             if (useRealBrowser() && StringUtils.isEmpty(title)) {
1168                 Thread.sleep(42);
1169                 assertEquals(expected.toString(), driver.getTitle());
1170                 return driver;
1171             }
1172             throw e;
1173         }
1174         return driver;
1175     }
1176 
1177     protected final WebDriver loadPageVerifyTextArea2(final String html) throws Exception {
1178         return loadPageTextArea2(html, getExpectedAlerts());
1179     }
1180 
1181     protected final WebDriver loadPageTextArea2(final String html, final String... expectedAlerts) throws Exception {
1182         final WebDriver driver = loadPage2(html);
1183         return verifyTextArea2(driver, expectedAlerts);
1184     }
1185 
1186     protected final WebDriver verifyTextArea2(final WebDriver driver,
1187             final String... expectedAlerts) throws Exception {
1188         final WebElement textArea = driver.findElement(By.id("myLog"));
1189 
1190         if (expectedAlerts.length == 0) {
1191             assertEquals("", textArea.getDomProperty("value"));
1192             return driver;
1193         }
1194 
1195         if (!useRealBrowser()
1196                 && expectedAlerts.length == 1
1197                 && expectedAlerts[0].startsWith("data:image/png;base64,")) {
1198             String value = textArea.getDomProperty("value");
1199             if (value.endsWith("\u00A7")) {
1200                 value = value.substring(0, value.length() - 1);
1201             }
1202             compareImages(expectedAlerts[0], value);
1203             return driver;
1204         }
1205 
1206         final StringBuilder expected = new StringBuilder();
1207         for (int i = 0; i < expectedAlerts.length; i++) {
1208             expected.append(expectedAlerts[i]).append('\u00A7');
1209         }
1210         verify(() -> textArea.getDomProperty("value"), expected.toString());
1211 
1212         return driver;
1213     }
1214 
1215     protected final String getJsVariableValue(final WebDriver driver, final String varName) throws Exception {
1216         final String script = "return String(" + varName + ")";
1217         final String result = (String) ((JavascriptExecutor) driver).executeScript(script);
1218 
1219         return result;
1220     }
1221 
1222     protected final WebDriver verifyJsVariable(final WebDriver driver, final String varName,
1223             final String expected) throws Exception {
1224         final String result = getJsVariableValue(driver, varName);
1225         assertEquals(expected, result);
1226 
1227         return driver;
1228     }
1229 
1230     protected final WebDriver verifyWindowName2(final Duration maxWaitTime, final WebDriver driver,
1231             final String... expectedAlerts) throws Exception {
1232         final long maxWait = System.currentTimeMillis() + maxWaitTime.toMillis();
1233 
1234         while (System.currentTimeMillis() < maxWait) {
1235             try {
1236                 return verifyWindowName2(driver, expectedAlerts);
1237             }
1238             catch (final AssertionError e) {
1239                 // ignore and wait
1240             }
1241         }
1242 
1243         return verifyWindowName2(driver, expectedAlerts);
1244     }
1245 
1246     protected final WebDriver verifyWindowName2(final WebDriver driver,
1247             final String... expectedAlerts) throws Exception {
1248         final StringBuilder expected = new StringBuilder();
1249         for (int i = 0; i < expectedAlerts.length; i++) {
1250             expected.append(expectedAlerts[i]).append('\u00A7');
1251         }
1252 
1253         return verifyJsVariable(driver, "window.top.name", expected.toString());
1254     }
1255 
1256     protected final WebDriver verifySessionStorage2(final WebDriver driver,
1257             final String... expectedAlerts) throws Exception {
1258         final StringBuilder expected = new StringBuilder();
1259         for (int i = 0; i < expectedAlerts.length; i++) {
1260             expected.append(expectedAlerts[i]).append('\u00A7');
1261         }
1262 
1263         return verifyJsVariable(driver, "sessionStorage.getItem('Log')", expected.toString());
1264     }
1265 
1266     /**
1267      * Defines the provided HTML as the response for {@link WebTestCase#URL_FIRST}
1268      * and loads the page with this URL using the current WebDriver version; finally, asserts that the
1269      * alerts equal the expected alerts.
1270      * @param html the HTML to use
1271      * @param maxWaitTime the maximum time to wait to get the alerts (in millis)
1272      * @return the web driver
1273      * @throws Exception if something goes wrong
1274      */
1275     protected final WebDriver loadPageWithAlerts2(final String html, final Duration maxWaitTime) throws Exception {
1276         return loadPageWithAlerts2(html, URL_FIRST, maxWaitTime);
1277     }
1278 
1279     /**
1280      * Same as {@link #loadPageWithAlerts2(String)}, but specifying the default URL.
1281      * @param html the HTML to use
1282      * @param url the URL to use to load the page
1283      * @return the web driver
1284      * @throws Exception if something goes wrong
1285      */
1286     protected final WebDriver loadPageWithAlerts2(final String html, final URL url) throws Exception {
1287         return loadPageWithAlerts2(html, url, DEFAULT_WAIT_TIME);
1288     }
1289 
1290     /**
1291      * Same as {@link #loadPageWithAlerts2(String, long)}, but specifying the default URL.
1292      * @param html the HTML to use
1293      * @param url the URL to use to load the page
1294      * @param maxWaitTime the maximum time to wait to get the alerts (in millis)
1295      * @return the web driver
1296      * @throws Exception if something goes wrong
1297      */
1298     protected final WebDriver loadPageWithAlerts2(final String html, final URL url, final Duration maxWaitTime)
1299             throws Exception {
1300         final WebDriver driver = loadPage2(html, url);
1301 
1302         verifyAlerts(maxWaitTime, driver, getExpectedAlerts());
1303         return driver;
1304     }
1305 
1306     /**
1307      * Verifies the captured alerts.
1308      * @param driver the driver instance
1309      * @param expectedAlerts the expected alerts
1310      * @throws Exception in case of failure
1311      */
1312     protected void verifyAlerts(final WebDriver driver, final String... expectedAlerts) throws Exception {
1313         verifyAlerts(DEFAULT_WAIT_TIME, driver, expectedAlerts);
1314     }
1315 
1316     /**
1317      * Verifies the captured alerts.
1318      *
1319      * @param maxWaitTime the maximum time to wait for the expected alert to be found
1320      * @param driver the driver instance
1321      * @param expected the expected alerts
1322      * @throws Exception in case of failure
1323      */
1324     protected void verifyAlerts(final Duration maxWaitTime, final WebDriver driver, final String... expected)
1325             throws Exception {
1326         final List<String> actualAlerts = getCollectedAlerts(maxWaitTime, driver, expected.length);
1327 
1328         assertEquals(expected.length, actualAlerts.size());
1329 
1330         if (!useRealBrowser()) {
1331             // check if we have data-image Url
1332             for (int i = 0; i < expected.length; i++) {
1333                 if (expected[i].startsWith("data:image/png;base64,")) {
1334                     // we have to compare element by element
1335                     for (int j = 0; j < expected.length; j++) {
1336                         if (expected[j].startsWith("data:image/png;base64,")) {
1337                             compareImages(expected[j], actualAlerts.get(j));
1338                         }
1339                         else {
1340                             assertEquals(expected[j], actualAlerts.get(j));
1341                         }
1342                     }
1343                     return;
1344                 }
1345             }
1346         }
1347 
1348         assertEquals(expected, actualAlerts);
1349     }
1350 
1351     /**
1352      * Same as {@link #loadPageWithAlerts2(String)} with additional servlet configuration.
1353      * @param html the HTML to use for the default response
1354      * @param servlets the additional servlets to configure with their mapping
1355      * @return the web driver
1356      * @throws Exception if something goes wrong
1357      */
1358     protected final WebDriver loadPageWithAlerts2(final String html,
1359             final Map<String, Class<? extends Servlet>> servlets) throws Exception {
1360         return loadPageWithAlerts2(html, URL_FIRST, DEFAULT_WAIT_TIME, servlets);
1361     }
1362 
1363     /**
1364      * Same as {@link #loadPageWithAlerts2(String, URL, long)}, but with additional servlet configuration.
1365      * @param html the HTML to use for the default page
1366      * @param url the URL to use to load the page
1367      * @param maxWaitTime the maximum time to wait to get the alerts (in millis)
1368      * @param servlets the additional servlets to configure with their mapping
1369      * @return the web driver
1370      * @throws Exception if something goes wrong
1371      */
1372     protected final WebDriver loadPageWithAlerts2(final String html, final URL url, final Duration maxWaitTime,
1373             final Map<String, Class<? extends Servlet>> servlets) throws Exception {
1374 
1375         expandExpectedAlertsVariables(URL_FIRST);
1376 
1377         final WebDriver driver = loadPage2(html, url, servlets);
1378         verifyAlerts(maxWaitTime, driver, getExpectedAlerts());
1379 
1380         return driver;
1381     }
1382 
1383     /**
1384      * Loads the provided URL serving responses from {@link #getMockWebConnection()}
1385      * and verifies that the captured alerts are correct.
1386      * @param url the URL to use to load the page
1387      * @return the web driver
1388      * @throws Exception if something goes wrong
1389      */
1390     protected final WebDriver loadPageWithAlerts2(final URL url) throws Exception {
1391         return loadPageWithAlerts2(url, DEFAULT_WAIT_TIME);
1392     }
1393 
1394     /**
1395      * Same as {@link #loadPageWithAlerts2(URL)}, but using with timeout.
1396      * @param url the URL to use to load the page
1397      * @param maxWaitTime the maximum time to wait to get the alerts (in millis)
1398      * @return the web driver
1399      * @throws Exception if something goes wrong
1400      */
1401     protected final WebDriver loadPageWithAlerts2(final URL url, final Duration maxWaitTime) throws Exception {
1402         startWebServer(getMockWebConnection(), null);
1403 
1404         final WebDriver driver = getWebDriver();
1405         driver.get(url.toExternalForm());
1406 
1407         verifyAlerts(maxWaitTime, driver, getExpectedAlerts());
1408         return driver;
1409     }
1410 
1411     /**
1412      * Gets the alerts collected by the driver.
1413      * Note: it currently works only if no new page has been loaded in the window
1414      * @param driver the driver
1415      * @return the collected alerts
1416      * @throws Exception in case of problem
1417      */
1418     protected List<String> getCollectedAlerts(final WebDriver driver) throws Exception {
1419         return getCollectedAlerts(driver, getExpectedAlerts().length);
1420     }
1421 
1422     /**
1423      * Gets the alerts collected by the driver.
1424      * Note: it currently works only if no new page has been loaded in the window
1425      * @param driver the driver
1426      * @param alertsLength the expected length of Alerts
1427      * @return the collected alerts
1428      * @throws Exception in case of problem
1429      */
1430     protected List<String> getCollectedAlerts(final WebDriver driver, final int alertsLength) throws Exception {
1431         return getCollectedAlerts(DEFAULT_WAIT_TIME, driver, alertsLength);
1432     }
1433 
1434     /**
1435      * Gets the alerts collected by the driver.
1436      * Note: it currently works only if no new page has been loaded in the window
1437      * @param maxWaitTime the maximum time to wait to get the alerts (in millis)
1438      * @param driver the driver
1439      * @param alertsLength the expected length of Alerts
1440      * @return the collected alerts
1441      * @throws Exception in case of problem
1442      */
1443     protected List<String> getCollectedAlerts(final Duration maxWaitTime,
1444             final WebDriver driver, final int alertsLength) throws Exception {
1445         final List<String> collectedAlerts = new ArrayList<>();
1446 
1447         long maxWait = System.currentTimeMillis() + maxWaitTime.toMillis();
1448 
1449         while (collectedAlerts.size() < alertsLength && System.currentTimeMillis() < maxWait) {
1450             try {
1451                 final Alert alert = driver.switchTo().alert();
1452                 final String text = alert.getText();
1453 
1454                 collectedAlerts.add(text);
1455                 alert.accept();
1456 
1457                 // handling of alerts requires some time
1458                 // at least for tests with many alerts we have to take this into account
1459                 maxWait += 100;
1460             }
1461             catch (final NoAlertPresentException e) {
1462                 Thread.sleep(10);
1463             }
1464         }
1465 
1466         return collectedAlerts;
1467     }
1468 
1469     /**
1470      * Returns the HtmlElement of the specified WebElement.
1471      * @param webElement the webElement
1472      * @return the HtmlElement
1473      * @throws Exception if an error occurs
1474      */
1475     protected HtmlElement toHtmlElement(final WebElement webElement) throws Exception {
1476         return (HtmlElement) ((HtmlUnitWebElement) webElement).getElement();
1477     }
1478 
1479     /**
1480      * Loads an expectation file for the specified browser search first for a browser specific resource
1481      * and falling back in a general resource.
1482      * @param resourcePrefix the start of the resource name
1483      * @param resourceSuffix the end of the resource name
1484      * @return the content of the file
1485      * @throws Exception in case of error
1486      */
1487     protected String loadExpectation(final String resourcePrefix, final String resourceSuffix) throws Exception {
1488         final Class<?> referenceClass = getClass();
1489         final BrowserVersion browserVersion = getBrowserVersion();
1490 
1491         String realBrowserNyiExpectation = null;
1492         String realNyiExpectation = null;
1493         final String browserExpectation;
1494         final String expectation;
1495         if (!useRealBrowser()) {
1496             // first try nyi
1497             final String browserSpecificNyiResource
1498                     = resourcePrefix + "." + browserVersion.getNickname() + "_NYI" + resourceSuffix;
1499             realBrowserNyiExpectation = loadContent(referenceClass.getResource(browserSpecificNyiResource));
1500 
1501             // next nyi without browser
1502             final String nyiResource = resourcePrefix + ".NYI" + resourceSuffix;
1503             realNyiExpectation = loadContent(referenceClass.getResource(nyiResource));
1504         }
1505 
1506         // implemented - browser specific
1507         final String browserSpecificResource = resourcePrefix + "." + browserVersion.getNickname() + resourceSuffix;
1508         browserExpectation = loadContent(referenceClass.getResource(browserSpecificResource));
1509 
1510         // implemented - all browsers
1511         final String resource = resourcePrefix + resourceSuffix;
1512         expectation = loadContent(referenceClass.getResource(resource));
1513 
1514         // check for duplicates
1515         if (realBrowserNyiExpectation != null) {
1516             if (realNyiExpectation != null) {
1517                 Assertions.assertNotEquals(realBrowserNyiExpectation, realNyiExpectation,
1518                         "Duplicate NYI Expectation for Browser " + browserVersion.getNickname());
1519             }
1520 
1521             if (browserExpectation == null) {
1522                 if (expectation != null) {
1523                     Assertions.assertNotEquals(realBrowserNyiExpectation, expectation,
1524                             "NYI Expectation matches the expected "
1525                                     + "result for Browser " + browserVersion.getNickname());
1526                 }
1527             }
1528             else {
1529                 Assertions.assertNotEquals(realBrowserNyiExpectation, browserExpectation,
1530                         "NYI Expectation matches the expected "
1531                                 + "browser specific result for Browser "
1532                                 + browserVersion.getNickname());
1533             }
1534 
1535             return realBrowserNyiExpectation;
1536         }
1537 
1538         if (realNyiExpectation != null) {
1539             if (browserExpectation == null) {
1540                 if (expectation != null) {
1541                     Assertions.assertNotEquals(realNyiExpectation, expectation,
1542                             "NYI Expectation matches the expected "
1543                                     + "result for Browser " + browserVersion.getNickname());
1544                 }
1545             }
1546             else {
1547                 Assertions.assertNotEquals(realNyiExpectation, browserExpectation,
1548                             "NYI Expectation matches the expected "
1549                                     + "browser specific result for Browser "
1550                                     + browserVersion.getNickname());
1551             }
1552             return realNyiExpectation;
1553         }
1554 
1555         if (browserExpectation != null) {
1556             if (expectation != null) {
1557                 Assertions.assertNotEquals(browserExpectation, expectation,
1558                             "Browser specific NYI Expectation matches the expected "
1559                                     + "result for Browser " + browserVersion.getNickname());
1560             }
1561             return browserExpectation;
1562         }
1563         return expectation;
1564     }
1565 
1566     private static String loadContent(final URL url) throws URISyntaxException, IOException {
1567         if (url == null) {
1568             return null;
1569         }
1570 
1571         final File file = new File(url.toURI());
1572         String content = FileUtils.readFileToString(file, UTF_8);
1573         content = StringUtils.replace(content, "\r\n", "\n");
1574         return content;
1575     }
1576 
1577     /**
1578      * Reads the number of JS threads remaining from unit tests run before.
1579      * This should be always 0, if {@link #isWebClientCached()} is {@code false}.
1580      * @throws InterruptedException in case of problems
1581      */
1582     @BeforeEach
1583     public void beforeTest() throws InterruptedException {
1584         if (!isWebClientCached()) {
1585             if (!getJavaScriptThreads().isEmpty()) {
1586                 Thread.sleep(200);
1587             }
1588             assertTrue("There are already JS threads running before the test", getJavaScriptThreads().isEmpty());
1589         }
1590     }
1591 
1592     /**
1593      * Asserts the current title is equal to the expectation string.
1594      * @param webdriver the driver in use
1595      * @param expected the expected object
1596      * @throws Exception in case of failure
1597      */
1598     protected void assertTitle(final WebDriver webdriver, final String expected) throws Exception {
1599         final long maxWait = System.currentTimeMillis() + DEFAULT_WAIT_TIME.toMillis();
1600 
1601         while (true) {
1602             final String title = webdriver.getTitle();
1603             try {
1604                 assertEquals(expected, title);
1605                 return;
1606             }
1607             catch (final org.opentest4j.AssertionFailedError e) {
1608                 if (expected.length() <= title.length()
1609                         || System.currentTimeMillis() > maxWait) {
1610                     throw e;
1611                 }
1612                 Thread.sleep(10);
1613             }
1614         }
1615     }
1616 
1617     /**
1618      * Release resources but DON'T close the browser if we are running with a real browser.
1619      * Note that HtmlUnitDriver is not cached by default, but that can be configured by {@link #isWebClientCached()}.
1620      */
1621     @AfterEach
1622     @Override
1623     public void releaseResources() {
1624         final List<String> unhandledAlerts = new ArrayList<>();
1625         if (webDriver_ != null) {
1626             UnhandledAlertException ex = null;
1627             do {
1628                 ex = null;
1629                 try {
1630                     // getTitle will do an implicit check for open alerts
1631                     webDriver_.getTitle();
1632                 }
1633                 catch (final NoSuchWindowException e) {
1634                     // ignore
1635                 }
1636                 catch (final UnhandledAlertException e) {
1637                     ex = e;
1638                     unhandledAlerts.add(e.getMessage());
1639                 }
1640             }
1641             while (ex != null);
1642         }
1643 
1644         super.releaseResources();
1645 
1646         if (!isWebClientCached()) {
1647             boolean rhino = false;
1648             if (webDriver_ != null) {
1649                 try {
1650                     rhino = getWebClient().getJavaScriptEngine() instanceof JavaScriptEngine;
1651                 }
1652                 catch (final Exception e) {
1653                     throw new RuntimeException(e);
1654                 }
1655                 webDriver_.quit();
1656                 webDriver_ = null;
1657             }
1658             if (rhino) {
1659                 assertTrue("There are still JS threads running after the test", getJavaScriptThreads().isEmpty());
1660             }
1661         }
1662 
1663         if (useRealBrowser()) {
1664             synchronized (WEB_DRIVERS_REAL_BROWSERS) {
1665                 final WebDriver driver = WEB_DRIVERS_REAL_BROWSERS.get(getBrowserVersion());
1666                 if (driver != null) {
1667                     try {
1668                         final String currentWindow = driver.getWindowHandle();
1669 
1670                         final Set<String> handles = driver.getWindowHandles();
1671                         // close all windows except the current one
1672                         handles.remove(currentWindow);
1673 
1674                         if (handles.size() > 0) {
1675                             for (final String handle : handles) {
1676                                 try {
1677                                     driver.switchTo().window(handle);
1678                                     driver.close();
1679                                 }
1680                                 catch (final NoSuchWindowException e) {
1681                                     LOG.error("Error switching to browser window; quit browser.", e);
1682                                     WEB_DRIVERS_REAL_BROWSERS.remove(getBrowserVersion());
1683                                     WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.remove(getBrowserVersion());
1684                                     driver.quit();
1685                                     return;
1686                                 }
1687                             }
1688 
1689                             // we have to force WebDriver to treat the remaining window
1690                             // as the one we like to work with from now on
1691                             // looks like a web driver issue to me (version 2.47.2)
1692                             driver.switchTo().window(currentWindow);
1693                         }
1694 
1695                         driver.manage().deleteAllCookies();
1696 
1697                         // in the remaining window, load a blank page
1698                         driver.get("about:blank");
1699                     }
1700                     catch (final NoSuchSessionException e) {
1701                         LOG.error("Error browser session no longer available.", e);
1702                         WEB_DRIVERS_REAL_BROWSERS.remove(getBrowserVersion());
1703                         WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.remove(getBrowserVersion());
1704                         return;
1705                     }
1706                     catch (final WebDriverException e) {
1707                         shutDownRealBrowsers();
1708                     }
1709                 }
1710             }
1711         }
1712         assertTrue("There are still unhandled alerts: " + String.join("; ", unhandledAlerts),
1713                         unhandledAlerts.isEmpty());
1714     }
1715 
1716     /**
1717      * Returns the underlying WebWindow of the specified driver.
1718      *
1719      * <b>Your test shouldn't depend primarily on WebClient</b>
1720      *
1721      * @return the current web window
1722      * @see #toHtmlElement(WebElement)
1723      */
1724     protected WebWindow getWebWindow() {
1725         return webDriver_.getCurrentWindow().getWebWindow();
1726     }
1727 
1728     /**
1729      * Whether {@link WebClient} is cached or not, defaults to {@code false}.
1730      *
1731      * <p>This is needed to be {@code true} for huge test class, as we could run out of sockets.
1732      *
1733      * @return whether {@link WebClient} is cached or not
1734      */
1735     protected boolean isWebClientCached() {
1736         return false;
1737     }
1738 
1739     /**
1740      * Configure {@link WebClientOptions#getTimeout()}.
1741      *
1742      * @return null if unchanged otherwise the timeout as int
1743      */
1744     protected Integer getWebClientTimeout() {
1745         return null;
1746     }
1747 
1748     protected Page getEnclosedPage() {
1749         return getWebWindow().getEnclosedPage();
1750     }
1751 
1752     protected WebClient getWebClient() {
1753         return webDriver_.getWebClient();
1754     }
1755 
1756     /**
1757      * Needed as Jetty starting from 9.4.4 expects UTF-8 encoding by default.
1758      */
1759     public static class AsciiEncodingFilter implements Filter {
1760 
1761         private static Charset CHARSET_;
1762 
1763         /**
1764          * {@inheritDoc}
1765          */
1766         @Override
1767         public void init(final FilterConfig filterConfig) throws ServletException {
1768         }
1769 
1770         /**
1771          * {@inheritDoc}
1772          */
1773         @Override
1774         public void doFilter(final ServletRequest request, final ServletResponse response, final FilterChain chain)
1775                 throws IOException, ServletException {
1776             if (request instanceof Request) {
1777                 ((Request) request).setQueryEncoding(CHARSET_.name());
1778             }
1779             chain.doFilter(request, response);
1780         }
1781 
1782         /**
1783          * {@inheritDoc}
1784          */
1785         @Override
1786         public void destroy() {
1787         }
1788     }
1789 }