View Javadoc
1   /*
2    * Copyright (c) 2002-2026 Gargoyle Software Inc.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    * https://www.apache.org/licenses/LICENSE-2.0
8    *
9    * Unless required by applicable law or agreed to in writing, software
10   * distributed under the License is distributed on an "AS IS" BASIS,
11   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12   * See the License for the specific language governing permissions and
13   * limitations under the License.
14   */
15  package org.htmlunit;
16  
17  import static java.nio.charset.StandardCharsets.ISO_8859_1;
18  import static java.nio.charset.StandardCharsets.UTF_8;
19  
20  import java.io.File;
21  import java.io.FileInputStream;
22  import java.io.IOException;
23  import java.net.URISyntaxException;
24  import java.net.URL;
25  import java.nio.charset.Charset;
26  import java.time.Duration;
27  import java.util.ArrayList;
28  import java.util.Arrays;
29  import java.util.Enumeration;
30  import java.util.HashMap;
31  import java.util.HashSet;
32  import java.util.List;
33  import java.util.Locale;
34  import java.util.Map;
35  import java.util.Properties;
36  import java.util.Set;
37  import java.util.concurrent.Executor;
38  import java.util.concurrent.Executors;
39  
40  import org.apache.commons.io.FileUtils;
41  import org.apache.commons.io.IOUtils;
42  import org.apache.commons.lang3.StringUtils;
43  import org.apache.commons.lang3.exception.ExceptionUtils;
44  import org.apache.commons.logging.Log;
45  import org.apache.commons.logging.LogFactory;
46  import org.eclipse.jetty.server.Server;
47  import org.htmlunit.MockWebConnection.RawResponseData;
48  import org.htmlunit.WebServerTestCase.SSLVariant;
49  import org.htmlunit.html.HtmlElement;
50  import org.htmlunit.javascript.JavaScriptEngine;
51  import org.htmlunit.junit.TestCaseCorrector;
52  import org.htmlunit.util.JettyServerUtils;
53  import org.htmlunit.util.NameValuePair;
54  import org.junit.jupiter.api.AfterAll;
55  import org.junit.jupiter.api.AfterEach;
56  import org.junit.jupiter.api.Assertions;
57  import org.junit.jupiter.api.BeforeEach;
58  import org.junit.jupiter.api.extension.ExtendWith;
59  import org.openqa.selenium.Alert;
60  import org.openqa.selenium.By;
61  import org.openqa.selenium.Dimension;
62  import org.openqa.selenium.JavascriptExecutor;
63  import org.openqa.selenium.NoAlertPresentException;
64  import org.openqa.selenium.NoSuchSessionException;
65  import org.openqa.selenium.NoSuchWindowException;
66  import org.openqa.selenium.UnhandledAlertException;
67  import org.openqa.selenium.WebDriver;
68  import org.openqa.selenium.WebDriverException;
69  import org.openqa.selenium.WebElement;
70  import org.openqa.selenium.chrome.ChromeDriver;
71  import org.openqa.selenium.chrome.ChromeDriverService;
72  import org.openqa.selenium.chrome.ChromeOptions;
73  import org.openqa.selenium.devtools.DevTools;
74  import org.openqa.selenium.devtools.v149.emulation.Emulation;
75  import org.openqa.selenium.edge.EdgeDriver;
76  import org.openqa.selenium.edge.EdgeDriverService;
77  import org.openqa.selenium.edge.EdgeOptions;
78  import org.openqa.selenium.firefox.FirefoxDriver;
79  import org.openqa.selenium.firefox.FirefoxDriverService;
80  import org.openqa.selenium.firefox.FirefoxOptions;
81  import org.openqa.selenium.firefox.FirefoxProfile;
82  import org.openqa.selenium.firefox.GeckoDriverService;
83  import org.openqa.selenium.htmlunit.HtmlUnitDriver;
84  import org.openqa.selenium.htmlunit.HtmlUnitWebElement;
85  import org.openqa.selenium.htmlunit.options.HtmlUnitDriverOptions;
86  import org.openqa.selenium.htmlunit.options.HtmlUnitOption;
87  import org.openqa.selenium.remote.UnreachableBrowserException;
88  import org.opentest4j.AssertionFailedError;
89  
90  import jakarta.servlet.Servlet;
91  import jakarta.servlet.ServletException;
92  import jakarta.servlet.http.HttpServlet;
93  import jakarta.servlet.http.HttpServletRequest;
94  import jakarta.servlet.http.HttpServletResponse;
95  
96  /**
97   * Base class for tests using WebDriver.
98   * <p>
99   * By default, this test runs with HtmlUnit, but this behavior can be changed by having a property file named
100  * "{@code test.properties}" in the HtmlUnit root directory.
101  * Sample (remove the part not matching your os):
102  * <pre>
103    browsers=hu,ff,chrome
104 
105    ff.bin=/usr/bin/firefox                              [Unix]
106    ff-esr.bin=/usr/bin/firefox-esr                      [Unix]
107    geckodriver.bin=/usr/bin/driver/geckodriver          [Unix]
108    chrome.bin=/path/to/chromedriver                     [Unix]
109    edge.bin=/path/to/chromedriver                       [Unix]
110 
111    geckodriver.bin=C:\\path\\to\\geckodriver.exe              [Windows]
112    ff.bin=C:\\path\\to\\Mozilla Firefox\\firefox.exe          [Windows]
113    ff-esr.bin=C:\\path\\to\\Mozilla Firefox ESR\\firefox.exe  [Windows]
114    chrome.bin=C:\\path\\to\\chromedriver.exe                  [Windows]
115    edge.bin=C:\\path\\to\\msedgedriver.exe                    [Windows]
116 
117    autofix=false
118    </pre>
119 
120  * The file could contain some properties:
121  * <ul>
122  *   <li>browsers: is a comma separated list contains any combination of
123  *     <ul>
124  *       <li>hu (for HtmlUnit with all browser versions),</li>
125  *       <li>hu-ff,</li>
126  *       <li>hu-ff-esr,</li>
127  *       <li>hu-chrome,</li>
128  *       <li>hu-edge,</li>
129  *       <li>ff, (running test using real Firefox),</li>
130  *       <li>ff-esr, (running test using real Firefox ESR),</li>
131  *       <li>chrome (running test using real Chrome),</li>
132  *       <li>edge (running test using real Edge),</li>
133  *     </ul>
134  *   </li>
135  *
136  *   <li>chrome.bin (mandatory if it does not exist in the <i>path</i>): is the location of the ChromeDriver binary (see
137  *   <a href="https://googlechromelabs.github.io/chrome-for-testing/last-known-good-versions-with-downloads.json">Chrome Driver downloads</a>)</li>
138  *   <li>geckodriver.bin (mandatory if it does not exist in the <i>path</i>): is the location of the GeckoDriver binary
139  *   (see <a href="https://github.com/mozilla/geckodriver/releases">Gecko Driver Releases</a>)</li>
140  *   <li>ff.bin (optional): is the location of the FF binary, in Windows use double back-slashes</li>
141  *   <li>ff-esr.bin (optional): is the location of the FF binary, in Windows use double back-slashes</li>
142  *   <li>edge.bin (mandatory if it does not exist in the <i>path</i>): is the location of the MicrosoftWebDriver binary
143  *   (see <a href="https://developer.microsoft.com/en-us/microsoft-edge/tools/webdriver/">Microsoft Edge WebDriver downloads</a>)</li>
144  *   <li>autofix (optional): if {@code true}, try to automatically fix the real browser expectations,
145  *   or add/remove {@code @NotYetImplemented} annotations, use with caution!</li>
146  * </ul>
147  *
148  * @author Marc Guillemot
149  * @author Ahmed Ashour
150  * @author Ronald Brill
151  * @author Frank Danek
152  */
153 @ExtendWith(TestCaseCorrector.class)
154 public abstract class WebDriverTestCase extends WebTestCase {
155 
156     private static final String LOG_EX_FUNCTION =
157             "  function logEx(e) {\n"
158             + "    let toStr = null;\n"
159             + "    if (toStr === null && e instanceof EvalError) { toStr = ''; }\n"
160             + "    if (toStr === null && e instanceof RangeError) { toStr = ''; }\n"
161             + "    if (toStr === null && e instanceof ReferenceError) { toStr = ''; }\n"
162             + "    if (toStr === null && e instanceof SyntaxError) { toStr = ''; }\n"
163             + "    if (toStr === null && e instanceof TypeError) { toStr = ''; }\n"
164             + "    if (toStr === null && e instanceof URIError) { toStr = ''; }\n"
165             + "    if (toStr === null && e instanceof AggregateError) { toStr = '/AggregateError'; }\n"
166             + "    if (toStr === null && typeof InternalError == 'function' "
167                           + "&& e instanceof InternalError) { toStr = '/InternalError'; }\n"
168             + "    if (toStr === null) {\n"
169             + "      let rx = /\\[object (.*)\\]/;\n"
170             + "      toStr = Object.prototype.toString.call(e);\n"
171             + "      let match = rx.exec(toStr);\n"
172             + "      if (match != null) { toStr = '/' + match[1]; }\n"
173             + "    }"
174             + "    log(e.name + toStr);\n"
175             + "  }\n";
176 
177     /**
178      * Function used in many tests.
179      */
180     public static final String LOG_TITLE_FUNCTION =
181             "  function log(msg) { window.document.title += msg + '\\u00a7'; }\n"
182             + LOG_EX_FUNCTION;
183 
184     /**
185      * Function used in many tests.
186      */
187     public static final String LOG_TITLE_FUNCTION_NORMALIZE =
188             "  function log(msg) { "
189                     + "msg = '' + msg; "
190                     + "msg = msg.replace(/ /g, '\\\\s'); "
191                     + "msg = msg.replace(/\\n/g, '\\\\n'); "
192                     + "msg = msg.replace(/\\r/g, '\\\\r'); "
193                     + "msg = msg.replace(/\\t/g, '\\\\t'); "
194                     + "msg = msg.replace(/\\u001e/g, '\\\\u001e'); "
195                     + "window.document.title += msg + '\u00A7';}\n"
196 
197                     + LOG_EX_FUNCTION;
198 
199     /**
200      * Function used in many tests.
201      */
202     public static final String LOG_WINDOW_NAME_FUNCTION =
203             "  function log(msg) { window.top.name += msg + '\\u00a7'; }\n"
204             + "  window.top.name = '';"
205             + LOG_EX_FUNCTION;
206 
207     /**
208      * Function used in many tests.
209      */
210     public static final String LOG_SESSION_STORAGE_FUNCTION =
211             "  function log(msg) { "
212             + "var l = sessionStorage.getItem('Log');"
213             + "sessionStorage.setItem('Log', (null === l?'':l) + msg + '\\u00a7'); }\n";
214 
215     /**
216      * Function used in many tests.
217      */
218     public static final String LOG_TEXTAREA_FUNCTION = "  function log(msg) { "
219             + "document.getElementById('myLog').value += msg + '\u00A7';}\n"
220             + LOG_EX_FUNCTION;
221 
222     /**
223      * HtmlSniped to insert text area used for logging.
224      */
225     public static final String LOG_TEXTAREA = "  <textarea id='myLog' cols='80' rows='22'></textarea>\n";
226 
227     /**
228      * The system property for automatically fixing the test case expectations.
229      */
230     public static final String AUTOFIX_ = "htmlunit.autofix";
231 
232     /**
233      * All browsers supported.
234      */
235     private static final List<BrowserVersion> ALL_BROWSERS_ = List.of(BrowserVersion.CHROME, BrowserVersion.EDGE, BrowserVersion.FIREFOX, BrowserVersion.FIREFOX_ESR);
236 
237     /**
238      * Browsers which run by default.
239      */
240     private static final BrowserVersion[] DEFAULT_RUNNING_BROWSERS_ =
241         {BrowserVersion.CHROME,
242             BrowserVersion.EDGE,
243             BrowserVersion.FIREFOX,
244             BrowserVersion.FIREFOX_ESR};
245 
246     private static final Log LOG = LogFactory.getLog(WebDriverTestCase.class);
247 
248     private static Set<String> BROWSERS_PROPERTIES_;
249     private static String CHROME_BIN_;
250     private static String EDGE_BIN_;
251     private static String GECKO_BIN_;
252     private static String FF_BIN_;
253     private static String FF_ESR_BIN_;
254 
255     /** The driver cache. */
256     protected static final Map<BrowserVersion, WebDriver> WEB_DRIVERS_ = new HashMap<>();
257 
258     /** The driver cache for real browsers. */
259     protected static final Map<BrowserVersion, WebDriver> WEB_DRIVERS_REAL_BROWSERS = new HashMap<>();
260     private static final Map<BrowserVersion, Integer> WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT = new HashMap<>();
261 
262     private static Server STATIC_SERVER_;
263     private static String STATIC_SERVER_STARTER_; // stack trace to save the server start code location
264     // second server for cross-origin tests.
265     private static Server STATIC_SERVER2_;
266     private static String STATIC_SERVER2_STARTER_; // stack trace to save the server start code location
267     // third server for multi-origin cross-origin tests.
268     private static Server STATIC_SERVER3_;
269     private static String STATIC_SERVER3_STARTER_; // stack trace to save the server start code location
270 
271     private static Boolean LAST_TEST_UsesMockWebConnection_;
272     private static final Executor EXECUTOR_POOL = Executors.newFixedThreadPool(4);
273 
274     private boolean useRealBrowser_;
275 
276     /**
277      * The HtmlUnitDriver.
278      */
279     private HtmlUnitDriver webDriver_;
280 
281     /**
282      * Override this function in a test class to ask for STATIC_SERVER2_ to be set up.
283      * @return true if two servers are needed.
284      */
285     protected boolean needThreeConnections() {
286         return false;
287     }
288 
289     /**
290      * @return the browser properties (and initializes them lazy)
291      */
292     public static Set<String> getBrowsersProperties() {
293         if (BROWSERS_PROPERTIES_ == null) {
294             try {
295                 final Properties properties = new Properties();
296                 final File file = new File("test.properties");
297                 if (file.exists()) {
298                     try (FileInputStream in = new FileInputStream(file)) {
299                         properties.load(in);
300                     }
301 
302                     String browsersValue = properties.getProperty("browsers");
303                     if (browsersValue == null || browsersValue.isEmpty()) {
304                         browsersValue = "hu";
305                     }
306                     BROWSERS_PROPERTIES_ = new HashSet<>(Arrays.asList(browsersValue.replaceAll(" ", "")
307                             .toLowerCase(Locale.ROOT).split(",")));
308                     CHROME_BIN_ = properties.getProperty("chrome.bin");
309                     EDGE_BIN_ = properties.getProperty("edge.bin");
310 
311                     GECKO_BIN_ = properties.getProperty("geckodriver.bin");
312                     FF_BIN_ = properties.getProperty("ff.bin");
313                     FF_ESR_BIN_ = properties.getProperty("ff-esr.bin");
314 
315                     final boolean autofix = Boolean.parseBoolean(properties.getProperty("autofix"));
316                     System.setProperty(AUTOFIX_, Boolean.toString(autofix));
317                 }
318             }
319             catch (final Exception e) {
320                 LOG.error("Error reading htmlunit.properties. Ignoring!", e);
321             }
322             if (BROWSERS_PROPERTIES_ == null) {
323                 BROWSERS_PROPERTIES_ = new HashSet<>(Arrays.asList("hu"));
324             }
325             if (BROWSERS_PROPERTIES_.contains("hu")) {
326                 for (final BrowserVersion browserVersion : DEFAULT_RUNNING_BROWSERS_) {
327                     BROWSERS_PROPERTIES_.add("hu-" + browserVersion.getNickname().toLowerCase());
328                 }
329             }
330         }
331         return BROWSERS_PROPERTIES_;
332     }
333 
334     /**
335      * @return the list of supported browsers
336      */
337     public static List<BrowserVersion> allBrowsers() {
338         return ALL_BROWSERS_;
339     }
340 
341     /**
342      * Configure the driver only once.
343      * @return the driver
344      */
345     protected WebDriver getWebDriver() {
346         final BrowserVersion browserVersion = getBrowserVersion();
347         WebDriver driver;
348         if (useRealBrowser()) {
349             synchronized (WEB_DRIVERS_REAL_BROWSERS) {
350                 driver = WEB_DRIVERS_REAL_BROWSERS.get(browserVersion);
351                 if (driver != null) {
352                     // there seems to be a memory leak at least with FF;
353                     // we have to restart sometimes
354                     Integer count = WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.get(browserVersion);
355                     if (null == count) {
356                         count = -1;
357                     }
358                     count += 1;
359                     if (count >= 1000) {
360                         shutDownReal(browserVersion);
361                         driver = null;
362                     }
363                     else {
364                         WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.put(browserVersion, count);
365                     }
366                 }
367 
368                 if (driver == null) {
369                     try {
370                         driver = buildWebDriver();
371                     }
372                     catch (final IOException e) {
373                         throw new RuntimeException(e);
374                     }
375 
376                     WEB_DRIVERS_REAL_BROWSERS.put(browserVersion, driver);
377                     WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.put(browserVersion, 0);
378                 }
379             }
380         }
381         else {
382             driver = WEB_DRIVERS_.get(browserVersion);
383             if (driver == null) {
384                 try {
385                     driver = buildWebDriver();
386                 }
387                 catch (final IOException e) {
388                     throw new RuntimeException(e);
389                 }
390 
391                 if (isWebClientCached()) {
392                     WEB_DRIVERS_.put(browserVersion, driver);
393                 }
394             }
395         }
396         return driver;
397     }
398 
399     /**
400      * Closes the drivers.
401      * @throws Exception If an error occurs
402      */
403     @AfterAll
404     public static void shutDownAll() throws Exception {
405         for (final WebDriver driver : WEB_DRIVERS_.values()) {
406             driver.quit();
407         }
408         WEB_DRIVERS_.clear();
409 
410         shutDownRealBrowsers();
411 
412         stopWebServers();
413     }
414 
415     /**
416      * Closes the real browser drivers.
417      */
418     private static void shutDownRealBrowsers() {
419         synchronized (WEB_DRIVERS_REAL_BROWSERS) {
420             for (final WebDriver driver : WEB_DRIVERS_REAL_BROWSERS.values()) {
421                 quit(driver);
422             }
423             WEB_DRIVERS_REAL_BROWSERS.clear();
424             WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.clear();
425         }
426     }
427 
428     /**
429      * Closes the real browser drivers.
430      * @param browser the real browser to close
431      */
432     private static void shutDownReal(final BrowserVersion browser) {
433         synchronized (WEB_DRIVERS_REAL_BROWSERS) {
434             final WebDriver driver = WEB_DRIVERS_REAL_BROWSERS.get(browser);
435             if (driver != null) {
436                 quit(driver);
437                 WEB_DRIVERS_REAL_BROWSERS.remove(browser);
438                 WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.remove(browser);
439             }
440         }
441     }
442 
443     private static void quit(final WebDriver driver) {
444         if (driver != null) {
445             try {
446                 driver.quit();
447             }
448             catch (final UnreachableBrowserException e) {
449                 LOG.error("Can't quit browser", e);
450                 // ignore, the browser is gone
451             }
452             catch (final NoClassDefFoundError e) {
453                 LOG.error("Can't quit browser", e);
454                 // ignore, the browser is gone
455             }
456             catch (final UnsatisfiedLinkError e) {
457                 LOG.error("Can't quit browser", e);
458                 // ignore, the browser is gone
459             }
460         }
461     }
462 
463     /**
464      * Asserts all static servers are null.
465      * @throws Exception if it fails
466      */
467     protected static void assertWebServersStopped() throws Exception {
468         Assertions.assertNull(STATIC_SERVER_, STATIC_SERVER_STARTER_);
469         Assertions.assertNull(STATIC_SERVER2_, STATIC_SERVER2_STARTER_);
470         Assertions.assertNull(STATIC_SERVER3_, STATIC_SERVER3_STARTER_);
471     }
472 
473     /**
474      * Stops all WebServers.
475      * @throws Exception if it fails
476      */
477     protected static void stopWebServers() throws Exception {
478         JettyServerUtils.stopServer(STATIC_SERVER_);
479         STATIC_SERVER_ = null;
480 
481         JettyServerUtils.stopServer(STATIC_SERVER2_);
482         STATIC_SERVER2_ = null;
483 
484         JettyServerUtils.stopServer(STATIC_SERVER3_);
485         STATIC_SERVER3_ = null;
486 
487         LAST_TEST_UsesMockWebConnection_ = null;
488     }
489 
490     /**
491      * @return whether to use real browser or not.
492      */
493     public boolean useRealBrowser() {
494         return useRealBrowser_;
495     }
496 
497     /**
498      * Sets whether to use real browser or not.
499      * @param useRealBrowser whether to use real browser or not
500      */
501     public void setUseRealBrowser(final boolean useRealBrowser) {
502         useRealBrowser_ = useRealBrowser;
503     }
504 
505     /**
506      * Builds a new WebDriver instance.
507      * @return the instance
508      * @throws IOException in case of exception
509      */
510     protected WebDriver buildWebDriver() throws IOException {
511         if (useRealBrowser()) {
512             if (BrowserVersion.EDGE.isSameBrowser(getBrowserVersion())) {
513                 final EdgeDriverService service = new EdgeDriverService.Builder()
514                         .withLogOutput(System.out)
515                         .usingDriverExecutable(new File(EDGE_BIN_))
516 
517                         .withAppendLog(true)
518                         .withReadableTimestamp(true)
519 
520                         .build();
521 
522                 final String locale = getBrowserVersion().getBrowserLocale().toLanguageTag();
523 
524                 final EdgeOptions options = new EdgeOptions();
525                 // BiDi
526                 // options.setCapability("webSocketUrl", true);
527 
528                 options.addArguments("--lang=" + locale);
529                 // https://stackoverflow.com/questions/11289597/webdriver-how-to-specify-preferred-languages-for-chrome
530                 options.setExperimentalOption("prefs", Map.of("intl.accept_languages", locale));
531                 options.addArguments("--remote-allow-origins=*");
532 
533                 // seems to be not required for edge
534                 // options.addArguments("--disable-search-engine-choice-screen");
535                 // see https://www.selenium.dev/blog/2024/chrome-browser-woes/
536                 // options.addArguments("--disable-features=OptimizationGuideModelDownloading,"
537                 //         + "OptimizationHintsFetching,OptimizationTargetPrediction,OptimizationHints");
538 
539                 final EdgeDriver edge = new EdgeDriver(service, options);
540 
541                 final DevTools devTools = edge.getDevTools();
542                 devTools.createSession();
543 
544                 final String tz = getBrowserVersion().getSystemTimezone().getID();
545                 devTools.send(Emulation.setTimezoneOverride(tz));
546 
547                 return edge;
548             }
549 
550             if (BrowserVersion.CHROME.isSameBrowser(getBrowserVersion())) {
551                 final ChromeDriverService service = new ChromeDriverService.Builder()
552                         .withLogOutput(System.out)
553                         .usingDriverExecutable(new File(CHROME_BIN_))
554 
555                         .withAppendLog(true)
556                         .withReadableTimestamp(true)
557 
558                         .build();
559 
560                 final String locale = getBrowserVersion().getBrowserLocale().toLanguageTag();
561 
562                 final ChromeOptions options = new ChromeOptions();
563                 // BiDi
564                 // options.setCapability("webSocketUrl", true);
565 
566                 options.addArguments("--lang=" + locale);
567                 // https://stackoverflow.com/questions/11289597/webdriver-how-to-specify-preferred-languages-for-chrome
568                 options.setExperimentalOption("prefs", Map.of("intl.accept_languages", locale));
569                 options.addArguments("--remote-allow-origins=*");
570                 options.addArguments("--disable-search-engine-choice-screen");
571                 // see https://www.selenium.dev/blog/2024/chrome-browser-woes/
572                 options.addArguments("--disable-features=OptimizationGuideModelDownloading,"
573                         + "OptimizationHintsFetching,OptimizationTargetPrediction,OptimizationHints");
574 
575                 final ChromeDriver chrome = new ChromeDriver(service, options);
576 
577                 final DevTools devTools = chrome.getDevTools();
578                 devTools.createSession();
579 
580                 final String tz = getBrowserVersion().getSystemTimezone().getID();
581                 devTools.send(Emulation.setTimezoneOverride(tz));
582 
583                 return chrome;
584             }
585 
586             if (BrowserVersion.FIREFOX.isSameBrowser(getBrowserVersion())) {
587                 return createFirefoxDriver(GECKO_BIN_, FF_BIN_);
588             }
589 
590             if (BrowserVersion.FIREFOX_ESR.isSameBrowser(getBrowserVersion())) {
591                 return createFirefoxDriver(GECKO_BIN_, FF_ESR_BIN_);
592             }
593 
594             throw new RuntimeException("Unexpected BrowserVersion: " + getBrowserVersion());
595         }
596 
597         if (webDriver_ == null) {
598             final HtmlUnitDriverOptions driverOptions = new HtmlUnitDriverOptions(getBrowserVersion());
599 
600             if (isWebClientCached()) {
601                 driverOptions.setCapability(HtmlUnitOption.optHistorySizeLimit, 0);
602             }
603 
604             if (getWebClientTimeout() != null) {
605                 driverOptions.setCapability(HtmlUnitOption.optTimeout, getWebClientTimeout());
606             }
607             webDriver_ = new HtmlUnitDriver(driverOptions);
608             webDriver_.setExecutor(EXECUTOR_POOL);
609         }
610         return webDriver_;
611     }
612 
613     private FirefoxDriver createFirefoxDriver(final String geckodriverBinary, final String binary) {
614         final FirefoxDriverService service = new GeckoDriverService.Builder()
615                 .withLogOutput(System.out)
616                 .usingDriverExecutable(new File(geckodriverBinary))
617                 .build();
618 
619         final FirefoxOptions options = new FirefoxOptions();
620         // BiDi
621         // options.setCapability("webSocketUrl", true);
622 
623         options.setBinary(binary);
624 
625         String locale = getBrowserVersion().getBrowserLocale().toLanguageTag();
626         locale = locale + "," + getBrowserVersion().getBrowserLocale().getLanguage();
627 
628         final FirefoxProfile profile = new FirefoxProfile();
629         profile.setPreference("intl.accept_languages", locale);
630         // no idea so far how to set this
631         // final String tz = getBrowserVersion().getSystemTimezone().getID();
632         // profile.setPreference("intl.tz", tz);
633         options.setProfile(profile);
634 
635         return new FirefoxDriver(service, options);
636     }
637 
638     /**
639      * Starts the web server delivering response from the provided connection.
640      * @param mockConnection the sources for responses
641      * @param serverCharset the {@link Charset} at the server side
642      * @throws Exception if a problem occurs
643      */
644     protected void startWebServer(final MockWebConnection mockConnection, final Charset serverCharset)
645             throws Exception {
646         if (Boolean.FALSE.equals(LAST_TEST_UsesMockWebConnection_)) {
647             stopWebServers();
648         }
649 
650         // The mock connection servlet call sit under both servers, so long as tests
651         // keep the URLs distinct.
652         final Map<String, Class<? extends Servlet>> servlets = new HashMap<>();
653         servlets.put("/*", MockWebConnectionServlet.class);
654 
655         LAST_TEST_UsesMockWebConnection_ = Boolean.TRUE;
656 
657         if (STATIC_SERVER_ == null) {
658 
659             final Server server = JettyServerUtils.startWebServer(PORT, "./", servlets, serverCharset, isBasicAuthentication(), SSLVariant.NONE);
660             STATIC_SERVER_STARTER_ = ExceptionUtils.getStackTrace(new Throwable("StaticServerStarter"));
661             STATIC_SERVER_ = server;
662         }
663         MockWebConnectionServlet.MockConnection_ = mockConnection;
664 
665         if (STATIC_SERVER2_ == null && needThreeConnections()) {
666             final Server server2 = JettyServerUtils.startWebServer(PORT2, "./", servlets, null, false, SSLVariant.NONE);
667             STATIC_SERVER2_STARTER_ = ExceptionUtils.getStackTrace(new Throwable("StaticServer2Starter"));
668             STATIC_SERVER2_ = server2;
669 
670             final Server server3 = JettyServerUtils.startWebServer(PORT3, "./", servlets, null, false, SSLVariant.NONE);
671             STATIC_SERVER3_STARTER_ = ExceptionUtils.getStackTrace(new Throwable("StaticServer3Starter"));
672             STATIC_SERVER3_ = server3;
673         }
674     }
675 
676     /**
677      * Returns whether to use basic authentication for all resources or not.
678      * The default implementation returns false.
679      * @return whether to use basic authentication or not
680      */
681     protected boolean isBasicAuthentication() {
682         return false;
683     }
684 
685     /**
686      * Starts the web server on the default {@link #PORT}.
687      * The given resourceBase is used to be the ROOT directory that serves the default context.
688      * <p><b>Don't forget to stop the returned HttpServer after the test</b>
689      *
690      * @param resourceBase the base of resources for the default context
691      * @param servlets map of {String, Class} pairs: String is the path spec, while class is the class
692      * @throws Exception if the test fails
693      */
694     protected static void startWebServer(final String resourceBase, final Map<String, Class<? extends Servlet>> servlets) throws Exception {
695         stopWebServers();
696         LAST_TEST_UsesMockWebConnection_ = Boolean.FALSE;
697 
698         STATIC_SERVER_STARTER_ = ExceptionUtils.getStackTrace(new Throwable("StaticServerStarter"));
699         STATIC_SERVER_ = JettyServerUtils.startWebServer(PORT, resourceBase, servlets, null, false, SSLVariant.NONE);
700     }
701 
702     /**
703      * Starts the <b>second</b> web server on the default {@link #PORT2}.
704      * The given resourceBase is used to be the ROOT directory that serves the default context.
705      * <p><b>Don't forget to stop the returned HttpServer after the test</b>
706      *
707      * @param resourceBase the base of resources for the default context
708      * @param servlets map of {String, Class} pairs: String is the path spec, while class is the class
709      * @throws Exception if the test fails
710      */
711     protected static void startWebServer2(final String resourceBase, final Map<String, Class<? extends Servlet>> servlets) throws Exception {
712         if (STATIC_SERVER2_ != null) {
713             JettyServerUtils.stopServer(STATIC_SERVER2_);
714         }
715         STATIC_SERVER2_STARTER_ = ExceptionUtils.getStackTrace(new Throwable("StaticServer2Starter"));
716         STATIC_SERVER2_ = JettyServerUtils.startWebServer(PORT2, resourceBase, servlets, null, false, SSLVariant.NONE);
717     }
718 
719     /**
720      * Servlet delivering content from a MockWebConnection.
721      */
722     public static class MockWebConnectionServlet extends HttpServlet {
723         private static MockWebConnection MockConnection_;
724 
725         static void setMockconnection(final MockWebConnection connection) {
726             MockConnection_ = connection;
727         }
728 
729         /**
730          * {@inheritDoc}
731          */
732         @Override
733         protected void service(final HttpServletRequest request, final HttpServletResponse response)
734                 throws ServletException, IOException {
735 
736             try {
737                 doService(request, response);
738             }
739             catch (final ServletException e) {
740                 throw e;
741             }
742             catch (final IOException e) {
743                 throw e;
744             }
745             catch (final Exception e) {
746                 throw new ServletException(e);
747             }
748         }
749 
750         private static void doService(final HttpServletRequest request, final HttpServletResponse response)
751                 throws Exception {
752             String url = request.getRequestURL().toString();
753             if (LOG.isDebugEnabled()) {
754                 LOG.debug(request.getMethod() + " " + url);
755             }
756 
757             if (url.endsWith("/favicon.ico")) {
758                 response.setStatus(HttpServletResponse.SC_NOT_FOUND);
759                 return;
760             }
761 
762             if (url.contains("/delay")) {
763                 final String delay = StringUtils.substringBetween(url, "/delay", "/");
764                 final int ms = Integer.parseInt(delay);
765                 if (LOG.isDebugEnabled()) {
766                     LOG.debug("Sleeping for " + ms + " before to deliver " + url);
767                 }
768                 Thread.sleep(ms);
769             }
770 
771             // copy parameters
772             final List<NameValuePair> requestParameters = new ArrayList<>();
773             try {
774                 for (final Enumeration<String> paramNames = request.getParameterNames();
775                         paramNames.hasMoreElements();) {
776                     final String name = paramNames.nextElement();
777                     final String[] values = request.getParameterValues(name);
778                     for (final String value : values) {
779                         requestParameters.add(new NameValuePair(name, value));
780                     }
781                 }
782             }
783             catch (final IllegalArgumentException e) {
784                 // Jetty 8.1.7 throws it in getParameterNames for a query like "cb=%%RANDOM_NUMBER%%"
785                 // => we should use a more low level test server
786                 requestParameters.clear();
787                 final String query = request.getQueryString();
788                 if (query != null) {
789                     url += "?" + query;
790                 }
791             }
792 
793             final String queryString = request.getQueryString();
794             if (StringUtils.isNotBlank(queryString)) {
795                 url = url + "?" + queryString;
796             }
797             final URL requestedUrl = new URL(url);
798             final WebRequest webRequest = new WebRequest(requestedUrl);
799 
800             final String method = request.getMethod().toUpperCase(Locale.ROOT);
801             webRequest.setHttpMethod(HttpMethod.valueOf(method));
802 
803             // copy headers
804             for (final Enumeration<String> en = request.getHeaderNames(); en.hasMoreElements();) {
805                 final String headerName = en.nextElement();
806                 final String headerValue = request.getHeader(headerName);
807                 webRequest.setAdditionalHeader(headerName, headerValue);
808             }
809 
810             if (requestParameters.isEmpty() && request.getContentLength() > 0) {
811                 final byte[] buffer = new byte[request.getContentLength()];
812                 IOUtils.read(request.getInputStream(), buffer, 0, buffer.length);
813 
814                 final String encoding = request.getCharacterEncoding();
815                 if (encoding == null) {
816                     webRequest.setRequestBody(new String(buffer, ISO_8859_1));
817                     webRequest.setCharset(null);
818                 }
819                 else {
820                     webRequest.setRequestBody(new String(buffer, encoding));
821                     webRequest.setCharset(Charset.forName(encoding));
822                 }
823             }
824             else {
825                 webRequest.setRequestParameters(requestParameters);
826             }
827 
828             // check content type if it is multipart enctype
829             if (request.getContentType() != null
830                         && request.getContentType().startsWith(FormEncodingType.MULTIPART.getName())) {
831                 webRequest.setEncodingType(FormEncodingType.MULTIPART);
832             }
833 
834             final RawResponseData resp = MockConnection_.getRawResponse(webRequest);
835 
836             // write WebResponse to HttpServletResponse
837             response.setStatus(resp.getStatusCode());
838 
839             boolean charsetInContentType = false;
840             for (final NameValuePair responseHeader : resp.getHeaders()) {
841                 final String headerName = responseHeader.getName();
842                 if (HttpHeader.CONTENT_TYPE.equals(headerName) && responseHeader.getValue().contains("charset=")) {
843                     charsetInContentType = true;
844                 }
845                 response.addHeader(headerName, responseHeader.getValue());
846             }
847 
848             if (resp.getByteContent() != null) {
849                 response.getOutputStream().write(resp.getByteContent());
850             }
851             else {
852                 if (!charsetInContentType) {
853                     response.setCharacterEncoding(resp.getCharset().name());
854                 }
855                 response.getWriter().print(resp.getStringContent());
856             }
857             response.flushBuffer();
858         }
859     }
860 
861     /**
862      * Same as {@link #loadPageWithAlerts2(String)}... but doesn't verify the alerts.
863      * @param html the HTML to use
864      * @return the web driver
865      * @throws Exception if something goes wrong
866      */
867     protected final WebDriver loadPage2(final String html) throws Exception {
868         return loadPage2(html, URL_FIRST);
869     }
870 
871     /**
872      * Same as {@link #loadPageWithAlerts2(String)}... but doesn't verify the alerts.
873      * @param html the HTML to use
874      * @param url the url to use to load the page
875      * @return the web driver
876      * @throws Exception if something goes wrong
877      */
878     protected final WebDriver loadPage2(final String html, final URL url) throws Exception {
879         return loadPage2(html, url, "text/html;charset=ISO-8859-1", ISO_8859_1, null);
880     }
881 
882     /**
883      * Same as {@link #loadPageWithAlerts2(String)}... but doesn't verify the alerts.
884      * @param html the HTML to use
885      * @param url the url to use to load the page
886      * @param contentType the content type to return
887      * @param charset the charset
888      * @return the web driver
889      * @throws Exception if something goes wrong
890      */
891     protected final WebDriver loadPage2(final String html, final URL url,
892             final String contentType, final Charset charset) throws Exception {
893         return loadPage2(html, url, contentType, charset, null);
894     }
895 
896     /**
897      * Same as {@link #loadPageWithAlerts2(String)}... but doesn't verify the alerts.
898      * @param html the HTML to use
899      * @param url the url to use to load the page
900      * @param contentType the content type to return
901      * @param charset the charset
902      * @param serverCharset the charset at the server side.
903      * @return the web driver
904      * @throws Exception if something goes wrong
905      */
906     protected final WebDriver loadPage2(final String html, final URL url,
907             final String contentType, final Charset charset, final Charset serverCharset) throws Exception {
908         getMockWebConnection().setResponse(url, html, contentType, charset);
909         return loadPage2(url, serverCharset);
910     }
911 
912     /**
913      * Load the page from the url.
914      * @param url the url to use to load the page
915      * @param serverCharset the charset at the server side.
916      * @return the web driver
917      * @throws Exception if something goes wrong
918      */
919     protected final WebDriver loadPage2(final URL url, final Charset serverCharset) throws Exception {
920         startWebServer(getMockWebConnection(), serverCharset);
921 
922         WebDriver driver = getWebDriver();
923         if (!(driver instanceof HtmlUnitDriver)) {
924             try {
925                 resizeIfNeeded(driver);
926             }
927             catch (final NoSuchSessionException e) {
928                 // maybe the driver was killed by the test before; setup a new one
929                 shutDownRealBrowsers();
930 
931                 driver = getWebDriver();
932                 resizeIfNeeded(driver);
933             }
934         }
935         driver.get(url.toExternalForm());
936 
937         return driver;
938     }
939 
940     /**
941      * Same as {@link #loadPage2(String)} with additional servlet configuration.
942      * @param html the HTML to use for the default response
943      * @param servlets the additional servlets to configure with their mapping
944      * @return the web driver
945      * @throws Exception if something goes wrong
946      */
947     protected final WebDriver loadPage2(final String html,
948             final Map<String, Class<? extends Servlet>> servlets) throws Exception {
949         return loadPage2(html, URL_FIRST, servlets, null);
950     }
951 
952     /**
953      * Same as {@link #loadPage2(String, URL)}, but with additional servlet configuration.
954      * @param html the HTML to use for the default page
955      * @param url the URL to use to load the page
956      * @param servlets the additional servlets to configure with their mapping
957      * @return the web driver
958      * @throws Exception if something goes wrong
959      */
960     protected final WebDriver loadPage2(final String html, final URL url,
961             final Map<String, Class<? extends Servlet>> servlets) throws Exception {
962         return loadPage2(html, url, servlets, null);
963     }
964 
965     /**
966      * Same as {@link #loadPage2(String, URL)}, but with additional servlet configuration.
967      * @param html the HTML to use for the default page
968      * @param url the URL to use to load the page
969      * @param servlets the additional servlets to configure with their mapping
970      * @param servlets2 the additional servlets to configure with their mapping for a second server
971      * @return the web driver
972      * @throws Exception if something goes wrong
973      */
974     protected final WebDriver loadPage2(final String html, final URL url,
975             final Map<String, Class<? extends Servlet>> servlets,
976             final Map<String, Class<? extends Servlet>> servlets2) throws Exception {
977         servlets.put("/*", MockWebConnectionServlet.class);
978         getMockWebConnection().setResponse(url, html);
979         MockWebConnectionServlet.MockConnection_ = getMockWebConnection();
980 
981         startWebServer("./", servlets);
982         if (servlets2 != null) {
983             startWebServer2("./", servlets2);
984         }
985 
986         WebDriver driver = getWebDriver();
987         if (!(driver instanceof HtmlUnitDriver)) {
988             try {
989                 resizeIfNeeded(driver);
990             }
991             catch (final NoSuchSessionException e) {
992                 // maybe the driver was killed by the test before; setup a new one
993                 shutDownRealBrowsers();
994 
995                 driver = getWebDriver();
996                 resizeIfNeeded(driver);
997             }
998         }
999         driver.get(url.toExternalForm());
1000 
1001         return driver;
1002     }
1003 
1004     protected void resizeIfNeeded(final WebDriver driver) {
1005         final Dimension size = driver.manage().window().getSize();
1006         if (size.getWidth() != 1272 || size.getHeight() != 768) {
1007             // only resize if needed because it may be quite expensive
1008             driver.manage().window().setSize(new Dimension(1272, 768));
1009         }
1010     }
1011 
1012     /**
1013      * Defines the provided HTML as the response for {@link WebTestCase#URL_FIRST}
1014      * and loads the page with this URL using the current WebDriver version; finally, asserts that the
1015      * alerts equal the expected alerts.
1016      * @param html the HTML to use
1017      * @return the web driver
1018      * @throws Exception if something goes wrong
1019      */
1020     protected final WebDriver loadPageWithAlerts2(final String html) throws Exception {
1021         return loadPageWithAlerts2(html, URL_FIRST, DEFAULT_WAIT_TIME);
1022     }
1023 
1024     /**
1025      * Defines the provided HTML as the response for {@link WebTestCase#URL_FIRST}
1026      * and loads the page with this URL using the current WebDriver version; finally, asserts that the
1027      * alerts equal the expected alerts.
1028      * @param html the HTML to use
1029      * @return the web driver
1030      * @throws Exception if something goes wrong
1031      */
1032     protected final WebDriver loadPageVerifyTitle2(final String html) throws Exception {
1033         return loadPageVerifyTitle2(html, getExpectedAlerts());
1034     }
1035 
1036     protected final WebDriver loadPageVerifyTitle2(final String html, final String... expectedAlerts) throws Exception {
1037         final WebDriver driver = loadPage2(html);
1038         return verifyTitle2(driver, expectedAlerts);
1039     }
1040 
1041     protected final WebDriver verifyTitle2(final Duration maxWaitTime, final WebDriver driver,
1042             final String... expectedAlerts) throws Exception {
1043 
1044         final StringBuilder expected = new StringBuilder();
1045         for (String expectedAlert : expectedAlerts) {
1046             expected.append(expectedAlert).append('\u00A7');
1047         }
1048         final String expectedTitle = expected.toString();
1049 
1050         final long maxWait = System.currentTimeMillis() + maxWaitTime.toMillis();
1051 
1052         while (System.currentTimeMillis() < maxWait) {
1053             try {
1054                 final String title = driver.getTitle();
1055                 assertEquals(expectedTitle, title);
1056                 return driver;
1057             }
1058             catch (final AssertionError e) {
1059                 // ignore and wait
1060             }
1061         }
1062 
1063         assertEquals(expectedTitle, driver.getTitle());
1064         return driver;
1065     }
1066 
1067     protected final WebDriver verifyTitle2(final WebDriver driver,
1068             final String... expectedAlerts) throws Exception {
1069         if (expectedAlerts.length == 0) {
1070             assertEquals("", driver.getTitle());
1071             return driver;
1072         }
1073 
1074         final StringBuilder expected = new StringBuilder();
1075         for (String expectedAlert : expectedAlerts) {
1076             expected.append(expectedAlert).append('\u00A7');
1077         }
1078 
1079         final String title = driver.getTitle();
1080         try {
1081             assertEquals(expected.toString(), title);
1082         }
1083         catch (final AssertionError e) {
1084             if (useRealBrowser() && StringUtils.isEmpty(title)) {
1085                 Thread.sleep(42);
1086                 assertEquals(expected.toString(), driver.getTitle());
1087                 return driver;
1088             }
1089             throw e;
1090         }
1091         return driver;
1092     }
1093 
1094     protected final WebDriver loadPageVerifyTextArea2(final String html) throws Exception {
1095         return loadPageTextArea2(html, getExpectedAlerts());
1096     }
1097 
1098     protected final WebDriver loadPageTextArea2(final String html, final String... expectedAlerts) throws Exception {
1099         final WebDriver driver = loadPage2(html);
1100         return verifyTextArea2(driver, expectedAlerts);
1101     }
1102 
1103     protected final WebDriver verifyTextArea2(final WebDriver driver,
1104             final String... expectedAlerts) throws Exception {
1105         final WebElement textArea = driver.findElement(By.id("myLog"));
1106         String value = textArea.getDomProperty("value");
1107 
1108         if (expectedAlerts.length == 0) {
1109             assertEquals("", value);
1110             return driver;
1111         }
1112 
1113         final StringBuilder expected = new StringBuilder();
1114         for (String expectedAlert : expectedAlerts) {
1115             expected.append(expectedAlert).append('\u00A7');
1116         }
1117 
1118         if (expectedAlerts.length == 1 && expectedAlerts[0].startsWith("data:image/png;base64,")) {
1119             try {
1120                 // fastpath
1121                 assertEquals(expected.toString(), value);
1122             } catch (final AssertionFailedError e) {
1123                 // starting wiht FF151 we have canvas fingerprinting protection
1124                 // therefore we have to compare the pictures
1125                 if (useRealBrowser()
1126                         && !(getBrowserVersion().isFirefox() && !getBrowserVersion().isFirefoxESR())) {
1127                     throw e;
1128                 }
1129 
1130                 // fails -> compare images
1131                 if (value.endsWith("\u00A7")) {
1132                     value = value.substring(0, value.length() - 1);
1133                 }
1134 
1135                 compareImages(expectedAlerts[0], value);
1136                 return driver;
1137             }
1138         }
1139 
1140         verify(() -> textArea.getDomProperty("value"), expected.toString());
1141         return driver;
1142     }
1143 
1144     protected final String getJsVariableValue(final WebDriver driver, final String varName) throws Exception {
1145         final String script = "return String(" + varName + ")";
1146         final String result = (String) ((JavascriptExecutor) driver).executeScript(script);
1147 
1148         return result;
1149     }
1150 
1151     protected final WebDriver verifyJsVariable(final WebDriver driver, final String varName,
1152             final String expected) throws Exception {
1153         final String result = getJsVariableValue(driver, varName);
1154         assertEquals(expected, result);
1155 
1156         return driver;
1157     }
1158 
1159     protected final WebDriver verifyWindowName2(final Duration maxWaitTime, final WebDriver driver,
1160             final String... expectedAlerts) throws Exception {
1161         final long maxWait = System.currentTimeMillis() + maxWaitTime.toMillis();
1162 
1163         while (System.currentTimeMillis() < maxWait) {
1164             try {
1165                 return verifyWindowName2(driver, expectedAlerts);
1166             }
1167             catch (final AssertionError e) {
1168                 // ignore and wait
1169             }
1170         }
1171 
1172         return verifyWindowName2(driver, expectedAlerts);
1173     }
1174 
1175     protected final WebDriver verifyWindowName2(final WebDriver driver,
1176             final String... expectedAlerts) throws Exception {
1177         final StringBuilder expected = new StringBuilder();
1178         for (String expectedAlert : expectedAlerts) {
1179             expected.append(expectedAlert).append('\u00A7');
1180         }
1181 
1182         return verifyJsVariable(driver, "window.top.name", expected.toString());
1183     }
1184 
1185     protected final WebDriver verifySessionStorage2(final WebDriver driver,
1186             final String... expectedAlerts) throws Exception {
1187         final StringBuilder expected = new StringBuilder();
1188         for (String expectedAlert : expectedAlerts) {
1189             expected.append(expectedAlert).append('\u00A7');
1190         }
1191 
1192         return verifyJsVariable(driver, "sessionStorage.getItem('Log')", expected.toString());
1193     }
1194 
1195     /**
1196      * Defines the provided HTML as the response for {@link WebTestCase#URL_FIRST}
1197      * and loads the page with this URL using the current WebDriver version; finally, asserts that the
1198      * alerts equal the expected alerts.
1199      * @param html the HTML to use
1200      * @param maxWaitTime the maximum time to wait to get the alerts (in millis)
1201      * @return the web driver
1202      * @throws Exception if something goes wrong
1203      */
1204     protected final WebDriver loadPageWithAlerts2(final String html, final Duration maxWaitTime) throws Exception {
1205         return loadPageWithAlerts2(html, URL_FIRST, maxWaitTime);
1206     }
1207 
1208     /**
1209      * Same as {@link #loadPageWithAlerts2(String)}, but specifying the default URL.
1210      * @param html the HTML to use
1211      * @param url the URL to use to load the page
1212      * @return the web driver
1213      * @throws Exception if something goes wrong
1214      */
1215     protected final WebDriver loadPageWithAlerts2(final String html, final URL url) throws Exception {
1216         return loadPageWithAlerts2(html, url, DEFAULT_WAIT_TIME);
1217     }
1218 
1219     /**
1220      * Same as {@link #loadPageWithAlerts2(String, long)}, but specifying the default URL.
1221      * @param html the HTML to use
1222      * @param url the URL to use to load the page
1223      * @param maxWaitTime the maximum time to wait to get the alerts (in millis)
1224      * @return the web driver
1225      * @throws Exception if something goes wrong
1226      */
1227     protected final WebDriver loadPageWithAlerts2(final String html, final URL url, final Duration maxWaitTime)
1228             throws Exception {
1229         final WebDriver driver = loadPage2(html, url);
1230 
1231         verifyAlerts(maxWaitTime, driver, getExpectedAlerts());
1232         return driver;
1233     }
1234 
1235     /**
1236      * Verifies the captured alerts.
1237      * @param driver the driver instance
1238      * @param expectedAlerts the expected alerts
1239      * @throws Exception in case of failure
1240      */
1241     protected void verifyAlerts(final WebDriver driver, final String... expectedAlerts) throws Exception {
1242         verifyAlerts(DEFAULT_WAIT_TIME, driver, expectedAlerts);
1243     }
1244 
1245     /**
1246      * Verifies the captured alerts.
1247      *
1248      * @param maxWaitTime the maximum time to wait for the expected alert to be found
1249      * @param driver the driver instance
1250      * @param expected the expected alerts
1251      * @throws Exception in case of failure
1252      */
1253     protected void verifyAlerts(final Duration maxWaitTime, final WebDriver driver, final String... expected)
1254             throws Exception {
1255         final List<String> actualAlerts = getCollectedAlerts(maxWaitTime, driver, expected.length);
1256 
1257         assertEquals(expected.length, actualAlerts.size());
1258 
1259         if (!useRealBrowser()) {
1260             // check if we have data-image Url
1261             for (String s : expected) {
1262                 if (s.startsWith("data:image/png;base64,")) {
1263                     // we have to compare element by element
1264                     for (int j = 0; j < expected.length; j++) {
1265                         if (expected[j].startsWith("data:image/png;base64,")) {
1266                             compareImages(expected[j], actualAlerts.get(j));
1267                         } else {
1268                             assertEquals(expected[j], actualAlerts.get(j));
1269                         }
1270                     }
1271                     return;
1272                 }
1273             }
1274         }
1275 
1276         assertEquals(expected, actualAlerts);
1277     }
1278 
1279     /**
1280      * Same as {@link #loadPageWithAlerts2(String)} with additional servlet configuration.
1281      * @param html the HTML to use for the default response
1282      * @param servlets the additional servlets to configure with their mapping
1283      * @return the web driver
1284      * @throws Exception if something goes wrong
1285      */
1286     protected final WebDriver loadPageWithAlerts2(final String html,
1287             final Map<String, Class<? extends Servlet>> servlets) throws Exception {
1288         return loadPageWithAlerts2(html, URL_FIRST, DEFAULT_WAIT_TIME, servlets);
1289     }
1290 
1291     /**
1292      * Same as {@link #loadPageWithAlerts2(String, URL, long)}, but with additional servlet configuration.
1293      * @param html the HTML to use for the default page
1294      * @param url the URL to use to load the page
1295      * @param maxWaitTime the maximum time to wait to get the alerts (in millis)
1296      * @param servlets the additional servlets to configure with their mapping
1297      * @return the web driver
1298      * @throws Exception if something goes wrong
1299      */
1300     protected final WebDriver loadPageWithAlerts2(final String html, final URL url, final Duration maxWaitTime,
1301             final Map<String, Class<? extends Servlet>> servlets) throws Exception {
1302 
1303         expandExpectedAlertsVariables(URL_FIRST);
1304 
1305         final WebDriver driver = loadPage2(html, url, servlets);
1306         verifyAlerts(maxWaitTime, driver, getExpectedAlerts());
1307 
1308         return driver;
1309     }
1310 
1311     /**
1312      * Loads the provided URL serving responses from {@link #getMockWebConnection()}
1313      * and verifies that the captured alerts are correct.
1314      * @param url the URL to use to load the page
1315      * @return the web driver
1316      * @throws Exception if something goes wrong
1317      */
1318     protected final WebDriver loadPageWithAlerts2(final URL url) throws Exception {
1319         return loadPageWithAlerts2(url, DEFAULT_WAIT_TIME);
1320     }
1321 
1322     /**
1323      * Same as {@link #loadPageWithAlerts2(URL)}, but using with timeout.
1324      * @param url the URL to use to load the page
1325      * @param maxWaitTime the maximum time to wait to get the alerts (in millis)
1326      * @return the web driver
1327      * @throws Exception if something goes wrong
1328      */
1329     protected final WebDriver loadPageWithAlerts2(final URL url, final Duration maxWaitTime) throws Exception {
1330         startWebServer(getMockWebConnection(), null);
1331 
1332         final WebDriver driver = getWebDriver();
1333         driver.get(url.toExternalForm());
1334 
1335         verifyAlerts(maxWaitTime, driver, getExpectedAlerts());
1336         return driver;
1337     }
1338 
1339     /**
1340      * Gets the alerts collected by the driver.
1341      * Note: it currently works only if no new page has been loaded in the window
1342      * @param driver the driver
1343      * @return the collected alerts
1344      * @throws Exception in case of problem
1345      */
1346     protected List<String> getCollectedAlerts(final WebDriver driver) throws Exception {
1347         return getCollectedAlerts(driver, getExpectedAlerts().length);
1348     }
1349 
1350     /**
1351      * Gets the alerts collected by the driver.
1352      * Note: it currently works only if no new page has been loaded in the window
1353      * @param driver the driver
1354      * @param alertsLength the expected length of Alerts
1355      * @return the collected alerts
1356      * @throws Exception in case of problem
1357      */
1358     protected List<String> getCollectedAlerts(final WebDriver driver, final int alertsLength) throws Exception {
1359         return getCollectedAlerts(DEFAULT_WAIT_TIME, driver, alertsLength);
1360     }
1361 
1362     /**
1363      * Gets the alerts collected by the driver.
1364      * Note: it currently works only if no new page has been loaded in the window
1365      * @param maxWaitTime the maximum time to wait to get the alerts (in millis)
1366      * @param driver the driver
1367      * @param alertsLength the expected length of Alerts
1368      * @return the collected alerts
1369      * @throws Exception in case of problem
1370      */
1371     protected List<String> getCollectedAlerts(final Duration maxWaitTime,
1372             final WebDriver driver, final int alertsLength) throws Exception {
1373         final List<String> collectedAlerts = new ArrayList<>();
1374 
1375         long maxWait = System.currentTimeMillis() + maxWaitTime.toMillis();
1376 
1377         while (collectedAlerts.size() < alertsLength && System.currentTimeMillis() < maxWait) {
1378             try {
1379                 final Alert alert = driver.switchTo().alert();
1380                 final String text = alert.getText();
1381 
1382                 collectedAlerts.add(text);
1383                 alert.accept();
1384 
1385                 // handling of alerts requires some time
1386                 // at least for tests with many alerts we have to take this into account
1387                 maxWait += 100;
1388             }
1389             catch (final NoAlertPresentException e) {
1390                 Thread.sleep(10);
1391             }
1392         }
1393 
1394         return collectedAlerts;
1395     }
1396 
1397     /**
1398      * Returns the HtmlElement of the specified WebElement.
1399      * @param webElement the webElement
1400      * @return the HtmlElement
1401      * @throws Exception if an error occurs
1402      */
1403     protected HtmlElement toHtmlElement(final WebElement webElement) throws Exception {
1404         return (HtmlElement) ((HtmlUnitWebElement) webElement).getElement();
1405     }
1406 
1407     /**
1408      * Loads an expectation file for the specified browser search first for a browser specific resource
1409      * and falling back in a general resource.
1410      * @param resourcePrefix the start of the resource name
1411      * @param resourceSuffix the end of the resource name
1412      * @return the content of the file
1413      * @throws Exception in case of error
1414      */
1415     protected String loadExpectation(final String resourcePrefix, final String resourceSuffix) throws Exception {
1416         final Class<?> referenceClass = getClass();
1417         final BrowserVersion browserVersion = getBrowserVersion();
1418 
1419         String realBrowserNyiExpectation = null;
1420         String realNyiExpectation = null;
1421         final String browserExpectation;
1422         final String expectation;
1423         if (!useRealBrowser()) {
1424             // first try nyi
1425             final String browserSpecificNyiResource
1426                     = resourcePrefix + "." + browserVersion.getNickname() + "_NYI" + resourceSuffix;
1427             realBrowserNyiExpectation = loadContent(referenceClass.getResource(browserSpecificNyiResource));
1428 
1429             // next nyi without browser
1430             final String nyiResource = resourcePrefix + ".NYI" + resourceSuffix;
1431             realNyiExpectation = loadContent(referenceClass.getResource(nyiResource));
1432         }
1433 
1434         // implemented - browser specific
1435         final String browserSpecificResource = resourcePrefix + "." + browserVersion.getNickname() + resourceSuffix;
1436         browserExpectation = loadContent(referenceClass.getResource(browserSpecificResource));
1437 
1438         // implemented - all browsers
1439         final String resource = resourcePrefix + resourceSuffix;
1440         expectation = loadContent(referenceClass.getResource(resource));
1441 
1442         // check for duplicates
1443         if (realBrowserNyiExpectation != null) {
1444             if (realNyiExpectation != null) {
1445                 Assertions.assertNotEquals(realBrowserNyiExpectation, realNyiExpectation,
1446                         "Duplicate NYI Expectation for Browser " + browserVersion.getNickname());
1447             }
1448 
1449             if (browserExpectation == null) {
1450                 if (expectation != null) {
1451                     Assertions.assertNotEquals(realBrowserNyiExpectation, expectation,
1452                             "NYI Expectation matches the expected "
1453                                     + "result for Browser " + browserVersion.getNickname());
1454                 }
1455             }
1456             else {
1457                 Assertions.assertNotEquals(realBrowserNyiExpectation, browserExpectation,
1458                         "NYI Expectation matches the expected "
1459                                 + "browser specific result for Browser "
1460                                 + browserVersion.getNickname());
1461             }
1462 
1463             return realBrowserNyiExpectation;
1464         }
1465 
1466         if (realNyiExpectation != null) {
1467             if (browserExpectation == null) {
1468                 if (expectation != null) {
1469                     Assertions.assertNotEquals(realNyiExpectation, expectation,
1470                             "NYI Expectation matches the expected "
1471                                     + "result for Browser " + browserVersion.getNickname());
1472                 }
1473             }
1474             else {
1475                 Assertions.assertNotEquals(realNyiExpectation, browserExpectation,
1476                             "NYI Expectation matches the expected "
1477                                     + "browser specific result for Browser "
1478                                     + browserVersion.getNickname());
1479             }
1480             return realNyiExpectation;
1481         }
1482 
1483         if (browserExpectation != null) {
1484             if (expectation != null) {
1485                 Assertions.assertNotEquals(browserExpectation, expectation,
1486                             "Browser specific NYI Expectation matches the expected "
1487                                     + "result for Browser " + browserVersion.getNickname());
1488             }
1489             return browserExpectation;
1490         }
1491         return expectation;
1492     }
1493 
1494     private static String loadContent(final URL url) throws URISyntaxException, IOException {
1495         if (url == null) {
1496             return null;
1497         }
1498 
1499         final File file = new File(url.toURI());
1500         final String content = FileUtils.readFileToString(file, UTF_8)
1501                                         .replace("\r\n", "\n");
1502         return content;
1503     }
1504 
1505     /**
1506      * Reads the number of JS threads remaining from unit tests run before.
1507      * This should be always 0, if {@link #isWebClientCached()} is {@code false}.
1508      * @throws InterruptedException in case of problems
1509      */
1510     @BeforeEach
1511     public void beforeTest() throws InterruptedException {
1512         if (!isWebClientCached()) {
1513             if (!getJavaScriptThreads().isEmpty()) {
1514                 Thread.sleep(200);
1515             }
1516             assertTrue("There are already JS threads running before the test", getJavaScriptThreads().isEmpty());
1517         }
1518     }
1519 
1520     /**
1521      * Asserts the current title is equal to the expectation string.
1522      * @param webdriver the driver in use
1523      * @param expected the expected object
1524      * @throws Exception in case of failure
1525      */
1526     protected void assertTitle(final WebDriver webdriver, final String expected) throws Exception {
1527         final long maxWait = System.currentTimeMillis() + DEFAULT_WAIT_TIME.toMillis();
1528 
1529         while (true) {
1530             final String title = webdriver.getTitle();
1531             try {
1532                 assertEquals(expected, title);
1533                 return;
1534             }
1535             catch (final org.opentest4j.AssertionFailedError e) {
1536                 if (expected.length() <= title.length()
1537                         || System.currentTimeMillis() > maxWait) {
1538                     throw e;
1539                 }
1540                 Thread.sleep(10);
1541             }
1542         }
1543     }
1544 
1545     /**
1546      * Release resources but DON'T close the browser if we are running with a real browser.
1547      * Note that HtmlUnitDriver is not cached by default, but that can be configured by {@link #isWebClientCached()}.
1548      */
1549     @AfterEach
1550     @Override
1551     public void releaseResources() {
1552         final List<String> unhandledAlerts = new ArrayList<>();
1553         if (webDriver_ != null) {
1554             UnhandledAlertException ex = null;
1555             do {
1556                 ex = null;
1557                 try {
1558                     // getTitle will do an implicit check for open alerts
1559                     webDriver_.getTitle();
1560                 }
1561                 catch (final NoSuchWindowException e) {
1562                     // ignore
1563                 }
1564                 catch (final UnhandledAlertException e) {
1565                     ex = e;
1566                     unhandledAlerts.add(e.getMessage());
1567                 }
1568             }
1569             while (ex != null);
1570         }
1571 
1572         super.releaseResources();
1573 
1574         if (!isWebClientCached()) {
1575             boolean rhino = false;
1576             if (webDriver_ != null) {
1577                 try {
1578                     rhino = getWebClient().getJavaScriptEngine() instanceof JavaScriptEngine;
1579                 }
1580                 catch (final Exception e) {
1581                     throw new RuntimeException(e);
1582                 }
1583                 webDriver_.quit();
1584                 webDriver_ = null;
1585             }
1586             if (rhino) {
1587                 assertTrue("There are still JS threads running after the test", getJavaScriptThreads().isEmpty());
1588             }
1589         }
1590 
1591         if (useRealBrowser()) {
1592             synchronized (WEB_DRIVERS_REAL_BROWSERS) {
1593                 final WebDriver driver = WEB_DRIVERS_REAL_BROWSERS.get(getBrowserVersion());
1594                 if (driver != null) {
1595                     try {
1596                         final String currentWindow = driver.getWindowHandle();
1597 
1598                         final Set<String> handles = driver.getWindowHandles();
1599                         // close all windows except the current one
1600                         handles.remove(currentWindow);
1601 
1602                         if (!handles.isEmpty()) {
1603                             for (final String handle : handles) {
1604                                 try {
1605                                     driver.switchTo().window(handle);
1606                                     driver.close();
1607                                 }
1608                                 catch (final NoSuchWindowException e) {
1609                                     LOG.error("Error switching to browser window; quit browser.", e);
1610                                     WEB_DRIVERS_REAL_BROWSERS.remove(getBrowserVersion());
1611                                     WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.remove(getBrowserVersion());
1612                                     driver.quit();
1613                                     return;
1614                                 }
1615                             }
1616 
1617                             // we have to force WebDriver to treat the remaining window
1618                             // as the one we like to work with from now on
1619                             // looks like a web driver issue to me (version 2.47.2)
1620                             driver.switchTo().window(currentWindow);
1621                         }
1622 
1623                         driver.manage().deleteAllCookies();
1624 
1625                         // in the remaining window, load a blank page
1626                         driver.get("about:blank");
1627                     }
1628                     catch (final NoSuchSessionException e) {
1629                         LOG.error("Error browser session no longer available.", e);
1630                         WEB_DRIVERS_REAL_BROWSERS.remove(getBrowserVersion());
1631                         WEB_DRIVERS_REAL_BROWSERS_USAGE_COUNT.remove(getBrowserVersion());
1632                         return;
1633                     }
1634                     catch (final WebDriverException e) {
1635                         shutDownRealBrowsers();
1636                     }
1637                 }
1638             }
1639         }
1640         assertTrue("There are still unhandled alerts: " + String.join("; ", unhandledAlerts),
1641                         unhandledAlerts.isEmpty());
1642     }
1643 
1644     /**
1645      * Returns the underlying WebWindow of the specified driver.
1646      *
1647      * <b>Your test shouldn't depend primarily on WebClient</b>
1648      *
1649      * @return the current web window
1650      * @see #toHtmlElement(WebElement)
1651      */
1652     protected WebWindow getWebWindow() {
1653         return webDriver_.getCurrentWindow().getWebWindow();
1654     }
1655 
1656     /**
1657      * Whether {@link WebClient} is cached or not, defaults to {@code false}.
1658      *
1659      * <p>This is needed to be {@code true} for huge test class, as we could run out of sockets.
1660      *
1661      * @return whether {@link WebClient} is cached or not
1662      */
1663     protected boolean isWebClientCached() {
1664         return false;
1665     }
1666 
1667     /**
1668      * Configure {@link WebClientOptions#getTimeout()}.
1669      *
1670      * @return null if unchanged otherwise the timeout as int
1671      */
1672     protected Integer getWebClientTimeout() {
1673         return null;
1674     }
1675 
1676     protected Page getEnclosedPage() {
1677         return getWebWindow().getEnclosedPage();
1678     }
1679 
1680     protected WebClient getWebClient() {
1681         return webDriver_.getWebClient();
1682     }
1683 }