View Javadoc
1   /*
2    * Copyright (c) 2002-2026 Gargoyle Software Inc.
3    *
4    * Licensed under the Apache License, Version 2.0 (the "License");
5    * you may not use this file except in compliance with the License.
6    * You may obtain a copy of the License at
7    * https://www.apache.org/licenses/LICENSE-2.0
8    *
9    * Unless required by applicable law or agreed to in writing, software
10   * distributed under the License is distributed on an "AS IS" BASIS,
11   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12   * See the License for the specific language governing permissions and
13   * limitations under the License.
14   */
15  package org.htmlunit.html;
16  
17  import static org.htmlunit.BrowserVersionFeatures.EVENT_FOCUS_ON_LOAD;
18  import static org.htmlunit.BrowserVersionFeatures.HTTP_HEADER_CH_UA;
19  import static org.htmlunit.html.DomElement.ATTRIBUTE_NOT_DEFINED;
20  
21  import java.io.File;
22  import java.io.IOException;
23  import java.io.ObjectInputStream;
24  import java.io.ObjectOutputStream;
25  import java.io.Serializable;
26  import java.net.MalformedURLException;
27  import java.net.URL;
28  import java.nio.charset.Charset;
29  import java.nio.charset.StandardCharsets;
30  import java.util.ArrayList;
31  import java.util.Arrays;
32  import java.util.Collection;
33  import java.util.Collections;
34  import java.util.Comparator;
35  import java.util.HashMap;
36  import java.util.HashSet;
37  import java.util.Iterator;
38  import java.util.LinkedHashSet;
39  import java.util.List;
40  import java.util.Locale;
41  import java.util.Map;
42  import java.util.Set;
43  import java.util.WeakHashMap;
44  import java.util.concurrent.ConcurrentHashMap;
45  
46  import org.apache.commons.lang3.StringUtils;
47  import org.apache.commons.logging.Log;
48  import org.apache.commons.logging.LogFactory;
49  import org.htmlunit.Cache;
50  import org.htmlunit.ElementNotFoundException;
51  import org.htmlunit.FailingHttpStatusCodeException;
52  import org.htmlunit.History;
53  import org.htmlunit.HttpHeader;
54  import org.htmlunit.OnbeforeunloadHandler;
55  import org.htmlunit.Page;
56  import org.htmlunit.ScriptResult;
57  import org.htmlunit.SgmlPage;
58  import org.htmlunit.TopLevelWindow;
59  import org.htmlunit.WebAssert;
60  import org.htmlunit.WebClient;
61  import org.htmlunit.WebClientOptions;
62  import org.htmlunit.WebRequest;
63  import org.htmlunit.WebResponse;
64  import org.htmlunit.WebWindow;
65  import org.htmlunit.corejs.javascript.Function;
66  import org.htmlunit.corejs.javascript.Script;
67  import org.htmlunit.corejs.javascript.Scriptable;
68  import org.htmlunit.corejs.javascript.ScriptableObject;
69  import org.htmlunit.corejs.javascript.VarScope;
70  import org.htmlunit.css.ComputedCssStyleDeclaration;
71  import org.htmlunit.css.CssStyleSheet;
72  import org.htmlunit.html.impl.SimpleRange;
73  import org.htmlunit.html.parser.HTMLParserDOMBuilder;
74  import org.htmlunit.http.HttpStatus;
75  import org.htmlunit.javascript.AbstractJavaScriptEngine;
76  import org.htmlunit.javascript.HtmlUnitScriptable;
77  import org.htmlunit.javascript.JavaScriptEngine;
78  import org.htmlunit.javascript.PostponedAction;
79  import org.htmlunit.javascript.host.Window;
80  import org.htmlunit.javascript.host.event.BeforeUnloadEvent;
81  import org.htmlunit.javascript.host.event.Event;
82  import org.htmlunit.javascript.host.event.EventTarget;
83  import org.htmlunit.javascript.host.html.HTMLDocument;
84  import org.htmlunit.protocol.javascript.JavaScriptURLConnection;
85  import org.htmlunit.util.MimeType;
86  import org.htmlunit.util.SerializableLock;
87  import org.htmlunit.util.UrlUtils;
88  import org.w3c.dom.Attr;
89  import org.w3c.dom.Comment;
90  import org.w3c.dom.DOMConfiguration;
91  import org.w3c.dom.DOMException;
92  import org.w3c.dom.DOMImplementation;
93  import org.w3c.dom.Document;
94  import org.w3c.dom.DocumentType;
95  import org.w3c.dom.Element;
96  import org.w3c.dom.EntityReference;
97  import org.w3c.dom.ProcessingInstruction;
98  
99  /**
100  * A representation of an HTML page returned from a server.
101  * <p>
102  * This class provides different methods to access the page's content like
103  * {@link #getForms()}, {@link #getAnchors()}, {@link #getElementById(String)}, ... as well as the
104  * very powerful inherited methods {@link #getByXPath(String)} and {@link #getFirstByXPath(String)}
105  * for fine grained user specific access to child nodes.
106  * </p>
107  * <p>
108  * Child elements allowing user interaction provide methods for this purpose like {@link HtmlAnchor#click()},
109  * {@link HtmlInput#type(String)}, {@link HtmlOption#setSelected(boolean)}, ...
110  * </p>
111  * <p>
112  * HtmlPage instances should not be instantiated directly. They will be returned by {@link WebClient#getPage(String)}
113  * when the content type of the server's response is <code>text/html</code> (or one of its variations).<br>
114  * <br>
115  * <b>Example:</b><br>
116  * <br>
117  * <code>
118  * final HtmlPage page = webClient.{@link WebClient#getPage(String) getPage}("http://mywebsite/some/page.html");
119  * </code>
120  * </p>
121  *
122  * @author Mike Bowler
123  * @author Alex Nikiforoff
124  * @author Noboru Sinohara
125  * @author David K. Taylor
126  * @author Andreas Hangler
127  * @author Christian Sell
128  * @author Chris Erskine
129  * @author Marc Guillemot
130  * @author Ahmed Ashour
131  * @author Daniel Gredler
132  * @author Dmitri Zoubkov
133  * @author Sudhan Moghe
134  * @author Ethan Glasser-Camp
135  * @author Tom Anderson
136  * @author Ronald Brill
137  * @author Frank Danek
138  * @author Joerg Werner
139  * @author Atsushi Nakagawa
140  * @author Rural Hunter
141  * @author Ronny Shapiro
142  * @author Lai Quang Duong
143  * @author Sven Strickroth
144  */
145 @SuppressWarnings("PMD.TooManyFields")
146 public class HtmlPage extends SgmlPage {
147 
148     private static final Log LOG = LogFactory.getLog(HtmlPage.class);
149 
150     private static final Comparator<DomElement> DOCUMENT_POSITION_COMPERATOR = new DocumentPositionComparator();
151 
152     private HTMLParserDOMBuilder domBuilder_;
153     private transient Charset originalCharset_;
154     private final Object lock_ = new SerializableLock(); // used for synchronization
155 
156     private Map<String, MappedElementIndexEntry> idMap_ = new ConcurrentHashMap<>();
157     private Map<String, MappedElementIndexEntry> nameMap_ = new ConcurrentHashMap<>();
158     // The id/name lookup index is built lazily on first use. Until then,
159     // notifyNodeAdded / fireAttributeChange skip the per-element index updates.
160     // Reads must call ensureMappedElementsBuilt() before consulting idMap_/nameMap_.
161     private boolean mappedElementsBuilt_;
162 
163     private List<BaseFrameElement> frameElements_ = new ArrayList<>();
164     private int parserCount_;
165     private int snippetParserCount_;
166     private int inlineSnippetParserCount_;
167     private Collection<HtmlAttributeChangeListener> attributeListeners_;
168     private List<PostponedAction> afterLoadActions_ = Collections.synchronizedList(new ArrayList<>());
169     private boolean cleaning_;
170     private HtmlBase base_;
171     private URL baseUrl_;
172     private List<AutoCloseable> autoCloseableList_;
173     private ElementFromPointHandler elementFromPointHandler_;
174     private DomElement elementWithFocus_;
175     private List<SimpleRange> selectionRanges_ = new ArrayList<>(3);
176 
177     private transient ComputedStylesCache computedStylesCache_;
178 
179     private static final HashSet<String> TABBABLE_TAGS =
180             new HashSet<>(Arrays.asList(HtmlAnchor.TAG_NAME, HtmlArea.TAG_NAME,
181                     HtmlButton.TAG_NAME, HtmlInput.TAG_NAME, HtmlObject.TAG_NAME,
182                     HtmlSelect.TAG_NAME, HtmlTextArea.TAG_NAME));
183     private static final HashSet<String> ACCEPTABLE_TAG_NAMES =
184             new HashSet<>(Arrays.asList(HtmlAnchor.TAG_NAME, HtmlArea.TAG_NAME,
185                     HtmlButton.TAG_NAME, HtmlInput.TAG_NAME, HtmlLabel.TAG_NAME,
186                     HtmlLegend.TAG_NAME, HtmlTextArea.TAG_NAME));
187 
188     /** Definition of special cases for the smart DomHtmlAttributeChangeListenerImpl. */
189     private static final Set<String> ATTRIBUTES_AFFECTING_PARENT = new HashSet<>(Arrays.asList(
190             "style",
191             "class",
192             "height",
193             "width"));
194 
195     static class DocumentPositionComparator implements Comparator<DomElement>, Serializable {
196         @Override
197         public int compare(final DomElement elt1, final DomElement elt2) {
198             final short relation = elt1.compareDocumentPosition(elt2);
199             if (relation == 0) {
200                 return 0; // same node
201             }
202             if ((relation & DOCUMENT_POSITION_CONTAINS) != 0 || (relation & DOCUMENT_POSITION_PRECEDING) != 0) {
203                 return 1;
204             }
205 
206             return -1;
207         }
208     }
209 
210     /**
211      * Creates an instance of HtmlPage.
212      * An HtmlPage instance is normally retrieved with {@link WebClient#getPage(String)}.
213      *
214      * @param webResponse the web response that was used to create this page
215      * @param webWindow the window that this page is being loaded into
216      */
217     public HtmlPage(final WebResponse webResponse, final WebWindow webWindow) {
218         super(webResponse, webWindow);
219     }
220 
221     /**
222      * {@inheritDoc}
223      */
224     @Override
225     public HtmlPage getPage() {
226         return this;
227     }
228 
229     /**
230      * {@inheritDoc}
231      */
232     @Override
233     public boolean hasCaseSensitiveTagNames() {
234         return false;
235     }
236 
237     /**
238      * Initialize this page.
239      * @throws IOException if an IO problem occurs
240      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
241      *         {@link org.htmlunit.WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set
242      *         to true.
243      */
244     @Override
245     public void initialize() throws IOException, FailingHttpStatusCodeException {
246         final WebWindow enclosingWindow = getEnclosingWindow();
247         final boolean isAboutBlank = getUrl() == UrlUtils.URL_ABOUT_BLANK;
248         if (isAboutBlank) {
249             // a frame contains first a faked "about:blank" before its real content specified by src gets loaded
250             if (enclosingWindow instanceof FrameWindow window
251                     && !window.getFrameElement().isContentLoaded()) {
252                 return;
253             }
254 
255             // save the URL that should be used to resolve relative URLs in this page
256             if (enclosingWindow instanceof TopLevelWindow topWindow) {
257                 final WebWindow openerWindow = topWindow.getOpener();
258                 if (openerWindow != null && openerWindow.getEnclosedPage() != null) {
259                     baseUrl_ = openerWindow.getEnclosedPage().getWebResponse().getWebRequest().getUrl();
260                 }
261             }
262         }
263 
264         if (!isAboutBlank) {
265             setReadyState(READY_STATE_INTERACTIVE);
266             getDocumentElement().setReadyState(READY_STATE_INTERACTIVE);
267             executeEventHandlersIfNeeded(Event.TYPE_READY_STATE_CHANGE);
268         }
269 
270         executeDeferredScriptsIfNeeded();
271 
272         executeEventHandlersIfNeeded(Event.TYPE_DOM_DOCUMENT_LOADED);
273 
274         // postponed actions are more or less the async scripts,
275         // they are running in real browsers whenever the download is done
276         processPostponedActionsIfNeeded();
277 
278         loadFrames();
279 
280         // don't set the ready state if we really load the blank page into the window
281         // see Node.initInlineFrameIfNeeded()
282         if (!isAboutBlank) {
283             setReadyState(READY_STATE_COMPLETE);
284             getDocumentElement().setReadyState(READY_STATE_COMPLETE);
285             executeEventHandlersIfNeeded(Event.TYPE_READY_STATE_CHANGE);
286         }
287 
288         // frame initialization has a different order
289         boolean isFrameWindow = enclosingWindow instanceof FrameWindow;
290         boolean isFirstPageInFrameWindow = false;
291         if (isFrameWindow) {
292             isFrameWindow = ((FrameWindow) enclosingWindow).getFrameElement() instanceof HtmlFrame;
293 
294             final History hist = enclosingWindow.getHistory();
295             if (hist.getLength() > 0 && UrlUtils.URL_ABOUT_BLANK == hist.getUrl(0)) {
296                 isFirstPageInFrameWindow = hist.getLength() <= 2;
297             }
298             else {
299                 isFirstPageInFrameWindow = enclosingWindow.getHistory().getLength() < 2;
300             }
301         }
302 
303         if (isFrameWindow && !isFirstPageInFrameWindow) {
304             executeEventHandlersIfNeeded(Event.TYPE_LOAD);
305         }
306 
307         for (final BaseFrameElement frameElement : new ArrayList<>(frameElements_)) {
308             if (frameElement instanceof HtmlFrame) {
309                 final Page page = frameElement.getEnclosedWindow().getEnclosedPage();
310                 if (page != null && page.isHtmlPage()) {
311                     ((HtmlPage) page).executeEventHandlersIfNeeded(Event.TYPE_LOAD);
312                 }
313             }
314         }
315 
316         if (!isFrameWindow) {
317             executeEventHandlersIfNeeded(Event.TYPE_LOAD);
318 
319             if (!isAboutBlank && enclosingWindow.getWebClient().isJavaScriptEnabled()
320                     && hasFeature(EVENT_FOCUS_ON_LOAD)) {
321                 final HtmlElement body = getBody();
322                 if (body != null) {
323                     final Event event = new Event((Window) enclosingWindow.getScriptableObject(), Event.TYPE_FOCUS);
324                     body.fireEvent(event);
325                 }
326             }
327         }
328 
329         try {
330             while (!afterLoadActions_.isEmpty()) {
331                 final PostponedAction action = afterLoadActions_.remove(0);
332                 action.execute();
333             }
334         }
335         catch (final IOException e) {
336             throw e;
337         }
338         catch (final Exception e) {
339             throw new RuntimeException(e);
340         }
341         executeRefreshIfNeeded();
342     }
343 
344     /**
345      * Adds an action that should be executed once the page has been loaded.
346      * @param action the action
347      */
348     void addAfterLoadAction(final PostponedAction action) {
349         afterLoadActions_.add(action);
350     }
351 
352     /**
353      * Clean up this page.
354      */
355     @Override
356     public void cleanUp() {
357         //To avoid endless recursion caused by window.close() in onUnload
358         if (cleaning_) {
359             return;
360         }
361 
362         cleaning_ = true;
363         try {
364             super.cleanUp();
365             executeEventHandlersIfNeeded(Event.TYPE_UNLOAD);
366             deregisterFramesIfNeeded();
367         }
368         finally {
369             cleaning_ = false;
370 
371             if (autoCloseableList_ != null) {
372                 for (final AutoCloseable closeable : new ArrayList<>(autoCloseableList_)) {
373                     try {
374                         closeable.close();
375                     }
376                     catch (final Exception e) {
377                         LOG.error("Closing the autoclosable " + closeable + " failed", e);
378                     }
379                 }
380             }
381         }
382     }
383 
384     /**
385      * {@inheritDoc}
386      */
387     @Override
388     public HtmlElement getDocumentElement() {
389         return (HtmlElement) super.getDocumentElement();
390     }
391 
392     /**
393      * Returns the document's {@code body} element.
394      *
395      * @return the document's {@code body} element, or {@code null} if it does
396      *         not exist
397      */
398     public HtmlBody getBody() {
399         final DomElement doc = getDocumentElement();
400         if (doc != null) {
401             for (final DomNode node : doc.getChildren()) {
402                 if (node instanceof HtmlBody body) {
403                     return body;
404                 }
405             }
406         }
407         return null;
408     }
409 
410     /**
411      * Returns the head element.
412      * @return the head element
413      */
414     public HtmlElement getHead() {
415         final DomElement doc = getDocumentElement();
416         if (doc != null) {
417             for (final DomNode node : doc.getChildren()) {
418                 if (node instanceof HtmlHead) {
419                     return (HtmlElement) node;
420                 }
421             }
422         }
423         return null;
424     }
425 
426     /**
427      * {@inheritDoc}
428      */
429     @Override
430     public Document getOwnerDocument() {
431         return null;
432     }
433 
434     /**
435      * {@inheritDoc}
436      * Not yet implemented.
437      */
438     @Override
439     public org.w3c.dom.Node importNode(final org.w3c.dom.Node importedNode, final boolean deep) {
440         throw new UnsupportedOperationException("HtmlPage.importNode is not yet implemented.");
441     }
442 
443     /**
444      * {@inheritDoc}
445      * Not yet implemented.
446      */
447     @Override
448     public String getInputEncoding() {
449         throw new UnsupportedOperationException("HtmlPage.getInputEncoding is not yet implemented.");
450     }
451 
452     /**
453      * {@inheritDoc}
454      */
455     @Override
456     public String getXmlEncoding() {
457         return null;
458     }
459 
460     /**
461      * {@inheritDoc}
462      */
463     @Override
464     public boolean getXmlStandalone() {
465         return false;
466     }
467 
468     /**
469      * {@inheritDoc}
470      * Not yet implemented.
471      */
472     @Override
473     public void setXmlStandalone(final boolean xmlStandalone) throws DOMException {
474         throw new UnsupportedOperationException("HtmlPage.setXmlStandalone is not yet implemented.");
475     }
476 
477     /**
478      * {@inheritDoc}
479      */
480     @Override
481     public String getXmlVersion() {
482         return null;
483     }
484 
485     /**
486      * {@inheritDoc}
487      * Not yet implemented.
488      */
489     @Override
490     public void setXmlVersion(final String xmlVersion) throws DOMException {
491         throw new UnsupportedOperationException("HtmlPage.setXmlVersion is not yet implemented.");
492     }
493 
494     /**
495      * {@inheritDoc}
496      * Not yet implemented.
497      */
498     @Override
499     public boolean getStrictErrorChecking() {
500         throw new UnsupportedOperationException("HtmlPage.getStrictErrorChecking is not yet implemented.");
501     }
502 
503     /**
504      * {@inheritDoc}
505      * Not yet implemented.
506      */
507     @Override
508     public void setStrictErrorChecking(final boolean strictErrorChecking) {
509         throw new UnsupportedOperationException("HtmlPage.setStrictErrorChecking is not yet implemented.");
510     }
511 
512     /**
513      * {@inheritDoc}
514      * Not yet implemented.
515      */
516     @Override
517     public String getDocumentURI() {
518         throw new UnsupportedOperationException("HtmlPage.getDocumentURI is not yet implemented.");
519     }
520 
521     /**
522      * {@inheritDoc}
523      * Not yet implemented.
524      */
525     @Override
526     public void setDocumentURI(final String documentURI) {
527         throw new UnsupportedOperationException("HtmlPage.setDocumentURI is not yet implemented.");
528     }
529 
530     /**
531      * {@inheritDoc}
532      * Not yet implemented.
533      */
534     @Override
535     public org.w3c.dom.Node adoptNode(final org.w3c.dom.Node source) throws DOMException {
536         throw new UnsupportedOperationException("HtmlPage.adoptNode is not yet implemented.");
537     }
538 
539     /**
540      * {@inheritDoc}
541      * Not yet implemented.
542      */
543     @Override
544     public DOMConfiguration getDomConfig() {
545         throw new UnsupportedOperationException("HtmlPage.getDomConfig is not yet implemented.");
546     }
547 
548     /**
549      * {@inheritDoc}
550      * Not yet implemented.
551      */
552     @Override
553     public org.w3c.dom.Node renameNode(final org.w3c.dom.Node newNode, final String namespaceURI,
554         final String qualifiedName) throws DOMException {
555         throw new UnsupportedOperationException("HtmlPage.renameNode is not yet implemented.");
556     }
557 
558     /**
559      * {@inheritDoc}
560      */
561     @Override
562     public Charset getCharset() {
563         if (originalCharset_ == null) {
564             originalCharset_ = getWebResponse().getContentCharset();
565         }
566         return originalCharset_;
567     }
568 
569     /**
570      * {@inheritDoc}
571      */
572     @Override
573     public String getContentType() {
574         return getWebResponse().getContentType();
575     }
576 
577     /**
578      * {@inheritDoc}
579      * Not yet implemented.
580      */
581     @Override
582     public DOMImplementation getImplementation() {
583         throw new UnsupportedOperationException("HtmlPage.getImplementation is not yet implemented.");
584     }
585 
586     /**
587      * {@inheritDoc}
588      * @param tagName the tag name, preferably in lowercase
589      */
590     @Override
591     public DomElement createElement(String tagName) {
592         if (tagName.indexOf(':') == -1) {
593             tagName = org.htmlunit.util.StringUtils.toRootLowerCase(tagName);
594         }
595         return getWebClient().getPageCreator().getHtmlParser().getFactory(tagName)
596                     .createElementNS(this, null, tagName, null);
597     }
598 
599     /**
600      * {@inheritDoc}
601      */
602     @Override
603     public DomElement createElementNS(final String namespaceURI, final String qualifiedName) {
604         return getWebClient().getPageCreator().getHtmlParser()
605                 .getElementFactory(this, namespaceURI, qualifiedName, false, true)
606                 .createElementNS(this, namespaceURI, qualifiedName, null);
607     }
608 
609     /**
610      * {@inheritDoc}
611      * Not yet implemented.
612      */
613     @Override
614     public Attr createAttributeNS(final String namespaceURI, final String qualifiedName) {
615         throw new UnsupportedOperationException("HtmlPage.createAttributeNS is not yet implemented.");
616     }
617 
618     /**
619      * {@inheritDoc}
620      * Not yet implemented.
621      */
622     @Override
623     public EntityReference createEntityReference(final String id) {
624         throw new UnsupportedOperationException("HtmlPage.createEntityReference is not yet implemented.");
625     }
626 
627     /**
628      * {@inheritDoc}
629      * Not yet implemented.
630      */
631     @Override
632     public ProcessingInstruction createProcessingInstruction(final String namespaceURI, final String qualifiedName) {
633         throw new UnsupportedOperationException("HtmlPage.createProcessingInstruction is not yet implemented.");
634     }
635 
636     /**
637      * {@inheritDoc}
638      */
639     @Override
640     public DomElement getElementById(final String elementId) {
641         if (elementId != null) {
642             ensureMappedElementsBuilt();
643             final MappedElementIndexEntry elements = idMap_.get(elementId);
644             if (elements != null) {
645                 return elements.first();
646             }
647         }
648         return null;
649     }
650 
651     /**
652      * Returns the {@link HtmlAnchor} with the specified name.
653      *
654      * @param name the name to search by
655      * @return the {@link HtmlAnchor} with the specified name
656      * @throws ElementNotFoundException if the anchor could not be found
657      */
658     public HtmlAnchor getAnchorByName(final String name) throws ElementNotFoundException {
659         return getDocumentElement().getOneHtmlElementByAttribute("a", DomElement.NAME_ATTRIBUTE, name);
660     }
661 
662     /**
663      * Returns the {@link HtmlAnchor} with the specified href.
664      *
665      * @param href the string to search by
666      * @return the HtmlAnchor
667      * @throws ElementNotFoundException if the anchor could not be found
668      */
669     public HtmlAnchor getAnchorByHref(final String href) throws ElementNotFoundException {
670         return getDocumentElement().getOneHtmlElementByAttribute("a", "href", href);
671     }
672 
673     /**
674      * Returns a list of all anchors contained in this page.
675      * @return the list of {@link HtmlAnchor} in this page
676      */
677     public List<HtmlAnchor> getAnchors() {
678         return getDocumentElement().getElementsByTagNameImpl("a");
679     }
680 
681     /**
682      * Returns the first anchor with the specified text.
683      * @param text the text to search for
684      * @return the first anchor that was found
685      * @throws ElementNotFoundException if no anchors are found with the specified text
686      */
687     public HtmlAnchor getAnchorByText(final String text) throws ElementNotFoundException {
688         WebAssert.notNull("text", text);
689 
690         for (final HtmlAnchor anchor : getAnchors()) {
691             if (text.equals(anchor.asNormalizedText())) {
692                 return anchor;
693             }
694         }
695         throw new ElementNotFoundException("a", "<text>", text);
696     }
697 
698     /**
699      * Returns the first form that matches the specified name.
700      * @param name the name to search for
701      * @return the first form
702      * @throws ElementNotFoundException If no forms match the specified result.
703      */
704     public HtmlForm getFormByName(final String name) throws ElementNotFoundException {
705         final List<HtmlForm> forms = getDocumentElement()
706                 .getElementsByAttribute("form", DomElement.NAME_ATTRIBUTE, name);
707         if (forms.isEmpty()) {
708             throw new ElementNotFoundException("form", DomElement.NAME_ATTRIBUTE, name);
709         }
710         return forms.get(0);
711     }
712 
713     /**
714      * Returns a list of all the forms in this page.
715      * @return all the forms in this page
716      */
717     public List<HtmlForm> getForms() {
718         return getDocumentElement().getElementsByTagNameImpl("form");
719     }
720 
721     /**
722      * Given a relative URL (ie <code>/foo</code>), returns a fully-qualified URL based on
723      * the URL that was used to load this page.
724      *
725      * @param relativeUrl the relative URL
726      * @return the fully-qualified URL for the specified relative URL
727      * @throws MalformedURLException if an error occurred when creating a URL object
728      */
729     public URL getFullyQualifiedUrl(String relativeUrl) throws MalformedURLException {
730         // to handle http: and http:/ in FF (Bug #474)
731         boolean incorrectnessNotified = false;
732         while (relativeUrl.startsWith("http:") && !relativeUrl.startsWith("http://")) {
733             if (!incorrectnessNotified) {
734                 notifyIncorrectness("Incorrect URL \"" + relativeUrl + "\" has been corrected");
735                 incorrectnessNotified = true;
736             }
737             relativeUrl = "http:/" + relativeUrl.substring(5);
738         }
739 
740         return WebClient.expandUrl(getBaseURL(), relativeUrl);
741     }
742 
743     /**
744      * Given a target attribute value, resolve the target using a base target for the page.
745      *
746      * @param elementTarget the target specified as an attribute of the element
747      * @return the resolved target to use for the element
748      */
749     public String getResolvedTarget(final String elementTarget) {
750         final String resolvedTarget;
751         if (base_ == null) {
752             resolvedTarget = elementTarget;
753         }
754         else if (elementTarget != null && !elementTarget.isEmpty()) {
755             resolvedTarget = elementTarget;
756         }
757         else {
758             resolvedTarget = base_.getTargetAttribute();
759         }
760         return resolvedTarget;
761     }
762 
763     /**
764      * Returns a list of ids (strings) that correspond to the tabbable elements
765      * in this page. Return them in the same order specified in {@link #getTabbableElements}
766      *
767      * @return the list of id's
768      */
769     public List<String> getTabbableElementIds() {
770         final List<String> list = new ArrayList<>();
771 
772         for (final HtmlElement element : getTabbableElements()) {
773             list.add(element.getId());
774         }
775 
776         return Collections.unmodifiableList(list);
777     }
778 
779     /**
780      * Returns a list of all elements that are tabbable in the order that will
781      * be used for tabbing.
782      * <p>
783      * The rules for determining tab order are as follows:
784      * </p>
785      * <ol>
786      *   <li>Those elements that support the tabindex attribute and assign a
787      *   positive value to it are navigated first. Navigation proceeds from the
788      *   element with the lowest tabindex value to the element with the highest
789      *   value. Values need not be sequential nor must they begin with any
790      *   particular value. Elements that have identical tabindex values should
791      *   be navigated in the order they appear in the character stream.</li>
792      *   <li>Those elements that do not support the tabindex attribute or
793      *   support it and assign it a value of "0" are navigated next. These
794      *   elements are navigated in the order they appear in the character
795      *   stream.</li>
796      *   <li>Elements that are disabled do not participate in the tabbing
797      *   order.</li>
798      * </ol>
799      * <p>
800      * Additionally, the value of tabindex must be within 0 and 32767. Any
801      * values outside this range will be ignored.
802      * </p>
803      * <p>
804      * The following elements support the <code>tabindex</code> attribute:
805      * A, AREA, BUTTON, INPUT, OBJECT, SELECT, and TEXTAREA.
806      * </p>
807      *
808      * @return all the tabbable elements in proper tab order
809      */
810     public List<HtmlElement> getTabbableElements() {
811         final List<HtmlElement> tabbableElements = new ArrayList<>();
812         for (final HtmlElement element : getHtmlElementDescendants()) {
813             final String tagName = element.getTagName();
814             if (TABBABLE_TAGS.contains(tagName)) {
815                 final boolean disabled = element.isDisabledElementAndDisabled();
816                 if (!disabled && !HtmlElement.TAB_INDEX_OUT_OF_BOUNDS.equals(element.getTabIndex())) {
817                     tabbableElements.add(element);
818                 }
819             }
820         }
821         tabbableElements.sort(createTabOrderComparator());
822         return Collections.unmodifiableList(tabbableElements);
823     }
824 
825     private static Comparator<HtmlElement> createTabOrderComparator() {
826         return (element1, element2) -> {
827             final Short i1 = element1.getTabIndex();
828             final Short i2 = element2.getTabIndex();
829 
830             final short index1;
831             if (i1 == null) {
832                 index1 = -1;
833             }
834             else {
835                 index1 = i1.shortValue();
836             }
837 
838             final short index2;
839             if (i2 == null) {
840                 index2 = -1;
841             }
842             else {
843                 index2 = i2.shortValue();
844             }
845 
846             final int result;
847             if (index1 > 0 && index2 > 0) {
848                 result = index1 - index2;
849             }
850             else if (index1 > 0) {
851                 result = -1;
852             }
853             else if (index2 > 0) {
854                 result = 1;
855             }
856             else if (index1 == index2) {
857                 result = 0;
858             }
859             else {
860                 result = index2 - index1;
861             }
862 
863             return result;
864         };
865     }
866 
867     /**
868      * Returns the HTML element that is assigned to the specified access key. An
869      * access key (aka mnemonic key) is used for keyboard navigation of the
870      * page.
871      * <p>
872      * Only the following HTML elements may have <code>accesskey</code>s defined: A, AREA,
873      * BUTTON, INPUT, LABEL, LEGEND, and TEXTAREA.
874      * </p>
875      *
876      * @param accessKey the key to look for
877      * @return the HTML element that is assigned to the specified key or null
878      *      if no elements can be found that match the specified key.
879      */
880     public HtmlElement getHtmlElementByAccessKey(final char accessKey) {
881         final List<HtmlElement> elements = getHtmlElementsByAccessKey(accessKey);
882         if (elements.isEmpty()) {
883             return null;
884         }
885         return elements.get(0);
886     }
887 
888     /**
889      * Returns all the HTML elements that are assigned to the specified access key. An
890      * access key (aka mnemonic key) is used for keyboard navigation of the
891      * page.
892      * <p>
893      * The HTML specification seems to indicate that one accesskey cannot be used
894      * for multiple elements however Internet Explorer does seem to support this.
895      * It's worth noting that Firefox does not support multiple elements with one
896      * access key so you are making your HTML browser specific if you rely on this
897      * feature.
898      * </p>
899      *
900      * <p>
901      * Only the following HTML elements may have <code>accesskey</code>s defined: A, AREA,
902      * BUTTON, INPUT, LABEL, LEGEND, and TEXTAREA.
903      * </p>
904      *
905      * @param accessKey the key to look for
906      * @return the elements that are assigned to the specified accesskey
907      */
908     public List<HtmlElement> getHtmlElementsByAccessKey(final char accessKey) {
909         final List<HtmlElement> elements = new ArrayList<>();
910 
911         final String searchString = Character.toString(accessKey).toLowerCase(Locale.ROOT);
912         for (final HtmlElement element : getHtmlElementDescendants()) {
913             if (ACCEPTABLE_TAG_NAMES.contains(element.getTagName())) {
914                 final String accessKeyAttribute = element.getAttributeDirect("accesskey");
915                 if (searchString.equalsIgnoreCase(accessKeyAttribute)) {
916                     elements.add(element);
917                 }
918             }
919         }
920 
921         return elements;
922     }
923 
924     /**
925      * <p>Executes the specified JavaScript code within the page. The usage would be similar to what can
926      * be achieved to execute JavaScript in the current page by entering "javascript:...some JS code..."
927      * in the URL field of a native browser.</p>
928      * <p><b>Note:</b> the provided code won't be executed if JavaScript has been disabled on the WebClient
929      * (see {@link org.htmlunit.WebClient#isJavaScriptEnabled()}).</p>
930      * @param sourceCode the JavaScript code to execute
931      * @return a ScriptResult which will contain both the current page (which may be different from
932      *         the previous page) and a JavaScript result object
933      */
934     public ScriptResult executeJavaScript(final String sourceCode) {
935         return executeJavaScript(sourceCode, "injected script", 1);
936     }
937 
938     /**
939      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
940      * <p>
941      * Execute the specified JavaScript if a JavaScript engine was successfully
942      * instantiated. If this JavaScript causes the current page to be reloaded
943      * (through location="" or form.submit()) then return the new page, otherwise
944      * return the current page.
945      * </p>
946      * <p><b>Please note:</b> Although this method is public, it is not intended for
947      * general execution of JavaScript. Users of HtmlUnit should interact with the pages
948      * as a user would by clicking on buttons or links and having the JavaScript event
949      * handlers execute as needed.
950      * </p>
951      *
952      * @param sourceCode the JavaScript code to execute
953      * @param sourceName the name for this chunk of code (will be displayed in error messages)
954      * @param startLine the line at which the script source starts
955      * @return a ScriptResult which will contain both the current page (which may be different from
956      *         the previous page) and a JavaScript result object.
957      */
958     public ScriptResult executeJavaScript(String sourceCode, final String sourceName, final int startLine) {
959         if (!getWebClient().isJavaScriptEnabled()) {
960             return new ScriptResult(JavaScriptEngine.UNDEFINED);
961         }
962 
963         if (org.htmlunit.util.StringUtils.startsWithIgnoreCase(sourceCode,
964                                                 JavaScriptURLConnection.JAVASCRIPT_PREFIX)) {
965             sourceCode = sourceCode.substring(JavaScriptURLConnection.JAVASCRIPT_PREFIX.length()).trim();
966             if (sourceCode.startsWith("return ")) {
967                 sourceCode = sourceCode.substring("return ".length());
968             }
969         }
970 
971         final Window window = getEnclosingWindow().getScriptableObject();
972         final VarScope scope = ScriptableObject.getTopLevelScope(window.getParentScope());
973 
974         final Object result = getWebClient().getJavaScriptEngine()
975                 .execute(this, scope, sourceCode, sourceName, startLine);
976         return new ScriptResult(result);
977     }
978 
979     /** Various possible external JavaScript file loading results. */
980     enum JavaScriptLoadResult {
981         /** The load was aborted and nothing was done. */
982         NOOP,
983         /** The load was aborted and nothing was done. */
984         NO_CONTENT,
985         /** The external JavaScript file was downloaded and compiled successfully. */
986         SUCCESS,
987         /** The external JavaScript file was not downloaded successfully. */
988         DOWNLOAD_ERROR,
989         /** The external JavaScript file was downloaded but was not compiled successfully. */
990         COMPILATION_ERROR
991     }
992 
993     /**
994      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
995      *
996      * @param srcAttribute the source attribute from the script tag
997      * @param scriptCharset the charset from the script tag
998      * @return the result of loading the specified external JavaScript file
999      * @throws FailingHttpStatusCodeException if the request's status code indicates a request
1000      *         failure and the {@link WebClient} was configured to throw exceptions on failing
1001      *         HTTP status codes
1002      */
1003     JavaScriptLoadResult loadExternalJavaScriptFile(final String srcAttribute,
1004                             final Charset scriptCharset, final boolean crossorigin)
1005         throws FailingHttpStatusCodeException {
1006 
1007         final WebClient client = getWebClient();
1008         if (org.htmlunit.util.StringUtils.isBlank(srcAttribute) || !client.isJavaScriptEnabled()) {
1009             return JavaScriptLoadResult.NOOP;
1010         }
1011 
1012         final URL scriptURL;
1013         try {
1014             scriptURL = getFullyQualifiedUrl(srcAttribute);
1015             final String protocol = scriptURL.getProtocol();
1016             if ("javascript".equals(protocol)) {
1017                 if (LOG.isInfoEnabled()) {
1018                     LOG.info("Ignoring script src [" + srcAttribute + "]");
1019                 }
1020                 return JavaScriptLoadResult.NOOP;
1021             }
1022             if (!"http".equals(protocol) && !"https".equals(protocol)
1023                     && !"data".equals(protocol) && !"file".equals(protocol)) {
1024                 client.getJavaScriptErrorListener().malformedScriptURL(this, srcAttribute,
1025                         new MalformedURLException("unknown protocol: '" + protocol + "'"));
1026                 return JavaScriptLoadResult.NOOP;
1027             }
1028         }
1029         catch (final MalformedURLException e) {
1030             client.getJavaScriptErrorListener().malformedScriptURL(this, srcAttribute, e);
1031             return JavaScriptLoadResult.NOOP;
1032         }
1033 
1034         final Object script;
1035         try {
1036             script = loadJavaScriptFromUrl(scriptURL, scriptCharset, crossorigin);
1037         }
1038         catch (final IOException e) {
1039             client.getJavaScriptErrorListener().loadScriptError(this, scriptURL, e);
1040             return JavaScriptLoadResult.DOWNLOAD_ERROR;
1041         }
1042         catch (final FailingHttpStatusCodeException e) {
1043             if (e.getStatusCode() == HttpStatus.NO_CONTENT_204) {
1044                 return JavaScriptLoadResult.NO_CONTENT;
1045             }
1046             client.getJavaScriptErrorListener().loadScriptError(this, scriptURL, e);
1047             throw e;
1048         }
1049 
1050         if (script == null) {
1051             return JavaScriptLoadResult.COMPILATION_ERROR;
1052         }
1053 
1054         final Window window = getEnclosingWindow().getScriptableObject();
1055         final VarScope scope = ScriptableObject.getTopLevelScope(window.getParentScope());
1056 
1057         @SuppressWarnings("unchecked")
1058         final AbstractJavaScriptEngine<Object> engine = (AbstractJavaScriptEngine<Object>) client.getJavaScriptEngine();
1059         engine.execute(this, scope, script);
1060         return JavaScriptLoadResult.SUCCESS;
1061     }
1062 
1063     /**
1064      * Loads JavaScript from the specified URL. This method may return {@code null} if
1065      * there is a problem loading the code from the specified URL.
1066      *
1067      * @param url the URL of the script
1068      * @param scriptCharset the charset from the script tag
1069      * @return the content of the file, or {@code null} if we ran into a compile error
1070      * @throws IOException if there is a problem downloading the JavaScript file
1071      * @throws FailingHttpStatusCodeException if the request's status code indicates a request
1072      *         failure and the {@link WebClient} was configured to throw exceptions on failing
1073      *         HTTP status codes
1074      */
1075     private Object loadJavaScriptFromUrl(final URL url, final Charset scriptCharset,
1076                     final boolean crossorigin) throws IOException,
1077         FailingHttpStatusCodeException {
1078 
1079         final WebRequest referringRequest = getWebResponse().getWebRequest();
1080 
1081         final WebClient client = getWebClient();
1082         final WebRequest request = new WebRequest(url);
1083         // copy all headers from the referring request
1084         request.setAdditionalHeaders(new HashMap<>(referringRequest.getAdditionalHeaders()));
1085 
1086         // at least overwrite this headers
1087         request.setAdditionalHeader(HttpHeader.ACCEPT, client.getBrowserVersion().getScriptAcceptHeader());
1088 
1089         request.setFetchDestination(WebRequest.FetchDestination.SCRIPT);
1090         request.setRequestingUrl(referringRequest.getUrl());
1091         request.setFetchModeOverride(WebRequest.FetchMode.NO_CORS);
1092 
1093         request.setRefererHeader(referringRequest.getUrl());
1094         request.setCharset(scriptCharset);
1095 
1096         // use info from script tag or fall back to utf-8
1097         // https://www.rfc-editor.org/rfc/rfc9239#section-4.2
1098         if (scriptCharset != null) {
1099             request.setDefaultResponseContentCharset(scriptCharset);
1100         }
1101         else {
1102             request.setDefaultResponseContentCharset(StandardCharsets.UTF_8);
1103         }
1104 
1105         if (crossorigin) {
1106             request.setFetchModeOverride(WebRequest.FetchMode.CORS);
1107 
1108             if (client.getBrowserVersion().hasFeature(HTTP_HEADER_CH_UA)) {
1109                 request.setAdditionalHeader(HttpHeader.ORIGIN,
1110                         UrlUtils.getUrlWithProtocolAndAuthority(url).toExternalForm());
1111             }
1112         }
1113 
1114         // our cache is a bit strange;
1115         // loadWebResponse check the cache for the web response
1116         // AND also fixes the request url for the following cache lookups
1117         final WebResponse response = client.loadWebResponse(request);
1118 
1119         // now we can look into the cache with the fixed request for
1120         // a cached script
1121         final Cache cache = client.getCache();
1122         final Object cachedScript = cache.getCachedObject(request);
1123         if (cachedScript instanceof Script) {
1124             return cachedScript;
1125         }
1126 
1127         client.printContentIfNecessary(response);
1128         client.throwFailingHttpStatusCodeExceptionIfNecessary(response);
1129 
1130         final int statusCode = response.getStatusCode();
1131         if (statusCode == HttpStatus.NO_CONTENT_204) {
1132             throw new FailingHttpStatusCodeException(response);
1133         }
1134 
1135         if (!response.isSuccess()) {
1136             throw new IOException("Unable to download JavaScript from '" + url + "' (status " + statusCode + ").");
1137         }
1138 
1139         final String contentType = response.getContentType();
1140         if (contentType != null) {
1141             if (MimeType.isObsoleteJavascriptMimeType(contentType)) {
1142                 getWebClient().getIncorrectnessListener().notify(
1143                         "Obsolete content type encountered: '" + contentType + "' "
1144                                 + "for remotely loaded JavaScript element at '" + url + "'.", this);
1145             }
1146             else if (!MimeType.isJavascriptMimeType(contentType)) {
1147                 getWebClient().getIncorrectnessListener().notify(
1148                         "Expect content type of '" + MimeType.TEXT_JAVASCRIPT + "' "
1149                                 + "for remotely loaded JavaScript element at '" + url + "', "
1150                                 + "but got '" + contentType + "'.", this);
1151             }
1152         }
1153 
1154         final Charset scriptEncoding = response.getContentCharset();
1155         final String scriptCode = response.getContentAsString(scriptEncoding);
1156         if (null != scriptCode) {
1157             final AbstractJavaScriptEngine<?> javaScriptEngine = client.getJavaScriptEngine();
1158 
1159             final Window window = getEnclosingWindow().getScriptableObject();
1160             final VarScope scope = ScriptableObject.getTopLevelScope(window.getParentScope());
1161 
1162             final Object script = javaScriptEngine.compile(this, scope, scriptCode, url.toExternalForm(), 1);
1163             if (script != null && cache.cacheIfPossible(request, response, script)) {
1164                 // no cleanup if the response is stored inside the cache
1165                 return script;
1166             }
1167 
1168             response.cleanUp();
1169             return script;
1170         }
1171 
1172         response.cleanUp();
1173         return null;
1174     }
1175 
1176     /**
1177      * Returns the title of this page or an empty string if the title wasn't specified.
1178      *
1179      * @return the title of this page or an empty string if the title wasn't specified
1180      */
1181     public String getTitleText() {
1182         final HtmlTitle titleElement = getTitleElement();
1183         if (titleElement != null) {
1184             return titleElement.asNormalizedText();
1185         }
1186         return "";
1187     }
1188 
1189     /**
1190      * Sets the text for the title of this page. If there is not a title element
1191      * on this page, then one has to be generated.
1192      * @param message the new text
1193      */
1194     public void setTitleText(final String message) {
1195         HtmlTitle titleElement = getTitleElement();
1196         if (titleElement == null) {
1197             LOG.debug("No title element, creating one");
1198             final HtmlHead head = (HtmlHead) getFirstChildElement(getDocumentElement(), HtmlHead.class);
1199             if (head == null) {
1200                 // perhaps should we create head too?
1201                 throw new IllegalStateException("Headelement was not defined for this page");
1202             }
1203             final Map<String, DomAttr> emptyMap = Collections.emptyMap();
1204             titleElement = new HtmlTitle(HtmlTitle.TAG_NAME, this, emptyMap);
1205             if (head.getFirstChild() != null) {
1206                 head.getFirstChild().insertBefore(titleElement);
1207             }
1208             else {
1209                 head.appendChild(titleElement);
1210             }
1211         }
1212 
1213         titleElement.setNodeValue(message);
1214     }
1215 
1216     /**
1217      * Gets the first child of startElement that is an instance of the given class.
1218      * @param startElement the parent element
1219      * @param clazz the class to search for
1220      * @return {@code null} if no child found
1221      */
1222     private static DomElement getFirstChildElement(final DomElement startElement, final Class<?> clazz) {
1223         if (startElement == null) {
1224             return null;
1225         }
1226         for (final DomElement element : startElement.getChildElements()) {
1227             if (clazz.isInstance(element)) {
1228                 return element;
1229             }
1230         }
1231 
1232         return null;
1233     }
1234 
1235     /**
1236      * Gets the first child of startElement or it's children that is an instance of the given class.
1237      * @param startElement the parent element
1238      * @param clazz the class to search for
1239      * @return {@code null} if no child found
1240      */
1241     private DomElement getFirstChildElementRecursive(final DomElement startElement, final Class<?> clazz) {
1242         if (startElement == null) {
1243             return null;
1244         }
1245         for (final DomElement element : startElement.getChildElements()) {
1246             if (clazz.isInstance(element)) {
1247                 return element;
1248             }
1249             final DomElement childFound = getFirstChildElementRecursive(element, clazz);
1250             if (childFound != null) {
1251                 return childFound;
1252             }
1253         }
1254 
1255         return null;
1256     }
1257 
1258     /**
1259      * Gets the title element for this page. Returns null if one is not found.
1260      *
1261      * @return the title element for this page or null if this is not one
1262      */
1263     private HtmlTitle getTitleElement() {
1264         return (HtmlTitle) getFirstChildElementRecursive(getDocumentElement(), HtmlTitle.class);
1265     }
1266 
1267     /**
1268      * Looks for and executes any appropriate event handlers. Looks for body and frame tags.
1269      * @param eventType either {@link Event#TYPE_LOAD}, {@link Event#TYPE_UNLOAD}, or {@link Event#TYPE_BEFORE_UNLOAD}
1270      * @return {@code true} if user accepted <code>onbeforeunload</code> (not relevant to other events)
1271      */
1272     private boolean executeEventHandlersIfNeeded(final String eventType) {
1273         // If JavaScript isn't enabled, there's nothing for us to do.
1274         if (!getWebClient().isJavaScriptEnabled()) {
1275             return true;
1276         }
1277 
1278         // Execute the specified event on the document element.
1279         final WebWindow window = getEnclosingWindow();
1280         if (window.getScriptableObject() instanceof Window) {
1281             final Event event;
1282             if (Event.TYPE_BEFORE_UNLOAD.equals(eventType)) {
1283                 event = new BeforeUnloadEvent(this, eventType);
1284             }
1285             else {
1286                 event = new Event(this, eventType);
1287             }
1288 
1289             // This is the same as DomElement.fireEvent() and was copied
1290             // here so it could be used with HtmlPage.
1291             if (LOG.isDebugEnabled()) {
1292                 LOG.debug("Firing " + event);
1293             }
1294 
1295             final EventTarget jsNode;
1296             if (Event.TYPE_DOM_DOCUMENT_LOADED.equals(eventType)) {
1297                 jsNode = getScriptableObject();
1298             }
1299             else if (Event.TYPE_READY_STATE_CHANGE.equals(eventType)) {
1300                 jsNode = getDocumentElement().getScriptableObject();
1301             }
1302             else {
1303                 // The load/beforeunload/unload events target Document but paths Window only (tested in Chrome/FF)
1304                 jsNode = window.getScriptableObject();
1305             }
1306 
1307             ((JavaScriptEngine) getWebClient().getJavaScriptEngine()).callSecured(cx -> jsNode.fireEvent(event), this);
1308 
1309             if (!isOnbeforeunloadAccepted(this, event)) {
1310                 return false;
1311             }
1312         }
1313 
1314         // If this page was loaded in a frame, execute the version of the event specified on the frame tag.
1315         if (window instanceof FrameWindow fw) {
1316             final BaseFrameElement frame = fw.getFrameElement();
1317 
1318             // if part of a document fragment, then the load event is not triggered
1319             if (Event.TYPE_LOAD.equals(eventType) && frame.getParentNode() instanceof DomDocumentFragment) {
1320                 return true;
1321             }
1322 
1323             if (frame.hasEventHandlers("on" + eventType)) {
1324                 if (LOG.isDebugEnabled()) {
1325                     LOG.debug("Executing on" + eventType + " handler for " + frame);
1326                 }
1327                 if (window.getScriptableObject() instanceof Window) {
1328                     final Event event;
1329                     if (Event.TYPE_BEFORE_UNLOAD.equals(eventType)) {
1330                         event = new BeforeUnloadEvent(frame, eventType);
1331                     }
1332                     else {
1333                         event = new Event(frame, eventType);
1334                     }
1335                     // This fires the "load" event for the <frame> element which, like all non-window
1336                     // load events, propagates up to Document but not Window.  The "load" event for
1337                     // <frameset> on the other hand, like that of <body>, is handled above where it is
1338                     // fired against Document and directed to Window.
1339                     frame.fireEvent(event);
1340 
1341                     if (!isOnbeforeunloadAccepted((HtmlPage) frame.getPage(), event)) {
1342                         return false;
1343                     }
1344                 }
1345             }
1346         }
1347 
1348         return true;
1349     }
1350 
1351     /**
1352      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1353      *
1354      * @return true if the OnbeforeunloadHandler has accepted to change the page
1355      */
1356     public boolean isOnbeforeunloadAccepted() {
1357         return executeEventHandlersIfNeeded(Event.TYPE_BEFORE_UNLOAD);
1358     }
1359 
1360     private boolean isOnbeforeunloadAccepted(final HtmlPage page, final Event event) {
1361         if (event instanceof BeforeUnloadEvent beforeUnloadEvent) {
1362             if (beforeUnloadEvent.isBeforeUnloadMessageSet()) {
1363                 final OnbeforeunloadHandler handler = getWebClient().getOnbeforeunloadHandler();
1364                 if (handler == null) {
1365                     LOG.warn("document.onbeforeunload() returned a string in event.returnValue,"
1366                             + " but no onbeforeunload handler installed.");
1367                 }
1368                 else {
1369                     final String message = JavaScriptEngine.toString(beforeUnloadEvent.getReturnValue());
1370                     return handler.handleEvent(page, message);
1371                 }
1372             }
1373         }
1374         return true;
1375     }
1376 
1377     /**
1378      * If a refresh has been specified either through a meta tag or an HTTP
1379      * response header, then perform that refresh.
1380      * @throws IOException if an IO problem occurs
1381      */
1382     private void executeRefreshIfNeeded() throws IOException {
1383         // If this page is not in a frame then a refresh has already happened,
1384         // most likely through the JavaScript onload handler, so we don't do a
1385         // second refresh.
1386         final WebWindow window = getEnclosingWindow();
1387         if (window == null) {
1388             return;
1389         }
1390 
1391         final String refreshString = getRefreshStringOrNull();
1392         if (refreshString == null || refreshString.isEmpty()) {
1393             return;
1394         }
1395 
1396         final double time;
1397         final URL url;
1398 
1399         final int index = StringUtils.indexOfAnyBut(refreshString, "0123456789.");
1400 
1401         if (index == -1) {
1402             // Format: <meta http-equiv='refresh' content='10'>
1403             try {
1404                 time = Double.parseDouble(refreshString);
1405             }
1406             catch (final NumberFormatException e) {
1407                 if (LOG.isErrorEnabled()) {
1408                     LOG.error("Malformed refresh string (no ';' but not a number): " + refreshString, e);
1409                 }
1410                 return;
1411             }
1412             url = getUrl();
1413         }
1414         else {
1415             // Format: <meta http-equiv='refresh' content='10;url=http://www.blah.com'>
1416             try {
1417                 time = Double.parseDouble(refreshString.substring(0, index));
1418             }
1419             catch (final NumberFormatException e) {
1420                 if (LOG.isErrorEnabled()) {
1421                     LOG.error("Malformed refresh string (no valid number before ';') " + refreshString, e);
1422                 }
1423                 return;
1424             }
1425 
1426             String urlPart = refreshString.substring(index);
1427             final char separator = urlPart.charAt(0);
1428             if (";, \r\n\t".indexOf(separator) >= 0) {
1429                 urlPart = StringUtils.stripStart(urlPart, ";, \r\n\t");
1430                 if (urlPart.toLowerCase(Locale.ROOT).startsWith("url")) {
1431                     urlPart = urlPart.substring(3);
1432                     urlPart = urlPart.trim();
1433 
1434                     if (urlPart.toLowerCase().startsWith("=")) {
1435                         urlPart = urlPart.substring(1);
1436                         urlPart = urlPart.trim();
1437                     }
1438                 }
1439 
1440                 if (org.htmlunit.util.StringUtils.isBlank(urlPart)) {
1441                     //content='10; URL=' is treated as content='10'
1442                     url = getUrl();
1443                 }
1444                 else {
1445                     if (urlPart.charAt(0) == '"' || urlPart.charAt(0) == 0x27) {
1446                         urlPart = urlPart.substring(1);
1447                     }
1448                     if (urlPart.charAt(urlPart.length() - 1) == '"' || urlPart.charAt(urlPart.length() - 1) == 0x27) {
1449                         urlPart = urlPart.substring(0, urlPart.length() - 1);
1450                     }
1451                     try {
1452                         url = getFullyQualifiedUrl(urlPart);
1453                     }
1454                     catch (final MalformedURLException e) {
1455                         if (LOG.isErrorEnabled()) {
1456                             LOG.error("Malformed URL in refresh string: " + refreshString, e);
1457                         }
1458                         return;
1459                     }
1460                 }
1461             }
1462             else {
1463                 if (LOG.isErrorEnabled()) {
1464                     LOG.error("Malformed refresh string (separator after time missing): " + refreshString);
1465                 }
1466                 return;
1467             }
1468         }
1469 
1470         processRefresh(url, time);
1471     }
1472 
1473     // this is different from what is done in org.htmlunit.WebClient.loadWebResponseFromWebConnection(WebRequest, int)
1474     // because there we are directly replacing the response before loading the response into the window
1475     // here we are replacing the page in the window (maybe after some time)
1476     private void processRefresh(final URL url, final double time) throws IOException {
1477         final WebClient webClient = getWebClient();
1478 
1479         final int refreshLimit = webClient.getOptions().getPageRefreshLimit();
1480         if (refreshLimit == 0) {
1481             final WebResponse webResponse = getWebResponse();
1482             throw new FailingHttpStatusCodeException("Too many redirects for "
1483                     + webResponse.getWebRequest().getUrl(), webResponse);
1484         }
1485 
1486         if (refreshLimit >= 0) {
1487             final StackTraceElement[] elements = new Exception().getStackTrace();
1488             int count = 0;
1489             final int elementCountLimit = refreshLimit > 50 ? 400 : refreshLimit > 10 ? 80 : 5;
1490             final int elementCount = elements.length;
1491 
1492             if (elementCount > elementCountLimit) {
1493                 for (int i = 0; i < elementCount; i++) {
1494                     if ("processRefresh".equals(elements[i].getMethodName())
1495                             && "org.htmlunit.html.HtmlPage".equals(elements[i].getClassName())) {
1496                         count++;
1497                         if (count >= refreshLimit) {
1498                             final WebResponse webResponse = getWebResponse();
1499                             throw new FailingHttpStatusCodeException(
1500                                             "Too many redirects (>= " + count + ") for "
1501                                                 + webResponse.getWebRequest().getUrl(), webResponse);
1502                         }
1503                     }
1504                 }
1505             }
1506         }
1507 
1508         webClient.getRefreshHandler().handleRefresh(this, url, (int) time);
1509     }
1510 
1511     /**
1512      * Returns an auto-refresh string if specified. This will look in both the meta
1513      * tags and inside the HTTP response headers.
1514      * @return the auto-refresh string
1515      */
1516     private String getRefreshStringOrNull() {
1517         final List<HtmlMeta> metaTags = getMetaTags("refresh");
1518         if (!metaTags.isEmpty()) {
1519             return metaTags.get(0).getContentAttribute().trim();
1520         }
1521         return getWebResponse().getResponseHeaderValue("Refresh");
1522     }
1523 
1524     private void processPostponedActionsIfNeeded() {
1525         if (!getWebClient().isJavaScriptEnabled()) {
1526             return;
1527         }
1528         getWebClient().getJavaScriptEngine().processPostponedActions();
1529     }
1530 
1531     /**
1532      * Executes any deferred scripts, if necessary.
1533      */
1534     private void executeDeferredScriptsIfNeeded() {
1535         if (!getWebClient().isJavaScriptEnabled()) {
1536             return;
1537         }
1538         final DomElement doc = getDocumentElement();
1539         final List<HtmlScript> scripts = new ArrayList<>();
1540 
1541         // don't call getElementsByTagName() here because it creates a live collection
1542         for (final HtmlElement elem : doc.getHtmlElementDescendants()) {
1543             if ("script".equals(elem.getLocalName()) && (elem instanceof HtmlScript script)) {
1544                 if (script.isDeferred() && ATTRIBUTE_NOT_DEFINED != script.getSrcAttribute()) {
1545                     scripts.add(script);
1546                 }
1547             }
1548         }
1549         for (final HtmlScript script : scripts) {
1550             ScriptElementSupport.executeScriptIfNeeded(script, true, true);
1551         }
1552     }
1553 
1554     /**
1555      * Deregister frames that are no longer in use.
1556      */
1557     public void deregisterFramesIfNeeded() {
1558         final List<BaseFrameElement> frameElementsCopy = new ArrayList<>(frameElements_);
1559         for (final BaseFrameElement frameElement : frameElementsCopy) {
1560             final WebWindow window = frameElement.getEnclosedWindow();
1561             getWebClient().deregisterWebWindow(window);
1562             final Page page = window.getEnclosedPage();
1563             if (page != null && page.isHtmlPage()) {
1564                 // seems quite silly, but for instance if the src attribute of an iframe is not
1565                 // set, the error only occurs when leaving the page
1566                 ((HtmlPage) page).deregisterFramesIfNeeded();
1567             }
1568         }
1569     }
1570 
1571     /**
1572      * Returns a list containing all the frames (from frame and iframe tags) in this page
1573      * in document order.
1574      * @return a list of {@link FrameWindow}
1575      */
1576     public List<FrameWindow> getFrames() {
1577         final List<BaseFrameElement> frameElements = new ArrayList<>(frameElements_);
1578         frameElements.sort(DOCUMENT_POSITION_COMPERATOR);
1579 
1580         final List<FrameWindow> list = new ArrayList<>(frameElements.size());
1581         for (final BaseFrameElement frameElement : frameElements) {
1582             list.add(frameElement.getEnclosedWindow());
1583         }
1584         return list;
1585     }
1586 
1587     /**
1588      * Returns the first frame contained in this page with the specified name.
1589      * @param name the name to search for
1590      * @return the first frame found
1591      * @throws ElementNotFoundException If no frame exist in this page with the specified name.
1592      */
1593     public FrameWindow getFrameByName(final String name) throws ElementNotFoundException {
1594         for (final BaseFrameElement frameElement : frameElements_) {
1595             final FrameWindow fw = frameElement.getEnclosedWindow();
1596             if (fw.getName().equals(name)) {
1597                 return fw;
1598             }
1599         }
1600 
1601         throw new ElementNotFoundException("frame or iframe", DomElement.NAME_ATTRIBUTE, name);
1602     }
1603 
1604     /**
1605      * Simulate pressing an access key. This may change the focus, may click buttons and may invoke
1606      * JavaScript.
1607      *
1608      * @param accessKey the key that will be pressed
1609      * @return the element that has the focus after pressing this access key or null if no element
1610      *         has the focus.
1611      * @throws IOException if an IO error occurs during the processing of this access key (this
1612      *         would only happen if the access key triggered a button which in turn caused a page load)
1613      */
1614     public DomElement pressAccessKey(final char accessKey) throws IOException {
1615         final HtmlElement element = getHtmlElementByAccessKey(accessKey);
1616         if (element != null) {
1617             element.focus();
1618             if (element instanceof HtmlAnchor
1619                     || element instanceof HtmlArea
1620                     || element instanceof HtmlButton
1621                     || element instanceof HtmlInput
1622                     || element instanceof HtmlLabel
1623                     || element instanceof HtmlLegend
1624                     || element instanceof HtmlTextArea) {
1625                 final Page newPage = element.click();
1626 
1627                 if (newPage != this && getFocusedElement() == element) {
1628                     // The page was reloaded therefore no element on this page will have the focus.
1629                     getFocusedElement().blur();
1630                 }
1631             }
1632         }
1633 
1634         return getFocusedElement();
1635     }
1636 
1637     /**
1638      * Move the focus to the next element in the tab order. To determine the specified tab
1639      * order, refer to {@link HtmlPage#getTabbableElements()}
1640      *
1641      * @return the element that has focus after calling this method
1642      */
1643     public HtmlElement tabToNextElement() {
1644         final List<HtmlElement> elements = getTabbableElements();
1645         if (elements.isEmpty()) {
1646             setFocusedElement(null);
1647             return null;
1648         }
1649 
1650         final HtmlElement elementToGiveFocus;
1651         final DomElement elementWithFocus = getFocusedElement();
1652         if (elementWithFocus == null) {
1653             elementToGiveFocus = elements.get(0);
1654         }
1655         else {
1656             final int index = elements.indexOf(elementWithFocus);
1657             if (index == -1) {
1658                 // The element with focus isn't on this page
1659                 elementToGiveFocus = elements.get(0);
1660             }
1661             else if (index == elements.size() - 1) {
1662                 // if at last jump to start
1663                 elementToGiveFocus = elements.get(0);
1664             }
1665             else {
1666                 elementToGiveFocus = elements.get(index + 1);
1667             }
1668         }
1669 
1670         setFocusedElement(elementToGiveFocus);
1671         return elementToGiveFocus;
1672     }
1673 
1674     /**
1675      * Move the focus to the previous element in the tab order. To determine the specified tab
1676      * order, refer to {@link HtmlPage#getTabbableElements()}
1677      *
1678      * @return the element that has focus after calling this method
1679      */
1680     public HtmlElement tabToPreviousElement() {
1681         final List<HtmlElement> elements = getTabbableElements();
1682         if (elements.isEmpty()) {
1683             setFocusedElement(null);
1684             return null;
1685         }
1686 
1687         final HtmlElement elementToGiveFocus;
1688         final DomElement elementWithFocus = getFocusedElement();
1689         if (elementWithFocus == null) {
1690             elementToGiveFocus = elements.get(elements.size() - 1);
1691         }
1692         else {
1693             final int index = elements.indexOf(elementWithFocus);
1694             if (index == -1) {
1695                 // The element with focus isn't on this page
1696                 elementToGiveFocus = elements.get(elements.size() - 1);
1697             }
1698             else if (index == 0) {
1699                 // first; back to the last
1700                 elementToGiveFocus = elements.get(elements.size() - 1);
1701             }
1702             else {
1703                 elementToGiveFocus = elements.get(index - 1);
1704             }
1705         }
1706 
1707         setFocusedElement(elementToGiveFocus);
1708         return elementToGiveFocus;
1709     }
1710 
1711     /**
1712      * Returns the HTML element with the specified ID. If more than one element
1713      * has this ID (not allowed by the HTML spec), then this method returns the
1714      * first one.
1715      *
1716      * @param elementId the ID value to search for
1717      * @param <E> the element type
1718      * @return the HTML element with the specified ID
1719      * @throws ElementNotFoundException if no element was found matching the specified ID
1720      */
1721     @SuppressWarnings("unchecked")
1722     public <E extends HtmlElement> E getHtmlElementById(final String elementId) throws ElementNotFoundException {
1723         final DomElement element = getElementById(elementId);
1724         if (element == null) {
1725             throw new ElementNotFoundException("*", DomElement.ID_ATTRIBUTE, elementId);
1726         }
1727         return (E) element;
1728     }
1729 
1730     /**
1731      * Returns the elements with the specified ID. If there are no elements
1732      * with the specified ID, this method returns an empty list. Please note that
1733      * the lists returned by this method are immutable.
1734      *
1735      * @param elementId the ID value to search for
1736      * @return the elements with the specified name attribute
1737      */
1738     public List<DomElement> getElementsById(final String elementId) {
1739         if (elementId != null) {
1740             ensureMappedElementsBuilt();
1741             final MappedElementIndexEntry elements = idMap_.get(elementId);
1742             if (elements != null) {
1743                 return new ArrayList<>(elements.elements());
1744             }
1745         }
1746         return Collections.emptyList();
1747     }
1748 
1749     /**
1750      * Returns the element with the specified name. If more than one element
1751      * has this name, then this method returns the first one.
1752      *
1753      * @param name the name value to search for
1754      * @param <E> the element type
1755      * @return the element with the specified name
1756      * @throws ElementNotFoundException if no element was found matching the specified name
1757      */
1758     @SuppressWarnings("unchecked")
1759     public <E extends DomElement> E getElementByName(final String name) throws ElementNotFoundException {
1760         if (name != null) {
1761             ensureMappedElementsBuilt();
1762             final MappedElementIndexEntry elements = nameMap_.get(name);
1763             if (elements != null) {
1764                 return (E) elements.first();
1765             }
1766         }
1767         throw new ElementNotFoundException("*", DomElement.NAME_ATTRIBUTE, name);
1768     }
1769 
1770     /**
1771      * Returns the elements with the specified name attribute. If there are no elements
1772      * with the specified name, this method returns an empty list. Please note that
1773      * the lists returned by this method are immutable.
1774      *
1775      * @param name the name value to search for
1776      * @return the elements with the specified name attribute
1777      */
1778     public List<DomElement> getElementsByName(final String name) {
1779         if (name != null) {
1780             ensureMappedElementsBuilt();
1781             final MappedElementIndexEntry elements = nameMap_.get(name);
1782             if (elements != null) {
1783                 return new ArrayList<>(elements.elements());
1784             }
1785         }
1786         return Collections.emptyList();
1787     }
1788 
1789     /**
1790      * Returns the elements with the specified string for their name or ID. If there are
1791      * no elements with the specified name or ID, this method returns an empty list.
1792      *
1793      * @param idAndOrName the value to search for
1794      * @return the elements with the specified string for their name or ID
1795      */
1796     public List<DomElement> getElementsByIdAndOrName(final String idAndOrName) {
1797         if (idAndOrName == null) {
1798             return Collections.emptyList();
1799         }
1800         ensureMappedElementsBuilt();
1801         final MappedElementIndexEntry list1 = idMap_.get(idAndOrName);
1802         final MappedElementIndexEntry list2 = nameMap_.get(idAndOrName);
1803         final List<DomElement> list = new ArrayList<>();
1804         if (list1 != null) {
1805             list.addAll(list1.elements());
1806         }
1807         if (list2 != null) {
1808             for (final DomElement elt : list2.elements()) {
1809                 if (!list.contains(elt)) {
1810                     list.add(elt);
1811                 }
1812             }
1813         }
1814         return list;
1815     }
1816 
1817     /**
1818      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1819      *
1820      * @param node the node that has just been added to the document
1821      */
1822     void notifyNodeAdded(final DomNode node) {
1823         if (node instanceof DomElement element1) {
1824             addMappedElement(element1, true);
1825 
1826             if (node instanceof BaseFrameElement element) {
1827                 frameElements_.add(element);
1828             }
1829 
1830             if (node.getFirstChild() != null) {
1831                 for (final Iterator<HtmlElement> iterator = node.new DescendantHtmlElementsIterator();
1832                         iterator.hasNext();) {
1833                     final HtmlElement child = iterator.next();
1834                     if (child instanceof BaseFrameElement element) {
1835                         frameElements_.add(element);
1836                     }
1837                 }
1838             }
1839 
1840             if ("base".equals(node.getNodeName())) {
1841                 calculateBase();
1842             }
1843         }
1844         node.onAddedToPage();
1845     }
1846 
1847     /**
1848      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
1849      *
1850      * @param node the node that has just been removed from the tree
1851      */
1852     void notifyNodeRemoved(final DomNode node) {
1853         if (node instanceof HtmlElement element) {
1854             removeMappedElement(element, true, true);
1855 
1856             if (node instanceof BaseFrameElement) {
1857                 frameElements_.remove(node);
1858             }
1859             for (final HtmlElement child : node.getHtmlElementDescendants()) {
1860                 if (child instanceof BaseFrameElement) {
1861                     frameElements_.remove(child);
1862                 }
1863             }
1864 
1865             if ("base".equals(node.getNodeName())) {
1866                 calculateBase();
1867             }
1868         }
1869     }
1870 
1871     /**
1872      * Adds an element to the ID and name maps, if necessary.
1873      * @param element the element to be added to the ID and name maps
1874      * @param recurse indicates if children must be added too
1875      */
1876     void addMappedElement(final DomElement element, final boolean recurse) {
1877         // Index is built lazily; skip while not built. ensureMappedElementsBuilt()
1878         // walks the tree once and populates everything on first read.
1879         if (!mappedElementsBuilt_) {
1880             return;
1881         }
1882         if (isAncestorOf(element)) {
1883             addElement(element, recurse);
1884         }
1885     }
1886 
1887     private void ensureMappedElementsBuilt() {
1888         if (mappedElementsBuilt_) {
1889             return;
1890         }
1891 
1892         final DomElement root = getDocumentElement();
1893         if (root != null) {
1894             addElement(root, true);
1895         }
1896 
1897         // Flip the flag only after the maps are populated, so a partial
1898         // failure mid-walk leaves us with built_=false and the next read
1899         // tries again rather than seeing a half-populated index.
1900         mappedElementsBuilt_ = true;
1901     }
1902 
1903     private void addElement(final DomElement element, final boolean recurse) {
1904         final String idValue = element.getAttribute(DomElement.ID_ATTRIBUTE);
1905         if (ATTRIBUTE_NOT_DEFINED != idValue) {
1906             MappedElementIndexEntry elements = idMap_.get(idValue);
1907             if (elements == null) {
1908                 elements = new MappedElementIndexEntry();
1909                 elements.add(element);
1910                 idMap_.put(idValue, elements);
1911             }
1912             else {
1913                 elements.add(element);
1914             }
1915         }
1916 
1917         final String nameValue = element.getAttribute(DomElement.NAME_ATTRIBUTE);
1918         if (ATTRIBUTE_NOT_DEFINED != nameValue) {
1919             MappedElementIndexEntry elements = nameMap_.get(nameValue);
1920             if (elements == null) {
1921                 elements = new MappedElementIndexEntry();
1922                 elements.add(element);
1923                 nameMap_.put(nameValue, elements);
1924             }
1925             else {
1926                 elements.add(element);
1927             }
1928         }
1929 
1930         if (recurse) {
1931             // poor man's approach - we don't use getChildElements()
1932             // to avoid a bunch of object constructions
1933             DomNode nextChild = element.getFirstChild();
1934             while (nextChild != null) {
1935                 if (nextChild instanceof DomElement domElement) {
1936                     addElement(domElement, true);
1937                 }
1938                 nextChild = nextChild.getNextSibling();
1939             }
1940         }
1941     }
1942 
1943     /**
1944      * Removes an element and optionally its children from the ID and name maps, if necessary.
1945      * @param element the element to be removed from the ID and name maps
1946      * @param recurse indicates if children must be removed too
1947      * @param descendant indicates of the element was descendant of this HtmlPage, but now its parent might be null
1948      */
1949     void removeMappedElement(final DomElement element, final boolean recurse, final boolean descendant) {
1950         // see addMappedElement: while the index is unbuilt, removals are also no-ops.
1951         if (!mappedElementsBuilt_) {
1952             return;
1953         }
1954         if (descendant || isAncestorOf(element)) {
1955             removeElement(element, recurse);
1956         }
1957     }
1958 
1959     private void removeElement(final DomElement element, final boolean recurse) {
1960         final String idValue = element.getAttribute(DomElement.ID_ATTRIBUTE);
1961         if (ATTRIBUTE_NOT_DEFINED != idValue) {
1962             final MappedElementIndexEntry elements = idMap_.remove(idValue);
1963             if (elements != null) {
1964                 elements.remove(element);
1965                 if (!elements.elements_.isEmpty()) {
1966                     idMap_.put(idValue, elements);
1967                 }
1968             }
1969         }
1970 
1971         final String nameValue = element.getAttribute(DomElement.NAME_ATTRIBUTE);
1972         if (ATTRIBUTE_NOT_DEFINED != nameValue) {
1973             final MappedElementIndexEntry elements = nameMap_.remove(nameValue);
1974             if (elements != null) {
1975                 elements.remove(element);
1976                 if (!elements.elements_.isEmpty()) {
1977                     nameMap_.put(nameValue, elements);
1978                 }
1979             }
1980         }
1981 
1982         if (recurse) {
1983             for (final DomElement child : element.getChildElements()) {
1984                 removeElement(child, true);
1985             }
1986         }
1987     }
1988 
1989     /**
1990      * Indicates if the attribute name indicates that the owning element is mapped.
1991      * @param document the owning document
1992      * @param attributeName the name of the attribute to consider
1993      * @return {@code true} if the owning element should be mapped in its owning page
1994      */
1995     static boolean isMappedElement(final Document document, final String attributeName) {
1996         return document instanceof HtmlPage
1997             && (DomElement.NAME_ATTRIBUTE.equals(attributeName) || DomElement.ID_ATTRIBUTE.equals(attributeName));
1998     }
1999 
2000     private void calculateBase() {
2001         final List<HtmlElement> baseElements = getDocumentElement().getStaticElementsByTagName("base");
2002 
2003         base_ = null;
2004         for (final HtmlElement baseElement : baseElements) {
2005             if (baseElement instanceof HtmlBase base) {
2006                 if (base_ != null) {
2007                     notifyIncorrectness("Multiple 'base' detected, only the first is used.");
2008                     break;
2009                 }
2010                 base_ = base;
2011             }
2012         }
2013     }
2014 
2015     /**
2016      * Loads the content of the contained frames. This is done after the page is completely loaded, to allow script
2017      * contained in the frames to reference elements from the page located after the closing &lt;/frame&gt; tag.
2018      * @throws FailingHttpStatusCodeException if the server returns a failing status code AND the property
2019      *         {@link WebClientOptions#setThrowExceptionOnFailingStatusCode(boolean)} is set to {@code true}
2020      */
2021     void loadFrames() throws FailingHttpStatusCodeException {
2022         for (final BaseFrameElement frameElement : new ArrayList<>(frameElements_)) {
2023             // test if the frame should really be loaded:
2024             // if a script has already changed its content, it should be skipped
2025             // use == and not equals(...) to identify initial content (versus URL set to "about:blank")
2026             if (frameElement.getEnclosedWindow() != null
2027                     && UrlUtils.URL_ABOUT_BLANK == frameElement.getEnclosedPage().getUrl()
2028                     && !frameElement.isContentLoaded()) {
2029                 frameElement.loadInnerPage();
2030             }
2031         }
2032     }
2033 
2034     /**
2035      * Gives a basic representation for debugging purposes.
2036      * @return a basic representation
2037      */
2038     @Override
2039     public String toString() {
2040         final StringBuilder builder = new StringBuilder()
2041             .append("HtmlPage(")
2042             .append(getUrl())
2043             .append(")@")
2044             .append(hashCode());
2045         return builder.toString();
2046     }
2047 
2048     /**
2049      * Gets the meta tag for a given {@code http-equiv} value.
2050      * @param httpEquiv the {@code http-equiv} value
2051      * @return a list of {@link HtmlMeta}
2052      */
2053     protected List<HtmlMeta> getMetaTags(final String httpEquiv) {
2054         if (getDocumentElement() == null) {
2055             return Collections.emptyList(); // weird case, for instance if document.documentElement has been removed
2056         }
2057         final List<HtmlMeta> tags = getDocumentElement().getStaticElementsByTagName("meta");
2058         final List<HtmlMeta> foundTags = new ArrayList<>();
2059         for (final HtmlMeta htmlMeta : tags) {
2060             if (httpEquiv.equalsIgnoreCase(htmlMeta.getHttpEquivAttribute())) {
2061                 foundTags.add(htmlMeta);
2062             }
2063         }
2064         return foundTags;
2065     }
2066 
2067     /**
2068      * Creates a clone of this instance, and clears cached state to be not shared with the original.
2069      *
2070      * @return a clone of this instance
2071      */
2072     @Override
2073     protected HtmlPage clone() {
2074         final HtmlPage result = (HtmlPage) super.clone();
2075         result.elementWithFocus_ = null;
2076 
2077         result.idMap_ = new ConcurrentHashMap<>();
2078         result.nameMap_ = new ConcurrentHashMap<>();
2079         result.mappedElementsBuilt_ = false;
2080 
2081         return result;
2082     }
2083 
2084     /**
2085      * {@inheritDoc}
2086      */
2087     @Override
2088     public HtmlPage cloneNode(final boolean deep) {
2089         // we need the ScriptObject clone before cloning the kids.
2090         final HtmlPage result = (HtmlPage) super.cloneNode(false);
2091         if (getWebClient().isJavaScriptEnabled()) {
2092             final HtmlUnitScriptable jsObjClone = getScriptableObject().clone();
2093             jsObjClone.setDomNode(result);
2094         }
2095 
2096         // if deep, clone the kids too, and re initialize parts of the clone
2097         if (deep) {
2098             // this was previously synchronized but that makes not sense, why
2099             // lock the source against a copy only one has a reference too,
2100             // because result is a local reference
2101             result.attributeListeners_ = null;
2102 
2103             result.selectionRanges_ = new ArrayList<>(3);
2104             // the original one is synchronized so we should do that here too, shouldn't we?
2105             result.afterLoadActions_ = Collections.synchronizedList(new ArrayList<>());
2106             result.frameElements_ = new ArrayList<>();
2107             for (DomNode child = getFirstChild(); child != null; child = child.getNextSibling()) {
2108                 result.appendChild(child.cloneNode(true));
2109             }
2110         }
2111         return result;
2112     }
2113 
2114     /**
2115      * Adds an HtmlAttributeChangeListener to the listener list.
2116      * The listener is registered for all attributes of all HtmlElements contained in this page.
2117      *
2118      * @param listener the attribute change listener to be added
2119      * @see #removeHtmlAttributeChangeListener(HtmlAttributeChangeListener)
2120      */
2121     public void addHtmlAttributeChangeListener(final HtmlAttributeChangeListener listener) {
2122         WebAssert.notNull("listener", listener);
2123         synchronized (lock_) {
2124             if (attributeListeners_ == null) {
2125                 attributeListeners_ = new LinkedHashSet<>();
2126             }
2127             attributeListeners_.add(listener);
2128         }
2129     }
2130 
2131     /**
2132      * Removes an HtmlAttributeChangeListener from the listener list.
2133      * This method should be used to remove HtmlAttributeChangeListener that were registered
2134      * for all attributes of all HtmlElements contained in this page.
2135      *
2136      * @param listener the attribute change listener to be removed
2137      * @see #addHtmlAttributeChangeListener(HtmlAttributeChangeListener)
2138      */
2139     public void removeHtmlAttributeChangeListener(final HtmlAttributeChangeListener listener) {
2140         WebAssert.notNull("listener", listener);
2141         synchronized (lock_) {
2142             if (attributeListeners_ != null) {
2143                 attributeListeners_.remove(listener);
2144             }
2145         }
2146     }
2147 
2148     /**
2149      * Notifies all registered listeners for the given event to add an attribute.
2150      * @param event the event to fire
2151      */
2152     void fireHtmlAttributeAdded(final HtmlAttributeChangeEvent event) {
2153         final List<HtmlAttributeChangeListener> listeners = safeGetAttributeListeners();
2154         if (listeners != null) {
2155             for (final HtmlAttributeChangeListener listener : listeners) {
2156                 listener.attributeAdded(event);
2157             }
2158         }
2159     }
2160 
2161     /**
2162      * Notifies all registered listeners for the given event to replace an attribute.
2163      * @param event the event to fire
2164      */
2165     void fireHtmlAttributeReplaced(final HtmlAttributeChangeEvent event) {
2166         final List<HtmlAttributeChangeListener> listeners = safeGetAttributeListeners();
2167         if (listeners != null) {
2168             for (final HtmlAttributeChangeListener listener : listeners) {
2169                 listener.attributeReplaced(event);
2170             }
2171         }
2172     }
2173 
2174     /**
2175      * Notifies all registered listeners for the given event to remove an attribute.
2176      * @param event the event to fire
2177      */
2178     void fireHtmlAttributeRemoved(final HtmlAttributeChangeEvent event) {
2179         final List<HtmlAttributeChangeListener> listeners = safeGetAttributeListeners();
2180         if (listeners != null) {
2181             for (final HtmlAttributeChangeListener listener : listeners) {
2182                 listener.attributeRemoved(event);
2183             }
2184         }
2185     }
2186 
2187     private List<HtmlAttributeChangeListener> safeGetAttributeListeners() {
2188         synchronized (lock_) {
2189             if (attributeListeners_ != null) {
2190                 return new ArrayList<>(attributeListeners_);
2191             }
2192             return null;
2193         }
2194     }
2195 
2196     /**
2197      * {@inheritDoc}
2198      */
2199     @Override
2200     protected void checkChildHierarchy(final org.w3c.dom.Node newChild) throws DOMException {
2201         if (newChild instanceof Element) {
2202             if (getDocumentElement() != null) {
2203                 throw new DOMException(DOMException.HIERARCHY_REQUEST_ERR,
2204                     "The Document may only have a single child Element.");
2205             }
2206         }
2207         else if (newChild instanceof DocumentType) {
2208             if (getDoctype() != null) {
2209                 throw new DOMException(DOMException.HIERARCHY_REQUEST_ERR,
2210                     "The Document may only have a single child DocumentType.");
2211             }
2212         }
2213         else if (!(newChild instanceof Comment || newChild instanceof ProcessingInstruction)) {
2214             throw new DOMException(DOMException.HIERARCHY_REQUEST_ERR,
2215                 "The Document may not have a child of this type: " + newChild.getNodeType());
2216         }
2217         super.checkChildHierarchy(newChild);
2218     }
2219 
2220     /**
2221      * Returns {@code true} if an HTML parser is operating on this page, adding content to it.
2222      * @return {@code true} if an HTML parser is operating on this page, adding content to it
2223      */
2224     public boolean isBeingParsed() {
2225         return parserCount_ > 0;
2226     }
2227 
2228     /**
2229      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2230      *
2231      * Called by the HTML parser to let the page know that it has started parsing some content for this page.
2232      */
2233     public void registerParsingStart() {
2234         parserCount_++;
2235     }
2236 
2237     /**
2238      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2239      *
2240      * Called by the HTML parser to let the page know that it has finished parsing some content for this page.
2241      */
2242     public void registerParsingEnd() {
2243         parserCount_--;
2244     }
2245 
2246     /**
2247      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2248      *
2249      * Returns {@code true} if an HTML parser is parsing a non-inline HTML snippet to add content
2250      * to this page. Non-inline content is content that is parsed for the page, but not in the
2251      * same stream as the page itself -- basically anything other than <code>document.write()</code>
2252      * or <code>document.writeln()</code>: <code>innerHTML</code>, <code>outerHTML</code>,
2253      * <code>document.createElement()</code>, etc.
2254      *
2255      * @return {@code true} if an HTML parser is parsing a non-inline HTML snippet to add content
2256      *         to this page
2257      */
2258     public boolean isParsingHtmlSnippet() {
2259         return snippetParserCount_ > 0;
2260     }
2261 
2262     /**
2263      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2264      *
2265      * Called by the HTML parser to let the page know that it has started parsing a non-inline HTML snippet.
2266      */
2267     public void registerSnippetParsingStart() {
2268         snippetParserCount_++;
2269     }
2270 
2271     /**
2272      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2273      *
2274      * Called by the HTML parser to let the page know that it has finished parsing a non-inline HTML snippet.
2275      */
2276     public void registerSnippetParsingEnd() {
2277         snippetParserCount_--;
2278     }
2279 
2280     /**
2281      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2282      *
2283      * Returns {@code true} if an HTML parser is parsing an inline HTML snippet to add content
2284      * to this page. Inline content is content inserted into the parser stream dynamically
2285      * while the page is being parsed (i.e. <code>document.write()</code> or <code>document.writeln()</code>).
2286      *
2287      * @return {@code true} if an HTML parser is parsing an inline HTML snippet to add content
2288      *         to this page
2289      */
2290     public boolean isParsingInlineHtmlSnippet() {
2291         return inlineSnippetParserCount_ > 0;
2292     }
2293 
2294     /**
2295      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2296      *
2297      * Called by the HTML parser to let the page know that it has started parsing an inline HTML snippet.
2298      */
2299     public void registerInlineSnippetParsingStart() {
2300         inlineSnippetParserCount_++;
2301     }
2302 
2303     /**
2304      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2305      *
2306      * Called by the HTML parser to let the page know that it has finished parsing an inline HTML snippet.
2307      */
2308     public void registerInlineSnippetParsingEnd() {
2309         inlineSnippetParserCount_--;
2310     }
2311 
2312     /**
2313      * Refreshes the page by sending the same parameters as previously sent to get this page.
2314      * @return the newly loaded page.
2315      * @throws IOException if an IO problem occurs
2316      */
2317     public Page refresh() throws IOException {
2318         return getWebClient().getPage(getWebResponse().getWebRequest());
2319     }
2320 
2321     /**
2322      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2323      * <p>
2324      * Parses the given string as would it belong to the content being parsed
2325      * at the current parsing position
2326      * </p>
2327      * @param string the HTML code to write in place
2328      */
2329     public void writeInParsedStream(final String string) {
2330         getDOMBuilder().pushInputString(string);
2331     }
2332 
2333     /**
2334      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2335      *
2336      * Sets the builder to allow page to send content from document.write(ln) calls.
2337      * @param htmlUnitDOMBuilder the builder
2338      */
2339     public void setDOMBuilder(final HTMLParserDOMBuilder htmlUnitDOMBuilder) {
2340         domBuilder_ = htmlUnitDOMBuilder;
2341     }
2342 
2343     /**
2344      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2345      *
2346      * Returns the current builder.
2347      * @return the current builder
2348      */
2349     public HTMLParserDOMBuilder getDOMBuilder() {
2350         return domBuilder_;
2351     }
2352 
2353     /**
2354      * <p>Returns all namespaces defined in the root element of this page.</p>
2355      * <p>The default namespace has a key of an empty string.</p>
2356      * @return all namespaces defined in the root element of this page
2357      */
2358     public Map<String, String> getNamespaces() {
2359         final org.w3c.dom.NamedNodeMap attributes = getDocumentElement().getAttributes();
2360         final Map<String, String> namespaces = new HashMap<>();
2361         for (int i = 0; i < attributes.getLength(); i++) {
2362             final Attr attr = (Attr) attributes.item(i);
2363             String name = attr.getName();
2364             if (name.startsWith("xmlns")) {
2365                 int startPos = 5;
2366                 if (name.length() > 5 && name.charAt(5) == ':') {
2367                     startPos = 6;
2368                 }
2369                 name = name.substring(startPos);
2370                 namespaces.put(name, attr.getValue());
2371             }
2372         }
2373         return namespaces;
2374     }
2375 
2376     /**
2377      * {@inheritDoc}
2378      */
2379     @Override
2380     public void setDocumentType(final DocumentType type) {
2381         super.setDocumentType(type);
2382     }
2383 
2384     /**
2385      * Saves the current page, with all images, to the specified location.
2386      * The default behavior removes all script elements.
2387      *
2388      * @param file file to write this page into
2389      * @throws IOException If an error occurs
2390      */
2391     public void save(final File file) throws IOException {
2392         new XmlSerializer().save(this, file);
2393     }
2394 
2395     /**
2396      * Returns whether the current page mode is in {@code quirks mode} or in {@code standards mode}.
2397      * @return true for {@code quirks mode}, false for {@code standards mode}
2398      */
2399     public boolean isQuirksMode() {
2400         return "BackCompat".equals(((HTMLDocument) getScriptableObject()).getCompatMode());
2401     }
2402 
2403     /**
2404      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2405      * {@inheritDoc}
2406      */
2407     @Override
2408     public boolean isAttachedToPage() {
2409         return true;
2410     }
2411 
2412     /**
2413      * {@inheritDoc}
2414      */
2415     @Override
2416     public boolean isHtmlPage() {
2417         return true;
2418     }
2419 
2420     /**
2421      * The base URL used to resolve relative URLs.
2422      * @return the base URL
2423      */
2424     public URL getBaseURL() {
2425         URL baseUrl;
2426         if (base_ == null) {
2427             baseUrl = getUrl();
2428             final WebWindow window = getEnclosingWindow();
2429             final boolean frame = window != null && window != window.getTopWindow();
2430             if (frame) {
2431                 final boolean frameSrcIsNotSet = baseUrl == UrlUtils.URL_ABOUT_BLANK;
2432                 final boolean frameSrcIsJs = "javascript".equals(baseUrl.getProtocol());
2433                 if (frameSrcIsNotSet || frameSrcIsJs) {
2434                     baseUrl = window.getTopWindow().getEnclosedPage().getWebResponse()
2435                         .getWebRequest().getUrl();
2436                 }
2437             }
2438             else if (baseUrl_ != null) {
2439                 baseUrl = baseUrl_;
2440             }
2441         }
2442         else {
2443             final String href = base_.getHrefAttribute().trim();
2444             if (org.htmlunit.util.StringUtils.isEmptyOrNull(href)) {
2445                 baseUrl = getUrl();
2446             }
2447             else {
2448                 final URL url = getUrl();
2449                 try {
2450                     if (href.startsWith("http://") || href.startsWith("https://")) {
2451                         baseUrl = new URL(href);
2452                     }
2453                     else if (href.startsWith("//")) {
2454                         baseUrl = new URL("%s:%s".formatted(url.getProtocol(), href));
2455                     }
2456                     else if (href.length() > 0 && href.charAt(0) == '/') {
2457                         final int port = Window.getPort(url);
2458                         baseUrl = new URL("%s://%s:%d%s".formatted(url.getProtocol(), url.getHost(), port, href));
2459                     }
2460                     else if (url.toString().endsWith("/")) {
2461                         baseUrl = new URL("%s%s".formatted(url, href));
2462                     }
2463                     else {
2464                         baseUrl = new URL(UrlUtils.resolveUrl(url, href));
2465                     }
2466                 }
2467                 catch (final MalformedURLException e) {
2468                     notifyIncorrectness("Invalid base url: \"" + href + "\", ignoring it");
2469                     baseUrl = url;
2470                 }
2471             }
2472         }
2473 
2474         return baseUrl;
2475     }
2476 
2477     /**
2478      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2479      *
2480      * Adds an {@link AutoCloseable}, which would be closed during the {@link #cleanUp()}.
2481      * @param autoCloseable the autoclosable
2482      */
2483     public void addAutoCloseable(final AutoCloseable autoCloseable) {
2484         if (autoCloseable == null) {
2485             return;
2486         }
2487 
2488         if (autoCloseableList_ == null) {
2489             autoCloseableList_ = new ArrayList<>();
2490         }
2491         autoCloseableList_.add(autoCloseable);
2492     }
2493 
2494     /**
2495      * {@inheritDoc}
2496      */
2497     @Override
2498     public boolean handles(final Event event) {
2499         if (Event.TYPE_BLUR.equals(event.getType()) || Event.TYPE_FOCUS.equals(event.getType())) {
2500             return true;
2501         }
2502         return super.handles(event);
2503     }
2504 
2505     /**
2506      * Sets the {@link ElementFromPointHandler}.
2507      * @param elementFromPointHandler the handler
2508      */
2509     public void setElementFromPointHandler(final ElementFromPointHandler elementFromPointHandler) {
2510         elementFromPointHandler_ = elementFromPointHandler;
2511     }
2512 
2513     /**
2514      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2515      *
2516      * Returns the element for the specified x coordinate and the specified y coordinate.
2517      *
2518      * @param x the x offset, in pixels
2519      * @param y the y offset, in pixels
2520      * @return the element for the specified x coordinate and the specified y coordinate
2521      */
2522     public HtmlElement getElementFromPoint(final int x, final int y) {
2523         if (elementFromPointHandler_ == null) {
2524             if (LOG.isWarnEnabled()) {
2525                 LOG.warn("ElementFromPointHandler was not specicifed for " + this);
2526             }
2527             if (x <= 0 || y <= 0) {
2528                 return null;
2529             }
2530             return getBody();
2531         }
2532         return elementFromPointHandler_.getElementFromPoint(this, x, y);
2533     }
2534 
2535     /**
2536      * Moves the focus to the specified element. This will trigger any relevant JavaScript
2537      * event handlers.
2538      *
2539      * @param newElement the element that will receive the focus, use {@code null} to remove focus from any element
2540      * @return true if the specified element now has the focus
2541      * @see #getFocusedElement()
2542      */
2543     public boolean setFocusedElement(final DomElement newElement) {
2544         return setFocusedElement(newElement, false);
2545     }
2546 
2547     /**
2548      * Moves the focus to the specified element. This will trigger any relevant JavaScript
2549      * event handlers.
2550      *
2551      * @param newElement the element that will receive the focus, use {@code null} to remove focus from any element
2552      * @param windowActivated - whether the enclosing window got focus resulting in specified element getting focus
2553      * @return true if the specified element now has the focus
2554      * @see #getFocusedElement()
2555      */
2556     public boolean setFocusedElement(final DomElement newElement, final boolean windowActivated) {
2557         if (elementWithFocus_ == newElement && !windowActivated) {
2558             // nothing to do
2559             return true;
2560         }
2561 
2562         final DomElement oldFocusedElement = elementWithFocus_;
2563         elementWithFocus_ = null;
2564 
2565         if (!windowActivated) {
2566             if (oldFocusedElement != null) {
2567                 oldFocusedElement.removeFocus();
2568                 oldFocusedElement.fireEvent(Event.TYPE_BLUR);
2569 
2570                 oldFocusedElement.fireEvent(Event.TYPE_FOCUS_OUT);
2571             }
2572         }
2573 
2574         elementWithFocus_ = newElement;
2575 
2576         // use newElement in the code below because element elementWithFocus_
2577         // might be changed by another thread
2578         if (newElement != null) {
2579             newElement.focus();
2580             newElement.fireEvent(Event.TYPE_FOCUS);
2581 
2582             newElement.fireEvent(Event.TYPE_FOCUS_IN);
2583         }
2584 
2585         // If a page reload happened as a result of the focus change then obviously this
2586         // element will not have the focus because its page has gone away.
2587         return this == getEnclosingWindow().getEnclosedPage();
2588     }
2589 
2590     /**
2591      * Returns the element with the focus or null if no element has the focus.
2592      * @return the element with focus or null
2593      * @see #setFocusedElement(DomElement)
2594      */
2595     public DomElement getFocusedElement() {
2596         return elementWithFocus_;
2597     }
2598 
2599     /**
2600      * <p><span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span></p>
2601      *
2602      * Sets the element with focus.
2603      * @param elementWithFocus the element with focus
2604      */
2605     public void setElementWithFocus(final DomElement elementWithFocus) {
2606         elementWithFocus_ = elementWithFocus;
2607     }
2608 
2609     /**
2610      * <p><span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span></p>
2611      * Returns the currently active element.
2612      *
2613      * @return the element that currently has focus, or the document's
2614      *         {@code body} element if no element has focus, or {@code null}
2615      *         if the document has no {@code body} element
2616      */
2617     public HtmlElement getActiveElement() {
2618         final DomElement activeElement = getFocusedElement();
2619         if (activeElement instanceof HtmlElement element) {
2620             return element;
2621         }
2622 
2623         final HtmlElement body = getBody();
2624         if (body != null) {
2625             return body;
2626         }
2627         return null;
2628     }
2629 
2630     /**
2631      * <p><span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span></p>
2632      *
2633      * <p>Returns the page's current selection ranges.</p>
2634      *
2635      * @return the page's current selection ranges
2636      */
2637     public List<SimpleRange> getSelectionRanges() {
2638         return selectionRanges_;
2639     }
2640 
2641     /**
2642      * <p><span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span></p>
2643      *
2644      * <p>Makes the specified selection range the *only* selection range on this page.</p>
2645      *
2646      * @param selectionRange the selection range
2647      */
2648     public void setSelectionRange(final SimpleRange selectionRange) {
2649         selectionRanges_.clear();
2650         selectionRanges_.add(selectionRange);
2651     }
2652 
2653     /**
2654      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2655      *
2656      * Execute a Function in the given context.
2657      *
2658      * @param function the JavaScript Function to call
2659      * @param thisObject the "this" object to be used during invocation
2660      * @param args the arguments to pass into the call
2661      * @param htmlElement the HTML element for which this script is being executed
2662      *        This element will be the context during the JavaScript execution. If null,
2663      *        the context will default to the page.
2664      * @return a ScriptResult which will contain both the current page (which may be different from
2665      *        the previous page) and a JavaScript result object.
2666      */
2667     public ScriptResult executeJavaScriptFunction(final Object function, final Object thisObject,
2668             final Object[] args, final DomNode htmlElement) {
2669         if (!getWebClient().isJavaScriptEnabled()) {
2670             return new ScriptResult(null);
2671         }
2672 
2673         final JavaScriptEngine engine = (JavaScriptEngine) getWebClient().getJavaScriptEngine();
2674         final Object result = engine.callFunction(this,
2675                                 (Function) function, (Scriptable) thisObject, args, htmlElement);
2676 
2677         return new ScriptResult(result);
2678     }
2679 
2680     private void writeObject(final ObjectOutputStream oos) throws IOException {
2681         oos.defaultWriteObject();
2682         oos.writeObject(originalCharset_ == null ? null : originalCharset_.name());
2683     }
2684 
2685     private void readObject(final ObjectInputStream ois) throws ClassNotFoundException, IOException {
2686         ois.defaultReadObject();
2687         final String charsetName = (String) ois.readObject();
2688         if (charsetName != null) {
2689             originalCharset_ = Charset.forName(charsetName);
2690         }
2691     }
2692 
2693     /**
2694      * {@inheritDoc}
2695      */
2696     @Override
2697     public void setNodeValue(final String value) {
2698         // Default behavior is to do nothing, overridden in some subclasses
2699     }
2700 
2701     /**
2702      * {@inheritDoc}
2703      */
2704     @Override
2705     public void setPrefix(final String prefix) {
2706         // Empty.
2707     }
2708 
2709     /**
2710      * {@inheritDoc}
2711      */
2712     @Override
2713     public void clearComputedStyles() {
2714         if (computedStylesCache_ != null) {
2715             computedStylesCache_.clear();
2716         }
2717     }
2718 
2719     /**
2720      * {@inheritDoc}
2721      */
2722     @Override
2723     public void clearComputedStyles(final DomElement element) {
2724         if (computedStylesCache_ != null) {
2725             computedStylesCache_.remove(element);
2726         }
2727     }
2728 
2729     /**
2730      * {@inheritDoc}
2731      */
2732     @Override
2733     public void clearComputedStylesUpToRoot(final DomElement element) {
2734         if (computedStylesCache_ != null) {
2735             computedStylesCache_.remove(element);
2736 
2737             DomNode parent = element.getParentNode();
2738             while (parent != null) {
2739                 computedStylesCache_.remove(parent);
2740                 parent = parent.getParentNode();
2741             }
2742         }
2743     }
2744 
2745     /**
2746      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2747      *
2748      * @param element the element to clear its cache
2749      * @param normalizedPseudo the pseudo attribute
2750      * @return the cached ComputedCssStyleDeclaration object or null
2751      */
2752     public ComputedCssStyleDeclaration getStyleFromCache(final DomElement element,
2753             final String normalizedPseudo) {
2754         return getCssPropertiesCache().get(element, normalizedPseudo);
2755     }
2756 
2757     /**
2758      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2759      *
2760      * Caches a ComputedCssStyleDeclaration object.
2761      * @param element the element to clear its cache
2762      * @param normalizedPseudo the pseudo attribute
2763      * @param style the ComputedCssStyleDeclaration to cache
2764      */
2765     public void putStyleIntoCache(final DomElement element, final String normalizedPseudo,
2766             final ComputedCssStyleDeclaration style) {
2767         getCssPropertiesCache().put(element, normalizedPseudo, style);
2768     }
2769 
2770     /**
2771      * <span style="color:red">INTERNAL API - SUBJECT TO CHANGE AT ANY TIME - USE AT YOUR OWN RISK.</span><br>
2772      *
2773      * @return a list of all styles from this page (&lt;style&gt; and &lt;link rel=stylesheet&gt;).
2774      *         This returns an empty list if css support is disabled in the web client options.
2775      */
2776     public List<CssStyleSheet> getStyleSheets() {
2777         final List<CssStyleSheet> styles = new ArrayList<>();
2778         if (getWebClient().getOptions().isCssEnabled()) {
2779             for (final HtmlElement htmlElement : getHtmlElementDescendants()) {
2780                 if (htmlElement instanceof HtmlStyle style) {
2781                     styles.add(style.getSheet());
2782                     continue;
2783                 }
2784 
2785                 if (htmlElement instanceof HtmlLink link) {
2786                     if (link.isStyleSheetLink()) {
2787                         styles.add(link.getSheet());
2788                     }
2789                 }
2790             }
2791         }
2792         return styles;
2793     }
2794 
2795     /**
2796      * Returns the computed styles cache for this page.
2797      *
2798      * @return the computed styles cache for this page
2799      */
2800     private ComputedStylesCache getCssPropertiesCache() {
2801         if (computedStylesCache_ == null) {
2802             computedStylesCache_ = new ComputedStylesCache();
2803 
2804             // maintain the style cache
2805             final DomHtmlAttributeChangeListenerImpl listener = new DomHtmlAttributeChangeListenerImpl();
2806             addDomChangeListener(listener);
2807             addHtmlAttributeChangeListener(listener);
2808         }
2809         return computedStylesCache_;
2810     }
2811 
2812     /**
2813      * <p>Listens for changes anywhere in the document and evicts cached computed styles whenever something relevant
2814      * changes. Note that the very lazy way of doing this (completely clearing the cache every time something happens)
2815      * results in very meager performance gains. In order to get good (but still correct) performance, we need to be
2816      * a little smarter.</p>
2817      *
2818      * <p>CSS 2.1 has the following <a href="http://www.w3.org/TR/CSS21/selector.html">selector types</a> (where "SN" is
2819      * shorthand for "the selected node"):</p>
2820      *
2821      * <ol>
2822      *   <li><em>Universal</em> (i.e. "*"): Affected by the removal of SN from the document.</li>
2823      *   <li><em>Type</em> (i.e. "div"): Affected by the removal of SN from the document.</li>
2824      *   <li><em>Descendant</em> (i.e. "div span"): Affected by changes to SN or to any of its ancestors.</li>
2825      *   <li><em>Child</em> (i.e. "div &gt; span"): Affected by changes to SN or to its parent.</li>
2826      *   <li><em>Adjacent Sibling</em> (i.e. "table + p"): Affected by changes to SN or its previous sibling.</li>
2827      *   <li><em>Attribute</em> (i.e. "div.up, div[class~=up]"): Affected by changes to an attribute of SN.</li>
2828      *   <li><em>ID</em> (i.e. "#header"): Affected by changes to the <code>id</code> attribute of SN.</li>
2829      *   <li><em>Pseudo-Elements and Pseudo-Classes</em> (i.e. "p:first-child"): Affected by changes to parent.</li>
2830      * </ol>
2831      *
2832      * <p>Together, these rules dictate that the smart (but still lazy) way of removing elements from the computed style
2833      * cache is as follows -- whenever a node changes in any way, the cache needs to be cleared of styles for nodes
2834      * which:</p>
2835      *
2836      * <ul>
2837      *   <li>are actually the same node as the node that changed</li>
2838      *   <li>are siblings of the node that changed</li>
2839      *   <li>are descendants of the node that changed</li>
2840      * </ul>
2841      *
2842      * <p>Additionally, whenever a <code>style</code> node or a <code>link</code> node
2843      * with <code>rel=stylesheet</code> is added or
2844      * removed, all elements should be removed from the computed style cache.</p>
2845      */
2846     private class DomHtmlAttributeChangeListenerImpl implements DomChangeListener, HtmlAttributeChangeListener {
2847 
2848         /**
2849          * Ctor.
2850          */
2851         DomHtmlAttributeChangeListenerImpl() {
2852             super();
2853         }
2854 
2855         /**
2856          * {@inheritDoc}
2857          */
2858         @Override
2859         public void nodeAdded(final DomChangeEvent event) {
2860             nodeChanged(event.getChangedNode(), null);
2861         }
2862 
2863         /**
2864          * {@inheritDoc}
2865          */
2866         @Override
2867         public void nodeDeleted(final DomChangeEvent event) {
2868             nodeChanged(event.getChangedNode(), null);
2869         }
2870 
2871         /**
2872          * {@inheritDoc}
2873          */
2874         @Override
2875         public void attributeAdded(final HtmlAttributeChangeEvent event) {
2876             nodeChanged(event.getHtmlElement(), event.getName());
2877         }
2878 
2879         /**
2880          * {@inheritDoc}
2881          */
2882         @Override
2883         public void attributeRemoved(final HtmlAttributeChangeEvent event) {
2884             nodeChanged(event.getHtmlElement(), event.getName());
2885         }
2886 
2887         /**
2888          * {@inheritDoc}
2889          */
2890         @Override
2891         public void attributeReplaced(final HtmlAttributeChangeEvent event) {
2892             nodeChanged(event.getHtmlElement(), event.getName());
2893         }
2894 
2895         private void nodeChanged(final DomNode changedNode, final String attribName) {
2896             // If a stylesheet was changed, all of our calculations could be off; clear the cache.
2897             if (changedNode instanceof HtmlStyle) {
2898                 clearComputedStyles();
2899                 return;
2900             }
2901             if (changedNode instanceof HtmlLink link) {
2902                 if (link.isStyleSheetLink()) {
2903                     clearComputedStyles();
2904                     return;
2905                 }
2906             }
2907 
2908             // Apparently it wasn't a stylesheet that changed; be semi-smart about what we evict and when.
2909             // null means that a node was added/removed; we always have to take care of this for the parents
2910             final boolean clearParents = attribName == null || ATTRIBUTES_AFFECTING_PARENT.contains(attribName);
2911             if (computedStylesCache_ != null) {
2912                 computedStylesCache_.nodeChanged(changedNode, clearParents);
2913             }
2914         }
2915     }
2916 
2917     /**
2918      * Cache computed styles when possible, because their calculation is very expensive.
2919      * We use a weak hash map because we don't want this cache to be the only reason
2920      * nodes are kept around in the JVM, if all other references to them are gone.
2921      */
2922     private static final class ComputedStylesCache implements Serializable {
2923         private transient WeakHashMap<DomElement, Map<String, ComputedCssStyleDeclaration>>
2924                     computedStyles_ = new WeakHashMap<>();
2925 
2926         /**
2927          * Ctor.
2928          */
2929         ComputedStylesCache() {
2930             super();
2931         }
2932 
2933         public synchronized ComputedCssStyleDeclaration get(final DomElement element,
2934                 final String normalizedPseudo) {
2935             final Map<String, ComputedCssStyleDeclaration> elementMap = computedStyles_.get(element);
2936             if (elementMap != null) {
2937                 return elementMap.get(normalizedPseudo);
2938             }
2939             return null;
2940         }
2941 
2942         public synchronized void put(final DomElement element,
2943                 final String normalizedPseudo, final ComputedCssStyleDeclaration style) {
2944             final Map<String, ComputedCssStyleDeclaration>
2945                     elementMap = computedStyles_.computeIfAbsent(element, k -> new WeakHashMap<>());
2946             elementMap.put(normalizedPseudo, style);
2947         }
2948 
2949         public synchronized void nodeChanged(final DomNode changed, final boolean clearParents) {
2950             final Iterator<Map.Entry<DomElement, Map<String, ComputedCssStyleDeclaration>>>
2951                     i = computedStyles_.entrySet().iterator();
2952             while (i.hasNext()) {
2953                 final Map.Entry<DomElement, Map<String, ComputedCssStyleDeclaration>> entry = i.next();
2954                 final DomElement node = entry.getKey();
2955                 if (changed == node
2956                     || changed.getParentNode() == node.getParentNode()
2957                     || changed.isAncestorOf(node)
2958                     || clearParents && node.isAncestorOf(changed)) {
2959                     i.remove();
2960                 }
2961             }
2962 
2963             // maybe this is a better solution but I have to think a bit more about this
2964             //
2965             //            if (computedStyles_.isEmpty()) {
2966             //                return;
2967             //            }
2968             //
2969             //            // remove all siblings
2970             //            DomNode parent = changed.getParentNode();
2971             //            if (parent != null) {
2972             //                for (DomNode sibling : parent.getChildNodes()) {
2973             //                    computedStyles_.remove(sibling.getScriptableObject());
2974             //                }
2975             //
2976             //                if (clearParents) {
2977             //                    // remove all parents
2978             //                    while (parent != null) {
2979             //                        computedStyles_.remove(parent.getScriptableObject());
2980             //                        parent = parent.getParentNode();
2981             //                    }
2982             //                }
2983             //            }
2984             //
2985             //            // remove changed itself and all descendants
2986             //            computedStyles_.remove(changed.getScriptableObject());
2987             //            for (DomNode descendant : changed.getDescendants()) {
2988             //                computedStyles_.remove(descendant.getScriptableObject());
2989             //            }
2990         }
2991 
2992         public synchronized void clear() {
2993             computedStyles_.clear();
2994         }
2995 
2996         public synchronized Map<String, ComputedCssStyleDeclaration> remove(final DomNode element) {
2997             return computedStyles_.remove(element);
2998         }
2999 
3000         private void readObject(final ObjectInputStream in) throws IOException, ClassNotFoundException {
3001             in.defaultReadObject();
3002             computedStyles_ = new WeakHashMap<>();
3003         }
3004     }
3005 
3006     private static final class MappedElementIndexEntry implements Serializable {
3007         private final ArrayList<DomElement> elements_;
3008         private boolean sorted_;
3009 
3010         MappedElementIndexEntry() {
3011             // we do not expect to many elements having the same id/name
3012             elements_ = new ArrayList<>(2);
3013             sorted_ = true;
3014         }
3015 
3016         void add(final DomElement element) {
3017             if (elements_.indexOf(element) == -1) {
3018                 elements_.add(element);
3019                 sorted_ = elements_.size() < 2;
3020             }
3021         }
3022 
3023         DomElement first() {
3024             if (elements_.isEmpty()) {
3025                 return null;
3026             }
3027 
3028             if (sorted_) {
3029                 return elements_.get(0);
3030             }
3031 
3032             elements_.sort(DOCUMENT_POSITION_COMPERATOR);
3033             sorted_ = true;
3034 
3035             return elements_.get(0);
3036         }
3037 
3038         List<DomElement> elements() {
3039             if (sorted_) {
3040                 return elements_;
3041             }
3042 
3043             elements_.sort(DOCUMENT_POSITION_COMPERATOR);
3044             sorted_ = true;
3045 
3046             return elements_;
3047         }
3048 
3049         void remove(final DomElement element) {
3050             elements_.remove(element);
3051             sorted_ = elements_.size() < 2;
3052         }
3053     }
3054 }