fara/browser/page_script.js

var MultimodalWebSurfer = MultimodalWebSurfer || (function() {
    let nextLabel = 10;

    let roleMapping = {
        "a": "link",
        "area": "link",
        "button": "button",
        "input, type=button": "button",
        "input, type=checkbox": "checkbox",
        "input, type=email": "textbox",
        "input, type=number": "spinbutton",
        "input, type=radio": "radio",
        "input, type=range": "slider",
        "input, type=reset": "button",
        "input, type=search": "searchbox",
        "input, type=submit": "button",
        "input, type=tel": "textbox",
        "input, type=text": "textbox",
        "input, type=url": "textbox",
        "search": "search",
        "select": "combobox",
        "option": "option",
        "textarea": "textbox"
    };

    let getCursor = function (elm) {
        return window.getComputedStyle(elm)["cursor"];
    };

    let isVisible = function (element) {
        return !!(element.offsetWidth || element.offsetHeight || element.getClientRects().length);
    };

    /**
     * Finds interactive elements in the regular DOM (excluding Shadow DOM)
     * Looks for elements that are:
     * 1. Standard interactive elements (inputs, buttons, links)
     * 2. Elements with ARIA roles indicating interactivity
     * 3. Elements with cursor styles suggesting interactivity
     *
     * @returns {Array} Array of DOM elements that are deemed interactive
     */
    let getInteractiveElementsNoShaddow = function () {
        let results = []
        let roles = ["scrollbar", "searchbox", "slider", "spinbutton", "switch", "tab", "treeitem", "button", "checkbox", "gridcell", "link", "menuitem", "menuitemcheckbox", "menuitemradio", "option", "progressbar", "radio", "textbox", "combobox", "menu", "tree", "treegrid", "grid", "listbox", "radiogroup", "widget"];
        let inertCursors = ["auto", "default", "none", "text", "vertical-text", "not-allowed", "no-drop"];

        // Get the main interactive elements
        let nodeList = document.querySelectorAll("input, select, textarea, button, [href], [onclick], [contenteditable], [tabindex]:not([tabindex='-1'])");
        for (let i = 0; i < nodeList.length; i++) { // Copy to something mutable
            // make sure not disabled and visible
            if (nodeList[i].disabled || !isVisible(nodeList[i])) {
                continue;
            }
            results.push(nodeList[i]);
        }

        // Anything not already included that has a suitable role
        nodeList = document.querySelectorAll("[role]");
        for (let i = 0; i < nodeList.length; i++) { // Copy to something mutable
            // make sure not disabled and visible
            if (nodeList[i].disabled || !isVisible(nodeList[i])) {
                continue;
            }
            if (results.indexOf(nodeList[i]) == -1) {
                let role = nodeList[i].getAttribute("role");
                if (roles.indexOf(role) > -1) {
                    results.push(nodeList[i]);
                }
            }
        }

        // Any element that changes the cursor to something implying interactivity
        nodeList = document.querySelectorAll("*");
        for (let i = 0; i < nodeList.length; i++) {
            let node = nodeList[i];
            if (node.disabled || !isVisible(node)) {
                continue;
            }

            // Cursor is default, or does not suggest interactivity
            let cursor = getCursor(node);
            if (inertCursors.indexOf(cursor) >= 0) {
                continue;
            }

            // Move up to the first instance of this cursor change
            let parent = node.parentNode;
            while (parent && getCursor(parent) == cursor) {
                node = parent;
                parent = node.parentNode;
            }

            // Add the node if it is new
            if (results.indexOf(node) == -1) {
                results.push(node);
            }
        }

        return results;
    };

    /**
     * Recursively gathers elements matching specified roles from both regular DOM and Shadow DOM
     * @param {Array} roles - Array of role selectors to match
     * @param {Document|ShadowRoot} root - Root element to start search from
     * @returns {Array} Array of matching elements
     */
    function gatherAllElements(roles, root = document) {
        const elements = [];
        const stack = [root];
        const selector = roles.join(",");

        while (stack.length > 0) {
            const currentRoot = stack.pop();

            // Add elements at current level
            elements.push(...Array.from(currentRoot.querySelectorAll(selector)));

            // Add shadow roots to stack
            currentRoot.querySelectorAll("*").forEach(el => {
                if (el.shadowRoot && el.shadowRoot.mode === "open") {
                    stack.push(el.shadowRoot);
                }
            });
        }

        return elements;
    }

    /**
     * Gets all interactive elements from both regular DOM and Shadow DOM
     * Filters elements to ensure they are visible and accessible
     * @returns {Array} Array of interactive elements
     */
    let getInteractiveElements = function () {
        // Get all elements that are interactive without the shadow DOM
        const interactive_roles = ["input", "option", "select", "textarea", "button", "href", "onclick", "contenteditable", "tabindex:not([tabindex='-1'])"];

        let results = [];

        let elements_no_shaddow = getInteractiveElementsNoShaddow();
        for (let i = 0; i < elements_no_shaddow.length; i++) {
            if (results.indexOf(elements_no_shaddow[i]) == -1) {
                // check if it has a rect
                let rects = elements_no_shaddow[i].getClientRects();
                for (const rect of rects) {
                    let x = rect.left + rect.width / 2;
                    let y = rect.top + rect.height / 2;
                    if (isTopmost(elements_no_shaddow[i], x, y)) {
                        results.push(elements_no_shaddow[i]);
                        break;
                    }
                }
            }
        }

        // From the shadow DOM get all interactive elements and options that are not in the no shadow list
        let elements_all = gatherAllElements(interactive_roles);

        // Filter and process interactive elements
        elements_all.forEach(element => {
            // if file, auto add
            if (element.tagName.toLowerCase() === "input" && element.getAttribute("type") == "file") {
                results.push(element);
                return;
            }
            // if option  auto add
            if (element.tagName.toLowerCase() === "option") {
                results.push(element);
                return;
            }
            if (element.disabled || !isVisible(element)) {
                return;
            }

            if (interactive_roles.includes(element.tagName.toLowerCase())) {
                results.push(element);
            }
        });

        return results;
    };

    /**
     * Assigns unique identifiers to interactive elements
     * @param {Array} elements - Array of elements to label
     * @returns {Array} Updated array of interactive elements
     */
    let labelElements = function (elements) {
        for (let i = 0; i < elements.length; i++) {
            if (!elements[i].hasAttribute("__elementId")) {
                elements[i].setAttribute("__elementId", "" + (nextLabel++));
            }
        }
        return getInteractiveElements();
    };

    /**
     * Checks if an element is the topmost element at given coordinates
     * @param {Element} element - Element to check
     * @param {number} x - X coordinate
     * @param {number} y - Y coordinate
     * @returns {boolean} True if element is topmost at coordinates
     */
    let isTopmost = function (element, x, y) {
        let hit = document.elementFromPoint(x, y);

        // Hack to handle elements outside the viewport
        if (hit === null) {
            return true;
        }

        while (hit) {
            if (hit == element) return true;
            hit = hit.parentNode;
        }
        return false;
    };

    let getFocusedElementId = function () {
        let elm = document.activeElement;
        while (elm) {
            if (elm.hasAttribute && elm.hasAttribute("__elementId")) {
                return elm.getAttribute("__elementId");
            }
            elm = elm.parentNode;
        }
        return null;
    };

    let trimmedInnerText = function (element) {
        if (!element) {
            return "";
        }
        let text = element.innerText;
        if (!text) {
            return "";
        }
        return text.trim();
    };

    let getApproximateAriaName = function (element) {
        if (element.hasAttribute("aria-label")) {
            return element.getAttribute("aria-label");
        }

        // check if element has span that is called label and grab the inner text
        if (element.querySelector("span.label")) {
            return element.querySelector("span.label").innerText;
        }

        // Check for aria labels
        if (element.hasAttribute("aria-labelledby")) {
            let buffer = "";
            let ids = element.getAttribute("aria-labelledby").split(" ");
            for (let i = 0; i < ids.length; i++) {
                let label = document.getElementById(ids[i]);
                if (label) {
                    buffer = buffer + " " + trimmedInnerText(label);
                }
            }
            return buffer.trim();
        }

        if (element.hasAttribute("aria-label")) {
            return element.getAttribute("aria-label");
        }

        // Check for labels
        if (element.hasAttribute("id")) {
            let label_id = element.getAttribute("id");
            let label = "";
            try {
                // Escape special characters in the ID
                let escaped_id = CSS.escape(label_id);
                let labels = document.querySelectorAll(`label[for="${escaped_id}"]`);
                for (let j = 0; j < labels.length; j++) {
                    label += labels[j].innerText + " ";
                }
                label = label.trim();
                if (label != "") {
                    return label;
                }
            } catch (e) {
                console.warn("Error finding label for element:", e);
            }
        }

        if (element.hasAttribute("name")) {
            return element.getAttribute("name");
        }

        if (element.parentElement && element.parentElement.tagName == "LABEL") {
            return element.parentElement.innerText;
        }

        // Check for alt text or titles
        if (element.hasAttribute("alt")) {
            return element.getAttribute("alt")
        }

        if (element.hasAttribute("title")) {
            return element.getAttribute("title")
        }

        return trimmedInnerText(element);
    };

    let getApproximateAriaRole = function (element) {
        let tag = element.tagName.toLowerCase();
        if (tag == "input" && element.hasAttribute("type")) {
            tag = tag + ", type=" + element.getAttribute("type");
        }

        if (element.hasAttribute("role")) {
            return [element.getAttribute("role"), tag];
        }
        else if (tag in roleMapping) {
            return [roleMapping[tag], tag];
        }
        else {
            return ["", tag];
        }
    };

    /**
     * Gets information about all interactive elements including their:
     * - Position and dimensions
     * - ARIA roles and names
     * - Tag names
     * - Scrollability
     *
     * @returns {Object} Map of element IDs to their properties
     */
    let getInteractiveRects = function () {
        let elements = labelElements(getInteractiveElements());
        let results = {};
        for (let i = 0; i < elements.length; i++) {
            let key = elements[i].getAttribute("__elementId");
            let rects = elements[i].getBoundingClientRect();

            // Skip options unless their select is focused
            if (elements[i].tagName.toLowerCase() === "option") {

                let select_focused = false;
                let select = elements[i].closest("select");
                if (select && select.hasAttribute("__elementId") &&
                    getFocusedElementId() === select.getAttribute("__elementId")) {
                    select_focused = true;
                }
                // check if option is visible without select being focused
                let option_visible = false;
                if (isVisible(elements[i])) {
                    option_visible = true;
                }
                // check if select is expanded even if not focused
                let select_expanded = false;
                if (select && select.hasAttribute("open")) {
                    select_expanded = true;
                }
                if (!(select_focused || option_visible || select_expanded)) {
                    continue;
                }
            }

            let ariaRole = getApproximateAriaRole(elements[i]);
            let ariaName = getApproximateAriaName(elements[i]);
            let vScrollable = elements[i].scrollHeight - elements[i].clientHeight >= 1;

            let record = {
                "tag_name": ariaRole[1],
                "role": ariaRole[0],
                "aria-name": ariaName,
                "v-scrollable": vScrollable,
                "rects": []
            };

            if (rects.length > 0) {
                for (const rect of rects) {
                    let x = rect.left + rect.width / 2;
                    let y = rect.top + rect.height / 2;
                    if (isTopmost(elements[i], x, y)) {
                        record["rects"].push(JSON.parse(JSON.stringify(rect)));
                    }
                }
            }
            else {
                record["rects"].push(JSON.parse(JSON.stringify(rects)));
            }

            results[key] = record;
        }
        return results;
    };

    /**
     * Gets current viewport information including dimensions and scroll positions
     * @returns {Object} Viewport properties
     */
    let getVisualViewport = function () {
        let vv = window.visualViewport;
        let de = document.documentElement;
        return {
            "height": vv ? vv.height : 0,
            "width": vv ? vv.width : 0,
            "offsetLeft": vv ? vv.offsetLeft : 0,
            "offsetTop": vv ? vv.offsetTop : 0,
            "pageLeft": vv ? vv.pageLeft : 0,
            "pageTop": vv ? vv.pageTop : 0,
            "scale": vv ? vv.scale : 0,
            "clientWidth": de ? de.clientWidth : 0,
            "clientHeight": de ? de.clientHeight : 0,
            "scrollWidth": de ? de.scrollWidth : 0,
            "scrollHeight": de ? de.scrollHeight : 0
        };
    };

    let _getMetaTags = function () {
        let meta = document.querySelectorAll("meta");
        let results = {};
        for (let i = 0; i < meta.length; i++) {
            let key = null;
            if (meta[i].hasAttribute("name")) {
                key = meta[i].getAttribute("name");
            }
            else if (meta[i].hasAttribute("property")) {
                key = meta[i].getAttribute("property");
            }
            else {
                continue;
            }
            if (meta[i].hasAttribute("content")) {
                results[key] = meta[i].getAttribute("content");
            }
        }
        return results;
    };

    let _getJsonLd = function () {
        let jsonld = [];
        let scripts = document.querySelectorAll('script[type="application/ld+json"]');
        for (let i = 0; i < scripts.length; i++) {
            jsonld.push(scripts[i].innerHTML.trim());
        }
        return jsonld;
    };

    // From: https://www.stevefenton.co.uk/blog/2022/12/parse-microdata-with-javascript/
    let _getMicrodata = function () {
        function sanitize(input) {
            return input.replace(/\s/gi, ' ').trim();
        }

        function addValue(information, name, value) {
            if (information[name]) {
                if (typeof information[name] === 'array') {
                    information[name].push(value);
                } else {
                    const arr = [];
                    arr.push(information[name]);
                    arr.push(value);
                    information[name] = arr;
                }
            } else {
                information[name] = value;
            }
        }

        function traverseItem(item, information) {
            const children = item.children;

            for (let i = 0; i < children.length; i++) {
                const child = children[i];

                if (child.hasAttribute('itemscope')) {
                    if (child.hasAttribute('itemprop')) {
                        const itemProp = child.getAttribute('itemprop');
                        const itemType = child.getAttribute('itemtype');

                        const childInfo = {
                            itemType: itemType
                        };

                        traverseItem(child, childInfo);

                        itemProp.split(' ').forEach(propName => {
                            addValue(information, propName, childInfo);
                        });
                    }

                } else if (child.hasAttribute('itemprop')) {
                    const itemProp = child.getAttribute('itemprop');
                    itemProp.split(' ').forEach(propName => {
                        if (propName === 'url') {
                            addValue(information, propName, child.href);
                        } else {
                            addValue(information, propName, sanitize(child.getAttribute("content") || child.content || child.textContent || child.src || ""));
                        }
                    });
                    traverseItem(child, information);
                } else {
                    traverseItem(child, information);
                }
            }
        }

        const microdata = [];

        document.querySelectorAll("[itemscope]").forEach(function (elem, i) {
            const itemType = elem.getAttribute('itemtype');
            const information = {
                itemType: itemType
            };
            traverseItem(elem, information);
            microdata.push(information);
        });

        return microdata;
    };

    let getPageMetadata = function () {
        let jsonld = _getJsonLd();
        let metaTags = _getMetaTags();
        let microdata = _getMicrodata();
        let results = {}
        if (jsonld.length > 0) {
            try {
                results["jsonld"] = JSON.parse(jsonld);
            }
            catch (e) {
                results["jsonld"] = jsonld;
            }
        }
        if (microdata.length > 0) {
            results["microdata"] = microdata;
        }
        for (let key in metaTags) {
            if (metaTags.hasOwnProperty(key)) {
                results["meta_tags"] = metaTags;
                break;
            }
        }
        return results;
    };

    /**
     * Extracts all visible text content from the viewport
     * Preserves basic formatting with newlines for block elements
     * @returns {string} Visible text content
     */
    let getVisibleText = function () {
        // Get the window's current viewport boundaries
        const viewportHeight = window.innerHeight || document.documentElement.clientHeight;
        const viewportWidth = window.innerWidth || document.documentElement.clientWidth;

        let textInView = "";
        const walker = document.createTreeWalker(
            document.body,
            NodeFilter.SHOW_TEXT,
            null,
            false
        );

        while (walker.nextNode()) {
            const textNode = walker.currentNode;
            // Create a range to retrieve bounding rectangles of the current text node
            const range = document.createRange();
            range.selectNodeContents(textNode);

            const rects = range.getClientRects();

            // Check if any rect is inside (or partially inside) the viewport
            for (const rect of rects) {
                const isVisible =
                    rect.width > 0 &&
                    rect.height > 0 &&
                    rect.bottom >= 0 &&
                    rect.right >= 0 &&
                    rect.top <= viewportHeight &&
                    rect.left <= viewportWidth;

                if (isVisible) {
                    textInView += textNode.nodeValue.replace(/\s+/g, " ");
                    // Is the parent a block element?
                    if (textNode.parentNode) {
                        const parent = textNode.parentNode;
                        const style = window.getComputedStyle(parent);
                        if (["inline", "hidden", "none"].indexOf(style.display) === -1) {
                            textInView += "\n";
                        }
                    }
                    break; // No need to check other rects once found visible
                }
            }
        }

        // Remove blank lines from textInView
        textInView = textInView.replace(/^\s*\n/gm, "").trim().replace(/\n+/g, "\n");
        return textInView;
    };

    // Public API
    return {
        getInteractiveRects: getInteractiveRects,
        getVisualViewport: getVisualViewport,
        getFocusedElementId: getFocusedElementId,
        getPageMetadata: getPageMetadata,
    };
})();