[py] run the BiDi tests against the Grid server

.github/workflows/ci-python.yml

@@ -74,6 +74,24 @@ jobs:
         --local_test_jobs 1
         --target_pattern_file=remote-targets.txt
 
+  remote-bidi-tests:
+    name: Remote BiDi Tests
+    uses: ./.github/workflows/bazel.yml
+    with:
+      name: Integration Tests Remote BiDi
+      needs-display: true
+      rerun-with-debug: true
+      download-artifact-name: targets
+      run: >
+        [ -s bazel-targets-py.txt ] || echo //py/... > bazel-targets-py.txt;
+        { echo "tests(//py:test-remote-bidi) intersect set("; cat bazel-targets-py.txt; echo ")"; } > query.txt;
+        bazel query --query_file=query.txt --output=label > remote-bidi-targets.txt;
+        ./scripts/github-actions/bazel-test-if-targets.sh
+        --keep_going
+        --flaky_test_attempts 3
+        --local_test_jobs 1
+        --target_pattern_file=remote-bidi-targets.txt
+
   browser-tests:
     name: Browser Tests
     uses: ./.github/workflows/bazel.yml

py/BUILD.bazel

@@ -1124,6 +1124,48 @@ FEATURE_SUITE_DEFS = {
     if BROWSER_TESTS[browser].get("bidi", False)
 ]
 
+# Generate test-<browser>-remote-bidi targets (chrome and firefox only).
+# Deliberately not folded into the test-<browser>-remote aggregate: BiDi over Grid's
+# websocket proxy is new coverage, so keep it out of the established remote job until
+# it has proven stable.
+[
+    py_test_suite(
+        name = "test-%s-remote-bidi" % browser,
+        size = "large",
+        srcs = BIDI_TESTS + BIDI_IMPLEMENTATIONS,
+        args = [
+            "--instafail",
+            "--bidi",
+            "--remote",
+        ] + BROWSERS[browser]["args"],
+        data = BROWSERS[browser]["data"] + [
+            ":java-location",
+            "//java/src/org/openqa/selenium/grid:selenium_server_deploy.jar",
+            "@bazel_tools//tools/jdk:current_java_runtime",
+        ],
+        env = {
+            "SE_BAZEL_JAVA_LOCATION": "$(rootpath :java-location)",
+        },
+        env_inherit = ["DISPLAY"],
+        tags = ["no-sandbox"] + BROWSERS[browser]["tags"] + [
+            "remote",
+            "%s-remote" % browser,
+        ],
+        target_compatible_with = BROWSERS[browser]["target_compatible_with"],
+        test_suffix = "%s-remote-bidi" % browser,
+        deps = [
+            ":bidi_protocol",  # bidi/protocol_tests.py imports the generated _bidi layer directly
+            ":common_alert",  # bidi/browsing_context_tests.py calls EC.alert_is_present()
+            ":init-tree",
+            ":webserver",
+        ] + BROWSER_TESTS[browser]["deps"] + TEST_DEPS,
+    )
+    for browser in [
+        "chrome",
+        "firefox",
+    ]
+]
+
 # Generate test-<browser>-remote-common targets (chrome and firefox only)
 [
     py_test_suite(
@@ -1151,11 +1193,11 @@ FEATURE_SUITE_DEFS = {
             "SE_BAZEL_JAVA_LOCATION": "$(rootpath :java-location)",
         },
         env_inherit = ["DISPLAY"],
-        tags = [
-            "no-sandbox",
+        tags = ["no-sandbox"] + BROWSERS[browser]["tags"] + [
             "remote",
             "%s-remote" % browser,
         ],
+        target_compatible_with = BROWSERS[browser]["target_compatible_with"],
         test_suffix = "%s-remote" % browser,
         deps = [
             ":init-tree",
@@ -1187,11 +1229,11 @@ FEATURE_SUITE_DEFS = {
             "SE_BAZEL_JAVA_LOCATION": "$(rootpath :java-location)",
         },
         env_inherit = ["DISPLAY"],
-        tags = [
-            "no-sandbox",
+        tags = ["no-sandbox"] + BROWSERS[browser]["tags"] + [
             "remote",
             "%s-remote" % browser,
         ],
+        target_compatible_with = BROWSERS[browser]["target_compatible_with"],
         test_suffix = "%s-remote-actions" % browser,
         deps = [
             ":init-tree",
@@ -1223,11 +1265,11 @@ FEATURE_SUITE_DEFS = {
             "SE_BAZEL_JAVA_LOCATION": "$(rootpath :java-location)",
         },
         env_inherit = ["DISPLAY"],
-        tags = [
-            "no-sandbox",
+        tags = ["no-sandbox"] + BROWSERS[browser]["tags"] + [
             "remote",
             "%s-remote" % browser,
         ],
+        target_compatible_with = BROWSERS[browser]["target_compatible_with"],
         test_suffix = "%s-remote-%s" % (browser, feature),
         deps = [
             ":init-tree",
@@ -1263,6 +1305,16 @@ test_suite(
     ],
 )
 
+# Kept out of :test-remote so BiDi-over-Grid runs as its own CI job while it is new.
+test_suite(
+    name = "test-remote-bidi",
+    tags = ["remote"],
+    tests = [
+        ":test-chrome-remote-bidi",
+        ":test-firefox-remote-bidi",
+    ],
+)
+
 py_test_suite(
     name = "test-webkitgtk-common",
     size = "large",

py/TESTING.md

@@ -43,6 +43,13 @@ bazel test //py:test/selenium/webdriver/common/window_tests-chrome
 # With BiDi protocol
 bazel test //py:test-chrome-bidi
 
+# Against a Grid server (chrome and firefox only). The suite starts its own
+# Selenium standalone server and talks to it with webdriver.Remote.
+bazel test //py:test-chrome-remote       # classic protocol
+bazel test //py:test-chrome-remote-bidi  # BiDi over Grid's websocket proxy
+bazel test //py:test-remote              # every classic remote suite
+bazel test //py:test-remote-bidi         # every BiDi remote suite
+
 # Test filters
 bazel test //py/... --test_tag_filters=chrome
 
@@ -96,6 +103,12 @@ pytest py/test/selenium/webdriver/chrome/ --driver chrome --headless -v
 ```
 > **Note:**
 > For running BiDi tests, use the `--bidi` flag.
+>
+> To run against a Grid server, add `--remote`. It starts a Selenium standalone
+> server and runs the tests through `webdriver.Remote`, so it needs the Grid jar
+> built first (`bazel build //java/src/org/openqa/selenium/grid:selenium_server_deploy.jar`).
+> `--bidi --remote` combine: Grid rewrites the `webSocketUrl` capability to its own
+> `/session/<id>/se/bidi` endpoint and proxies the socket through to the node.
 
 ## Skipping Tests
 
@@ -146,6 +159,7 @@ modules, you will find the main fixtures in `conftest.py`:
 | `webserver` | Test HTTP server reference |
 | `clean_driver` | Fresh driver without parametrization |
 | `clean_options` | Fresh browser options instance |
+| `headless` | Whether the browser was started headless, for tests asserting on behavior a headless browser does not model (e.g. window focus) |
 
 ## Test Organization
 

py/conftest.py

@@ -490,6 +490,45 @@ def stop_driver(self):
             driver_to_stop.quit()
 
 
+def _skip_unless_remote(request, is_remote):
+    """Skip tests living under ``test/selenium/webdriver/remote/`` unless ``--remote`` is set."""
+    if request.node.path.parts[-2] == "remote" and not is_remote:
+        pytest.skip("Remote tests require the --remote flag")
+
+
+def _apply_xfail_markers(request, driver_class, is_remote):
+    """Honor the ``xfail_<driver>`` / ``xfail_remote`` markers for the driver under test.
+
+    Raises the resulting pytest outcome directly, so callers need no follow-up branching.
+
+    Args:
+        request: The pytest request for the test being set up.
+        driver_class: Name of the driver under test, used to build the marker name.
+        is_remote: Whether the run targets a remote server, enabling ``xfail_remote``.
+    """
+    marker = request.node.get_closest_marker(f"xfail_{driver_class.lower()}")
+    if marker is None and is_remote:
+        marker = request.node.get_closest_marker("xfail_remote")
+    if marker is None:
+        return
+
+    kwargs = dict(marker.kwargs)
+
+    # A falsy condition means the xfail does not apply to this run.
+    condition = kwargs.pop("condition", True)
+    if callable(condition):
+        condition = condition()
+    if not condition:
+        return
+
+    # run=False means skip outright rather than run and expect failure.
+    if not kwargs.pop("run", True):
+        pytest.skip()
+
+    kwargs.pop("raises", None)
+    pytest.xfail(**kwargs)
+
+
 @pytest.fixture
 def driver(request, server):
     global selenium_driver
@@ -504,34 +543,14 @@ def driver(request, server):
     if not selenium_driver.is_platform_valid:
         pytest.skip(f"{driver_class} tests can only run on {selenium_driver.exe_platform}")
 
-    # skip tests in the 'remote' directory if not running with --remote flag
-    if request.node.path.parts[-2] == "remote" and not selenium_driver.is_remote:
-        pytest.skip("Remote tests require the --remote flag")
+    _skip_unless_remote(request, selenium_driver.is_remote)
 
     # skip tests for drivers that don't support BiDi when --bidi is enabled
     if selenium_driver.bidi:
         if driver_class.lower() not in selenium_driver.supported_bidi_drivers:
             pytest.skip(f"{driver_class} does not support BiDi")
 
-    # conditionally mark tests as expected to fail based on driver
-    marker = request.node.get_closest_marker(f"xfail_{driver_class.lower()}")
-    # Also check for xfail_remote when running with --remote
-    if marker is None and selenium_driver.is_remote:
-        marker = request.node.get_closest_marker("xfail_remote")
-    if marker is not None:
-        kwargs = dict(marker.kwargs)
-        # Support condition kwarg - if condition is False, skip the xfail
-        condition = kwargs.pop("condition", True)
-        if callable(condition):
-            condition = condition()
-        if condition:
-            if "run" in kwargs:
-                if not kwargs["run"]:
-                    pytest.skip()
-                    yield
-                    return
-            kwargs.pop("raises", None)
-            pytest.xfail(**kwargs)
+    _apply_xfail_markers(request, driver_class, selenium_driver.is_remote)
 
     # For BiDi tests, only restart driver when explicitly marked as needing fresh driver.
     # Tests marked with @pytest.mark.needs_fresh_driver get full driver restart for test isolation.
@@ -600,6 +619,17 @@ def load(self, name):
     return Pages()
 
 
+@pytest.fixture
+def headless(request):
+    """Whether the browser under test was started headless.
+
+    Without a window manager some behavior is not modelled at all - notably window
+    focus, which headless Chromium reports as always held by the current window - so
+    tests that assert on it need to know which mode they are running in.
+    """
+    return bool(request.config.option.headless)
+
+
 @pytest.fixture(autouse=True, scope="session")
 def server(request):
     is_remote = request.config.getoption("remote")
@@ -676,20 +706,7 @@ def clean_driver(request):
         raise Exception("This test requires a --driver to be specified.")
     driver_reference = getattr(webdriver, driver_class)
 
-    # conditionally mark tests as expected to fail based on driver
-    marker = request.node.get_closest_marker(f"xfail_{driver_class.lower()}")
-    # Also check for xfail_remote when running with --remote
-    if marker is None and request.config.getoption("remote"):
-        marker = request.node.get_closest_marker("xfail_remote")
-    if marker is not None:
-        kwargs = dict(marker.kwargs)
-        if "run" in kwargs:
-            if not kwargs["run"]:
-                pytest.skip()
-                yield
-                return
-        kwargs.pop("raises", None)
-        pytest.xfail(**kwargs)
+    _apply_xfail_markers(request, driver_class, request.config.getoption("remote"))
 
     yield driver_reference
 
@@ -721,10 +738,7 @@ def firefox_options(request):
     if driver_class != "firefox":
         pytest.skip(f"This test requires Firefox. Got {driver_class}")
 
-    # skip tests in the 'remote' directory if not running with --remote flag
-    is_remote = request.config.getoption("remote")
-    if request.node.path.parts[-2] == "remote" and not is_remote:
-        pytest.skip("Remote tests require the --remote flag")
+    _skip_unless_remote(request, request.config.getoption("remote"))
 
     options = Driver.clean_options("firefox", request)
 
@@ -742,10 +756,7 @@ def chromium_options(request):
     if driver_class not in ("chrome", "edge"):
         pytest.skip(f"This test requires Chrome or Edge. Got {driver_class}")
 
-    # skip tests in the 'remote' directory if not running with --remote flag
-    is_remote = request.config.getoption("remote")
-    if request.node.path.parts[-2] == "remote" and not is_remote:
-        pytest.skip("Remote tests require the --remote flag")
+    _skip_unless_remote(request, request.config.getoption("remote"))
 
     options = Driver.clean_options(driver_class, request)
 

py/test/selenium/webdriver/common/_bidi/browsing_context_tests.py

@@ -21,6 +21,7 @@
 
 import pytest
 
+from selenium.common.exceptions import TimeoutException
 from selenium.webdriver.common._bidi.browser import Browser
 from selenium.webdriver.common._bidi.browsing_context import (
     BoxClipRectangle,
@@ -195,15 +196,25 @@ def test_close_tab(driver):
     bc.close(context=tab1)
 
 
-def test_activate_browsing_context(driver):
+def test_activate_browsing_context(driver, headless):
     bc = BrowsingContext(driver)
     window1 = driver.current_window_handle
     window2 = bc.create(type=CreateType.WINDOW).context
 
-    assert not driver.execute_script("return document.hasFocus();")
+    # Focus is handed over asynchronously, so poll rather than reading once. Headless
+    # Chromium has no window manager and never hands it over at all, so skip there
+    # instead of failing; headless Firefox does, and keeps the full assertions.
+    try:
+        WebDriverWait(driver, 5).until_not(lambda d: d.execute_script("return document.hasFocus();"))
+    except TimeoutException:
+        if headless:
+            bc.close(context=window2)
+            pytest.skip("this headless browser does not move focus between windows")
+        raise
 
     bc.activate(context=window1)
 
+    WebDriverWait(driver, 5).until(lambda d: d.execute_script("return document.hasFocus();"))
     assert driver.execute_script("return document.hasFocus();")
 
     bc.close(context=window2)

py/test/selenium/webdriver/common/_bidi/input_tests.py

@@ -382,6 +382,10 @@ def test_wheel_scroll_with_duration(driver, pages):
     )
     Input(driver).perform_actions(context=driver.current_window_handle, actions=[wheel_actions])
 
+    # duration spreads the scroll over 500ms, so it can still be in flight when
+    # perform_actions returns; Firefox was observed one pixel short.
+    WebDriverWait(driver, 5).until(lambda d: d.execute_script("return window.pageYOffset;") == 100)
+
     assert driver.execute_script("return window.pageYOffset;") == 100
 
 

py/test/selenium/webdriver/common/bidi/browsing_context_tests.py

@@ -22,6 +22,7 @@
 
 import pytest
 
+from selenium.common.exceptions import TimeoutException
 from selenium.webdriver.common.bidi.browsing_context import ReadinessState
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.window import WindowTypes
@@ -205,17 +206,27 @@ def test_close_tab(driver):
     driver.browsing_context.close(tab1)
 
 
-def test_activate_browsing_context(driver):
+def test_activate_browsing_context(driver, headless):
     """Test activating a browsing context."""
     window1 = driver.current_window_handle
     # 2nd window is focused
     window2 = driver.browsing_context.create(type=WindowTypes.WINDOW)
 
+    # Focus is handed over asynchronously, so poll rather than reading once. Headless
+    # Chromium has no window manager and never hands it over at all, so skip there
+    # instead of failing; headless Firefox does, and keeps the full assertions.
     # We did not switch the driver, so we are running the script to check focus on 1st window
-    assert not driver.execute_script("return document.hasFocus();")
+    try:
+        WebDriverWait(driver, 5).until_not(lambda d: d.execute_script("return document.hasFocus();"))
+    except TimeoutException:
+        if headless:
+            driver.browsing_context.close(window2)
+            pytest.skip("this headless browser does not move focus between windows")
+        raise
 
     driver.browsing_context.activate(window1)
 
+    WebDriverWait(driver, 5).until(lambda d: d.execute_script("return document.hasFocus();"))
     assert driver.execute_script("return document.hasFocus();")
 
     # Clean up