diff --git a/.github/workflows/ci-python.yml b/.github/workflows/ci-python.yml index 805c07e9ba1a8..288e97cea731f 100644 --- a/.github/workflows/ci-python.yml +++ b/.github/workflows/ci-python.yml @@ -74,6 +74,24 @@ jobs: --local_test_jobs 1 --target_pattern_file=remote-targets.txt + remote-bidi-tests: + name: Remote BiDi Tests + uses: ./.github/workflows/bazel.yml + with: + name: Integration Tests Remote BiDi + needs-display: true + rerun-with-debug: true + download-artifact-name: targets + run: > + [ -s bazel-targets-py.txt ] || echo //py/... > bazel-targets-py.txt; + { echo "tests(//py:test-remote-bidi) intersect set("; cat bazel-targets-py.txt; echo ")"; } > query.txt; + bazel query --query_file=query.txt --output=label > remote-bidi-targets.txt; + ./scripts/github-actions/bazel-test-if-targets.sh + --keep_going + --flaky_test_attempts 3 + --local_test_jobs 1 + --target_pattern_file=remote-bidi-targets.txt + browser-tests: name: Browser Tests uses: ./.github/workflows/bazel.yml diff --git a/py/BUILD.bazel b/py/BUILD.bazel index 704cf26f4b0fc..b9b1e4c63de21 100644 --- a/py/BUILD.bazel +++ b/py/BUILD.bazel @@ -1124,6 +1124,48 @@ FEATURE_SUITE_DEFS = { if BROWSER_TESTS[browser].get("bidi", False) ] +# Generate test--remote-bidi targets (chrome and firefox only). +# Deliberately not folded into the test--remote aggregate: BiDi over Grid's +# websocket proxy is new coverage, so keep it out of the established remote job until +# it has proven stable. +[ + py_test_suite( + name = "test-%s-remote-bidi" % browser, + size = "large", + srcs = BIDI_TESTS + BIDI_IMPLEMENTATIONS, + args = [ + "--instafail", + "--bidi", + "--remote", + ] + BROWSERS[browser]["args"], + data = BROWSERS[browser]["data"] + [ + ":java-location", + "//java/src/org/openqa/selenium/grid:selenium_server_deploy.jar", + "@bazel_tools//tools/jdk:current_java_runtime", + ], + env = { + "SE_BAZEL_JAVA_LOCATION": "$(rootpath :java-location)", + }, + env_inherit = ["DISPLAY"], + tags = ["no-sandbox"] + BROWSERS[browser]["tags"] + [ + "remote", + "%s-remote" % browser, + ], + target_compatible_with = BROWSERS[browser]["target_compatible_with"], + test_suffix = "%s-remote-bidi" % browser, + deps = [ + ":bidi_protocol", # bidi/protocol_tests.py imports the generated _bidi layer directly + ":common_alert", # bidi/browsing_context_tests.py calls EC.alert_is_present() + ":init-tree", + ":webserver", + ] + BROWSER_TESTS[browser]["deps"] + TEST_DEPS, + ) + for browser in [ + "chrome", + "firefox", + ] +] + # Generate test--remote-common targets (chrome and firefox only) [ py_test_suite( @@ -1151,11 +1193,11 @@ FEATURE_SUITE_DEFS = { "SE_BAZEL_JAVA_LOCATION": "$(rootpath :java-location)", }, env_inherit = ["DISPLAY"], - tags = [ - "no-sandbox", + tags = ["no-sandbox"] + BROWSERS[browser]["tags"] + [ "remote", "%s-remote" % browser, ], + target_compatible_with = BROWSERS[browser]["target_compatible_with"], test_suffix = "%s-remote" % browser, deps = [ ":init-tree", @@ -1187,11 +1229,11 @@ FEATURE_SUITE_DEFS = { "SE_BAZEL_JAVA_LOCATION": "$(rootpath :java-location)", }, env_inherit = ["DISPLAY"], - tags = [ - "no-sandbox", + tags = ["no-sandbox"] + BROWSERS[browser]["tags"] + [ "remote", "%s-remote" % browser, ], + target_compatible_with = BROWSERS[browser]["target_compatible_with"], test_suffix = "%s-remote-actions" % browser, deps = [ ":init-tree", @@ -1223,11 +1265,11 @@ FEATURE_SUITE_DEFS = { "SE_BAZEL_JAVA_LOCATION": "$(rootpath :java-location)", }, env_inherit = ["DISPLAY"], - tags = [ - "no-sandbox", + tags = ["no-sandbox"] + BROWSERS[browser]["tags"] + [ "remote", "%s-remote" % browser, ], + target_compatible_with = BROWSERS[browser]["target_compatible_with"], test_suffix = "%s-remote-%s" % (browser, feature), deps = [ ":init-tree", @@ -1263,6 +1305,16 @@ test_suite( ], ) +# Kept out of :test-remote so BiDi-over-Grid runs as its own CI job while it is new. +test_suite( + name = "test-remote-bidi", + tags = ["remote"], + tests = [ + ":test-chrome-remote-bidi", + ":test-firefox-remote-bidi", + ], +) + py_test_suite( name = "test-webkitgtk-common", size = "large", diff --git a/py/TESTING.md b/py/TESTING.md index 6a9fe19690f3b..3e0a2c023c6e2 100644 --- a/py/TESTING.md +++ b/py/TESTING.md @@ -43,6 +43,13 @@ bazel test //py:test/selenium/webdriver/common/window_tests-chrome # With BiDi protocol bazel test //py:test-chrome-bidi +# Against a Grid server (chrome and firefox only). The suite starts its own +# Selenium standalone server and talks to it with webdriver.Remote. +bazel test //py:test-chrome-remote # classic protocol +bazel test //py:test-chrome-remote-bidi # BiDi over Grid's websocket proxy +bazel test //py:test-remote # every classic remote suite +bazel test //py:test-remote-bidi # every BiDi remote suite + # Test filters bazel test //py/... --test_tag_filters=chrome @@ -96,6 +103,12 @@ pytest py/test/selenium/webdriver/chrome/ --driver chrome --headless -v ``` > **Note:** > For running BiDi tests, use the `--bidi` flag. +> +> To run against a Grid server, add `--remote`. It starts a Selenium standalone +> server and runs the tests through `webdriver.Remote`, so it needs the Grid jar +> built first (`bazel build //java/src/org/openqa/selenium/grid:selenium_server_deploy.jar`). +> `--bidi --remote` combine: Grid rewrites the `webSocketUrl` capability to its own +> `/session//se/bidi` endpoint and proxies the socket through to the node. ## Skipping Tests @@ -146,6 +159,7 @@ modules, you will find the main fixtures in `conftest.py`: | `webserver` | Test HTTP server reference | | `clean_driver` | Fresh driver without parametrization | | `clean_options` | Fresh browser options instance | +| `headless` | Whether the browser was started headless, for tests asserting on behavior a headless browser does not model (e.g. window focus) | ## Test Organization diff --git a/py/conftest.py b/py/conftest.py index 198fc6d9ada48..5b5d8c3dc29d7 100644 --- a/py/conftest.py +++ b/py/conftest.py @@ -490,6 +490,45 @@ def stop_driver(self): driver_to_stop.quit() +def _skip_unless_remote(request, is_remote): + """Skip tests living under ``test/selenium/webdriver/remote/`` unless ``--remote`` is set.""" + if request.node.path.parts[-2] == "remote" and not is_remote: + pytest.skip("Remote tests require the --remote flag") + + +def _apply_xfail_markers(request, driver_class, is_remote): + """Honor the ``xfail_`` / ``xfail_remote`` markers for the driver under test. + + Raises the resulting pytest outcome directly, so callers need no follow-up branching. + + Args: + request: The pytest request for the test being set up. + driver_class: Name of the driver under test, used to build the marker name. + is_remote: Whether the run targets a remote server, enabling ``xfail_remote``. + """ + marker = request.node.get_closest_marker(f"xfail_{driver_class.lower()}") + if marker is None and is_remote: + marker = request.node.get_closest_marker("xfail_remote") + if marker is None: + return + + kwargs = dict(marker.kwargs) + + # A falsy condition means the xfail does not apply to this run. + condition = kwargs.pop("condition", True) + if callable(condition): + condition = condition() + if not condition: + return + + # run=False means skip outright rather than run and expect failure. + if not kwargs.pop("run", True): + pytest.skip() + + kwargs.pop("raises", None) + pytest.xfail(**kwargs) + + @pytest.fixture def driver(request, server): global selenium_driver @@ -504,34 +543,14 @@ def driver(request, server): if not selenium_driver.is_platform_valid: pytest.skip(f"{driver_class} tests can only run on {selenium_driver.exe_platform}") - # skip tests in the 'remote' directory if not running with --remote flag - if request.node.path.parts[-2] == "remote" and not selenium_driver.is_remote: - pytest.skip("Remote tests require the --remote flag") + _skip_unless_remote(request, selenium_driver.is_remote) # skip tests for drivers that don't support BiDi when --bidi is enabled if selenium_driver.bidi: if driver_class.lower() not in selenium_driver.supported_bidi_drivers: pytest.skip(f"{driver_class} does not support BiDi") - # conditionally mark tests as expected to fail based on driver - marker = request.node.get_closest_marker(f"xfail_{driver_class.lower()}") - # Also check for xfail_remote when running with --remote - if marker is None and selenium_driver.is_remote: - marker = request.node.get_closest_marker("xfail_remote") - if marker is not None: - kwargs = dict(marker.kwargs) - # Support condition kwarg - if condition is False, skip the xfail - condition = kwargs.pop("condition", True) - if callable(condition): - condition = condition() - if condition: - if "run" in kwargs: - if not kwargs["run"]: - pytest.skip() - yield - return - kwargs.pop("raises", None) - pytest.xfail(**kwargs) + _apply_xfail_markers(request, driver_class, selenium_driver.is_remote) # For BiDi tests, only restart driver when explicitly marked as needing fresh driver. # Tests marked with @pytest.mark.needs_fresh_driver get full driver restart for test isolation. @@ -600,6 +619,17 @@ def load(self, name): return Pages() +@pytest.fixture +def headless(request): + """Whether the browser under test was started headless. + + Without a window manager some behavior is not modelled at all - notably window + focus, which headless Chromium reports as always held by the current window - so + tests that assert on it need to know which mode they are running in. + """ + return bool(request.config.option.headless) + + @pytest.fixture(autouse=True, scope="session") def server(request): is_remote = request.config.getoption("remote") @@ -676,20 +706,7 @@ def clean_driver(request): raise Exception("This test requires a --driver to be specified.") driver_reference = getattr(webdriver, driver_class) - # conditionally mark tests as expected to fail based on driver - marker = request.node.get_closest_marker(f"xfail_{driver_class.lower()}") - # Also check for xfail_remote when running with --remote - if marker is None and request.config.getoption("remote"): - marker = request.node.get_closest_marker("xfail_remote") - if marker is not None: - kwargs = dict(marker.kwargs) - if "run" in kwargs: - if not kwargs["run"]: - pytest.skip() - yield - return - kwargs.pop("raises", None) - pytest.xfail(**kwargs) + _apply_xfail_markers(request, driver_class, request.config.getoption("remote")) yield driver_reference @@ -721,10 +738,7 @@ def firefox_options(request): if driver_class != "firefox": pytest.skip(f"This test requires Firefox. Got {driver_class}") - # skip tests in the 'remote' directory if not running with --remote flag - is_remote = request.config.getoption("remote") - if request.node.path.parts[-2] == "remote" and not is_remote: - pytest.skip("Remote tests require the --remote flag") + _skip_unless_remote(request, request.config.getoption("remote")) options = Driver.clean_options("firefox", request) @@ -742,10 +756,7 @@ def chromium_options(request): if driver_class not in ("chrome", "edge"): pytest.skip(f"This test requires Chrome or Edge. Got {driver_class}") - # skip tests in the 'remote' directory if not running with --remote flag - is_remote = request.config.getoption("remote") - if request.node.path.parts[-2] == "remote" and not is_remote: - pytest.skip("Remote tests require the --remote flag") + _skip_unless_remote(request, request.config.getoption("remote")) options = Driver.clean_options(driver_class, request) diff --git a/py/test/selenium/webdriver/common/_bidi/browsing_context_tests.py b/py/test/selenium/webdriver/common/_bidi/browsing_context_tests.py index e7e88dadf68ee..3137d5668ae2a 100644 --- a/py/test/selenium/webdriver/common/_bidi/browsing_context_tests.py +++ b/py/test/selenium/webdriver/common/_bidi/browsing_context_tests.py @@ -21,6 +21,7 @@ import pytest +from selenium.common.exceptions import TimeoutException from selenium.webdriver.common._bidi.browser import Browser from selenium.webdriver.common._bidi.browsing_context import ( BoxClipRectangle, @@ -195,15 +196,25 @@ def test_close_tab(driver): bc.close(context=tab1) -def test_activate_browsing_context(driver): +def test_activate_browsing_context(driver, headless): bc = BrowsingContext(driver) window1 = driver.current_window_handle window2 = bc.create(type=CreateType.WINDOW).context - assert not driver.execute_script("return document.hasFocus();") + # Focus is handed over asynchronously, so poll rather than reading once. Headless + # Chromium has no window manager and never hands it over at all, so skip there + # instead of failing; headless Firefox does, and keeps the full assertions. + try: + WebDriverWait(driver, 5).until_not(lambda d: d.execute_script("return document.hasFocus();")) + except TimeoutException: + if headless: + bc.close(context=window2) + pytest.skip("this headless browser does not move focus between windows") + raise bc.activate(context=window1) + WebDriverWait(driver, 5).until(lambda d: d.execute_script("return document.hasFocus();")) assert driver.execute_script("return document.hasFocus();") bc.close(context=window2) diff --git a/py/test/selenium/webdriver/common/_bidi/input_tests.py b/py/test/selenium/webdriver/common/_bidi/input_tests.py index 900b7d338cc0d..927dcd83e0faf 100644 --- a/py/test/selenium/webdriver/common/_bidi/input_tests.py +++ b/py/test/selenium/webdriver/common/_bidi/input_tests.py @@ -382,6 +382,10 @@ def test_wheel_scroll_with_duration(driver, pages): ) Input(driver).perform_actions(context=driver.current_window_handle, actions=[wheel_actions]) + # duration spreads the scroll over 500ms, so it can still be in flight when + # perform_actions returns; Firefox was observed one pixel short. + WebDriverWait(driver, 5).until(lambda d: d.execute_script("return window.pageYOffset;") == 100) + assert driver.execute_script("return window.pageYOffset;") == 100 diff --git a/py/test/selenium/webdriver/common/bidi/browsing_context_tests.py b/py/test/selenium/webdriver/common/bidi/browsing_context_tests.py index 86e3d11af0341..e996fe0d5a043 100644 --- a/py/test/selenium/webdriver/common/bidi/browsing_context_tests.py +++ b/py/test/selenium/webdriver/common/bidi/browsing_context_tests.py @@ -22,6 +22,7 @@ import pytest +from selenium.common.exceptions import TimeoutException from selenium.webdriver.common.bidi.browsing_context import ReadinessState from selenium.webdriver.common.by import By from selenium.webdriver.common.window import WindowTypes @@ -205,17 +206,27 @@ def test_close_tab(driver): driver.browsing_context.close(tab1) -def test_activate_browsing_context(driver): +def test_activate_browsing_context(driver, headless): """Test activating a browsing context.""" window1 = driver.current_window_handle # 2nd window is focused window2 = driver.browsing_context.create(type=WindowTypes.WINDOW) + # Focus is handed over asynchronously, so poll rather than reading once. Headless + # Chromium has no window manager and never hands it over at all, so skip there + # instead of failing; headless Firefox does, and keeps the full assertions. # We did not switch the driver, so we are running the script to check focus on 1st window - assert not driver.execute_script("return document.hasFocus();") + try: + WebDriverWait(driver, 5).until_not(lambda d: d.execute_script("return document.hasFocus();")) + except TimeoutException: + if headless: + driver.browsing_context.close(window2) + pytest.skip("this headless browser does not move focus between windows") + raise driver.browsing_context.activate(window1) + WebDriverWait(driver, 5).until(lambda d: d.execute_script("return document.hasFocus();")) assert driver.execute_script("return document.hasFocus();") # Clean up diff --git a/py/test/selenium/webdriver/common/bidi/input_tests.py b/py/test/selenium/webdriver/common/bidi/input_tests.py index 97ab7f0848870..aa3b00ce8b28b 100644 --- a/py/test/selenium/webdriver/common/bidi/input_tests.py +++ b/py/test/selenium/webdriver/common/bidi/input_tests.py @@ -636,6 +636,10 @@ def test_wheel_scroll_with_duration(driver, pages): driver.input.perform_actions(driver.current_window_handle, [wheel_actions]) + # duration spreads the scroll over 500ms, so it can still be in flight when + # perform_actions returns; Firefox was observed one pixel short. + WebDriverWait(driver, 5).until(lambda d: d.execute_script("return window.pageYOffset;") == 100) + scroll_y = driver.execute_script("return window.pageYOffset;") assert scroll_y == 100