diff --git a/.github/workflows/migrations-check.yml b/.github/workflows/migrations-check.yml index f2a33010e34a..6584eef58e6b 100644 --- a/.github/workflows/migrations-check.yml +++ b/.github/workflows/migrations-check.yml @@ -9,8 +9,53 @@ on: - master jobs: + # Applying all ~1800 migrations from an empty database takes ~9 minutes, and + # 99% of commits on master touch no migration at all, so the overwhelming + # majority of runs re-derive a schema identical to the previous one. + # + # models.py is included deliberately: a model changed without a matching + # migration is exactly what this workflow exists to catch. + # + # Only pull_request is gated. push, merge_group and workflow_dispatch always + # run the full check, so nothing is permanently skipped -- in particular a + # RunPython migration that imports application code can still be broken by a + # code-only change, and master will catch that. + changed: + name: detect migration changes + runs-on: ubuntu-24.04 + outputs: + migrations: ${{ steps.detect.outputs.migrations }} + steps: + - uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: detect + id: detect + shell: bash + run: | + if [[ "${{ github.event_name }}" != "pull_request" ]]; then + echo "migrations=true" >> "$GITHUB_OUTPUT" + echo "Not a pull request; running the full check." + exit 0 + fi + base="origin/${{ github.base_ref }}" + git fetch --no-tags --depth=1 origin "${{ github.base_ref }}" || true + changed=$(git diff --name-only "$(git merge-base "$base" HEAD)" HEAD) + echo "$changed" | grep -E '(/migrations/[^/]+\.py$|/models\.py$)' > /tmp/hits || true + if [[ -s /tmp/hits ]]; then + echo "migrations=true" >> "$GITHUB_OUTPUT" + echo "Migration or model files changed:" + sed 's/^/ /' /tmp/hits + else + echo "migrations=false" >> "$GITHUB_OUTPUT" + echo "No migration or model files changed; skipping the migration check." + fi + check_migrations: name: check migrations + needs: changed + if: needs.changed.outputs.migrations == 'true' runs-on: ${{ matrix.os }} strategy: matrix: @@ -45,7 +90,9 @@ jobs: MYSQL_DATABASE: "edxapp" MYSQL_USER: "edxapp001" MYSQL_PASSWORD: "password" - MYSQL_RANDOM_ROOT_PASSWORD: true + # Fixed rather than random so the schema-cache fallback below can drop + # and recreate the database. This database is discarded with the job. + MYSQL_ROOT_PASSWORD: "rootpw" options: >- --health-cmd "mysqladmin ping" --health-interval 10s @@ -97,16 +144,66 @@ jobs: run: | uv tree + # Applying every migration from an empty database is ~9 minutes and almost + # all of that work is identical between runs. Cache the resulting schema + # (including the django_migrations table, which records what has already + # been applied) and reload it instead. + # + # restore-keys matters more than the exact key here: on a PR that adds one + # migration the exact key misses, an older schema is restored, and the + # migrate below applies only the delta. Both LMS and CMS point at the same + # database, so one dump covers both. + - name: Restore cached migrated schema + uses: actions/cache@v4 + with: + path: migrations-schema.sql + key: migschema-${{ matrix.mysql-version }}-${{ matrix.django-version }}-${{ hashFiles('**/migrations/*.py', 'uv.lock') }} + restore-keys: | + migschema-${{ matrix.mysql-version }}-${{ matrix.django-version }}- + - name: Run Tests env: LMS_CFG: lms/envs/minimal.yml # This is from the LMS dir on purpose since we don't need anything different for the CMS yet. STUDIO_CFG: lms/envs/minimal.yml run: | - echo "Running the LMS migrations." - uv run ./manage.py lms migrate - echo "Running the CMS migrations." - uv run ./manage.py cms migrate + set -o pipefail + + load_cached_schema() { + if [[ -s migrations-schema.sql ]]; then + echo "Restoring cached schema ($(du -h migrations-schema.sql | cut -f1))." + docker exec -i ${{ job.services.mysql.id }} mysql -uedxapp001 -ppassword edxapp < migrations-schema.sql + else + echo "No cached schema; migrating from an empty database." + fi + } + + reset_database() { + docker exec ${{ job.services.mysql.id }} mysql -uroot -prootpw -e " + DROP DATABASE IF EXISTS edxapp; + CREATE DATABASE edxapp; + GRANT ALL ON edxapp.* TO 'edxapp001'@'%';" + } + + run_migrations() { + uv run ./manage.py lms migrate && uv run ./manage.py cms migrate + } + + load_cached_schema + if ! run_migrations; then + # A restored schema can be unusable -- a migration squashed, renamed + # or removed since the dump was taken leaves django_migrations + # describing a graph that no longer exists. Fall back to the cold + # path rather than reporting a migration failure that is really a + # stale cache, which would be indistinguishable from a real break. + echo "::warning::Incremental migration failed; retrying from an empty database." + reset_database + run_migrations + fi + + echo "Dumping schema for the next run." + docker exec ${{ job.services.mysql.id }} mysqldump -uedxapp001 -ppassword \ + --no-tablespaces --single-transaction --routines edxapp > migrations-schema.sql # This job aggregates test results. It's the required check for branch protection. # https://github.com/marketplace/actions/alls-green#why @@ -115,6 +212,7 @@ jobs: name: Migrations checks successful if: always() needs: + - changed - check_migrations runs-on: ubuntu-24.04 steps: @@ -123,3 +221,7 @@ jobs: uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe with: jobs: ${{ toJSON(needs) }} + # check_migrations is skipped on pull requests that touch no migration + # or model file. allowed-skips keeps this required check green in that + # case, while a genuine failure still fails it. + allowed-skips: check_migrations diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index ef667d755673..8d1c6dc6cd72 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -21,6 +21,10 @@ jobs: name: ${{ matrix.shard_name }}(py=${{ matrix.python-version }},dj=${{ matrix.django-version }},mongo=${{ matrix.mongo-version }}) runs-on: ${{ matrix.os-version }} strategy: + # One shard failing shouldn't cancel the other nine. At ~20 min a run, + # seeing every failing shard at once is worth more than the runner + # minutes saved by bailing early. + fail-fast: false matrix: python-version: - "3.12" @@ -119,6 +123,12 @@ jobs: echo "report_log_arg=" >> $GITHUB_ENV fi + - name: report available parallelism + shell: bash + run: | + echo "nproc: $(nproc)" + uv run python -c "import os, psutil; print('logical:', os.cpu_count(), 'physical:', psutil.cpu_count(logical=False))" + - name: run tests shell: bash run: | @@ -126,7 +136,20 @@ jobs: # (~3.7k lines of noise). The .coverage data file is still written and # uploaded as an artifact for the combined `coverage` job below, which # is what reports to codecov. - uv run python -Wd -m pytest -p no:randomly --ds=${{ env.settings_path }} ${{ env.unit_test_paths }} \ + # + # -n logical spreads the shard across every vCPU on the runner; without + # it pytest used a single core and left ~75% of the runner idle. + # + # --dist loadfile groups by module, not by class. ModuleStoreIsolationMixin + # keeps process-global stacks (__old_modulestores, __old_contentstores, + # __settings_overrides) on top of Django's override_settings stack, and both + # SharedModuleStoreTestCase.setUpClass and ModuleStoreTestCase.setUp push onto + # them, so unwinding has to stay strictly LIFO. Splitting a module's classes + # across workers (loadscope) produced interleavings that broke that nesting + # and left settings restored to the wrong object. Keeping whole files together + # preserves the within-file ordering the stacks rely on. + uv run python -Wd -m pytest -p no:randomly -n logical --dist loadfile \ + --ds=${{ env.settings_path }} ${{ env.unit_test_paths }} \ --cov=. --cov-report="" ${{ env.report_log_arg }} - name: Upload pytest timing report @@ -164,66 +187,16 @@ jobs: path: reports/${{ matrix.shard_name }}_${{ matrix.python-version }}_${{ matrix.django-version }}_${{ matrix.mongo-version }}_${{ matrix.os-version }}.coverage overwrite: true - collect-and-verify: + verify-shard-coverage: + name: verify shard coverage runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v7 - - - name: install system requirements - run: | - sudo apt-get update && sudo apt-get install libxmlsec1-dev - - - name: Install uv - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 - with: - enable-cache: true - python-version: "3.12" - - - name: install requirements - run: make test-requirements - - - name: collect tests from all modules - shell: bash - run: | - echo "root_cms_unit_tests_count=$(uv run pytest --disable-warnings --collect-only --ds=cms.envs.test cms/ -q | head -n -2 | wc -l)" >> $GITHUB_ENV - echo "root_lms_unit_tests_count=$(uv run pytest --disable-warnings --collect-only --ds=lms.envs.test lms/ openedx/ common/djangoapps/ xmodule/ -q | head -n -2 | wc -l)" >> $GITHUB_ENV - - - name: get GHA unit test paths - shell: bash - run: | - echo "cms_unit_test_paths=$(uv run python scripts/gha_unit_tests_collector.py --cms-only)" >> $GITHUB_ENV - echo "lms_unit_test_paths=$(uv run python scripts/gha_unit_tests_collector.py --lms-only)" >> $GITHUB_ENV - - - name: collect tests from GHA unit test shards - shell: bash - run: | - echo "cms_unit_tests_count=$(uv run pytest --disable-warnings --collect-only --ds=cms.envs.test ${{ env.cms_unit_test_paths }} -q | head -n -2 | wc -l)" >> $GITHUB_ENV - echo "lms_unit_tests_count=$(uv run pytest --disable-warnings --collect-only --ds=lms.envs.test ${{ env.lms_unit_test_paths }} -q | head -n -2 | wc -l)" >> $GITHUB_ENV - - - name: add unit tests count - shell: bash - run: | - echo "root_all_unit_tests_count=$((${{ env.root_cms_unit_tests_count }}+${{ env.root_lms_unit_tests_count }}))" >> $GITHUB_ENV - echo "shards_all_unit_tests_count=$((${{ env.cms_unit_tests_count }}+${{ env.lms_unit_tests_count }}))" >> $GITHUB_ENV - - - name: print unit tests count - shell: bash - run: | - echo CMS unit tests from root: ${{ env.root_cms_unit_tests_count }} - echo LMS unit tests from root: ${{ env.root_lms_unit_tests_count }} - echo CMS unit tests from shards: ${{ env.cms_unit_tests_count }} - echo LMS unit tests from shards: ${{ env.lms_unit_tests_count }} - echo All root unit tests count: ${{ env.root_all_unit_tests_count }} - echo All shards unit tests count: ${{ env.shards_all_unit_tests_count }} - - - name: fail the check - shell: bash - if: ${{ env.root_all_unit_tests_count != env.shards_all_unit_tests_count }} - run: | - echo "::error title='Unit test modules in unit-test-shards.json (unit-tests.yml workflow) are outdated'::unit tests running in unit-tests - workflow don't match the count for unit tests for entire openedx-platform suite, please update the unit-test-shards.json under .github/workflows - to add any missing apps and match the count. for more details please take a look at scripts/gha-shards-readme.md" - exit 1 + # Replaces collect-and-verify, which installed every dependency and ran + # two full pytest collections (~2 min) to compare test counts. This reads + # the file tree in a few seconds and names the directory that is missing. + - name: check every test directory belongs to a shard + run: python3 scripts/verify_test_suite_coverage.py # This job aggregates test results. It's the required check for branch protection. # https://github.com/marketplace/actions/alls-green#why @@ -232,7 +205,7 @@ jobs: name: Unit tests successful runs-on: ubuntu-24.04 if: always() - needs: [run-tests] + needs: [run-tests, verify-shard-coverage] steps: - name: Decide whether the needed jobs succeeded or failed # uses: re-actors/alls-green@v1.2.1 diff --git a/common/djangoapps/student/management/tests/test_recover_account.py b/common/djangoapps/student/management/tests/test_recover_account.py index bd1cbf3a8e9b..b2adebec70da 100644 --- a/common/djangoapps/student/management/tests/test_recover_account.py +++ b/common/djangoapps/student/management/tests/test_recover_account.py @@ -4,9 +4,11 @@ import re from tempfile import NamedTemporaryFile +import crum import pytest from django.conf import settings from django.contrib.auth import get_user_model +from django.contrib.sites.models import Site from django.core import mail from django.core.files.uploadedfile import SimpleUploadedFile from django.core.management import CommandError, call_command @@ -31,6 +33,23 @@ def setUp(self): super().setUp() self.user = UserFactory.create(username='amy', email='amy@edx.com', password='password') + # recover_account sends an ACE message, and the ace template tags resolve + # the "current" request through crum's thread-local, which no test resets. + # emulate_http_request() sets .site on the request it finds there, so + # whether rendering succeeds depends on what an earlier test happened to + # leave behind: a request without .site makes the tags raise, the command + # swallows it as "Unable to send email", and mail.outbox stays empty. + # Install a well-formed request instead of inheriting one. + request = self.request_factory.get('/') + request.site = Site.objects.get_current() + request.user = self.user + previous_request = crum.get_current_request() + crum.set_current_request(request) + # Restore whatever was there rather than clearing to None: other tests in + # this process may be relying on it, and it is not this class's business + # to change that -- only to stop depending on it itself. + self.addCleanup(crum.set_current_request, previous_request) + def _write_test_csv(self, csv, lines): """Write a test csv file with the lines provided""" csv.write(b"username,current_email,desired_email\n") diff --git a/common/djangoapps/student/tests/test_activate_account.py b/common/djangoapps/student/tests/test_activate_account.py index 3fd6a1b432e8..43c5e18cb372 100644 --- a/common/djangoapps/student/tests/test_activate_account.py +++ b/common/djangoapps/student/tests/test_activate_account.py @@ -7,6 +7,7 @@ from django.conf import settings from django.contrib.auth.models import User # pylint: disable=imported-auth-user +from django.core.cache import cache from django.test import TestCase, override_settings from django.urls import reverse from django.utils.http import urlencode @@ -24,6 +25,13 @@ class TestActivateAccount(TestCase): def setUp(self): super().setUp() + # django_ratelimit counts attempts in the Django cache, which is not + # rolled back between tests the way the database is. Left over counts + # from earlier tests in the same process make the login views answer + # 400/429 with "Too many failed login attempts". Serial ordering hid + # this; under pytest-xdist a different mix of tests shares the worker. + # test_reset_password.py in this package already does the same thing. + cache.clear() self.username = "jack" self.email = "jack@fake.edx.org" self.password = "test-password" diff --git a/common/djangoapps/student/tests/test_models.py b/common/djangoapps/student/tests/test_models.py index d57f4abfd5ac..9c4ed7a34b01 100644 --- a/common/djangoapps/student/tests/test_models.py +++ b/common/djangoapps/student/tests/test_models.py @@ -185,7 +185,7 @@ def test_upgrade_deadline_with_schedule(self): assert enrollment.upgrade_deadline == enrollment.schedule.upgrade_deadline @skip_unless_lms - @ddt.data(*(set(CourseMode.ALL_MODES) - set(CourseMode.AUDIT_MODES))) + @ddt.data(*sorted(set(CourseMode.ALL_MODES) - set(CourseMode.AUDIT_MODES))) def test_upgrade_deadline_for_non_upgradeable_enrollment(self, mode): """ The property should return None if an upgrade cannot be upgraded. """ enrollment = CourseEnrollmentFactory(course_id=self.course.id, mode=mode) # pylint: disable=no-member @@ -831,7 +831,7 @@ def setUp(self): super().setUp() self.course = CourseFactory.create() - @ddt.data(*(set(CourseMode.ALL_MODES) - set(CourseMode.AUDIT_MODES))) + @ddt.data(*sorted(set(CourseMode.ALL_MODES) - set(CourseMode.AUDIT_MODES))) def test_paid_user_not_downgraded_on_activation(self, mode): """ Make sure that students who are already enrolled + have paid do not get downgraded to audit mode diff --git a/common/djangoapps/util/tests/test_sandboxing.py b/common/djangoapps/util/tests/test_sandboxing.py index b044a1c5be67..08b78318307b 100644 --- a/common/djangoapps/util/tests/test_sandboxing.py +++ b/common/djangoapps/util/tests/test_sandboxing.py @@ -10,7 +10,7 @@ from opaque_keys.edx.locator import CourseLocator, LibraryLocator, LibraryLocatorV2 from xmodule.contentstore.django import contentstore -from xmodule.modulestore.tests.django_utils import upload_file_to_course +from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase, upload_file_to_course from xmodule.util.sandboxing import SandboxService, can_execute_unsafe_code @@ -45,9 +45,16 @@ def test_courselikes_with_unsafe_code_default(self): @ddt.ddt -class SandboxServiceTest(TestCase): +class SandboxServiceTest(SharedModuleStoreTestCase): """ Test SandboxService methods. + + SharedModuleStoreTestCase rather than TestCase: this class writes to the + contentstore in setUpClass, and plain TestCase gives it no contentstore + isolation -- it just reads whatever settings.CONTENTSTORE happens to hold. If + an earlier test has left an override_settings frame masking that setting, the + upload raises here, setUpClass dies with the class-level atomic already open, + and every later test in the worker fails with TransactionManagementError. """ PYTHON_LIB_FILENAME = 'test_python_lib.zip' PYTHON_LIB_SOURCE_FILE = './common/test/data/uploads/python_lib.zip' @@ -116,11 +123,14 @@ def test_no_python_lib_zip(self): assert self.sandbox_service.get_python_lib_zip() is None -class SandboxServiceForLibrariesV2Test(TestCase): +class SandboxServiceForLibrariesV2Test(SharedModuleStoreTestCase): """ Test SandboxService methods for V2 Content Libraries. (Lacks tests for anything other than python_lib_zip) + + SharedModuleStoreTestCase for the same reason as SandboxServiceTest above: + it builds a SandboxService around the contentstore in setUpClass. """ @classmethod diff --git a/lms/djangoapps/instructor/tests/test_api.py b/lms/djangoapps/instructor/tests/test_api.py index a9a6eee0d979..f8e9ec322805 100644 --- a/lms/djangoapps/instructor/tests/test_api.py +++ b/lms/djangoapps/instructor/tests/test_api.py @@ -154,6 +154,11 @@ ) +# These are sets because the membership checks below are the common use. Any +# ddt parameterization over them must sort first: set iteration order for str +# depends on PYTHONHASHSEED, which differs per process, so ddt would generate a +# different index -> endpoint mapping in each pytest-xdist worker and collection +# would not match across workers. INSTRUCTOR_GET_ENDPOINTS = { 'get_anon_ids', 'get_issued_certificates', @@ -338,7 +343,7 @@ def setUp(self): global_user = GlobalStaffFactory() self.client.login(username=global_user.username, password=self.TEST_PASSWORD) - @ddt.data(*INSTRUCTOR_POST_ENDPOINTS) + @ddt.data(*sorted(INSTRUCTOR_POST_ENDPOINTS)) def test_endpoints_reject_get(self, data): """ Tests that POST endpoints are rejected with 405 when using GET. @@ -349,7 +354,7 @@ def test_endpoints_reject_get(self, data): assert response.status_code == 405, \ f'Endpoint {data} returned status code {response.status_code} instead of a 405. It should not allow GET.' - @ddt.data(*INSTRUCTOR_GET_ENDPOINTS) + @ddt.data(*sorted(INSTRUCTOR_GET_ENDPOINTS)) def test_endpoints_accept_get(self, data): """ Tests that GET endpoints are not rejected with 405 when using GET. diff --git a/lms/djangoapps/learner_home/test_serializers.py b/lms/djangoapps/learner_home/test_serializers.py index 6bde2958c482..12f909304df4 100644 --- a/lms/djangoapps/learner_home/test_serializers.py +++ b/lms/djangoapps/learner_home/test_serializers.py @@ -55,6 +55,13 @@ from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase from xmodule.modulestore.tests.factories import CourseFactory +# Fixed URLs for ddt parameterization. These must be stable across processes: +# ddt puts trivial values (str/bool/None/numbers) into the generated test name, +# so a random URL would produce different node IDs in each pytest-xdist worker. +TEST_URL_A = "example.com/test-url-a" +TEST_URL_B = "example.com/test-url-b" +TEST_URL_C = "example.com/test-url-c" + class LearnerDashboardBaseTest(SharedModuleStoreTestCase): """Base class for common setup""" @@ -395,11 +402,14 @@ def test_audit_access_expired(self, expiration_datetime, should_be_expired): self.assertEqual(output["isAuditAccessExpired"], should_be_expired) # noqa: PT009 + # Literal URLs, not random_url(): ddt embeds trivial values in the generated + # test name, so a per-process random URL gives each pytest-xdist worker a + # different node ID and collection no longer matches across workers. @ddt.data( - (random_url(), True, uuid4(), True), + (TEST_URL_A, True, uuid4(), True), (None, True, uuid4(), False), - (random_url(), False, uuid4(), False), - (random_url(), True, None, False), + (TEST_URL_B, False, uuid4(), False), + (TEST_URL_C, True, None, False), ) @ddt.unpack def test_user_can_upgrade( @@ -647,9 +657,10 @@ def test_is_downloadable(self, cert_status, is_downloadable_expected): # Then isDownloadable should be calculated correctly self.assertEqual(output_data["isDownloadable"], is_downloadable_expected) # noqa: PT009 + # Literal URLs -- see the note on test_user_can_upgrade above. @ddt.data( - (True, random_url()), - (False, random_url()), + (True, TEST_URL_A), + (False, TEST_URL_B), (True, None), (False, None), ) diff --git a/openedx/core/djangoapps/courseware_api/tests/test_views.py b/openedx/core/djangoapps/courseware_api/tests/test_views.py index 3f4f5fae0ebc..84841fab8473 100644 --- a/openedx/core/djangoapps/courseware_api/tests/test_views.py +++ b/openedx/core/djangoapps/courseware_api/tests/test_views.py @@ -89,8 +89,14 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): - super().tearDownClass() + # Delete the course before ending modulestore isolation, not after. + # super().tearDownClass() tears the isolation down and drops the mongo + # collections, so a delete_course() after it operates on a modulestore + # that no longer holds this course and raises -- which aborts the rest + # of tearDownClass and leaves the class's settings override on the + # global stack for the remainder of the process. cls.store.delete_course(cls.course.id, cls.user.id) + super().tearDownClass() def setUp(self): super().setUp() diff --git a/openedx/core/djangoapps/safe_sessions/tests/test_middleware.py b/openedx/core/djangoapps/safe_sessions/tests/test_middleware.py index 0f0e3ad1ddd3..14d00adbee5c 100644 --- a/openedx/core/djangoapps/safe_sessions/tests/test_middleware.py +++ b/openedx/core/djangoapps/safe_sessions/tests/test_middleware.py @@ -453,12 +453,19 @@ def test_warn_on_user_change_in_both(self): @patch("openedx.core.djangoapps.safe_sessions.middleware.set_custom_attribute") def test_warn_with_verbose_logging(self, mock_set_custom_attribute): self.set_up_for_success() - self.request.user = UserFactory.create() + # The ids are read back from the users rather than hard-coded as 1 and 2: + # they come from an auto-increment column, so they are only 1 and 2 when + # nothing earlier in the process has created a user. That holds in a + # fixed serial order and stops holding as soon as the order changes. + session_user = self.user + request_user = self.request.user = UserFactory.create() with self.assert_logged('SafeCookieData: Changing request user. ', log_level='warning'): SafeSessionMiddleware(get_response=lambda request: None).process_response( self.request, self.client.response ) - mock_set_custom_attribute.assert_has_calls([call('safe_sessions.user_id_list', '1,2')]) + mock_set_custom_attribute.assert_has_calls([ + call('safe_sessions.user_id_list', f'{session_user.id},{request_user.id}') + ]) @patch("openedx.core.djangoapps.safe_sessions.middleware.LOG_REQUEST_USER_CHANGES", False) def test_warn_without_verbose_logging(self): @@ -474,7 +481,10 @@ def test_warn_without_verbose_logging(self): @patch("openedx.core.djangoapps.safe_sessions.middleware.cache") def test_user_change_with_header_logging(self, mock_cache): self.set_up_for_success() - self.request.user = UserFactory.create() + # See the note in test_warn_with_verbose_logging: these ids are + # auto-increment values, not constants. + session_user = self.user + request_user = self.request.user = UserFactory.create() with self.assert_logged('SafeCookieData: Changing request user. ', log_level='warning'): SafeSessionMiddleware(get_response=lambda request: None).process_response( self.request, self.client.response @@ -483,8 +493,8 @@ def test_user_change_with_header_logging(self, mock_cache): # simply assert that the cache is set (here) and checked (below). mock_cache.set_many.assert_called_with( { - 'safe_sessions.middleware.recent_user_change_detected_1': True, - 'safe_sessions.middleware.recent_user_change_detected_2': True + f'safe_sessions.middleware.recent_user_change_detected_{session_user.id}': True, + f'safe_sessions.middleware.recent_user_change_detected_{request_user.id}': True }, 300 ) @@ -494,7 +504,9 @@ def test_user_change_with_header_logging(self, mock_cache): # Note: The test cache is not returning True because it is not retaining its values # for some reason. Rather than asserting that we log the header appropriately, we'll # simply verify that we are checking the cache. - mock_cache.get.assert_called_with('safe_sessions.middleware.recent_user_change_detected_1', False) + mock_cache.get.assert_called_with( + f'safe_sessions.middleware.recent_user_change_detected_{session_user.id}', False + ) @override_settings(LOG_REQUEST_USER_CHANGE_HEADERS=True) @patch("openedx.core.djangoapps.safe_sessions.middleware.LOG_REQUEST_USER_CHANGES", True) diff --git a/openedx/core/djangoapps/user_authn/views/tests/test_filters.py b/openedx/core/djangoapps/user_authn/views/tests/test_filters.py index 41fe122bfdc8..2f23cced5fda 100644 --- a/openedx/core/djangoapps/user_authn/views/tests/test_filters.py +++ b/openedx/core/djangoapps/user_authn/views/tests/test_filters.py @@ -4,6 +4,7 @@ from unittest.mock import Mock, patch from django.contrib.auth import get_user_model +from django.core.cache import cache from django.test import override_settings from django.urls import reverse from openedx_filters import PipelineStep @@ -565,6 +566,13 @@ class PostLoginRedirectFiltersTest(UserAPITestCase): def setUp(self): # pylint: disable=arguments-differ super().setUp() + # django_ratelimit counts attempts in the Django cache, which is not + # rolled back between tests the way the database is. Left over counts + # from earlier tests in the same process make the login views answer + # 400/429 with "Too many failed login attempts". Serial ordering hid + # this; under pytest-xdist a different mix of tests shares the worker. + # test_reset_password.py in this package already does the same thing. + cache.clear() self.user = UserFactory.create( username="test", email="test@example.com", diff --git a/openedx/core/djangoapps/user_authn/views/tests/test_login.py b/openedx/core/djangoapps/user_authn/views/tests/test_login.py index 24c13a6996b0..a1e69b95f6b4 100644 --- a/openedx/core/djangoapps/user_authn/views/tests/test_login.py +++ b/openedx/core/djangoapps/user_authn/views/tests/test_login.py @@ -989,6 +989,13 @@ class LoginSessionViewTest(OpenEdxEventsTestMixin, ApiTestCase): def setUp(self): super().setUp() + # django_ratelimit counts attempts in the Django cache, which is not + # rolled back between tests the way the database is. Left over counts + # from earlier tests in the same process make the login views answer + # 400/429 with "Too many failed login attempts". Serial ordering hid + # this; under pytest-xdist a different mix of tests shares the worker. + # test_reset_password.py in this package already does the same thing. + cache.clear() self.url = reverse("user_api_login_session", kwargs={'api_version': 'v1'}) self.url_v2 = reverse("user_api_login_session", kwargs={'api_version': 'v2'}) self.user = UserFactory.create(username=self.USERNAME, email=self.EMAIL, password=self.PASSWORD) diff --git a/openedx/core/djangoapps/user_authn/views/tests/test_logout.py b/openedx/core/djangoapps/user_authn/views/tests/test_logout.py index e29a8d6e292b..9964339edcf4 100644 --- a/openedx/core/djangoapps/user_authn/views/tests/test_logout.py +++ b/openedx/core/djangoapps/user_authn/views/tests/test_logout.py @@ -8,6 +8,7 @@ import ddt import nh3 from django.conf import settings +from django.core.cache import cache from django.test import TestCase from django.test.utils import override_settings from django.urls import reverse @@ -26,6 +27,13 @@ class LogoutTests(TestCase): def setUp(self): """ Create a course and user, then log in. """ super().setUp() + # django_ratelimit counts attempts in the Django cache, which is not + # rolled back between tests the way the database is. Left over counts + # from earlier tests in the same process make the login views answer + # 400/429 with "Too many failed login attempts". Serial ordering hid + # this; under pytest-xdist a different mix of tests shares the worker. + # test_reset_password.py in this package already does the same thing. + cache.clear() self.user = UserFactory() self.client.login(username=self.user.username, password='test') diff --git a/openedx/core/djangolib/testing/utils.py b/openedx/core/djangolib/testing/utils.py index 3b263c2b1a53..45212fa4a3af 100644 --- a/openedx/core/djangolib/testing/utils.py +++ b/openedx/core/djangolib/testing/utils.py @@ -16,7 +16,7 @@ import crum from django.conf import settings from django.contrib import sites -from django.core.cache import caches +from django.core.cache import InvalidCacheBackendError, caches from django.core.exceptions import ImproperlyConfigured from django.db import DEFAULT_DB_ALIAS, connections from django.test import RequestFactory, TestCase, override_settings @@ -96,6 +96,15 @@ class CacheIsolationMixin: __settings_overrides = [] __old_settings = [] + # Number of cache isolations this exact class currently has open. Read from + # cls.__dict__ so subclasses never see (or decrement) a parent's count. + _CACHE_ISOLATION_DEPTH_ATTR = '_cache_isolation_depth_count' + + @classmethod + def _cache_isolation_depth(cls): + """How many cache isolations this exact class currently has open.""" + return cls.__dict__.get(cls._CACHE_ISOLATION_DEPTH_ATTR, 0) + @classmethod def setUpClass(cls): super().setUpClass() @@ -151,6 +160,7 @@ def start_cache_isolation(cls): override = override_settings(CACHES=cache_settings) override.__enter__() # pylint: disable=unnecessary-dunder-call cls.__settings_overrides.append(override) + setattr(cls, cls._CACHE_ISOLATION_DEPTH_ATTR, cls._cache_isolation_depth() + 1) assert settings.CACHES == cache_settings @@ -166,6 +176,15 @@ def end_cache_isolation(cls): # Make sure that cache contents don't leak out after the isolation is ended cls.clear_caches() + # Only unwind what this exact class pushed. The stacks below are shared + # by every subclass, so an unguarded pop can take another class's + # override off the stack -- and override_settings.__exit__ restores + # settings._wrapped to the value *that* frame captured, silently + # discarding every override layered above it. + if cls._cache_isolation_depth() <= 0: + return + setattr(cls, cls._CACHE_ISOLATION_DEPTH_ATTR, cls._cache_isolation_depth() - 1) + if cls.__settings_overrides: cls.__settings_overrides.pop().__exit__(None, None, None) assert settings.CACHES == cls.__old_settings.pop() @@ -180,7 +199,19 @@ def clear_caches(cls): # accessed using caches[name] previously, so we loop # over our list of overridden caches, instead. for cache in settings.CACHES: - caches[cache].clear() + try: + caches[cache].clear() + except (InvalidCacheBackendError, AttributeError): + # settings.CACHES and Django's connection handler can disagree + # about which aliases exist: an override_settings frame that adds + # caches (ModuleStoreIsolationMixin adds course_index_cache and + # friends) fires setting_changed on the way out, which resets the + # handler, and a concurrent frame can leave the two out of step. + # An alias that cannot be resolved has no cache to clear, so + # there is nothing to do -- but raising here fails the test from + # inside cleanup, which is how one transient disagreement turned + # into a hundred-plus InvalidCacheBackendError failures a run. + continue # The sites framework caches in a module-level dictionary. # Clear that. diff --git a/openedx/core/pytest_hooks.py b/openedx/core/pytest_hooks.py index 16afe851ac0b..411ed04d8812 100644 --- a/openedx/core/pytest_hooks.py +++ b/openedx/core/pytest_hooks.py @@ -47,6 +47,14 @@ def pytest_sessionfinish(session): Since multiple pytests are running, this makes sure warnings from different run are not overwritten """ + # Under pytest-xdist this hook fires on every worker as well as on the + # controller. Only the controller holds the aggregated report, so the + # workers bail out here; otherwise they race each other for the next free + # file name and leave behind partial files that the warning report job + # would then double-count. + if os.environ.get("PYTEST_XDIST_WORKER"): + return + dir_path = "test_root/log" file_name_postfix = "pytest_warnings" num = 0 diff --git a/openedx/features/content_type_gating/tests/test_partitions.py b/openedx/features/content_type_gating/tests/test_partitions.py index 7b1721c9b0fb..ffa1c7da02f5 100644 --- a/openedx/features/content_type_gating/tests/test_partitions.py +++ b/openedx/features/content_type_gating/tests/test_partitions.py @@ -16,7 +16,15 @@ class TestContentTypeGatingPartition(CacheIsolationTestCase): # pylint: disable=missing-class-docstring - def setUp(self): # pylint: disable=super-method-not-called + def setUp(self): + # CacheIsolationTestCase.setUp() is what clears the caches and registers + # the cleanup that clears them again. Skipping it opted this class out of + # the isolation its base class exists to provide, so ContentTypeGatingConfig + # -- a ConfigurationModel, which caches current() -- was answered from + # whatever an earlier test had cached. That made + # test_create_content_gating_partition_disabled see enabled=True and build + # a partition where it asserts None. + super().setUp() self.course_key = CourseKey.from_string('course-v1:test+course+key') CourseOverviewFactory.create(id=self.course_key) @@ -49,6 +57,13 @@ def test_create_content_gating_partition_disabled(self): def test_create_content_gating_partition_no_scheme_installed(self): mock_course = Mock(id=self.course_key, user_partitions={}) + # enabled_for_course() starts with `if not correct_modes_for_fbe(...)`, + # so the course needs audit and verified modes before the config's + # enabled=True has any effect. Without them this returns None at the + # first branch and never reaches the code under test. This test asserted None and so passed either way, but was + # never actually exercising the UserPartitionError branch it patches. + CourseModeFactory.create(course_id=mock_course.id, mode_slug='audit') + CourseModeFactory.create(course_id=mock_course.id, mode_slug='verified') ContentTypeGatingConfig.objects.create(enabled=True, enabled_as_of=datetime(2018, 1, 1)) with patch( @@ -62,6 +77,14 @@ def test_create_content_gating_partition_partition_id_used(self): mock_course = Mock( id=self.course_key, user_partitions={Mock(name='partition', id=CONTENT_GATING_PARTITION_ID): object()} ) + # enabled_for_course() starts with `if not correct_modes_for_fbe(...)`, + # so the course needs audit and verified modes before the config's + # enabled=True has any effect. Without them this returns None at the + # first branch and never reaches the code under test. It passed + # previously only because the ConfigurationModel cache was polluted by + # test_create_content_gating_partition_happy_path, which does create them. + CourseModeFactory.create(course_id=mock_course.id, mode_slug='audit') + CourseModeFactory.create(course_id=mock_course.id, mode_slug='verified') ContentTypeGatingConfig.objects.create(enabled=True, enabled_as_of=datetime(2018, 1, 1)) with patch('openedx.features.content_type_gating.partitions.LOG') as mock_log: diff --git a/scripts/verify_test_suite_coverage.py b/scripts/verify_test_suite_coverage.py new file mode 100644 index 000000000000..2d81d55bea9c --- /dev/null +++ b/scripts/verify_test_suite_coverage.py @@ -0,0 +1,60 @@ +""" +Fail if any test file belongs to no shard in unit-test-shards.json. + +That file is maintained by hand and cannot simply be the repository roots: a few +apps under openedx/ -- content_staging, content_tagging -- are Studio-only and +raise at import under lms settings, so the lms and cms shards each cover part of +the tree rather than either covering all of it. The list therefore drifts when a +new Django app is added, and a drifting list means tests that silently never run. + +Replaces collect-and-verify, which compared test *counts* between the shard list +and the roots. That needed every Python dependency and two full pytest +collections (~2 minutes) to report only that two numbers disagreed. Comparing +paths reads the file tree in a few seconds and names the directory at fault. +""" +import json +import pathlib +import sys + +SHARDS_JSON = '.github/workflows/unit-test-shards.json' +ROOTS = ('lms', 'cms', 'openedx', 'common/djangoapps', 'xmodule') +TEST_GLOBS = ('test_*.py', 'tests.py', 'tests_*.py', '*_tests.py') +# Mirrors norecursedirs in pyproject.toml, plus trees that hold fixtures rather +# than collectable tests. +SKIP = ('node_modules', '/envs/', '/migrations/', 'test_root', '/.git/', '/features/') + + +def covered_paths(): + with open(SHARDS_JSON) as shards_file: + shards = json.load(shards_file) + return tuple({path for shard in shards.values() for path in shard['paths']}) + + +def find_test_files(): + for root in ROOTS: + for glob in TEST_GLOBS: + for path in pathlib.Path(root).rglob(glob): + text = str(path) + if not any(skip in f'/{text}' for skip in SKIP): + yield text + + +def main(): + prefixes = covered_paths() + uncovered = sorted({ + str(pathlib.Path(f).parent) for f in find_test_files() + if not f.startswith(prefixes) + }) + if uncovered: + print("::error title=unit-test-shards.json is out of date::" + "These directories contain tests that no shard in " + f"{SHARDS_JSON} covers, so they never run in CI. Add them to a " + "shard -- a cms.envs.test one if they need Studio settings.") + for directory in uncovered: + print(f" {directory}") + sys.exit(1) + print(f"All test files are covered by {len(prefixes)} shard paths.") + + +if __name__ == "__main__": + main() diff --git a/xmodule/modulestore/tests/django_utils.py b/xmodule/modulestore/tests/django_utils.py index d4a2030ac496..563223d62509 100644 --- a/xmodule/modulestore/tests/django_utils.py +++ b/xmodule/modulestore/tests/django_utils.py @@ -6,6 +6,7 @@ import copy import functools import os +import warnings from contextlib import contextmanager from enum import Enum from mimetypes import guess_type @@ -260,6 +261,21 @@ def enable_signals_by_name(cls, *signal_names): signal.enable() +def _contentstore_is_readable(): + """Can settings.CONTENTSTORE be read right now? + + UserSettingsHolder raises AttributeError for a setting in its _deleted set, + so getattr with a default is not enough on its own -- the point is to + observe whether the lookup succeeds, not what it returns. + """ + try: + # The attribute access *is* the probe; both linters need telling. + settings.CONTENTSTORE # noqa: B018 # pylint: disable=pointless-statement + except AttributeError: + return False + return True + + class ModuleStoreIsolationMixin(CacheIsolationMixin, SignalIsolationMixin): """ A mixin to be used by TestCases that want to isolate their use of the @@ -300,6 +316,16 @@ def my_test(self): __old_modulestores = [] __old_contentstores = [] + # Number of isolations this exact class currently has open. Read from + # cls.__dict__ rather than as a normal attribute so that subclasses never + # see (and never decrement) a parent's count. + _ISOLATION_DEPTH_ATTR = '_modulestore_isolation_depth_count' + + @classmethod + def _modulestore_isolation_depth(cls): + """How many isolations this exact class currently has open.""" + return cls.__dict__.get(cls._ISOLATION_DEPTH_ATTR, 0) + @classmethod def start_modulestore_isolation(cls): """ @@ -307,21 +333,68 @@ def start_modulestore_isolation(cls): :py:meth:`end_modulestore_isolation` is called, this modulestore will be flushed (all content will be deleted). """ + if not _contentstore_is_readable(): + # Warn rather than fail. This isolation overrides CONTENTSTORE + # anyway, and the old value is only kept to assert against at + # teardown, so a missing one is recoverable here. Raising instead + # turned a single upstream problem into hundreds of downstream + # errors and hid where it came from. + warnings.warn( + f"settings.CONTENTSTORE was missing when {cls.__name__} started " + "modulestore isolation; an earlier override_settings frame is " + "masking it. Proceeding, since this isolation overrides it.", + RuntimeWarning, + stacklevel=2, + ) + cls.disable_all_signals() cls.enable_signals_by_name(*cls.ENABLED_SIGNALS) cls.start_cache_isolation() - override = override_settings( - MODULESTORE=cls.MODULESTORE(), - CONTENTSTORE=cls.CONTENTSTORE(), - ) - cls.__old_modulestores.append(copy.deepcopy(settings.MODULESTORE)) - cls.__old_contentstores.append(copy.deepcopy(settings.CONTENTSTORE)) - override.__enter__() # pylint: disable=unnecessary-dunder-call + # Cache isolation is now held but the modulestore isolation is not, and + # end_modulestore_isolation() keys off the modulestore depth: if it is + # zero that call returns immediately and never unwinds the cache. So + # anything that raises between here and the depth increment below would + # strand a CACHES override for the rest of the process, which surfaces + # later as InvalidCacheBackendError or KeyError on a named cache rather + # than as an error here. cls.MODULESTORE() and cls.CONTENTSTORE() are + # arbitrary callables and copy.deepcopy is not total, so this is a real + # window, not a theoretical one. Undo the cache isolation and the signal + # changes if it happens. + try: + override = override_settings( + MODULESTORE=cls.MODULESTORE(), + CONTENTSTORE=cls.CONTENTSTORE(), + ) + + # getattr with a default: these snapshots exist only to assert against + # at teardown, and the override replaces both settings regardless, so a + # missing value must not stop isolation from being established. + old_modulestore = copy.deepcopy(getattr(settings, 'MODULESTORE', None)) + old_contentstore = copy.deepcopy(getattr(settings, 'CONTENTSTORE', None)) + override.__enter__() # pylint: disable=unnecessary-dunder-call + except Exception: + cls.end_cache_isolation() + cls.enable_all_signals() + raise + + # settings is global and now mutated. Record the isolation before doing + # anything else that can raise, so that a failure below is still + # unwindable; otherwise the override leaks for the rest of the process + # and every later start_modulestore_isolation() reads a settings object + # whose CONTENTSTORE has been restored away. + cls.__old_modulestores.append(old_modulestore) + cls.__old_contentstores.append(old_contentstore) cls.__settings_overrides.append(override) - XMODULE_FACTORY_LOCK.enable() - clear_existing_modulestores() - cls.store = modulestore() + setattr(cls, cls._ISOLATION_DEPTH_ATTR, cls._modulestore_isolation_depth() + 1) + + try: + XMODULE_FACTORY_LOCK.enable() + clear_existing_modulestores() + cls.store = modulestore() + except Exception: + cls.end_modulestore_isolation() + raise @classmethod def end_modulestore_isolation(cls): @@ -329,15 +402,40 @@ def end_modulestore_isolation(cls): Delete all content in the Modulestore, and reset the Modulestore settings from before :py:meth:`start_modulestore_isolation` was called. + + Does nothing if this class has no isolation open, so that it is safe to + call both from an explicit tearDownClass and from a registered cleanup. """ - drop_mongo_collections() # pylint: disable=no-value-for-parameter - XMODULE_FACTORY_LOCK.disable() - cls.__settings_overrides.pop().__exit__(None, None, None) + if cls._modulestore_isolation_depth() <= 0: + return + setattr(cls, cls._ISOLATION_DEPTH_ATTR, cls._modulestore_isolation_depth() - 1) + + # Everything below must run even if an earlier step raises. Unwinding the + # settings override is the part that matters: an override_settings object + # is single-use (Django's disable() does `del self.wrapped`), so a frame + # that is skipped here can never be unwound later, and every subsequent + # test in this process reads a settings object that has been rewound past + # the modulestore overrides. drop_mongo_collections() in particular can + # fail under load, which is how one bad teardown used to poison a whole + # xdist worker. + try: + drop_mongo_collections() # pylint: disable=no-value-for-parameter + finally: + try: + XMODULE_FACTORY_LOCK.disable() + override = cls.__settings_overrides.pop() + old_modulestore = cls.__old_modulestores.pop() + old_contentstore = cls.__old_contentstores.pop() + + override.__exit__(None, None, None) + if old_modulestore is not None: + assert settings.MODULESTORE == old_modulestore + if old_contentstore is not None: + assert settings.CONTENTSTORE == old_contentstore - assert settings.MODULESTORE == cls.__old_modulestores.pop() - assert settings.CONTENTSTORE == cls.__old_contentstores.pop() - cls.end_cache_isolation() - cls.enable_all_signals() + finally: + cls.end_cache_isolation() + cls.enable_all_signals() @staticmethod def allow_transaction_exception(): @@ -458,10 +556,16 @@ def setUpTestData(cls): """ cls.start_modulestore_isolation() - # Now yield to allow the test class to run its setUpClass() setup code. - yield - # Now call the base class, which calls back into the test class's setUpTestData(). - super().setUpClass() + try: + # Now yield to allow the test class to run its setUpClass() setup code. + yield + # Now call the base class, which calls back into the test class's setUpTestData(). + super().setUpClass() + except Exception: + # unittest skips tearDownClass when setUpClass raises, so without this + # the isolation above would leak into every later class in the process. + cls.end_modulestore_isolation() + raise @classmethod def setUpClass(cls): @@ -471,6 +575,11 @@ def setUpClass(cls): """ super().setUpClass() cls.start_modulestore_isolation() + # tearDownClass is skipped entirely when a subclass's setUpClass raises + # after this point; a class cleanup still runs, so register one as a + # backstop. end_modulestore_isolation() is a no-op once the isolation + # has already been ended, so the normal path is unaffected. + cls.addClassCleanup(cls.end_modulestore_isolation) @classmethod def tearDownClass(cls): diff --git a/xmodule/modulestore/tests/factories.py b/xmodule/modulestore/tests/factories.py index f3f4cdf86d7c..ccb2b5079636 100644 --- a/xmodule/modulestore/tests/factories.py +++ b/xmodule/modulestore/tests/factories.py @@ -46,7 +46,11 @@ class XModuleFactoryLock(threading.local): def __init__(self): super().__init__() - self._enabled = False + # A count rather than a flag: modulestore isolation nests (a + # SharedModuleStoreTestCase class-level isolation can contain further + # isolations), and with a plain boolean the inner disable() switched + # factories off while the outer scope was still using them. + self._depth = 0 def enable(self): """ @@ -54,20 +58,20 @@ def enable(self): where the modulestore will be reset at the end of the test (such as inside ModuleStoreTestCase). """ - self._enabled = True + self._depth += 1 def disable(self): """ Disable XModuleFactories. This should be called once the data from the factory has been cleaned up. """ - self._enabled = False + self._depth = max(0, self._depth - 1) def is_enabled(self): """ Return whether XModuleFactories are enabled. """ - return self._enabled + return self._depth > 0 XMODULE_FACTORY_LOCK = XModuleFactoryLock()