SONARPY-4604 Moved from tox to uv

GitOrigin-RevId: 387df7b0632f99c8e50cc172fc1fb3ea1842bcdb

Author: joke1196

python-frontend/pom.xml

@@ -179,12 +179,12 @@
             </configuration>
             <executions>
               <execution>
-                <id>tox-runner</id>
+                <id>serializer-runner</id>
                 <phase>generate-resources</phase>
                 <configuration>
                   <executable>python</executable>
                   <arguments>
-                      <argument>runners/tox_runner.py</argument>
+                      <argument>runners/serializer_runner.py</argument>
                       <argument>--skip_tests</argument>
                       <argument>${skipTests}</argument>
                       <argument>--fail_fast</argument>

python-frontend/typeshed_serializer/README.md

@@ -2,38 +2,48 @@ To run:
 
 in `python-frontend/src/main/protobuf` directory, run: ```protoc -I=. --python_out=../../../typeshed_serializer/serializer/proto_out ./symbols.proto```
 
-```pip install -r requirements.txt```
+```uv sync```
 
+## Run the serializer
+
+```bash
+uv run python runners/serializer_runner.py
+```
+
+## Run the test suite
+
+```bash
+uv run python -m pytest tests/
+```
 
 ## Rebuild only custom symbols
 
 ```bash
-tox -e selective-serialize -- custom
+uv run python -m utils.folder_manager custom && uv run python -m serializer.typeshed_serializer custom
 ```
 
 ## Run a custom test
 
 ```bash
-tox -e py314 -- tests/runners/test_tox_runner.py
+uv run python -m pytest tests/runners/test_serializer_runner.py
 ```
 ## Run one specific unit test
 
 ```bash
-tox -e py314 -- tests/runners/test_tox_runner.py::ToxRunnerTest::test_dry_run_unchanged_checksums -v
+uv run python -m pytest tests/runners/test_serializer_runner.py::RunnerTest::test_dry_run_unchanged_checksums -v
 ```
 
-## Dry run of tox_runner
+## Dry run of runner
 
-- Use python, not tox.
-- Will show which calls to the `tox` module would have been triggered, depending on the checksums and file-system state.
+- Will show which calls would have been triggered, depending on the checksums and file-system state.
 - Will not perform any change.
 
 ```bash
-python runners/tox_runner.py --dry_run true 
+uv run python runners/serializer_runner.py --dry_run true
 ```
 
 Can also be run in fail fast mode, to reflect the checksum validation used by the QA workflow
 
 ```bash
-python runners/tox_runner.py --dry_run true --fail_fast true  
+uv run python runners/serializer_runner.py --dry_run true --fail_fast true
 ```

python-frontend/typeshed_serializer/docker/Dockerfile

@@ -8,14 +8,14 @@ RUN apt-get update && \
 # This command will print the version of protoc, confirming its installation
 RUN protoc --version
 
-# Install tox
-RUN pip install --no-cache-dir tox
+# Install uv
+RUN pip install --no-cache-dir uv
 
 # Add typeshed serializer to the image container
 COPY --chmod=777 ./docker/cmd.sh /sonar-python/cmd.sh
 
 # Make run script executable
 RUN chmod +x /sonar-python/cmd.sh
 
-# Run runners/tox_runner.py when the container launches
+# Run runners/runner.py when the container launches
 CMD ["/sonar-python/cmd.sh"]

python-frontend/typeshed_serializer/docker/cmd.sh

@@ -7,6 +7,6 @@ cd /sonar-python/python-frontend/src/main/protobuf
 protoc -I=. --python_out=../../../typeshed_serializer/serializer/proto_out ./symbols.proto
 
 cd /sonar-python/python-frontend/typeshed_serializer
-# Recreate the env to make sure that latest dependencies will be downloaded
-tox --recreate --notest
-python runners/tox_runner.py --skip_tests=false
+# Sync dependencies from uv.lock
+uv sync
+uv run python runners/serializer_runner.py --skip_tests=false

python-frontend/typeshed_serializer/pyproject.toml

@@ -0,0 +1,92 @@
+#
+# SonarQube Python Plugin
+# Copyright (C) SonarSource Sàrl
+# mailto:info AT sonarsource DOT com
+#
+# You can redistribute and/or modify this program under the terms of
+# the Sonar Source-Available License Version 1, as published by SonarSource Sàrl.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See the Sonar Source-Available License for more details.
+#
+# You should have received a copy of the Sonar Source-Available License
+# along with this program; if not, see https://sonarsource.com/license/ssal/
+#
+
+[project]
+name = "typeshed-serializer"
+version = "0.1.0"
+requires-python = ">=3.14"
+dependencies = [
+  # Keep the mypy 2.x semantic-model migration separate from the Python 3.14 host upgrade.
+  "mypy==1.19.1",
+  "mypy-extensions==1.1.0",
+  "protobuf==5.29.3",
+  "six==1.16.0",
+  "toml==0.10.2",
+  "typing-extensions==4.16.0",
+  "pytest==9.1.1",
+  "pytest-cov==7.1.0",
+
+  # Stub dependencies
+  "atomicwrites==1.4.1",
+  "backports-abc==0.5",
+  "certifi==2023.5.7",
+  "characteristic==14.3.0",
+  "click==8.1.3",
+  "cryptography==36.0.1",
+  "dataclasses==0.6",
+  "emoji==2.1.0",
+  "types-emoji==2.1.0.3",
+  "enum34==1.1.10",
+  "filelock==3.4.0",
+  "first==2.0.2",
+  "Flask==2.3.2",
+  "Flask-Cors==3.0.10",
+  "types-Flask-Cors==3.0.10.5",
+  "freezegun==1.2.2",
+  "frozenlist==1.4.1",
+  "geoip2==4.7.0",
+  "ipaddress==1.0.23",
+  "itsdangerous==2.1.2",
+  "Jinja2==3.1.2",
+  "kazoo==2.9.0",
+  "MarkupSafe==2.1.2",
+  "maxminddb==2.3.0",
+  "orjson==3.12.0",
+  "pandas==2.3.3",
+  "pandas-stubs==2.2.3.241009",
+  "pathlib==1.0.1",
+  "PyJWT==2.7.0",
+  "PyYAML==6.0.1",
+  "pyspark==3.5.4",
+  "types-PyYAML==6.0.12.20240311",
+  "requests==2.30.0",
+  "types-requests==2.30.0.0",
+  "Routes==2.5.1",
+  "termcolor==2.3.0",
+  "tornado==6.3.1",
+  "Werkzeug==2.3.4",
+  "xxhash==3.2.0",
+  # These versions preserve the FastAPI/Starlette APIs modeled by existing rules.
+  # They are build-only dependencies; the analyzer does not ship or execute them.
+  "fastapi==0.98.0",
+  # FastAPI requires Pydantic v1. Custom Pydantic modules are serialized separately
+  # and take precedence over imported modules when the analyzer loads descriptors.
+  "pydantic==1.10.26",
+  "starlette==0.27.0",
+
+  # Lock the transitive dependency from pandas and pandas-stubs. NumPy 2.5 stubs
+  # use Python 3.12 type statements, while the serializer also targets Python 3.10.
+  "numpy==2.4.2",
+]
+
+[tool.pytest.ini_options]
+addopts = "--cov=serializer --cov=runners --cov-report xml:cov.xml --ignore=resources --cov-branch"
+
+[[tool.uv.index]]
+name = "repox"
+url = "https://repox.jfrog.io/artifactory/api/pypi/sonarsource-pypi/simple"
+default = true

python-frontend/typeshed_serializer/requirements.txt

@@ -65,7 +65,7 @@
     },
 }
 
-logger = logging.getLogger('tox_runner')
+logger = logging.getLogger('runner')
 handler = logging.StreamHandler(sys.stdout)
 log_formatter = logging.Formatter(fmt='%(name)s [%(levelname)s] --- %(message)s ---')
 logger.setLevel(logging.INFO)
@@ -91,7 +91,7 @@ def fetch_resource_file_names(folder_name: str, file_extension: str) -> list[str
 
 
 def fetch_config_file_names() -> list[str]:
-    return [os.path.join(CURRENT_PATH, '../requirements.txt'), os.path.join(CURRENT_PATH, '../tox.ini')]
+    return [os.path.join(CURRENT_PATH, '../pyproject.toml'), os.path.join(CURRENT_PATH, '../uv.lock')]
 
 def fetch_source_file_names(folder_path: str) -> list[str]:
     filenames = fetch_python_file_names(folder_path)
@@ -238,26 +238,31 @@ def compute_and_compare_checksums(check_type: str, folder_name: str, folder_file
 
 
 
-def get_serialize_command_to_run(previous_source_checksum: Optional[str], current_sources_checksum: str, changed_serializers: List[str]) -> Optional[List[str]]:
-    """Determine the serialization command to run based on checksums and changed serializers.
+def get_serialize_command_to_run(previous_source_checksum: Optional[str], current_sources_checksum: str, changed_serializers: List[str]) -> Optional[List[List[str]]]:
+    """Determine the serialization commands to run based on checksums and changed serializers.
 
     Args:
         previous_source_checksum: Previous checksum of source files
         current_sources_checksum: Current checksum of source files
         changed_serializers: List of serializers that have changed
 
     Returns:
-        Command to run as list of strings, or None if no serialization needed
+        List of commands to run sequentially as list of string lists, or None if no serialization needed
     """
     if previous_source_checksum != current_sources_checksum:
         # Serializer code has changed - run full serialization
         logger.info('SERIALIZER CODE HAS CHANGED - STARTING FULL TYPESHED SERIALIZATION')
-        return ['tox', '-e', 'serialize']
+        return [
+            ['uv', 'run', 'python', '-m', 'utils.folder_manager'],
+            ['uv', 'run', 'python', '-m', 'serializer.typeshed_serializer'],
+        ]
     elif changed_serializers:
         logger.info(f"STARTING SELECTIVE TYPESHED SERIALIZATION FOR: {','.join(changed_serializers)}")
-        # Run selective serialization through tox environment
         serializers_arg = ','.join(changed_serializers)
-        return ['tox', '-e', 'selective-serialize', '--', serializers_arg]
+        return [
+            ['uv', 'run', 'python', '-m', 'utils.folder_manager', serializers_arg],
+            ['uv', 'run', 'python', '-m', 'serializer.typeshed_serializer', serializers_arg],
+        ]
     else:
         logger.info('SKIPPING TYPESHED SERIALIZATION')
         return None
@@ -276,33 +281,45 @@ def update_folder_checksums_for_changed_serializers(changed_serializers: List[st
                 write_folder_checksum(folder_name, source_checksum, binary_checksum)
 
 
+def _run_serialize(commands: List[List[str]], full_serialization: bool, changed_serializers: List[str], dry_run: bool) -> None:
+    if dry_run:
+        for cmd in commands:
+            logger.info(f'DRY RUN: Would execute: {" ".join(cmd)}')
+        return
+    for cmd in commands:
+        subprocess.run(cmd, check=True)
+    if full_serialization:
+        update_all_checksums()
+    else:
+        update_folder_checksums_for_changed_serializers(changed_serializers)
+
+
+def _run_tests(dry_run: bool) -> None:
+    test_cmd = ['uv', 'run', 'python', '-m', 'pytest', 'tests/']
+    if dry_run:
+        logger.info(f'DRY RUN: Would run: {" ".join(test_cmd)}')
+    else:
+        subprocess.run(test_cmd, check=True)
+
+
 def main(skip_tests=False, fail_fast=False, dry_run=False):
-    # Check if serializer source code has changed
     source_files = fetch_source_file_names(SERIALIZER_PATH)
     current_sources_checksum = compute_checksum(source_files, normalize_text_files)
     previous_sources_checksum = read_previous_checksum(SERIALIZER_SOURCE_CHECKSUM_FILE)
-    serializer_sources_changed = compute_and_compare_checksums("SERIALIZER_SOURCE", SERIALIZER_PATH, source_files, normalize_text_files, previous_sources_checksum)
+    compute_and_compare_checksums("SERIALIZER_SOURCE", SERIALIZER_PATH, source_files, normalize_text_files, previous_sources_checksum)
 
-    if serializer_sources_changed and fail_fast :
+    if previous_sources_checksum != current_sources_checksum and fail_fast:
         raise RuntimeError('INCONSISTENT SOURCES CHECKSUMS')
 
     changed_serializers = detect_required_serializations(fail_fast)
-    serialize_command_to_run = get_serialize_command_to_run(previous_sources_checksum, current_sources_checksum, changed_serializers)
+    full_serialization = previous_sources_checksum != current_sources_checksum
+    serialize_commands = get_serialize_command_to_run(previous_sources_checksum, current_sources_checksum, changed_serializers)
 
-    # Execute or display the serialize command
-    if serialize_command_to_run:
-        if dry_run:
-            logger.info(f'DRY RUN: Would execute: {" ".join(serialize_command_to_run)}')
-        else:
-            _ = subprocess.run(serialize_command_to_run, check=True)
-            update_folder_checksums_for_changed_serializers(changed_serializers)
+    if serialize_commands:
+        _run_serialize(serialize_commands, full_serialization, changed_serializers, dry_run)
 
-    # Run tests after serialization (unless skip_tests=True)
     if not skip_tests:
-        if dry_run:
-            logger.info('DRY RUN: Would run test')
-        else:
-            _ = subprocess.run(['tox', '-e', 'py314'], check=True)
+        _run_tests(dry_run)
 
 
 if __name__ == '__main__':

…ypeshed_serializer/runners/tox_runner.py …_serializer/runners/serializer_runner.pypython-frontend/typeshed_serializer/runners/tox_runner.py renamed to python-frontend/typeshed_serializer/runners/serializer_runner.py python-frontend/typeshed_serializer/runners/tox_runner.py renamed to python-frontend/typeshed_serializer/runners/serializer_runner.py

@@ -1 +1 @@
-6f00fdc8583bdda94825a556205a538829ee31371722518613d50800d2ddf380
+4384976d2c4b1c24d007d635f81e8080d05481397e998148f379151591d510f7