diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..d311028 --- /dev/null +++ b/.flake8 @@ -0,0 +1,5 @@ +[flake8] +max-line-length = 99 +exclude = + .venv + diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml new file mode 100644 index 0000000..ad494fb --- /dev/null +++ b/.github/workflows/main.yaml @@ -0,0 +1,68 @@ +name: main + +on: + push: + branches: [main, develop] + pull_request: + +jobs: + flake8: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4.1.0 + with: + python-version: 3.12 + + - name: Install flake8 + run: pip --disable-pip-version-check install flake8 + + - name: Lint with flake8 + run: flake8 --count + + tests: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13"] + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e . + pip install -r requirements-test.txt + + - name: Test with pytest + run: | + make test + + # - name: Upload coverage reports to Codecov + # if: (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/develop') && matrix.python-version == '3.12' + # uses: codecov/codecov-action@v4.0.1 + # with: + # token: ${{ secrets.CODECOV_TOKEN }} + + build-and-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4.1.0 + + - name: Build docker image + run: make docker-build + + - name: Run tests + run: | + make docker-ci-test diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..712dd12 --- /dev/null +++ b/.gitignore @@ -0,0 +1,169 @@ + +# Created by https://www.toptal.com/developers/gitignore/api/python +# Edit at https://www.toptal.com/developers/gitignore?templates=python + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +.idea/ + +#OSX +.DS_Store + +#Jupyter +.ipynb + +#terraform +.terraform +.terraform.lock.hcl +terraform.tfstate +terraform.tfstate.backup + +# project stuff +scripts/config.sh +test/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..768c280 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,47 @@ +repos: + +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-ast + - id: check-added-large-files + - id: check-yaml + args: [--unsafe] + - id: check-toml + - id: debug-statements + - id: end-of-file-fixer + - id: fix-byte-order-marker + - id: name-tests-test + args: [--pytest-test-first] + - id: mixed-line-ending + - id: trailing-whitespace + +- repo: local + hooks: + - id: lint + name: check code standards (lint) + entry: make lint + types: [python] + language: system + pass_filenames: false + + - id: codespell + name: check code for misspellings (codespell) + entry: codespell + types: [text] + language: python + pass_filenames: false + additional_dependencies: + - tomli + + - id: typecheck + name: check static types (mypy) + entry: make typecheck + types: [python] + language: system + pass_filenames: false + +#- repo: https://github.com/commitizen-tools/commitizen +# rev: v4.2.2 +# hooks: +# - id: commitizen diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..280f5c6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,69 @@ +# --------------------------------------------------------------------------------------- +# BASE IMAGE +# --------------------------------------------------------------------------------------- +FROM python:3.12.10-slim-bookworm AS base + +# Setup a volume for configuration and authtentication. +VOLUME ["/root/.config"] + +# Update system and install build tools. Remove unneeded stuff afterwards. +# Upgrade PIP. +# Create working directory. +RUN apt-get update && \ + apt-get install -y --no-install-recommends gcc g++ build-essential && \ + rm -rf /var/lib/apt/lists/* && \ + pip install --upgrade pip && \ + mkdir -p /opt/project + +# Set working directory. +WORKDIR /opt/project + +# --------------------------------------------------------------------------------------- +# DEPENDENCIES IMAGE (installed project dependencies) +# --------------------------------------------------------------------------------------- +# We do this first so when we modify code while development, this layer is reused +# from cache and only the layer installing the package executes again. +FROM base AS deps +COPY requirements.txt . +RUN pip install -r requirements.txt + +# --------------------------------------------------------------------------------------- +# Apache Beam integration IMAGE +# --------------------------------------------------------------------------------------- +FROM deps AS beam +# Copy files from official SDK image, including script/dependencies. +# IMPORTANT: This version must match the one in requirements.txt +COPY --from=apache/beam_python3.12_sdk:2.64.0 /opt/apache/beam /opt/apache/beam + +# Set the entrypoint to Apache Beam SDK launcher. +ENTRYPOINT ["/opt/apache/beam/boot"] + +# --------------------------------------------------------------------------------------- +# PRODUCTION IMAGE +# --------------------------------------------------------------------------------------- +# If you need Apache Beam integration, replace "deps" base image with "beam". +FROM deps AS prod + +COPY . /opt/project +RUN pip install . && \ + rm -rf /root/.cache/pip && \ + rm -rf /opt/project/* + +# --------------------------------------------------------------------------------------- +# DEVELOPMENT IMAGE (editable install and development tools) +# --------------------------------------------------------------------------------------- +# If you need Apache Beam integration, replace "deps" base image with "beam". +FROM deps AS dev + +COPY . /opt/project +RUN pip install -e .[lint,test,dev,build] + +# --------------------------------------------------------------------------------------- +# TEST IMAGE (This checks that package is properly installed in prod image) +# --------------------------------------------------------------------------------------- +FROM prod AS test + +COPY ./tests /opt/project/tests +COPY ./requirements-test.txt /opt/project/ + +RUN pip install -r requirements-test.txt \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..e33d4e8 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,6 @@ +include LICENSE +include README.md + +recursive-include src/**/assets * +recursive-exclude * __pycache__ +recursive-exclude * *.py[cod] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..2139d2a --- /dev/null +++ b/Makefile @@ -0,0 +1,154 @@ +.DEFAULT_GOAL:=help + +VENV_NAME:=.venv +REQS_PROD:=requirements.txt +DOCKER_DEV_SERVICE:=dev +DOCKER_CI_TEST_SERVICE:=test + +GCP_PROJECT:=world-fishing-827 +GCP_DOCKER_VOLUME:=gcp + +sources = src + +# --------------------- +# DOCKER +# --------------------- + +.PHONY: docker-build ## Builds docker image. +docker-build: + docker compose build + +.PHONY: docker-volume ## Creates the docker volume for GCP. +docker-volume: + docker volume create --name ${GCP_DOCKER_VOLUME} + +.PHONY: docker-gcp ## gcp: Authenticates to google cloud and configure the project. +docker-gcp: + make docker-volume + docker compose run gcloud auth application-default login + docker compose run gcloud config set project ${GCP_PROJECT} + docker compose run gcloud auth application-default set-quota-project ${GCP_PROJECT} + +.PHONY: docker-ci-test ## Runs tests using prod image, exporting coverage.xml report. +docker-ci-test: + docker compose run --rm ${DOCKER_CI_TEST_SERVICE} + +.PHONY: docker-shell ## Enters to docker container shell. +docker-shell: + docker compose run --rm -it ${DOCKER_DEV_SERVICE} + +.PHONY: reqs ## Compiles requirements.txt with pip-tools. +reqs: + docker compose run --rm ${DOCKER_DEV_SERVICE} -c \ + 'pip-compile -o ${REQS_PROD} -v' + +.PHONY: reqs-upgrade ## Upgrades requirements.txt with pip-tools. +reqs-upgrade: + docker compose run --rm ${DOCKER_DEV_SERVICE} -c \ + 'pip-compile -o ${REQS_PROD} -U -v' + +# --------------------- +# VIRTUAL ENVIRONMENT +# --------------------- + +.PHONY: venv ## Creates virtual environment. +venv: + python -m venv ${VENV_NAME} + +.PHONY: install ## Install the package and dependencies for local development. +install: + python -m pip install -U pip + python -m pip install -e .[lint,dev,build] + python -m pip install -r requirements-test.txt + +.PHONY: test ## Run all unit tests exporting coverage.xml report. +test: + python -m pytest -m "not integration" --cov-report term --cov-report=xml --cov=$(sources) + +# --------------------- +# QUALITY CHECKS +# --------------------- + +.PHONY: hooks ## Install and pre-commit hooks. +hooks: + python -m pre_commit install --install-hooks + python -m pre_commit install --hook-type commit-msg + +.PHONY: format ## Auto-format python source files according with PEP8. +format: + python -m black $(sources) + python -m ruff check --fix $(sources) + python -m ruff format $(sources) + +.PHONY: lint ## Lint python source files. +lint: + python -m ruff check $(sources) + python -m ruff format --check $(sources) + python -m black $(sources) --check --diff + +.PHONY: codespell ## Use Codespell to do spell checking. +codespell: + python -m codespell_lib + +.PHONY: typecheck ## Perform type-checking. +typecheck: + python -m mypy + +.PHONY: audit ## Use pip-audit to scan for known vulnerabilities. +audit: + python -m pip_audit . + +.PHONY: pre-commit ## Run all pre-commit hooks. +pre-commit: + python -m pre_commit run --all-files + +.PHONY: all ## Run the standard set of checks performed in CI. +all: lint codespell typecheck audit test + +# --------------------- +# PACKAGE BUILD +# --------------------- + + +.PHONY: build ## Build a source distribution and a wheel distribution. +build: all clean + python -m build + +.PHONY: publish ## Publish the distribution to PyPI. +publish: build + python -m twine upload dist/* --verbose + +.PHONY: clean ## Clear local caches and build artifacts. +clean: + # remove Python file artifacts + rm -rf `find . -name __pycache__` + rm -f `find . -type f -name '*.py[co]'` + rm -f `find . -type f -name '*~'` + rm -f `find . -type f -name '.*~'` + rm -rf .cache + rm -rf .mypy_cache + rm -rf .ruff_cache + # remove build artifacts + rm -rf build + rm -rf dist + rm -rf `find . -name '*.egg-info'` + rm -rf `find . -name '*.egg'` + # remove test and coverage artifacts + rm -rf .tox/ + rm -f .coverage + rm -f .coverage.* + rm -rf coverage.* + rm -rf htmlcov/ + rm -rf .pytest_cache + rm -rf htmlcov + + +# --------------------- +# HELP +# --------------------- + +.PHONY: help ## Display this message +help: + @grep -E \ + '^.PHONY: .*?## .*$$' $(MAKEFILE_LIST) | \ + awk 'BEGIN {FS = ".PHONY: |## "}; {printf "\033[36m%-19s\033[0m %s\n", $$2, $$3}' \ No newline at end of file diff --git a/README.md b/README.md index 99d8932..ab84e25 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,277 @@ -# python-app-template -A template for python (dockerized) applications. +

python-app-template

+ +

+ + + + + + + + Python versions + + + Last release + +

+ +A template for Python applications. + +**Features**: +* :white_check_mark: Standard Python project structure & packaging. +* :white_check_mark: Dependency management with [pip-tools]. +* :white_check_mark: Tools for quality checks: documentation, [PEP8], typehints, codespell. +* :white_check_mark: **[Optional]** pre-commit hooks to enforce automatic quality checks. +* :white_check_mark: Dockerization with focus in image size optimization. +* :white_check_mark: Continuous Integration (CI) workflows (GitHub Actions). +* :white_check_mark: Continuous Deployment (CI) workflows (Google Cloud Build). +* :white_check_mark: Makefile with shortcuts to increase development speed. +* :white_check_mark: README badges with project information. +* :white_check_mark: Development workflow documentation. +* :white_check_mark: Support for [Apache Beam] integrated pipelines. + + +[Apache Beam]: https://beam.apache.org +[branch protection rules]: https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/managing-protected-branches/managing-a-branch-protection-rule +[codecov]: https://about.codecov.io +[docker compose]: https://docs.docker.com/compose/install/linux/ +[Google BigQuery]: https://cloud.google.com/bigquery +[Google Dataflow]: https://cloud.google.com/products/dataflow?hl=en +[Markdown alerts]: https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax#alerts +[mermaid diagram]: https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/creating-diagrams +[PEP8]: https://peps.python.org/pep-0008/ +[pip-tools]: https://pip-tools.readthedocs.io/en/stable/ +[pre-commit]: https://pre-commit.com +[pytest]: https://docs.pytest.org/en/stable/ +[used by pytest-cov]: https://pytest-cov.readthedocs.io/en/latest/config.html + +[cli.py]: src/python_app_template/cli.py +[docs/contributing]: docs/contributing +[examples]: examples/ + + +[.github/]: .github/ +[docs/]: docs/ +[notebooks/]: notebooks/ +[src/python_app_template/]: src/python_app_template +[src/python_app_template/assets/]: src/python_app_template/assets/ +[tests/]: tests/ + +[.coveragerc]: .coveragerc +[.flake8]: .flake8 +[.gitignore]: .gitignore +[.pre-commit-config.yaml]: .pre-commit-config.yaml +[activate_venv.sh]: activate-venv.sh +[cloudbuild.yaml]: cloudbuild.yaml +[codecov.yml]: codecov.yml +[docker-compose.yml]: docker-compose.yml +[Dockerfile]: Dockerfile +[GIT-WORKFLOW.md]: GIT-WORKFLOW.md +[LICENSE]: LICENSE +[Makefile]: Makefile +[MANIFEST.in]: MANIFEST.in +[pyproject.toml]: pyproject.toml +[pytest.ini]: pytest.ini +[requirements.txt]: requirements.txt +[requirements-test.txt]: requirements-test.txt +[README.md]: README.md +[setup.py]: setup.py + + +[Preparing the environment]: docs/contributing/ENVIRONMENT.md +[Making changes]: docs/contributing/MAKING-CHANGES.md +[Git Workflow]: docs/contributing/GIT-WORKFLOW.md +[Managing dependencies]: docs/contributing/DEPENDENCIES.md +[How to deploy]: docs/contributing/HOW-TO-DEPLOY.md + +## Introduction + +
+ +_Write the motivation for this application here._ +_When applicable, include citations for relevant articles or resources for further reading._ + +The motivation for this repository is to provide a robust, +well-documented template for Python applications, +including but not limited to GFW data pipelines. + +Goals of this template include: +* Reducing the overhead of creating a new Dockerized Python application. +* Minimizing the effort required to transition prototypes to production. +* Establishing consistent development standards across projects. + +## Usage + +_Write the relevant documentation on how to use this application here. +Ideally, organize it into sections_. + +_Assume in this section that a package has been published in PYPI +or a Docker image to the registry._ + +_Use [Markdown alerts] to highlight important information._ + +
+ +Example: +> [!CAUTION] + Using HTML sections breaks the rendering of [Markdown alerts] alerts. + To avoid this, place them outside `
` sections, for example. + +### Minimum Requirements + +This template can be used at various stages of development: +**Proof of Concept**, **Prototype**, and **Production**. +Each stage has its own minimum quality requirements +and incrementally builds upon the requirements of the previous stages. + + +1. 💡 **Proof of Concept** + - Replace all instances of `python-app-template` with your project's name. + - Add the following sections to the `README.md`: + - **Introduction** + - **Usage** + - Declare required dependencies in [pyproject.toml]. + Use the `dev` extra for development-only dependencies. + - Store data files (e.g., `.txt`, `.json`, `.csv`) in the `src/your_project/assets/` directory. + +2. 🛠️ **Prototype** + - Set up [branch protection rules] for `main` and `develop` branches. + - ☑️ Restrict deletions. + - ☑️ Require a pull request before merging. + - ☑️ Require approvals. + - ☑️ Require conversation resolution before merging. + - ☑️ Require status checks to pass. Add GitHub actions to checks. + - ☑️ Require branches to be up to date before merging. + - ☑️ Block force pushes. + - ☑️ Allowed merge methods: only Merge. + - Enforce the [Git Workflow] to maintain consistent branching and collaboration practices. + - Set up Google Cloud Build triggers to automatically publish the Docker image upon merges. + - Re-compile `requirements.txt` for Docker using: + ``` + make reqs + ``` + - Update the `README.md` to include: + - **Output Schema** + +3. 🚀 **Production** + - Install and configure pre-commit hooks. See [Preparing the environment]. + - Add unit tests to ensure code reliability. + - Write thorough documentation: + - A complete `README.md`. + - Docstrings for all **public** modules, classes, methods and functions. + + + +### Repository Overview + +_This section applies only to the template and provides an overview of the repository contents._. + +#### Directories + +This is a brief summary of all the relevant directories of the repository. + +
+ +| Directory | Description | +| ------------------------------- | ------------------------------------------------------------------------------- | +|[.github/] | Configuration for GitHub actions. | +|[docs/] | Markdown files with detailed documentation. | +|[notebooks/] | All jupyter notebooks go here. | +|[src/python_app_template/] | All source code go here. | +|[src/python_app_template/assets/]| All data files go here. | +|[tests/] | All tests go here. | + +
+ +#### Files + +This is a brief summary of all the relevant files of the repository. + +
+ +| File | Description | +| -------------------------------| ------------------------------------------------------------------------------- | +|[.flake8] | Configuration for [PEP8] checker. | +|[.gitignore] | List of files and directories to be ignored by git. | +|[.pre-commit-config.yaml] | Configuration to automate software quality checks. | +|[activate_venv.sh] | Simple shortcut to enter virtual environment. | +|[cloudbuild.yaml] | Configuration to build and publish docker images in Google Cloud. | +|[codecov.yml] | Configuration for [codecov] GitHub integration. | +|[docker-compose.yml] | Configuration for [docker compose]. | +|[Dockerfile] | Instructions for building the Docker image. | +|[LICENSE] | The software license. | +|[Makefile] | Set of commands to simplify development. | +|[MANIFEST.in] | Set of patterns to include or exclude files from installed package. | +|[pyproject.toml] | Modern Python packaging configuration file. | +|[requirements.txt] | Full set of compiled production dependencies (pinned to specific versions). | +|[requirements-test.txt] | High level test dependencies needed to test the installed package. | +|[README.md] | This file. | +|[setup.py] | Legacy Python packaging config file, kept for compatibility with [Apache Beam]. | + +
+ +
+ +### Using the CLI + +_Write instructions on how to use the CLI of the application here._ + +#### Config file example + +_**Optional**_. +_Provide an example of an input configuration file._ + +## Output schema. + +_**Optional**_. +_Discuss any relevant details about the schema. +Provide a link to the schema definition._ + +## Data persistence pattern + +_**Optional**_. +_Explain the data persistence pattern used in this application._ + +## How to Contribute + +Please read the guidelines in [docs/contributing] folder: +1. [Preparing the environment] +2. [Git Workflow] +3. [Making changes] +4. [Managing dependencies] +5. [How to deploy] + +## Implementation details + +_**Optional**_. +_This section is for describing implementation details, primarily for developers._ + +### Most relevant modules + +_**Optional**_. +_Use this section to describe the most important modules of your application._ + +Example: +
+ +| Module | Description | +| --- | --- | +| [cli.py] | Defines the application CLI. | + +
+ +### Flow chart + +_**Optional**_. +_Use this section to display a [mermaid diagram] that explains the implementation._ + +## References + +_**Optional**_. +_Provide any relevant references to support the explanations in this README. +Here we provide some examples._ + + +[1] Welch H., Clavelle T., White T. D., Cimino M. A., Van Osdel J., Hochberg T., et al. (2022). Hot spots of unseen fishing vessels. Sci. Adv. 8 (44), eabq2109. doi: 10.1126/sciadv.abq2109 + +[2] J. H. Ford, B. Bergseth, C. Wilcox, Chasing the fish oil—Do bunker vessels hold the key to fisheries crime networks? Front. Mar. Sci. 5, 267 (2018). \ No newline at end of file diff --git a/activate-venv.sh b/activate-venv.sh new file mode 100755 index 0000000..cb3e6be --- /dev/null +++ b/activate-venv.sh @@ -0,0 +1,3 @@ +# Usage: . activate-venv.sh (DOT-SPACE-A-TAB) + +. .venv/bin/activate diff --git a/cloudbuild.yaml b/cloudbuild.yaml new file mode 100644 index 0000000..63c33e5 --- /dev/null +++ b/cloudbuild.yaml @@ -0,0 +1,22 @@ +steps: + - name: gcr.io/cloud-builders/docker + id: build + args: + - build + - '-t' + - '${_IMAGE_NAME}:${TAG_NAME}' + - '-t' + - '${_IMAGE_NAME}:latest' + - '-f' + - Dockerfile + - '--target' + - prod + - . +images: + - '${_IMAGE_NAME}:${TAG_NAME}' + - '${_IMAGE_NAME}:latest' +timeout: 600s +substitutions: + _IMAGE_NAME: 'gcr.io/${PROJECT_ID}/github.com/globalfishingwatch/python-app-template' +options: + dynamic_substitutions: true diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000..334b6cd --- /dev/null +++ b/codecov.yml @@ -0,0 +1,4 @@ +coverage: # Avoid failing CI jobs because of coverage. + status: + patch: off + project: off \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..fe26248 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,26 @@ +services: + gcloud: + image: google/cloud-sdk:latest + volumes: + - "gcp:/root/.config/" + entrypoint: gcloud + + dev: + build: + context: . + target: dev + volumes: + - ".:/opt/project" + - "gcp:/root/.config/" + entrypoint: /bin/bash + + test: + # Runs tests using the production Docker image. + # Intended to be executed in the GitHub CI environment. + build: + context: . + target: test + entrypoint: "pytest -v" +volumes: + gcp: + external: true diff --git a/docs/contributing/DEPENDENCIES.md b/docs/contributing/DEPENDENCIES.md new file mode 100644 index 0000000..6afbc5d --- /dev/null +++ b/docs/contributing/DEPENDENCIES.md @@ -0,0 +1,32 @@ +[pip-tools]: https://pip-tools.readthedocs.io/en/stable/ + +[pyproject.toml]: pyproject.toml +[requirements.txt]: requirements.txt +[requirements-test.txt]: requirements-test.txt + +### Updating dependencies + +The [requirements.txt] file contains all transitive dependencies pinned to specific versions. +It is automatically generated using [pip-tools], +based on the dependencies specified in [pyproject.toml]. +This process ensures reproducibility, +allowing the application to run consistently across different environments. + +Use [pyproject.toml] to define high-level dependencies with flexible version constraints +(e.g., ~=1.2, >=1.0, <2.0, ...). +Do not modify [requirements.txt] manually. + +To re-compile dependencies, just run +```shell +make reqs +``` + +If you want to upgrade all dependencies to latest compatible versions, just run: +```shell +make reqs-upgrade +``` +
+ +> [!NOTE] +> Remember that if you change the [requirements.txt], +you need to rebuild the docker image (`make docker-build`) in order to use it locally. diff --git a/docs/contributing/ENVIRONMENT.md b/docs/contributing/ENVIRONMENT.md new file mode 100644 index 0000000..6d6f1c2 --- /dev/null +++ b/docs/contributing/ENVIRONMENT.md @@ -0,0 +1,61 @@ +[docker compose]: https://docs.docker.com/compose/install/linux/ +[docker official instructions]: https://docs.docker.com/engine/install/ +[PEP8]: https://peps.python.org/pep-0008/ +[pre-commit]: https://pre-commit.com + +[Makefile]: Makefile + +### Preparing the environment + +Install Docker Engine using the [docker official instructions] (avoid snap packages) +and the [docker compose] plugin. No other system dependencies are required. + +1. First, clone the repository. +```shell +git clone https://github.com/GlobalFishingWatch/python-app-template.git +``` + +2. Make sure you can build the docker image: +```shell +make docker-build +``` + +3. In order to be able to connect to BigQuery, authenticate and configure the project: +```shell +make docker-gcp +``` + +4. Create virtual environment and activate it: +```shell +make venv +./.venv/bin/activate +``` + +5. Install dependencies and the python package: +```shell +make install +``` + +6. (Optional) Install [pre-commit] hooks: +```shell +make hooks +``` +This step is strongly recommended to maintain code quality and prevent technical debt, +particularly regarding documentation, [PEP8] compliance, spelling, and type-hinting. +If this feels too rigid for your current workflow, +at a minimum, make regular use of the provided [Makefile] commands: +`format`, `lint`, `codespell`, and `typecheck`. + +[PEP8] checks will be enforced in the GitHub CI. + +7. Make sure you can run unit tests: +```shell +make test +``` + + +> [!NOTE] +> Alternatively, + you can handle all the development inside a docker container + without installing dependencies in a virtual environment. + Use `make docker-shell` to enter a docker container. \ No newline at end of file diff --git a/docs/contributing/GIT-WORKFLOW.md b/docs/contributing/GIT-WORKFLOW.md new file mode 100644 index 0000000..fa4934e --- /dev/null +++ b/docs/contributing/GIT-WORKFLOW.md @@ -0,0 +1,69 @@ +# Git workflow summary: + +[Git Flow]: https://nvie.com/posts/a-successful-git-branching-model/ +[Semantic Versioning]: https://semver.org + +> [!IMPORTANT] +In the following, **X**, **Y** and **Z** refer to **MAJOR**, **MINOR** and **PATCH** of [Semantic Versioning]. + +We use [Git Flow] as our branching strategy, +which is well suited for projects with long release cycles. +In this document we present a summary of the strategy. + +These are the 5 types of branches used in this strategy: +| Name | Type | Purpose | +|-----------------|-----------|----------------------------------------------------------------------------------| +| `main` | Permanent | Represents the production-ready state; all releases originate here. | +| `develop` | Permanent | The integration branch for ongoing development; features are merged here. | +| `feature/*` | Temporary | Branches for developing new features, branched off from `develop`. | +| `release/X.Y.Z` | Temporary | Branches for preparing a new production release, branched off from `develop`. | +| `hotfix/*` | Temporary | Branches for critical fixes to the production version, branched off from `main`. | + +
+ +## **Temporary branches**: + +Temporary branches are used to integrate features, releases and hotfixes to the permanent branches. +Names should be descriptive and concise, all lowercase and with words separated by hyphens "-". +Optionally, feature branch names can be prefixed with JIRA ticket instead of the `feature/` prefix. +For example, you can use something like `PIPELINE-2020-name-of-the-branch`. + +### **Feature workflow**: + +1. Create a branch from `develop`. +2. Work on the feature. +3. Rebase on-top of `develop`. +4. Push changes and open a PR. Ask for a review. +5. Merge branch to `develop` with a merge commit. + +To maintain a clear _semi-linear_ history in `develop`, +we rebase feature branches on top of `develop` before merging. +The merge should be done **forcing a merge commit**, +otherwise would be a fast-forward merge (because we rebased) +and the history would be linear instead of semi-linear, +losing the context of the branch. +This is enforced in the GitHub UI, +but locally is done with: +```shell +git checkout develop +git pull +git merge branch_name --no-ff +``` + +### **Release workflow**: + +1. Create a branch named `release/X.Y.Z` from `develop`. +2. Perform all steps needed to make the release. +3. Push changes and open a PR. Ask for a review. +4. Merge `release/X.Y.Z` to `main` and also to `develop`. +5. Create a release from `main`. The tag should be named `vX.Y.Z`. + +### `Hotfix workflow`: + +1. Create a branch named `hotfix/your-branch-name` from `main`. +2. Work on the fix. Perform steps needed to make the release. +3. Push changes and open a PR. Ask for a review. +4. Merge `hotfix/your-branch-name` to `main` and also to `develop`. +5. Create a release from `main`. The tag should be named `vX.Y.Z`. + +
diff --git a/docs/contributing/HOW-TO-DEPLOY.md b/docs/contributing/HOW-TO-DEPLOY.md new file mode 100644 index 0000000..0faa58d --- /dev/null +++ b/docs/contributing/HOW-TO-DEPLOY.md @@ -0,0 +1,5 @@ +### How to deploy + +A Google Cloud build that publishes a Docker image will be triggered in the following cases: +- Pushing a commit to the `main` or `develop` branches (this includes merges). +- Creating a tag. \ No newline at end of file diff --git a/docs/contributing/MAKING-CHANGES.md b/docs/contributing/MAKING-CHANGES.md new file mode 100644 index 0000000..885fbcf --- /dev/null +++ b/docs/contributing/MAKING-CHANGES.md @@ -0,0 +1,28 @@ +[How to Write a Git Commit Message]: https://cbea.ms/git-commit/ +[interactive rebase]: https://git-scm.com/book/en/v2/Git-Tools-Rewriting-History +[pre-commit]: https://pre-commit.com + +[GIT-WORKFLOW.md]: GIT-WORKFLOW.md +[Makefile]: Makefile + +
+ +### Making changes + +Create a branch and a Pull Request (PR) following the workflow defined in [GIT-WORKFLOW.md]. + +**When working on a branch, try to follow this guidelines:** +- Write clear commit messages. See [How to Write a Git Commit Message]. +- Use [interactive rebase] to maintain the commit history of your branch clean. +- If you are not using [pre-commit] hooks, + use the provided [Makefile] commands (`format`, `lint`, `codespell`, `typecheck`) + as much as possible to maintain code quality. +- Add unit tests for each piece of code: + * Avoid connecting to external services during unit tests. Use mocks as needed. + * Ensure unit tests run as fast as possible. + +**When submitting a PR, ensure it meets the following criteria:** +- The PR targets the correct base branch (this depends on the chosen Git workflow). +- The title and body clearly explain **what** the PR does and **why** it’s necessary. +- The body includes a link to the related JIRA ticket, + facilitating integration between the ticket and the PR. \ No newline at end of file diff --git a/docs/contributing/README.md b/docs/contributing/README.md new file mode 100644 index 0000000..62c3b3f --- /dev/null +++ b/docs/contributing/README.md @@ -0,0 +1,7 @@ +## How to Contribute + +These guidelines are intended to maintain code quality, +clearly communicate the status of the codebase, +and standardize the development workflow. +Use your best judgment and feel free to propose changes +to the guidelines by submitting a pull request. \ No newline at end of file diff --git a/notebooks/.gitkeep b/notebooks/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..0e9206d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,170 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +where = ["src"] +exclude = ["tests*"] +namespaces = false + +[tool.setuptools.package-data] +"assets" = ["*"] + +[project] +name = "python-app-template" +version = "0.1.0" +description = "A template for python (dockerized) applications." +readme = "README.md" +license = "Apache-2.0" +authors = [ + { name = "Tomás J. Link", email = "tomas.link@globalfishingwatch.org" }, +] +maintainers = [ + { name = "Tomás J. Link", email = "tomas.link@globalfishingwatch.org" }, +] +classifiers = [ + "Operating System :: OS Independent", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + +] +requires-python = ">= 3.9" +dependencies = [ + "pyyaml~=6.0", + "rich~=13.9", + "rich-argparse~=1.6", +] + +[project.urls] +Homepage = "https://github.com/GlobalFishingWatch/python-app-template" +Documentation = "https://globalfishingwatch.github.io/python-app-template/" +Changelog = "https://github.com/GlobalFishingWatch/python-app-template/blob/main/CHANGELOG.md" +Repository = "https://github.com/GlobalFishingWatch/python-app-template" +Issues = "https://github.com/GlobalFishingWatch/python-app-template/issues" + +[project.scripts] +python-app-template = "python_app_template.cli:main" + +[project.optional-dependencies] +# Linting and code quality tools +lint = [ + "black~=25.1", # Code formatting tool. + "isort~=6.0", # Python imports sorting tool. + "mypy~=1.15", # Static type checker. + "pydocstyle~=6.3", # Python docstring style checker. + "ruff~=0.11", # Linter and code analysis tool. + "codespell[toml]~=2.4", # Spell checker for code. + "flake8~=7.0", # Simple PEP8 checker. + "types-PyYAML", # MyPy stubs for pyyaml. +] + +# Development workflow and tools +dev = [ + "pre-commit~=4.2", # Framework for managing pre-commit hooks. + "pip-tools~=7.0", # Freezing dependencies for production containers. + "pip-audit~=2.8", # Audit for finding vulnerabilities in dependencies. +] + +# Build tools +build = [ + "build~=1.2", # Python PEP 517 compliant build system. + "setuptools~=78.1", # Python packaging library. + "twine~=6.1", # For uploading Python packages to PyPI. +] + +[tool.ruff] +fix = true +line-length = 99 +src = ["src", "tests"] +target-version = "py312" + +[tool.ruff.format] +docstring-code-format = true + +[tool.ruff.lint] +unfixable = [] +select = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings + "F", # pyflakes + "RUF", # Ruff-specific rules + "ANN", # flake8-annotations + "C", # flake8-comprehensions + "B", # flake8-bugbear + "I", # isort + "D", # pydocstyle +] +ignore = [ + "E501", # line too long, handled by black + "C901", # too complex + "ANN401", # Dynamically typed expressions (typing.Any) are disallowed in `**kwargs` +] + +[tool.ruff.lint.per-file-ignores] +"__init__.py" = ["F401"] + +[tool.ruff.lint.isort] +lines-after-imports = 2 +lines-between-types = 1 +known-first-party = ["gfw", "tests"] + +[tool.ruff.lint.pydocstyle] +convention = "google" + +[tool.black] +target-version = ["py312"] +line-length = 99 + +[tool.isort] +profile = "black" +line_length = 99 +known_first_party = ["gfw"] +lines_after_imports = 2 +lines_between_sections = 1 +lines_between_types = 1 +ensure_newline_before_comments = true +force_sort_within_sections = true +src_paths = ["src", "tests"] + +[tool.pydocstyle] +convention = "google" + +[tool.mypy] +strict = true +ignore_missing_imports = true +files = "src" +disallow_untyped_calls = false + +[tool.pytest.ini_options] +minversion = "6.0" +testpaths = ["tests"] +addopts = "-v --cov=src --cov-report=term-missing" + +[tool.coverage.run] +source = ["src", "tests"] +branch = true +parallel = true +context = "${CONTEXT}" + +[tool.coverage.report] +precision = 0 +skip_empty = true +ignore_errors = false +show_missing = true +exclude_lines = [ + # Have to re-enable the standard pragma + "pragma: no cover", + # Don't complain if tests don't hit defensive assertion code: + "raise AssertionError", + "raise NotImplementedError", + "AbstractMethodError", + # Don't complain if non-runnable code isn't run: + "if 0:", + "if __name__ == .__main__.:", + "if TYPE_CHECKING:", +] + +[tool.codespell] +skip = '.git,env*,venv*,.venv*, build*,tmp*' \ No newline at end of file diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 0000000..b0c47c5 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,7 @@ +# We want to be able to install only test dependencies, for example when testing an already installed package. +# Currently, is not possible to install only dependencies from pyproject.toml. +# For that, we use this file to specify the test dependencies required. + +pytest~=8.3 # Core testing framework. +pytest-cov~=6.0 # Coverage plugin for pytest. +pytest-mock~=3.14 # Mocking plugin for pytest. diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..c8542a3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,20 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --output-file=requirements.txt +# +markdown-it-py==3.0.0 + # via rich +mdurl==0.1.2 + # via markdown-it-py +pygments==2.19.1 + # via rich +pyyaml==6.0.2 + # via python-app-template (setup.py) +rich==13.9.4 + # via + # python-app-template (setup.py) + # rich-argparse +rich-argparse==1.7.0 + # via python-app-template (setup.py) diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..c9d1cfa --- /dev/null +++ b/setup.py @@ -0,0 +1,4 @@ +"""This setup.py is added for compatibility with Apache Beam dataflow pipelines.""" +import setuptools + +setuptools.setup() diff --git a/src/python_app_template/__init__.py b/src/python_app_template/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/python_app_template/assets/data.json b/src/python_app_template/assets/data.json new file mode 100644 index 0000000..dcb9714 --- /dev/null +++ b/src/python_app_template/assets/data.json @@ -0,0 +1,3 @@ +{ + "value": 1234 +} \ No newline at end of file diff --git a/src/python_app_template/cli.py b/src/python_app_template/cli.py new file mode 100644 index 0000000..251f8f1 --- /dev/null +++ b/src/python_app_template/cli.py @@ -0,0 +1,19 @@ +import sys +import logging + +from .version import __version__ + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +def cli(args: list): + logger.info("Starting APP (v{})...".format(__version__)) + + +def main(): + cli(sys.argv[1:]) + + +if __name__ == "__main__": + main() diff --git a/src/python_app_template/version.py b/src/python_app_template/version.py new file mode 100644 index 0000000..769e7cb --- /dev/null +++ b/src/python_app_template/version.py @@ -0,0 +1,6 @@ +"""Holds the current version of the program.""" + +import importlib.metadata + + +__version__ = importlib.metadata.version("python-app-template") diff --git a/tests/test_assets.py b/tests/test_assets.py new file mode 100644 index 0000000..1abbad9 --- /dev/null +++ b/tests/test_assets.py @@ -0,0 +1,11 @@ +import json +from importlib import resources + +from python_app_template import assets + + +def test_data(): + with open(resources.files(assets) / "data.json") as file: + data = json.load(file) + + assert data == {"value": 1234} diff --git a/tests/test_version.py b/tests/test_version.py new file mode 100644 index 0000000..5f193f8 --- /dev/null +++ b/tests/test_version.py @@ -0,0 +1,5 @@ +from python_app_template import version + + +def test_version(): + assert isinstance(version.__version__, str)