Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 18 additions & 2 deletions cronjobs/src/commands/_git_export_git_tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -283,7 +283,8 @@ def delete_old_tags(
repo: pygit2.Repository, max_age_days: int, min_tags_per_collection: int = 2
) -> list[str]:
"""
Delete old tags from the repository, keeping the most recent `min_tags_per_collection` tags for each collection.
Delete old tags from the repository, keeping the most recent `min_tags_per_collection` tags for each collection,
as well as the oldest tag of each collection (except on the `common` branch).

Return the list of deleted tag names.
"""
Expand All @@ -304,6 +305,17 @@ def delete_old_tags(
timestamp = int(timestamp)
group_by_collection.setdefault(collection, []).append((ref_name, timestamp))

# The oldest tag of each collection is kept forever so that clients that come back with
# a timestamp older than our retention period still obtain the tombstones of the records
# deleted since then (see `get_collection_changeset()` in `git-reader/app.py`).
# The `v1/common` branch is excluded, because it has to remain truncatable in order
# to prune the LFS objects that it references (see `truncate_branch()` usage).
pinned_tags = {
tags[0][0]
for collection, tags in group_by_collection.items()
if not collection.endswith("/timestamps/common")
}

# For each collection, we find all the tags that are older than
# threshold. We keep the most recent `min_tags_per_collection` old tags
# to make sure clients can catch up with synchronization.
Expand All @@ -314,7 +326,11 @@ def delete_old_tags(
kept_count = 0
for ref_name, timestamp in reversed(tags):
age_days = (now_ts - timestamp) / (60 * 60 * 24 * 1000)
if age_days < max_age_days or kept_count < min_tags_per_collection:
if (
age_days < max_age_days
or kept_count < min_tags_per_collection
or ref_name in pinned_tags
):
kept_count += 1
continue

Expand Down
28 changes: 28 additions & 0 deletions cronjobs/tests/commands/test_git_export_git_tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -302,6 +302,34 @@ def test_delete_new_and_old_tags(tmp_repo):
assert f"refs/tags/{t}" not in repo.references


def test_delete_old_tags_keeps_oldest_of_collection(tmp_repo):
repo = tmp_repo
now_ts = int(time.time() * 1000)

commit = tmp_repo.revparse_single("main")
tags = [
f"v1/timestamps/main/password-rules/{now_ts - i * 86400000}"
for i in range(1, 11)
]
for tag in tags:
repo.create_tag(
tag,
commit.id,
pygit2.GIT_OBJECT_COMMIT,
pygit2.Signature("Tester", "test@example.com"),
"A tag",
)

deleted = delete_old_tags(repo, max_age_days=5, min_tags_per_collection=2)

assert len(tags) == 10
assert len(deleted) == 5
# Unlike on the `common` branch, the oldest tag of a collection is never deleted.
assert f"refs/tags/{tags[-1]}" in repo.references
for t in tags[4:-1]:
assert f"refs/tags/{t}" not in repo.references


@pytest.fixture
def repo_with_tagged_commits(tmp_repo):
repo = tmp_repo
Expand Down
Loading