diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..78e8f7e --- /dev/null +++ b/.editorconfig @@ -0,0 +1,8 @@ +root = true + +[*.py] +profile = black +indent_style = space +indent_size = 4 +src_paths = src,tests + diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 0000000..5776dc4 --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,3 @@ +[settings] +profile=black +src_paths=src,tests diff --git a/Makefile b/Makefile index 1ad0bdd..7951b53 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help run build preview aggregate aggregate-repo aggregate-update-repo aggregate-repo-single aggregate-update-repo-single test-aggregate-local clean clean-projects clean-aggregated-git test test-unit test-integration check spelling linkcheck woke +.PHONY: help run build preview aggregate aggregate-repo aggregate-update-repo aggregate-repo-single aggregate-update-repo-single test-aggregate-local clean clean-projects clean-aggregated-git test test-unit test-integration check spelling linkcheck woke glossary glossary-check format help: @echo "Garden Linux Documentation Hub - Available targets:" @@ -18,6 +18,8 @@ help: @echo " spelling - Check spelling with codespell" @echo " linkcheck - Check links with lychee" @echo " woke - Check inclusive language with woke" + @echo " glossary - Process glossary links in documentation" + @echo " glossary-check - Validate glossary structure" @echo "" @echo " Documentation Aggregation:" @echo " aggregate-local - Aggregate from local repos (file:// URLs in repos-config.local.json)" @@ -48,6 +50,10 @@ build: install clean aggregate preview: pnpm run docs:preview +format: + black src/ tests/ + isort src/ tests/ + # Testing test: test-unit test-integration @echo "All tests passed!" @@ -76,6 +82,25 @@ woke: @echo "Running inclusive language checks..." @pnpm run docs:woke +glossary: + @echo "Processing glossary links..." + @python3 -c "import sys, importlib.util; \ + from pathlib import Path; \ + spec = importlib.util.spec_from_file_location('auto_glossary', 'src/aggregation/auto_glossary.py'); \ + mod = importlib.util.module_from_spec(spec); \ + spec.loader.exec_module(mod); \ + mod.process_glossary_links(Path('docs'))" + +glossary-check: + @echo "Validating glossary structure..." + @python3 -c "import importlib.util; \ + from pathlib import Path; \ + spec = importlib.util.spec_from_file_location('auto_glossary', 'src/aggregation/auto_glossary.py'); \ + mod = importlib.util.module_from_spec(spec); \ + spec.loader.exec_module(mod); \ + mod.AutoGlossary(Path('docs/reference/glossary.md')); \ + print(' ✓ Glossary structure valid')" + # Documentation Aggregation aggregate-local: @echo "Aggregating from local repositories (relative paths)..." diff --git a/docs/contributing/documentation/auto-glossary.md b/docs/contributing/documentation/auto-glossary.md new file mode 100644 index 0000000..3f8eaba --- /dev/null +++ b/docs/contributing/documentation/auto-glossary.md @@ -0,0 +1,211 @@ +--- +title: "Documentation Auto Glossary" +description: "Automatically link common idioms to their glossary entry" +order: 13 +related_topics: + - /contributing/documentation/documentation_workflow.md + - /contributing/documentation/writing_good_docs.md + - /contributing/documentation/adding-repos.md + - /contributing/documentation/working-locally.md + - /contributing/documentation/ci-architecture.md + - /contributing/documentation/ci-workflows-reference.md + - /contributing/documentation/configuration.md + - /contributing/documentation/technical.md + - /contributing/documentation/testing.md + - /contributing/documentation/vitepress-features.md +--- + +# Auto Glossary + +The documentation system includes an automatic glossary linker that converts marked terms into links pointing to the glossary page. + +## Quick Start + +Mark terms in your documentation using the marker format: + +```markdown +Deploy {glossary:Gardenlinux} on {glossary:AWS} using {glossary:KVM}. +``` + +After aggregation, this becomes: + +```markdown +Deploy [Gardenlinux](/reference/glossary#gardenlinux) on [AWS](/reference/glossary#aws) using [KVM](/reference/glossary#kvm). +``` + +## Function + +During aggregation (`make aggregate`), the system: + +1. Parses `docs/reference/glossary.md` to extract all level-3 headers as terms +2. Extracts aliases from terms with parenthesized expansions (e.g., `ADR (Architecture Decision Record)`) +3. Scans all markdown files for glossary markers +4. Replaces markers with markdown links to glossary anchors +5. Preserves code blocks, inline code, and existing links + +## Marking Terms + +Use the format where the term name matches a glossary entry: + +```markdown +{glossary:AWS} +{glossary:Garden Linux} +{glossary:ADR (Architecture Decision Record)} +``` + +Term matching is case-insensitive but preserves your original formatting: + +- Input: `{glossary:aws}` produces `[aws](/reference/glossary#aws)` +- Input: `{glossary:AWS}` produces `[AWS](/reference/glossary#aws)` + +::: details Developer Information +The auto glossary system is designed to accept a new custom `entry_format` through which +the shortcode `{glossary:*}` can be replaced with a new custom pattern. + +Check the source code for more information. +::: + +## Protected Regions + +The system leaves certain content unchanged: + +- Fenced code blocks (triple backticks) +- Inline code (single backticks) +- Existing markdown links + +Glossary markers inside these regions are not processed. + +## Adding Glossary Terms + +To add a new term: + +1. Edit `docs/reference/glossary.md` +2. Add a level-3 header: + +```markdown +### New Term Name + +Definition of the term. +``` + +The anchor is generated automatically (lowercase, hyphens replace spaces). Reference it with the marker format in your documentation. + +## Alias Support + +Terms with parenthesized expansions create aliases automatically. + +Given this glossary entry: + +```markdown +### ADR (Architecture Decision Record) +``` + +The system creates: +- Main term: `ADR (Architecture Decision Record)` +- Alias: `Architecture Decision Record` +- Alias: `ADR` + +All three can be used in markers and link to the same glossary entry. + +## Auto-Linking + + + +Auto-linking creates links for glossary terms without requiring explicit markers. This feature is disabled by default. + +With auto-linking enabled, input text: + +``` +Deploy Gardenlinux on AWS using KVM virtualization. +``` + +Becomes: + +```markdown +Deploy [Garden Linux](/reference/glossary#garden-linux) on [AWS](/reference/glossary#aws) using [KVM](/reference/glossary#kvm) virtualization. +``` + +Auto-linking rules: +- Links only the first occurrence of each term +- Matches longer terms first (`Garden Linux` before `Linux`) +- Case-insensitive matching +- Respects word boundaries +- Preserves code blocks and inline code +- Does not modify existing links + +::: warning +This feature is disabled by default as it is highly experimental and intended for research work only. + +**Contributors must not rely on this feature when contributing documentation.** +::: + +## Makefile Targets + +Process glossary links manually: + +```bash +make glossary +``` + +Validate glossary structure: + +```bash +make glossary-check +``` + +## Warnings + +**Term not found:** + +``` +[Warning][auto-glossary] Term 'xyz' not found in glossary (referenced in file.md) +``` + +The marker remains unchanged. Add the term to the glossary, fix the spelling, or remove the marker. + +**Anchor collision:** + +``` +[Warning][auto-glossary] Anchor collision detected: + 'Term 1' and 'Term 2' both generate anchor 'term-1-2' +``` + +Rename one of the terms in the glossary. + +## Anchor Generation + +Terms are converted to VitePress-compatible anchors: + +- Convert to lowercase +- Replace spaces and special characters with hyphens +- Collapse consecutive hyphens +- Strip leading and trailing hyphens +- Normalize Unicode to ASCII + +Examples: +- `AWS` becomes `aws` +- `Garden Linux` becomes `garden-linux` +- `ADR (Architecture Decision Record)` becomes `adr-architecture-decision-record` + +## Technical Reference + +Module location: `src/aggregation/auto_glossary.py` + +Main components: +- `AutoGlossary` class for glossary processing +- `process_glossary_links()` function for batch processing +- `GLOSSARY_ENTRY_FORMAT` constant defining the default marker format + +Integration points: +- Runs as part of `src/aggregate.py` after release notes generation +- Makefile provides `glossary` and `glossary-check` targets + +## Testing + +Run the test suite: + +```bash +python3 -m pytest tests/unit/test_auto_glossary*.py tests/unit/test_generate_anchor.py -v +``` + +Test coverage includes format validation, anchor generation, alias extraction, auto-linking, and integration tests. diff --git a/requirements.txt b/requirements.txt index 68a74e3..664b2b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,6 @@ codespell==2.4.2 +black +isort pytest pyyaml # glrd @ git+https://github.com/gardenlinux/glrd.git@v4.2.0 diff --git a/src/aggregate.py b/src/aggregate.py index a7f27ab..2a36b42 100755 --- a/src/aggregate.py +++ b/src/aggregate.py @@ -11,13 +11,13 @@ import tempfile from pathlib import Path -from aggregation import (DocsFetcher, copy_targeted_docs, load_config, - save_config) -from aggregation.structure import verify_internal_links +from aggregation import DocsFetcher, copy_targeted_docs, load_config, save_config +from aggregation.auto_glossary import process_glossary_links from aggregation.flavor_matrix import generate_flavor_matrix_docs from aggregation.github_api import GitHubAPIError, list_repo_releases from aggregation.release_notes import generate_release_notes_docs from aggregation.releases import generate_release_docs +from aggregation.structure import verify_internal_links def transform_repo_docs( @@ -227,9 +227,7 @@ def main() -> int: repo_names = {repo.name for repo in repos} if args.repo not in repo_names: if args.single: - parser.error( - f"Repository '{args.repo}' not found in config" - ) + parser.error(f"Repository '{args.repo}' not found in config") else: print(f"WARNING: Repository '{args.repo}' not found in config") @@ -308,7 +306,9 @@ def main() -> int: ) return 1 existing_gh_tags = {r["tag_name"].lstrip("v") for r in gh_releases} - print(f" Fetched {len(gh_releases)} GitHub release(s), {len(existing_gh_tags)} unique tag(s)") + print( + f" Fetched {len(gh_releases)} GitHub release(s), {len(existing_gh_tags)} unique tag(s)" + ) # Generate release documentation from GLRD print(f"\n{'='*60}") @@ -322,6 +322,12 @@ def main() -> int: print(f"{'='*60}\n") generate_release_notes_docs(docs_dir, gh_releases) + # Process glossary links + print(f"\n{'='*60}") + print("Processing glossary links...") + print(f"{'='*60}\n") + process_glossary_links(docs_dir) + # Summary print(f"\n{'='*60}") print("Documentation aggregation complete!") diff --git a/src/aggregation/__init__.py b/src/aggregation/__init__.py index 9f765c6..b969228 100644 --- a/src/aggregation/__init__.py +++ b/src/aggregation/__init__.py @@ -1,16 +1,20 @@ """Aggregation package for docs-ng documentation aggregation.""" # Re-export commonly used functions for backward compatibility with tests +from .auto_glossary import AutoGlossary, process_glossary_links from .config import load_config, save_config from .fetcher import DocsFetcher from .flavor_matrix import generate_flavor_matrix_docs from .models import AggregateResult, RepoConfig from .releases import generate_release_docs from .sphinx_builder import build_sphinx_markdown -from .structure import (copy_targeted_docs, process_all_markdown, - verify_internal_links) -from .transformer import (ensure_frontmatter, parse_frontmatter, - quote_yaml_value, rewrite_links) +from .structure import copy_targeted_docs, process_all_markdown, verify_internal_links +from .transformer import ( + ensure_frontmatter, + parse_frontmatter, + quote_yaml_value, + rewrite_links, +) __all__ = [ # Models @@ -36,4 +40,7 @@ "generate_flavor_matrix_docs", # Sphinx builder "build_sphinx_markdown", + # Auto Glossary + "AutoGlossary", + "process_glossary_links", ] diff --git a/src/aggregation/auto_glossary.py b/src/aggregation/auto_glossary.py new file mode 100644 index 0000000..20607cc --- /dev/null +++ b/src/aggregation/auto_glossary.py @@ -0,0 +1,361 @@ +import re +import unicodedata +import uuid +from pathlib import Path +from typing import Any, Dict, Match, Pattern, Tuple + +# Default format for marking glossary entries in markdown +GLOSSARY_ENTRY_FORMAT = "{glossary:*}" + + +class AutoGlossary: + """Links marked terms to glossary entries. + + Processes markdown files to convert glossary term markers into links + to the glossary page. Supports configurable marker formats. + """ + + def __init__(self, glossary_path: Path, entry_format: str | None = None): + """Initialize the glossary linker. + + :param glossary_path: Path to the glossary markdown file + :param entry_format: Format string for glossary markers (uses GLOSSARY_ENTRY_FORMAT if None). + The * character acts as placeholder for the term name. + """ + self.glossary_path: Path = glossary_path + # term -> (anchor, display_name) + self.terms: Dict[str, Tuple[str, str]] = {} + # alias -> canonical_term + self.aliases: Dict[str, str] = {} + + # Set entry format (use default if not provided) + self.entry_format: str = entry_format if entry_format else GLOSSARY_ENTRY_FORMAT + + # Validate entry format contains the placeholder + if "*" not in self.entry_format: + raise ValueError( + f"Entry format must contain '*' as placeholder for term name. " + f"Got: '{self.entry_format}'" + ) + + # Build regex pattern from entry format + self._entry_pattern = self._build_pattern_from_format() + + self._parse_glossary() + + def _build_pattern_from_format(self) -> re.Pattern: + """Build a regex pattern from the entry format string. + + :return: Compiled regex pattern with named group 'term' for the captured term + """ + # Escape special regex characters except * + escaped: str = re.escape(self.entry_format) + + # Replace the escaped \* with a named capture group for the term + # Match non-greedy to handle multiple occurrences on same line + # Allow alphanumeric, spaces, hyphens, underscores, parentheses in term names + pattern_str: str = escaped.replace(r"\*", r"(?P[a-zA-Z0-9\s\-_()]+?)") + + # Compile and return + return re.compile(pattern_str) + + def get_entry_format_example(self, term: str = "example") -> str: + """Get an example of how to mark a term with the current entry format. + + :param term: Example term to show in the format + :return: String showing how the term should be marked + """ + return self.entry_format.replace("*", term) + + def _parse_glossary(self) -> None: + """Parse glossary.md to extract terms and anchors. + + Reads the glossary file and builds an index of terms mapped to their anchors. + Terms are stored in lowercase for case-insensitive matching. + Extracts aliases from parenthesized expansions. + """ + if not self.glossary_path.exists(): + raise FileNotFoundError(f"Glossary file not found: {self.glossary_path}") + + content: str = self.glossary_path.read_text(encoding="utf-8") + + # Pattern to match level-3 headers + header_pattern = re.compile(r"^###\s+(.+)$", re.MULTILINE) + + anchors_seen: dict[str, str] = {} + + for match in header_pattern.finditer(content): + term = match.group(1).strip() + + if not term: + continue + + # Generate anchor for this term + anchor: str = self._generate_anchor(term) + + # Check for anchor collisions + if anchor in anchors_seen: + original_term: str = anchors_seen[anchor] + print( + f"[Warning][auto-glossary] Anchor collision detected:\n" + f" '{original_term}' and '{term}' both generate anchor '{anchor}'" + ) + else: + anchors_seen[anchor] = term + + # Store term with lowercase key for case-insensitive lookup + term_key: str = term.lower() + self.terms[term_key] = (anchor, term) + + # Extract aliases from parenthesized expansions + # e.g., "ADR (Architecture Decision Record)" -> alias "Architecture Decision Record" + parenthesis_match: Match[str] | None = re.match( + r"^([^(]+)\s*\(([^)]+)\)$", term + ) + if parenthesis_match: + abbreviation: str | Any = parenthesis_match.group(1).strip() + expansion: str | Any = parenthesis_match.group(2).strip() + + # Store expansion as alias pointing to the full term + expansion_key: str | Any = expansion.lower() + if expansion_key not in self.terms: + self.aliases[expansion_key] = term_key + + # Also store abbreviation alone as alias + abbr_key: str = abbreviation.lower() + if abbr_key != term_key and abbr_key not in self.terms: + self.aliases[abbr_key] = term_key + + if not self.terms: + print( + f"[Warning][auto-glossary] No glossary terms found in {self.glossary_path}" + ) + + def _generate_anchor(self, term: str) -> str: + """Generate VitePress-compatible anchor from term. + + :param term: The glossary term to convert + :return: URL-safe anchor string + :raises ValueError: If term is empty or results in empty anchor + """ + if not term or not term.strip(): + raise ValueError( + "[Error][auto-glossary] Term cannot be empty or whitespace-only!" + ) + + # Normalize by removing accented characters etc. + normalized: str = unicodedata.normalize("NFD", term.strip()) + ascii_only: str = "".join( + char for char in normalized if unicodedata.category(char) != "Mn" + ) + + # Replace whitespaces and special characters with hyphens; Strip leading/trailing hyphens. + anchor = ascii_only.lower().replace(" ", "-") + anchor = re.sub(r"[^a-z0-9-]", "-", anchor) + anchor = re.sub(r"-+", "-", anchor) + anchor = anchor.strip("-") + + if not anchor: + raise ValueError( + f"[Error][auto-glossary] Could not generate valid anchor from term: '{term}'" + "Result was empty after removing special characters" + ) + + if len(anchor) > 100: + print( + f"[Warning][auto-glossary] Generated anchor is very long ({len(anchor)} chars): '{anchor[:50]}...'" + ) + + return anchor + + def link_terms( + self, content: str, file_path: str = "", auto_link: bool = False + ) -> str: + """Process markdown content and link glossary terms. + + :param content: Markdown content to process + :param file_path: Relative path of the file for logging + :param auto_link: Whether to auto-link first occurrence of known terms + :return: Content with glossary markers replaced by markdown links + """ + if "glossary.md" in file_path: + return content + + # Extract code blocks and inline code to protect them from modification + protected_regions: dict[str, str] = {} + pattern_id: str = uuid.uuid4().hex[:8] + placeholder_pattern: str = f"<<>>" + + # Pattern for fenced code blocks (``` or ~~~) + code_block_pattern: Pattern[str] = re.compile( + r"^(`{3,}|~{3,})[^\n]*\n[\s\S]*?^\1\s*$", re.MULTILINE + ) + + # Pattern for inline code (`code`) + inline_code_pattern: Pattern[str] = re.compile(r"(`[^`\n]+`)") + + # Pattern for existing markdown links ([text](url)) + link_pattern: Pattern[str] = re.compile(r"(\[([^\]]+)\]\([^\)]+\))") + + # Store and replace code blocks + def protect_region(match): + idx: int = len(protected_regions) + placeholder: str = placeholder_pattern.format(idx) + protected_regions[placeholder] = match.group(0) + return placeholder + + work_content: str = code_block_pattern.sub(protect_region, content) + work_content = link_pattern.sub(protect_region, work_content) + work_content = inline_code_pattern.sub(protect_region, work_content) + + # Process glossary markers + def replace_glossary_marker(match): + term = match.group("term").strip() + term_lower = term.lower() + + # Check if term exists in glossary + if term_lower in self.terms: + anchor, display_name = self.terms[term_lower] + return f"[{term}](/reference/glossary#{anchor})" + elif term_lower in self.aliases: + canonical = self.aliases[term_lower] + anchor, display_name = self.terms[canonical] + return f"[{term}](/reference/glossary#{anchor})" + else: + print( + f"[Warning][auto-glossary] Term '{term}' not found in glossary " + f"(referenced in {file_path})" + ) + # Return original marker if term not found + return match.group(0) + + work_content: str = self._entry_pattern.sub( + replace_glossary_marker, work_content + ) + + # Auto-link first occurrence of each term + if auto_link: + work_content = self._auto_link_terms(work_content, file_path) + + # Restore protected regions + for placeholder, original in protected_regions.items(): + work_content = work_content.replace(placeholder, original) + + return work_content + + def _auto_link_terms(self, content: str, file_path: str = "") -> str: + """Auto-link first occurrence of glossary terms. + + :param content: Markdown content to process + :param file_path: Relative path for logging + :return: Content with first occurrences linked + """ + # Build list of terms sorted by length (longest first) to handle overlapping terms + all_terms = [] + + # Add main terms + for term_key, (anchor, display_name) in self.terms.items(): + all_terms.append((display_name, anchor)) + + # Add aliases (but use actual alias text for matching) + for alias_key, canonical_key in self.aliases.items(): + anchor, _ = self.terms[canonical_key] + # Reconstruct proper case for alias + alias_words = alias_key.split() + alias_display = " ".join(word.capitalize() for word in alias_words) + all_terms.append((alias_display, anchor)) + + # Sort by length descending to match longer terms first + all_terms.sort(key=lambda x: len(x[0]), reverse=True) + + # Find all matches first, then apply them + matches_to_link = [] + linked_terms = set() + + for term_text, anchor in all_terms: + term_lower = term_text.lower() + if term_lower in linked_terms: + continue + + # Create pattern for whole word match + pattern = re.compile(r"\b(" + re.escape(term_text) + r")\b", re.IGNORECASE) + + # Find first occurrence + match = pattern.search(content) + if match: + # Check if this overlaps with any existing match + overlaps = False + for existing_start, existing_end, _, _ in matches_to_link: + if not ( + match.end() <= existing_start or match.start() >= existing_end + ): + overlaps = True + break + + if not overlaps: + matches_to_link.append( + (match.start(), match.end(), match.group(1), anchor) + ) + linked_terms.add(term_lower) + + # Sort matches by position (reverse order to maintain positions) + matches_to_link.sort(key=lambda x: x[0], reverse=True) + + # Apply all links from end to start + result = content + for start, end, matched_text, anchor in matches_to_link: + link = f"[{matched_text}](/reference/glossary#{anchor})" + result = result[:start] + link + result[end:] + + return result + + def _is_valid_term(self, term: str) -> bool: + """Check if term exists in glossary. + + :param term: Term name to check + :return: True if term exists in glossary or aliases + """ + return term.lower() in self.terms or term.lower() in self.aliases + + +def process_glossary_links( + docs_dir: Path, auto_link: bool = False, entry_format: str | None = None +) -> int: + """Process all markdown files in docs directory for glossary linking. + + :param docs_dir: Root documentation directory + :param auto_link: Whether to auto-link first occurrence of known terms + :param entry_format: Format string for glossary markers + :return: Number of files processed successfully + """ + glossary_path: Path = docs_dir / "reference" / "glossary.md" + + if not glossary_path.exists(): + print("[Warning][auto-glossary] Glossary not found, skipping glossary linking!") + return 1 + + linker: AutoGlossary = AutoGlossary(glossary_path, entry_format=entry_format) + processed: int = 0 + + print(f"[INFO] Using glossary entry format: {linker.entry_format}") + print(f"[INFO] Example usage: {linker.get_entry_format_example('AWS')}") + + # Process all markdown files except glossary itself + for md_file in docs_dir.rglob("*.md"): + if md_file.name == "glossary.md": + continue + + try: + content: str = md_file.read_text(encoding="utf-8") + linked_content: str = linker.link_terms( + content, str(md_file.relative_to(docs_dir)), auto_link=auto_link + ) + + if linked_content != content: + md_file.write_text(linked_content, encoding="utf-8") + processed += 1 + except Exception as e: + print(f"[Warning][auto-glossary] Error processing {md_file.name}: {e}") + + print(f"[Success] Auto-Glossary processed {processed} files successfully.") + return processed diff --git a/src/aggregation/config.py b/src/aggregation/config.py index e6b278a..7af3f8f 100644 --- a/src/aggregation/config.py +++ b/src/aggregation/config.py @@ -4,7 +4,7 @@ import sys from typing import List -from .models import RepoConfig, _DEFAULT_MEDIA_DIRECTORIES +from .models import _DEFAULT_MEDIA_DIRECTORIES, RepoConfig def load_config(config_path: str) -> List[RepoConfig]: diff --git a/src/aggregation/fetcher.py b/src/aggregation/fetcher.py index ccf2568..fdbf18f 100644 --- a/src/aggregation/fetcher.py +++ b/src/aggregation/fetcher.py @@ -128,11 +128,15 @@ def _fetch_remote( if repo.structure == "sphinx": print(f" Building Sphinx Markdown from {repo.docs_path}/") sphinx_ok = build_sphinx_markdown( - temp_dir, repo.docs_path, output_dir, + temp_dir, + repo.docs_path, + output_dir, target_map=repo.target_map or None, ) if not sphinx_ok: - print(f" Warning: Sphinx build failed; falling back to raw docs copy") + print( + f" Warning: Sphinx build failed; falling back to raw docs copy" + ) self._copy_docs(docs_source, output_dir) else: print(f" Copying docs to {output_dir}") @@ -185,18 +189,22 @@ def _fetch_local( file=sys.stderr, ) return False - + # Copy docs directory (or build Sphinx Markdown first) docs_source = repo_abs_path / repo.docs_path if docs_source.exists(): if repo.structure == "sphinx": print(f" Building Sphinx Markdown from {repo.docs_path}/") sphinx_ok = build_sphinx_markdown( - repo_abs_path, repo.docs_path, output_dir, + repo_abs_path, + repo.docs_path, + output_dir, target_map=repo.target_map or None, ) if not sphinx_ok: - print(f" Warning: Sphinx build failed; falling back to raw docs copy") + print( + f" Warning: Sphinx build failed; falling back to raw docs copy" + ) self._copy_docs(docs_source, output_dir) else: print(f" Copying docs from {repo.docs_path}/") diff --git a/src/aggregation/github_api.py b/src/aggregation/github_api.py index 47e499a..0af91a1 100644 --- a/src/aggregation/github_api.py +++ b/src/aggregation/github_api.py @@ -113,9 +113,7 @@ def get_json(url: str) -> Any: ) from exc -def list_repo_releases( - owner: str, repo: str, per_page: int = 100 -) -> list[dict]: +def list_repo_releases(owner: str, repo: str, per_page: int = 100) -> list[dict]: """Fetch every release for *owner*/*repo* from the GitHub Releases API. Paginates automatically until GitHub returns an empty page. The full list diff --git a/src/aggregation/models.py b/src/aggregation/models.py index e75faff..c225574 100644 --- a/src/aggregation/models.py +++ b/src/aggregation/models.py @@ -3,7 +3,6 @@ from dataclasses import dataclass, field from typing import Dict, List - _DEFAULT_MEDIA_DIRECTORIES = [".media", "assets"] diff --git a/src/aggregation/release_notes.py b/src/aggregation/release_notes.py index 2d2881b..c8e1ef3 100644 --- a/src/aggregation/release_notes.py +++ b/src/aggregation/release_notes.py @@ -120,7 +120,10 @@ def generate_release_notes_docs(docs_dir: Path, releases: list[dict]) -> bool: print(f" Removed: {md_file.relative_to(docs_dir)}") if not releases: - print("Warning: No releases provided to generate_release_notes_docs", file=sys.stderr) + print( + "Warning: No releases provided to generate_release_notes_docs", + file=sys.stderr, + ) return False # Query GLRD to determine release status diff --git a/src/aggregation/releases.py b/src/aggregation/releases.py index 85bb8e8..aa10add 100644 --- a/src/aggregation/releases.py +++ b/src/aggregation/releases.py @@ -180,9 +180,7 @@ def generate_release_table( if has_minor: # Build the normalized version string for tag comparison. if "patch" in version_obj: - version_str = ( - f"{version_obj['major']}.{version_obj['minor']}.{version_obj['patch']}" - ) + version_str = f"{version_obj['major']}.{version_obj['minor']}.{version_obj['patch']}" else: version_str = f"{version_obj['major']}.{version_obj['minor']}" @@ -274,7 +272,9 @@ def generate_release_docs(docs_dir: Path, existing_gh_tags: set[str]) -> bool: ) return False - active_table = generate_release_table(active_data, active_versions, existing_gh_tags) + active_table = generate_release_table( + active_data, active_versions, existing_gh_tags + ) active_gantt = generate_mermaid_gantt(active_data) active_timeline = get_timeline_section(active_gantt, "Release Timeline") @@ -306,7 +306,9 @@ def generate_release_docs(docs_dir: Path, existing_gh_tags: set[str]) -> bool: print(f" Updated: {release_path}") if archived_data is not None: - archived_table = generate_release_table(archived_data, active_versions, existing_gh_tags) + archived_table = generate_release_table( + archived_data, active_versions, existing_gh_tags + ) archived_gantt = generate_mermaid_gantt(archived_data) archived_timeline = get_timeline_section( archived_gantt, "Archived Releases Timeline" diff --git a/src/aggregation/sphinx_builder.py b/src/aggregation/sphinx_builder.py index f82e969..4037231 100644 --- a/src/aggregation/sphinx_builder.py +++ b/src/aggregation/sphinx_builder.py @@ -90,14 +90,19 @@ def build_sphinx_markdown( env["NO_COLOR"] = "1" src_dir = repo_dir / "src" if src_dir.is_dir(): - env["PYTHONPATH"] = ( - str(src_dir) + os.pathsep + env.get("PYTHONPATH", "") - ) + env["PYTHONPATH"] = str(src_dir) + os.pathsep + env.get("PYTHONPATH", "") print(f" [sphinx] PYTHONPATH prepended with: {src_dir}") result = subprocess.run( - [sys.executable, "-m", "sphinx", "-M", "markdown", - str(docs_source), str(build_dir)], + [ + sys.executable, + "-m", + "sphinx", + "-M", + "markdown", + str(docs_source), + str(build_dir), + ], capture_output=True, text=True, cwd=str(docs_source), @@ -235,7 +240,9 @@ def _copy_manual_markdown( # Copy flat into output_dir root — placement is driven by github_target_path dest = output_dir / md_file.name shutil.copy2(md_file, dest) - print(f" [sphinx] Carried over manual markdown: {md_file.relative_to(docs_source)}") + print( + f" [sphinx] Carried over manual markdown: {md_file.relative_to(docs_source)}" + ) copied += 1 if copied: diff --git a/src/aggregation/structure.py b/src/aggregation/structure.py index 9f42fce..316aa99 100644 --- a/src/aggregation/structure.py +++ b/src/aggregation/structure.py @@ -4,7 +4,7 @@ from pathlib import Path from typing import List -from .transformer import (ensure_frontmatter, parse_frontmatter, rewrite_links) +from .transformer import ensure_frontmatter, parse_frontmatter, rewrite_links def copy_targeted_docs( @@ -135,9 +135,13 @@ def copy_targeted_docs( # Used to colocate nested media dirs with retargeted markdown files. source_to_target_parents: dict[Path, set[Path]] = {} for src_rel, target_rel in targeted_files: - src_parent = Path(src_rel).parent # e.g. Path("overview") - target_parent = Path(target_rel).parent # e.g. Path("reference/supporting_tools") - source_to_target_parents.setdefault(src_parent, set()).add(target_parent) + src_parent = Path(src_rel).parent # e.g. Path("overview") + target_parent = Path( + target_rel + ).parent # e.g. Path("reference/supporting_tools") + source_to_target_parents.setdefault(src_parent, set()).add( + target_parent + ) for media_dir_name in media_dirs: # Recursively find all instances of this media directory in the source @@ -163,18 +167,32 @@ def copy_targeted_docs( else: # Nested media directory: look up source parent in mapping # to colocate media with the retargeted markdown file(s). - media_source_parent = rel_path.parent # e.g. Path("overview") + media_source_parent = ( + rel_path.parent + ) # e.g. Path("overview") if media_source_parent in source_to_target_parents: - for target_parent in source_to_target_parents[media_source_parent]: - target_media = docs_path / target_parent / media_dir_name - target_media.parent.mkdir(parents=True, exist_ok=True) - shutil.copytree(media_dir, target_media, dirs_exist_ok=True) - print(f" ✓ Copied media: {target_parent / media_dir_name}") + for target_parent in source_to_target_parents[ + media_source_parent + ]: + target_media = ( + docs_path / target_parent / media_dir_name + ) + target_media.parent.mkdir( + parents=True, exist_ok=True + ) + shutil.copytree( + media_dir, target_media, dirs_exist_ok=True + ) + print( + f" ✓ Copied media: {target_parent / media_dir_name}" + ) else: # No mapping found: fall back to source-relative placement target_media = docs_path / rel_path target_media.parent.mkdir(parents=True, exist_ok=True) - shutil.copytree(media_dir, target_media, dirs_exist_ok=True) + shutil.copytree( + media_dir, target_media, dirs_exist_ok=True + ) print(f" ✓ Copied media: {rel_path}") else: print(" No files with 'github_target_path:' frontmatter found") @@ -243,7 +261,12 @@ def verify_internal_links( or link.startswith("https://") or link.startswith("#") or link.startswith("mailto:") - or (":" in link and not link.startswith("/") and not link.startswith("./") and not link.startswith("../")) + or ( + ":" in link + and not link.startswith("/") + and not link.startswith("./") + and not link.startswith("../") + ) ): continue diff --git a/src/aggregation/transformer.py b/src/aggregation/transformer.py index 9d6386b..2c9f237 100644 --- a/src/aggregation/transformer.py +++ b/src/aggregation/transformer.py @@ -61,9 +61,7 @@ def replace_link(match): if file_dir: dir_depth = len(file_dir.split("/")) if levels_up > dir_depth: - new_link = ( - f"{github_base}/{repo_name}/blob/main/{stripped_link}" - ) + new_link = f"{github_base}/{repo_name}/blob/main/{stripped_link}" return f"[{text}]({new_link})" # Inside docs tree: leave relative link unchanged for VitePress diff --git a/tests/integration/test_aggregation.py b/tests/integration/test_aggregation.py index ef86440..83d0fac 100644 --- a/tests/integration/test_aggregation.py +++ b/tests/integration/test_aggregation.py @@ -4,13 +4,14 @@ from pathlib import Path import pytest -from aggregation import DocsFetcher, RepoConfig, process_all_markdown +from aggregation import DocsFetcher, RepoConfig, process_all_markdown # --------------------------------------------------------------------------- # Shared helpers # --------------------------------------------------------------------------- + def _make_output_dir(tmp_path: Path) -> Path: """Create and return a fresh output directory under tmp_path.""" output_dir = tmp_path / "output" @@ -25,22 +26,22 @@ def _make_fetcher(project_root: Path) -> DocsFetcher: class TestDocsFetcher: """Integration tests for DocsFetcher.""" - + def test_fetch_local_with_temp_dir(self, tmp_path): """Test fetching from a local directory structure.""" # Create a mock local repository repo_path = tmp_path / "mock-repo" docs_path = repo_path / "docs" docs_path.mkdir(parents=True) - + # Create some test files (docs_path / "index.md").write_text("# Test Documentation\n\nContent here.") (docs_path / "guide.md").write_text("# Guide\n\nSome guide content.") - + subdir = docs_path / "tutorials" subdir.mkdir() (subdir / "tutorial1.md").write_text("# Tutorial 1\n\nTutorial content.") - + # Create repo config repo = RepoConfig( name="test-repo", @@ -48,19 +49,19 @@ def test_fetch_local_with_temp_dir(self, tmp_path): docs_path="docs", ref="", ) - + output_dir = _make_output_dir(tmp_path) result = _make_fetcher(tmp_path).fetch(repo, output_dir) - + # Verify success assert result.success is True assert result.resolved_commit is None # Local repos don't have commits - + # Verify files were copied assert (output_dir / "index.md").exists() assert (output_dir / "guide.md").exists() assert (output_dir / "tutorials" / "tutorial1.md").exists() - + # Verify content assert "Test Documentation" in (output_dir / "index.md").read_text() @@ -145,52 +146,52 @@ def test_fetch_local_resolves_relative_path(self, tmp_path): class TestMarkdownProcessing: """Integration tests for markdown processing.""" - + def test_process_all_markdown(self, tmp_path): """Test processing markdown files in a directory.""" # Create test directory structure target_dir = tmp_path / "target" target_dir.mkdir() - + # Create test markdown files (target_dir / "README.md").write_text( "# README\n\n[Link](./guide.md)\n[External](https://example.com)" ) (target_dir / "index.md").write_text("# Index\n\nContent") - + subdir = target_dir / "docs" subdir.mkdir() (subdir / "guide.md").write_text("# Guide\n\n[Back](../README.md)") - + # Process the markdown process_all_markdown(str(target_dir), "test-repo") - + # Verify README was renamed to index (but we already have index.md, so it won't be) # The function only renames if index.md doesn't exist assert (target_dir / "README.md").exists() - + # Verify links were rewritten in index.md (which was already there) index_content = (target_dir / "index.md").read_text() assert "# Index" in index_content - + # Verify guide links were rewritten guide_content = (subdir / "guide.md").read_text() assert "README" in guide_content - + def test_process_markdown_with_frontmatter(self, tmp_path): """Test that frontmatter is properly handled.""" target_dir = tmp_path / "target" target_dir.mkdir() - + # Create markdown with problematic frontmatter (target_dir / "test.md").write_text( "---\ntitle: Test: Example\ntags: tag1, tag2\n---\n\n# Content" ) - + # Process process_all_markdown(str(target_dir), "test-repo") - + # Verify frontmatter was fixed content = (target_dir / "test.md").read_text() assert '"Test: Example"' in content # Colon should be quoted - assert '"tag1, tag2"' in content # Comma should be quoted \ No newline at end of file + assert '"tag1, tag2"' in content # Comma should be quoted diff --git a/tests/unit/test_auto_glossary_advanced.py b/tests/unit/test_auto_glossary_advanced.py new file mode 100644 index 0000000..31e3b75 --- /dev/null +++ b/tests/unit/test_auto_glossary_advanced.py @@ -0,0 +1,292 @@ +"""Tests for alias extraction and auto-linking features.""" + +from pathlib import Path + +import pytest + +from aggregation.auto_glossary import AutoGlossary + + +class TestAliasExtraction: + """Tests for automatic alias extraction from glossary terms.""" + + @pytest.fixture + def glossary_with_aliases(self, tmp_path): + """Create glossary with terms that have aliases.""" + glossary = tmp_path / "glossary.md" + glossary.write_text("""# Glossary + +### ADR (Architecture Decision Record) + +A document that captures decisions. + +### CIS (Center for Internet Security) + +A security framework. + +### Lima (Linux Machines) + +A VM tool for macOS. + +### OCI (OCI Image Format) + +Container image format. +""") + return glossary + + def test_extract_aliases_from_parentheses(self, glossary_with_aliases): + """Test that aliases are extracted from parenthesized expansions.""" + linker = AutoGlossary(glossary_with_aliases) + + # Main term should exist + assert "adr (architecture decision record)" in linker.terms + + # Expansion should be an alias + assert "architecture decision record" in linker.aliases + assert ( + linker.aliases["architecture decision record"] + == "adr (architecture decision record)" + ) + + # Abbreviation alone should be an alias + assert "adr" in linker.aliases + assert linker.aliases["adr"] == "adr (architecture decision record)" + + def test_multiple_aliases(self, glossary_with_aliases): + """Test that multiple terms with aliases work correctly.""" + linker = AutoGlossary(glossary_with_aliases) + + # Check CIS aliases + assert "cis" in linker.aliases + assert "center for internet security" in linker.aliases + + # Check Lima aliases + assert "lima" in linker.aliases + assert "linux machines" in linker.aliases + + def test_link_via_alias(self, glossary_with_aliases): + """Test linking using an alias instead of main term.""" + linker = AutoGlossary(glossary_with_aliases) + + # Link using expansion alias + content = "See {glossary:Architecture Decision Record} for details." + result = linker.link_terms(content) + assert ( + "[Architecture Decision Record](/reference/glossary#adr-architecture-decision-record)" + in result + ) + + # Link using abbreviation alias + content2 = "Use {glossary:ADR} format." + result2 = linker.link_terms(content2) + assert "[ADR](/reference/glossary#adr-architecture-decision-record)" in result2 + + def test_alias_case_insensitive(self, glossary_with_aliases): + """Test that alias matching is case-insensitive.""" + linker = AutoGlossary(glossary_with_aliases) + + content = "Use {glossary:adr} or {glossary:ARCHITECTURE DECISION RECORD}." + result = linker.link_terms(content) + + assert "[adr](/reference/glossary#adr-architecture-decision-record)" in result + assert ( + "[ARCHITECTURE DECISION RECORD](/reference/glossary#adr-architecture-decision-record)" + in result + ) + + +class TestAutoLinking: + """Tests for automatic term linking without explicit markers.""" + + @pytest.fixture + def glossary_for_autolink(self, tmp_path): + """Create glossary for auto-link testing.""" + glossary = tmp_path / "glossary.md" + glossary.write_text("""# Glossary + +### AWS + +Amazon Web Services. + +### Garden Linux + +A Linux distribution. + +### KVM + +Kernel-based Virtual Machine. + +### Azure + +Microsoft Azure platform. +""") + return glossary + + def test_auto_link_first_occurrence(self, glossary_for_autolink): + """Test that first occurrence of term is auto-linked.""" + linker = AutoGlossary(glossary_for_autolink) + + content = "Deploy on AWS. AWS is a cloud platform." + result = linker.link_terms(content, auto_link=True) + + # First AWS should be linked + assert "[AWS](/reference/glossary#aws)" in result + # Second AWS should remain unlinked + assert result.count("[AWS](/reference/glossary#aws)") == 1 + assert result.count("AWS") == 2 + + def test_auto_link_multiword_term(self, glossary_for_autolink): + """Test auto-linking multi-word terms.""" + linker = AutoGlossary(glossary_for_autolink) + + content = "Garden Linux is great. Use Garden Linux for deployment." + result = linker.link_terms(content, auto_link=True) + + # First occurrence should be linked + assert "[Garden Linux](/reference/glossary#garden-linux)" in result + # Only one occurrence should be linked + assert result.count("[Garden Linux](/reference/glossary#garden-linux)") == 1 + + def test_auto_link_multiple_terms(self, glossary_for_autolink): + """Test auto-linking multiple different terms.""" + linker = AutoGlossary(glossary_for_autolink) + + content = "Deploy Garden Linux on AWS using KVM virtualization." + result = linker.link_terms(content, auto_link=True) + + assert "[Garden Linux](/reference/glossary#garden-linux)" in result + assert "[AWS](/reference/glossary#aws)" in result + assert "[KVM](/reference/glossary#kvm)" in result + + def test_auto_link_case_insensitive(self, glossary_for_autolink): + """Test that auto-linking is case-insensitive.""" + linker = AutoGlossary(glossary_for_autolink) + + content = "Use aws for deployment." + result = linker.link_terms(content, auto_link=True) + + # Should link even with lowercase + assert "[aws](/reference/glossary#aws)" in result + + def test_auto_link_preserves_existing_markers(self, glossary_for_autolink): + """Test that explicit markers take precedence over auto-linking.""" + linker = AutoGlossary(glossary_for_autolink) + + content = "Use {glossary:AWS} and AWS again." + result = linker.link_terms(content, auto_link=True) + + # Marker should be processed + assert "[AWS](/reference/glossary#aws)" in result + # Auto-link should still work for second occurrence + # (but only first unlinked occurrence) + assert result.count("[AWS](/reference/glossary#aws)") >= 1 + + def test_auto_link_disabled_by_default(self, glossary_for_autolink): + """Test that auto-linking is disabled by default.""" + linker = AutoGlossary(glossary_for_autolink) + + content = "Deploy on AWS using KVM." + result = linker.link_terms(content, auto_link=False) + + # No links should be created without markers + assert "[AWS]" not in result + assert "[KVM]" not in result + assert result == content + + def test_auto_link_preserves_code_blocks(self, glossary_for_autolink): + """Test that auto-linking doesn't affect code blocks.""" + linker = AutoGlossary(glossary_for_autolink) + + content = """Deploy AWS outside code. + +```bash +# Do not link AWS in code +echo "AWS" +``` +""" + result = linker.link_terms(content, auto_link=True) + + # AWS outside code should be linked + lines = result.split("\n") + assert "[AWS](/reference/glossary#aws)" in lines[0] + # AWS in code should not be linked + assert "# Do not link AWS in code" in result + assert 'echo "AWS"' in result + + def test_auto_link_preserves_inline_code(self, glossary_for_autolink): + """Test that auto-linking doesn't affect inline code.""" + linker = AutoGlossary(glossary_for_autolink) + + content = "Use AWS with `AWS` config." + result = linker.link_terms(content, auto_link=True) + + # First AWS should be linked + assert "[AWS](/reference/glossary#aws)" in result + # Inline code AWS should not be linked + assert "`AWS`" in result + + def test_auto_link_word_boundaries(self, glossary_for_autolink): + """Test that auto-linking respects word boundaries.""" + linker = AutoGlossary(glossary_for_autolink) + + content = "AWSXYZ is not AWS. Use AWS here." + result = linker.link_terms(content, auto_link=True) + + # AWSXYZ should not be linked + assert "AWSXYZ" in result + assert "[AWSXYZ]" not in result + # Standalone AWS should be linked + assert "[AWS](/reference/glossary#aws)" in result + + def test_auto_link_longest_match_first(self, glossary_for_autolink): + """Test that longer terms are matched before shorter ones.""" + linker = AutoGlossary(glossary_for_autolink) + + # "Garden Linux" should be matched before "Linux" if both were terms + content = "Garden Linux is great." + result = linker.link_terms(content, auto_link=True) + + # Whole phrase should be linked + assert "[Garden Linux](/reference/glossary#garden-linux)" in result + # Should not have partial matches + assert result.count("[") == 1 + + +class TestAliasAndAutoLinkCombined: + """Test interaction between aliases and auto-linking.""" + + @pytest.fixture + def combined_glossary(self, tmp_path): + """Create glossary with aliases for auto-link testing.""" + glossary = tmp_path / "glossary.md" + glossary.write_text("""# Glossary + +### ADR (Architecture Decision Record) + +A document format. + +### VM (Virtual Machine) + +Virtualization technology. +""") + return glossary + + def test_auto_link_via_alias(self, combined_glossary): + """Test that aliases can be auto-linked.""" + linker = AutoGlossary(combined_glossary) + + # Use expansion in text + content = "An Architecture Decision Record documents choices." + result = linker.link_terms(content, auto_link=True) + + # Should link via alias + assert "/reference/glossary#adr-architecture-decision-record" in result + + def test_auto_link_abbreviation_alias(self, combined_glossary): + """Test auto-linking abbreviation aliases.""" + linker = AutoGlossary(combined_glossary) + + content = "Use ADR format for documentation." + result = linker.link_terms(content, auto_link=True) + + assert "[ADR](/reference/glossary#adr-architecture-decision-record)" in result diff --git a/tests/unit/test_auto_glossary_format.py b/tests/unit/test_auto_glossary_format.py new file mode 100644 index 0000000..4c8b97f --- /dev/null +++ b/tests/unit/test_auto_glossary_format.py @@ -0,0 +1,153 @@ +"""Unit tests for AutoGlossary entry format functionality.""" + +from pathlib import Path + +import pytest + +from aggregation.auto_glossary import GLOSSARY_ENTRY_FORMAT, AutoGlossary + + +class TestGlossaryEntryFormat: + """Tests for glossary entry format configuration.""" + + def test_default_format(self, tmp_path): + """Test that default format is used when none is provided.""" + glossary = tmp_path / "glossary.md" + glossary.write_text("# Glossary\n\n### Test Term\n\nDefinition.") + + linker = AutoGlossary(glossary) + assert linker.entry_format == GLOSSARY_ENTRY_FORMAT + assert linker.entry_format == "{glossary:*}" + + def test_custom_format(self, tmp_path): + """Test that custom format can be provided.""" + glossary = tmp_path / "glossary.md" + glossary.write_text("# Glossary\n\n### Test Term\n\nDefinition.") + + custom_format = "{{*}}" + linker = AutoGlossary(glossary, entry_format=custom_format) + assert linker.entry_format == custom_format + + def test_format_validation_requires_placeholder(self, tmp_path): + """Test that format must contain * placeholder.""" + glossary = tmp_path / "glossary.md" + glossary.write_text("# Glossary\n\n### Test Term\n\nDefinition.") + + with pytest.raises(ValueError, match="must contain '\\*' as placeholder"): + AutoGlossary(glossary, entry_format="{glossary:term}") + + def test_get_entry_format_example_default(self, tmp_path): + """Test getting example with default format.""" + glossary = tmp_path / "glossary.md" + glossary.write_text("# Glossary\n\n### AWS\n\nAmazon Web Services.") + + linker = AutoGlossary(glossary) + example = linker.get_entry_format_example("AWS") + assert example == "{glossary:AWS}" + + def test_get_entry_format_example_custom(self, tmp_path): + """Test getting example with custom format.""" + glossary = tmp_path / "glossary.md" + glossary.write_text("# Glossary\n\n### KVM\n\nKernel-based VM.") + + linker = AutoGlossary(glossary, entry_format="[[g:*]]") + example = linker.get_entry_format_example("KVM") + assert example == "[[g:KVM]]" + + def test_pattern_building_default_format(self, tmp_path): + """Test that regex pattern is built correctly from default format.""" + glossary = tmp_path / "glossary.md" + glossary.write_text("# Glossary\n\n### Test\n\nDefinition.") + + linker = AutoGlossary(glossary) + + # Test the pattern matches the format correctly + match = linker._entry_pattern.search("{glossary:AWS}") + assert match is not None + assert match.group("term") == "AWS" + + def test_pattern_building_custom_format(self, tmp_path): + """Test that regex pattern is built correctly from custom format.""" + glossary = tmp_path / "glossary.md" + glossary.write_text("# Glossary\n\n### Test\n\nDefinition.") + + linker = AutoGlossary(glossary, entry_format="{{*}}") + + # Test the pattern matches the custom format + match = linker._entry_pattern.search("{{Azure}}") + assert match is not None + assert match.group("term") == "Azure" + + def test_pattern_captures_multiword_terms(self, tmp_path): + """Test that pattern captures multi-word terms.""" + glossary = tmp_path / "glossary.md" + glossary.write_text("# Glossary\n\n### Garden Linux\n\nA Linux distribution.") + + linker = AutoGlossary(glossary) + + match = linker._entry_pattern.search("{glossary:Garden Linux}") + assert match is not None + assert match.group("term") == "Garden Linux" + + def test_pattern_captures_terms_with_parentheses(self, tmp_path): + """Test that pattern captures terms with parentheses.""" + glossary = tmp_path / "glossary.md" + glossary.write_text( + "# Glossary\n\n### ADR (Architecture Decision Record)\n\nDefinition." + ) + + linker = AutoGlossary(glossary) + + match = linker._entry_pattern.search( + "{glossary:ADR (Architecture Decision Record)}" + ) + assert match is not None + assert match.group("term") == "ADR (Architecture Decision Record)" + + def test_pattern_does_not_match_without_format(self, tmp_path): + """Test that plain terms without format markers are not matched.""" + glossary = tmp_path / "glossary.md" + glossary.write_text("# Glossary\n\n### AWS\n\nDefinition.") + + linker = AutoGlossary(glossary) + + # Plain term should not match + match = linker._entry_pattern.search("AWS is a cloud provider") + assert match is None + + def test_multiple_formats_in_content(self, tmp_path): + """Test finding multiple formatted terms in content.""" + glossary = tmp_path / "glossary.md" + glossary.write_text("# Glossary\n\n### AWS\n### Azure\n### GCP") + + linker = AutoGlossary(glossary) + + content = "Use {glossary:AWS} or {glossary:Azure} or {glossary:GCP}." + matches = list(linker._entry_pattern.finditer(content)) + + assert len(matches) == 3 + assert matches[0].group("term") == "AWS" + assert matches[1].group("term") == "Azure" + assert matches[2].group("term") == "GCP" + + def test_alternative_format_brackets(self, tmp_path): + """Test alternative format using double brackets.""" + glossary = tmp_path / "glossary.md" + glossary.write_text("# Glossary\n\n### Kubernetes\n\nDefinition.") + + linker = AutoGlossary(glossary, entry_format="[[*]]") + + match = linker._entry_pattern.search("[[Kubernetes]]") + assert match is not None + assert match.group("term") == "Kubernetes" + + def test_alternative_format_shorthand(self, tmp_path): + """Test alternative format using shorthand notation.""" + glossary = tmp_path / "glossary.md" + glossary.write_text("# Glossary\n\n### Docker\n\nDefinition.") + + linker = AutoGlossary(glossary, entry_format="[g:*]") + + match = linker._entry_pattern.search("[g:Docker]") + assert match is not None + assert match.group("term") == "Docker" diff --git a/tests/unit/test_auto_glossary_integration.py b/tests/unit/test_auto_glossary_integration.py new file mode 100644 index 0000000..c080abf --- /dev/null +++ b/tests/unit/test_auto_glossary_integration.py @@ -0,0 +1,253 @@ +"""Integration tests for AutoGlossary functionality.""" + +from pathlib import Path + +import pytest + +from aggregation.auto_glossary import AutoGlossary, process_glossary_links + + +class TestAutoGlossaryIntegration: + """Integration tests for complete glossary linking workflow.""" + + @pytest.fixture + def glossary_file(self, tmp_path): + """Create a test glossary file.""" + glossary = tmp_path / "reference" + glossary.mkdir() + glossary_md = glossary / "glossary.md" + glossary_md.write_text("""--- +title: "Glossary" +--- + +# Glossary + +## A + +### AWS + +Amazon Web Services. A cloud platform. + +### Azure + +Microsoft Azure cloud platform. + +## G + +### Garden Linux + +A Debian-based Linux distribution. + +### GitHub Actions + +Continuous integration platform. + +## K + +### KVM + +Kernel-based Virtual Machine. +""") + return glossary_md + + def test_parse_glossary_basic(self, glossary_file): + """Test that glossary terms are parsed correctly.""" + linker = AutoGlossary(glossary_file) + + assert "aws" in linker.terms + assert "azure" in linker.terms + assert "garden linux" in linker.terms + assert "github actions" in linker.terms + assert "kvm" in linker.terms + + # Check anchor generation + anchor, display_name = linker.terms["aws"] + assert anchor == "aws" + assert display_name == "AWS" + + anchor, display_name = linker.terms["garden linux"] + assert anchor == "garden-linux" + assert display_name == "Garden Linux" + + def test_link_terms_basic(self, glossary_file): + """Test basic term linking.""" + linker = AutoGlossary(glossary_file) + + content = "Use {glossary:AWS} for cloud deployment." + result = linker.link_terms(content) + + assert "[AWS](/reference/glossary#aws)" in result + assert "{glossary:AWS}" not in result + + def test_link_terms_multiple(self, glossary_file): + """Test linking multiple terms in same content.""" + linker = AutoGlossary(glossary_file) + + content = ( + "Deploy {glossary:Garden Linux} on {glossary:AWS} or {glossary:Azure}." + ) + result = linker.link_terms(content) + + assert "[Garden Linux](/reference/glossary#garden-linux)" in result + assert "[AWS](/reference/glossary#aws)" in result + assert "[Azure](/reference/glossary#azure)" in result + + def test_link_terms_case_insensitive(self, glossary_file): + """Test that term matching is case-insensitive.""" + linker = AutoGlossary(glossary_file) + + content = "Use {glossary:aws} or {glossary:AWS} or {glossary:Aws}." + result = linker.link_terms(content) + + # All variants should be linked + assert result.count("[aws](/reference/glossary#aws)") == 1 + assert result.count("[AWS](/reference/glossary#aws)") == 1 + assert result.count("[Aws](/reference/glossary#aws)") == 1 + + def test_link_terms_preserves_code_blocks(self, glossary_file): + """Test that code blocks are not modified.""" + linker = AutoGlossary(glossary_file) + + content = """# Example + +```bash +# Do not link {glossary:AWS} in code +echo "AWS" +``` + +Regular text with {glossary:AWS}. +""" + result = linker.link_terms(content) + + # Code block should remain unchanged + assert "# Do not link {glossary:AWS} in code" in result + # But regular text should be linked + assert "[AWS](/reference/glossary#aws)" in result + + def test_link_terms_preserves_inline_code(self, glossary_file): + """Test that inline code is not modified.""" + linker = AutoGlossary(glossary_file) + + content = "Use `{glossary:AWS}` configuration or {glossary:AWS} directly." + result = linker.link_terms(content) + + # Inline code should remain unchanged + assert "`{glossary:AWS}`" in result + # But regular text should be linked + assert "[AWS](/reference/glossary#aws)" in result + + def test_link_terms_preserves_existing_links(self, glossary_file): + """Test that existing markdown links are not modified.""" + linker = AutoGlossary(glossary_file) + + content = "See [AWS docs](https://aws.amazon.com) and {glossary:AWS}." + result = linker.link_terms(content) + + # Existing link should remain unchanged + assert "[AWS docs](https://aws.amazon.com)" in result + # Glossary marker should be linked + assert "[AWS](/reference/glossary#aws)" in result + + def test_link_terms_unknown_term_warning(self, glossary_file, capsys): + """Test that unknown terms produce warnings.""" + linker = AutoGlossary(glossary_file) + + content = "Use {glossary:UnknownTerm} here." + result = linker.link_terms(content, file_path="test.md") + + # Unknown term should remain unchanged + assert "{glossary:UnknownTerm}" in result + # Should produce warning + captured = capsys.readouterr() + assert "UnknownTerm" in captured.out + assert "not found" in captured.out + + def test_custom_entry_format(self, glossary_file): + """Test using custom entry format.""" + linker = AutoGlossary(glossary_file, entry_format="[[*]]") + + content = "Use [[AWS]] for deployment." + result = linker.link_terms(content) + + assert "[AWS](/reference/glossary#aws)" in result + assert "[[AWS]]" not in result + + def test_skip_glossary_file_itself(self, glossary_file): + """Test that glossary file itself is not modified.""" + linker = AutoGlossary(glossary_file) + + content = "### AWS\n\nUse {glossary:AWS} for testing." + result = linker.link_terms(content, file_path="reference/glossary.md") + + # Should remain unchanged + assert result == content + assert "{glossary:AWS}" in result + + def test_process_glossary_links_integration(self, tmp_path): + """Test the complete process_glossary_links function.""" + docs_dir = tmp_path + + # Create glossary + ref_dir = docs_dir / "reference" + ref_dir.mkdir() + glossary = ref_dir / "glossary.md" + glossary.write_text("""# Glossary + +### Test Term + +A test term definition. +""") + + # Create test markdown file + test_file = docs_dir / "test.md" + test_file.write_text("This is a {glossary:Test Term} example.") + + # Process files + count = process_glossary_links(docs_dir) + + assert count == 1 + + # Check that file was modified + result = test_file.read_text() + assert "[Test Term](/reference/glossary#test-term)" in result + assert "{glossary:Test Term}" not in result + + def test_process_glossary_links_no_changes(self, tmp_path): + """Test that files without markers are not modified.""" + docs_dir = tmp_path + + # Create glossary + ref_dir = docs_dir / "reference" + ref_dir.mkdir() + glossary = ref_dir / "glossary.md" + glossary.write_text("# Glossary\n\n### Test\n\nDefinition.") + + # Create file without glossary markers + test_file = docs_dir / "test.md" + test_file.write_text("This is plain content.") + + # Process files + count = process_glossary_links(docs_dir) + + # No files should be modified + assert count == 0 + assert test_file.read_text() == "This is plain content." + + def test_multiword_terms_with_parentheses(self, glossary_file): + """Test terms with parentheses like 'ADR (Architecture Decision Record)'.""" + # Add term with parentheses to glossary + glossary_content = glossary_file.read_text() + glossary_content += ( + "\n### ADR (Architecture Decision Record)\n\nA documentation pattern.\n" + ) + glossary_file.write_text(glossary_content) + + linker = AutoGlossary(glossary_file) + + content = "See {glossary:ADR (Architecture Decision Record)} for details." + result = linker.link_terms(content) + + assert ( + "[ADR (Architecture Decision Record)](/reference/glossary#adr-architecture-decision-record)" + in result + ) diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 27d0d22..557277c 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -5,7 +5,8 @@ from pathlib import Path import pytest -from aggregation import load_config, save_config, RepoConfig + +from aggregation import RepoConfig, load_config, save_config from aggregation.models import _DEFAULT_MEDIA_DIRECTORIES diff --git a/tests/unit/test_generate_anchor.py b/tests/unit/test_generate_anchor.py new file mode 100644 index 0000000..f17fac7 --- /dev/null +++ b/tests/unit/test_generate_anchor.py @@ -0,0 +1,139 @@ +"""Unit tests for _generate_anchor method.""" + +from pathlib import Path + +import pytest + +from aggregation.auto_glossary import AutoGlossary + + +class TestGenerateAnchor: + """Tests for VitePress-compatible anchor generation.""" + + @pytest.fixture + def linker(self, tmp_path): + """Create a basic AutoGlossary instance for testing.""" + glossary = tmp_path / "glossary.md" + glossary.write_text("# Glossary\n\n### Test\n\nDefinition.") + return AutoGlossary(glossary) + + def test_simple_term(self, linker): + """Test simple single-word term.""" + assert linker._generate_anchor("AWS") == "aws" + assert linker._generate_anchor("Azure") == "azure" + + def test_multiword_term(self, linker): + """Test multi-word terms preserve hyphens.""" + assert linker._generate_anchor("Garden Linux") == "garden-linux" + assert linker._generate_anchor("Bare Metal") == "bare-metal" + + def test_term_with_parentheses(self, linker): + """Test terms with parentheses.""" + result = linker._generate_anchor("ADR (Architecture Decision Record)") + assert result == "adr-architecture-decision-record" + + result = linker._generate_anchor("CIS (Center for Internet Security)") + assert result == "cis-center-for-internet-security" + + def test_term_with_hyphens(self, linker): + """Test terms that already contain hyphens.""" + assert linker._generate_anchor("end-to-end") == "end-to-end" + assert linker._generate_anchor("Just-in-Time") == "just-in-time" + + def test_term_with_special_characters(self, linker): + """Test removal of special characters.""" + assert linker._generate_anchor("Test: Example") == "test-example" + assert linker._generate_anchor("Test/Example") == "test-example" + assert linker._generate_anchor("Test.Example") == "test-example" + + def test_term_with_numbers(self, linker): + """Test terms containing numbers.""" + assert linker._generate_anchor("IPv4") == "ipv4" + assert linker._generate_anchor("HTTP/2") == "http-2" + + def test_leading_trailing_whitespace(self, linker): + """Test that whitespace is stripped.""" + assert linker._generate_anchor(" AWS ") == "aws" + assert linker._generate_anchor(" Garden Linux ") == "garden-linux" + + def test_consecutive_spaces(self, linker): + """Test that consecutive spaces become single hyphen.""" + assert linker._generate_anchor("Garden Linux") == "garden-linux" + assert ( + linker._generate_anchor("Test Multiple Spaces") + == "test-multiple-spaces" + ) + + def test_consecutive_hyphens(self, linker): + """Test that consecutive hyphens are collapsed.""" + assert linker._generate_anchor("Test--Example") == "test-example" + assert linker._generate_anchor("Test---Example") == "test-example" + + def test_leading_trailing_hyphens(self, linker): + """Test that leading/trailing hyphens are removed.""" + assert linker._generate_anchor("-AWS-") == "aws" + assert linker._generate_anchor("--Test--") == "test" + + def test_mixed_special_characters(self, linker): + """Test complex terms with mixed special characters.""" + # Parentheses, commas, ampersands, etc. + result = linker._generate_anchor("SAP (S/4HANA, ERP & CRM)") + assert result == "sap-s-4hana-erp-crm" + + def test_underscores(self, linker): + """Test handling of underscores.""" + # VitePress typically converts underscores to hyphens or removes them + assert linker._generate_anchor("my_term") in ["my-term", "myterm"] + + def test_real_glossary_terms(self, linker): + """Test against actual terms from the glossary.""" + real_terms = { + "ADR (Architecture Decision Record)": "adr-architecture-decision-record", + "Architecture": "architecture", + "AWS": "aws", + "Azure": "azure", + "Bare Metal": "bare-metal", + "Builder": "builder", + "Build Flavor String": "build-flavor-string", + "CIS (Center for Internet Security)": "cis-center-for-internet-security", + "Cloud Image": "cloud-image", + "Garden Linux": "garden-linux", + "GitHub Actions": "github-actions", + "KVM": "kvm", + "Lima (Linux Machines)": "lima-linux-machines", + "LTS Kernel": "lts-kernel", + "OCI (OCI Image Format)": "oci-oci-image-format", + "Patch Version": "patch-version", + "Secure Boot": "secure-boot", + "SELinux": "selinux", + "TPM2": "tpm2", + } + + for term, expected_anchor in real_terms.items(): + result = linker._generate_anchor(term) + assert ( + result == expected_anchor + ), f"Failed for '{term}': got '{result}', expected '{expected_anchor}'" + + def test_unicode_handling(self, linker): + """Test handling of unicode characters.""" + # Should convert or remove non-ASCII characters + result = linker._generate_anchor("Café") + assert result in ["cafe", "caf"] # Either remove or transliterate + + def test_empty_and_whitespace_only(self, linker): + """Test edge cases with empty/whitespace strings.""" + with pytest.raises(ValueError, match="empty or whitespace-only"): + linker._generate_anchor("") + + with pytest.raises(ValueError, match="empty or whitespace-only"): + linker._generate_anchor(" ") + + with pytest.raises(ValueError, match="empty or whitespace-only"): + linker._generate_anchor("\t\n") + + def test_case_preservation_then_lowercase(self, linker): + """Test that case is properly converted to lowercase.""" + assert linker._generate_anchor("CamelCase") == "camelcase" + assert linker._generate_anchor("UPPERCASE") == "uppercase" + assert linker._generate_anchor("MixedCASE") == "mixedcase" diff --git a/tests/unit/test_github_api.py b/tests/unit/test_github_api.py index fccea7b..17a13d1 100644 --- a/tests/unit/test_github_api.py +++ b/tests/unit/test_github_api.py @@ -12,11 +12,11 @@ from aggregation.github_api import GitHubAPIError, get_json, list_repo_releases - # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- + def _make_response(body: bytes, status: int = 200) -> MagicMock: """Return a mock context-manager response that yields *body*.""" resp = MagicMock() @@ -45,6 +45,7 @@ def _make_http_error(code: int, headers: dict | None = None) -> urllib.error.HTT # get_json # --------------------------------------------------------------------------- + class TestGetJson: """Tests for get_json.""" @@ -137,6 +138,7 @@ def test_raises_on_invalid_json(self): # list_repo_releases # --------------------------------------------------------------------------- + class TestListRepoReleases: """Tests for list_repo_releases.""" diff --git a/tests/unit/test_models.py b/tests/unit/test_models.py index 6b61ccc..ee28614 100644 --- a/tests/unit/test_models.py +++ b/tests/unit/test_models.py @@ -1,7 +1,8 @@ """Unit tests for aggregation.models module.""" import pytest -from aggregation import RepoConfig, AggregateResult + +from aggregation import AggregateResult, RepoConfig from aggregation.models import _DEFAULT_MEDIA_DIRECTORIES diff --git a/tests/unit/test_release_filter.py b/tests/unit/test_release_filter.py index ebad7bb..2c617c5 100644 --- a/tests/unit/test_release_filter.py +++ b/tests/unit/test_release_filter.py @@ -4,12 +4,14 @@ from aggregation.releases import generate_release_table - # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- -def _make_release(major: int, minor: int | None = None, patch: int | None = None) -> dict: + +def _make_release( + major: int, minor: int | None = None, patch: int | None = None +) -> dict: """Build a minimal GLRD release dict.""" version: dict = {"major": major} if minor is not None: @@ -31,6 +33,7 @@ def _make_releases_data(*releases) -> dict: # Filtering: minor rows # --------------------------------------------------------------------------- + class TestGenerateReleaseTableFiltering: """generate_release_table skips minor rows absent from existing_gh_tags.""" @@ -47,8 +50,11 @@ def test_minor_row_skipped_when_tag_absent(self): assert "1877.14" not in table # Table should be effectively empty (header rows only) data_rows = [ - line for line in table.splitlines() - if line.startswith("|") and not line.startswith("|:") and "Version" not in line + line + for line in table.splitlines() + if line.startswith("|") + and not line.startswith("|:") + and "Version" not in line ] assert data_rows == [] @@ -61,9 +67,9 @@ def test_major_row_always_included(self): def test_mixed_major_and_minor(self): """Major rows pass through; unmatched minor rows are dropped.""" data = _make_releases_data( - _make_release(2150), # major-only — always kept - _make_release(2150, 1, 0), # minor — tag present - _make_release(2150, 2, 0), # minor — tag absent + _make_release(2150), # major-only — always kept + _make_release(2150, 1, 0), # minor — tag present + _make_release(2150, 2, 0), # minor — tag absent ) table = generate_release_table( data, diff --git a/tests/unit/test_structure.py b/tests/unit/test_structure.py index 7c37a71..c6410db 100644 --- a/tests/unit/test_structure.py +++ b/tests/unit/test_structure.py @@ -3,6 +3,7 @@ from pathlib import Path import pytest + from aggregation.structure import copy_targeted_docs, verify_internal_links @@ -72,9 +73,7 @@ def test_retargeted_media_colocated_with_target(self, tmp_path): (source / "overview" / "README.md").write_text(content) (source / "overview" / "assets" / "x.png").write_bytes(b"\x89PNG") - copy_targeted_docs( - str(source), str(docs), "glrd", media_dirs=["assets"] - ) + copy_targeted_docs(str(source), str(docs), "glrd", media_dirs=["assets"]) # Targeted file copied to correct location assert (docs / "reference" / "supporting_tools" / "glrd.md").exists() @@ -104,9 +103,7 @@ def test_identity_mapped_media_unchanged(self, tmp_path): (source / "tutorials" / "cloud" / "first-boot-aws.md").write_text(content) (source / "tutorials" / "assets" / "img.png").write_bytes(b"\x89PNG") - copy_targeted_docs( - str(source), str(docs), "gardenlinux", media_dirs=["assets"] - ) + copy_targeted_docs(str(source), str(docs), "gardenlinux", media_dirs=["assets"]) # Targeted file at its correct location assert (docs / "tutorials" / "cloud" / "first-boot-aws.md").exists() @@ -158,15 +155,9 @@ def test_no_errors_when_linked_file_also_shipped(self, tmp_path): docs.mkdir() content_a = ( - "---\n" - "github_target_path: docs/reference/a.md\n" - "---\n\n[B](b.md)\n" - ) - content_b = ( - "---\n" - "github_target_path: docs/reference/b.md\n" - "---\n\n# B\n" + "---\n" "github_target_path: docs/reference/a.md\n" "---\n\n[B](b.md)\n" ) + content_b = "---\n" "github_target_path: docs/reference/b.md\n" "---\n\n# B\n" (source / "a.md").write_text(content_a) (source / "b.md").write_text(content_b) diff --git a/tests/unit/test_transformer.py b/tests/unit/test_transformer.py index 7b0e653..e178a88 100644 --- a/tests/unit/test_transformer.py +++ b/tests/unit/test_transformer.py @@ -1,7 +1,9 @@ """Unit tests for aggregation.transformer module.""" -import pytest from pathlib import Path + +import pytest + from aggregation import ( ensure_frontmatter, parse_frontmatter, @@ -48,7 +50,9 @@ def test_absolute_path_redirects_to_github(self): """Test that absolute paths redirect to GitHub.""" content = "[File](/some/path/README.md)" result = rewrite_links(content, "gardenlinux", "") - assert "github.com/gardenlinux/gardenlinux/blob/main/some/path/README.md" in result + assert ( + "github.com/gardenlinux/gardenlinux/blob/main/some/path/README.md" in result + ) def test_bare_filename_left_unchanged(self): """Bare filenames without path separators are left unchanged for VitePress."""