Merge pull request #133 from dotkernel/llms-generate

Updates to llms-generator

Author: bidi47

.gitignore

@@ -56,3 +56,8 @@ public/uploads/
 # Generated by bin/generate-packages (daily cron); not tracked so it never conflicts on pull
 /public/dotkernel-packages.json
 /public/dotkernel-packages.json.tmp
+
+/public/llms-full.txt
+/public/llms.txt
+/public/feed.xml
+/public/sitemap.xml

src/App/src/Factory/LlmsGeneratorFactory.php

@@ -24,6 +24,7 @@ public function __invoke(ContainerInterface $container): LlmsGenerator
             $config['llms']['sourceDir'],
             $config['llms']['indexFile'],
             $config['application']['url'] ?? '',
+            $config['llms']['pagesDir'] ?? null,
         );
     }
 }

src/App/src/Service/LlmsGenerator.php

@@ -9,9 +9,12 @@
 use RuntimeException;
 
 use function array_key_exists;
+use function basename;
 use function count;
+use function explode;
 use function file_get_contents;
 use function file_put_contents;
+use function glob;
 use function implode;
 use function preg_match;
 use function sprintf;
@@ -65,6 +68,7 @@ public function __construct(
         private readonly string $sourceDir,
         private readonly string $outputFile,
         private readonly string $baseUrl,
+        private readonly ?string $pagesDir = null,
     ) {
     }
 
@@ -91,7 +95,10 @@ public function write(): int
             $sections[] = $this->buildCategorySection($slug, $posts);
         }
 
-        $written = $this->buildHeader() . implode("\n", $sections);
+        $written = $this->buildHeader()
+            . $this->buildPagesSection()
+            . "## Categories\n\n"
+            . implode("\n", $sections);
 
         if (file_put_contents($this->outputFile, $written) === false) {
             throw new RuntimeException('Unable to write llms.txt.');
@@ -163,53 +170,98 @@ private function buildCategorySection(string $slug, array $entries): string
      */
     private function resolveFrontMatter(string $categorySlug, Post $post): array
     {
-        $result   = ['title' => $post->getTitle(), 'description' => $post->getExcerpt()];
         $path     = sprintf('%s/%s/%s.md', $this->sourceDir, $categorySlug, $post->getSlug());
         $contents = @file_get_contents($path);
 
-        if ($contents === false) {
-            return $result;
-        }
+        return [
+            'title'       => $contents === false
+                ? $post->getTitle()
+                : $this->extractFrontMatterField($contents, 'title') ?? $post->getTitle(),
+            'description' => $contents === false
+                ? $post->getExcerpt()
+                : $this->extractFrontMatterField($contents, 'description') ?? $post->getExcerpt(),
+        ];
+    }
 
-        if (preg_match('/^title:\s*"(.*)"\s*$/m', $contents, $matches) === 1) {
-            $result['title'] = trim($matches[1]);
+    private function buildPagesSection(): string
+    {
+        if ($this->pagesDir === null) {
+            return '';
         }
 
-        if (preg_match('/^description:\s*"(.*)"\s*$/m', $contents, $matches) === 1) {
-            $result['description'] = trim($matches[1]);
-        }
+        $pages = [];
+        foreach (glob($this->pagesDir . '/*.md') ?: [] as $path) {
+            $contents = file_get_contents($path);
+            if ($contents === false) {
+                continue;
+            }
 
-        return $result;
-    }
+            $title       = $this->extractFrontMatterField($contents, 'title');
+            $description = $this->extractFrontMatterField($contents, 'description');
+            if ($title === null || $description === null) {
+                continue;
+            }
 
-    private function buildHeader(): string
-    {
-        return <<<HEADER
-        # Dotkernel Light
+            $pages[] = [
+                'title'       => trim(explode('|', $title, 2)[0]),
+                'slug'        => basename($path, '.md'),
+                'description' => $description,
+            ];
+        }
 
-        > Dotkernel Light is the technical blog for the Dotkernel headless PHP platform - a PSR-15 compliant 
-        application built on Mezzio and Laminas components. It publishes architecture write-ups, how-tos, 
-        and release notes for the platform applications - Dotkernel API, Admin and Queue - and for the standalone 
-        Dotkernel Light skeleton.
+        if ($pages === []) {
+            return '';
+        }
 
-        Content spans foundational PHP/middleware architecture (PSR-7, PSR-15, request lifecycle, dependency injection),
-         practical how-tos (Doctrine migrations, CORS, authentication, caching), and the history/release notes of the 
-         Dotkernel ecosystem going back to 2008. Posts are organized by category and attributed to an author; 
-         URLs follow the pattern `/{category-slug}/{post-slug}/`.
+        usort($pages, static fn (array $a, array $b): int => strcasecmp($a['title'], $b['title']));
 
-        ## Docs
+        $lines = ['## Pages', ''];
+        foreach ($pages as $page) {
+            $lines[] = sprintf(
+                '- [%s](%s/%s/): %s',
+                $page['title'],
+                $this->baseUrl,
+                $page['slug'],
+                $page['description'],
+            );
+        }
 
-        - [Blog]({$this->baseUrl}/blog/): full list of posts, most recent first, paginated
-        - [Categories]({$this->baseUrl}/categories/): all categories with post counts
-        - [About]({$this->baseUrl}/about/): the team behind Dotkernel - how the team works, its commitment to open 
-        source and the PHP community, and how it uses AI under guardrails
-        - [OSS Package Lifecycle]({$this->baseUrl}/dotkernel-packages-oss-lifecycle/): support/maintenance status of 
-        Dotkernel's open-source packages
-        - [Contact]({$this->baseUrl}/contact/)
+        return implode("\n", $lines) . "\n\n";
+    }
 
-        ## Categories
+    private function extractFrontMatterField(string $contents, string $field): ?string
+    {
+        if (preg_match(sprintf('/^%s:\s*"(.*)"\s*$/m', $field), $contents, $matches) === 1) {
+            return trim($matches[1]);
+        }
 
+        return null;
+    }
 
-        HEADER;
+    private function buildHeader(): string
+    {
+        $intro = '> Dotkernel Light is the technical blog for the Dotkernel headless PHP platform - a PSR-15 '
+            . 'compliant application built on Mezzio and Laminas components. It publishes architecture '
+            . 'write-ups, how-tos, and release notes for the platform applications - Dotkernel API, Admin '
+            . 'and Queue - and for the standalone Dotkernel Light skeleton.';
+
+        $body = 'Content spans foundational PHP/middleware architecture (PSR-7, PSR-15, request lifecycle, '
+            . 'dependency injection), practical how-tos (Doctrine migrations, CORS, authentication, caching), '
+            . 'and the history/release notes of the Dotkernel ecosystem going back to 2008. Posts are organized '
+            . 'by category and attributed to an author; URLs follow the pattern `/{category-slug}/{post-slug}/`.';
+
+        $about = 'the team behind Dotkernel - how the team works, its commitment to open source and the PHP '
+            . 'community, and how it uses AI under guardrails';
+
+        return "# Dotkernel Light\n\n"
+            . $intro . "\n\n"
+            . $body . "\n\n"
+            . "## Docs\n\n"
+            . "- [Blog]({$this->baseUrl}/blog/): full list of posts, most recent first, paginated\n"
+            . "- [Categories]({$this->baseUrl}/categories/): all categories with post counts\n"
+            . "- [About]({$this->baseUrl}/about/): {$about}\n"
+            . "- [OSS Package Lifecycle]({$this->baseUrl}/dotkernel-packages-oss-lifecycle/): "
+            . "support/maintenance status of Dotkernel's open-source packages\n"
+            . "- [Contact]({$this->baseUrl}/contact/)\n\n";
     }
 }

test/Unit/App/Factory/LlmsGeneratorFactoryTest.php

@@ -24,15 +24,35 @@ public function testInvokeAppliesTheConfiguredValues(): void
             'llms'        => [
                 'sourceDir' => '/tmp/md-articles',
                 'indexFile' => '/tmp/llms.txt',
+                'pagesDir'  => '/tmp/md-pages',
             ],
             'application' => ['url' => 'https://example.test'],
         ]));
 
         $this->assertSame('/tmp/llms.txt', $generator->getOutputFile());
         $this->assertSame('/tmp/md-articles', $this->readProperty($generator, 'sourceDir'));
+        $this->assertSame('/tmp/md-pages', $this->readProperty($generator, 'pagesDir'));
         $this->assertSame('https://example.test', $this->readProperty($generator, 'baseUrl'));
     }
 
+    /**
+     * A config predating the static-page markdown simply skips that section.
+     *
+     * @throws Exception
+     */
+    public function testInvokeLeavesThePagesDirectoryUnsetWhenItIsNotConfigured(): void
+    {
+        $generator = (new LlmsGeneratorFactory())($this->createContainer([
+            'llms'        => [
+                'sourceDir' => '/tmp/md-articles',
+                'indexFile' => '/tmp/llms.txt',
+            ],
+            'application' => ['url' => 'https://example.test'],
+        ]));
+
+        $this->assertNull($this->readProperty($generator, 'pagesDir'));
+    }
+
     /**
      * The application URL is read from `application.url` - the site's single URL key. A config
      * without it must not blow up, because the placeholder substitution is optional.

test/Unit/App/Service/LlmsGeneratorTest.php

@@ -32,6 +32,7 @@ class LlmsGeneratorTest extends UnitTest
 {
     private string $workDir;
     private string $sourceDir;
+    private string $pagesDir;
     private string $outputFile;
 
     protected function setUp(): void
@@ -46,9 +47,11 @@ protected function setUp(): void
         );
 
         $this->sourceDir  = $this->workDir . DIRECTORY_SEPARATOR . 'md-articles';
+        $this->pagesDir   = $this->workDir . DIRECTORY_SEPARATOR . 'md-pages';
         $this->outputFile = $this->workDir . DIRECTORY_SEPARATOR . 'llms.txt';
 
         mkdir($this->sourceDir, 0775, true);
+        mkdir($this->pagesDir, 0775, true);
     }
 
     protected function tearDown(): void
@@ -263,6 +266,101 @@ public function testWriteFallsBackToThePostEntityWhenTheArticleFileIsMissing():
         $this->assertStringContainsString(': Raw excerpt.', $output);
     }
 
+    /**
+     * @throws Exception
+     */
+    public function testWriteOmitsThePagesSectionWhenNoPagesDirectoryIsConfigured(): void
+    {
+        $this->writePage('api', 'Dotkernel API | Tagline');
+
+        $this->createGenerator(pagesDir: null)->write();
+
+        $this->assertStringNotContainsString('## Pages', $this->writtenOutput());
+    }
+
+    /**
+     * @throws Exception
+     */
+    public function testWriteOmitsThePagesSectionWhenThePagesDirectoryIsEmpty(): void
+    {
+        $this->createGenerator()->write();
+
+        $this->assertStringNotContainsString('## Pages', $this->writtenOutput());
+    }
+
+    /**
+     * @throws Exception
+     */
+    public function testWriteIncludesEachPageSortedAlphabeticallyByTitle(): void
+    {
+        $this->writePage('api', 'Zebra page | Some tagline');
+        $this->writePage('admin', 'Alpha page | Some tagline');
+
+        $this->createGenerator(baseUrl: 'https://example.test')->write();
+
+        $output  = $this->writtenOutput();
+        $alphaAt = mb_strpos($output, '[Alpha page]');
+        $zebraAt = mb_strpos($output, '[Zebra page]');
+
+        $this->assertStringContainsString('## Pages', $output);
+        $this->assertNotFalse($alphaAt);
+        $this->assertNotFalse($zebraAt);
+        $this->assertLessThan($zebraAt, $alphaAt);
+    }
+
+    /**
+     * @throws Exception
+     */
+    public function testWriteTrimsThePageTitleAtTheTaglineSeparator(): void
+    {
+        $this->writePage('api', 'Dotkernel API | Open-source REST API skeleton for PHP', 'The description.');
+
+        $this->createGenerator(baseUrl: 'https://example.test')->write();
+
+        $this->assertStringContainsString(
+            '- [Dotkernel API](https://example.test/api/): The description.',
+            $this->writtenOutput()
+        );
+        $this->assertStringNotContainsString('Open-source REST API skeleton for PHP]', $this->writtenOutput());
+    }
+
+    /**
+     * @throws Exception
+     */
+    public function testWriteSkipsAPageMissingATitleOrDescription(): void
+    {
+        $this->writePage('api', 'Dotkernel API | Tagline', null);
+        $this->writePage('admin', 'Dotkernel Admin | Tagline', 'Has a description.');
+
+        $this->createGenerator()->write();
+
+        $output = $this->writtenOutput();
+        $this->assertStringNotContainsString('[Dotkernel API]', $output);
+        $this->assertStringContainsString('[Dotkernel Admin]', $output);
+    }
+
+    /**
+     * @throws Exception
+     */
+    public function testWritePlacesThePagesSectionBetweenDocsAndCategories(): void
+    {
+        $this->writePage('api', 'Dotkernel API | Tagline');
+        $posts = [$this->createPost('A post', 'a-post', 'dotkernel', 'Dotkernel')];
+
+        $this->createGenerator($posts)->write();
+
+        $output       = $this->writtenOutput();
+        $docsAt       = mb_strpos($output, '## Docs');
+        $pagesAt      = mb_strpos($output, '## Pages');
+        $categoriesAt = mb_strpos($output, '## Categories');
+
+        $this->assertNotFalse($docsAt);
+        $this->assertNotFalse($pagesAt);
+        $this->assertNotFalse($categoriesAt);
+        $this->assertLessThan($pagesAt, $docsAt);
+        $this->assertLessThan($categoriesAt, $pagesAt);
+    }
+
     /**
      * @throws Exception
      */
@@ -298,6 +396,7 @@ private function createGenerator(
         array $posts = [],
         ?string $outputFile = null,
         string $baseUrl = 'https://example.test',
+        ?string $pagesDir = '',
     ): LlmsGenerator {
         $postRepository = $this->createStub(PostRepository::class);
         $postRepository->method('getPublishedPosts')->willReturn($posts);
@@ -307,6 +406,7 @@ private function createGenerator(
             $this->sourceDir,
             $outputFile ?? $this->outputFile,
             $baseUrl,
+            $pagesDir === '' ? $this->pagesDir : $pagesDir,
         );
     }
 
@@ -346,6 +446,15 @@ private function writeArticle(string $categorySlug, string $slug, string $title,
         );
     }
 
+    private function writePage(string $slug, string $title, ?string $description = 'A description.'): void
+    {
+        $frontMatter = $description === null
+            ? sprintf("---\ntitle: \"%s\"\n---\n\nBody.\n", $title)
+            : sprintf("---\ntitle: \"%s\"\ndescription: \"%s\"\n---\n\nBody.\n", $title, $description);
+
+        file_put_contents($this->pagesDir . DIRECTORY_SEPARATOR . $slug . '.md', $frontMatter);
+    }
+
     private function writtenOutput(): string
     {
         $contents = file_get_contents($this->outputFile);