diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..050fa4c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +.git +.github +*.md +LICENSE diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 0000000..6d474c3 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,77 @@ +# Copilot Instructions — devops-ia/steampipe + +## Project Overview + +This repo maintains a community Docker image for [Steampipe](https://steampipe.io). Turbot stopped publishing official Docker images after v0.22.0, so we build from their pre-compiled binaries. + +The image is a thin wrapper: it downloads the Steampipe binary, sets up the runtime environment (UID 9193, directories, env vars), and provides a default CMD. + +## Architecture + +``` +Dockerfile → builds the image (ARG STEAMPIPE_VERSION controls version) +README.md → documents flags, env vars, quickstart, Kubernetes notes +cli-snapshot.json → machine-readable snapshot of CLI behavior (auto-generated) +scripts/ → extraction and comparison tools (do not modify) +package.json → semantic-release config (do not modify) +``` + +## When Upstream Releases a New Version + +1. Review the behavioral diff in the PR comment +2. Update `README.md`: + - Add new CLI flags to the flag tables + - Remove deprecated/removed flags + - Add new environment variables to the env var table + - Remove dropped environment variables + - Update the version in example commands if relevant +3. Update `Dockerfile`: + - Add/remove ENV vars if defaults changed + - Update HEALTHCHECK if the service behavior changed +4. Do NOT update `ARG STEAMPIPE_VERSION` (updatecli handles this) +5. Do NOT modify `cli-snapshot.json` (CI regenerates it) + +## Files You SHOULD Modify + +- `README.md` — flag tables, env var tables, examples +- `Dockerfile` — ENV defaults, HEALTHCHECK, EXPOSE + +## Files You MUST NOT Modify + +- `.github/workflows/` — CI/CD pipelines +- `package.json` — semantic-release config +- `cli-snapshot.json` — auto-generated by CI +- `scripts/` — extraction tools +- `LICENSE` + +## How to Build and Test + +```bash +# Build +docker build -t steampipe:test . + +# Smoke test +docker run --rm steampipe:test steampipe --version + +# Service test +docker run --rm -d --name sp-test steampipe:test steampipe service start --foreground --database-listen network +sleep 10 +docker exec sp-test bash -c 'echo > /dev/tcp/localhost/9193' && echo "OK" +docker stop sp-test +``` + +## Documentation Format + +Flag tables use this format: +```markdown +| Flag | Description | Default | +|------|-------------|---------| +| `--foreground` | Run in foreground (required for containers) | — | +``` + +Env var tables use this format: +```markdown +| Variable | Image default | Description | +|----------|--------------|-------------| +| `STEAMPIPE_UPDATE_CHECK` | `false` | Disable update checking | +``` diff --git a/.github/updatecli/steampipe.yaml b/.github/updatecli/steampipe.yaml new file mode 100644 index 0000000..7ad7363 --- /dev/null +++ b/.github/updatecli/steampipe.yaml @@ -0,0 +1,23 @@ +--- +name: Bump Steampipe version + +sources: + steampipe: + kind: githubRelease + spec: + owner: turbot + repository: steampipe + token: '{{ requiredEnv "GITHUB_TOKEN" }}' + versionFilter: + kind: semver + pattern: ">=2.0.0" + +targets: + dockerfile: + name: "Update Steampipe version in Dockerfile" + kind: file + sourceid: steampipe + spec: + file: Dockerfile + matchpattern: 'ARG STEAMPIPE_VERSION=.*' + replacepattern: 'ARG STEAMPIPE_VERSION={{ source "steampipe" }}' diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml new file mode 100644 index 0000000..2d9d775 --- /dev/null +++ b/.github/workflows/copilot-setup-steps.yml @@ -0,0 +1,29 @@ +name: Copilot Setup Steps + +on: + workflow_dispatch: + push: + paths: + - .github/workflows/copilot-setup-steps.yml + pull_request: + paths: + - .github/workflows/copilot-setup-steps.yml + +jobs: + copilot-setup-steps: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Install tools + run: | + sudo apt-get update -qq && sudo apt-get install -y -qq jq > /dev/null + echo "jq $(jq --version), docker $(docker --version)" + + - name: Build Docker image + run: docker build -t steampipe:local . + + - name: Validate + run: docker run --rm steampipe:local steampipe --version diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml new file mode 100644 index 0000000..2a4657b --- /dev/null +++ b/.github/workflows/docker-build.yml @@ -0,0 +1,282 @@ +name: Build and Push Docker Image + +env: + DOCKERHUB_USER: devopsiaci + DOCKERHUB_REPO: steampipe + GHCR_REGISTRY: ghcr.io + GHCR_REPO: ${{ github.repository }} + +on: + workflow_dispatch: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + test: + name: Test + runs-on: ubuntu-latest + permissions: + contents: read + security-events: write + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Lint Dockerfile + uses: hadolint/hadolint-action@54c9adbab1582c2ef04b2016b760714a4bfde3cf # v3.1.0 + with: + dockerfile: Dockerfile + config: .hadolint.yaml + + - name: Unit tests + run: | + pip install -r tests/requirements.txt + python3 -m pytest tests/ \ + --cov=compare_snapshots \ + --cov-report=term-missing \ + --cov-fail-under=90 + + - name: Build test image + uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0 + with: + context: . + load: true + tags: steampipe:test + + - name: Smoke test + run: | + docker run --rm steampipe:test steampipe --version + + - name: Container structure tests + run: | + docker run --rm \ + -v "$PWD/structure-tests.yaml:/structure-tests.yaml:ro" \ + -v /var/run/docker.sock:/var/run/docker.sock \ + gcr.io/gcp-runtimes/container-structure-test:latest \ + test --image steampipe:test --config /structure-tests.yaml + + - name: Security scan (Trivy) + uses: aquasecurity/trivy-action@57a97c7e7821a5776cebc9bb87c984fa69cba8f1 # v0.35.0 + with: + image-ref: steampipe:test + format: sarif + output: trivy-results.sarif + exit-code: "1" + severity: CRITICAL + ignore-unfixed: true + + - name: Upload Trivy SARIF results + if: always() + uses: github/codeql-action/upload-sarif@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18 + with: + sarif_file: trivy-results.sarif + + behavior-check: + name: Behavior Check + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Detect version change + id: version + run: | + git fetch origin ${{ github.base_ref }} --depth=1 + OLD_VERSION=$(git show origin/${{ github.base_ref }}:Dockerfile 2>/dev/null | grep -oP 'ARG STEAMPIPE_VERSION=\K.*' || echo "") + NEW_VERSION=$(grep -oP 'ARG STEAMPIPE_VERSION=\K.*' Dockerfile) + echo "old=$OLD_VERSION" >> "$GITHUB_OUTPUT" + echo "new=$NEW_VERSION" >> "$GITHUB_OUTPUT" + if [ "$OLD_VERSION" != "$NEW_VERSION" ]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + echo "Version change detected: $OLD_VERSION → $NEW_VERSION" + else + echo "changed=false" >> "$GITHUB_OUTPUT" + echo "No version change" + fi + + - name: Build test image + if: steps.version.outputs.changed == 'true' + uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0 + with: + context: . + load: true + tags: steampipe:test + + - name: Extract CLI snapshot + if: steps.version.outputs.changed == 'true' + run: | + docker run --rm --network none \ + -v "$PWD/scripts:/scripts:ro" \ + steampipe:test bash /scripts/extract-cli-snapshot.sh > /tmp/cli-snapshot-new.json + + - name: Extract env vars from upstream source + if: steps.version.outputs.changed == 'true' + run: | + ENV_VARS=$(bash scripts/extract-env-vars.sh "${{ steps.version.outputs.new }}") + jq --argjson env_vars "$ENV_VARS" '. + {env_vars: $env_vars}' /tmp/cli-snapshot-new.json > /tmp/cli-snapshot-full.json + + - name: Compare snapshots + id: diff + if: steps.version.outputs.changed == 'true' + run: | + python3 scripts/compare_snapshots.py \ + cli-snapshot.json /tmp/cli-snapshot-full.json \ + --output-md /tmp/behavior-diff.md \ + --output-json /tmp/behavior-diff.json \ + && echo "has_changes=false" >> "$GITHUB_OUTPUT" \ + || echo "has_changes=true" >> "$GITHUB_OUTPUT" + + - name: Comment on PR + if: steps.version.outputs.changed == 'true' + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const fs = require('fs'); + const md = fs.readFileSync('/tmp/behavior-diff.md', 'utf8'); + const marker = ''; + + // Find and update existing comment, or create new one + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + const existing = comments.find(c => c.body.includes(marker)); + + const body = `${marker}\n${md}`; + if (existing) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existing.id, + body, + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body, + }); + } + + release: + name: Release + needs: [test] + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + permissions: + attestations: write + artifact-metadata: write + contents: write + id-token: write + issues: write + packages: write + pull-requests: write + + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 + + - name: Semantic Release + id: semantic + uses: cycjimmy/semantic-release-action@b12c8f6015dc215fe37bc154d4ad456dd3833c90 # v6.0.0 + with: + tag_format: 'v${version}' + extra_plugins: | + @semantic-release/changelog + @semantic-release/git + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Set Docker metadata + id: meta + if: steps.semantic.outputs.new_release_published == 'true' + uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0 + with: + images: | + ${{ env.DOCKERHUB_USER }}/${{ env.DOCKERHUB_REPO }} + ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_REPO }} + labels: | + org.opencontainers.image.maintainer='amartingarcia,ialejandro' + org.opencontainers.image.title='Steampipe' + org.opencontainers.image.description='Steampipe CLI — Use SQL to query cloud APIs' + org.opencontainers.image.vendor='devops-ia' + tags: | + type=raw,value=${{ steps.semantic.outputs.new_release_git_tag }} + + - name: Set up QEMU + if: steps.semantic.outputs.new_release_published == 'true' + uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 + + - name: Set up Docker Buildx + if: steps.semantic.outputs.new_release_published == 'true' + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + + - name: Cache Docker layers + if: steps.semantic.outputs.new_release_published == 'true' + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + - name: "[DOCKERHUB] Log in" + if: steps.semantic.outputs.new_release_published == 'true' + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: "[GHCR] Log in" + if: steps.semantic.outputs.new_release_published == 'true' + uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0 + with: + registry: ${{ env.GHCR_REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push Docker image + id: push + if: steps.semantic.outputs.new_release_published == 'true' + uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0 + with: + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + context: . + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/amd64,linux/arm64 + push: true + sbom: true + tags: ${{ steps.meta.outputs.tags }} + + - name: "[DOCKERHUB] Update registry description" + if: steps.semantic.outputs.new_release_published == 'true' + uses: peter-evans/dockerhub-description@1b9a80c056b620d92cedb9d9b5a223409c68ddfa # v5.0.0 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + repository: ${{ env.DOCKERHUB_USER }}/${{ env.DOCKERHUB_REPO }} + + - name: "[GHCR] Generate artifact attestation" + if: steps.semantic.outputs.new_release_published == 'true' + uses: actions/attest@59d89421af93a897026c735860bf21b6eb4f7b26 # v4.1.0 + with: + subject-name: ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_REPO }} + subject-digest: ${{ steps.push.outputs.digest }} + push-to-registry: true + + - name: Move Docker cache + if: steps.semantic.outputs.new_release_published == 'true' + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a8c07fa --- /dev/null +++ b/.gitignore @@ -0,0 +1,16 @@ +# OS +.DS_Store +Thumbs.db + +# IDE +.vscode/ +.idea/ +.coverage + +__pycache__/ + +# OpenSpec — local workflow tooling, not part of the image +openspec/ +OPENSPEC-GUIDE.md +.github/prompts/ +.github/skills/ \ No newline at end of file diff --git a/.hadolint.yaml b/.hadolint.yaml new file mode 100644 index 0000000..b6b5fc4 --- /dev/null +++ b/.hadolint.yaml @@ -0,0 +1,9 @@ +# Hadolint configuration +# https://github.com/hadolint/hadolint + +ignore: + # DL3008: Pin versions in apt-get install — acceptable for base system tools + # in CI-built images where reproducibility is handled by pinning the base image tag + - DL3008 + # DL3009: Delete apt-get lists — handled explicitly in our RUN layer + - DL3009 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..058b292 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,97 @@ +# Contributing + +Thank you for your interest in contributing! This repo builds and publishes a community Docker image for [Steampipe](https://steampipe.io). + +## Prerequisites + +- Docker 20.10+ +- `bash`, `jq`, `python3`, `pip` +- `helm` (optional, for chart testing) + +## Build the image locally + +```bash +# Build with the default Steampipe version from the Dockerfile +docker build -t steampipe:dev . + +# Build with a specific version +docker build --build-arg STEAMPIPE_VERSION=2.4.1 -t steampipe:dev . +``` + +## Run the test suite + +### Unit tests (no Docker needed) + +```bash +pip install -r tests/requirements.txt +python3 -m pytest tests/ --cov=compare_snapshots --cov-report=term-missing +``` + +### Lint the Dockerfile + +```bash +docker run --rm -i hadolint/hadolint < Dockerfile +``` + +### Container structure tests (requires built image) + +```bash +docker run --rm \ + -v "$PWD/structure-tests.yaml:/structure-tests.yaml:ro" \ + -v /var/run/docker.sock:/var/run/docker.sock \ + gcr.io/gcp-runtimes/container-structure-test:latest \ + test --image steampipe:dev --config /structure-tests.yaml +``` + +### Security scan + +```bash +docker run --rm \ + -v /var/run/docker.sock:/var/run/docker.sock \ + aquasec/trivy image --severity CRITICAL --ignore-unfixed steampipe:dev +``` + +## How releases work + +Releases are **fully automated** — do not bump versions manually. + +1. [updatecli](https://www.updatecli.io/) detects new Steampipe releases and opens a PR updating `ARG STEAMPIPE_VERSION` in the `Dockerfile`. +2. The PR CI runs all tests. +3. On merge to `main`, [semantic-release](https://semantic-release.gitbook.io/) reads conventional commits, bumps the chart version, and publishes to GHCR and Docker Hub automatically. + +## Commit message format + +This repo uses [Conventional Commits](https://www.conventionalcommits.org/): + +``` +feat: add support for multi-arch builds +fix: correct plugin directory permissions +chore: bump steampipe to 2.5.0 +docs: add GCP plugin configuration example +``` + +| Type | When to use | +|------|------------| +| `feat` | New feature or capability | +| `fix` | Bug fix | +| `chore` | Maintenance (version bumps, CI tweaks) | +| `docs` | Documentation only | +| `refactor` | Code restructure without behaviour change | + +## Reporting bugs + +Please [open an issue](https://github.com/devops-ia/steampipe/issues/new) with: + +- Steampipe image version +- Docker version (`docker --version`) +- Steps to reproduce +- Expected vs actual behaviour +- Relevant logs (`docker logs steampipe`) + +## Pull requests + +1. Fork the repo and create a branch: `git checkout -b feat/my-improvement` +2. Make your changes +3. Run the tests (see above) +4. Commit using Conventional Commits format +5. Open a PR against `main` diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..46b3dfb --- /dev/null +++ b/Dockerfile @@ -0,0 +1,46 @@ +FROM debian:bookworm-slim + +ARG STEAMPIPE_VERSION=2.4.1 +ARG TARGETARCH + +LABEL maintainer="amartingarcia, ialejandro" +LABEL org.opencontainers.image.title="Steampipe" +LABEL org.opencontainers.image.description="Steampipe CLI — Use SQL to query cloud APIs" +LABEL org.opencontainers.image.source="https://github.com/devops-ia/steampipe" +LABEL org.opencontainers.image.vendor="devops-ia" +LABEL org.opencontainers.image.url="https://steampipe.io" + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# hadolint ignore=DL3008,DL3005 +RUN apt-get update && \ + apt-get upgrade -y --no-install-recommends && \ + apt-get install -y --no-install-recommends ca-certificates curl jq && \ + rm -rf /var/lib/apt/lists/* + +RUN curl -fsSL "https://github.com/turbot/steampipe/releases/download/v${STEAMPIPE_VERSION}/steampipe_linux_${TARGETARCH}.tar.gz" \ + | tar -xz -C /usr/local/bin && \ + chmod +x /usr/local/bin/steampipe + +# UID 9193, GID 0 (OpenShift compatible) +RUN useradd -u 9193 -g 0 -d /home/steampipe -m -s /bin/bash steampipe + +RUN mkdir -p /home/steampipe/.steampipe/{config,internal,logs,plugins} \ + /workspace && \ + chown -R 9193:0 /home/steampipe /workspace && \ + chmod -R g=u /home/steampipe /workspace + +ENV STEAMPIPE_UPDATE_CHECK=false \ + STEAMPIPE_TELEMETRY=none \ + STEAMPIPE_INSTALL_DIR=/home/steampipe/.steampipe \ + STEAMPIPE_LOG_LEVEL=warn + +USER 9193 +WORKDIR /home/steampipe + +EXPOSE 9193 + +HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \ + CMD bash -c 'echo > /dev/tcp/localhost/9193' || exit 1 + +CMD ["steampipe", "service", "start", "--foreground"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..b6eddf8 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 DevOps Solutions + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index 58757e4..def83a2 100644 --- a/README.md +++ b/README.md @@ -1 +1,162 @@ -# steampipe \ No newline at end of file +# steampipe + +[![CI](https://github.com/devops-ia/steampipe/actions/workflows/docker-build.yml/badge.svg)](https://github.com/devops-ia/steampipe/actions/workflows/docker-build.yml) +[![GitHub release](https://img.shields.io/github/v/release/devops-ia/steampipe)](https://github.com/devops-ia/steampipe/releases) +[![Docker Hub](https://img.shields.io/docker/v/devopsiaci/steampipe?label=Docker%20Hub&logo=docker)](https://hub.docker.com/r/devopsiaci/steampipe) +[![Docker Pulls](https://img.shields.io/docker/pulls/devopsiaci/steampipe?logo=docker)](https://hub.docker.com/r/devopsiaci/steampipe) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) + +Community Docker image for [Steampipe](https://steampipe.io) — use SQL to instantly query cloud services (AWS, Azure, GCP and more). + +> **Why this image?** Turbot [stopped publishing official Docker images](https://steampipe.io/docs/managing/containers) after Steampipe v0.22.0. This project provides multi-arch container images built from official pre-compiled binaries. + +## Quick start + +```bash +# Run Steampipe as a service (PostgreSQL endpoint on port 9193) +docker run -d --name steampipe \ + -p 9193:9193 \ + ghcr.io/devops-ia/steampipe:2.4.1 \ + steampipe service start --foreground --database-listen network + +# Install a plugin +docker exec steampipe steampipe plugin install aws + +# Connect with any PostgreSQL client +psql -h localhost -p 9193 -U steampipe -d steampipe +``` + +## Image details + +| Property | Value | +|----------|-------| +| Base image | `debian:bookworm-slim` | +| Architectures | `linux/amd64`, `linux/arm64` | +| User | `steampipe` (UID 9193, GID 0) | +| Port | 9193 (PostgreSQL) | +| Default CMD | `steampipe service start --foreground` | + +## Registries + +```bash +# GitHub Container Registry +docker pull ghcr.io/devops-ia/steampipe:2.4.1 + +# Docker Hub +docker pull devopsiaci/steampipe:2.4.1 +``` + +## Documentation + +| Topic | Description | +|-------|-------------| +| [Getting Started](docs/getting-started.md) | Docker, Docker Compose, plugin install, first query | +| [Configuration](docs/configuration.md) | Environment variables, `.spc` plugin configs, memory tuning | +| [Kubernetes](docs/kubernetes.md) | Helm chart, OpenShift, Secrets, health checks | +| [Examples](docs/examples.md) | AWS queries, multi-cloud, Python/Node/Go clients, security audits | +| [Troubleshooting](docs/troubleshooting.md) | Connection errors, permissions, OOM, debug mode | + +## Versioning + +Image versions track upstream Steampipe releases 1:1. New versions are detected automatically via [updatecli](https://www.updatecli.io/) and published after merge. + +## Contributing + +See [CONTRIBUTING.md](CONTRIBUTING.md). + +## License + +MIT — see [LICENSE](LICENSE). + +Community Docker image for [Steampipe](https://steampipe.io) — use SQL to instantly query cloud services (AWS, Azure, GCP and more). + +> **Why this image?** Turbot [stopped publishing official Docker images](https://steampipe.io/docs/managing/containers) after Steampipe v0.22.0. This project provides multi-arch container images built from official pre-compiled binaries. + +## Quick start + +```bash +# Run Steampipe as a service (PostgreSQL endpoint on port 9193) +docker run -d --name steampipe \ + -p 9193:9193 \ + ghcr.io/devops-ia/steampipe:2.4.1 \ + service start --foreground --database-listen network + +# Install a plugin +docker exec steampipe steampipe plugin install aws + +# Connect with any PostgreSQL client +psql -h localhost -p 9193 -U steampipe -d steampipe +``` + +## Image details + +| Property | Value | +|----------|-------| +| Base image | `debian:bookworm-slim` | +| Architectures | `linux/amd64`, `linux/arm64` | +| User | `steampipe` (UID 9193, GID 0) | +| Port | 9193 (PostgreSQL) | +| Entrypoint | `steampipe` | +| Default CMD | `service start --foreground` | + +## Registries + +```bash +# GitHub Container Registry +docker pull ghcr.io/devops-ia/steampipe:2.4.1 + +# Docker Hub +docker pull devopsiaci/steampipe:2.4.1 +``` + +## `service start` flags + +| Flag | Description | Default | +|------|-------------|---------| +| `--foreground` | Run in foreground (required for containers) | — | +| `--database-listen` | Accept connections from `local` or `network` | `local` | +| `--database-port` | PostgreSQL port | `9193` | +| `--database-password` | Set database password | random | +| `--show-password` | Show password in logs | `false` | + +## Environment variables + +Container-optimized defaults are pre-configured. Override as needed: + +| Variable | Image default | Description | +|----------|--------------|-------------| +| `STEAMPIPE_UPDATE_CHECK` | `false` | Disable update checking | +| `STEAMPIPE_TELEMETRY` | `none` | Disable telemetry | +| `STEAMPIPE_LOG_LEVEL` | `warn` | Logging level | +| `STEAMPIPE_DATABASE_PASSWORD` | random | Database password | +| `STEAMPIPE_MEMORY_MAX_MB` | `1024` | Process memory soft limit (MB) | +| `STEAMPIPE_PLUGIN_MEMORY_MAX_MB` | `1024` | Per-plugin memory soft limit (MB) | +| `STEAMPIPE_CACHE` | `true` | Enable/disable query cache | +| `STEAMPIPE_CACHE_TTL` | `300` | Cache TTL in seconds | +| `STEAMPIPE_QUERY_TIMEOUT` | `240` | Query timeout in seconds | +| `STEAMPIPE_INSTALL_DIR` | `/home/steampipe/.steampipe` | Installation directory | +| `STEAMPIPE_DIAGNOSTIC_LEVEL` | `NONE` | Diagnostic level (`ALL`, `NONE`) | + +Full reference: [Steampipe Environment Variables](https://steampipe.io/docs/reference/env-vars/overview) + +## Kubernetes / Helm + +This image is designed to work with the [helm-steampipe](https://github.com/devops-ia/helm-steampipe) Helm chart: + +- **UID 9193 / GID 0** — compatible with OpenShift restricted SCC +- **Directory structure** matches chart volume mounts (`/home/steampipe/.steampipe/{config,internal,logs,plugins}`, `/workspace`) +- **Shell available** (`/bin/bash`, `/bin/sh`) for init container scripts + +## Versioning + +Image versions track upstream Steampipe releases 1:1: + +| Image tag | Steampipe version | +|-----------|-------------------| +| `2.4.1` | [v2.4.1](https://github.com/turbot/steampipe/releases/tag/v2.4.1) | + +New versions are detected automatically via [updatecli](https://www.updatecli.io/) and published after merge. + +## License + +MIT — see [LICENSE](LICENSE). \ No newline at end of file diff --git a/cli-snapshot.json b/cli-snapshot.json new file mode 100644 index 0000000..58cb376 --- /dev/null +++ b/cli-snapshot.json @@ -0,0 +1,76 @@ +{ + "version": "2.4.1", + "snapshot_date": "2026-04-10", + "subcommands": [ + "completion", + "login", + "plugin", + "query", + "service" + ], + "service_start_flags": [ + "--database-listen", + "--database-password", + "--database-port", + "--foreground", + "--help", + "--show-password" + ], + "query_flags": [ + "--export", + "--header", + "--help", + "--input", + "--max-parallel", + "--output", + "--progress", + "--query-timeout", + "--search-path", + "--search-path-prefix", + "--separator", + "--share", + "--snapshot", + "--snapshot-location", + "--snapshot-tag", + "--snapshot-title", + "--timing" + ], + "plugin_flags": [ + "--help", + "--progress" + ], + "env_vars": [ + "PIPES_HOST", + "PIPES_INSTALL_DIR", + "PIPES_TOKEN", + "STEAMPIPE_CACHE", + "STEAMPIPE_CACHE_MAX_SIZE_MB", + "STEAMPIPE_CACHE_MAX_TTL", + "STEAMPIPE_CACHE_TTL", + "STEAMPIPE_CONFIG_DUMP", + "STEAMPIPE_CONNECTION_WATCHER", + "STEAMPIPE_DASHBOARD_START_TIMEOUT", + "STEAMPIPE_DATABASE_PASSWORD", + "STEAMPIPE_DATABASE_SSL_PASSWORD", + "STEAMPIPE_DATABASE_START_TIMEOUT", + "STEAMPIPE_DISPLAY_WIDTH", + "STEAMPIPE_INITDB_DATABASE_NAME", + "STEAMPIPE_INSTALL_DIR", + "STEAMPIPE_MAX_PARALLEL", + "STEAMPIPE_MEMORY_MAX_MB", + "STEAMPIPE_MOD_LOCATION", + "STEAMPIPE_PLUGIN_MEMORY_MAX_MB", + "STEAMPIPE_PLUGIN_START_TIMEOUT", + "STEAMPIPE_QUERY_TIMEOUT", + "STEAMPIPE_SNAPSHOT_LOCATION", + "STEAMPIPE_TELEMETRY", + "STEAMPIPE_UPDATE_CHECK", + "STEAMPIPE_WORKSPACE", + "STEAMPIPE_WORKSPACE_CHDIR", + "STEAMPIPE_WORKSPACE_DATABASE", + "STEAMPIPE_WORKSPACE_PROFILES_LOCATION" + ], + "help_text_hash": "pending-docker-build", + "service_help_hash": "pending-docker-build", + "query_help_hash": "pending-docker-build" +} diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..cdf77d1 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,142 @@ +# Configuration + +## Environment variables + +All container-optimized defaults are pre-configured. Override any variable with `-e KEY=value` or in your Compose file. + +| Variable | Image default | Description | +|----------|--------------|-------------| +| `STEAMPIPE_UPDATE_CHECK` | `false` | Disable automatic update checks | +| `STEAMPIPE_TELEMETRY` | `none` | Disable telemetry (`none` or `info`) | +| `STEAMPIPE_LOG_LEVEL` | `warn` | Log verbosity (`trace`, `debug`, `info`, `warn`, `error`) | +| `STEAMPIPE_DATABASE_PASSWORD` | random | PostgreSQL password for the `steampipe` user | +| `STEAMPIPE_DATABASE_PORT` | `9193` | PostgreSQL port | +| `STEAMPIPE_MEMORY_MAX_MB` | `1024` | Soft memory limit for the Steampipe process | +| `STEAMPIPE_PLUGIN_MEMORY_MAX_MB` | `1024` | Soft memory limit per plugin | +| `STEAMPIPE_CACHE` | `true` | Enable/disable query result cache | +| `STEAMPIPE_CACHE_TTL` | `300` | Cache TTL in seconds | +| `STEAMPIPE_QUERY_TIMEOUT` | `240` | Query timeout in seconds | +| `STEAMPIPE_MAX_PARALLEL` | `10` | Maximum parallel query executions | +| `STEAMPIPE_INSTALL_DIR` | `/home/steampipe/.steampipe` | Steampipe home directory | +| `STEAMPIPE_DIAGNOSTIC_LEVEL` | `NONE` | Diagnostic level (`ALL` or `NONE`) | + +Full reference: [Steampipe Environment Variables](https://steampipe.io/docs/reference/env-vars/overview) + +## Set a fixed database password + +```bash +docker run -d --name steampipe \ + -p 9193:9193 \ + -e STEAMPIPE_DATABASE_PASSWORD=supersecret \ + -e STEAMPIPE_DATABASE_LISTEN=network \ + ghcr.io/devops-ia/steampipe:2.4.1 \ + steampipe service start --foreground --database-listen network +``` + +## Plugin configuration (.spc files) + +Plugins are configured via HCL files (`.spc`) mounted into `/home/steampipe/.steampipe/config/`. + +### AWS plugin — credentials via environment variables + +Create `aws.spc`: + +```hcl +connection "aws" { + plugin = "aws" + regions = ["us-east-1", "eu-west-1"] +} +``` + +Mount it and pass credentials as environment variables: + +```bash +docker run -d --name steampipe \ + -p 9193:9193 \ + -v "$PWD/aws.spc:/home/steampipe/.steampipe/config/aws.spc:ro" \ + -e AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE \ + -e AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY \ + -e AWS_DEFAULT_REGION=us-east-1 \ + ghcr.io/devops-ia/steampipe:2.4.1 \ + steampipe service start --foreground --database-listen network +``` + +### AWS plugin — credentials via mounted profile + +```bash +docker run -d --name steampipe \ + -p 9193:9193 \ + -v "$HOME/.aws:/home/steampipe/.aws:ro" \ + -v "$PWD/aws.spc:/home/steampipe/.steampipe/config/aws.spc:ro" \ + ghcr.io/devops-ia/steampipe:2.4.1 \ + steampipe service start --foreground --database-listen network +``` + +### GCP plugin + +```hcl +connection "gcp" { + plugin = "gcp" + project = "my-project-id" +} +``` + +```bash +docker run -d --name steampipe \ + -p 9193:9193 \ + -v "$PWD/gcp.spc:/home/steampipe/.steampipe/config/gcp.spc:ro" \ + -v "$PWD/service-account.json:/home/steampipe/.config/gcloud/application_default_credentials.json:ro" \ + ghcr.io/devops-ia/steampipe:2.4.1 \ + steampipe service start --foreground --database-listen network +``` + +### Multiple connections (aggregator) + +```hcl +connection "aws_dev" { + plugin = "aws" + regions = ["us-east-1"] + # profile = "dev" +} + +connection "aws_prod" { + plugin = "aws" + regions = ["us-east-1", "eu-west-1"] + # profile = "prod" +} + +connection "aws_all" { + plugin = "aws" + type = "aggregator" + connections = ["aws_dev", "aws_prod"] +} +``` + +## Kubernetes Secrets for plugin credentials + +In Kubernetes, store credentials as Secrets and inject them as environment variables or mounted files. See [Kubernetes](kubernetes.md) for full examples. + +## Memory tuning + +For large datasets or many concurrent queries, increase the memory limits: + +```bash +docker run -d --name steampipe \ + -p 9193:9193 \ + -e STEAMPIPE_MEMORY_MAX_MB=4096 \ + -e STEAMPIPE_PLUGIN_MEMORY_MAX_MB=2048 \ + -e STEAMPIPE_MAX_PARALLEL=20 \ + ghcr.io/devops-ia/steampipe:2.4.1 \ + steampipe service start --foreground --database-listen network +``` + +Also set Docker memory limits to match: + +```bash +docker run -d --name steampipe \ + --memory=6g --memory-swap=6g \ + -p 9193:9193 \ + -e STEAMPIPE_MEMORY_MAX_MB=4096 \ + ghcr.io/devops-ia/steampipe:2.4.1 \ + steampipe service start --foreground --database-listen network +``` diff --git a/docs/examples.md b/docs/examples.md new file mode 100644 index 0000000..8805571 --- /dev/null +++ b/docs/examples.md @@ -0,0 +1,184 @@ +# Examples + +## Query AWS resources + +Install the AWS plugin and run a query: + +```bash +docker exec steampipe steampipe plugin install aws + +# List all S3 buckets +docker exec steampipe steampipe query \ + "select name, region, creation_date from aws_s3_bucket order by creation_date desc" + +# Find public S3 buckets +docker exec steampipe steampipe query \ + "select name, region from aws_s3_bucket where bucket_policy_is_public = true" + +# List EC2 instances by state +docker exec steampipe steampipe query \ + "select instance_id, instance_type, instance_state, region from aws_ec2_instance order by instance_state" +``` + +## Query multiple clouds + +```bash +docker exec steampipe steampipe plugin install aws azure gcp + +# AWS vs Azure: compare running VMs +docker exec steampipe steampipe query " + select 'aws' as cloud, instance_id as id, instance_type as size, region + from aws_ec2_instance where instance_state = 'running' + union all + select 'azure', id, size, location + from azure_compute_virtual_machine where power_state = 'running' + order by cloud, region +" +``` + +## Use psql for complex queries + +```bash +# Connect interactively +psql -h localhost -p 9193 -U steampipe -d steampipe + +# Run a file +psql -h localhost -p 9193 -U steampipe -d steampipe -f my-query.sql + +# One-liner with output formatting +psql -h localhost -p 9193 -U steampipe -d steampipe \ + -c "select name, region from aws_s3_bucket" \ + --csv > buckets.csv +``` + +## Connect from application code + +### Python + +```python +import psycopg2 + +conn = psycopg2.connect( + host="localhost", + port=9193, + dbname="steampipe", + user="steampipe", + password="your-password", + sslmode="disable", +) +cur = conn.cursor() +cur.execute("SELECT name, region FROM aws_s3_bucket") +for row in cur.fetchall(): + print(row) +conn.close() +``` + +### Node.js + +```javascript +const { Client } = require("pg"); + +const client = new Client({ + host: "localhost", + port: 9193, + database: "steampipe", + user: "steampipe", + password: "your-password", + ssl: false, +}); + +await client.connect(); +const res = await client.query("SELECT name, region FROM aws_s3_bucket"); +console.log(res.rows); +await client.end(); +``` + +### Go + +```go +package main + +import ( + "database/sql" + "fmt" + _ "github.com/lib/pq" +) + +func main() { + db, _ := sql.Open("postgres", + "host=localhost port=9193 dbname=steampipe user=steampipe password=your-password sslmode=disable") + defer db.Close() + + rows, _ := db.Query("SELECT name, region FROM aws_s3_bucket") + defer rows.Close() + for rows.Next() { + var name, region string + rows.Scan(&name, ®ion) + fmt.Printf("%s (%s)\n", name, region) + } +} +``` + +## Steampipe + Powerpipe together + +Run Steampipe as the query backend and Powerpipe as the dashboard frontend: + +```bash +# See examples/docker-compose-with-powerpipe.yml +docker compose -f examples/docker-compose-with-powerpipe.yml up -d + +# Install AWS plugin in Steampipe +docker compose exec steampipe steampipe plugin install aws + +# Install a compliance mod in Powerpipe +docker compose exec powerpipe powerpipe mod install github.com/turbot/steampipe-mod-aws-compliance + +# Open the dashboard +open http://localhost:9033 +``` + +## Export query results + +```bash +# JSON +docker exec steampipe steampipe query \ + "select * from aws_s3_bucket" --output json > buckets.json + +# CSV +docker exec steampipe steampipe query \ + "select * from aws_s3_bucket" --output csv > buckets.csv + +# Markdown table +docker exec steampipe steampipe query \ + "select name, region from aws_s3_bucket limit 10" --output table +``` + +## Security audit example + +```sql +-- Find IAM users with console access and no MFA +SELECT + user_name, + create_date, + password_last_used +FROM aws_iam_user +WHERE + password_enabled = true + AND mfa_enabled = false +ORDER BY create_date; + +-- Find security groups with unrestricted inbound access +SELECT + group_id, + group_name, + description, + region +FROM aws_vpc_security_group +WHERE + EXISTS ( + SELECT 1 + FROM jsonb_array_elements(ip_permissions) AS p + WHERE p->>'IpRanges' LIKE '%0.0.0.0/0%' + ) +ORDER BY region, group_name; +``` diff --git a/docs/getting-started.md b/docs/getting-started.md new file mode 100644 index 0000000..ae1aef5 --- /dev/null +++ b/docs/getting-started.md @@ -0,0 +1,111 @@ +# Getting Started + +## Prerequisites + +- [Docker](https://docs.docker.com/get-docker/) 20.10+ +- A cloud provider account (AWS, Azure, GCP, etc.) for querying + +## Pull the image + +```bash +# GitHub Container Registry (recommended) +docker pull ghcr.io/devops-ia/steampipe:2.4.1 + +# Docker Hub +docker pull devopsiaci/steampipe:2.4.1 +``` + +## Run as a query shell + +Execute a one-off interactive SQL session: + +```bash +docker run -it --rm \ + ghcr.io/devops-ia/steampipe:2.4.1 \ + steampipe query +``` + +## Run as a PostgreSQL service + +Start Steampipe as a persistent background service accessible on port 9193: + +```bash +docker run -d --name steampipe \ + -p 9193:9193 \ + ghcr.io/devops-ia/steampipe:2.4.1 \ + steampipe service start --foreground --database-listen network +``` + +Verify it's running: + +```bash +docker logs steampipe +# Look for: "Database is now running" +``` + +## Install a plugin + +Plugins are installed at runtime and stored in a volume for persistence: + +```bash +# Create a named volume so plugins survive container restarts +docker volume create steampipe-data + +docker run -d --name steampipe \ + -p 9193:9193 \ + -v steampipe-data:/home/steampipe/.steampipe \ + -e STEAMPIPE_DATABASE_PASSWORD=mypassword \ + ghcr.io/devops-ia/steampipe:2.4.1 \ + steampipe service start --foreground --database-listen network + +# Install the AWS plugin +docker exec steampipe steampipe plugin install aws + +# List installed plugins +docker exec steampipe steampipe plugin list +``` + +## Connect with a SQL client + +Once the service is running, connect with any PostgreSQL-compatible client: + +```bash +# psql +psql -h localhost -p 9193 -U steampipe -d steampipe + +# DBeaver, TablePlus, DataGrip — use these connection settings: +# Host: localhost +# Port: 9193 +# Database: steampipe +# User: steampipe +# Password: (see STEAMPIPE_DATABASE_PASSWORD or docker logs) +``` + +## Run with Docker Compose + +See [`examples/docker-compose.yml`](../examples/docker-compose.yml) for a ready-to-use Compose setup. + +```bash +cd examples +docker compose up -d +docker compose exec steampipe steampipe plugin install aws +``` + +## Verify the query engine works + +```bash +docker exec steampipe steampipe query "select 1 as test" +# Expected output: +# +------+ +# | test | +# +------+ +# | 1 | +# +------+ +``` + +## Next steps + +- [Configuration](configuration.md) — env vars, plugin credentials, memory tuning +- [Examples](examples.md) — real-world queries and use cases +- [Kubernetes](kubernetes.md) — deploy with Helm +- [Troubleshooting](troubleshooting.md) — common problems and fixes diff --git a/docs/kubernetes.md b/docs/kubernetes.md new file mode 100644 index 0000000..bf05675 --- /dev/null +++ b/docs/kubernetes.md @@ -0,0 +1,183 @@ +# Kubernetes + +This image is designed for the [helm-steampipe](https://github.com/devops-ia/helm-steampipe) Helm chart but also works with plain Kubernetes manifests. + +## Install with Helm + +```bash +helm repo add devops-ia https://devops-ia.github.io/helm-charts +helm repo update + +helm install steampipe devops-ia/steampipe \ + --set image.repository=ghcr.io/devops-ia/steampipe \ + --set image.tag=2.4.1 \ + --set bbdd.enabled=true \ + --set bbdd.listen=network \ + --namespace steampipe \ + --create-namespace +``` + +## Upgrade the image version + +```bash +helm upgrade steampipe devops-ia/steampipe \ + --set image.tag=2.5.0 \ + --reuse-values +``` + +## Custom values file + +```yaml +# values.yaml +image: + repository: ghcr.io/devops-ia/steampipe + tag: "2.4.1" + +bbdd: + enabled: true + listen: network + port: 9193 + +resources: + requests: + cpu: "250m" + memory: "512Mi" + limits: + cpu: "2000m" + memory: "2Gi" + +env: + - name: STEAMPIPE_MEMORY_MAX_MB + value: "1536" + - name: STEAMPIPE_PLUGIN_MEMORY_MAX_MB + value: "1024" + - name: STEAMPIPE_DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: steampipe-credentials + key: password +``` + +```bash +helm install steampipe devops-ia/steampipe -f values.yaml \ + --namespace steampipe --create-namespace +``` + +## Inject plugin credentials via Secrets + +```yaml +# secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: aws-credentials + namespace: steampipe +type: Opaque +stringData: + AWS_ACCESS_KEY_ID: "AKIAIOSFODNN7EXAMPLE" + AWS_SECRET_ACCESS_KEY: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + AWS_DEFAULT_REGION: "us-east-1" +``` + +Reference the Secret in your Helm values: + +```yaml +envFrom: + - secretRef: + name: aws-credentials +``` + +## Mount a plugin .spc file via ConfigMap + +```yaml +# configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: steampipe-plugin-config + namespace: steampipe +data: + aws.spc: | + connection "aws" { + plugin = "aws" + regions = ["us-east-1", "eu-west-1"] + } +``` + +Reference it in your Helm values: + +```yaml +extraVolumes: + - name: plugin-config + configMap: + name: steampipe-plugin-config + +extraVolumeMounts: + - name: plugin-config + mountPath: /home/steampipe/.steampipe/config/aws.spc + subPath: aws.spc + readOnly: true +``` + +## Plugin installation via init container + +Install plugins before the main container starts using the chart's `initContainer.plugins` value: + +```yaml +initContainer: + plugins: + - aws + - azure + - gcp +``` + +## OpenShift compatibility + +The image runs as **UID 9193 / GID 0** — compatible with OpenShift's restricted Security Context Constraint (SCC) without modifications. + +```yaml +# No securityContext overrides needed for OpenShift restricted SCC +securityContext: {} +``` + +## Health check + +The PostgreSQL endpoint serves as the health check: + +```yaml +livenessProbe: + exec: + command: + - pg_isready + - -h + - localhost + - -p + - "9193" + - -U + - steampipe + initialDelaySeconds: 30 + periodSeconds: 10 + +readinessProbe: + exec: + command: + - pg_isready + - -h + - localhost + - -p + - "9193" + - -U + - steampipe + initialDelaySeconds: 15 + periodSeconds: 5 +``` + +## Connect from another pod + +```bash +# From any pod in the same namespace +psql -h steampipe -p 9193 -U steampipe -d steampipe + +# Using the full service DNS +psql -h steampipe.steampipe.svc.cluster.local -p 9193 -U steampipe -d steampipe +``` diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md new file mode 100644 index 0000000..3e60b0e --- /dev/null +++ b/docs/troubleshooting.md @@ -0,0 +1,162 @@ +# Troubleshooting + +## Container exits immediately + +**Symptom:** `docker run` exits right away with code 0 or 1. + +**Cause:** Missing `--foreground` flag when running as a service. + +```bash +# Wrong — exits after starting the background daemon +docker run ghcr.io/devops-ia/steampipe:2.4.1 steampipe service start + +# Correct — blocks in foreground (required for containers) +docker run ghcr.io/devops-ia/steampipe:2.4.1 \ + steampipe service start --foreground --database-listen network +``` + +--- + +## Cannot connect on port 9193 + +**Symptom:** `psql: error: connection to server on socket failed: Connection refused` + +**Cause 1:** Steampipe started without `--database-listen network` — by default it only listens on `localhost` inside the container. + +```bash +# Must include --database-listen network to accept external connections +docker run -d -p 9193:9193 \ + ghcr.io/devops-ia/steampipe:2.4.1 \ + steampipe service start --foreground --database-listen network +``` + +**Cause 2:** Service not yet ready — PostgreSQL bootstraps on first start (~30 seconds). + +```bash +# Wait for the service to be ready +until docker exec steampipe pg_isready -h localhost -p 9193 -U steampipe; do + echo "Waiting for Steampipe..."; sleep 5 +done +``` + +**Cause 3:** Port not published in `docker run`. + +```bash +# Publish the port explicitly +docker run -d -p 9193:9193 ... +``` + +--- + +## Plugin not found after install + +**Symptom:** `Error: could not find plugin 'aws'` after `steampipe plugin install aws`. + +**Cause:** Plugin was installed in a container that was later removed; no persistent volume. + +```bash +# Use a named volume for the Steampipe home directory +docker volume create steampipe-data + +docker run -d --name steampipe \ + -p 9193:9193 \ + -v steampipe-data:/home/steampipe/.steampipe \ + ghcr.io/devops-ia/steampipe:2.4.1 \ + steampipe service start --foreground --database-listen network + +# Plugins installed now survive container recreation +docker exec steampipe steampipe plugin install aws +``` + +--- + +## Permission denied errors + +**Symptom:** `permission denied` writing to `/home/steampipe/.steampipe/` or plugin directories. + +**Cause:** Volume mounted with wrong ownership; the container runs as UID 9193. + +```bash +# Fix ownership on the host before mounting +sudo chown -R 9193:0 ./steampipe-data + +# Or use Docker's --user flag is NOT recommended — always run as UID 9193 +``` + +In Kubernetes, use an `initContainer` to fix permissions: + +```yaml +initContainers: + - name: fix-permissions + image: busybox + command: ["chown", "-R", "9193:0", "/home/steampipe/.steampipe"] + volumeMounts: + - name: steampipe-data + mountPath: /home/steampipe/.steampipe +``` + +--- + +## Memory / OOM errors + +**Symptom:** Container is killed with exit code 137, or queries fail with out-of-memory errors. + +**Cause:** Default memory limits are conservative for large cloud accounts. + +```bash +# Increase memory limits +docker run -d --name steampipe \ + --memory=4g \ + -p 9193:9193 \ + -e STEAMPIPE_MEMORY_MAX_MB=3072 \ + -e STEAMPIPE_PLUGIN_MEMORY_MAX_MB=2048 \ + ghcr.io/devops-ia/steampipe:2.4.1 \ + steampipe service start --foreground --database-listen network +``` + +In Kubernetes: + +```yaml +resources: + requests: + memory: "1Gi" + limits: + memory: "4Gi" +env: + - name: STEAMPIPE_MEMORY_MAX_MB + value: "3072" + - name: STEAMPIPE_PLUGIN_MEMORY_MAX_MB + value: "2048" +``` + +--- + +## Query timeout + +**Symptom:** `Error: query timed out` for large queries. + +```bash +# Increase timeout (default 240s) +docker exec steampipe steampipe query \ + --query-timeout 600 \ + "select * from aws_s3_bucket" + +# Or set permanently via env var +docker run -e STEAMPIPE_QUERY_TIMEOUT=600 ... +``` + +--- + +## Debug mode + +Enable detailed logging to diagnose unexpected behaviour: + +```bash +docker run -d --name steampipe \ + -p 9193:9193 \ + -e STEAMPIPE_LOG_LEVEL=debug \ + ghcr.io/devops-ia/steampipe:2.4.1 \ + steampipe service start --foreground --database-listen network + +docker logs -f steampipe +``` diff --git a/examples/aws.spc b/examples/aws.spc new file mode 100644 index 0000000..982dc48 --- /dev/null +++ b/examples/aws.spc @@ -0,0 +1,34 @@ +# AWS plugin configuration example. +# Mount this file to: /home/steampipe/.steampipe/config/aws.spc +# +# Credentials are read from environment variables (recommended for containers): +# AWS_ACCESS_KEY_ID +# AWS_SECRET_ACCESS_KEY +# AWS_DEFAULT_REGION +# +# Or from a mounted ~/.aws/credentials file. + +connection "aws" { + plugin = "aws" + regions = ["us-east-1", "eu-west-1", "ap-southeast-1"] +} + +# Multi-account setup with an aggregator: +# +# connection "aws_dev" { +# plugin = "aws" +# profile = "dev" +# regions = ["us-east-1"] +# } +# +# connection "aws_prod" { +# plugin = "aws" +# profile = "prod" +# regions = ["us-east-1", "eu-west-1"] +# } +# +# connection "aws_all" { +# plugin = "aws" +# type = "aggregator" +# connections = ["aws_dev", "aws_prod"] +# } diff --git a/examples/docker-compose-with-powerpipe.yml b/examples/docker-compose-with-powerpipe.yml new file mode 100644 index 0000000..09836a9 --- /dev/null +++ b/examples/docker-compose-with-powerpipe.yml @@ -0,0 +1,42 @@ +services: + steampipe: + image: ghcr.io/devops-ia/steampipe:2.4.1 + container_name: steampipe + command: steampipe service start --foreground --database-listen network + ports: + - "9193:9193" + volumes: + - steampipe-data:/home/steampipe/.steampipe + environment: + STEAMPIPE_DATABASE_PASSWORD: steampipe + STEAMPIPE_UPDATE_CHECK: "false" + STEAMPIPE_TELEMETRY: none + STEAMPIPE_LOG_LEVEL: warn + healthcheck: + test: ["CMD-SHELL", "pg_isready -h localhost -p 9193 -U steampipe"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + restart: unless-stopped + + powerpipe: + image: ghcr.io/devops-ia/powerpipe:1.5.1 + container_name: powerpipe + ports: + - "9033:9033" + volumes: + - workspace:/workspace + environment: + POWERPIPE_DATABASE: "postgres://steampipe:steampipe@steampipe:9193/steampipe" + POWERPIPE_UPDATE_CHECK: "false" + POWERPIPE_TELEMETRY: none + POWERPIPE_LISTEN: network + depends_on: + steampipe: + condition: service_healthy + restart: unless-stopped + +volumes: + steampipe-data: + workspace: diff --git a/examples/docker-compose.yml b/examples/docker-compose.yml new file mode 100644 index 0000000..ef13d03 --- /dev/null +++ b/examples/docker-compose.yml @@ -0,0 +1,24 @@ +services: + steampipe: + image: ghcr.io/devops-ia/steampipe:2.4.1 + container_name: steampipe + command: steampipe service start --foreground --database-listen network + ports: + - "9193:9193" + volumes: + - steampipe-data:/home/steampipe/.steampipe + environment: + STEAMPIPE_DATABASE_PASSWORD: steampipe + STEAMPIPE_UPDATE_CHECK: "false" + STEAMPIPE_TELEMETRY: none + STEAMPIPE_LOG_LEVEL: warn + healthcheck: + test: ["CMD-SHELL", "pg_isready -h localhost -p 9193 -U steampipe"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + restart: unless-stopped + +volumes: + steampipe-data: diff --git a/package.json b/package.json new file mode 100644 index 0000000..4b91009 --- /dev/null +++ b/package.json @@ -0,0 +1,21 @@ +{ + "name": "steampipe", + "private": true, + "release": { + "branches": ["main"], + "tagFormat": "v${version}", + "plugins": [ + "@semantic-release/commit-analyzer", + "@semantic-release/release-notes-generator", + "@semantic-release/changelog", + [ + "@semantic-release/git", + { + "assets": ["CHANGELOG.md"], + "message": "chore(release): ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}" + } + ], + "@semantic-release/github" + ] + } +} diff --git a/scripts/compare_snapshots.py b/scripts/compare_snapshots.py new file mode 100644 index 0000000..cfdccbd --- /dev/null +++ b/scripts/compare_snapshots.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +"""Compare two CLI snapshots and generate a semantic diff. + +Usage: + python3 scripts/compare-snapshots.py old.json new.json [--output-md diff.md] [--output-json diff.json] + +Outputs: + - JSON diff to stdout (or --output-json) + - Markdown summary to --output-md (optional) + +Exit codes: + 0 — no behavioral changes + 1 — behavioral changes detected + 2 — error +""" + +import argparse +import json +import sys +from pathlib import Path + + +def diff_lists(old: list, new: list) -> dict: + """Compare two sorted lists, return added/removed.""" + old_set = set(old) + new_set = set(new) + added = sorted(new_set - old_set) + removed = sorted(old_set - new_set) + return {"added": added, "removed": removed} + + +_SKIP_KEYS = {"version", "snapshot_date"} +_HASH_SUFFIX = "_hash" + + +def _array_keys(snapshot: dict) -> list[str]: + """Return all keys whose values are lists (skip metadata/hash keys).""" + return sorted( + k for k, v in snapshot.items() + if isinstance(v, list) and k not in _SKIP_KEYS + ) + + +def _hash_keys(snapshot: dict) -> list[str]: + """Return all keys that are hash fields.""" + return sorted( + k for k, v in snapshot.items() + if isinstance(v, str) and k.endswith(_HASH_SUFFIX) + ) + + +def compare(old: dict, new: dict) -> dict: + """Generate semantic diff between two snapshots (generic, works for any CLI).""" + # Collect all array keys from both snapshots + all_array_keys = sorted(set(_array_keys(old)) | set(_array_keys(new))) + all_hash_keys = sorted(set(_hash_keys(old)) | set(_hash_keys(new))) + + result: dict = { + "has_changes": False, + "version_change": f"{old.get('version', '?')} → {new.get('version', '?')}", + "categories": {}, + "hash_changes": [], + } + + for key in all_array_keys: + d = diff_lists(old.get(key, []), new.get(key, [])) + result["categories"][key] = d + if d["added"] or d["removed"]: + result["has_changes"] = True + + for key in all_hash_keys: + if old.get(key) != new.get(key): + result["hash_changes"].append(key.removesuffix(_HASH_SUFFIX).replace("_", " ")) + + return result + + +def render_diff_table(label: str, diff: dict) -> str: + """Render a single section's added/removed as markdown.""" + lines = [] + if diff["added"] or diff["removed"]: + lines.append(f"\n### {label}") + lines.append("| Change | Item |") + lines.append("|--------|------|") + for item in diff["removed"]: + lines.append(f"| ❌ Removed | `{item}` |") + for item in diff["added"]: + lines.append(f"| ➕ Added | `{item}` |") + return "\n".join(lines) + + +def render_markdown(diff: dict) -> str: + """Render full markdown summary.""" + lines = [f"## 🔍 CLI Behavioral Changes: {diff['version_change']}\n"] + + if not diff["has_changes"] and not diff["hash_changes"]: + lines.append("✅ **No behavioral changes detected.**") + lines.append( + "\nThe new version has the same CLI flags, env vars, and subcommands." + ) + return "\n".join(lines) + + for key, section_diff in diff["categories"].items(): + label = key.replace("_", " ").title() + table = render_diff_table(label, section_diff) + if table: + lines.append(table) + + if diff["hash_changes"]: + lines.append(f"\n### Help text changes") + lines.append( + f"Help text changed for: {', '.join(diff['hash_changes'])}. " + "Review `--help` output for wording/description updates." + ) + + if diff["has_changes"]: + lines.append("\n---") + lines.append("### ⚠️ Action needed") + lines.append( + "These changes may require updates to `README.md` and `Dockerfile` ENV.\n" + ) + lines.append("@copilot Please review the behavioral changes above and:") + lines.append("1. Update `README.md` flag and env var tables") + lines.append("2. Update `Dockerfile` ENV section if defaults changed") + lines.append("3. Ensure documented behavior matches the current CLI") + + return "\n".join(lines) + + +def main(): + parser = argparse.ArgumentParser(description="Compare CLI snapshots") + parser.add_argument("old", help="Path to old cli-snapshot.json") + parser.add_argument("new", help="Path to new cli-snapshot.json") + parser.add_argument("--output-md", help="Write markdown summary to file") + parser.add_argument("--output-json", help="Write JSON diff to file") + args = parser.parse_args() + + try: + old = json.loads(Path(args.old).read_text()) + new = json.loads(Path(args.new).read_text()) + except (json.JSONDecodeError, FileNotFoundError) as e: + print(f"ERROR: {e}", file=sys.stderr) + sys.exit(2) + + diff = compare(old, new) + + # Output JSON (flatten categories for backward compat) + output = {k: v for k, v in diff.items() if k != "categories"} + output.update(diff["categories"]) + diff_json = json.dumps(output, indent=2) + if args.output_json: + Path(args.output_json).write_text(diff_json + "\n") + else: + print(diff_json) + + # Output markdown + if args.output_md: + md = render_markdown(diff) + Path(args.output_md).write_text(md + "\n") + + # Exit code: 1 if behavioral changes, 0 if none + sys.exit(1 if diff["has_changes"] else 0) + + +if __name__ == "__main__": + main() diff --git a/scripts/extract-cli-snapshot.sh b/scripts/extract-cli-snapshot.sh new file mode 100755 index 0000000..0ad320a --- /dev/null +++ b/scripts/extract-cli-snapshot.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +# Extract a behavioral snapshot of a Cobra CLI. +# Runs INSIDE the Docker container (mount this script as a volume). +# +# Usage: +# docker run --rm --network none \ +# -v "$PWD/scripts:/scripts:ro" \ +# steampipe:test bash /scripts/extract-cli-snapshot.sh +# +# Override binary: CLI_BIN=steampipe bash /scripts/extract-cli-snapshot.sh +# Override depth: MAX_DEPTH=3 bash /scripts/extract-cli-snapshot.sh +# +# Outputs JSON to stdout. Env vars are NOT included (extracted separately +# from upstream source code by extract-env-vars.sh). + +set -euo pipefail + +# Suppress color, update checks, and telemetry for deterministic output. +export NO_COLOR=1 +export STEAMPIPE_UPDATE_CHECK=false STEAMPIPE_TELEMETRY=none +export POWERPIPE_UPDATE_CHECK=false POWERPIPE_TELEMETRY=none +export TERM=dumb + +readonly CLI="${CLI_BIN:-steampipe}" +readonly MAX_DEPTH="${MAX_DEPTH:-4}" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +# Strip ANSI escape sequences from stdin. +_strip_ansi() { + sed 's/\x1b\[[0-9;]*[mGKHF]//g' +} + +# Emit clean --help text for a command. +# Exits non-zero if the command itself fails (not just "no subcommands"). +_help() { + "$@" --help 2>&1 | _strip_ansi +} + +# Parse subcommand names from help text on stdin. +# Prints one name per line; empty output (exit 0) = no Available Commands section. +_parse_subcommands() { + awk ' + /^[[:space:]]*Available Commands:/ { in_section=1; next } + in_section && /^[^[:space:]]/ { exit } + in_section && /^[[:space:]]+[a-z]/ { print $1 } + ' | grep -xv 'help' || true +} + +# --------------------------------------------------------------------------- +# Recursive emitter +# --------------------------------------------------------------------------- + +# Emit one JSON fragment per command visited (flags + help hash), then recurse. +# Args: depth key_prefix [cmd_words...] +_emit() { + local depth="$1" + local key_prefix="$2" + shift 2 + # $@ = full command words, e.g.: steampipe service start + + # Capture help text ONCE; derive flags, subcommands, and hash from it. + local help_text + if ! help_text=$(_help "$@"); then + return 0 # command failed — skip without poisoning the snapshot + fi + + local flags + flags=$(printf '%s\n' "$help_text" \ + | grep -oE -- '--[a-z][a-z0-9-]+' \ + | sort -u \ + | jq -R . | jq -s . \ + || printf '[]') + + local hash + hash=$(printf '%s' "$help_text" | sha256sum | awk '{print $1}') + + jq -n \ + --arg kf "${key_prefix}_flags" \ + --argjson vf "$flags" \ + --arg kh "${key_prefix}_help_hash" \ + --arg vh "$hash" \ + '{($kf): $vf, ($kh): $vh}' + + [[ "$depth" -ge "$MAX_DEPTH" ]] && return 0 + + local subcmds + subcmds=$(printf '%s\n' "$help_text" | _parse_subcommands) || return 0 + + while IFS= read -r sub; do + [[ -z "$sub" ]] && continue + _emit $(( depth + 1 )) "${key_prefix}_${sub}" "$@" "$sub" + done <<< "$subcmds" +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +VERSION=$("$CLI" --version 2>&1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' || printf 'unknown') + +# Capture top-level help once. +TOP_HELP=$(_help "$CLI") +TOP_HASH=$(printf '%s' "$TOP_HELP" | sha256sum | awk '{print $1}') +TOP_CMDS=$(printf '%s\n' "$TOP_HELP" | _parse_subcommands) +SUBCMDS_JSON=$(printf '%s\n' "$TOP_CMDS" | jq -R . | jq -s .) + +# Stream all JSON fragments then merge into one object. +{ + jq -n \ + --arg version "$VERSION" \ + --arg date "$(date -u +%Y-%m-%d)" \ + --arg help_hash "$TOP_HASH" \ + --argjson subs "$SUBCMDS_JSON" \ + '{version: $version, snapshot_date: $date, help_text_hash: $help_hash, subcommands: $subs}' + + while IFS= read -r cmd; do + [[ -z "$cmd" ]] && continue + _emit 1 "$cmd" "$CLI" "$cmd" + done <<< "$TOP_CMDS" + +} | jq -s 'reduce .[] as $item ({}; . + $item)' diff --git a/scripts/extract-env-vars.sh b/scripts/extract-env-vars.sh new file mode 100755 index 0000000..70a1a41 --- /dev/null +++ b/scripts/extract-env-vars.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Extract environment variable names from upstream Steampipe source code. +# Runs on the CI runner (not inside the container). +# +# Usage: +# bash scripts/extract-env-vars.sh 2.4.1 +# +# Outputs a JSON array of env var names to stdout. + +set -euo pipefail + +VERSION="${1:?Usage: $0 }" + +# Fetch env.go from the tagged release +ENV_GO_URL="https://raw.githubusercontent.com/turbot/steampipe/v${VERSION}/pkg/constants/env.go" + +ENV_VARS=$(curl -sfL "$ENV_GO_URL" \ + | grep -oE '"(STEAMPIPE_[A-Z_]+|PIPES_[A-Z_]+)"' \ + | tr -d '"' \ + | sort -u \ + | jq -R . | jq -s .) + +if [ "$ENV_VARS" = "[]" ] || [ -z "$ENV_VARS" ]; then + echo "ERROR: No env vars extracted from $ENV_GO_URL" >&2 + exit 1 +fi + +echo "$ENV_VARS" diff --git a/structure-tests.yaml b/structure-tests.yaml new file mode 100644 index 0000000..81f100f --- /dev/null +++ b/structure-tests.yaml @@ -0,0 +1,80 @@ +schemaVersion: "2.0.0" + +# Validate binary +fileExistenceTests: + - name: "steampipe binary exists" + path: "/usr/local/bin/steampipe" + shouldExist: true + permissions: "-rwxr-xr-x" + + - name: "steampipe home dir exists" + path: "/home/steampipe" + shouldExist: true + + - name: "steampipe install dir exists" + path: "/home/steampipe/.steampipe" + shouldExist: true + + - name: "workspace dir exists" + path: "/workspace" + shouldExist: true + +# Validate binary runs +fileContentTests: [] + +# Validate image metadata +metadataTest: + labels: + - key: "org.opencontainers.image.title" + value: "Steampipe" + - key: "org.opencontainers.image.vendor" + value: "devops-ia" + - key: "org.opencontainers.image.source" + value: "https://github.com/devops-ia/steampipe" + exposedPorts: + - "9193" + cmd: + - "steampipe" + - "service" + - "start" + - "--foreground" + workdir: "/home/steampipe" + user: "9193" + +# Validate ENV vars and binary behavior +commandTests: + - name: "STEAMPIPE_UPDATE_CHECK is false" + command: "sh" + args: ["-c", "printenv STEAMPIPE_UPDATE_CHECK"] + expectedOutput: ["false"] + exitCode: 0 + + - name: "STEAMPIPE_TELEMETRY is none" + command: "sh" + args: ["-c", "printenv STEAMPIPE_TELEMETRY"] + expectedOutput: ["none"] + exitCode: 0 + + - name: "STEAMPIPE_INSTALL_DIR is set" + command: "sh" + args: ["-c", "printenv STEAMPIPE_INSTALL_DIR"] + expectedOutput: ["/home/steampipe/.steampipe"] + exitCode: 0 + + - name: "STEAMPIPE_LOG_LEVEL is warn" + command: "sh" + args: ["-c", "printenv STEAMPIPE_LOG_LEVEL"] + expectedOutput: ["warn"] + exitCode: 0 + + - name: "steampipe --version returns version string" + command: "steampipe" + args: ["--version"] + expectedOutput: + - "Steampipe v[0-9]+\\.[0-9]+\\.[0-9]+" + exitCode: 0 + + - name: "steampipe --help exits 0" + command: "steampipe" + args: ["--help"] + exitCode: 0 diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..e78bfcf --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,5 @@ +"""pytest configuration — adds scripts/ to sys.path for imports.""" +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent / "scripts")) diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 0000000..661d893 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,2 @@ +pytest>=8.0 +pytest-cov>=5.0 diff --git a/tests/test_compare_snapshots.py b/tests/test_compare_snapshots.py new file mode 100644 index 0000000..3a42cf5 --- /dev/null +++ b/tests/test_compare_snapshots.py @@ -0,0 +1,356 @@ +"""Unit tests for scripts/compare-snapshots.py.""" + +import json +import sys +from pathlib import Path + +import pytest + +from compare_snapshots import compare, diff_lists, render_markdown + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +@pytest.fixture +def steampipe_snapshot_v1(): + return { + "version": "2.4.1", + "snapshot_date": "2026-01-01", + "subcommands": ["completion", "login", "plugin", "query", "service"], + "service_start_flags": ["--database-listen", "--database-port", "--foreground", "--help", "--show-password"], + "query_flags": ["--export", "--help", "--output", "--search-path", "--timing"], + "plugin_flags": ["--help", "--progress"], + "env_vars": ["STEAMPIPE_CACHE", "STEAMPIPE_TELEMETRY", "STEAMPIPE_UPDATE_CHECK"], + "help_text_hash": "abc123", + "service_help_hash": "def456", + "query_help_hash": "ghi789", + } + + +@pytest.fixture +def steampipe_snapshot_v2_no_changes(steampipe_snapshot_v1): + """Same content, just different version label.""" + return {**steampipe_snapshot_v1, "version": "2.5.0"} + + +@pytest.fixture +def steampipe_snapshot_v2_with_changes(): + return { + "version": "2.5.0", + "snapshot_date": "2026-02-01", + "subcommands": ["completion", "login", "plugin", "query", "service", "workspace"], + "service_start_flags": ["--database-listen", "--database-port", "--foreground", "--help"], + "query_flags": ["--export", "--help", "--output", "--search-path", "--timing", "--var"], + "plugin_flags": ["--help", "--progress"], + "env_vars": ["STEAMPIPE_CACHE", "STEAMPIPE_NEW_VAR", "STEAMPIPE_UPDATE_CHECK"], + "help_text_hash": "changed_hash", + "service_help_hash": "def456", + "query_help_hash": "ghi789", + } + + +@pytest.fixture +def powerpipe_snapshot_v1(): + """Powerpipe uses different key names — tests the dynamic detection.""" + return { + "version": "1.5.1", + "snapshot_date": "2026-01-01", + "subcommands": ["benchmark", "completion", "mod", "server"], + "server_flags": ["--help", "--listen", "--port"], + "benchmark_run_flags": ["--dry-run", "--export", "--help", "--output"], + "mod_flags": ["--help"], + "env_vars": ["POWERPIPE_LISTEN", "POWERPIPE_PORT"], + "help_text_hash": "pp_abc", + "server_help_hash": "pp_def", + "benchmark_help_hash": "pp_ghi", + } + + +# --------------------------------------------------------------------------- +# diff_lists +# --------------------------------------------------------------------------- + +class TestDiffLists: + def test_added_items(self): + d = diff_lists(["a", "b"], ["a", "b", "c"]) + assert d["added"] == ["c"] + assert d["removed"] == [] + + def test_removed_items(self): + d = diff_lists(["a", "b", "c"], ["a", "b"]) + assert d["added"] == [] + assert d["removed"] == ["c"] + + def test_both_added_and_removed(self): + d = diff_lists(["a", "b"], ["b", "c"]) + assert d["added"] == ["c"] + assert d["removed"] == ["a"] + + def test_no_changes(self): + d = diff_lists(["a", "b"], ["a", "b"]) + assert d["added"] == [] + assert d["removed"] == [] + + def test_empty_lists(self): + d = diff_lists([], []) + assert d["added"] == [] + assert d["removed"] == [] + + def test_result_is_sorted(self): + d = diff_lists(["z", "a"], ["z", "b", "c"]) + assert d["added"] == ["b", "c"] + assert d["removed"] == ["a"] + + +# --------------------------------------------------------------------------- +# compare — no changes +# --------------------------------------------------------------------------- + +class TestCompareNoChanges: + def test_identical_snapshots_has_no_changes(self, steampipe_snapshot_v1): + result = compare(steampipe_snapshot_v1, steampipe_snapshot_v1) + assert result["has_changes"] is False + + def test_version_label_only_has_no_changes(self, steampipe_snapshot_v1, steampipe_snapshot_v2_no_changes): + result = compare(steampipe_snapshot_v1, steampipe_snapshot_v2_no_changes) + assert result["has_changes"] is False + + def test_version_change_is_reported(self, steampipe_snapshot_v1, steampipe_snapshot_v2_no_changes): + result = compare(steampipe_snapshot_v1, steampipe_snapshot_v2_no_changes) + assert "2.4.1" in result["version_change"] + assert "2.5.0" in result["version_change"] + + def test_no_hash_changes_when_identical(self, steampipe_snapshot_v1): + result = compare(steampipe_snapshot_v1, steampipe_snapshot_v1) + assert result["hash_changes"] == [] + + +# --------------------------------------------------------------------------- +# compare — with behavioral changes +# --------------------------------------------------------------------------- + +class TestCompareWithChanges: + def test_has_changes_true(self, steampipe_snapshot_v1, steampipe_snapshot_v2_with_changes): + result = compare(steampipe_snapshot_v1, steampipe_snapshot_v2_with_changes) + assert result["has_changes"] is True + + def test_added_subcommand_detected(self, steampipe_snapshot_v1, steampipe_snapshot_v2_with_changes): + result = compare(steampipe_snapshot_v1, steampipe_snapshot_v2_with_changes) + assert "workspace" in result["categories"]["subcommands"]["added"] + + def test_removed_flag_detected(self, steampipe_snapshot_v1, steampipe_snapshot_v2_with_changes): + result = compare(steampipe_snapshot_v1, steampipe_snapshot_v2_with_changes) + assert "--show-password" in result["categories"]["service_start_flags"]["removed"] + + def test_added_flag_detected(self, steampipe_snapshot_v1, steampipe_snapshot_v2_with_changes): + result = compare(steampipe_snapshot_v1, steampipe_snapshot_v2_with_changes) + assert "--var" in result["categories"]["query_flags"]["added"] + + def test_added_env_var_detected(self, steampipe_snapshot_v1, steampipe_snapshot_v2_with_changes): + result = compare(steampipe_snapshot_v1, steampipe_snapshot_v2_with_changes) + assert "STEAMPIPE_NEW_VAR" in result["categories"]["env_vars"]["added"] + + def test_removed_env_var_detected(self, steampipe_snapshot_v1, steampipe_snapshot_v2_with_changes): + result = compare(steampipe_snapshot_v1, steampipe_snapshot_v2_with_changes) + assert "STEAMPIPE_TELEMETRY" in result["categories"]["env_vars"]["removed"] + + def test_hash_change_detected(self, steampipe_snapshot_v1, steampipe_snapshot_v2_with_changes): + result = compare(steampipe_snapshot_v1, steampipe_snapshot_v2_with_changes) + assert len(result["hash_changes"]) > 0 + + +# --------------------------------------------------------------------------- +# compare — dynamic key detection (powerpipe) +# --------------------------------------------------------------------------- + +class TestCompareDynamicKeys: + def test_powerpipe_keys_detected(self, powerpipe_snapshot_v1): + result = compare(powerpipe_snapshot_v1, powerpipe_snapshot_v1) + assert "server_flags" in result["categories"] + assert "benchmark_run_flags" in result["categories"] + assert "mod_flags" in result["categories"] + + def test_no_hardcoded_steampipe_keys_required(self, powerpipe_snapshot_v1): + """compare() must work with powerpipe keys, not just steampipe keys.""" + result = compare(powerpipe_snapshot_v1, powerpipe_snapshot_v1) + assert result["has_changes"] is False + # steampipe-specific keys should NOT appear if not in snapshot + assert "service_start_flags" not in result["categories"] + assert "query_flags" not in result["categories"] + + def test_keys_only_in_new_snapshot_are_detected(self, powerpipe_snapshot_v1): + old = {**powerpipe_snapshot_v1} + new = {**powerpipe_snapshot_v1, "new_category": ["item-a", "item-b"]} + result = compare(old, new) + assert "new_category" in result["categories"] + assert result["categories"]["new_category"]["added"] == ["item-a", "item-b"] + assert result["has_changes"] is True + + +# --------------------------------------------------------------------------- +# render_markdown +# --------------------------------------------------------------------------- + +class TestRenderMarkdown: + def test_no_changes_message(self, steampipe_snapshot_v1): + diff = compare(steampipe_snapshot_v1, steampipe_snapshot_v1) + md = render_markdown(diff) + assert "No behavioral changes detected" in md + assert "Action needed" not in md + + def test_changes_include_tables(self, steampipe_snapshot_v1, steampipe_snapshot_v2_with_changes): + diff = compare(steampipe_snapshot_v1, steampipe_snapshot_v2_with_changes) + md = render_markdown(diff) + assert "❌ Removed" in md + assert "➕ Added" in md + + def test_changes_include_copilot_mention(self, steampipe_snapshot_v1, steampipe_snapshot_v2_with_changes): + diff = compare(steampipe_snapshot_v1, steampipe_snapshot_v2_with_changes) + md = render_markdown(diff) + assert "@copilot" in md + + def test_hash_change_section_present(self, steampipe_snapshot_v1, steampipe_snapshot_v2_with_changes): + diff = compare(steampipe_snapshot_v1, steampipe_snapshot_v2_with_changes) + md = render_markdown(diff) + assert "Help text changes" in md + + def test_version_in_header(self, steampipe_snapshot_v1, steampipe_snapshot_v2_with_changes): + diff = compare(steampipe_snapshot_v1, steampipe_snapshot_v2_with_changes) + md = render_markdown(diff) + assert "2.4.1" in md + assert "2.5.0" in md + + +class TestMain: + """Test main() in-process via mocking to get coverage tracking.""" + + def test_main_no_changes_exits_0(self, steampipe_snapshot_v1, tmp_path, monkeypatch): + import compare_snapshots as cs + old_file = tmp_path / "old.json" + new_file = tmp_path / "new.json" + old_file.write_text(json.dumps(steampipe_snapshot_v1)) + new_file.write_text(json.dumps(steampipe_snapshot_v1)) + + monkeypatch.setattr(sys, "argv", ["compare_snapshots.py", str(old_file), str(new_file)]) + with pytest.raises(SystemExit) as exc: + cs.main() + assert exc.value.code == 0 + + def test_main_with_changes_exits_1(self, steampipe_snapshot_v1, steampipe_snapshot_v2_with_changes, tmp_path, monkeypatch): + import compare_snapshots as cs + old_file = tmp_path / "old.json" + new_file = tmp_path / "new.json" + old_file.write_text(json.dumps(steampipe_snapshot_v1)) + new_file.write_text(json.dumps(steampipe_snapshot_v2_with_changes)) + + monkeypatch.setattr(sys, "argv", ["compare_snapshots.py", str(old_file), str(new_file)]) + with pytest.raises(SystemExit) as exc: + cs.main() + assert exc.value.code == 1 + + def test_main_bad_json_exits_2(self, tmp_path, monkeypatch): + import compare_snapshots as cs + bad_file = tmp_path / "bad.json" + bad_file.write_text("not valid {{{") + good_file = tmp_path / "good.json" + good_file.write_text("{}") + + monkeypatch.setattr(sys, "argv", ["compare_snapshots.py", str(bad_file), str(good_file)]) + with pytest.raises(SystemExit) as exc: + cs.main() + assert exc.value.code == 2 + + def test_main_writes_output_files(self, steampipe_snapshot_v1, steampipe_snapshot_v2_with_changes, tmp_path, monkeypatch): + import compare_snapshots as cs + old_file = tmp_path / "old.json" + new_file = tmp_path / "new.json" + out_json = tmp_path / "diff.json" + out_md = tmp_path / "diff.md" + old_file.write_text(json.dumps(steampipe_snapshot_v1)) + new_file.write_text(json.dumps(steampipe_snapshot_v2_with_changes)) + + monkeypatch.setattr(sys, "argv", [ + "compare_snapshots.py", str(old_file), str(new_file), + "--output-json", str(out_json), + "--output-md", str(out_md), + ]) + with pytest.raises(SystemExit): + cs.main() + + assert out_json.exists() + assert out_md.exists() + data = json.loads(out_json.read_text()) + assert data["has_changes"] is True + assert "CLI Behavioral Changes" in out_md.read_text() + + +# --------------------------------------------------------------------------- +# CLI invocation via subprocess (exit codes) +# --------------------------------------------------------------------------- + +class TestExitCodes: + def _run(self, old_data, new_data, tmp_path): + import subprocess + old_file = tmp_path / "old.json" + new_file = tmp_path / "new.json" + old_file.write_text(json.dumps(old_data)) + new_file.write_text(json.dumps(new_data)) + script = Path(__file__).parent.parent / "scripts" / "compare_snapshots.py" + result = subprocess.run( + [sys.executable, str(script), str(old_file), str(new_file)], + capture_output=True, + ) + return result + + def test_exit_0_when_no_changes(self, steampipe_snapshot_v1, tmp_path): + result = self._run(steampipe_snapshot_v1, steampipe_snapshot_v1, tmp_path) + assert result.returncode == 0 + + def test_exit_1_when_changes(self, steampipe_snapshot_v1, steampipe_snapshot_v2_with_changes, tmp_path): + result = self._run(steampipe_snapshot_v1, steampipe_snapshot_v2_with_changes, tmp_path) + assert result.returncode == 1 + + def test_exit_2_on_missing_file(self, tmp_path): + import subprocess + missing = tmp_path / "missing.json" + good_file = tmp_path / "good.json" + good_file.write_text("{}") + script = Path(__file__).parent.parent / "scripts" / "compare_snapshots.py" + result = subprocess.run( + [sys.executable, str(script), str(missing), str(good_file)], + capture_output=True, + ) + assert result.returncode == 2 + + def test_output_json_file_written(self, steampipe_snapshot_v1, steampipe_snapshot_v2_with_changes, tmp_path): + import subprocess + old_file = tmp_path / "old.json" + new_file = tmp_path / "new.json" + out_json = tmp_path / "diff.json" + old_file.write_text(json.dumps(steampipe_snapshot_v1)) + new_file.write_text(json.dumps(steampipe_snapshot_v2_with_changes)) + script = Path(__file__).parent.parent / "scripts" / "compare_snapshots.py" + subprocess.run( + [sys.executable, str(script), str(old_file), str(new_file), "--output-json", str(out_json)], + capture_output=True, + ) + assert out_json.exists() + data = json.loads(out_json.read_text()) + assert data["has_changes"] is True + + def test_output_md_file_written(self, steampipe_snapshot_v1, steampipe_snapshot_v2_with_changes, tmp_path): + import subprocess + old_file = tmp_path / "old.json" + new_file = tmp_path / "new.json" + out_md = tmp_path / "diff.md" + old_file.write_text(json.dumps(steampipe_snapshot_v1)) + new_file.write_text(json.dumps(steampipe_snapshot_v2_with_changes)) + script = Path(__file__).parent.parent / "scripts" / "compare_snapshots.py" + subprocess.run( + [sys.executable, str(script), str(old_file), str(new_file), "--output-md", str(out_md)], + capture_output=True, + ) + assert out_md.exists() + assert "CLI Behavioral Changes" in out_md.read_text() diff --git a/trivy.yaml b/trivy.yaml new file mode 100644 index 0000000..8455965 --- /dev/null +++ b/trivy.yaml @@ -0,0 +1,9 @@ +# Trivy configuration +# https://trivy.dev/docs/references/configuration/config-file/ + +scan: + # Skip the steampipe binary from vulnerability scanning. + # CVEs in the Go binary are upstream Turbot's responsibility; + # we cannot patch a pre-compiled third-party binary. + skip-files: + - usr/local/bin/steampipe