From 0bea115dc6375f711ea88807ad3baeddb8bb6d3d Mon Sep 17 00:00:00 2001 From: Douglas Barahona Date: Thu, 3 Sep 2026 16:27:25 -0600 Subject: [PATCH] perf: parallelize ClamAV virus scans --- Dockerfile | 2 +- README.md | 4 +- scripts/virus-scan | 118 +++++++++++++++++++++++----- tests/test-virus-scan.sh | 163 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 265 insertions(+), 22 deletions(-) create mode 100644 tests/test-virus-scan.sh diff --git a/Dockerfile b/Dockerfile index f4d6e8f..2b84b68 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,7 @@ RUN apt-get update && \ build-essential \ ca-certificates \ clamav \ + clamav-daemon \ clamav-freshclam \ curl \ fonts-liberation \ @@ -204,4 +205,3 @@ COPY ./entrypoint.sh /entrypoint.sh RUN chmod +x /entrypoint.sh ENTRYPOINT ["/entrypoint.sh"] - diff --git a/README.md b/README.md index 8f712f7..4788e7d 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ - php - rsync - shellcheck -- clamscan +- clamdscan - kubectl - aws-cli - azure-cli @@ -90,7 +90,7 @@ The `scripts` directory contains useful tools that can help test applications an - `all-scripts`: Runs all the included and additional custom scripts inside the `/custom-scripts` directory. - `php-syntax`: Checks the syntax of all PHP files inside the `workdir` -- `virus-scan`: Runs `clamscan` against the `workdir`. +- `virus-scan`: Starts a temporary ClamAV daemon and runs a parallel `clamdscan` against the `workdir`. - `slack-message`: Sends Slack notifications via webhook. ### Using slack-message diff --git a/scripts/virus-scan b/scripts/virus-scan index 0d38666..9286291 100644 --- a/scripts/virus-scan +++ b/scripts/virus-scan @@ -1,6 +1,9 @@ #!/bin/bash -# Use ClamAV to do a virus scan of the repo. Only display files where a virus is found +set -uo pipefail + +# Use a short-lived ClamAV daemon to scan the repository in parallel. Only +# display files where a virus is found. # Colors # shellcheck disable=SC1117 @@ -8,35 +11,112 @@ end="\033[0m" red="\033[0;31m" green="\033[0;32m" -function red { - echo -e "${red}${1}${end}" +red() { + echo -e "${red}${1}${end}" +} + +green() { + echo -e "${green}${1}${end}" } -function green { - echo -e "${green}${1}${end}" +scanner_error() { + red "Virus scanner internal error." } +for binary in clamd clamdscan; do + if ! command -v "${binary}" > /dev/null 2>&1; then + scanner_error + exit 0 + fi +done + +database_dir="${CLAMAV_DB_DIR:-/var/lib/clamav}" +if [[ ! -d "${database_dir}" ]]; then + scanner_error + exit 0 +fi + +runtime_dir="$(mktemp -d "${TMPDIR:-/tmp}/virus-scan.XXXXXX")" +if [[ -z "${runtime_dir}" || ! -d "${runtime_dir}" ]]; then + scanner_error + exit 0 +fi + +clamd_config="${runtime_dir}/clamd.conf" +clamd_log="${runtime_dir}/clamd.log" +clamd_pid="" + +cleanup() { + local exit_status=$? + trap - EXIT + + if [[ -n "${clamd_pid}" ]] && kill -0 "${clamd_pid}" 2>/dev/null; then + kill "${clamd_pid}" 2>/dev/null || true + wait "${clamd_pid}" 2>/dev/null || true + fi + + if [[ -d "${runtime_dir}" ]]; then + rm -r -- "${runtime_dir}" + fi + exit "${exit_status}" +} +trap cleanup EXIT + +cat > "${clamd_config}" < "${clamd_log}" 2>&1 & +clamd_pid=$! + +clamd_ready=false +for _ in {1..30}; do + if clamdscan --config-file="${clamd_config}" --ping=1 > /dev/null 2>&1; then + clamd_ready=true + break + fi + + if ! kill -0 "${clamd_pid}" 2>/dev/null; then + break + fi + + sleep 1 +done + +if [[ "${clamd_ready}" != "true" ]]; then + [[ -f "${clamd_log}" ]] && cat "${clamd_log}" >&2 + scanner_error + exit 0 +fi +clamdscan \ + --config-file="${clamd_config}" \ + --multiscan \ + --infected \ + "${PWD}" virus_status=$? echo "-------" echo "" -if [ $virus_status -eq 0 ] -then - green "Clean - no viruses found" - echo "" - exit 0 -elif [ $virus_status -eq 1 ] -then - red "**** INFECTED FILE FOUND!!! **** PLEASE SEE REPORT ABOVE ****" - echo "" - exit 1 +if [[ "${virus_status}" -eq 0 ]]; then + green "Clean - no viruses found" + echo "" + exit 0 +elif [[ "${virus_status}" -eq 1 ]]; then + red "**** INFECTED FILE FOUND!!! **** PLEASE SEE REPORT ABOVE ****" + echo "" + exit 1 else - red "Virus scanner internal error." - echo "" - exit 0 # don't block a deploy because the virus scan program is broken + scanner_error + echo "" + exit 0 # don't block a deploy because the virus scan program is broken fi diff --git a/tests/test-virus-scan.sh b/tests/test-virus-scan.sh new file mode 100644 index 0000000..87333e0 --- /dev/null +++ b/tests/test-virus-scan.sh @@ -0,0 +1,163 @@ +#!/usr/bin/env bash + +set -uo pipefail + +TEST_DIR="$(mktemp -d "/private/tmp/virus-scan-tests.XXXXXX")" +readonly TEST_DIR +REPOSITORY_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +readonly REPOSITORY_DIR +readonly SCRIPT_PATH="${REPOSITORY_DIR}/scripts/virus-scan" + +failures=0 + +cleanup() { + if [[ -d "${TEST_DIR}" ]]; then + rm -r -- "${TEST_DIR}" + fi +} +trap cleanup EXIT + +fail() { + printf 'not ok - %s\n' "$1" >&2 + failures=$((failures + 1)) +} + +assert_status() { + local expected="$1" + local actual="$2" + local description="$3" + + if [[ "${actual}" -ne "${expected}" ]]; then + fail "${description}: expected exit ${expected}, got ${actual}" + fi +} + +assert_file_contains() { + local file="$1" + local pattern="$2" + local description="$3" + + if [[ ! -f "${file}" ]] || ! grep -Eq -- "${pattern}" "${file}"; then + fail "${description}" + fi +} + +create_fake_binaries() { + local bin_dir="$1" + + mkdir -p "${bin_dir}" + + cat > "${bin_dir}/clamd" <<'EOF' +#!/usr/bin/env bash +set -u + +if [[ "${FAKE_START_MODE:-success}" == "failure" ]]; then + exit 2 +fi + +config_file="" +while [[ $# -gt 0 ]]; do + case "$1" in + -c|--config-file) + config_file="$2" + shift 2 + ;; + *) + shift + ;; + esac +done + +cp "${config_file}" "${TEST_STATE}/clamd.conf" +printf '%s\n' "$$" > "${TEST_STATE}/clamd.pid" +trap 'touch "${TEST_STATE}/clamd.stopped"; exit 0' TERM INT +while :; do + sleep 0.05 +done +EOF + + cat > "${bin_dir}/clamdscan" <<'EOF' +#!/usr/bin/env bash +set -u + +printf '%q ' "$@" >> "${TEST_STATE}/clamdscan.calls" +printf '\n' >> "${TEST_STATE}/clamdscan.calls" + +for argument in "$@"; do + if [[ "${argument}" == --ping* ]]; then + for _ in {1..50}; do + [[ -f "${TEST_STATE}/clamd.pid" ]] && exit 0 + sleep 0.01 + done + exit 2 + fi +done + +exit "${FAKE_SCAN_EXIT:-0}" +EOF + + chmod +x "${bin_dir}/clamd" "${bin_dir}/clamdscan" +} + +run_scan_case() { + local name="$1" + local scan_exit="$2" + local start_mode="$3" + local expected_exit="$4" + local state_dir="${TEST_DIR}/${name}" + local bin_dir="${state_dir}/bin" + local work_dir="${state_dir}/work" + local output_file="${state_dir}/output" + local actual_exit + + mkdir -p \ + "${state_dir}/database" \ + "${state_dir}/tmp" \ + "${work_dir}/.composer-cache" \ + "${work_dir}/node_modules_cache" + create_fake_binaries "${bin_dir}" + + set +e + ( + cd "${work_dir}" || exit 99 + PATH="${bin_dir}:${PATH}" \ + TEST_STATE="${state_dir}" \ + TMPDIR="${state_dir}/tmp" \ + CLAMAV_DB_DIR="${state_dir}/database" \ + FAKE_SCAN_EXIT="${scan_exit}" \ + FAKE_START_MODE="${start_mode}" \ + bash "${SCRIPT_PATH}" + ) > "${output_file}" 2>&1 + actual_exit=$? + set -e + + assert_status "${expected_exit}" "${actual_exit}" "${name}" +} + +run_scan_case clean 0 success 0 +assert_file_contains "${TEST_DIR}/clean/clamdscan.calls" '--multiscan' 'clean scan requests multiscan' +assert_file_contains "${TEST_DIR}/clean/clamdscan.calls" '--infected' 'clean scan only reports infected files' +assert_file_contains "${TEST_DIR}/clean/clamd.conf" 'ExcludePath .*\\.composer-cache' 'composer cache exclusion is configured for clamd' +assert_file_contains "${TEST_DIR}/clean/clamd.conf" 'ExcludePath .*node_modules_cache' 'node modules cache exclusion is configured for clamd' +assert_file_contains "${TEST_DIR}/clean/output" 'Clean - no viruses found' 'clean scan reports success' +[[ -f "${TEST_DIR}/clean/clamd.stopped" ]] || fail 'clean scan stops the temporary daemon' + +run_scan_case infected 1 success 1 +assert_file_contains "${TEST_DIR}/infected/output" 'INFECTED FILE FOUND' 'infected scan reports malware' +[[ -f "${TEST_DIR}/infected/clamd.stopped" ]] || fail 'infected scan stops the temporary daemon' + +run_scan_case scan_error 2 success 0 +assert_file_contains "${TEST_DIR}/scan_error/output" 'Virus scanner internal error' 'scan errors remain fail-open' +[[ -f "${TEST_DIR}/scan_error/clamd.stopped" ]] || fail 'scan errors stop the temporary daemon' + +run_scan_case startup_error 0 failure 0 +assert_file_contains "${TEST_DIR}/startup_error/output" 'Virus scanner internal error' 'daemon startup errors remain fail-open' + +assert_file_contains "${REPOSITORY_DIR}/Dockerfile" 'clamav-daemon' 'container installs the ClamAV daemon package' + +if [[ "${failures}" -gt 0 ]]; then + printf '%s test assertion(s) failed\n' "${failures}" >&2 + exit 1 +fi + +printf 'ok - virus-scan behavior\n'