Improve virus scan performance with clamdscan

Dockerfile

@@ -1,6 +1,6 @@
 # base image should be a PHP Debian container, such as php:7.4-buster
 ARG PHP_IMG
 FROM $PHP_IMG
 
 RUN apt-get update && \
     apt-get install -y \
@@ -8,6 +8,7 @@
       build-essential \
       ca-certificates \
       clamav \
+      clamav-daemon \
       clamav-freshclam \
       curl \
       fonts-liberation \
@@ -204,4 +205,3 @@
 RUN chmod +x /entrypoint.sh
 
 ENTRYPOINT ["/entrypoint.sh"]
-

README.md

@@ -16,7 +16,7 @@
 - php
 - rsync
 - shellcheck
-- clamscan
+- clamdscan
 - kubectl
 - aws-cli
 - azure-cli
@@ -90,7 +90,7 @@ The `scripts` directory contains useful tools that can help test applications an
 
 - `all-scripts`: Runs all the included and additional custom scripts inside the `/custom-scripts` directory.
 - `php-syntax`: Checks the syntax of all PHP files inside the `workdir`
-- `virus-scan`: Runs `clamscan` against the `workdir`.
+- `virus-scan`: Starts a temporary ClamAV daemon and runs a parallel `clamdscan` against the `workdir`.
 - `slack-message`: Sends Slack notifications via webhook.
 
 ### Using slack-message

scripts/virus-scan

@@ -1,42 +1,122 @@
 #!/bin/bash
 
-# Use ClamAV to do a virus scan of the repo. Only display files where a virus is found
+set -uo pipefail
+
+# Use a short-lived ClamAV daemon to scan the repository in parallel. Only
+# display files where a virus is found.
 
 # Colors
 # shellcheck disable=SC1117
 end="\033[0m"
 red="\033[0;31m"
 green="\033[0;32m"
 
-function red {
-  echo -e "${red}${1}${end}"
+red() {
+	echo -e "${red}${1}${end}"
+}
+
+green() {
+	echo -e "${green}${1}${end}"
 }
 
-function green {
-  echo -e "${green}${1}${end}"
+scanner_error() {
+	red "Virus scanner internal error."
 }
 
+for binary in clamd clamdscan; do
+	if ! command -v "${binary}" > /dev/null 2>&1; then
+		scanner_error
+		exit 0
+	fi
+done
+
+database_dir="${CLAMAV_DB_DIR:-/var/lib/clamav}"
+if [[ ! -d "${database_dir}" ]]; then
+	scanner_error
+	exit 0
+fi
+
+runtime_dir="$(mktemp -d "${TMPDIR:-/tmp}/virus-scan.XXXXXX")"
+if [[ -z "${runtime_dir}" || ! -d "${runtime_dir}" ]]; then
+	scanner_error
+	exit 0
+fi
+
+clamd_config="${runtime_dir}/clamd.conf"
+clamd_log="${runtime_dir}/clamd.log"
+clamd_pid=""
+
+cleanup() {
+	local exit_status=$?
+	trap - EXIT
+
+	if [[ -n "${clamd_pid}" ]] && kill -0 "${clamd_pid}" 2>/dev/null; then
+		kill "${clamd_pid}" 2>/dev/null || true
+		wait "${clamd_pid}" 2>/dev/null || true
+	fi
+
+	if [[ -d "${runtime_dir}" ]]; then
+		rm -r -- "${runtime_dir}"
+	fi
+	exit "${exit_status}"
+}
+trap cleanup EXIT
+
+cat > "${clamd_config}" <<EOF
+LocalSocket ${runtime_dir}/clamd.sock
+FixStaleSocket yes
+DatabaseDirectory ${database_dir}
+Foreground yes
+PidFile ${runtime_dir}/clamd.pid
+ExcludePath (^|/)\.composer-cache(/|$)
+ExcludePath (^|/)node_modules_cache(/|$)
+EOF
+
 green "#### Starting Virus Scan ####"
 
-clamscan --exclude-dir ./.composer-cache --exclude-dir ./node_modules_cache -riz .
+clamd -c "${clamd_config}" > "${clamd_log}" 2>&1 &
+clamd_pid=$!
+
+clamd_ready=false
+for _ in {1..30}; do
+	if clamdscan --config-file="${clamd_config}" --ping=1 > /dev/null 2>&1; then
+		clamd_ready=true
+		break
+	fi
+
+	if ! kill -0 "${clamd_pid}" 2>/dev/null; then
+		break
+	fi
+
+	sleep 1
+done
+
+if [[ "${clamd_ready}" != "true" ]]; then
+	[[ -f "${clamd_log}" ]] && cat "${clamd_log}" >&2
+	scanner_error
+	exit 0
+fi
 
+clamdscan \
+	--config-file="${clamd_config}" \
+	--multiscan \
+	--infected \
+	"${PWD}"
 virus_status=$?
 
 echo "-------"
 echo ""
 
-if [ $virus_status -eq 0 ]
-then
-  green "Clean - no viruses found"
-  echo ""
-  exit 0
-elif [ $virus_status -eq 1 ]
-then
-  red "**** INFECTED FILE FOUND!!! **** PLEASE SEE REPORT ABOVE ****"
-  echo ""
-  exit 1
+if [[ "${virus_status}" -eq 0 ]]; then
+	green "Clean - no viruses found"
+	echo ""
+	exit 0
+elif [[ "${virus_status}" -eq 1 ]]; then
+	red "**** INFECTED FILE FOUND!!! **** PLEASE SEE REPORT ABOVE ****"
+	echo ""
+	exit 1
 else
-  red "Virus scanner internal error."
-  echo ""
-  exit 0 # don't block a deploy because the virus scan program is broken
+	scanner_error
+	echo ""
+	exit 0 # don't block a deploy because the virus scan program is broken
 fi

tests/test-virus-scan.sh

@@ -0,0 +1,163 @@
+#!/usr/bin/env bash
+
+set -uo pipefail
+
+TEST_DIR="$(mktemp -d "/private/tmp/virus-scan-tests.XXXXXX")"
+readonly TEST_DIR
+REPOSITORY_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+readonly REPOSITORY_DIR
+readonly SCRIPT_PATH="${REPOSITORY_DIR}/scripts/virus-scan"
+
+failures=0
+
+cleanup() {
+	if [[ -d "${TEST_DIR}" ]]; then
+		rm -r -- "${TEST_DIR}"
+	fi
+}
+trap cleanup EXIT
+
+fail() {
+	printf 'not ok - %s\n' "$1" >&2
+	failures=$((failures + 1))
+}
+
+assert_status() {
+	local expected="$1"
+	local actual="$2"
+	local description="$3"
+
+	if [[ "${actual}" -ne "${expected}" ]]; then
+		fail "${description}: expected exit ${expected}, got ${actual}"
+	fi
+}
+
+assert_file_contains() {
+	local file="$1"
+	local pattern="$2"
+	local description="$3"
+
+	if [[ ! -f "${file}" ]] || ! grep -Eq -- "${pattern}" "${file}"; then
+		fail "${description}"
+	fi
+}
+
+create_fake_binaries() {
+	local bin_dir="$1"
+
+	mkdir -p "${bin_dir}"
+
+	cat > "${bin_dir}/clamd" <<'EOF'
+#!/usr/bin/env bash
+set -u
+
+if [[ "${FAKE_START_MODE:-success}" == "failure" ]]; then
+	exit 2
+fi
+
+config_file=""
+while [[ $# -gt 0 ]]; do
+	case "$1" in
+		-c|--config-file)
+			config_file="$2"
+			shift 2
+			;;
+		*)
+			shift
+			;;
+	esac
+done
+
+cp "${config_file}" "${TEST_STATE}/clamd.conf"
+printf '%s\n' "$$" > "${TEST_STATE}/clamd.pid"
+trap 'touch "${TEST_STATE}/clamd.stopped"; exit 0' TERM INT
+while :; do
+	sleep 0.05
+done
+EOF
+
+	cat > "${bin_dir}/clamdscan" <<'EOF'
+#!/usr/bin/env bash
+set -u
+
+printf '%q ' "$@" >> "${TEST_STATE}/clamdscan.calls"
+printf '\n' >> "${TEST_STATE}/clamdscan.calls"
+
+for argument in "$@"; do
+	if [[ "${argument}" == --ping* ]]; then
+		for _ in {1..50}; do
+			[[ -f "${TEST_STATE}/clamd.pid" ]] && exit 0
+			sleep 0.01
+		done
+		exit 2
+	fi
+done
+
+exit "${FAKE_SCAN_EXIT:-0}"
+EOF
+
+	chmod +x "${bin_dir}/clamd" "${bin_dir}/clamdscan"
+}
+
+run_scan_case() {
+	local name="$1"
+	local scan_exit="$2"
+	local start_mode="$3"
+	local expected_exit="$4"
+	local state_dir="${TEST_DIR}/${name}"
+	local bin_dir="${state_dir}/bin"
+	local work_dir="${state_dir}/work"
+	local output_file="${state_dir}/output"
+	local actual_exit
+
+	mkdir -p \
+		"${state_dir}/database" \
+		"${state_dir}/tmp" \
+		"${work_dir}/.composer-cache" \
+		"${work_dir}/node_modules_cache"
+	create_fake_binaries "${bin_dir}"
+
+	set +e
+	(
+		cd "${work_dir}" || exit 99
+		PATH="${bin_dir}:${PATH}" \
+			TEST_STATE="${state_dir}" \
+			TMPDIR="${state_dir}/tmp" \
+			CLAMAV_DB_DIR="${state_dir}/database" \
+			FAKE_SCAN_EXIT="${scan_exit}" \
+			FAKE_START_MODE="${start_mode}" \
+			bash "${SCRIPT_PATH}"
+	) > "${output_file}" 2>&1
+	actual_exit=$?
+	set -e
+
+	assert_status "${expected_exit}" "${actual_exit}" "${name}"
+}
+
+run_scan_case clean 0 success 0
+assert_file_contains "${TEST_DIR}/clean/clamdscan.calls" '--multiscan' 'clean scan requests multiscan'
+assert_file_contains "${TEST_DIR}/clean/clamdscan.calls" '--infected' 'clean scan only reports infected files'
+assert_file_contains "${TEST_DIR}/clean/clamd.conf" 'ExcludePath .*\\.composer-cache' 'composer cache exclusion is configured for clamd'
+assert_file_contains "${TEST_DIR}/clean/clamd.conf" 'ExcludePath .*node_modules_cache' 'node modules cache exclusion is configured for clamd'
+assert_file_contains "${TEST_DIR}/clean/output" 'Clean - no viruses found' 'clean scan reports success'
+[[ -f "${TEST_DIR}/clean/clamd.stopped" ]] || fail 'clean scan stops the temporary daemon'
+
+run_scan_case infected 1 success 1
+assert_file_contains "${TEST_DIR}/infected/output" 'INFECTED FILE FOUND' 'infected scan reports malware'
+[[ -f "${TEST_DIR}/infected/clamd.stopped" ]] || fail 'infected scan stops the temporary daemon'
+
+run_scan_case scan_error 2 success 0
+assert_file_contains "${TEST_DIR}/scan_error/output" 'Virus scanner internal error' 'scan errors remain fail-open'
+[[ -f "${TEST_DIR}/scan_error/clamd.stopped" ]] || fail 'scan errors stop the temporary daemon'
+
+run_scan_case startup_error 0 failure 0
+assert_file_contains "${TEST_DIR}/startup_error/output" 'Virus scanner internal error' 'daemon startup errors remain fail-open'
+
+assert_file_contains "${REPOSITORY_DIR}/Dockerfile" 'clamav-daemon' 'container installs the ClamAV daemon package'
+
+if [[ "${failures}" -gt 0 ]]; then
+	printf '%s test assertion(s) failed\n' "${failures}" >&2
+	exit 1
+fi
+
+printf 'ok - virus-scan behavior\n'