diff --git a/.github/workflows/marketplace_classpath.yml b/.github/workflows/marketplace_classpath.yml new file mode 100644 index 00000000000..e11d8acca10 --- /dev/null +++ b/.github/workflows/marketplace_classpath.yml @@ -0,0 +1,179 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- + +# Check that every marketplace plugin still works when a user installs it. +# +# Two failure modes, neither of which any other job here can see: +# +# * a plugin zip that ships its dependencies but no plugin jar — an assembly +# that matches no artifact produces exactly this, silently; +# * a plugin whose own classes reference a package that is on no classpath once +# installed, because a dependency was mis-scoped or dropped by a wildcard +# and so landed neither in the plugin's lib/ nor in lib/core. +# +# Both only fail in a user's install. Issue #8036 (hop-tech-parquet) was the second +# kind: it worked only because another plugin happened to put the same Hadoop jars +# in lib/core, and broke when that plugin left the default client. +# +# This job builds nothing. It downloads the published client and plugin zips and +# checks those, which is why it costs about three minutes instead of a full +# assembly build. The consequence is that it runs after a merge, not on a PR: a +# pull request does not produce plugin zips (pr_build_code.yml passes +# -Dassemblies=false -Dmodule.zips=false), so there is nothing for it to look at +# until the nightly deploy publishes new snapshots. +# +# The client zip is the baseline, deliberately, and the container image is not. The +# image is built from a client that has already had install-wave1-plugins.sh run +# over it, so every plugin is present and each one can borrow the others' jars — +# which is precisely how this class of bug hides. +name: Marketplace plugin classpath + +on: + schedule: + # Snapshots are published by the "Hop Orchestration Platform" job, which deploys + # on merge rather than on a schedule — three to five times on a normal weekday. + # (The daily IT job runs `clean install` and publishes nothing, so its schedule + # is not what this job should track.) The fetch always resolves the newest + # snapshot from maven-metadata.xml, so a run is never stale relative to main; it + # is only ever as old as the last merge. Two slots keep the worst-case delay at + # about twelve hours: midday catches the European morning's merges, late evening + # catches the rest of the day. + - cron: '0 12,22 * * *' + workflow_dispatch: + inputs: + version: + description: 'Version to check (default: this branch''s project version, e.g. 2.20.0-SNAPSHOT). A release like 2.19.0 checks what users actually downloaded.' + required: false + default: '' + type: string + +concurrency: + group: marketplace-classpath-${{ github.ref }} + cancel-in-progress: true + +jobs: + check: + name: Check marketplace plugin zips + runs-on: ubuntu-latest + env: + # The nightly integration-test run's test report. "lastCompletedBuild" keeps the + # links current without plumbing build numbers between two CI systems. The API + # hangs off this root, while the human-readable per-suite pages sit one level + # down under "(root)" — that segment is required for the pages and must not be + # used for the API. + IT_REPORT_ROOT: 'https://ci-builds.apache.org/job/Hop/job/Hop-integration-tests/lastCompletedBuild/testReport' + + steps: + - uses: actions/checkout@v4 + + # jdeps is a JDK tool: a JRE does not have it. + - name: Set up JDK 21 + uses: actions/setup-java@v4 + with: + java-version: '21' + distribution: 'temurin' + + - name: Resolve version + id: version + run: | + set -euo pipefail + version='${{ github.event.inputs.version }}' + if [ -z "$version" ]; then + # -N keeps this to the root pom rather than loading the whole reactor. + version=$(mvn -N -q -DforceStdout help:evaluate -Dexpression=project.version) + fi + [ -n "$version" ] || { echo "could not determine the project version"; exit 1; } + echo "version=$version" >> "$GITHUB_OUTPUT" + echo "checking $version" + + # Releases come from the ASF dist archive, snapshots from the snapshot + # repository. The plugin list is read from the client's own + # full-client-env.yaml, so the list and the baseline are always the same + # build. + - name: Download client and marketplace plugin zips + run: | + ./tools/fetch-apache-marketplace.sh \ + --version '${{ steps.version.outputs.version }}' \ + --dest "$RUNNER_TEMP/marketplace" + + # A renamed integration-test project silently turns a report link into a 404, + # which reads as "covered" when it is not. Warn rather than fail: Jenkins being + # unreachable says nothing about the plugin zips this job is here to check. + # Derived from optional-plugins.yaml and the integration-tests/ directories, so + # a new plugin whose IT project follows the naming convention is picked up + # without editing anything. + - name: Pair plugins with their integration-test projects + run: | + ./tools/marketplace-it-suites.sh | tee "$RUNNER_TEMP/it-suites.txt" + + - name: Verify the integration-test suite names still exist + continue-on-error: true + run: | + set -uo pipefail + suites=$(curl -fsSL --max-time 60 \ + "$IT_REPORT_ROOT/api/json?tree=suites\[name\]" | tr ',' '\n' | + sed -n 's/.*"name":"\([^"]*\)".*/\1/p' | sort -u) + if [ -z "$suites" ]; then + echo "::warning::could not read the Jenkins test report; skipping the link check" + exit 0 + fi + missing=0 + while read -r plugin suite; do + case "$plugin" in ''|\#*) continue ;; esac + if ! printf '%s\n' "$suites" | grep -qxF "$suite"; then + echo "::warning::$plugin maps to integration-test suite '$suite', which is not in the latest run" + missing=$((missing + 1)) + fi + done < "$RUNNER_TEMP/it-suites.txt" + echo "$missing stale mapping(s)" + + - name: Check + run: | + ./tools/check-plugin-classpath.sh \ + --client "$RUNNER_TEMP/marketplace/client.zip" \ + --plugins "$RUNNER_TEMP/marketplace/plugins.txt" \ + --label '${{ steps.version.outputs.version }}' \ + --junit reports/marketplace-classpath.xml \ + --report reports/summary.md \ + --it-suites "$RUNNER_TEMP/it-suites.txt" \ + --it-base "$IT_REPORT_ROOT/(root)" + + # Rendered on the run's summary page, so the per-plugin status is the first + # thing a reader sees rather than something buried in the log. + - name: Publish summary + if: always() + run: cat reports/summary.md >> "$GITHUB_STEP_SUMMARY" || true + + # Proves the check can still fail. Without this a packaging change that makes + # every plugin unanalysable would show up as a green run. + - name: Self test + run: | + ./tools/check-plugin-classpath.sh \ + --client "$RUNNER_TEMP/marketplace/client.zip" \ + --plugins "$RUNNER_TEMP/marketplace/plugins.txt" \ + --label '${{ steps.version.outputs.version }}' \ + --self-test + + - name: Upload report + if: always() + uses: actions/upload-artifact@v4 + with: + name: marketplace-classpath-report + path: reports/ + if-no-files-found: ignore diff --git a/tools/check-plugin-classpath.sh b/tools/check-plugin-classpath.sh new file mode 100755 index 00000000000..308ec06b425 --- /dev/null +++ b/tools/check-plugin-classpath.sh @@ -0,0 +1,491 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Fail when a marketplace plugin zip cannot work once it is installed. +# +# Two failure modes, both silent in every other part of the build: +# +# 1. The zip ships no plugin jar at all. Its dependencies are there, its +# version.xml is there, and installing it adds nothing that Hop can load. +# An assembly that matches no artifact produces exactly this and +# maven-assembly-plugin does not warn. +# +# 2. The plugin's own classes reference a package that is on no classpath once +# the plugin is installed, because a dependency was mis-scoped or dropped by +# a wildcard and so landed neither in the plugin's lib/ nor in the +# client's lib/core. The plugin then dies with NoClassDefFoundError on first +# use, in the user's install, not in CI. +# +# Each plugin is checked against a baseline client that represents what its users +# actually have. That baseline matters: with every plugin present, one plugin's +# lib/core contribution covers another's gap, which is how these bugs survive. +# For a marketplace plugin the baseline is the plain client. For a plugin that +# ships inside a client, install its companions with --install first. +# +# jdeps is run over the plugin's own jars only, never over the third-party jars it +# bundles: those are full of optional dependencies that are on no classpath by +# design, and several are modular, which makes jdeps abort with a module resolution +# error rather than report. Restricting the input to first-party classes keeps the +# check silent when healthy and loud when a plugin is genuinely broken. +# +# Usage: +# tools/check-plugin-classpath.sh --client --plugins [options] +# +# --client Baseline client to install onto. Required. +# --plugins Plugin list, one per line: " ". +# A '|' separator is accepted too. '#' comments and blank +# lines are ignored. Required. +# --install Install this zip into the baseline before checking +# anything. Repeatable. Use for the plugins that ship +# inside the client the marketplace plugins install onto. +# --allowlist Default: tools/plugin-classpath-allowlist.txt next to this +# script. +# --label Name for the run in output and JUnit. Default "plugins". +# --junit Also write a JUnit report, one testcase per plugin. +# --report Also write a Markdown status table, one row per plugin. +# --it-suites " " pairs naming the integration-test +# project that covers a plugin needing a live service. Those +# plugins report as needs-service rather than as a gap. +# --it-base Base URL for the test report, to turn the suite name into a +# link. Without it the suite is named but not linked. +# --plugin Check only this artifactId. +# --allow-missing Do not fail on plugins whose zip was not built. +# --self-test Prove the check can still fail, then exit. +# +set -euo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +CLIENT="" +PLUGIN_LIST="" +ALLOWLIST="${HERE}/plugin-classpath-allowlist.txt" +LABEL="plugins" +JUNIT="" +REPORT="" +IT_SUITES="" +IT_BASE="" +ONLY_PLUGIN="" +ALLOW_MISSING=false +SELF_TEST=false +INSTALL_ZIPS=() + +die() { + echo "ERROR: $*" >&2 + exit 2 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --client) shift; CLIENT="${1:-}"; [[ -n "${CLIENT}" ]] || die "--client needs a value" ;; + --plugins) shift; PLUGIN_LIST="${1:-}"; [[ -n "${PLUGIN_LIST}" ]] || die "--plugins needs a value" ;; + --install) shift; [[ -n "${1:-}" ]] || die "--install needs a value"; INSTALL_ZIPS+=("$1") ;; + --allowlist) shift; ALLOWLIST="${1:-}"; [[ -n "${ALLOWLIST}" ]] || die "--allowlist needs a value" ;; + --label) shift; LABEL="${1:-}"; [[ -n "${LABEL}" ]] || die "--label needs a value" ;; + --junit) shift; JUNIT="${1:-}"; [[ -n "${JUNIT}" ]] || die "--junit needs a value" ;; + --report) shift; REPORT="${1:-}"; [[ -n "${REPORT}" ]] || die "--report needs a value" ;; + --it-suites) shift; IT_SUITES="${1:-}"; [[ -n "${IT_SUITES}" ]] || die "--it-suites needs a value" ;; + --it-base) shift; IT_BASE="${1:-}"; [[ -n "${IT_BASE}" ]] || die "--it-base needs a value" ;; + --plugin) shift; ONLY_PLUGIN="${1:-}"; [[ -n "${ONLY_PLUGIN}" ]] || die "--plugin needs an artifactId" ;; + --allow-missing) ALLOW_MISSING=true ;; + --self-test) SELF_TEST=true ;; + -h | --help) sed -n '17,65p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) die "unknown argument: $1" ;; + esac + shift +done + +[[ -n "${CLIENT}" ]] || die "--client is required" +[[ -n "${PLUGIN_LIST}" ]] || die "--plugins is required" +[[ -f "${PLUGIN_LIST}" ]] || die "plugin list not found: ${PLUGIN_LIST}" +command -v jdeps >/dev/null 2>&1 || die "jdeps not on PATH — this needs a JDK, not a JRE" +command -v unzip >/dev/null 2>&1 || die "unzip not on PATH" + +WORK="$(mktemp -d "${TMPDIR:-/tmp}/plugin-classpath.XXXXXX")" +trap 'rm -rf "${WORK}"' EXIT + +# ---------------------------------------------------------------- baseline + +BASE="${WORK}/baseline" +if [[ -d "${CLIENT}" ]]; then + mkdir -p "${BASE}" + # Copied rather than used in place: --install writes into the baseline, and a + # check must never mutate the caller's client. + cp -R "${CLIENT}/." "${BASE}/" +elif [[ -f "${CLIENT}" ]]; then + echo "==> Unpacking $(basename "${CLIENT}")" + unzip -q -o "${CLIENT}" -d "${BASE}" +else + die "--client is neither a file nor a directory: ${CLIENT}" +fi + +# Client zips unpack to a single top directory; extracted container roots do not. +if [[ ! -d "${BASE}/lib" ]]; then + inner="$(find "${BASE}" -maxdepth 2 -type d -name lib -print -quit 2>/dev/null || true)" + [[ -n "${inner}" ]] || die "no lib/ directory found under ${CLIENT}" + BASE="$(dirname "${inner}")" +fi + +for zip in ${INSTALL_ZIPS[@]+"${INSTALL_ZIPS[@]}"}; do + [[ -f "${zip}" ]] || die "--install zip not found: ${zip}" + echo " installing into baseline: $(basename "${zip}")" + unzip -q -o "${zip}" -d "${BASE}" +done + +BASE_CP="" +while IFS= read -r jar; do + BASE_CP="${BASE_CP}${jar}:" +done < <(find "${BASE}/lib" "${BASE}/plugins" -name '*.jar' 2>/dev/null | sort) +[[ -n "${BASE_CP}" ]] || die "no jars found in the baseline client at ${BASE}" + +# ---------------------------------------------------------------- helpers + +# Jars that are the plugin's own code: directly under plugins///, +# as opposed to its bundled dependencies, which the assembly puts in that +# directory's lib/. +own_jars() { find "$1/plugins" -name '*.jar' 2>/dev/null | grep -v '/lib/' | sort || true; } + +# What a zip is, before deciding whether it is broken: +# plugin — ships first-party jars, check them +# nojar — ships dependencies under plugins/**/lib but no plugin jar. Broken: +# installing it adds libraries and nothing that Hop can load. +# library — ships no plugins/ tree at all, only lib/ jars. A shared library +# published as a zip, with nothing of its own to check. +# empty — no jars anywhere +classify() { + local dir="$1" + if [[ -n "$(own_jars "${dir}")" ]]; then echo plugin; return; fi + if [[ -n "$(find "${dir}/plugins" -name '*.jar' 2>/dev/null | head -1)" ]]; then echo nojar; return; fi + if [[ -n "$(find "${dir}" -name '*.jar' 2>/dev/null | head -1)" ]]; then echo library; return; fi + echo empty +} + +# $2, when set, is a jar basename to withhold from the classpath (--self-test). +plugin_cp() { + local dir="$1" withhold="${2:-}" cp="" jar + while IFS= read -r jar; do + [[ -n "${withhold}" && "$(basename "${jar}")" == "${withhold}" ]] && continue + cp="${cp}${jar}:" + done < <(find "${dir}" -name '*.jar' 2>/dev/null | sort) + if [[ -z "${withhold}" ]]; then + cp="${cp}${BASE_CP}" + else + while IFS= read -r jar; do + [[ "$(basename "${jar}")" == "${withhold}" ]] && continue + cp="${cp}${jar}:" + done < <(find "${BASE}/lib" "${BASE}/plugins" -name '*.jar' 2>/dev/null | sort) + fi + printf '%s\n' "${cp}" +} + +run_jdeps() { + local dir="$1" withhold="${2:-}" cp jars + cp="$(plugin_cp "${dir}" "${withhold}")" + jars="$(own_jars "${dir}")" + [[ -n "${jars}" ]] || return 3 + # shellcheck disable=SC2086 + jdeps -q --multi-release 21 -cp "${cp}" ${jars} 2>&1 || true +} + +# "sourcePackage missingPackage" per unresolved reference. jdeps also emits a bare +# " -> not found" summary; only the indented package lines name the source +# package, which is what makes a finding actionable. +parse_missing() { awk '/ not found$/ && /^[[:space:]]/ { print $1, $3 }' | sort -u; } + +allowed() { + local id="$1" pkg="$2" + [[ -f "${ALLOWLIST}" ]] || return 1 + awk -v id="${id}" -v pkg="${pkg}" ' + /^[[:space:]]*#/ { next } + NF < 2 { next } + $1 == id && (pkg == $2 || index(pkg, $2 ".") == 1) { found = 1; exit } + END { exit found ? 0 : 1 } + ' "${ALLOWLIST}" +} + +# The integration-test project covering this plugin, if it needs a live service. +it_suite() { + [[ -n "${IT_SUITES}" && -f "${IT_SUITES}" ]] || return 1 + awk -v id="$1" '/^[[:space:]]*#/ { next } NF < 2 { next } $1 == id { print $2; found = 1; exit } + END { exit found ? 0 : 1 }' "${IT_SUITES}" +} + +xml_escape() { sed -e 's/&/\&/g' -e 's//\>/g'; } + +# ---------------------------------------------------------------- plugin list + +IDS=() +ZIPS=() +while IFS= read -r line || [[ -n "${line}" ]]; do + line="${line%%#*}" + line="$(printf '%s' "${line}" | tr '|' ' ')" + # shellcheck disable=SC2086 + set -- ${line} + [[ $# -ge 2 ]] || continue + [[ -z "${ONLY_PLUGIN}" || "$1" == "${ONLY_PLUGIN}" ]] || continue + IDS+=("$1") + ZIPS+=("$2") +done <"${PLUGIN_LIST}" + +[[ ${#IDS[@]} -gt 0 ]] || die "no plugins to check from ${PLUGIN_LIST}" + +# ---------------------------------------------------------------- self test + +# Picks a plugin whose own classes resolve against a jar the plugin itself ships, +# withholds that jar, and expects the missing package to be reported. Derived from +# the build rather than hard-coded, so it keeps working as plugins come and go. +if [[ "${SELF_TEST}" == true ]]; then + echo "==> Self test (${LABEL})" + i=0 + while [[ ${i} -lt ${#IDS[@]} ]]; do + id="${IDS[$i]}"; zip="${ZIPS[$i]}"; i=$((i + 1)) + [[ -f "${zip}" ]] || continue + dir="${WORK}/st"; rm -rf "${dir}"; mkdir -p "${dir}" + unzip -q -o "${zip}" -d "${dir}" + [[ "$(classify "${dir}")" == plugin ]] || continue + out="$(run_jdeps "${dir}")" || continue + echo "${out}" | grep -q '^Exception in thread' && continue + [[ -n "$(echo "${out}" | parse_missing)" ]] && continue + + # Dependency jars only. The plugin's own jars are jdeps *input*, so withholding + # one from the classpath changes nothing and would fail the self test spuriously. + own_jars "${dir}" | xargs -n1 basename 2>/dev/null | sort -u >"${WORK}/own.txt" + find "${dir}" -name '*.jar' -exec basename {} \; | sort -u | + grep -Fxv -f "${WORK}/own.txt" >"${WORK}/shipped.txt" || true + # jdeps names JDK modules in the same column ("java.base"), so keep real jars only. + canary="$(echo "${out}" | + awk 'NF == 4 && $2 == "->" && $4 ~ /\.jar$/ { print $4 }' | + sort -u | grep -Fx -f "${WORK}/shipped.txt" | head -1 || true)" + [[ -n "${canary}" ]] || continue + + echo " canary plugin: ${id}" + echo " withholding: ${canary}" + findings="$(run_jdeps "${dir}" "${canary}" | parse_missing || true)" + if [[ -z "${findings}" ]]; then + echo + echo "SELF TEST FAILED: withholding ${canary} produced no finding." + exit 1 + fi + echo " reported:" + echo "${findings}" | sed 's/^/ /' + echo + echo "Self test passed." + exit 0 + done + die "no plugin with a usable canary jar; build the plugin zips first" +fi + +# ---------------------------------------------------------------- main + +echo "==> Checking ${#IDS[@]} ${LABEL} plugin(s)" +echo " baseline: ${CLIENT}" +echo + +ok=0; allowedcount=0; failed=0; missing=0; errored=0; libs=0 +CASES="${WORK}/cases" +: >"${CASES}" + +# One line per plugin. Detail can be multi-line, which would otherwise turn every +# continuation line into its own testcase, so newlines are folded onto a record +# separator here and unfolded when the JUnit report is written. +# Fields are separated by a unit separator, not a tab: tab is an IFS whitespace +# character, so `read` collapses runs of it and an empty detail field would silently +# shift every column after it. Embedded newlines fold onto a record separator so one +# plugin stays one line. +record() { + printf '%s\037%s\037%s\037%s\n' \ + "$1" "$2" "$(printf '%s' "$3" | tr '\n' '\036')" "${4:-0}" >>"${CASES}" +} + +# Sample pipelines and workflows the zip ships. Plugins put these under +# config/projects/samples via src/main/samples, and the client pre-registers that +# project, so they are what a per-plugin smoke test would run. +sample_count() { + find "$1" \( -name '*.hpl' -o -name '*.hwf' \) 2>/dev/null | wc -l | tr -d ' ' +} + +i=0 +while [[ ${i} -lt ${#IDS[@]} ]]; do + id="${IDS[$i]}"; zip="${ZIPS[$i]}"; i=$((i + 1)) + + if [[ ! -f "${zip}" ]]; then + echo " MISSING ${id} — ${zip}" + missing=$((missing + 1)) + record "${id}" skipped "zip not built: ${zip}" + continue + fi + + dir="${WORK}/stage"; rm -rf "${dir}"; mkdir -p "${dir}" + unzip -q -o "${zip}" -d "${dir}" + samples="$(sample_count "${dir}")" + + case "$(classify "${dir}")" in + library) + echo " LIBRARY ${id} (no plugin tree, nothing to check)" + libs=$((libs + 1)) + record "${id}" skipped "shared library zip" "${samples}" + continue + ;; + nojar) + echo " FAIL ${id}" + echo " the zip ships dependencies but no plugin jar" + echo " installing it adds nothing Hop can load" + echo " check the assembly matches the module's groupId:artifactId" + failed=$((failed + 1)) + record "${id}" failure "zip ships dependencies under plugins/**/lib but no plugin jar" "${samples}" + continue + ;; + empty) + echo " FAIL ${id} — the zip contains no jars at all" + failed=$((failed + 1)) + record "${id}" failure "zip contains no jars" "${samples}" + continue + ;; + esac + + out="$(run_jdeps "${dir}")" + + # jdeps refusing to run must never read as a pass: that is how a plugin quietly + # stops being checked. + if echo "${out}" | grep -q '^Exception in thread'; then + detail="$(echo "${out}" | grep '^Exception in thread' | head -1)" + echo " ERROR ${id} — jdeps could not analyse the plugin:" + echo " ${detail}" + errored=$((errored + 1)) + record "${id}" error "${detail}" "${samples}" + continue + fi + + hits="$(echo "${out}" | parse_missing)" + if [[ -z "${hits}" ]]; then + echo " OK ${id}" + ok=$((ok + 1)) + record "${id}" pass "" "${samples}" + continue + fi + + plugin_failed=false + shown="" + detail="" + while read -r src pkg; do + [[ -n "${src}" ]] || continue + if allowed "${id}" "${pkg}"; then + shown="${shown} allowed: ${pkg}"$'\n' + else + shown="${shown} ${src} -> ${pkg}"$'\n' + detail="${detail}${src} -> ${pkg}"$'\n' + plugin_failed=true + fi + done <<<"${hits}" + + if [[ "${plugin_failed}" == true ]]; then + echo " FAIL ${id}" + printf '%s' "${shown}" + failed=$((failed + 1)) + record "${id}" failure "${detail}" "${samples}" + else + echo " ALLOWED ${id}" + printf '%s' "${shown}" + allowedcount=$((allowedcount + 1)) + record "${id}" pass "" "${samples}" + fi +done + +echo +echo "Summary (${LABEL}): ok=${ok} allowed=${allowedcount} library=${libs} failed=${failed} errored=${errored} missing=${missing}" + +# ---------------------------------------------------------------- junit + +if [[ -n "${JUNIT}" ]]; then + mkdir -p "$(dirname "${JUNIT}")" + total=$(wc -l <"${CASES}" | tr -d ' ') + { + echo '' + printf '\n' \ + "${LABEL}" "${total}" "${failed}" "${errored}" "$((missing + libs))" + while IFS=$'\037' read -r id status detail _samples; do + printf ' ' "${LABEL}" "${id}" + case "${status}" in + pass) ;; + failure) printf '' "$(printf '%s' "${detail}" | tr '\036' '\n')" ;; + error) printf '' "$(printf '%s' "${detail}" | tr '\036' '\n')" ;; + skipped) printf '' "$(printf '%s' "${detail}" | tr '\036' ' ' | xml_escape)" ;; + esac + printf '\n' + done <"${CASES}" + echo '' + } >"${JUNIT}" + echo "JUnit report: ${JUNIT}" +fi + +# ---------------------------------------------------------------- report + +if [[ -n "${REPORT}" ]]; then + mkdir -p "$(dirname "${REPORT}")" + { + printf '### Marketplace plugins — %s\n\n' "${LABEL}" + printf '| Plugin | Classpath | Samples |\n|---|---|---|\n' + while IFS=$'\037' read -r id status detail samples; do + case "${status}" in + pass) cp_cell="pass" ;; + failure) cp_cell="**fail**" ;; + error) cp_cell="**error**" ;; + skipped) cp_cell="not checked — $(printf '%s' "${detail}" | tr '\036' ' ')" ;; + *) cp_cell="${status}" ;; + esac + + # A plugin an integration test already covers points at that test rather than + # reading as untested — most of them need a live service and cannot run here at + # all. It is evidence, not a pass: those tests run against a full distribution, + # where a plugin can borrow another plugin's jars — the very thing the classpath + # column exists to rule out. + if suite="$(it_suite "${id}")"; then + if [[ -n "${IT_BASE}" ]]; then + samples_cell="covered by IT — [\`${suite}\` ↗](${IT_BASE%/}/${suite}/)" + else + samples_cell="covered by IT — ${suite}" + fi + elif [[ "${samples:-0}" -gt 0 ]]; then + samples_cell="not run (${samples})" + else + samples_cell="no sample" + fi + printf '| %s | %s | %s |\n' "${id}" "${cp_cell}" "${samples_cell}" + done <"${CASES}" + + printf '\nClasspath: ok=%s allowed=%s library=%s failed=%s errored=%s missing=%s\n' \ + "${ok}" "${allowedcount}" "${libs}" "${failed}" "${errored}" "${missing}" + printf '\n%s\n' '- **no sample** — ships no pipeline under `config/projects/samples`, so nothing can smoke-test it.' + printf '%s\n' '- **not run** — ships samples; running them per plugin is not wired up yet.' + printf '%s\n' '- **covered by IT** — not smoke-tested here (these mostly need a live service), but exercised by the nightly integration tests. Supporting evidence only: those run against a full distribution, where a plugin can borrow another plugin'"'"'s jars.' + } >"${REPORT}" + echo "Report: ${REPORT}" +fi + +status=0 +if [[ ${failed} -gt 0 || ${errored} -gt 0 ]]; then + status=1 +fi +if [[ ${missing} -gt 0 ]]; then + if [[ "${ALLOW_MISSING}" == true ]]; then + echo "Ignoring ${missing} plugin zip(s) that were not built (--allow-missing)." + else + echo "${missing} plugin zip(s) were not built, so they were not checked." + status=1 + fi +fi + +[[ ${status} -eq 0 ]] && echo "Plugin classpath check passed (${LABEL})." +exit ${status} diff --git a/tools/fetch-apache-marketplace.sh b/tools/fetch-apache-marketplace.sh new file mode 100755 index 00000000000..4fd0bee5b82 --- /dev/null +++ b/tools/fetch-apache-marketplace.sh @@ -0,0 +1,202 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Download an Apache Hop client and the marketplace plugin zips that go with it, +# so tools/check-plugin-classpath.sh can check them the way a user's install would +# see them. +# +# The client zip is the right baseline and the container image is not: the image is +# built from a client that already had every marketplace plugin installed into it +# (Apache Hop's Jenkinsfile unpacks the client, runs install-wave1-plugins.sh, then +# builds the image from that directory). Checking against the image would let each +# plugin borrow the jars of all the others, which is exactly the bug being hunted. +# The published client zip is what a user downloads, and it is plugin-free. +# +# The plugin list is read from the client's own full-client-env.yaml rather than +# from Apache Hop's optional-plugins.yaml over the network, so the list and the +# baseline are always the same version. +# +# Releases and snapshots are both supported. A release client comes from the ASF +# dist archive; a -SNAPSHOT client comes from the snapshot repository, where the +# assemblies are published under the artifactId hop-client (the deploy-snapshots +# excludes drop hop-assemblies*, which that name does not match). Snapshot files are +# timestamped, so their real names have to be resolved per artifact from +# maven-metadata.xml — the plain --SNAPSHOT.zip name is a 404. +# +# Usage: +# tools/fetch-apache-marketplace.sh --version 2.19.0 --dest [--only ]... +# tools/fetch-apache-marketplace.sh --version 2.20.0-SNAPSHOT --dest +# tools/fetch-apache-marketplace.sh --version 2.19.0 --dest --client-only +# +# Writes /client.zip, /zips/*.zip and /plugins.txt (the +# " " list that check-plugin-classpath.sh --plugins expects). +# +# --client-only stops after the client, for callers that just need a baseline to +# install something else onto. +# +set -euo pipefail + +VERSION="" +DEST="" +CLIENT_ONLY=false +ONLY=() + +die() { echo "ERROR: $*" >&2; exit 2; } + +while [[ $# -gt 0 ]]; do + case "$1" in + --version) shift; VERSION="${1:-}"; [[ -n "${VERSION}" ]] || die "--version needs a value" ;; + --dest) shift; DEST="${1:-}"; [[ -n "${DEST}" ]] || die "--dest needs a value" ;; + --only) shift; [[ -n "${1:-}" ]] || die "--only needs an artifactId"; ONLY+=("$1") ;; + --client-only) CLIENT_ONLY=true ;; + -h | --help) sed -n '17,50p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) die "unknown argument: $1" ;; + esac + shift +done + +[[ -n "${VERSION}" ]] || die "--version is required" +[[ -n "${DEST}" ]] || die "--dest is required" +command -v curl >/dev/null 2>&1 || die "curl not on PATH" +command -v unzip >/dev/null 2>&1 || die "unzip not on PATH" + +# archive.apache.org keeps every release; downloads.apache.org only the current one, +# so a pinned older version resolves on the archive and not on the mirror. +DIST_URLS=( + "https://archive.apache.org/dist/hop/${VERSION}/apache-hop-client-${VERSION}.zip" + "https://downloads.apache.org/hop/${VERSION}/apache-hop-client-${VERSION}.zip" +) +# The marketplace itself resolves plugins from the ASF group first and Central as a +# fallback (see MarketplaceConfig in Apache Hop); mirror that order here so this +# check fails on the same artifacts a user's install would get. +REPO_URLS=( + "https://repository.apache.org/content/groups/public" + "https://repo1.maven.org/maven2" +) + +case "${VERSION}" in +*-SNAPSHOT) IS_SNAPSHOT=true ;; +*) IS_SNAPSHOT=false ;; +esac + +mkdir -p "${DEST}/zips" +CLIENT_ZIP="${DEST}/client.zip" + +fetch() { + local out="$1" url + shift + for url in "$@"; do + if curl -fsSL --retry 3 --retry-delay 2 -o "${out}.part" "${url}"; then + mv "${out}.part" "${out}" + printf '%s\n' "${url}" + return 0 + fi + done + rm -f "${out}.part" + return 1 +} + +# A snapshot version directory holds timestamped files only; maven-metadata.xml +# names the current one. Each artifact is resolved separately: they are usually from +# the same deploy run, but nothing guarantees it, and reusing one artifact's +# timestamp for another silently 404s. +snapshot_name() { + local repo="$1" art="$2" value + value="$(curl -fsSL --retry 2 --max-time 60 \ + "${repo}/org/apache/hop/${art}/${VERSION}/maven-metadata.xml" 2>/dev/null | + tr '<' '\n' | sed -n 's:^value>::p' | tail -1 || true)" + [[ -n "${value}" ]] || return 1 + printf '%s-%s.zip\n' "${art}" "${value}" +} + +# Candidate URLs for one artifact's zip, across both repositories. +zip_urls() { + local art="$1" repo name + for repo in "${REPO_URLS[@]}"; do + if [[ "${IS_SNAPSHOT}" == true ]]; then + name="$(snapshot_name "${repo}" "${art}")" || continue + else + name="${art}-${VERSION}.zip" + fi + printf '%s\n' "${repo}/org/apache/hop/${art}/${VERSION}/${name}" + done +} + +if [[ -f "${CLIENT_ZIP}" ]]; then + echo "==> Client already downloaded: ${CLIENT_ZIP}" +elif [[ "${IS_SNAPSHOT}" == true ]]; then + echo "==> Downloading Apache Hop ${VERSION} client from the snapshot repository" + urls=() + while IFS= read -r u; do urls+=("$u"); done < <(zip_urls hop-client) + [[ ${#urls[@]} -gt 0 ]] || die "no snapshot client published for ${VERSION}" + from="$(fetch "${CLIENT_ZIP}" "${urls[@]}")" || + die "could not download the snapshot client for ${VERSION}" + echo " from ${from}" +else + echo "==> Downloading Apache Hop ${VERSION} client" + from="$(fetch "${CLIENT_ZIP}" "${DIST_URLS[@]}")" || + die "could not download the client zip for ${VERSION} (tried ${DIST_URLS[*]})" + echo " from ${from}" +fi + +if [[ "${CLIENT_ONLY}" == true ]]; then + echo " client: ${CLIENT_ZIP}" + exit 0 +fi + +# full-client-env.yaml is generated from optional-plugins.yaml and ships in the +# client, so the list matches the baseline by construction. +ENV_FILE="$(unzip -Z1 "${CLIENT_ZIP}" '*full-client-env.yaml' 2>/dev/null | head -1 || true)" +[[ -n "${ENV_FILE}" ]] || + die "full-client-env.yaml not found in the client zip; this Hop version predates the marketplace registry" +unzip -p "${CLIENT_ZIP}" "${ENV_FILE}" >"${DEST}/full-client-env.yaml" + +ARTIFACTS="$(awk '/^[[:space:]]*-[[:space:]]*artifactId:/ { print $3 }' "${DEST}/full-client-env.yaml")" +[[ -n "${ARTIFACTS}" ]] || die "no artifactIds in full-client-env.yaml" + +wanted() { + [[ ${#ONLY[@]} -eq 0 ]] && return 0 + local a + for a in "${ONLY[@]}"; do [[ "$a" == "$1" ]] && return 0; done + return 1 +} + +: >"${DEST}/plugins.txt" +count=0 +failed=0 +for art in ${ARTIFACTS}; do + wanted "${art}" || continue + zip="${DEST}/zips/${art}-${VERSION}.zip" + if [[ ! -f "${zip}" ]]; then + urls=() + while IFS= read -r u; do urls+=("$u"); done < <(zip_urls "${art}") + if [[ ${#urls[@]} -eq 0 ]] || ! fetch "${zip}" "${urls[@]}" >/dev/null; then + # Not fatal on its own: the check reports it as a missing zip, which is a + # more useful failure than aborting the whole fetch here. + echo " MISSING ${art}" + failed=$((failed + 1)) + continue + fi + fi + printf '%s %s\n' "${art}" "${zip}" >>"${DEST}/plugins.txt" + count=$((count + 1)) +done + +echo "==> ${count} plugin zip(s) ready in ${DEST}/zips" +[[ ${failed} -gt 0 ]] && echo " ${failed} could not be downloaded" +echo " client: ${CLIENT_ZIP}" +echo " plugins: ${DEST}/plugins.txt" +exit 0 diff --git a/tools/marketplace-it-aliases.txt b/tools/marketplace-it-aliases.txt new file mode 100644 index 00000000000..487eb1e3b3e --- /dev/null +++ b/tools/marketplace-it-aliases.txt @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Exceptions for tools/marketplace-it-suites.sh. +# +# That script pairs a marketplace plugin with the integration-test project covering +# it by taking the last segment of the plugin's modulePath and looking for a +# directory of that name under integration-tests/. Twelve of the sixteen pairs fall +# out of that rule on their own; these four are the ones where the two trees chose +# different names. +# +# Format: +# Add a line only when the names genuinely differ — a new plugin whose IT project +# matches its module name needs nothing here. + +beam beam_directrunner +google gcp +mongodb mongo +script scripting diff --git a/tools/marketplace-it-suites.sh b/tools/marketplace-it-suites.sh new file mode 100755 index 00000000000..5b4400f3673 --- /dev/null +++ b/tools/marketplace-it-suites.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Pair each marketplace plugin with the integration-test project that covers it, for +# tools/check-plugin-classpath.sh --it-suites. +# +# Most marketplace plugins cannot be smoke-tested from a GitHub runner: they need a +# database, a cloud account or a cluster. The integration tests cover those, so the +# report links to the matching suite instead of leaving the row looking untested. +# +# Both sides of the pairing are already in the repository, so this is derived rather +# than maintained by hand: the plugin list and its modulePath come from +# optional-plugins.yaml (via list-marketplace-plugins.sh) and the projects are the +# directories under integration-tests/. A plugin is paired when the last segment of +# its modulePath matches a project name. Only the handful of cases where the two +# trees picked different names need an entry in marketplace-it-aliases.txt, so a new +# plugin that follows the convention is picked up with no edit here. +# +# The project directory name is also the suite name Jenkins publishes, which is what +# makes the report link work. +# +# Usage: +# tools/marketplace-it-suites.sh # " " per line +# +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +LIST="${ROOT}/tools/list-marketplace-plugins.sh" +ALIASES="${ROOT}/tools/marketplace-it-aliases.txt" +IT_DIR="${ROOT}/integration-tests" + +[[ -x "${LIST}" ]] || chmod +x "${LIST}" 2>/dev/null || true + +alias_for() { + [[ -f "${ALIASES}" ]] || return 1 + awk -v n="$1" '/^[[:space:]]*#/ { next } NF < 2 { next } $1 == n { print $2; found = 1; exit } + END { exit found ? 0 : 1 }' "${ALIASES}" +} + +while IFS='|' read -r artifact module; do + [[ -n "${artifact}" && -n "${module}" ]] || continue + name="${module##*/}" + suite="$(alias_for "${name}" || true)" + [[ -n "${suite}" ]] || suite="${name}" + [[ -d "${IT_DIR}/${suite}" ]] || continue + printf '%s %s\n' "${artifact}" "${suite}" +done < <("${LIST}") diff --git a/tools/plugin-classpath-allowlist.txt b/tools/plugin-classpath-allowlist.txt new file mode 100644 index 00000000000..7a2ef09ab32 --- /dev/null +++ b/tools/plugin-classpath-allowlist.txt @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Packages that tools/check-plugin-classpath.sh is allowed to leave unresolved. +# +# Format: +# A prefix matches the package itself and everything below it, so +# "edu.stanford.nlp" also covers "edu.stanford.nlp.simple". +# +# Every entry needs a comment saying why the jar legitimately does not ship. +# The default answer to a new unresolved package is to fix the packaging, not +# to add a line here. + +# stanford-corenlp is GPLv3 (ASF category X) and is deliberately kept out of the +# plugin zip; users download it themselves. See the hop.pluginlib.exclude* +# properties and the comment above them in plugins/transforms/stanfordnlp/pom.xml. +hop-transform-stanfordnlp edu.stanford.nlp