_helm-e2e.yml

# Copyright (C) 2024 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

name: Helm Chart E2e Test For Call
permissions:
  contents: read

on:
  workflow_call:
    inputs:
      example:
        default: "chatqna"
        required: true
        type: string
        description: "example to test, chatqna or common/asr"
      hardware:
        default: "xeon"
        required: true
        type: string
      dockerhub:
        default: "false"
        required: false
        type: string
        description: "Set to true if you want to use released docker images at dockerhub. By default using internal docker registry."
      mode:
        default: "CD"
        description: "Whether the test range is CI, CD or CICD"
        required: false
        type: string
      tag:
        default: "latest"
        required: false
        type: string
      version:
        default: "0-latest"
        required: false
        type: string

jobs:
  get-test-case:
    runs-on: ubuntu-latest
    outputs:
      value_files: ${{ steps.get-test-files.outputs.value_files }}
      CHECKOUT_REF: ${{ steps.get-checkout-ref.outputs.CHECKOUT_REF }}
    steps:
      - name: Get checkout ref
        id: get-checkout-ref
        run: |
          if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "pull_request_target" ]; then
            CHECKOUT_REF=refs/pull/${{ github.event.number }}/merge
          else
            CHECKOUT_REF=${{ github.ref }}
          fi
          echo "CHECKOUT_REF=${CHECKOUT_REF}" >> $GITHUB_OUTPUT
          echo "checkout ref ${CHECKOUT_REF}"

      - name: Checkout Repo
        uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2
        with:
          ref: ${{ steps.get-checkout-ref.outputs.CHECKOUT_REF }}
          fetch-depth: 0

      - name: Get test Services
        id: get-test-files
        run: |
          set -x
          if [ "${{ inputs.mode }}" = "CI" ]; then
            base_commit=${{ github.event.pull_request.base.sha }}
            merged_commit=$(git log -1 --format='%H')
            values_files=$(git diff --name-only ${base_commit} ${merged_commit} | \
            grep "${{ inputs.example }}/kubernetes/helm" | \
            grep "values.yaml" |\
            sort -u)
            echo $values_files
          elif [ "${{ inputs.mode }}" = "CD" ]; then
            values_files=$(ls ${{ inputs.example }}/kubernetes/helm/*values.yaml || true)
          fi
          value_files="["
          for file in ${values_files}; do
            if [ -f "$file" ]; then
              filename=$(basename "$file")
              if [[ "$filename" == *"gaudi"* ]]; then
                if [[ "${{ inputs.hardware }}" == "gaudi" ]]; then
                  value_files="${value_files}\"${filename}\","
                fi
              elif [[ "$filename" == *"rocm"* ]]; then
                if [[ "${{ inputs.hardware }}" == "rocm" ]]; then
                  value_files="${value_files}\"${filename}\","
                fi
              elif [[ "$filename" == *"nv"* ]]; then
                continue
              else
                if [[ "${{ inputs.hardware }}" == "xeon" ]]; then
                  value_files="${value_files}\"${filename}\","
                fi
              fi
            fi
          done
          value_files="${value_files%,}]"

          echo "value_files=${value_files}"
          echo "value_files=${value_files}" >> $GITHUB_OUTPUT

  helm-test:
    needs: [get-test-case]
    if: ${{ needs.get-test-case.outputs.value_files != '[]' }}
    strategy:
      matrix:
        value_file: ${{ fromJSON(needs.get-test-case.outputs.value_files) }}
      fail-fast: false
    runs-on: k8s-${{ inputs.hardware }}
    continue-on-error: true
    steps:
      - name: Clean Up Working Directory
        run: |
          echo "value_file=${{ matrix.value_file }}"
          sudo rm -rf ${{github.workspace}}/*

      - name: Get checkout ref
        id: get-checkout-ref
        run: |
          if [ "${{ github.event_name }}" == "pull_request" ] || [ "${{ github.event_name }}" == "pull_request_target" ]; then
            CHECKOUT_REF=refs/pull/${{ github.event.number }}/merge
          else
            CHECKOUT_REF=${{ github.ref }}
          fi
          echo "CHECKOUT_REF=${CHECKOUT_REF}" >> $GITHUB_OUTPUT
          echo "checkout ref ${CHECKOUT_REF}"

      - name: Checkout Repo
        uses: actions/checkout@1e31de5234b9f8995739874a8ce0492dc87873e2
        with:
          ref: ${{ steps.get-checkout-ref.outputs.CHECKOUT_REF }}
          fetch-depth: 0

      - name: Set variables
        env:
          example: ${{ inputs.example }}
        run: |
          if [[ ! "$example" =~ ^[a-zA-Z0-9]{1,20}$ ]] || [[ "$example" =~ \.\. ]] || [[ "$example" == -* || "$example" == *- ]]; then
            echo "Error: Invalid input - only lowercase alphanumeric and internal hyphens allowed"
            exit 1
          fi
          # SAFE_PREFIX="kb-"
          CHART_NAME="${SAFE_PREFIX}$(echo "$example" | tr '[:upper:]' '[:lower:]')"
          RAND_SUFFIX=$(openssl rand -hex 2 | tr -dc 'a-f0-9')

          cat <<EOF >> $GITHUB_ENV
          CHART_NAME=${CHART_NAME}
          RELEASE_NAME=${CHART_NAME}-$(date +%s)
          NAMESPACE=ns-${CHART_NAME}-${RAND_SUFFIX}
          ROLLOUT_TIMEOUT_SECONDS=600s
          TEST_TIMEOUT_SECONDS=600s
          KUBECTL_TIMEOUT_SECONDS=60s
          should_cleanup=false
          skip_validate=false
          CHART_FOLDER=${example}/kubernetes/helm
          EOF

          echo "Generated safe variables:" >> $GITHUB_STEP_SUMMARY
          echo "- CHART_NAME: ${CHART_NAME}" >> $GITHUB_STEP_SUMMARY

      - name: Helm install
        id: install
        env:
          GOOGLE_CSE_ID: ${{ secrets.GOOGLE_CSE_ID }}
          GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
          HUGGINGFACEHUB_API_TOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
          HFTOKEN: ${{ secrets.HUGGINGFACEHUB_API_TOKEN }}
          value_file: ${{ matrix.value_file }}
        run: |
          set -xe
          echo "should_cleanup=true" >> $GITHUB_ENV
          if [[ ! -f ${{ github.workspace }}/${{ env.CHART_FOLDER }}/${value_file} ]]; then
            echo "No value file found, exiting test!"
            echo "skip_validate=true" >> $GITHUB_ENV
            echo "should_cleanup=false" >> $GITHUB_ENV
            exit 0
          fi

          for img in `helm template -n $NAMESPACE $RELEASE_NAME oci://ghcr.io/opea-project/charts/${CHART_NAME} -f ${{ inputs.example }}/kubernetes/helm/${value_file} --version ${{ inputs.version }} | grep 'image:' | grep 'opea/' | awk '{print $2}' | xargs`;
          do
            # increase helm install wait for for vllm-gaudi case
            if [[ $img == *"vllm-gaudi"* ]]; then
              ROLLOUT_TIMEOUT_SECONDS=900s
            fi
          done
          if ! helm install \
            --create-namespace \
            --namespace $NAMESPACE \
            $RELEASE_NAME \
            oci://ghcr.io/opea-project/charts/${CHART_NAME} \
            --set global.HUGGINGFACEHUB_API_TOKEN=${HFTOKEN} \
            --set global.modelUseHostPath=/data2/hf_model \
            --set GOOGLE_API_KEY=${{ env.GOOGLE_API_KEY}} \
            --set GOOGLE_CSE_ID=${{ env.GOOGLE_CSE_ID}} \
            --set web-retriever.GOOGLE_API_KEY=${{ env.GOOGLE_API_KEY}} \
            --set web-retriever.GOOGLE_CSE_ID=${{ env.GOOGLE_CSE_ID}} \
            -f ${{ inputs.example }}/kubernetes/helm/${value_file} \
            --version ${{ inputs.version }} \
            --wait --timeout "$ROLLOUT_TIMEOUT_SECONDS"; then
            echo "Failed to install chart ${{ inputs.example }}"
            echo "skip_validate=true" >> $GITHUB_ENV
            .github/workflows/scripts/k8s-utils.sh dump_pods_status $NAMESPACE
            exit 1
          fi

      - name: Validate e2e test
        if: always()
        run: |
          set -xe
          if $skip_validate; then
            echo "Skip validate"
          else
            LOG_PATH=/home/$(whoami)/helm-logs
            chart=${{ env.CHART_NAME }}
            mkdir -p ${LOG_PATH}
            helm test -n $NAMESPACE $RELEASE_NAME --logs --timeout "$TEST_TIMEOUT_SECONDS" | tee ${LOG_PATH}/charts-${chart}.log
            exit_code=$?
            if [ $exit_code -ne 0 ]; then
                echo "Chart ${chart} test failed, please check the logs in ${LOG_PATH}!"
                exit 1
            fi

            echo "Checking response results, make sure the output is reasonable. "
            teststatus=false
            if [[ -f $LOG_PATH/charts-${chart}.log ]] && \
            [[ $(grep -c "^Phase:.*Failed" $LOG_PATH/charts-${chart}.log) != 0 ]]; then
                teststatus=false
                ${{ github.workspace }}/.github/workflows/scripts/k8s-utils.sh dump_all_pod_logs $NAMESPACE
            else
                teststatus=true
            fi

            if [ $teststatus == false ]; then
                echo "Response check failed, please check the logs in artifacts!"
                exit 1
            else
                echo "Response check succeeded!"
                exit 0
            fi
          fi

      - name: Helm uninstall
        if: always()
        run: |
          if $should_cleanup; then
            helm uninstall $RELEASE_NAME --namespace $NAMESPACE
            if ! kubectl delete ns $NAMESPACE --timeout=$KUBECTL_TIMEOUT_SECONDS; then
              kubectl delete pods --namespace $NAMESPACE --force --grace-period=0 --all
              kubectl delete ns $NAMESPACE --force --grace-period=0 --timeout=$KUBECTL_TIMEOUT_SECONDS
            fi
          fi