diff --git a/.github/workflows/build_static_site.sh b/.github/workflows/build_static_site.sh index 95068aca..0da5ad15 100644 --- a/.github/workflows/build_static_site.sh +++ b/.github/workflows/build_static_site.sh @@ -1,124 +1,170 @@ -set -e -#build_static_site.sh +#!/usr/bin/env bash +set -euo pipefail -# Do we need to build? -cd ./splashkitonline -# only build if the SplashKitWasm folder has changed, or if this was a push to a branch (so branches always build) -if ! git diff --quiet $(git merge-base "origin/main" "$1").."$1" -- SplashKitWasm &>/dev/null || [ "$2" == "push" ]; then - cd ../ +# build_static_site.sh + +ROOT_DIR="$(pwd)" +REPO_DIR="$ROOT_DIR/splashkitonline" +PREBUILT_DIR="$ROOT_DIR/prebuilt" + +echo "========================================" +echo "Build Static Site Script" +echo "Repo dir: $REPO_DIR" +echo "========================================" + +cd "$REPO_DIR" + +# Only build WASM binaries if SplashKitWasm changed, or if this is a push +if ! git diff --quiet "$(git merge-base "origin/main" "$1")..$1" -- SplashKitWasm &>/dev/null || [ "$2" = "push" ]; then + cd "$ROOT_DIR" echo "========================================" - echo "Downloading Compilation Pre-builts (To improve...these should be buildable too)" + echo "Downloading Compilation Pre-builts" echo "========================================" - mkdir -p ./splashkitonline/SplashKitWasm/prebuilt/cxx/compiler/ - cd ./splashkitonline/SplashKitWasm/prebuilt/cxx/compiler/ + + mkdir -p "$REPO_DIR/SplashKitWasm/prebuilt/cxx/compiler" + cd "$REPO_DIR/SplashKitWasm/prebuilt/cxx/compiler" + wget -O clang++.js https://raw.githubusercontent.com/WhyPenguins/SplashkitOnline/github-live/Browser_IDE/compilers/cxx/bin/clang++.js wget -O clang.wasm.lzma https://raw.githubusercontent.com/WhyPenguins/SplashkitOnline/github-live/Browser_IDE/compilers/cxx/bin/clang.wasm.lzma wget -O wasm-ld.js https://raw.githubusercontent.com/WhyPenguins/SplashkitOnline/github-live/Browser_IDE/compilers/cxx/bin/wasm-ld.js wget -O lld.wasm.lzma https://raw.githubusercontent.com/WhyPenguins/SplashkitOnline/github-live/Browser_IDE/compilers/cxx/bin/lld.wasm.lzma wget -O sysroot.zip https://github.com/WhyPenguins/SplashkitOnline/raw/refs/heads/cxx_language_backend_binaries/SplashKitWasm/prebuilt/sysroot.zip - # decompress them - silly since they'll just be re-compressed again, but it is what it is for now... - xz -d clang.wasm.lzma - xz -d lld.wasm.lzma - cd ../../../../../ + # Decompress downloaded wasm archives + xz -d -f clang.wasm.lzma + xz -d -f lld.wasm.lzma + + cd "$ROOT_DIR" echo "========================================" echo "Set Up Compilation Environment" echo "========================================" sudo apt-get -qq update - sudo apt-get install -y build-essential cmake libpng-dev libcurl4-openssl-dev libsdl2-dev libsdl2-mixer-dev libsdl2-gfx-dev libsdl2-image-dev libsdl2-net-dev libsdl2-ttf-dev libmikmod-dev libbz2-dev libflac-dev libvorbis-dev libwebp-dev + sudo apt-get install -y \ + build-essential \ + cmake \ + libpng-dev \ + libcurl4-openssl-dev \ + libsdl2-dev \ + libsdl2-mixer-dev \ + libsdl2-gfx-dev \ + libsdl2-image-dev \ + libsdl2-net-dev \ + libsdl2-ttf-dev \ + libmikmod-dev \ + libbz2-dev \ + libflac-dev \ + libvorbis-dev \ + libwebp-dev + git clone https://github.com/emscripten-core/emsdk.git ./emsdk/emsdk install 3.1.48 - echo "========================================" echo "Build SplashKit WASM Libraries" echo "========================================" - cd emsdk + cd "$ROOT_DIR/emsdk" ./emsdk activate 3.1.48 + # shellcheck disable=SC1091 source ./emsdk_env.sh - cd ../ - mkdir -p ./splashkitonline/SplashKitWasm/out/cxx/compiler/ # this one is due to a mistake in old CMakeLists, can be removed soon - # build this as well... - + cd "$ROOT_DIR" - cd ./splashkitonline/SplashKitWasm/cmake/ + mkdir -p "$REPO_DIR/SplashKitWasm/out/cxx/compiler" - emcmake cmake -G "Unix Makefiles" -DENABLE_JS_BACKEND=ON -DENABLE_CPP_BACKEND=ON -DENABLE_FUNCTION_OVERLOADING=ON -DCOMPRESS_BACKENDS=ON . + cd "$REPO_DIR/SplashKitWasm/cmake" + emcmake cmake -G "Unix Makefiles" \ + -DENABLE_JS_BACKEND=ON \ + -DENABLE_CPP_BACKEND=ON \ + -DENABLE_FUNCTION_OVERLOADING=ON \ + -DCOMPRESS_BACKENDS=ON . emmake make -j8 - cd ../../../ + cd "$ROOT_DIR" else - echo "========================================" echo "Using Precompiled Binaries from Main" echo "========================================" - cd ../ - - # Rather than building, we'll just grab the compiled binaries from the main release. - # To do this, we'll copy over all completely untracked files from it, which will correctly - # handle if the PR has deleted files since branching from main. - # Perhaps there's a cleaner way :) + cd "$REPO_DIR" - # first let's get a list of files _not_ to copy - cd ./splashkitonline - TRACKED_FILES=$(git log --pretty=format: --name-only --diff-filter=A -- Browser_IDE| sort - | sed '/^$/d') + # Get a list of tracked files so we only overlay prebuilt generated/static output + TRACKED_FILES=$(git log --pretty=format: --name-only --diff-filter=A -- . | sort -u | sed '/^$/d') EXCLUDE_FILE=$(mktemp) - echo "$TRACKED_FILES" | sed "s|^Browser_IDE||" > "$EXCLUDE_FILE" - - # add some explicit excludes - echo "/codemirror-5.65.15" >> "$EXCLUDE_FILE" - echo "/jszip" >> "$EXCLUDE_FILE" - echo "/babel" >> "$EXCLUDE_FILE" - echo "/split.js" >> "$EXCLUDE_FILE" - echo "/mime" >> "$EXCLUDE_FILE" - echo "/DemoProjects" >> "$EXCLUDE_FILE" - echo "/node_modules" >> "$EXCLUDE_FILE" - - cd ../ - - mkdir prebuilt - cd prebuilt - # Download main's latest release + echo "$TRACKED_FILES" > "$EXCLUDE_FILE" + + # Explicit excludes + { + echo "/codemirror-5.65.15" + echo "/jszip" + echo "/babel" + echo "/split.js" + echo "/mime" + echo "/node_modules" + echo "/prebuilt" + } >> "$EXCLUDE_FILE" + + cd "$ROOT_DIR" + + rm -rf "$PREBUILT_DIR" + mkdir -p "$PREBUILT_DIR" + cd "$PREBUILT_DIR" + wget "https://github.com/$3/releases/download/branch%2Fmain/splashkitonline-static-site-branch_main.zip" unzip splashkitonline-static-site-branch_main.zip rm splashkitonline-static-site-branch_main.zip - cd ../ - # copy in all the untracked files! - rsync -av --progress --exclude-from="$EXCLUDE_FILE" "prebuilt/" "splashkitonline/Browser_IDE/" + cd "$ROOT_DIR" -fi + # Copy in all untracked/generated files from the release into repo root + rsync -av --progress --exclude-from="$EXCLUDE_FILE" "$PREBUILT_DIR/" "$REPO_DIR/" + rm -f "$EXCLUDE_FILE" +fi echo "========================================" echo "Install Node Dependencies" echo "========================================" -cd ./splashkitonline/Browser_IDE - +cd "$REPO_DIR" npm install -cd ../../ +echo "========================================" +echo "Re-Structure Static Site" +echo "========================================" +# If changed, remember to update explicit excludes above +if [ -d node_modules/codemirror ]; then + rm -rf codemirror-5.65.15 + mv node_modules/codemirror codemirror-5.65.15 +fi +if [ -d node_modules/jszip/dist ]; then + rm -rf jszip + mv node_modules/jszip/dist jszip +fi + +if [ -d node_modules/@babel/standalone ]; then + rm -rf babel + mv node_modules/@babel/standalone babel +fi + +if [ -d node_modules/split.js/dist ]; then + rm -rf split.js + mv node_modules/split.js/dist split.js +fi + +if [ -d node_modules/mime/dist ]; then + rm -rf mime + mv node_modules/mime/dist mime +fi + +if [ -d js-lzma/data ]; then + rm -rf js-lzma/data +fi echo "========================================" -echo "Re-Structure Static Site" -echo "========================================" -cd ./splashkitonline/Browser_IDE - -# if changed, remember to update the explicit excludes above -mv node_modules/codemirror codemirror-5.65.15 -mv node_modules/jszip/dist jszip -mv node_modules/@babel/standalone babel -mv node_modules/split.js/dist split.js -mv node_modules/mime/dist mime -rm -rf external/js-lzma/data -mv ../DemoProjects DemoProjects - -cd ../ +echo "Static site preparation complete" +echo "========================================" \ No newline at end of file diff --git a/.github/workflows/build_static_site.yml b/.github/workflows/build_static_site.yml index 75c66d51..04b84295 100644 --- a/.github/workflows/build_static_site.yml +++ b/.github/workflows/build_static_site.yml @@ -1,20 +1,18 @@ name: Build Static Site on: - # Runs on pushes targeting the default branch push: branches: ["deployed", "main"] - pull_request_target: # use trusted workflow (as opposed to `pull_request`) + pull_request_target: types: - - labeled # workaround for manual triggering...if we actually start using labels this might need to be removed - - unlabeled # same + - labeled + - unlabeled - opened - reopened - synchronize - closed - # Allows you to run this workflow manually from the Actions tab workflow_dispatch: permissions: {} @@ -24,15 +22,25 @@ concurrency: cancel-in-progress: true jobs: - # Creates a build of the website and uploads it as an artifact - # May be running untrusted code, so it's been given very restricted permissions - # Hopefully it can't do anything too bad... build: if: ${{ github.event_name != 'pull_request_target' || github.event.action != 'closed' }} permissions: contents: read runs-on: ubuntu-latest + outputs: + artifact_name: ${{ steps.vars.outputs.artifact_name }} + steps: + - name: Determine artifact name + id: vars + shell: bash + run: | + if [ "${{ github.event_name }}" = "pull_request_target" ]; then + echo "artifact_name=preview-${{ github.event.pull_request.head.sha }}" >> "$GITHUB_OUTPUT" + else + echo "artifact_name=preview-${{ github.sha }}" >> "$GITHUB_OUTPUT" + fi + - name: Use Node.js 20 uses: actions/setup-node@v4 with: @@ -44,10 +52,10 @@ jobs: uses: actions/checkout@v4 with: path: splashkitonline - submodules: 'recursive' - ref: ${{ github.event.pull_request.head.ref }} - repository: ${{ github.event.pull_request.head.repo.full_name }} - fetch-depth: '0' + submodules: recursive + ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.ref || github.ref_name }} + repository: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name || github.repository }} + fetch-depth: 0 env: GITHUB_TOKEN: '' @@ -61,20 +69,19 @@ jobs: - name: Build static site working-directory: ./ + shell: bash run: | - GITHUB_SHA=${{ github.sha }} - if [ "${{ github.event_name }}" == "pull_request_target" ]; then + GITHUB_SHA="${{ github.sha }}" + if [ "${{ github.event_name }}" = "pull_request_target" ]; then GITHUB_SHA="${{ github.event.pull_request.head.sha }}" fi cd ./splashkitonline - # only run the `build_static_site.sh` in the PR if the user changed it - otherwise use the latest in main. - if ! git diff --quiet $(git merge-base "origin/main" "$GITHUB_SHA").."$GITHUB_SHA" -- .github/workflows/ &>/dev/null; then - script_path="./splashkitonline/.github/workflows/build_static_site.sh" # use the PR's + if ! git diff --quiet "$(git merge-base "origin/main" "$GITHUB_SHA")..$GITHUB_SHA" -- .github/workflows/ &>/dev/null; then + script_path="./.github/workflows/build_static_site.sh" else - script_path="./script/.github/workflows/build_static_site.sh" # use main's + script_path="../script/.github/workflows/build_static_site.sh" fi - cd ../ echo "using $script_path" bash "$script_path" "$GITHUB_SHA" "${{ github.event_name }}" "${{ github.repository }}" @@ -84,21 +91,16 @@ jobs: - name: Upload the build as an artifact uses: actions/upload-artifact@v4.4.3 with: - name: preview-${{ github.sha }} - path: ./splashkitonline/Browser_IDE + name: ${{ steps.vars.outputs.artifact_name }} + path: ./splashkitonline retention-days: 1 compression-level: 8 overwrite: true + if-no-files-found: error - # This part takes that artifact and uploads it as a release - # Perhaps this can be merged into the previous job - just seperated it out for - # potential security concerns (if for instance scripts in the previous job poisoned something) - # We never run the artifact here, so should be completely safe this way. - # Todo: Investigate! deploy: needs: build - if: ${{ always() && !failure() && !cancelled() }} # run this job even if the previous one is skipped - # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages + if: ${{ always() && !failure() && !cancelled() }} permissions: contents: write id-token: write @@ -107,8 +109,9 @@ jobs: name: static-site url: ${{ steps.deployment.outputs.page_url }} runs-on: ubuntu-latest + steps: - - name: Checkout scripts # This way it's safe... + - name: Checkout scripts uses: actions/checkout@v4 with: path: script @@ -120,51 +123,49 @@ jobs: uses: actions/checkout@v4 with: path: repo - submodules: 'false' - fetch-depth: '0' + submodules: false + fetch-depth: 0 - # Note, we download it _into_ the git folder (so a .git is in a parent dir) - # The edit_release.py script will upload the assets in release_assets.txt, _relative_ to where the script is - # called from. However, it also updates tags in the repo, and so a `.git` directory needs to be somewhere. - # This is all a bit messy, but it works for now. Can be someone else's job to untangle it :P - name: Download built site if: ${{ github.event_name != 'pull_request_target' || github.event.action != 'closed' }} uses: actions/download-artifact@v4.1.8 with: - name: preview-${{ github.sha }} + name: ${{ needs.build.outputs.artifact_name }} path: ./repo/built-site - name: Create/Update release if: ${{ github.event_name != 'pull_request_target' || github.event.action != 'closed' }} working-directory: ./repo/built-site + shell: bash run: | - sudo apt-get install zip sed + sudo apt-get install -y zip sed - if [ "${{ github.event_name }}" == "pull_request_target" ]; then - echo " PR Release! ${{ github.event.number }}" + if [ "${{ github.event_name }}" = "pull_request_target" ]; then + echo "PR Release! ${{ github.event.number }}" tag_name="pr/${{ github.event.number }}" else echo "Branch Release! ${{ github.ref_name }}" tag_name="branch/${{ github.ref_name }}" fi - file_name=$(echo $tag_name | sed "s#/#_#") - zip -r "../splashkitonline-static-site-$file_name.zip" ./ # note: this name is expected in download_github_site_previews.py + file_name=$(echo "$tag_name" | sed 's#/#_#') + + zip -r "../splashkitonline-static-site-$file_name.zip" ./ echo "../splashkitonline-static-site-$file_name.zip" >> ../../script/.github/workflows/release_assets.txt python3 ../../script/.github/workflows/edit_release.py "update" "${{ secrets.GITHUB_TOKEN }}" "${{ github.repository }}" "$tag_name" "${{ github.sha }}" yes "../../script/.github/workflows/release_assets.txt" "# Static Site Build ($tag_name)\nA static build of $tag_name, used in the live deployment :smiley:" - repoSiteURL=$(echo ${{ github.repository }} | sed 's/\//.github.io\//') - echo "repoSiteURL=$repoSiteURL" >> $GITHUB_ENV + repoSiteURL=$(echo "${{ github.repository }}" | sed 's/\//.github.io\//') + echo "repoSiteURL=$repoSiteURL" >> "$GITHUB_ENV" - name: Or delete the release... if: ${{ github.event_name == 'pull_request_target' && github.event.action == 'closed' }} working-directory: ./repo + shell: bash run: | - echo " PR Release Delete! ${{ github.ref_name }}" + echo "PR Release Delete! ${{ github.ref_name }}" python3 ../script/.github/workflows/edit_release.py "delete" "${{ secrets.GITHUB_TOKEN }}" "${{ github.repository }}" "pr/${{ github.event.number }}" - # The following bits assume the GitHub Pages site is about to be updated with this new release preview - name: Leave a link on the PR if: ${{ success() && github.event_name == 'pull_request_target' && github.event.action != 'closed' }} uses: marocchino/sticky-pull-request-comment@v2 @@ -185,4 +186,4 @@ jobs: | :whale2: **PR Preview!** | | :-----: | | The preview is no more! | - | Congrats if this was merged! :smile: | + | Congrats if this was merged! :smile: | \ No newline at end of file diff --git a/compilers/cxx/cxxCompilerClangWebWorker.js b/compilers/cxx/cxxCompilerClangWebWorker.js index f18e2dd1..22d46b02 100644 --- a/compilers/cxx/cxxCompilerClangWebWorker.js +++ b/compilers/cxx/cxxCompilerClangWebWorker.js @@ -2,10 +2,10 @@ let SKO = null; importScripts('./../../jszip/jszip.min.js'); importScripts('./../../javascript/communication/communication.js'); -importScripts('./../../external/js-lzma/src/wlzma.js'); -importScripts('./../../external/js-lzma/src/lzma.shim.js'); +importScripts('./../../js-lzma/src/wlzma.js'); +importScripts('./../../js-lzma/src/lzma.shim.js'); -self.wlzmaCustomPath = "./../../external/js-lzma/src/wlzma.wrk.js"; +self.wlzmaCustomPath = "./../../js-lzma/src/wlzma.wrk.js"; self.downloadRootPath = "./../../"; importScripts('./../../javascript/middleware/downloadHandler.js'); diff --git a/git grep -n lzma.js b/git grep -n lzma.js new file mode 100644 index 00000000..f9228bd9 --- /dev/null +++ b/git grep -n lzma.js @@ -0,0 +1,13 @@ +.github/workflows/build_static_site.sh:121:rm -rf external/js-lzma/data +.gitmodules:6:[submodule "js-lzma"] +.gitmodules:7: path = js-lzma +.gitmodules:8: url = https://github.com/WhyPenguins/js-lzma +compilers/cxx/cxxCompilerClangWebWorker.js:5:importScripts('./../../external/js-lzma/src/wlzma.js'); +compilers/cxx/cxxCompilerClangWebWorker.js:6:importScripts('./../../external/js-lzma/src/lzma.shim.js'); +compilers/cxx/cxxCompilerClangWebWorker.js:8:self.wlzmaCustomPath = "./../../external/js-lzma/src/wlzma.wrk.js"; +javascript/languages/languageDefinitions.js:103: "external/js-lzma/src/wlzma.js", +javascript/languages/languageDefinitions.js:104: "external/js-lzma/src/lzma.shim.js", +javascript/middleware/downloadHandler.js:3:let wlzmaPath = self['wlzmaCustomPath'] || "./external/js-lzma/src/wlzma.wrk.js"; +temp.txt:1:[submodule Browser_IDE/external/js-lzma"] +temp.txt:2: path = external/js-lzma +temp.txt:3: url = https://github.com/WhyPenguins/js-lzma diff --git a/javascript/languages/languageDefinitions.js b/javascript/languages/languageDefinitions.js index 1be615fd..18e619b0 100644 --- a/javascript/languages/languageDefinitions.js +++ b/javascript/languages/languageDefinitions.js @@ -100,8 +100,8 @@ let SplashKitOnlineLanguageDefinitions = [ "runtimes/cxx/bin/SplashKitBackendWASMCPP.worker.js", ], compilerFiles: [ - "external/js-lzma/src/wlzma.js", - "external/js-lzma/src/lzma.shim.js", + "js-lzma/src/wlzma.js", + "js-lzma/src/lzma.shim.js", "compilers/cxx/cxxCompiler.js", ], runtimeSizeAprox: 0, // user's compiled code becomes the 'runtime' diff --git a/javascript/middleware/downloadHandler.js b/javascript/middleware/downloadHandler.js index 15bbf1ee..1efef829 100644 --- a/javascript/middleware/downloadHandler.js +++ b/javascript/middleware/downloadHandler.js @@ -1,6 +1,6 @@ "use strict"; -let wlzmaPath = self['wlzmaCustomPath'] || "./external/js-lzma/src/wlzma.wrk.js"; +let wlzmaPath = self['wlzmaCustomPath'] || "./js-lzma/src/wlzma.wrk.js"; let downloadRootPath = self['downloadRootPath'] || "./"; // url patch map diff --git a/js-lzma b/js-lzma new file mode 160000 index 00000000..3060ba8a --- /dev/null +++ b/js-lzma @@ -0,0 +1 @@ +Subproject commit 3060ba8a5b5e6f4e7276d937b5e214979bf9f52b diff --git a/temp.txt b/temp.txt index b731f1f1..295c1ca2 100644 --- a/temp.txt +++ b/temp.txt @@ -1,3 +1,3 @@ -[submodule Browser_IDE/external/js-lzma"] - path = external/js-lzma +[submodule Browser_IDE/js-lzma"] + path = js-lzma url = https://github.com/WhyPenguins/js-lzma