wasm: add experimental browser runtime and SDK

.emscripten-version

@@ -0,0 +1 @@
+6.0.9

.github/workflows/wasm-browser.yaml

@@ -0,0 +1,75 @@
+name: Browser WebAssembly
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - '.emscripten-version'
+      - '.github/workflows/wasm-browser.yaml'
+      - 'CMakeLists.txt'
+      - 'cmake/**'
+      - 'include/**'
+      - 'src/**'
+      - 'lib/**'
+      - 'plugins/**'
+      - 'sdk/browser/**'
+      - 'tests/wasm/**'
+
+permissions:
+  contents: read
+
+jobs:
+  wasm:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix:
+        sanitizer: [none, address]
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: Install build prerequisites
+        run: sudo apt-get update && sudo apt-get install -y cmake make flex bison perl pkg-config
+      - name: Install pinned Emscripten SDK
+        shell: bash
+        run: |
+          set -euo pipefail
+          git init "$RUNNER_TEMP/flb-emsdk"
+          git -C "$RUNNER_TEMP/flb-emsdk" remote add origin https://github.com/emscripten-core/emsdk.git
+          git -C "$RUNNER_TEMP/flb-emsdk" fetch --depth=1 origin 5eb0bde7585670252e8ba05e9d361627bffd08b5
+          git -C "$RUNNER_TEMP/flb-emsdk" checkout --detach FETCH_HEAD
+          read -r flb_sdk_version < .emscripten-version
+          "$RUNNER_TEMP/flb-emsdk/emsdk" install "$flb_sdk_version"
+          "$RUNNER_TEMP/flb-emsdk/emsdk" activate "$flb_sdk_version"
+      - name: Build and test
+        shell: bash
+        env:
+          FLB_SANITIZER: ${{ matrix.sanitizer }}
+        run: |
+          source "$RUNNER_TEMP/flb-emsdk/emsdk_env.sh"
+          set -euo pipefail
+          flb_cmake_flags=()
+          if [[ "$FLB_SANITIZER" == address ]]; then
+            flb_cmake_flags+=(-DLIBCO_TESTS_ASAN=ON -DCMAKE_C_FLAGS=-fsanitize=address
+              '-DCMAKE_EXE_LINKER_FLAGS=-fsanitize=address -sALLOW_MEMORY_GROWTH=1')
+          fi
+          emcmake cmake -S . -B build-wasm -DFLB_WASM_BROWSER=ON \
+            -DFLB_DEBUG=OFF -DFLB_RELEASE=OFF -DCMAKE_BUILD_TYPE=MinSizeRel "${flb_cmake_flags[@]}"
+          cmake --build build-wasm -j8
+          ctest --test-dir build-wasm --output-on-failure
+          node --test tests/wasm/sdk_unit_test.mjs tests/wasm/sdk_worker_unit_test.mjs
+      - name: Test browser lifecycle and startup failures
+        shell: bash
+        run: |
+          set -euo pipefail
+          python3 -m venv "$RUNNER_TEMP/flb-browser-tests"
+          "$RUNNER_TEMP/flb-browser-tests/bin/python" -m pip install playwright==1.62.0
+          "$RUNNER_TEMP/flb-browser-tests/bin/python" -m playwright install --with-deps chromium
+          flb_browser=$("$RUNNER_TEMP/flb-browser-tests/bin/python" -c \
+            'from playwright.sync_api import sync_playwright; p = sync_playwright().start(); print(p.chromium.executable_path); p.stop()')
+          "$RUNNER_TEMP/flb-browser-tests/bin/python" tests/wasm/sdk_test.py \
+            --build-dir build-wasm --browser "$flb_browser"
+          "$RUNNER_TEMP/flb-browser-tests/bin/python" tests/wasm/sdk_test.py \
+            --build-dir build-wasm --browser "$flb_browser" --pthread-only
+          "$RUNNER_TEMP/flb-browser-tests/bin/python" tests/wasm/browser_page_test.py \
+            --build-dir build-wasm --browser "$flb_browser"

.gitignore

@@ -8,6 +8,10 @@ lib/jemalloc
 tests/internal/flb_tests_internal.h
 tests/runtime/flb_tests_runtime.h
 build/*
+/build-wasm/
+/build-wasm-asan/
+/build-wasm-release/
+/build-wasm-*/
 include/fluent-bit/flb_info.h
 include/fluent-bit/flb_plugins.h
 include/fluent-bit/flb_version.h

CMakeLists.txt

@@ -1,6 +1,13 @@
 cmake_minimum_required(VERSION 3.12)
 project(fluent-bit C)
 
+option(FLB_WASM_BROWSER "Enable the experimental Emscripten browser build profile" OFF)
+if(FLB_WASM_BROWSER)
+  include(cmake/wasm-browser.cmake)
+elseif(CMAKE_SYSTEM_NAME STREQUAL "Emscripten")
+  message(FATAL_ERROR "Emscripten builds require -DFLB_WASM_BROWSER=ON")
+endif()
+
 include(CheckCCompilerFlag)
 
 # CMP0069 ensures that LTO is enabled for all compilers
@@ -130,9 +137,15 @@ endif()
 INCLUDE(TestBigEndian)
 include(GNUInstallDirs)
 include(ExternalProject)
-include(cmake/FindJournald.cmake)
+if(NOT FLB_WASM_BROWSER)
+  include(cmake/FindJournald.cmake)
+endif()
 include(cmake/FindMonkey.cmake)
 include(cmake/macros.cmake)
+if(FLB_WASM_BROWSER)
+  include(cmake/wasm-openssl.cmake)
+  include(cmake/wasm-filesystem.cmake)
+endif()
 include(cmake/platform_feature_checks.cmake)
 set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/sanitizers-cmake/cmake" ${CMAKE_MODULE_PATH})
 find_package(Sanitizers)
@@ -204,6 +217,10 @@ option(FLB_SQLDB               "Enable SQL embedded DB"       Yes)
 option(FLB_HTTP_SERVER         "Enable HTTP Server"           Yes)
 option(FLB_BACKTRACE           "Enable stacktrace support"    Yes)
 option(FLB_LUAJIT              "Enable Lua Scripting support" Yes)
+option(FLB_WASM_LUA            "Enable portable Lua in the browser build" No)
+if(FLB_WASM_LUA AND NOT FLB_WASM_BROWSER)
+  message(FATAL_ERROR "FLB_WASM_LUA requires FLB_WASM_BROWSER=ON")
+endif()
 option(FLB_RECORD_ACCESSOR     "Enable record accessor"       Yes)
 option(FLB_SIGNV4              "Enable AWS Signv4 support"    Yes)
 option(FLB_AWS                 "Enable AWS support"           Yes)
@@ -1179,7 +1196,11 @@ else()
 endif ()
 
 # Configuration file YAML format support
-if(FLB_CONFIG_YAML)
+if(FLB_CONFIG_YAML AND FLB_WASM_BROWSER)
+  include(cmake/wasm-libyaml.cmake)
+  set(FLB_HAVE_LIBYAML 1)
+  FLB_DEFINITION(FLB_HAVE_LIBYAML)
+elseif(FLB_CONFIG_YAML)
   find_package(PkgConfig)
   # libyaml's corresponding pkg-config file is yaml-0.1.pc.
   # We should search it first.
@@ -1387,6 +1408,10 @@ if(FLB_LUAJIT)
     include(cmake/luajit.cmake)
   endif()
   FLB_DEFINITION(FLB_HAVE_LUAJIT)
+  FLB_DEFINITION(FLB_HAVE_LUA)
+elseif(FLB_WASM_LUA)
+  include(cmake/wasm-lua.cmake)
+  FLB_DEFINITION(FLB_HAVE_LUA)
 endif()
 
 # PostgreSQL
@@ -1588,6 +1613,12 @@ if(FLB_TESTS_INTERNAL)
   add_subdirectory(tests/internal/)
 endif()
 
+if(FLB_WASM_BROWSER)
+  enable_testing()
+  include(cmake/wasm-sdk.cmake)
+  add_subdirectory(tests/wasm)
+endif()
+
 # Installer Generation (Cpack)
 # ============================
 

cmake/headers.cmake

@@ -45,6 +45,11 @@ include_directories(
   ${FLB_PATH_ROOT_BINARY_DIR}/lib/monkey/include/monkey/
   )
 
+if(NOT FLB_HTTP_SERVER)
+  # mk_core is configured directly under the Fluent Bit project in this mode.
+  include_directories(${FLB_PATH_ROOT_BINARY_DIR}/include/monkey)
+endif()
+
 if(FLB_YYJSON)
   include_directories(${FLB_PATH_ROOT_SOURCE}/${FLB_PATH_LIB_YYJSON}/src)
 endif()

cmake/plugins_options.cmake

@@ -1,5 +1,16 @@
 macro(DEFINE_OPTION option_name description default_value)
     set(temp_value ${default_value})
+    if(FLB_WASM_BROWSER)
+        list(FIND FLB_WASM_BROWSER_PLUGINS "${option_name}" wasm_plugin_index)
+        if(wasm_plugin_index EQUAL -1)
+            if(${option_name})
+                message(FATAL_ERROR "${option_name} is not supported by the browser build profile")
+            endif()
+            set(temp_value OFF)
+        else()
+            set(temp_value ON)
+        endif()
+    endif()
     if(FLB_MINIMAL)
         set(temp_value OFF)
     endif()

cmake/wasm-browser.cmake

@@ -0,0 +1,109 @@
+# Experimental browser cross-compilation profile, distinct from FLB_WASM
+# (which embeds WAMR in a native Fluent Bit process).
+if(NOT CMAKE_SYSTEM_NAME STREQUAL "Emscripten")
+  message(FATAL_ERROR
+    "FLB_WASM_BROWSER requires Emscripten. Use: "
+    "emcmake cmake -S . -B build-wasm -DFLB_WASM_BROWSER=ON")
+endif()
+
+# Keep compiler and runtime shims in lockstep. Check before downloading or
+# compiling dependencies; a different SDK needs the complete validation matrix.
+file(STRINGS "${CMAKE_CURRENT_LIST_DIR}/../.emscripten-version" FLB_WASM_EMSCRIPTEN_VERSION LIMIT_COUNT 1)
+if(NOT EMSCRIPTEN_VERSION VERSION_EQUAL FLB_WASM_EMSCRIPTEN_VERSION)
+  message(FATAL_ERROR
+    "FLB_WASM_BROWSER requires Emscripten ${FLB_WASM_EMSCRIPTEN_VERSION} (found '${EMSCRIPTEN_VERSION}')")
+endif()
+
+# Reject incompatible overrides instead of silently producing a native or
+# browser-incompatible build. These are profile constraints, not defaults.
+foreach(option_name
+    FLB_ALL FLB_DEV FLB_BINARY FLB_SHARED_LIB FLB_EXAMPLES
+    FLB_TESTS_RUNTIME FLB_TESTS_INTERNAL FLB_TESTS_INTERNAL_FUZZ
+    FLB_TESTS_OSSFUZZ FLB_BENCHMARKS FLB_COVERAGE
+    FLB_TLS FLB_AWS FLB_SIGNV4 FLB_AWS_ERROR_REPORTER
+    FLB_HTTP_SERVER FLB_CHUNK_TRACE FLB_SQLDB
+    FLB_JEMALLOC FLB_BACKTRACE FLB_VALGRIND FLB_MTRACE FLB_INOTIFY
+    FLB_LUAJIT FLB_WASM FLB_WAMRC FLB_WASM_STACK_PROTECT
+    FLB_KAFKA FLB_ZIG FLB_RIPSER FLB_USE_RIPSER
+    FLB_PROXY_GO FLB_CUSTOM_CALYPTIA FLB_AVRO_ENCODER FLB_ARROW
+    FLB_UNICODE_ENCODER FLB_USE_SIMDUTF FLB_STATIC_CONF
+    FLB_EVENT_LOOP_EPOLL FLB_EVENT_LOOP_KQUEUE
+    FLB_EVENT_LOOP_SELECT FLB_EVENT_LOOP_LIBEVENT
+    FLB_PREFER_SYSTEM_LIBS FLB_PREFER_SYSTEM_LIB_BACKTRACE
+    FLB_PREFER_SYSTEM_LIB_CARES FLB_PREFER_SYSTEM_LIB_JEMALLOC
+    FLB_PREFER_SYSTEM_LIB_KAFKA FLB_PREFER_SYSTEM_LIB_LUAJIT
+    FLB_PREFER_SYSTEM_LIB_MSGPACK FLB_PREFER_SYSTEM_LIB_NGHTTP2
+    FLB_PREFER_SYSTEM_LIB_SQLITE FLB_PREFER_SYSTEM_LIB_ZSTD)
+  if(${option_name})
+    message(FATAL_ERROR "${option_name} is not supported by the browser build profile")
+  endif()
+  set(${option_name} OFF CACHE BOOL "Disabled by the browser build profile")
+endforeach()
+
+set(FLB_SIMD "Off" CACHE STRING "Enable SIMD support (On, Off, Auto)")
+set(FLB_IPO "Off" CACHE STRING "Build with interprocedural optimization")
+set(FLB_SECURITY "Off" CACHE STRING "Build with security optimizations")
+set(FLB_EVENT_LOOP_POLL ON CACHE BOOL "Enable poll(2) event loop backend")
+set(CIO_BACKEND_FILESYSTEM ON CACHE BOOL "Enable browser virtual filesystem chunks" FORCE)
+set(FLB_CONFIG_YAML ON CACHE BOOL "Enable browser YAML configuration")
+set(FLB_WASM_LUA ON CACHE BOOL "Enable portable Lua in the browser build")
+set(LIBCO_TESTS ON CACHE BOOL "Build coroutine tests")
+
+# Emscripten supplies linkable stubs for several unavailable OS services.
+# Compilation probes cannot establish that these services work in a browser.
+foreach(feature FLB_HAVE_FORK FLB_HAVE_SYS_WAIT_H FLB_HAVE_ACCEPT4 FLB_HAVE_UNIX_SOCKET)
+  set(${feature} OFF CACHE INTERNAL "Unavailable in browsers" FORCE)
+endforeach()
+
+# Candidate local-data plugins and the Fetch-backed HTTP output. Inclusion is a compilation
+# target, not a claim of browser runtime support. Keep new plugins opt-in.
+set(FLB_WASM_BROWSER_PLUGINS
+  FLB_IN_DUMMY
+  FLB_IN_EMITTER
+  FLB_IN_EVENT_TYPE
+  FLB_IN_FLUENTBIT_METRICS
+  FLB_IN_FLUENTBIT_LOGS
+  FLB_IN_LIB
+  FLB_IN_RANDOM
+  FLB_IN_STORAGE_BACKLOG
+  FLB_PROCESSOR_CONTENT_MODIFIER
+  FLB_PROCESSOR_CUMULATIVE_TO_DELTA
+  FLB_PROCESSOR_LABELS
+  FLB_PROCESSOR_METRICS_SELECTOR
+  FLB_PROCESSOR_OPENTELEMETRY_ENVELOPE
+  FLB_PROCESSOR_SQL
+  FLB_PROCESSOR_SAMPLING
+  FLB_FILTER_ALTER_SIZE
+  FLB_FILTER_EXPECT
+  FLB_FILTER_GREP
+  FLB_FILTER_LOG_TO_METRICS
+  FLB_FILTER_LUA
+  FLB_FILTER_MODIFY
+  FLB_FILTER_MULTILINE
+  FLB_FILTER_NEST
+  FLB_FILTER_PARSER
+  FLB_FILTER_RECORD_MODIFIER
+  FLB_FILTER_REWRITE_TAG
+  FLB_FILTER_STDOUT
+  FLB_FILTER_THROTTLE
+  FLB_FILTER_TYPE_CONVERTER
+  FLB_OUT_COUNTER
+  FLB_OUT_FLOWCOUNTER
+  FLB_OUT_HTTP
+  FLB_OUT_LOKI
+  FLB_OUT_OPENTELEMETRY
+  FLB_OUT_LIB
+  FLB_OUT_NULL
+  FLB_OUT_STDOUT)
+
+if(NOT FLB_WASM_LUA)
+  list(REMOVE_ITEM FLB_WASM_BROWSER_PLUGINS FLB_FILTER_LUA)
+endif()
+
+# Compile every dependency with the same threading ABI. The embedding API
+# must dispatch engine operations to a worker, never the browser main thread.
+add_compile_options(-pthread)
+set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -pthread")
+set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -pthread")
+
+message(STATUS "Experimental browser profile: worker runtime, virtual filesystem, and Fetch HTTPS output")

cmake/wasm-filesystem.cmake

@@ -0,0 +1,6 @@
+# Browser filesystem glue, kept outside the portable ChunkIO library.
+add_library(flb-wasm-filesystem INTERFACE)
+target_link_libraries(flb-wasm-filesystem INTERFACE
+  "-lidbfs.js" "--pre-js=${PROJECT_SOURCE_DIR}/src/wasm/storage.js")
+set_property(TARGET flb-wasm-filesystem PROPERTY INTERFACE_LINK_DEPENDS
+  "${PROJECT_SOURCE_DIR}/src/wasm/storage.js")

cmake/wasm-libyaml.cmake

@@ -0,0 +1,25 @@
+# Keep host pkg-config and native libraries out of the browser build.
+set(FLB_WASM_YAML_PREFIX "${CMAKE_BINARY_DIR}/wasm-yaml")
+file(MAKE_DIRECTORY "${FLB_WASM_YAML_PREFIX}/include")
+
+ExternalProject_Add(flb-wasm-libyaml
+  EXCLUDE_FROM_ALL TRUE
+  URL https://codeload.github.com/yaml/libyaml/tar.gz/refs/tags/0.2.5
+  URL_HASH SHA256=fa240dbf262be053f3898006d502d514936c818e422afdcf33921c63bed9bf2e
+  PREFIX "${CMAKE_BINARY_DIR}/wasm-libyaml"
+  CMAKE_ARGS
+    "-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}"
+    "-DCMAKE_INSTALL_PREFIX=${FLB_WASM_YAML_PREFIX}"
+    "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
+    "-DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} -pthread"
+    -DCMAKE_POLICY_VERSION_MINIMUM=3.5
+    -DBUILD_SHARED_LIBS=OFF -DBUILD_TESTING=OFF
+  BUILD_BYPRODUCTS "${FLB_WASM_YAML_PREFIX}/lib/libyaml.a"
+  LOG_DOWNLOAD ON LOG_CONFIGURE ON LOG_BUILD ON LOG_INSTALL ON)
+
+add_library(yaml STATIC IMPORTED GLOBAL)
+set_target_properties(yaml PROPERTIES
+  IMPORTED_LOCATION "${FLB_WASM_YAML_PREFIX}/lib/libyaml.a"
+  INTERFACE_INCLUDE_DIRECTORIES "${FLB_WASM_YAML_PREFIX}/include"
+  INTERFACE_COMPILE_DEFINITIONS YAML_DECLARE_STATIC)
+add_dependencies(yaml flb-wasm-libyaml)

cmake/wasm-lua-source/CMakeLists.txt

@@ -0,0 +1,13 @@
+cmake_minimum_required(VERSION 3.12)
+project(flb_portable_lua C)
+
+if(NOT EXISTS "${LUA_SOURCE_DIR}/src/lapi.c")
+  message(FATAL_ERROR "LUA_SOURCE_DIR must contain the pinned Lua source release")
+endif()
+file(GLOB lua_sources "${LUA_SOURCE_DIR}/src/*.c")
+list(REMOVE_ITEM lua_sources "${LUA_SOURCE_DIR}/src/lua.c" "${LUA_SOURCE_DIR}/src/luac.c")
+add_library(flb-lua STATIC ${lua_sources})
+target_include_directories(flb-lua PUBLIC "${LUA_SOURCE_DIR}/src")
+install(TARGETS flb-lua ARCHIVE DESTINATION lib)
+install(FILES "${LUA_SOURCE_DIR}/src/lua.h" "${LUA_SOURCE_DIR}/src/luaconf.h"
+  "${LUA_SOURCE_DIR}/src/lauxlib.h" "${LUA_SOURCE_DIR}/src/lualib.h" DESTINATION include)

cmake/wasm-lua.cmake

@@ -0,0 +1,29 @@
+# A portable interpreter, not LuaJIT: do not link host libraries or patch lib/.
+set(FLB_WASM_LUA_PREFIX "${CMAKE_BINARY_DIR}/wasm-lua")
+file(MAKE_DIRECTORY "${FLB_WASM_LUA_PREFIX}/include")
+
+ExternalProject_Add(flb-wasm-lua-source
+  EXCLUDE_FROM_ALL TRUE
+  URL https://www.lua.org/ftp/lua-5.4.9.tar.gz
+  URL_HASH SHA256=2335b6c582a52654f94612bf10d2f4672805d05329aa6568b1d8cd9e5c6fb8e6
+  PREFIX "${CMAKE_BINARY_DIR}/wasm-lua-source"
+  CONFIGURE_COMMAND ${CMAKE_COMMAND}
+    -S "${CMAKE_CURRENT_LIST_DIR}/wasm-lua-source" -B <BINARY_DIR>
+    "-DLUA_SOURCE_DIR=<SOURCE_DIR>"
+    "-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}"
+    "-DCMAKE_INSTALL_PREFIX=${FLB_WASM_LUA_PREFIX}"
+    "-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}"
+    "-DCMAKE_C_FLAGS=${CMAKE_C_FLAGS} -pthread"
+  BUILD_COMMAND ${CMAKE_COMMAND} --build <BINARY_DIR> --parallel 8
+  INSTALL_COMMAND ${CMAKE_COMMAND} --install <BINARY_DIR>
+  BUILD_BYPRODUCTS "${FLB_WASM_LUA_PREFIX}/lib/libflb-lua.a"
+  LOG_DOWNLOAD ON LOG_CONFIGURE ON LOG_BUILD ON LOG_INSTALL ON)
+
+add_library(flb-wasm-lua-static STATIC IMPORTED GLOBAL)
+set_target_properties(flb-wasm-lua-static PROPERTIES
+  IMPORTED_LOCATION "${FLB_WASM_LUA_PREFIX}/lib/libflb-lua.a"
+  INTERFACE_INCLUDE_DIRECTORIES "${FLB_WASM_LUA_PREFIX}/include")
+add_dependencies(flb-wasm-lua-static flb-wasm-lua-source)
+include_directories("${FLB_WASM_LUA_PREFIX}/include")
+set(LUAJIT_LIBRARIES flb-wasm-lua-static)
+message(STATUS "Browser Lua: portable Lua 5.4.9 interpreter; no LuaJIT/FFI")