From 608fcc66d841379f1cdbc1a0628c087bdbf705ea Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 31 Aug 2026 21:26:29 -0600 Subject: [PATCH 1/3] bin: generate YAML configuration from CLI options Signed-off-by: Eduardo Silva --- src/fluent-bit.c | 331 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 330 insertions(+), 1 deletion(-) diff --git a/src/fluent-bit.c b/src/fluent-bit.c index b74d68313f9..97f89539929 100644 --- a/src/fluent-bit.c +++ b/src/fluent-bit.c @@ -136,6 +136,8 @@ static void flb_help(int rc, struct flb_config *config) print_opt(" --supervisor", "run under a supervising parent process"); #endif print_opt("-D, --dry-run", "dry run"); + print_opt("-G, --generate-config", + "generate YAML configuration from command-line options"); print_opt_i("-f, --flush=SECONDS", "flush timeout in seconds", FLB_CONFIG_FLUSH_SECS); print_opt("-C, --custom=CUSTOM", "enable a custom plugin"); @@ -993,10 +995,250 @@ static int parse_trace_pipeline(flb_ctx_t *ctx, const char *pipeline, char **tra } #endif +static void yaml_write_indent(FILE *stream, int indentation) +{ + int index; + + for (index = 0; index < indentation; index++) { + fputc(' ', stream); + } +} + +static void yaml_write_quoted_string(FILE *stream, const char *value) +{ + unsigned char character; + + fputc('"', stream); + + while (*value != '\0') { + character = (unsigned char) *value; + + switch (character) { + case '\\': + fputs("\\\\", stream); + break; + case '"': + fputs("\\\"", stream); + break; + case '\n': + fputs("\\n", stream); + break; + case '\r': + fputs("\\r", stream); + break; + case '\t': + fputs("\\t", stream); + break; + case '\b': + fputs("\\b", stream); + break; + case '\f': + fputs("\\f", stream); + break; + default: + if (character < 0x20 || character == 0x7f) { + fprintf(stream, "\\x%02x", character); + } + else { + fputc(character, stream); + } + } + + value++; + } + + fputc('"', stream); +} + +static int yaml_write_properties(FILE *stream, struct cfl_kvlist *properties, + int indentation, int sequence_entry) +{ + int first = FLB_TRUE; + struct cfl_list *head; + struct cfl_kvpair *property; + + if (cfl_list_size(&properties->list) == 0) { + yaml_write_indent(stream, indentation); + fputs("- {}\n", stream); + return 0; + } + + cfl_list_foreach(head, &properties->list) { + property = cfl_list_entry(head, struct cfl_kvpair, _head); + if (property->val->type != CFL_VARIANT_STRING) { + return -1; + } + + yaml_write_indent(stream, indentation); + if (sequence_entry == FLB_TRUE && first == FLB_TRUE) { + fputs("- ", stream); + } + else if (sequence_entry == FLB_TRUE) { + fputs(" ", stream); + } + + yaml_write_quoted_string(stream, property->key); + fputs(": ", stream); + yaml_write_quoted_string(stream, property->val->data.as_string); + fputc('\n', stream); + first = FLB_FALSE; + } + + return 0; +} + +static int yaml_write_section_list(FILE *stream, const char *name, + struct mk_list *sections, int indentation) +{ + int result; + struct mk_list *head; + struct flb_cf_section *section; + + if (mk_list_size(sections) == 0) { + return 0; + } + + yaml_write_indent(stream, indentation); + fprintf(stream, "%s:\n", name); + + mk_list_foreach(head, sections) { + section = mk_list_entry(head, struct flb_cf_section, _head_section); + result = yaml_write_properties(stream, section->properties, + indentation + 2, FLB_TRUE); + if (result != 0) { + return result; + } + } + + return 0; +} + +static int yaml_write_string_list(FILE *stream, const char *name, + struct mk_list *entries) +{ + struct mk_list *head; + struct flb_slist_entry *entry; + + if (mk_list_size(entries) == 0) { + return 0; + } + + fprintf(stream, "%s:\n", name); + mk_list_foreach(head, entries) { + entry = mk_list_entry(head, struct flb_slist_entry, _head); + fputs(" - ", stream); + yaml_write_quoted_string(stream, entry->str); + fputc('\n', stream); + } + + return 0; +} + +#ifdef FLB_HAVE_STREAM_PROCESSOR +static int yaml_write_stream_processor_tasks(FILE *stream, struct mk_list *tasks) +{ + int index = 0; + struct mk_list *head; + struct flb_slist_entry *entry; + + if (mk_list_size(tasks) == 0) { + return 0; + } + + fputs("stream_processor:\n", stream); + mk_list_foreach(head, tasks) { + entry = mk_list_entry(head, struct flb_slist_entry, _head); + fprintf(stream, " - name: \"flb-console:%d\"\n", index++); + fputs(" exec: ", stream); + yaml_write_quoted_string(stream, entry->str); + fputc('\n', stream); + } + + return 0; +} +#endif + +static int generate_yaml_config(FILE *stream, struct flb_cf *cf, + struct flb_config *config) +{ + int result; + int has_pipeline; + int has_content; + + has_content = FLB_FALSE; + + if (mk_list_size(&config->external_plugins) > 0) { + result = yaml_write_string_list(stream, "plugins", &config->external_plugins); + if (result != 0) { + return result; + } + has_content = FLB_TRUE; + } + +#ifdef FLB_HAVE_STREAM_PROCESSOR + if (mk_list_size(&config->stream_processor_tasks) > 0) { + result = yaml_write_stream_processor_tasks(stream, + &config->stream_processor_tasks); + if (result != 0) { + return result; + } + has_content = FLB_TRUE; + } +#endif + + if (cfl_list_size(&cf->service->properties->list) > 0) { + fputs("service:\n", stream); + result = yaml_write_properties(stream, cf->service->properties, 2, FLB_FALSE); + if (result != 0) { + return result; + } + has_content = FLB_TRUE; + } + + if (mk_list_size(&cf->customs) > 0) { + result = yaml_write_section_list(stream, "customs", &cf->customs, 0); + if (result != 0) { + return result; + } + has_content = FLB_TRUE; + } + + has_pipeline = mk_list_size(&cf->inputs) > 0 || + mk_list_size(&cf->filters) > 0 || + mk_list_size(&cf->outputs) > 0; + if (has_pipeline == FLB_TRUE) { + fputs("pipeline:\n", stream); + has_content = FLB_TRUE; + + result = yaml_write_section_list(stream, "inputs", &cf->inputs, 2); + if (result != 0) { + return result; + } + + result = yaml_write_section_list(stream, "filters", &cf->filters, 2); + if (result != 0) { + return result; + } + + result = yaml_write_section_list(stream, "outputs", &cf->outputs, 2); + if (result != 0) { + return result; + } + } + + if (has_content == FLB_FALSE) { + fputs("{}\n", stream); + } + + return ferror(stream) ? -1 : 0; +} + static int flb_main_run(int argc, char **argv) { int opt; int ret; + int generate_config = FLB_FALSE; + int log_level_set = FLB_FALSE; flb_sds_t json; /* handle plugin properties: -1 = none, 0 = input, 1 = output */ @@ -1035,6 +1277,7 @@ static int flb_main_run(int argc, char **argv) char *trace_input = NULL; char *trace_output = NULL; struct mk_list *trace_props = NULL; + int trace_config_set = FLB_FALSE; #endif /* Setup long-options */ @@ -1045,6 +1288,7 @@ static int flb_main_run(int argc, char **argv) { "daemon", no_argument , NULL, 'd' }, #endif { "dry-run", no_argument , NULL, 'D' }, + { "generate-config", no_argument , NULL, 'G' }, { "flush", required_argument, NULL, 'f' }, { "http", no_argument , NULL, 'H' }, #ifndef FLB_SYSTEM_WINDOWS @@ -1130,7 +1374,7 @@ static int flb_main_run(int argc, char **argv) /* Parse the command line options */ while ((opt = getopt_long(argc, argv, "b:c:dDf:C:i:m:M:o:R:r:F:p:e:" - "t:T:l:vw:qVhJL:HP:s:SWYZ", + "t:T:l:vw:qVhJGL:HP:s:SWYZ", long_opts, NULL)) != -1) { switch (opt) { @@ -1151,6 +1395,9 @@ static int flb_main_run(int argc, char **argv) case 'D': config->dry_run = FLB_TRUE; break; + case 'G': + generate_config = FLB_TRUE; + break; case 'e': ret = flb_plugin_load_router(optarg, config); if (ret == -1) { @@ -1288,12 +1535,14 @@ static int flb_main_run(int argc, char **argv) exit(EXIT_SUCCESS); case 'v': config->verbose++; + log_level_set = FLB_TRUE; break; case 'w': config->workdir = flb_strdup(optarg); break; case 'q': config->verbose = FLB_LOG_OFF; + log_level_set = FLB_TRUE; break; case 's': flb_cf_section_property_add(cf_opts, service->properties, FLB_CONF_STR_CORO_STACK_SIZE, 0, optarg, 0); @@ -1317,21 +1566,25 @@ static int flb_main_run(int argc, char **argv) flb_cf_section_property_add(cf_opts, service->properties, FLB_CONF_STR_ENABLE_CHUNK_TRACE, 0, "on", 0); break; case FLB_LONG_TRACE: + trace_config_set = FLB_TRUE; parse_trace_pipeline(ctx, optarg, &trace_input, &trace_output, &trace_props); break; case FLB_LONG_TRACE_INPUT: + trace_config_set = FLB_TRUE; if (trace_input != NULL) { flb_free(trace_input); } trace_input = flb_strdup(optarg); break; case FLB_LONG_TRACE_OUTPUT: + trace_config_set = FLB_TRUE; if (trace_output != NULL) { flb_free(trace_output); } trace_output = flb_strdup(optarg); break; case FLB_LONG_TRACE_OUTPUT_PROPERTY: + trace_config_set = FLB_TRUE; if (trace_props == NULL) { trace_props = flb_calloc(1, sizeof(struct mk_list)); flb_kv_init(trace_props); @@ -1348,6 +1601,82 @@ static int flb_main_run(int argc, char **argv) } #endif /* !FLB_HAVE_STATIC_CONF */ + if (generate_config == FLB_TRUE) { + ret = 0; + + if (cfg_file != NULL) { + fprintf(stderr, "--generate-config cannot be used with --config\n"); + ret = -1; + } + else if (config->workdir != NULL) { + fprintf(stderr, "--generate-config cannot preserve --workdir\n"); + ret = -1; + } +#ifdef FLB_HAVE_CHUNK_TRACE + else if (trace_config_set == FLB_TRUE) { + fprintf(stderr, + "--generate-config cannot preserve startup trace options\n"); + ret = -1; + } +#endif + else if (log_level_set == FLB_TRUE) { + if (config->verbose == FLB_LOG_OFF) { + flb_cf_section_property_add(cf_opts, service->properties, + FLB_CONF_STR_LOGLEVEL, 0, "off", 0); + } + else if (config->verbose == FLB_LOG_ERROR) { + flb_cf_section_property_add(cf_opts, service->properties, + FLB_CONF_STR_LOGLEVEL, 0, "error", 0); + } + else if (config->verbose == FLB_LOG_WARN) { + flb_cf_section_property_add(cf_opts, service->properties, + FLB_CONF_STR_LOGLEVEL, 0, "warn", 0); + } + else if (config->verbose == FLB_LOG_INFO) { + flb_cf_section_property_add(cf_opts, service->properties, + FLB_CONF_STR_LOGLEVEL, 0, "info", 0); + } + else if (config->verbose == FLB_LOG_DEBUG) { + flb_cf_section_property_add(cf_opts, service->properties, + FLB_CONF_STR_LOGLEVEL, 0, "debug", 0); + } + else { + flb_cf_section_property_add(cf_opts, service->properties, + FLB_CONF_STR_LOGLEVEL, 0, "trace", 0); + } + } + + if (ret == 0) { + ret = generate_yaml_config(stdout, cf_opts, config); + if (ret != 0) { + fprintf(stderr, "failed to generate YAML configuration\n"); + } + } + +#ifdef FLB_HAVE_CHUNK_TRACE + if (trace_input != NULL) { + flb_free(trace_input); + } + if (trace_output != NULL) { + flb_free(trace_output); + } + if (trace_props != NULL) { + flb_kv_release(trace_props); + flb_free(trace_props); + } +#endif + + flb_free(cfg_file); + flb_cf_destroy(cf_opts); + flb_destroy(ctx); + + if (ret != 0) { + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; + } + set_log_level_from_env(config); if (config->verbose != FLB_LOG_OFF) { From 7fb7f6f467c2aee3d9f7247d82590b08a746e674 Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 31 Aug 2026 21:26:34 -0600 Subject: [PATCH 2/3] tests: runtime_shell: cover YAML config generation Signed-off-by: Eduardo Silva --- tests/runtime_shell/CMakeLists.txt | 1 + tests/runtime_shell/generate_config.sh | 37 ++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) create mode 100755 tests/runtime_shell/generate_config.sh diff --git a/tests/runtime_shell/CMakeLists.txt b/tests/runtime_shell/CMakeLists.txt index 34cacd1c5ff..96c3df73856 100644 --- a/tests/runtime_shell/CMakeLists.txt +++ b/tests/runtime_shell/CMakeLists.txt @@ -73,6 +73,7 @@ else() set(UNIT_TESTS_SH custom_calyptia.sh dry_run_invalid_property.sh + generate_config.sh in_dummy_expect.sh in_tail_expect.sh in_http_tls_expect.sh diff --git a/tests/runtime_shell/generate_config.sh b/tests/runtime_shell/generate_config.sh new file mode 100755 index 00000000000..5de59878ab2 --- /dev/null +++ b/tests/runtime_shell/generate_config.sh @@ -0,0 +1,37 @@ +#!/bin/sh + +test_generate_config() { + output_file="${TMPDIR:-/tmp}/fluent-bit-generate-config-$$.yaml" + error_file="${TMPDIR:-/tmp}/fluent-bit-generate-config-$$.err" + check_file="${TMPDIR:-/tmp}/fluent-bit-generate-config-check-$$.out" + + "$FLB_BIN" -f 2.5 -i dummy -p 'dummy={"message":"a:b # c"}' \ + -t 'generated.*' -o stdout -p 'format=json_lines' \ + --generate-config > "$output_file" 2> "$error_file" + result=$? + + assertEquals "configuration generation should succeed" 0 "$result" + assertTrue "generated configuration should not write to stderr" \ + "[ ! -s '$error_file' ]" + assertTrue "service settings should be generated" \ + "grep -Fq '\"flush\": \"2.5\"' '$output_file'" + assertTrue "input plugin should be generated" \ + "grep -Fq '\"name\": \"dummy\"' '$output_file'" + assertTrue "special YAML characters should be quoted" \ + "grep -Fq '\"dummy\": \"{\\\"message\\\":\\\"a:b # c\\\"}\"' '$output_file'" + assertTrue "tag should be generated" \ + "grep -Fq '\"tag\": \"generated.*\"' '$output_file'" + assertTrue "output property should be generated" \ + "grep -Fq '\"format\": \"json_lines\"' '$output_file'" + + "$FLB_BIN" --dry-run -c "$output_file" > "$check_file" 2>&1 + result=$? + + assertEquals "generated YAML configuration should pass validation" 0 "$result" + assertTrue "generated YAML validation should report success" \ + "grep -Fq 'configuration test is successful' '$check_file'" + + rm -f "$output_file" "$error_file" "$check_file" +} + +. "$FLB_RUNTIME_SHELL_PATH/runtime_shell.env" From cf8332c41890deae501cd406fc373a189e225bee Mon Sep 17 00:00:00 2001 From: Eduardo Silva Date: Mon, 31 Aug 2026 21:26:34 -0600 Subject: [PATCH 3/3] tests: integration: cover CLI config generation Signed-off-by: Eduardo Silva --- .../tests/test_cli_config_generation.py | 236 ++++++++++++++++++ 1 file changed, 236 insertions(+) create mode 100644 tests/integration/scenarios/cli_config_generation/tests/test_cli_config_generation.py diff --git a/tests/integration/scenarios/cli_config_generation/tests/test_cli_config_generation.py b/tests/integration/scenarios/cli_config_generation/tests/test_cli_config_generation.py new file mode 100644 index 00000000000..c1b536f8b7e --- /dev/null +++ b/tests/integration/scenarios/cli_config_generation/tests/test_cli_config_generation.py @@ -0,0 +1,236 @@ +import os +import subprocess +from pathlib import Path + +import yaml + +from utils.fluent_bit_manager import _default_binary_path +from utils.valgrind import assert_valgrind_clean + + +BINARY = os.environ.get("FLUENT_BIT_BINARY") or _default_binary_path() +REPO_ROOT = Path(__file__).resolve().parents[5] +VALGRIND = bool(os.environ.get("VALGRIND")) +VALGRIND_STRICT = bool(os.environ.get("VALGRIND_STRICT")) + + +def _run_fluent_bit(arguments, tmp_path, name, timeout=15): + command = [BINARY, *arguments] + valgrind_log = tmp_path / f"valgrind-{name}.log" + + if VALGRIND: + command = [ + "valgrind", + f"--log-file={valgrind_log}", + "--leak-check=full", + "--show-leak-kinds=all", + *command, + ] + timeout *= 6 + + result = subprocess.run( + command, + capture_output=True, + text=True, + timeout=timeout, + check=False, + ) + + if VALGRIND_STRICT: + assert_valgrind_clean(valgrind_log) + + return result + + +def _supported_options(): + result = subprocess.run( + [BINARY, "--help"], + capture_output=True, + text=True, + timeout=15, + check=False, + ) + assert result.returncode == 0 + return result.stdout + + +def test_generate_config_preserves_cli_configuration(tmp_path): + help_text = _supported_options() + storage_path = tmp_path / "storage" + log_path = tmp_path / "fluent-bit.log" + parser_path = REPO_ROOT / "conf" / "parsers.conf" + external_plugin = REPO_ROOT / "build" / "test_logs_go.so" + storage_path.mkdir() + + arguments = [ + "-b", str(storage_path), + "-f", "2.5", + "-C", "calyptia", + "-p", "api: key=a:b # c", + "-i", "dummy", + "-p", 'dummy={"message":"cli"}', + "-t", "cli.generated", + "-G", + "-i", "cpu", + "-p", "interval_sec=60", + "-F", "modify", + "-m", "cli.*", + "-p", "add=source cli", + "-o", "stdout", + "-m", "cli.*", + "-p", "format=json_lines", + "-o", "http", + "-m", "other.*", + "-p", "host=localhost", + "-p", "port=9090", + "-R", str(parser_path), + "-l", str(log_path), + "-s", "49152", + "-Y", + "-W", + "--enable-fips", + "-vv", + ] + + if "--daemon" in help_text: + arguments.append("-d") + if "--http" in help_text: + arguments.extend(["-H", "-L", "127.0.0.1", "-P", "2021"]) + if "--enable-chunk-trace" in help_text: + arguments.append("-Z") + if "--sp-task" in help_text: + arguments.extend(["-T", "SELECT * FROM STREAM:dummy.0;"]) + if os.name == "nt": + arguments.extend(["-M", "1024"]) + if external_plugin.exists() and not VALGRIND: + arguments.extend(["-e", str(external_plugin)]) + + result = _run_fluent_bit(arguments, tmp_path, "all-options") + assert result.returncode == 0, result.stderr + assert result.stderr == "" + + generated = yaml.safe_load(result.stdout) + service = generated["service"] + + assert service["storage.path"] == str(storage_path) + assert service["flush"] == "2.5" + assert service["parsers_file"] == str(parser_path) + assert service["log_file"] == str(log_path) + assert service["coro_stack_size"] == "49152" + assert service["hot_reload"] == "on" + assert service["hot_reload.ensure_thread_safety"] == "off" + assert service["security.fips_mode"] == "on" + assert service["log_level"] == "trace" + + if "--daemon" in help_text: + assert service["daemon"] == "on" + if "--http" in help_text: + assert service["http_server"] == "on" + assert service["http_listen"] == "127.0.0.1" + assert service["http_port"] == "2021" + if "--enable-chunk-trace" in help_text: + assert service["enable_chunk_trace"] == "on" + if os.name == "nt": + assert service["windows.maxstdio"] == "1024" + + assert generated["customs"] == [ + {"name": "calyptia", "api: key": "a:b # c"} + ] + assert generated["pipeline"] == { + "inputs": [ + { + "name": "dummy", + "dummy": '{"message":"cli"}', + "tag": "cli.generated", + }, + {"name": "cpu", "interval_sec": "60"}, + ], + "filters": [ + {"name": "modify", "match": "cli.*", "add": "source cli"} + ], + "outputs": [ + {"name": "stdout", "match": "cli.*", "format": "json_lines"}, + { + "name": "http", + "match": "other.*", + "host": "localhost", + "port": "9090", + }, + ], + } + + if "--sp-task" in help_text: + assert generated["stream_processor"] == [ + { + "name": "flb-console:0", + "exec": "SELECT * FROM STREAM:dummy.0;", + } + ] + if external_plugin.exists() and not VALGRIND: + assert generated["plugins"] == [str(external_plugin)] + + +def test_generate_config_preserves_quiet_log_level(tmp_path): + result = _run_fluent_bit( + ["-i", "dummy", "-o", "null", "-q", "--generate-config"], + tmp_path, + "quiet", + ) + + assert result.returncode == 0, result.stderr + generated = yaml.safe_load(result.stdout) + assert generated["service"]["log_level"] == "off" + + +def test_generate_config_rejects_nonportable_cli_options(tmp_path): + config_path = tmp_path / "source.yaml" + config_path.write_text("{}\n", encoding="utf-8") + + cases = [ + (["-c", str(config_path), "-G"], "cannot be used with --config"), + (["-w", str(tmp_path), "-G"], "cannot preserve --workdir"), + ] + + if "--trace-input" in _supported_options(): + cases.append( + (["--trace-input", "dummy.0", "-G"], + "cannot preserve startup trace options") + ) + + for index, (arguments, expected_error) in enumerate(cases): + result = _run_fluent_bit( + arguments, + tmp_path, + f"unsupported-{index}", + ) + assert result.returncode != 0 + assert expected_error in result.stderr + assert result.stdout == "" + + +def test_generated_config_runs_as_a_pipeline(tmp_path): + generation = _run_fluent_bit( + [ + "-i", "dummy", + "-p", "samples=1", + "-t", "generated.runtime", + "-o", "exit", + "-m", "generated.*", + "-p", "flush_count=1", + "-G", + ], + tmp_path, + "runtime-generation", + ) + assert generation.returncode == 0, generation.stderr + + config_path = tmp_path / "generated.yaml" + config_path.write_text(generation.stdout, encoding="utf-8") + + execution = _run_fluent_bit( + ["-c", str(config_path)], + tmp_path, + "runtime-execution", + timeout=15, + ) + assert execution.returncode == 0, execution.stdout + execution.stderr