From 2d0a5d64f41f2ba58425a16622e25fc8d7125835 Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Tue, 10 Mar 2026 10:50:32 -0400 Subject: [PATCH 1/7] Initial Yaml Support --- detection_rules/action_connector.py | 17 +++++ detection_rules/exception.py | 17 +++++ detection_rules/kbwrap.py | 54 +++++++++++++--- detection_rules/main.py | 97 +++++++++++++++++++++++++---- detection_rules/rule.py | 14 +++++ pyproject.toml | 2 +- 6 files changed, 180 insertions(+), 21 deletions(-) diff --git a/detection_rules/action_connector.py b/detection_rules/action_connector.py index 7f98f58e268..8951458ad6c 100644 --- a/detection_rules/action_connector.py +++ b/detection_rules/action_connector.py @@ -11,6 +11,7 @@ from typing import Any import pytoml # type: ignore[reportMissingTypeStubs] +import yaml from marshmallow import EXCLUDE from .config import parse_rules_config @@ -111,6 +112,22 @@ def save_toml(self) -> None: sorted_dict = dict(sorted(contents_dict.items(), key=lambda item: item[0] != "metadata")) pytoml.dump(sorted_dict, f) # type: ignore[reportUnknownMemberType] + def save_yaml(self, path: Path | None = None) -> None: + """Save the action to a YAML file.""" + target_path = path or self.path + if not target_path: + raise ValueError(f"Can't save action for {self.name} without a path") + if target_path.suffix not in (".yaml", ".yml"): + target_path = target_path.with_suffix(".yaml") + with target_path.open("w", encoding="utf-8", newline="\n") as f: + api_format = self.contents.to_api_format() + # If single item, write as dict; if multiple, write as list + content = api_format[0] if len(api_format) == 1 else api_format + output = yaml.safe_dump(content, default_flow_style=False, sort_keys=True) + _ = f.write(output) + if not output.endswith("\n"): + _ = f.write("\n") + def parse_action_connector_results_from_api( results: list[dict[str, Any]], diff --git a/detection_rules/exception.py b/detection_rules/exception.py index f67531a5763..5d1a060b06f 100644 --- a/detection_rules/exception.py +++ b/detection_rules/exception.py @@ -11,6 +11,7 @@ from typing import Any, get_args import pytoml # type: ignore[reportMissingTypeStubs] +import yaml from marshmallow import EXCLUDE, ValidationError, validates_schema from .config import parse_rules_config @@ -227,6 +228,22 @@ def save_toml(self) -> None: sorted_dict = dict(sorted(contents_dict.items(), key=lambda item: item[0] != "metadata")) pytoml.dump(sorted_dict, f) # type: ignore[reportUnknownMemberType] + def save_yaml(self, path: Path | None = None) -> None: + """Save the exception to a YAML file.""" + target_path = path or self.path + if not target_path: + raise ValueError(f"Can't save exception {self.name} without a path") + if target_path.suffix not in (".yaml", ".yml"): + target_path = target_path.with_suffix(".yaml") + with target_path.open("w", encoding="utf-8", newline="\n") as f: + api_format = self.contents.to_api_format() + # If single item, write as dict; if multiple, write as list + content = api_format[0] if len(api_format) == 1 else api_format + output = yaml.safe_dump(content, default_flow_style=False, sort_keys=True) + _ = f.write(output) + if not output.endswith("\n"): + _ = f.write("\n") + def parse_exceptions_results_from_api( results: list[dict[str, Any]], diff --git a/detection_rules/kbwrap.py b/detection_rules/kbwrap.py index ed2a2e1c76c..e02259f4fbf 100644 --- a/detection_rules/kbwrap.py +++ b/detection_rules/kbwrap.py @@ -214,7 +214,7 @@ def _process_imported_items( @kibana_group.command("export-rules") -@click.option("--directory", "-d", required=True, type=Path, help="Directory to export rules to") +@click.option("--directory", "-d", required=False, type=Path, help="Directory to export rules to") @click.option( "--action-connectors-directory", "-acd", required=False, type=Path, help="Directory to export action connectors to" ) @@ -230,6 +230,13 @@ def _process_imported_items( @click.option("--export-action-connectors", "-ac", is_flag=True, help="Include action connectors in export") @click.option("--export-exceptions", "-e", is_flag=True, help="Include exceptions in export") @click.option("--skip-errors", "-s", is_flag=True, help="Skip errors when exporting rules") +@click.option( + "--save-yaml-dir", + "-syd", + required=False, + type=Path, + help="Save exported rules and objects as YAML into this directory using original filenames", +) @click.option("--strip-version", "-sv", is_flag=True, help="Strip the version fields from all rules") @click.option( "--no-tactic-filename", @@ -260,7 +267,7 @@ def _process_imported_items( @click.pass_context def kibana_export_rules( # noqa: PLR0912, PLR0913, PLR0915 ctx: click.Context, - directory: Path, + directory: Path | None, action_connectors_directory: Path | None, exceptions_directory: Path | None, default_author: str, @@ -269,6 +276,7 @@ def kibana_export_rules( # noqa: PLR0912, PLR0913, PLR0915 export_action_connectors: bool = False, export_exceptions: bool = False, skip_errors: bool = False, + save_yaml_dir: Path | None = None, strip_version: bool = False, no_tactic_filename: bool = False, local_creation_date: bool = False, @@ -278,9 +286,20 @@ def kibana_export_rules( # noqa: PLR0912, PLR0913, PLR0915 load_rule_loading: bool = False, ) -> list[TOMLRule]: """Export rules from Kibana.""" + + def _raise_missing_path(message: str) -> None: + raise ValueError(message) + kibana = ctx.obj["kibana"] kibana_include_details = export_exceptions or export_action_connectors or custom_rules_only or export_query + if not directory and not save_yaml_dir: + raise click.UsageError("One of --directory or --save-yaml-dir must be provided.") + + output_directory = directory or save_yaml_dir + if not output_directory: + raise click.UsageError("Unable to determine output directory.") + # Only allow one of rule_id or rule_name if rule_name and rule_id: raise click.UsageError("Cannot use --rule-id and --rule-name together. Please choose one.") @@ -326,7 +345,7 @@ def kibana_export_rules( # noqa: PLR0912, PLR0913, PLR0915 click.echo("Warning: Action Connector export requested, but no Action Connector directory found") if results: - directory.mkdir(parents=True, exist_ok=True) + output_directory.mkdir(parents=True, exist_ok=True) else: click.echo("No rules found to export") return [] @@ -376,7 +395,7 @@ def kibana_export_rules( # noqa: PLR0912, PLR0913, PLR0915 tactic_name = first_tactic if not no_tactic_filename else None # type: ignore[reportUnknownMemberType] rule_name = rulename_to_filename(rule_resource.get("name"), tactic_name=tactic_name) # type: ignore[reportUnknownMemberType] - save_path = directory / f"{rule_name}" + save_path = output_directory / f"{rule_name}" # Get local rule data if load_rule_loading is enabled. If not enabled rules variable will be None. local_rule: dict[str, Any] = params.get("rule", {}) @@ -461,7 +480,14 @@ def kibana_export_rules( # noqa: PLR0912, PLR0913, PLR0915 saved: list[TOMLRule] = [] for rule in exported: try: - rule.save_toml() + if save_yaml_dir: + rule_path = rule.path + if isinstance(rule_path, Path): + rule.save_yaml(save_yaml_dir / rule_path.name) + else: + _raise_missing_path(f"Can't save rule {rule.name} ({rule.id}) without a path") + else: + rule.save_toml() except Exception as e: if skip_errors: print(f"- skipping {rule.contents.data.name} - {type(e).__name__}") @@ -474,7 +500,14 @@ def kibana_export_rules( # noqa: PLR0912, PLR0913, PLR0915 saved_exceptions: list[TOMLException] = [] for exception in exceptions: try: - exception.save_toml() + if save_yaml_dir: + exception_path = exception.path + if isinstance(exception_path, Path): + exception.save_yaml(save_yaml_dir / exception_path.name) + else: + _raise_missing_path(f"Can't save exception {exception.name} without a path") + else: + exception.save_toml() except Exception as e: if skip_errors: print(f"- skipping {exception.rule_name} - {type(e).__name__}") # type: ignore[reportUnknownMemberType] @@ -487,7 +520,10 @@ def kibana_export_rules( # noqa: PLR0912, PLR0913, PLR0915 saved_action_connectors: list[TOMLActionConnector] = [] for action in action_connectors: try: - action.save_toml() + if save_yaml_dir: + action.save_yaml(save_yaml_dir / action.path.name) + else: + action.save_toml() except Exception as e: if skip_errors: print(f"- skipping {action.name} - {type(e).__name__}") @@ -501,11 +537,11 @@ def kibana_export_rules( # noqa: PLR0912, PLR0913, PLR0915 click.echo(f"{len(exported)} rules converted") click.echo(f"{len(exceptions)} exceptions exported") click.echo(f"{len(action_connectors)} action connectors exported") - click.echo(f"{len(saved)} rules saved to {directory}") + click.echo(f"{len(saved)} rules saved to {output_directory}") click.echo(f"{len(saved_exceptions)} exception lists saved to {exceptions_directory}") click.echo(f"{len(saved_action_connectors)} action connectors saved to {action_connectors_directory}") if errors: - err_file = directory / "_errors.txt" + err_file = output_directory / "_errors.txt" _ = err_file.write_text("\n".join(errors)) click.echo(f"{len(errors)} errors saved to {err_file}") diff --git a/detection_rules/main.py b/detection_rules/main.py index 4636fa01826..1d28096edd9 100644 --- a/detection_rules/main.py +++ b/detection_rules/main.py @@ -21,6 +21,7 @@ from semver import Version from .action_connector import ( + TOMLActionConnector, TOMLActionConnectorContents, build_action_connector_objects, parse_action_connector_results_from_api, @@ -28,7 +29,7 @@ from .attack import build_threat_map_entry from .cli_utils import multi_collection, rule_prompt from .config import load_current_package_version, parse_rules_config -from .exception import TOMLExceptionContents, build_exception_objects, parse_exceptions_results_from_api +from .exception import TOMLException, TOMLExceptionContents, build_exception_objects, parse_exceptions_results_from_api from .generic_loader import GenericCollection from .misc import ( add_client, @@ -502,6 +503,60 @@ def view_rule( return rule +def _export_rules_as_yaml( # noqa: PLR0913 + rules: RuleCollection, + yaml_directory: Path, + downgrade_version: definitions.SemVer | None = None, + verbose: bool = True, + skip_unsupported: bool = False, + include_metadata: bool = False, + include_action_connectors: bool = False, + include_exceptions: bool = False, +) -> None: + """Export rules and exceptions into a directory of YAML files.""" + from .rule import downgrade_contents_from_rule + + unsupported: list[str] = [] + + for rule in rules: + contents_override = None + if downgrade_version: + try: + contents_override = downgrade_contents_from_rule( + rule, downgrade_version, include_metadata=include_metadata + ) + except ValueError as e: + if skip_unsupported: + unsupported.append(f"{e}: {rule.id} - {rule.name}") + continue + raise + + rule_path = yaml_directory / rulename_to_filename(rule.name) + rule_path.parent.mkdir(parents=True, exist_ok=True) + rule.save_yaml(rule_path, contents_override=contents_override) + + export_types: list[type[Any]] = [] + if include_exceptions: + export_types.append(TOMLException) + if include_action_connectors: + export_types.append(TOMLActionConnector) + + if export_types: + cl = GenericCollection.default() + for d in cl.items: + if any(isinstance(d, export_type) for export_type in export_types): + save_yaml = getattr(d, "save_yaml", None) + if callable(save_yaml) and isinstance(d.path, Path): + _ = save_yaml(yaml_directory / d.path.name) + + if verbose: + click.echo(f"Exported {len(rules) - len(unsupported)} rules into {yaml_directory}") + + if skip_unsupported and unsupported: + unsupported_str = "\n- ".join(unsupported) + click.echo(f"Skipped {len(unsupported)} unsupported rules: \n- {unsupported_str}") + + def _export_rules( # noqa: PLR0913 rules: RuleCollection, outfile: Path, @@ -596,6 +651,13 @@ def _export_rules( # noqa: PLR0913 @click.option( "--include-exceptions", "-e", type=bool, is_flag=True, default=False, help="Include Exceptions Lists in export" ) +@click.option( + "--save-yaml-dir", + "-syd", + type=Path, + required=False, + help="Optional directory to export individual YAML files instead of NDJSON", +) def export_rules_from_repo( # noqa: PLR0913 rules: RuleCollection, outfile: Path, @@ -605,6 +667,7 @@ def export_rules_from_repo( # noqa: PLR0913 include_metadata: bool, include_action_connectors: bool, include_exceptions: bool, + save_yaml_dir: Path | None, ) -> RuleCollection: """Export rule(s) and exception(s) into an importable ndjson file.""" if len(rules) == 0: @@ -621,16 +684,28 @@ def export_rules_from_repo( # noqa: PLR0913 new_contents = dataclasses.replace(rule.contents, data=new_data) rules.add_rule(TOMLRule(contents=new_contents)) - outfile.parent.mkdir(exist_ok=True) - _export_rules( - rules=rules, - outfile=outfile, - downgrade_version=stack_version, - skip_unsupported=skip_unsupported, - include_metadata=include_metadata, - include_action_connectors=include_action_connectors, - include_exceptions=include_exceptions, - ) + if save_yaml_dir: + save_yaml_dir.mkdir(parents=True, exist_ok=True) + _export_rules_as_yaml( + rules=rules, + yaml_directory=save_yaml_dir, + downgrade_version=stack_version, + skip_unsupported=skip_unsupported, + include_metadata=include_metadata, + include_action_connectors=include_action_connectors, + include_exceptions=include_exceptions, + ) + else: + outfile.parent.mkdir(exist_ok=True) + _export_rules( + rules=rules, + outfile=outfile, + downgrade_version=stack_version, + skip_unsupported=skip_unsupported, + include_metadata=include_metadata, + include_action_connectors=include_action_connectors, + include_exceptions=include_exceptions, + ) return rules diff --git a/detection_rules/rule.py b/detection_rules/rule.py index c7624478ea7..396f29fb13d 100644 --- a/detection_rules/rule.py +++ b/detection_rules/rule.py @@ -22,6 +22,7 @@ import eql # type: ignore[reportMissingTypeStubs] import kql # type: ignore[reportMissingTypeStubs] import marshmallow +import yaml from marko.block import Document as MarkoDocument from marko.ext.gfm import gfm from marshmallow import ValidationError, pre_load, validates_schema @@ -1728,6 +1729,19 @@ def save_json(self, path: Path, include_version: bool = True) -> None: json.dump(self.contents.to_api_format(include_version=include_version), f, sort_keys=True, indent=2) _ = f.write("\n") + def save_yaml(self, path: Path, contents_override: dict[str, Any] | None = None) -> None: + """Save the rule in YAML format.""" + path = path.with_suffix(".yaml") + with path.absolute().open("w", newline="\n") as f: + output = yaml.safe_dump( + contents_override if contents_override is not None else self.contents.to_api_format(), + sort_keys=True, + default_flow_style=False, + ) + _ = f.write(output) + if not output.endswith("\n"): + _ = f.write("\n") + @dataclass(frozen=True) class DeprecatedRuleContents(BaseRuleContents): diff --git a/pyproject.toml b/pyproject.toml index e62363e67a4..08a599c012a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "detection_rules" -version = "1.5.50" +version = "1.5.51" description = "Detection Rules is the home for rules used by Elastic Security. This repository is used for the development, maintenance, testing, validation, and release of rules for Elastic Security’s Detection Engine." readme = "README.md" requires-python = ">=3.12" From dfd61421b6b9ba58d664d967b82ed857c66141d0 Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Tue, 10 Mar 2026 11:14:39 -0400 Subject: [PATCH 2/7] bigfix in yaml extension paths --- detection_rules/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detection_rules/utils.py b/detection_rules/utils.py index eedd45d3191..90e2ef66d51 100644 --- a/detection_rules/utils.py +++ b/detection_rules/utils.py @@ -353,7 +353,7 @@ def load_rule_contents(rule_file: Path, single_only: bool = False) -> list[Any]: return contents or [{}] if extension == ".toml": rule = pytoml.loads(raw_text) # type: ignore[reportUnknownVariableType] - elif extension.lower() in ("yaml", "yml"): + elif extension.lower() in (".yaml", ".yml"): rule = load_dump(str(rule_file)) else: return [] From 65656cb30af920acb4903a2aeb78e2d3eb8ebbb3 Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Tue, 10 Mar 2026 15:31:53 -0400 Subject: [PATCH 3/7] fix merge --- pyproject.toml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index aeb96408d30..f2c47dd2c4f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,6 @@ [project] name = "detection_rules" -<<<<<<< yaml_support -version = "1.5.57" -======= -version = "1.6.0" ->>>>>>> main +version = "1.6.1" description = "Detection Rules is the home for rules used by Elastic Security. This repository is used for the development, maintenance, testing, validation, and release of rules for Elastic Security’s Detection Engine." readme = "README.md" requires-python = ">=3.12" From 772d7ee1bef32ef969b578aaa4b09c283b86f10e Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Tue, 10 Mar 2026 16:54:20 -0400 Subject: [PATCH 4/7] Add defaults when not specified --- detection_rules/exception.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/detection_rules/exception.py b/detection_rules/exception.py index 5d1a060b06f..55aecf50a9a 100644 --- a/detection_rules/exception.py +++ b/detection_rules/exception.py @@ -6,7 +6,7 @@ from collections import defaultdict from dataclasses import dataclass -from datetime import datetime +from datetime import UTC, datetime from pathlib import Path from typing import Any, get_args @@ -177,8 +177,19 @@ def from_exceptions_dict( # Format date to match schema container = exceptions_dict["container"] - creation_date = datetime.strptime(container["created_at"], "%Y-%m-%dT%H:%M:%S.%fZ").strftime("%Y/%m/%d") # noqa: DTZ007 - updated_date = datetime.strptime(container["updated_at"], "%Y-%m-%dT%H:%M:%S.%fZ").strftime("%Y/%m/%d") # noqa: DTZ007 + now_date = datetime.now(UTC).strftime("%Y/%m/%d") + created_at = container.get("created_at") + updated_at = container.get("updated_at") + creation_date = ( + datetime.strptime(created_at, "%Y-%m-%dT%H:%M:%S.%fZ").strftime("%Y/%m/%d") # noqa: DTZ007 + if created_at + else now_date + ) + updated_date = ( + datetime.strptime(updated_at, "%Y-%m-%dT%H:%M:%S.%fZ").strftime("%Y/%m/%d") # noqa: DTZ007 + if updated_at + else now_date + ) metadata = { "creation_date": creation_date, "list_name": exceptions_dict["container"]["name"], From 8c1e7e60eddc99832d5f8a8361570c7644cb2396 Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Thu, 9 Apr 2026 12:57:09 -0400 Subject: [PATCH 5/7] Added util function to reduce code reuse --- detection_rules/action_connector.py | 16 +++++----------- detection_rules/exception.py | 16 +++++----------- detection_rules/rule.py | 13 ++----------- detection_rules/utils.py | 18 ++++++++++++++++++ 4 files changed, 30 insertions(+), 33 deletions(-) diff --git a/detection_rules/action_connector.py b/detection_rules/action_connector.py index 8951458ad6c..35aaffe933e 100644 --- a/detection_rules/action_connector.py +++ b/detection_rules/action_connector.py @@ -11,12 +11,12 @@ from typing import Any import pytoml # type: ignore[reportMissingTypeStubs] -import yaml from marshmallow import EXCLUDE from .config import parse_rules_config from .mixins import MarshmallowDataclassMixin from .schemas import definitions +from .utils import ensure_yaml_suffix, save_yaml RULES_CONFIG = parse_rules_config() @@ -117,16 +117,10 @@ def save_yaml(self, path: Path | None = None) -> None: target_path = path or self.path if not target_path: raise ValueError(f"Can't save action for {self.name} without a path") - if target_path.suffix not in (".yaml", ".yml"): - target_path = target_path.with_suffix(".yaml") - with target_path.open("w", encoding="utf-8", newline="\n") as f: - api_format = self.contents.to_api_format() - # If single item, write as dict; if multiple, write as list - content = api_format[0] if len(api_format) == 1 else api_format - output = yaml.safe_dump(content, default_flow_style=False, sort_keys=True) - _ = f.write(output) - if not output.endswith("\n"): - _ = f.write("\n") + api_format = self.contents.to_api_format() + # If single item, write as dict; if multiple, write as list + content = api_format[0] if len(api_format) == 1 else api_format + save_yaml(ensure_yaml_suffix(target_path), content) def parse_action_connector_results_from_api( diff --git a/detection_rules/exception.py b/detection_rules/exception.py index 55aecf50a9a..37f7f210815 100644 --- a/detection_rules/exception.py +++ b/detection_rules/exception.py @@ -11,12 +11,12 @@ from typing import Any, get_args import pytoml # type: ignore[reportMissingTypeStubs] -import yaml from marshmallow import EXCLUDE, ValidationError, validates_schema from .config import parse_rules_config from .mixins import MarshmallowDataclassMixin from .schemas import definitions +from .utils import ensure_yaml_suffix, save_yaml RULES_CONFIG = parse_rules_config() @@ -244,16 +244,10 @@ def save_yaml(self, path: Path | None = None) -> None: target_path = path or self.path if not target_path: raise ValueError(f"Can't save exception {self.name} without a path") - if target_path.suffix not in (".yaml", ".yml"): - target_path = target_path.with_suffix(".yaml") - with target_path.open("w", encoding="utf-8", newline="\n") as f: - api_format = self.contents.to_api_format() - # If single item, write as dict; if multiple, write as list - content = api_format[0] if len(api_format) == 1 else api_format - output = yaml.safe_dump(content, default_flow_style=False, sort_keys=True) - _ = f.write(output) - if not output.endswith("\n"): - _ = f.write("\n") + api_format = self.contents.to_api_format() + # If single item, write as dict; if multiple, write as list + content = api_format[0] if len(api_format) == 1 else api_format + save_yaml(ensure_yaml_suffix(target_path), content) def parse_exceptions_results_from_api( diff --git a/detection_rules/rule.py b/detection_rules/rule.py index a5381ee40dc..b341dff9fe6 100644 --- a/detection_rules/rule.py +++ b/detection_rules/rule.py @@ -22,7 +22,6 @@ import eql # type: ignore[reportMissingTypeStubs] import kql # type: ignore[reportMissingTypeStubs] import marshmallow -import yaml from marko.block import Document as MarkoDocument from marko.ext.gfm import gfm from marshmallow import ValidationError, pre_load, validates_schema @@ -1745,16 +1744,8 @@ def save_json(self, path: Path, include_version: bool = True) -> None: def save_yaml(self, path: Path, contents_override: dict[str, Any] | None = None) -> None: """Save the rule in YAML format.""" - path = path.with_suffix(".yaml") - with path.absolute().open("w", newline="\n") as f: - output = yaml.safe_dump( - contents_override if contents_override is not None else self.contents.to_api_format(), - sort_keys=True, - default_flow_style=False, - ) - _ = f.write(output) - if not output.endswith("\n"): - _ = f.write("\n") + data = contents_override if contents_override is not None else self.contents.to_api_format() + utils.save_yaml(path.with_suffix(".yaml"), data, use_absolute_path=True) @dataclass(frozen=True) diff --git a/detection_rules/utils.py b/detection_rules/utils.py index ffea69d445a..9b5d84db15c 100644 --- a/detection_rules/utils.py +++ b/detection_rules/utils.py @@ -27,6 +27,7 @@ import click import eql.utils # type: ignore[reportMissingTypeStubs] import pytoml # type: ignore[reportMissingTypeStubs] +import yaml from eql.utils import load_dump # type: ignore[reportMissingTypeStubs] from github.Repository import Repository @@ -136,6 +137,23 @@ def save_etc_dump(contents: dict[str, Any], path: list[str], sort_keys: bool = T eql.utils.save_dump(contents, path) # type: ignore[reportUnknownVariableType] +def ensure_yaml_suffix(path: Path) -> Path: + """If ``path`` has no YAML extension, use ``.yaml``; keep ``.yaml`` / ``.yml`` unchanged.""" + if path.suffix in (".yaml", ".yml"): + return path + return path.with_suffix(".yaml") + + +def save_yaml(path: Path, data: Any, *, use_absolute_path: bool = False) -> None: + """Write ``data`` as YAML with sorted keys, block style, UTF-8, and a trailing newline.""" + out_path = path.absolute() if use_absolute_path else path + with out_path.open("w", encoding="utf-8", newline="\n") as f: + output = yaml.safe_dump(data, sort_keys=True, default_flow_style=False) + _ = f.write(output) + if not output.endswith("\n"): + _ = f.write("\n") + + # Top-level _config.yaml key -> DR_BYPASS_* env var set when true at load time OPTIONAL_ELASTIC_VALIDATION_BYPASS_ENV: dict[str, str] = { "bypass_note_validation_and_parse": "DR_BYPASS_NOTE_VALIDATION_AND_PARSE", From 41b56db8888f4ecb6e2a22ee3ac874fc885c9656 Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Thu, 9 Apr 2026 16:22:48 -0400 Subject: [PATCH 6/7] Simplify directory workflow for export rules --- CLI.md | 33 ++++++++++++++++++------------ detection_rules/kbwrap.py | 43 ++++++++++++++++----------------------- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/CLI.md b/CLI.md index 37960b936f7..2621da5ef95 100644 --- a/CLI.md +++ b/CLI.md @@ -413,21 +413,25 @@ python -m detection_rules kibana import-rules -d test-export-rules -o Toml formatted rule files can also be imported into Kibana through Kibana security app via a consolidated ndjson file which is exported from detection rules. +For this command, **`-d` / `--directory`** selects **input**: directories to load rules from (same as other multi-collection commands). **`--outfile` / `-o`** is the **NDJSON output path** when you are not using YAML mode. **`--save-yaml-dir` / `-syd`** writes **per-rule (and related) YAML files** into that directory instead of producing a single NDJSON file; when `-syd` is set, `-o` is unused. + +Default NDJSON path when `-o` is omitted: `exports/.ndjson` under the detection-rules repository root. + ```console Usage: detection_rules export-rules-from-repo [OPTIONS] Export rule(s) and exception(s) into an importable ndjson file. Options: - -f, --rule-file FILE - -d, --directory DIRECTORY Recursively load rules from a directory - -id, --rule-id TEXT + -f, --rule-file FILE Rule file(s) to load (repeatable) + -d, --directory DIRECTORY Recursively load rules from a directory (repeatable) + -id, --rule-id TEXT Load prebuilt rules matching these IDs (repeatable) -nt, --no-tactic-filename Allow rule filenames without tactic prefix. Use this if rules have been exported with this flag. - -o, --outfile PATH Name of file for exported rules - -r, --replace-id Replace rule IDs with new IDs before export - --stack-version [7.8|7.9|7.10|7.11|7.12|7.13|7.14|7.15|7.16|8.0|8.1|8.2|8.3|8.4|8.5|8.6|8.7|8.8|8.9|8.10|8.11|8.12|8.13|8.14|8.15|8.16|8.17|8.18|9.0] - Downgrade a rule version to be compatible with older instances of Kibana - -s, --skip-unsupported If `--stack-version` is passed, skip rule types which are unsupported (an error will be raised otherwise) + -o, --outfile PATH NDJSON file path for exported rules (ignored if --save-yaml-dir is set) + -syd, --save-yaml-dir PATH Export individual YAML files into this directory instead of NDJSON + -r, --replace-id Replace rule IDs with new UUIDs before export + --stack-version [7.8|...|9.0] Downgrade rule payloads for older Kibana (see `export-rules-from-repo --help` for full list) + -s, --skip-unsupported With `--stack-version`, skip unsupported rule types instead of erroring --include-metadata Add metadata to the exported rules -ac, --include-action-connectors Include Action Connectors in export @@ -500,18 +504,19 @@ Usage: detection_rules kibana export-rules [OPTIONS] Export rules from Kibana. Options: - -d, --directory PATH Directory to export rules to [required] + -d, --directory PATH Directory to write exported rules to [required] -acd, --action-connectors-directory PATH - Directory to export action connectors to + Directory to export action connectors to (defaults from rules config if omitted) -ed, --exceptions-directory PATH - Directory to export exceptions to + Directory to export exceptions to (defaults from rules config if omitted) -da, --default-author TEXT Default author for rules missing one - -r, --rule-id TEXT Optional Rule IDs to restrict export to - -rn, --rule-name TEXT Optional Rule name to restrict export to (KQL, case-insensitive, supports wildcards) + -r, --rule-id TEXT Optional rule ID(s) to restrict export to (repeatable) + -rn, --rule-name TEXT Optional rule name filter (KQL, case-insensitive, wildcards); mutually exclusive with `--rule-id` -ac, --export-action-connectors Include action connectors in export -e, --export-exceptions Include exceptions in export -s, --skip-errors Skip errors when exporting rules + -sy, --save-as-yaml Write rules (and exported exceptions/connectors when requested) as YAML under `--directory` instead of TOML -sv, --strip-version Strip the version fields from all rules -nt, --no-tactic-filename Exclude tactic prefix in exported filenames for rules. Use same flag for import-rules to prevent warnings and disable its unit test. -lc, --local-creation-date Preserve the local creation date of the rule @@ -523,6 +528,8 @@ Options: ``` +**Note:** `kibana export-rules` **`--directory` / `-d`** is the **output** directory only. It is unrelated to **`export-rules-from-repo`**, where **`-d`** means **input** rule directories. + Example of a rule exporting, with errors skipped ``` diff --git a/detection_rules/kbwrap.py b/detection_rules/kbwrap.py index 67003b3630e..3c1d58164bd 100644 --- a/detection_rules/kbwrap.py +++ b/detection_rules/kbwrap.py @@ -208,7 +208,7 @@ def _process_imported_items( @kibana_group.command("export-rules") -@click.option("--directory", "-d", required=False, type=Path, help="Directory to export rules to") +@click.option("--directory", "-d", required=True, type=Path, help="Directory to export rules to") @click.option( "--action-connectors-directory", "-acd", required=False, type=Path, help="Directory to export action connectors to" ) @@ -225,11 +225,11 @@ def _process_imported_items( @click.option("--export-exceptions", "-e", is_flag=True, help="Include exceptions in export") @click.option("--skip-errors", "-s", is_flag=True, help="Skip errors when exporting rules") @click.option( - "--save-yaml-dir", - "-syd", - required=False, - type=Path, - help="Save exported rules and objects as YAML into this directory using original filenames", + "--save-as-yaml", + "-sy", + is_flag=True, + default=False, + help="Save exported rules and objects as YAML into --directory (instead of TOML)", ) @click.option("--strip-version", "-sv", is_flag=True, help="Strip the version fields from all rules") @click.option( @@ -261,7 +261,7 @@ def _process_imported_items( @click.pass_context def kibana_export_rules( # noqa: PLR0912, PLR0913, PLR0915 ctx: click.Context, - directory: Path | None, + directory: Path, action_connectors_directory: Path | None, exceptions_directory: Path | None, default_author: str, @@ -270,7 +270,7 @@ def kibana_export_rules( # noqa: PLR0912, PLR0913, PLR0915 export_action_connectors: bool = False, export_exceptions: bool = False, skip_errors: bool = False, - save_yaml_dir: Path | None = None, + save_as_yaml: bool = False, strip_version: bool = False, no_tactic_filename: bool = False, local_creation_date: bool = False, @@ -287,13 +287,6 @@ def _raise_missing_path(message: str) -> None: kibana = ctx.obj["kibana"] kibana_include_details = export_exceptions or export_action_connectors or custom_rules_only or export_query - if not directory and not save_yaml_dir: - raise click.UsageError("One of --directory or --save-yaml-dir must be provided.") - - output_directory = directory or save_yaml_dir - if not output_directory: - raise click.UsageError("Unable to determine output directory.") - # Only allow one of rule_id or rule_name if rule_name and rule_id: raise click.UsageError("Cannot use --rule-id and --rule-name together. Please choose one.") @@ -339,7 +332,7 @@ def _raise_missing_path(message: str) -> None: click.echo("Warning: Action Connector export requested, but no Action Connector directory found") if results: - output_directory.mkdir(parents=True, exist_ok=True) + directory.mkdir(parents=True, exist_ok=True) else: click.echo("No rules found to export") return [] @@ -389,7 +382,7 @@ def _raise_missing_path(message: str) -> None: tactic_name = first_tactic if not no_tactic_filename else None # type: ignore[reportUnknownMemberType] rule_name = rulename_to_filename(rule_resource.get("name"), tactic_name=tactic_name) # type: ignore[reportUnknownMemberType] - save_path = output_directory / f"{rule_name}" + save_path = directory / f"{rule_name}" # Get local rule data if load_rule_loading is enabled. If not enabled rules variable will be None. local_rule: dict[str, Any] = params.get("rule", {}) @@ -474,10 +467,10 @@ def _raise_missing_path(message: str) -> None: saved: list[TOMLRule] = [] for rule in exported: try: - if save_yaml_dir: + if save_as_yaml: rule_path = rule.path if isinstance(rule_path, Path): - rule.save_yaml(save_yaml_dir / rule_path.name) + rule.save_yaml(directory / rule_path.name) else: _raise_missing_path(f"Can't save rule {rule.name} ({rule.id}) without a path") else: @@ -494,10 +487,10 @@ def _raise_missing_path(message: str) -> None: saved_exceptions: list[TOMLException] = [] for exception in exceptions: try: - if save_yaml_dir: + if save_as_yaml: exception_path = exception.path if isinstance(exception_path, Path): - exception.save_yaml(save_yaml_dir / exception_path.name) + exception.save_yaml(directory / exception_path.name) else: _raise_missing_path(f"Can't save exception {exception.name} without a path") else: @@ -514,8 +507,8 @@ def _raise_missing_path(message: str) -> None: saved_action_connectors: list[TOMLActionConnector] = [] for action in action_connectors: try: - if save_yaml_dir: - action.save_yaml(save_yaml_dir / action.path.name) + if save_as_yaml: + action.save_yaml(directory / action.path.name) else: action.save_toml() except Exception as e: @@ -531,11 +524,11 @@ def _raise_missing_path(message: str) -> None: click.echo(f"{len(exported)} rules converted") click.echo(f"{len(exceptions)} exceptions exported") click.echo(f"{len(action_connectors)} action connectors exported") - click.echo(f"{len(saved)} rules saved to {output_directory}") + click.echo(f"{len(saved)} rules saved to {directory}") click.echo(f"{len(saved_exceptions)} exception lists saved to {exceptions_directory}") click.echo(f"{len(saved_action_connectors)} action connectors saved to {action_connectors_directory}") if errors: - err_file = output_directory / "_errors.txt" + err_file = directory / "_errors.txt" _ = err_file.write_text("\n".join(errors)) click.echo(f"{len(errors)} errors saved to {err_file}") From 80d14e511af1f604fb09ca0847908456e05a977f Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Fri, 10 Apr 2026 11:01:21 -0400 Subject: [PATCH 7/7] patch bump --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3338d0704cc..19db32ee0e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "detection_rules" -version = "1.6.16" +version = "1.6.17" description = "Detection Rules is the home for rules used by Elastic Security. This repository is used for the development, maintenance, testing, validation, and release of rules for Elastic Security’s Detection Engine." readme = "README.md" requires-python = ">=3.12"