[FR] [DAC] Initial Yaml Support

CLI.md

@@ -413,21 +413,25 @@ python -m detection_rules kibana import-rules -d test-export-rules -o
 Toml formatted rule files can also be imported into Kibana through Kibana security app via a consolidated ndjson file
 which is exported from detection rules.
 
+For this command, **`-d` / `--directory`** selects **input**: directories to load rules from (same as other multi-collection commands). **`--outfile` / `-o`** is the **NDJSON output path** when you are not using YAML mode. **`--save-yaml-dir` / `-syd`** writes **per-rule (and related) YAML files** into that directory instead of producing a single NDJSON file; when `-syd` is set, `-o` is unused.
+
+Default NDJSON path when `-o` is omitted: `exports/<timestamp>.ndjson` under the detection-rules repository root.
+
 ```console
 Usage: detection_rules export-rules-from-repo [OPTIONS]
 
   Export rule(s) and exception(s) into an importable ndjson file.
 
 Options:
-  -f, --rule-file FILE
-  -d, --directory DIRECTORY       Recursively load rules from a directory
-  -id, --rule-id TEXT
+  -f, --rule-file FILE            Rule file(s) to load (repeatable)
+  -d, --directory DIRECTORY      Recursively load rules from a directory (repeatable)
+  -id, --rule-id TEXT             Load prebuilt rules matching these IDs (repeatable)
   -nt, --no-tactic-filename       Allow rule filenames without tactic prefix. Use this if rules have been exported with this flag.
-  -o, --outfile PATH              Name of file for exported rules
-  -r, --replace-id                Replace rule IDs with new IDs before export
-  --stack-version [7.8|7.9|7.10|7.11|7.12|7.13|7.14|7.15|7.16|8.0|8.1|8.2|8.3|8.4|8.5|8.6|8.7|8.8|8.9|8.10|8.11|8.12|8.13|8.14|8.15|8.16|8.17|8.18|9.0]
-                                  Downgrade a rule version to be compatible with older instances of Kibana
-  -s, --skip-unsupported          If `--stack-version` is passed, skip rule types which are unsupported (an error will be raised otherwise)
+  -o, --outfile PATH              NDJSON file path for exported rules (ignored if --save-yaml-dir is set)
+  -syd, --save-yaml-dir PATH      Export individual YAML files into this directory instead of NDJSON
+  -r, --replace-id                Replace rule IDs with new UUIDs before export
+  --stack-version [7.8|...|9.0]   Downgrade rule payloads for older Kibana (see `export-rules-from-repo --help` for full list)
+  -s, --skip-unsupported          With `--stack-version`, skip unsupported rule types instead of erroring
   --include-metadata              Add metadata to the exported rules
   -ac, --include-action-connectors
                                   Include Action Connectors in export
@@ -500,18 +504,19 @@ Usage: detection_rules kibana export-rules [OPTIONS]
   Export rules from Kibana.
 
 Options:
-  -d, --directory PATH            Directory to export rules to  [required]
+  -d, --directory PATH            Directory to write exported rules to  [required]
   -acd, --action-connectors-directory PATH
-                                  Directory to export action connectors to
+                                  Directory to export action connectors to (defaults from rules config if omitted)
   -ed, --exceptions-directory PATH
-                                  Directory to export exceptions to
+                                  Directory to export exceptions to (defaults from rules config if omitted)
   -da, --default-author TEXT      Default author for rules missing one
-  -r, --rule-id TEXT              Optional Rule IDs to restrict export to
-  -rn, --rule-name TEXT           Optional Rule name to restrict export to (KQL, case-insensitive, supports wildcards)
+  -r, --rule-id TEXT              Optional rule ID(s) to restrict export to (repeatable)
+  -rn, --rule-name TEXT           Optional rule name filter (KQL, case-insensitive, wildcards); mutually exclusive with `--rule-id`
   -ac, --export-action-connectors
                                   Include action connectors in export
   -e, --export-exceptions         Include exceptions in export
   -s, --skip-errors               Skip errors when exporting rules
+  -sy, --save-as-yaml             Write rules (and exported exceptions/connectors when requested) as YAML under `--directory` instead of TOML
   -sv, --strip-version            Strip the version fields from all rules
   -nt, --no-tactic-filename       Exclude tactic prefix in exported filenames for rules. Use same flag for import-rules to prevent warnings and disable its unit test.
   -lc, --local-creation-date      Preserve the local creation date of the rule
@@ -523,6 +528,8 @@ Options:
 
 ```
 
+**Note:** `kibana export-rules` **`--directory` / `-d`** is the **output** directory only. It is unrelated to **`export-rules-from-repo`**, where **`-d`** means **input** rule directories.
+
 Example of a rule exporting, with errors skipped
 
 ```

detection_rules/action_connector.py

@@ -16,6 +16,7 @@
 from .config import parse_rules_config
 from .mixins import MarshmallowDataclassMixin
 from .schemas import definitions
+from .utils import ensure_yaml_suffix, save_yaml
 
 RULES_CONFIG = parse_rules_config()
 
@@ -111,6 +112,16 @@ def save_toml(self) -> None:
             sorted_dict = dict(sorted(contents_dict.items(), key=lambda item: item[0] != "metadata"))
             pytoml.dump(sorted_dict, f)  # type: ignore[reportUnknownMemberType]
 
+    def save_yaml(self, path: Path | None = None) -> None:
+        """Save the action to a YAML file."""
+        target_path = path or self.path
+        if not target_path:
+            raise ValueError(f"Can't save action for {self.name} without a path")
+        api_format = self.contents.to_api_format()
+        # If single item, write as dict; if multiple, write as list
+        content = api_format[0] if len(api_format) == 1 else api_format
+        save_yaml(ensure_yaml_suffix(target_path), content)
+
 
 def parse_action_connector_results_from_api(
     results: list[dict[str, Any]],

detection_rules/exception.py

@@ -6,7 +6,7 @@
 
 from collections import defaultdict
 from dataclasses import dataclass
-from datetime import datetime
+from datetime import UTC, datetime
 from pathlib import Path
 from typing import Any, get_args
 
@@ -16,6 +16,7 @@
 from .config import parse_rules_config
 from .mixins import MarshmallowDataclassMixin
 from .schemas import definitions
+from .utils import ensure_yaml_suffix, save_yaml
 
 RULES_CONFIG = parse_rules_config()
 
@@ -176,8 +177,19 @@ def from_exceptions_dict(
 
         # Format date to match schema
         container = exceptions_dict["container"]
-        creation_date = datetime.strptime(container["created_at"], "%Y-%m-%dT%H:%M:%S.%fZ").strftime("%Y/%m/%d")  # noqa: DTZ007
-        updated_date = datetime.strptime(container["updated_at"], "%Y-%m-%dT%H:%M:%S.%fZ").strftime("%Y/%m/%d")  # noqa: DTZ007
+        now_date = datetime.now(UTC).strftime("%Y/%m/%d")
+        created_at = container.get("created_at")
+        updated_at = container.get("updated_at")
+        creation_date = (
+            datetime.strptime(created_at, "%Y-%m-%dT%H:%M:%S.%fZ").strftime("%Y/%m/%d")  # noqa: DTZ007
+            if created_at
+            else now_date
+        )
+        updated_date = (
+            datetime.strptime(updated_at, "%Y-%m-%dT%H:%M:%S.%fZ").strftime("%Y/%m/%d")  # noqa: DTZ007
+            if updated_at
+            else now_date
+        )
         metadata = {
             "creation_date": creation_date,
             "list_name": exceptions_dict["container"]["name"],
@@ -227,6 +239,16 @@ def save_toml(self) -> None:
             sorted_dict = dict(sorted(contents_dict.items(), key=lambda item: item[0] != "metadata"))
             pytoml.dump(sorted_dict, f)  # type: ignore[reportUnknownMemberType]
 
+    def save_yaml(self, path: Path | None = None) -> None:
+        """Save the exception to a YAML file."""
+        target_path = path or self.path
+        if not target_path:
+            raise ValueError(f"Can't save exception {self.name} without a path")
+        api_format = self.contents.to_api_format()
+        # If single item, write as dict; if multiple, write as list
+        content = api_format[0] if len(api_format) == 1 else api_format
+        save_yaml(ensure_yaml_suffix(target_path), content)
+
 
 def parse_exceptions_results_from_api(
     results: list[dict[str, Any]],

detection_rules/kbwrap.py

@@ -224,6 +224,13 @@ def _process_imported_items(
 @click.option("--export-action-connectors", "-ac", is_flag=True, help="Include action connectors in export")
 @click.option("--export-exceptions", "-e", is_flag=True, help="Include exceptions in export")
 @click.option("--skip-errors", "-s", is_flag=True, help="Skip errors when exporting rules")
+@click.option(
+    "--save-as-yaml",
+    "-sy",
+    is_flag=True,
+    default=False,
+    help="Save exported rules and objects as YAML into --directory (instead of TOML)",
+)
 @click.option("--strip-version", "-sv", is_flag=True, help="Strip the version fields from all rules")
 @click.option(
     "--no-tactic-filename",
@@ -263,6 +270,7 @@ def kibana_export_rules(  # noqa: PLR0912, PLR0913, PLR0915
     export_action_connectors: bool = False,
     export_exceptions: bool = False,
     skip_errors: bool = False,
+    save_as_yaml: bool = False,
     strip_version: bool = False,
     no_tactic_filename: bool = False,
     local_creation_date: bool = False,
@@ -272,6 +280,10 @@ def kibana_export_rules(  # noqa: PLR0912, PLR0913, PLR0915
     load_rule_loading: bool = False,
 ) -> list[TOMLRule]:
     """Export rules from Kibana."""
+
+    def _raise_missing_path(message: str) -> None:
+        raise ValueError(message)
+
     kibana = ctx.obj["kibana"]
     kibana_include_details = export_exceptions or export_action_connectors or custom_rules_only or export_query
 
@@ -455,7 +467,14 @@ def kibana_export_rules(  # noqa: PLR0912, PLR0913, PLR0915
     saved: list[TOMLRule] = []
     for rule in exported:
         try:
-            rule.save_toml()
+            if save_as_yaml:
+                rule_path = rule.path
+                if isinstance(rule_path, Path):
+                    rule.save_yaml(directory / rule_path.name)
+                else:
+                    _raise_missing_path(f"Can't save rule {rule.name} ({rule.id}) without a path")
+            else:
+                rule.save_toml()
         except Exception as e:
             if skip_errors:
                 print(f"- skipping {rule.contents.data.name} - {type(e).__name__}")
@@ -468,7 +487,14 @@ def kibana_export_rules(  # noqa: PLR0912, PLR0913, PLR0915
     saved_exceptions: list[TOMLException] = []
     for exception in exceptions:
         try:
-            exception.save_toml()
+            if save_as_yaml:
+                exception_path = exception.path
+                if isinstance(exception_path, Path):
+                    exception.save_yaml(directory / exception_path.name)
+                else:
+                    _raise_missing_path(f"Can't save exception {exception.name} without a path")
+            else:
+                exception.save_toml()
         except Exception as e:
             if skip_errors:
                 print(f"- skipping {exception.rule_name} - {type(e).__name__}")  # type: ignore[reportUnknownMemberType]
@@ -481,7 +507,10 @@ def kibana_export_rules(  # noqa: PLR0912, PLR0913, PLR0915
     saved_action_connectors: list[TOMLActionConnector] = []
     for action in action_connectors:
         try:
-            action.save_toml()
+            if save_as_yaml:
+                action.save_yaml(directory / action.path.name)
+            else:
+                action.save_toml()
         except Exception as e:
             if skip_errors:
                 print(f"- skipping {action.name} - {type(e).__name__}")

detection_rules/main.py

@@ -21,14 +21,15 @@
 from semver import Version
 
 from .action_connector import (
+    TOMLActionConnector,
     TOMLActionConnectorContents,
     build_action_connector_objects,
     parse_action_connector_results_from_api,
 )
 from .attack import build_threat_map_entry
 from .cli_utils import multi_collection, rule_prompt
 from .config import load_current_package_version, parse_rules_config
-from .exception import TOMLExceptionContents, build_exception_objects, parse_exceptions_results_from_api
+from .exception import TOMLException, TOMLExceptionContents, build_exception_objects, parse_exceptions_results_from_api
 from .generic_loader import GenericCollection
 from .misc import (
     add_client,
@@ -512,6 +513,60 @@ def view_rule(
     return rule
 
 
+def _export_rules_as_yaml(  # noqa: PLR0913
+    rules: RuleCollection,
+    yaml_directory: Path,
+    downgrade_version: definitions.SemVer | None = None,
+    verbose: bool = True,
+    skip_unsupported: bool = False,
+    include_metadata: bool = False,
+    include_action_connectors: bool = False,
+    include_exceptions: bool = False,
+) -> None:
+    """Export rules and exceptions into a directory of YAML files."""
+    from .rule import downgrade_contents_from_rule
+
+    unsupported: list[str] = []
+
+    for rule in rules:
+        contents_override = None
+        if downgrade_version:
+            try:
+                contents_override = downgrade_contents_from_rule(
+                    rule, downgrade_version, include_metadata=include_metadata
+                )
+            except ValueError as e:
+                if skip_unsupported:
+                    unsupported.append(f"{e}: {rule.id} - {rule.name}")
+                    continue
+                raise
+
+        rule_path = yaml_directory / rulename_to_filename(rule.name)
+        rule_path.parent.mkdir(parents=True, exist_ok=True)
+        rule.save_yaml(rule_path, contents_override=contents_override)
+
+    export_types: list[type[Any]] = []
+    if include_exceptions:
+        export_types.append(TOMLException)
+    if include_action_connectors:
+        export_types.append(TOMLActionConnector)
+
+    if export_types:
+        cl = GenericCollection.default()
+        for d in cl.items:
+            if any(isinstance(d, export_type) for export_type in export_types):
+                save_yaml = getattr(d, "save_yaml", None)
+                if callable(save_yaml) and isinstance(d.path, Path):
+                    _ = save_yaml(yaml_directory / d.path.name)
+
+    if verbose:
+        click.echo(f"Exported {len(rules) - len(unsupported)} rules into {yaml_directory}")
+
+        if skip_unsupported and unsupported:
+            unsupported_str = "\n- ".join(unsupported)
+            click.echo(f"Skipped {len(unsupported)} unsupported rules: \n- {unsupported_str}")
+
+
 def _export_rules(  # noqa: PLR0913
     rules: RuleCollection,
     outfile: Path,
@@ -611,6 +666,13 @@ def _export_rules(  # noqa: PLR0913
 @click.option(
     "--include-exceptions", "-e", type=bool, is_flag=True, default=False, help="Include Exceptions Lists in export"
 )
+@click.option(
+    "--save-yaml-dir",
+    "-syd",
+    type=Path,
+    required=False,
+    help="Optional directory to export individual YAML files instead of NDJSON",
+)
 def export_rules_from_repo(  # noqa: PLR0913
     rules: RuleCollection,
     outfile: Path,
@@ -620,6 +682,7 @@ def export_rules_from_repo(  # noqa: PLR0913
     include_metadata: bool,
     include_action_connectors: bool,
     include_exceptions: bool,
+    save_yaml_dir: Path | None,
 ) -> RuleCollection:
     """Export rule(s) and exception(s) into an importable ndjson file."""
     if len(rules) == 0:
@@ -636,16 +699,28 @@ def export_rules_from_repo(  # noqa: PLR0913
             new_contents = dataclasses.replace(rule.contents, data=new_data)
             rules.add_rule(TOMLRule(contents=new_contents))
 
-    outfile.parent.mkdir(exist_ok=True)
-    _export_rules(
-        rules=rules,
-        outfile=outfile,
-        downgrade_version=stack_version,
-        skip_unsupported=skip_unsupported,
-        include_metadata=include_metadata,
-        include_action_connectors=include_action_connectors,
-        include_exceptions=include_exceptions,
-    )
+    if save_yaml_dir:
+        save_yaml_dir.mkdir(parents=True, exist_ok=True)
+        _export_rules_as_yaml(
+            rules=rules,
+            yaml_directory=save_yaml_dir,
+            downgrade_version=stack_version,
+            skip_unsupported=skip_unsupported,
+            include_metadata=include_metadata,
+            include_action_connectors=include_action_connectors,
+            include_exceptions=include_exceptions,
+        )
+    else:
+        outfile.parent.mkdir(exist_ok=True)
+        _export_rules(
+            rules=rules,
+            outfile=outfile,
+            downgrade_version=stack_version,
+            skip_unsupported=skip_unsupported,
+            include_metadata=include_metadata,
+            include_action_connectors=include_action_connectors,
+            include_exceptions=include_exceptions,
+        )
 
     return rules
 

detection_rules/rule.py

@@ -1742,6 +1742,11 @@ def save_json(self, path: Path, include_version: bool = True) -> None:
             json.dump(self.contents.to_api_format(include_version=include_version), f, sort_keys=True, indent=2)
             _ = f.write("\n")
 
+    def save_yaml(self, path: Path, contents_override: dict[str, Any] | None = None) -> None:
+        """Save the rule in YAML format."""
+        data = contents_override if contents_override is not None else self.contents.to_api_format()
+        utils.save_yaml(path.with_suffix(".yaml"), data, use_absolute_path=True)
+
 
 @dataclass(frozen=True)
 class DeprecatedRuleContents(BaseRuleContents):