Add outcome-aware evaluation loop for Cisco Firepower response

Solutions/Cisco Firepower EStreamer/Analytic Rules/CiscoFirepower-IDS-Signature-HighPriority.yaml

@@ -0,0 +1,76 @@
+id: a1c9e026-9bb7-4c42-a3a1-83faa3ca26a6
+name: Cisco Firepower - IDS signature high priority classification
+description: |
+  'Detects classic Cisco Firepower / Snort-family signature hits (Generator ID not equal to SnortML GID 411)
+  with high-priority classifications commonly associated with malware C2, privilege gain, or network trojans.
+  These events are stronger signature true-positive candidates than ML-only (GID 411) paths and may justify
+  gated remediation after analyst or policy review - prefer HITL Gate/Prove over ungated BlockIP automation.
+  Pair with "Cisco Firepower - SnortML GID 411 ML-only high alert" and "Cisco Firepower - Signature and ML corroboration".'
+severity: High
+status: Available
+requiredDataConnectors:
+  - connectorId: CefAma
+    dataTypes:
+      - CommonSecurityLog
+queryFrequency: 15m
+queryPeriod: 15m
+triggerOperator: gt
+triggerThreshold: 0
+tactics:
+  - CommandAndControl
+  - Execution
+relevantTechniques:
+  - T1071
+  - T1203
+query: |
+  let HighPriorityClassifications = dynamic([
+      "A Network Trojan was Detected",
+      "A Network Trojan was detected",
+      "Successful Administrator Privilege Gain",
+      "Successful User Privilege Gain",
+      "Attempted Administrator Privilege Gain",
+      "Attempted User Privilege Gain",
+      "Known malware command and control traffic",
+      "Malware Command and Control Activity Detected",
+      "Known malicious file or file based exploit",
+      "Known client side exploit attempt",
+      "Large Scale Information Leak"
+  ]);
+  CommonSecurityLog
+  | where DeviceVendor =~ "Cisco"
+  | where DeviceProduct has_any ("Firepower", "Secure Firewall", "FTD", "NGFW")
+  | extend Combined = strcat(
+      tostring(Message), " ",
+      tostring(AdditionalExtensions), " ",
+      tostring(Activity), " ",
+      tostring(DeviceEventClassID), " ",
+      tostring(column_ifexists("FlexString1", "")), " ",
+      tostring(column_ifexists("FlexString2", "")), " ",
+      tostring(column_ifexists("DeviceCustomString1", "")), " ",
+      tostring(column_ifexists("DeviceCustomString2", "")), " ",
+      tostring(column_ifexists("DeviceCustomString3", "")), " ",
+      tostring(DeviceAction)
+    )
+  | extend ParsedGid = toint(extract(@"(?i)(?:gid|generator[\s_-]?id)[\s:=]*(\d+)", 1, Combined))
+  | where ParsedGid != 411 or isnull(ParsedGid)
+  | where not(Combined has_any ("SnortML", "snortml", "is_ml_only"))
+  | where Combined has_any (HighPriorityClassifications)
+      or Activity has_any (HighPriorityClassifications)
+  | extend HostCustomEntity = DeviceName, SrcIpCustomEntity = SourceIP, DstIpCustomEntity = DestinationIP
+  | project TimeGenerated, DeviceName, SourceIP, DestinationIP, DestinationPort, Activity, DeviceAction,
+      DeviceEventClassID, Message, AdditionalExtensions, HostCustomEntity, SrcIpCustomEntity, DstIpCustomEntity
+entityMappings:
+  - entityType: Host
+    fieldMappings:
+      - identifier: FullName
+        columnName: HostCustomEntity
+  - entityType: IP
+    fieldMappings:
+      - identifier: Address
+        columnName: SrcIpCustomEntity
+  - entityType: IP
+    fieldMappings:
+      - identifier: Address
+        columnName: DstIpCustomEntity
+version: 1.0.2
+kind: Scheduled

Solutions/Cisco Firepower EStreamer/Analytic Rules/CiscoFirepower-Signal-Mix-Drift.yaml

@@ -0,0 +1,47 @@
+id: 6ff65bb5-53bd-4ffb-a62a-25ea71c04eed
+name: Cisco Firepower - SnortML signal mix drift
+description: |
+  Detects a material increase in the proportion of SnortML GID 411 events compared with the preceding seven-day baseline.
+  This is a detection-quality regression signal, not evidence that an individual event is malicious. Investigate collector,
+  sensor, model, policy, and traffic changes before modifying response automation. ML-only events must not trigger automatic containment.
+severity: Medium
+status: Available
+requiredDataConnectors:
+  - connectorId: CefAma
+    dataTypes:
+      - CommonSecurityLog
+queryFrequency: 1h
+queryPeriod: 8d
+triggerOperator: gt
+triggerThreshold: 0
+tactics:
+  - DefenseEvasion
+relevantTechniques:
+  - T1562
+query: |
+  let FirepowerEvents = materialize(
+    CommonSecurityLog
+    | where TimeGenerated >= ago(8d)
+    | where DeviceVendor =~ "Cisco"
+    | where DeviceProduct has_any ("Firepower", "Secure Firewall", "FTD", "NGFW")
+    | extend Combined = strcat(tostring(Message), " ", tostring(AdditionalExtensions), " ", tostring(Activity), " ", tostring(DeviceEventClassID), " ", tostring(column_ifexists("FlexString1", "")), " ", tostring(column_ifexists("FlexString2", "")), " ", tostring(column_ifexists("DeviceCustomString1", "")), " ", tostring(column_ifexists("DeviceCustomString2", "")), " ", tostring(column_ifexists("DeviceCustomString3", "")))
+    | extend ParsedGid = toint(extract(@"(?i)(?:gid|generator[\s_-]?id)[\s:=]*(\d+)", 1, Combined))
+    | extend IsMlOnly = ParsedGid == 411 or Combined has "is_ml_only"
+  );
+  let Recent = FirepowerEvents
+    | where TimeGenerated >= ago(1h)
+    | summarize RecentTotal=count(), RecentMl=countif(IsMlOnly)
+    | extend RecentRatio=iff(RecentTotal == 0, 0.0, todouble(RecentMl) / RecentTotal);
+  let Baseline = FirepowerEvents
+    | where TimeGenerated between (ago(8d) .. ago(1d))
+    | summarize BaselineTotal=count(), BaselineMl=countif(IsMlOnly)
+    | extend BaselineRatio=iff(BaselineTotal == 0, 0.0, todouble(BaselineMl) / BaselineTotal);
+  Recent
+  | extend JoinKey=1
+  | join kind=inner (Baseline | extend JoinKey=1) on JoinKey
+  | where RecentTotal >= 20 and BaselineTotal >= 100
+  | where RecentRatio >= 0.25 and RecentRatio >= (BaselineRatio * 2.0)
+  | project TimeGenerated=now(), RecentTotal, RecentMl, RecentRatio, BaselineTotal, BaselineMl, BaselineRatio,
+      DriftMultiple=round(RecentRatio / iff(BaselineRatio == 0.0, 0.0001, BaselineRatio), 2)
+version: 1.0.0
+kind: Scheduled

Solutions/Cisco Firepower EStreamer/Analytic Rules/CiscoFirepower-Signature-And-ML-Corroboration.yaml

@@ -0,0 +1,90 @@
+id: 511445a6-6f4c-4e6a-a655-76c25b66597b
+name: Cisco Firepower - Signature and ML corroboration
+description: |
+  'Detects dual-signal corroboration on Cisco Firepower CEF: a classic high-priority IDS classification
+  (Generator ID not SnortML GID 411) co-occurring with an ML-only (GID 411 / SnortML) alert for the same
+  source and destination within a short window.
+  Signature + ML corroboration is a stronger remediation candidate than ML-only paths.
+  Prefer Gate/Prove HITL before BlockIP playbooks. Do not equate standalone ML confidence to signature TP.'
+severity: High
+status: Available
+requiredDataConnectors:
+  - connectorId: CefAma
+    dataTypes:
+      - CommonSecurityLog
+queryFrequency: 15m
+queryPeriod: 30m
+triggerOperator: gt
+triggerThreshold: 0
+tactics:
+  - CommandAndControl
+  - Exfiltration
+relevantTechniques:
+  - T1071
+  - T1041
+query: |
+  let lookback = 30m;
+  let HighPriorityClassifications = dynamic([
+      "A Network Trojan was Detected",
+      "A Network Trojan was detected",
+      "Successful Administrator Privilege Gain",
+      "Successful User Privilege Gain",
+      "Attempted Administrator Privilege Gain",
+      "Attempted User Privilege Gain",
+      "Known malware command and control traffic",
+      "Malware Command and Control Activity Detected",
+      "Large Scale Information Leak"
+  ]);
+  let Base = CommonSecurityLog
+  | where TimeGenerated > ago(lookback)
+  | where DeviceVendor =~ "Cisco"
+  | where DeviceProduct has_any ("Firepower", "Secure Firewall", "FTD", "NGFW")
+  | where isnotempty(SourceIP) and isnotempty(DestinationIP)
+  | extend Combined = strcat(
+      tostring(Message), " ",
+      tostring(AdditionalExtensions), " ",
+      tostring(Activity), " ",
+      tostring(DeviceEventClassID), " ",
+      tostring(column_ifexists("FlexString1", "")), " ",
+      tostring(column_ifexists("FlexString2", "")), " ",
+      tostring(column_ifexists("DeviceCustomString1", "")), " ",
+      tostring(column_ifexists("DeviceCustomString2", ""))
+    )
+  | extend ParsedGid = toint(extract(@"(?i)(?:gid|generator[\s_-]?id)[\s:=]*(\d+)", 1, Combined))
+  | extend IsMlOnly = ParsedGid == 411 or Combined has "is_ml_only"
+  | extend IsSignatureHigh = not(IsMlOnly)
+      and not(Combined has_any ("SnortML", "snortml"))
+      and (
+          Combined has_any (HighPriorityClassifications)
+          or Activity has_any (HighPriorityClassifications)
+      );
+  let Signatures = Base
+  | where IsSignatureHigh
+  | summarize SigTime=max(TimeGenerated), SigActivity=take_any(Activity), SigMessage=take_any(Message), DeviceName=take_any(DeviceName)
+      by SourceIP, DestinationIP, DestinationPort=tostring(DestinationPort), TimeBin=bin(TimeGenerated, 1m);
+  let MlOnly = Base
+  | where IsMlOnly and not(Combined has "is_corroborated")
+  | summarize MlTime=max(TimeGenerated), MlActivity=take_any(Activity), MlMessage=take_any(Message)
+      by SourceIP, DestinationIP, DestinationPort=tostring(DestinationPort), TimeBin=bin(TimeGenerated, 1m);
+  Signatures
+  | join kind=inner MlOnly on SourceIP, DestinationIP
+  | where abs(datetime_diff('minute', SigTime, MlTime)) <= 5
+  | summarize arg_max(SigTime, *) by SourceIP, DestinationIP
+  | extend HostCustomEntity = DeviceName, SrcIpCustomEntity = SourceIP, DstIpCustomEntity = DestinationIP
+  | project SigTime, MlTime, DeviceName, SourceIP, DestinationIP, SigActivity, MlActivity, SigMessage, MlMessage,
+      HostCustomEntity, SrcIpCustomEntity, DstIpCustomEntity
+entityMappings:
+  - entityType: Host
+    fieldMappings:
+      - identifier: FullName
+        columnName: HostCustomEntity
+  - entityType: IP
+    fieldMappings:
+      - identifier: Address
+        columnName: SrcIpCustomEntity
+  - entityType: IP
+    fieldMappings:
+      - identifier: Address
+        columnName: DstIpCustomEntity
+version: 1.0.2
+kind: Scheduled

Solutions/Cisco Firepower EStreamer/Analytic Rules/CiscoFirepower-SnortML-GID411-MLOnly.yaml

@@ -0,0 +1,60 @@
+id: bab70c8d-220e-46dc-aef2-1411eb43284e
+name: Cisco Firepower - SnortML GID 411 ML-only high alert
+description: |
+  'Detects Cisco Firepower / Snort-family intrusion events generated by SnortML (Generator ID / GID 411).
+  SnortML scores are machine-learning probability signals and must not be treated as equivalent to a classic Snort signature true positive (typically GID 1).
+  High ML-only confidence should escalate for corroboration - not automatic containment via BlockIP playbooks.
+  Pair with "Cisco Firepower - IDS signature high priority classification" and "Cisco Firepower - Signature and ML corroboration".
+  Related portable encodings: OCSF is_ml_only (ocsf-schema#1732), SigmaHQ/sigma#6237, elastic/detection-rules#6662.'
+severity: Medium
+status: Available
+requiredDataConnectors:
+  - connectorId: CefAma
+    dataTypes:
+      - CommonSecurityLog
+queryFrequency: 15m
+queryPeriod: 15m
+triggerOperator: gt
+triggerThreshold: 0
+tactics:
+  - CommandAndControl
+  - Exfiltration
+relevantTechniques:
+  - T1071
+  - T1041
+query: |
+  CommonSecurityLog
+  | where DeviceVendor =~ "Cisco"
+  | where DeviceProduct has_any ("Firepower", "Secure Firewall", "FTD", "NGFW")
+  | extend Combined = strcat(
+      tostring(Message), " ",
+      tostring(AdditionalExtensions), " ",
+      tostring(Activity), " ",
+      tostring(DeviceEventClassID), " ",
+      tostring(column_ifexists("FlexString1", "")), " ",
+      tostring(column_ifexists("FlexString2", "")), " ",
+      tostring(column_ifexists("DeviceCustomString1", "")), " ",
+      tostring(column_ifexists("DeviceCustomString2", "")), " ",
+      tostring(column_ifexists("DeviceCustomString3", ""))
+    )
+  | extend ParsedGid = toint(extract(@"(?i)(?:gid|generator[\s_-]?id)[\s:=]*(\d+)", 1, Combined))
+  | where ParsedGid == 411 or Combined has "is_ml_only"
+  | where not(Combined has "is_corroborated")
+  | extend HostCustomEntity = DeviceName, SrcIpCustomEntity = SourceIP, DstIpCustomEntity = DestinationIP
+  | project TimeGenerated, DeviceName, SourceIP, DestinationIP, DestinationPort, Activity, DeviceAction,
+      DeviceEventClassID, Message, AdditionalExtensions, HostCustomEntity, SrcIpCustomEntity, DstIpCustomEntity
+entityMappings:
+  - entityType: Host
+    fieldMappings:
+      - identifier: FullName
+        columnName: HostCustomEntity
+  - entityType: IP
+    fieldMappings:
+      - identifier: Address
+        columnName: SrcIpCustomEntity
+  - entityType: IP
+    fieldMappings:
+      - identifier: Address
+        columnName: DstIpCustomEntity
+version: 1.0.2
+kind: Scheduled

Solutions/Cisco Firepower EStreamer/Data/Solution_Cisco Firepower EStreamer.json

@@ -5,7 +5,16 @@
   "Description": "The [Cisco Firepower](https://www.cisco.com/site/in/en/products/security/firewalls/index.html) eStreamer Solution for Microsoft Sentinel works with [Cisco Event Streamer](https://github.com/CiscoSecurity/fp-05-microsoft-sentinel-connector) (also known as eStreamer) and allows you to stream System intrusion, discovery and connection data from Firepower Management Center (FMC) or managed device to Microsoft Sentinel \r\n eStreamer is a Client Server API designed for the Cisco Firepower NGFW Solution. The eStreamer client requests detailed event data on behalf of the SIEM or logging solution in the Common Event Format (CEF).\n\n This solution is dependent on the Common Event Format solution containing the CEF via AMA connector to collect the logs. The CEF solution will be installed as part of this solution installation.\n\n**NOTE: **Microsoft recommends installation of CEF via AMA Connector. The existing connectors are about to be deprecated by Aug 31, 2024.",
   "Data Connectors": [
     "Data Connectors/CiscoFirepowerEStreamerCollector.json",
-	"Data Connectors/template_CiscoFirepowerEStreamerAMA.json"
+    "Data Connectors/template_CiscoFirepowerEStreamerAMA.json"
+  ],
+  "Analytic Rules": [
+    "Analytic Rules/CiscoFirepower-SnortML-GID411-MLOnly.yaml",
+    "Analytic Rules/CiscoFirepower-IDS-Signature-HighPriority.yaml",
+    "Analytic Rules/CiscoFirepower-Signature-And-ML-Corroboration.yaml",
+    "Analytic Rules/CiscoFirepower-Signal-Mix-Drift.yaml"
+  ],
+  "Workbooks": [
+    "Workbooks/CiscoFirepowerDetectionResponseQuality.json"
   ],
   "Playbooks": [
     "Playbooks/CiscoFirepowerConnector/azuredeploy.json",
@@ -15,10 +24,10 @@
   ],
   "dependentDomainSolutionIds": [
     "azuresentinel.azure-sentinel-solution-commoneventformat"
-   ],
+  ],
   "BasePath": "C:\\Github\\Azure-Sentinel\\Solutions\\Cisco Firepower EStreamer",
-  "Version": "3.0.1",
+  "Version": "3.1.0",
   "Metadata": "SolutionMetadata.json",
   "TemplateSpec": true,
   "Is1Pconnector": false
-}
+}

Solutions/Cisco Firepower EStreamer/Evaluation/FirepowerOutcome-v1.md

@@ -0,0 +1,38 @@
+# Cisco Firepower response outcome contract v1
+
+This contract makes detection and response decisions machine-readable without introducing a new data store or an AI dependency. Playbooks emit the record in a Microsoft Sentinel incident comment. Humans remain authoritative for containment decisions.
+
+## Record format
+
+```text
+[FirepowerOutcome:v1] signal=<ml-only|signature|corroborated|unknown>; decision=<policy-denied|approved|rejected|not-required|unknown>; containment=<not-attempted|succeeded|failed|unchanged>; reason=<stable-reason-code>; ruleVersion=<version>; policyVersion=1.0.0
+```
+
+Values are deliberately bounded. Free-form analyst explanation may follow the record but must not replace it.
+
+## Safety invariants
+
+1. `signal=ml-only` cannot produce automatic containment. The Teams HITL path requires an explicit analyst decision and records that decision before any change.
+2. An AI-generated recommendation cannot modify production analytics, policies, or Firepower objects directly.
+3. Candidate changes must be replayed against the cases below and reviewed by a human.
+4. Every promoted change records its rule, playbook, policy, and evaluation-corpus versions.
+5. Ambiguous parsing fails closed to `signal=unknown` and cannot silently become an automatic containment path.
+
+## Deterministic evaluation cases
+
+| Case | Evidence | Expected signal | Expected decision/outcome |
+|---|---|---|---|
+| E01 | GID 411 only | `ml-only` | `policy-denied/not-attempted` |
+| E02 | `is_ml_only` only | `ml-only` | `policy-denied/not-attempted` |
+| E03 | GID 1 and high-priority classification | `signature` | eligible for policy-controlled response |
+| E04 | GID 411 plus independent signature for the same flow/window | `corroborated` | eligible for HITL response |
+| E05 | malformed or missing GID | `unknown` | no automatic containment |
+| E06 | no IP entity | any | `not-required/not-attempted` |
+| E07 | Firepower object does not exist | any eligible | `approved/failed` |
+| E08 | Firepower update succeeds | any eligible | `approved/succeeded` |
+| E09 | analyst rejects Teams card | any | `rejected/not-attempted` |
+| E10 | Teams approval expires | any | `unknown/not-attempted` |
+
+## Controlled improvement loop
+
+Outcome records and workbook trends may be used by an external agent to propose KQL, mapping, threshold, or playbook changes. A proposal must include the triggering evidence, a diff, replay results for every evaluation case, cost impact, safety-invariant results, and a rollback condition. Promotion occurs only through a reviewed pull request and canary deployment.