diff --git a/README.md b/README.md index 2ed8c5c..73e1752 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,13 @@ This Go-based MCP server acts as a bridge between AI applications and Collibra, - [`get_asset_details`](pkg/tools/get_asset_details/) - Retrieve detailed information about specific assets by UUID, including the asset's assignable attribute schema (every attribute it can hold, including empty ones) - [`get_business_term_data`](pkg/tools/get_business_term_data/) - Trace a business term back to its connected physical data assets - [`get_column_semantics`](pkg/tools/get_column_semantics/) - Retrieve data attributes, measures, and business assets connected to a column +- [`get_data_quality_rule`](pkg/tools/get_dq_rule/) - Read the definition of a single DQ rule (monitor) on a job — its type, SQL, filter, tolerance and active/suppressed state +- [`get_data_quality_rule_results`](pkg/tools/get_dq_rule_results/) - Read a rule's per-run results after a job run — score, breaking/passing record counts, pass/fail status and any exception. Paginated (`offset`/`limit`), newest first by default +- [`validate_data_quality_rule`](pkg/tools/validate_dq_rule/) - Validate a rule's SQL/definition against the source before saving or running it, so a malformed rule is caught up front. Returns whether the rule is valid plus the engine's message. Requires `edgeSiteId`/`connectionId`/`schemaName` (from `prepare_create_data_quality_job`) +- [`list_data_quality_rule_templates`](pkg/tools/list_dq_rule_templates/) - List the DQ rule templates (built-in + custom) available in the connected environment via the public DQ API. Each is a parameterized SQL pattern deployable via `deploy_data_quality_rule_template` (by its `ruleTemplateName`). Filters: `name`, `dimension`, `isSystem` (built-in vs custom); paginated +- [`get_data_quality_rule_template`](pkg/tools/get_dq_rule_template/) - Read a single DQ rule template by `ruleTemplateName` — its parameterized SQL, dialect, dimensions, default tolerance, built-in (`isSystem`) flag and deployed-rule count +- [`find_data_quality_rules`](pkg/tools/find_dq_rules/) - Search existing DQ rules (monitors) across jobs. Filter by exact `jobName` and/or `columnName` (combine both to detect rules already on a target column) or a rule-name substring. Returns each rule's job, column, type, status and SQL; paginated +- [`generate_data_quality_rule_sql`](pkg/tools/generate_dq_rule_sql/) - Turn a plain-language description of a check into rule SQL (Text2SQL / Collibra DQ AI), so a rule can be authored without writing SQL. Returns a single SQL string (validate before use). Requires `edgeSiteId`/`connectionId` (from `prepare_create_data_quality_job`) - [`get_lineage_downstream`](pkg/tools/get_lineage_downstream/) - Get downstream technical lineage (consumers) for a data entity - [`get_lineage_entity`](pkg/tools/get_lineage_entity/) - Get metadata about a specific entity in the technical lineage graph - [`get_lineage_transformation`](pkg/tools/get_lineage_transformation/) - Get details and logic of a specific data transformation @@ -27,6 +34,7 @@ This Go-based MCP server acts as a bridge between AI applications and Collibra, - [`prepare_create_asset`](pkg/tools/prepare_create_asset/) - Read-only companion to `create_asset`: enumerate available asset types and domains, resolve a UUID/publicId/displayName for either, and hydrate the scoped attribute and relation schema for a chosen pair - [`pull_data_contract_manifest`](pkg/tools/pull_data_contract_manifest/) - Download manifest for a data contract - [`search_asset_keyword`](pkg/tools/search_asset_keyword/) - Wildcard keyword search for assets; filters (status, community, domain, domain type, asset type, created-by) accept names or UUIDs +- [`search_catalog_columns`](pkg/tools/search_catalog_columns/) - Find catalog Column assets by metadata that keyword search can't filter on — Description/Data Type (attribute values), a Data Steward role, or relations to a Business Term/Business Rule/Data Element/Data Attribute (by name); AND-combined. Uses the DGC Knowledge Graph GraphQL API (must be enabled on the instance). Classification-tag filtering is not supported - [`search_data_class`](pkg/tools/search_data_classes/) - Search for data classes with filters. **Requires:** `dgc.data-classes-read` - [`search_data_classification_match`](pkg/tools/search_data_classification_matches/) - Search for associations between data classes and assets. **Requires:** `dgc.classify`, `dgc.catalog` - [`search_lineage_entities`](pkg/tools/search_lineage_entities/) - Search for entities in the technical lineage graph @@ -37,6 +45,8 @@ This Go-based MCP server acts as a bridge between AI applications and Collibra, - [`add_data_classification_match`](pkg/tools/add_data_classification_match/) - Associate a data class with an asset. **Requires:** `dgc.classify`, `dgc.catalog` - [`create_assessment`](pkg/tools/create_assessment/) - Conduct a new assessment from a template (given by name or UUID) in the Assessments application. Returns the template's (unanswered) questions to fill in afterward with `edit_assessment` — no separate prepare step needed - [`create_asset`](pkg/tools/create_asset/) - Create a new asset of any type. Resolves `assetType` (UUID, publicId, or display name), `domain` (UUID or name), `status` (UUID or name), and attributes (by name or typeId) server-side; converts Markdown to HTML for `RICH_TEXT` attributes; gates on duplicate-name (default `allowDuplicate: false`) +- [`create_data_quality_rule`](pkg/tools/create_dq_rule/) - Create a data quality rule (monitor) on an existing DQ job. `monitorType` is `FREEFORM_SQL` (full SQL query) or `SIMPLE_SQL` (single-column check); defaults to active and not suppressed. Confirm checkpoint: `confirm=false` (default) returns a preview of the rule + SQL without creating; `confirm=true` creates. Uses the DQ monitoring API and requires permission to create rules on the target job. **Experimental** (`data-quality` feature flag) +- [`deploy_data_quality_rule_template`](pkg/tools/deploy_dq_rule_template/) - Instantiate a rule template as concrete rules across one or more job/column targets (bulk). The DQ service resolves dialect-specific SQL and names each rule `{templateName}_{columnName}`. Confirm checkpoint: `confirm=false` (default) previews the template + targets without deploying; `confirm=true` deploys. Requires permission to deploy templates and create rules on the target jobs. **Experimental** (`data-quality` feature flag) - [`edit_assessment`](pkg/tools/edit_assessment/) - Edit a conducted assessment (identified by name or UUID) via a list of typed operations, applied as a single atomic PATCH (all-or-nothing): - `set_answer` - set a question's answer by `questionId`: TEXT/HTML/EXPRESSION/NUMBER/BOOLEAN/DATE via `value`, or ITEMS (choice) via `items`; supply `answerType` for a not-yet-answered question (an already-answered question's type is inferred). ASSETS/USERORGROUPS/ATTACHMENTS answer types are not yet supported - `set_status` - move status (`DRAFT`, `SUBMITTED`, `OBSOLETE`) diff --git a/cmd/chip/experimental.go b/cmd/chip/experimental.go index 9106eed..15266bc 100644 --- a/cmd/chip/experimental.go +++ b/cmd/chip/experimental.go @@ -19,7 +19,7 @@ import ( var knownExperimentalFeatures = map[string]string{ skills.FeatureName: "Embedded skill catalog served via list_collibra_skills and load_collibra_skill.", tools.ContextSpecificationsFeature: "Context specification tools: list_context_specifications, get_context_specification, and contextSpecificationId parameter on get_asset_details.", - tools.DataQualityFeatureName: "Data Quality job tool (create_data_quality_job: discovery + preview + create in one) that creates and queues DQ jobs.", + tools.DataQualityFeatureName: "Data quality tools: create data quality jobs (create_data_quality_job: discovery + preview + create in one); create, validate, read and search rules; per-run rule results; rule templates (list, read, deploy); plain-language (Text2SQL) rule generation; and catalog column search.", } // validateExperimental warns (without exiting) when the user enabled an diff --git a/go.mod b/go.mod index 314e163..b661198 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect - github.com/rogpeppe/go-internal v1.10.0 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/sagikazarmark/locafero v0.12.0 // indirect github.com/segmentio/asm v1.1.3 // indirect github.com/segmentio/encoding v0.5.4 // indirect @@ -26,7 +26,7 @@ require ( github.com/yosida95/uritemplate/v3 v3.0.2 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/oauth2 v0.35.0 // indirect - golang.org/x/sys v0.41.0 // indirect - golang.org/x/text v0.32.0 // indirect + golang.org/x/sys v0.45.0 // indirect + golang.org/x/text v0.34.0 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go.sum b/go.sum index 94fbd25..41bffe3 100644 --- a/go.sum +++ b/go.sum @@ -30,8 +30,7 @@ github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0 github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4= github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI= github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc= @@ -58,10 +57,10 @@ go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= -golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= -golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU= -golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/clients/dgc_kg_client.go b/pkg/clients/dgc_kg_client.go new file mode 100644 index 0000000..5ab0eb8 --- /dev/null +++ b/pkg/clients/dgc_kg_client.go @@ -0,0 +1,211 @@ +package clients + +import ( + "context" + "encoding/json" + "fmt" + "net/http" +) + +// This client queries the DGC Knowledge Graph GraphQL API +// (POST /graphql/knowledgeGraph/v1) to find catalog Column assets by metadata +// that the public REST search cannot filter on — attribute values, assigned +// responsibilities (roles), and relations to other assets. It is built against +// the deployed KG schema (assets(where: AssetFilter)); if a target environment +// runs a different KG version the filter shapes may differ. + +const kgEndpoint = "/graphql/knowledgeGraph/v1" + +// Column asset type and the OOTB (is_system) relation-type public ids that link +// a Column to each catalog asset type, with the direction of the relation +// relative to the Column and which end holds the other asset. +const ( + kgColumnAssetType = "Column" + + // Business Term (Business Asset) --represents--> Column (Data Asset): the + // Business Term is the source, so from the Column it is an incoming relation. + relBusinessTermPublicID = "BusinessAssetRepresentsDataAsset" + // Column (Asset) --governed by--> Business Rule (Governance Asset): Column is + // the source, so it is an outgoing relation with the rule as the target. + relBusinessRulePublicID = "AssetGovernedByGovernanceAsset" + // Data Element --targets--> Data Element (lineage); Column is a Data Element. + relDataElementPublicID = "DataElementTargetsDataElement" + // Data Attribute --represents--> Column: Data Attribute is the source, so from + // the Column it is an incoming relation. + relDataAttributePublicID = "DataAttributeRepresentsColumn" + + // Attribute type names used for the value filters. + attrDescription = "Description" + attrDataType = "Data Type" +) + +// CatalogColumnSearchParams are the metadata filters, ANDed together. Empty +// fields are omitted. A specific attribute-type name / relation public-id is +// applied per field (see constants above). +type CatalogColumnSearchParams struct { + Domain string + Community string + Description string + DataType string + StewardRole string + BusinessTerm string + BusinessRule string + DataElement string + DataAttribute string + Limit int + Offset int +} + +// CatalogColumn is one matching column returned by the search. +type CatalogColumn struct { + ID string `json:"id"` + FullName string `json:"fullName"` + DisplayName string `json:"displayName"` + Type struct { + Name string `json:"name"` + } `json:"type"` + Domain struct { + Name string `json:"name"` + } `json:"domain"` +} + +type kgSearchResponse struct { + Data struct { + Assets []CatalogColumn `json:"assets"` + } `json:"data"` + Errors []struct { + Message string `json:"message"` + } `json:"errors"` +} + +const kgColumnsQuery = `query Search($where: AssetFilter, $limit: Int, $offset: Int) { + assets(where: $where, limit: $limit, offset: $offset) { + id + fullName + displayName + type { name } + domain { name } + } +}` + +// eq builds a StringFilter {eq: v}; contains builds {contains: v}. +func kgEq(v string) map[string]any { return map[string]any{"name": map[string]any{"eq": v}} } +func kgStringEq(v string) map[string]any { return map[string]any{"eq": v} } + +// relationFragment builds an incoming/outgoing relation filter fragment matching +// the other end's displayName. dir is "incomingRelations" or "outgoingRelations"; +// end is "source" or "target" (the end that holds the related asset). +func relationFragment(dir, publicID, end, value string) map[string]any { + return map[string]any{ + dir: map[string]any{ + "typePublicId": publicID, + "any": map[string]any{ + end: map[string]any{"displayName": kgStringEq(value)}, + }, + }, + } +} + +// stringAttributeFragment matches a string attribute of the given type name whose +// value contains v. +func stringAttributeFragment(attrTypeName, v string) map[string]any { + return map[string]any{ + "stringAttributes": map[string]any{ + "any": map[string]any{ + "type": kgEq(attrTypeName), + "stringValue": map[string]any{"contains": v}, + }, + }, + } +} + +// buildWhere assembles the AssetFilter. Fragments that reuse the same top-level +// key (stringAttributes, incomingRelations, outgoingRelations) cannot coexist in +// one object, so all fragments are chained through the singular `_and`. +func buildWhere(p CatalogColumnSearchParams) map[string]any { + var frags []map[string]any + + // Always scope to Column. + frags = append(frags, map[string]any{"type": kgEq(kgColumnAssetType)}) + + if p.Domain != "" || p.Community != "" { + domain := map[string]any{} + if p.Domain != "" { + domain["name"] = kgStringEq(p.Domain) + } + if p.Community != "" { + domain["parent"] = map[string]any{"name": kgStringEq(p.Community)} + } + frags = append(frags, map[string]any{"domain": domain}) + } + if p.Description != "" { + frags = append(frags, stringAttributeFragment(attrDescription, p.Description)) + } + if p.DataType != "" { + frags = append(frags, stringAttributeFragment(attrDataType, p.DataType)) + } + if p.StewardRole != "" { + frags = append(frags, map[string]any{ + "responsibilities": map[string]any{ + "any": map[string]any{"role": kgEq(p.StewardRole)}, + }, + }) + } + if p.BusinessTerm != "" { + frags = append(frags, relationFragment("incomingRelations", relBusinessTermPublicID, "source", p.BusinessTerm)) + } + if p.BusinessRule != "" { + frags = append(frags, relationFragment("outgoingRelations", relBusinessRulePublicID, "target", p.BusinessRule)) + } + if p.DataElement != "" { + frags = append(frags, relationFragment("outgoingRelations", relDataElementPublicID, "target", p.DataElement)) + } + if p.DataAttribute != "" { + frags = append(frags, relationFragment("incomingRelations", relDataAttributePublicID, "source", p.DataAttribute)) + } + + // Fold the fragments into a single object chained by `_and`. + root := frags[0] + cur := root + for _, f := range frags[1:] { + cur["_and"] = f + cur = f + } + return root +} + +// SearchCatalogColumns finds catalog Column assets matching the given metadata +// filters via the Knowledge Graph GraphQL API. Requires that KG endpoint to be +// available on the target instance. +func SearchCatalogColumns(ctx context.Context, client *http.Client, params CatalogColumnSearchParams) ([]CatalogColumn, error) { + limit := params.Limit + if limit <= 0 { + limit = 25 + } + body := map[string]any{ + "query": kgColumnsQuery, + "variables": map[string]any{ + "where": buildWhere(params), + "limit": limit, + "offset": params.Offset, + }, + } + respBody, status, err := dqDo(ctx, client, http.MethodPost, kgEndpoint, body) + if err != nil { + return nil, fmt.Errorf("searching catalog columns: %w", err) + } + if status != http.StatusOK { + if status == http.StatusNotFound { + return nil, fmt.Errorf("searching catalog columns: knowledge graph endpoint not available on this instance (HTTP 404): %s", string(respBody)) + } + return nil, fmt.Errorf("searching catalog columns: unexpected status %d: %s", status, string(respBody)) + } + var resp kgSearchResponse + if err := json.Unmarshal(respBody, &resp); err != nil { + return nil, fmt.Errorf("searching catalog columns: decoding response: %w", err) + } + if len(resp.Errors) > 0 { + return nil, fmt.Errorf("searching catalog columns: graphql error: %s", resp.Errors[0].Message) + } + return resp.Data.Assets, nil +} diff --git a/pkg/clients/dq_ai_client.go b/pkg/clients/dq_ai_client.go new file mode 100644 index 0000000..3974dd1 --- /dev/null +++ b/pkg/clients/dq_ai_client.go @@ -0,0 +1,50 @@ +package clients + +import ( + "context" + "encoding/json" + "fmt" + "net/http" +) + +// Text2SQLRequest is the request body for POST /rest/dq/internal/v1/ai/text2sql. +// It turns a natural-language rule description into rule SQL. The table is +// identified by jobName; columns give the relevant column context. This endpoint +// is internal-only (not part of the public DQ API) and is not in the OAS spec. +type Text2SQLRequest struct { + EdgeSiteID string `json:"edgeSiteId"` + ConnectionID string `json:"connectionId"` + Query string `json:"query"` + JobName string `json:"jobName"` + Columns []string `json:"columns"` +} + +// Text2SQLResponse is the response: a single generated SQL string. There is no +// separate filter/WHERE-predicate field — the engine returns one query. +type Text2SQLResponse struct { + SQLQuery string `json:"sqlQuery"` +} + +// GenerateDQRuleSQL turns a natural-language description into rule SQL — +// POST /rest/dq/internal/v1/ai/text2sql. +func GenerateDQRuleSQL(ctx context.Context, client *http.Client, request Text2SQLRequest) (*Text2SQLResponse, error) { + respBody, status, err := dqDo(ctx, client, http.MethodPost, "/rest/dq/internal/v1/ai/text2sql", request) + if err != nil { + return nil, fmt.Errorf("generating dq rule sql: %w", err) + } + if status != http.StatusOK { + switch status { + case http.StatusBadRequest: + return nil, fmt.Errorf("generating dq rule sql: bad request (the description could not be turned into valid SQL): %s", string(respBody)) + case http.StatusForbidden: + return nil, fmt.Errorf("generating dq rule sql: missing permission to use DQ AI: %s", string(respBody)) + default: + return nil, fmt.Errorf("generating dq rule sql: unexpected status %d: %s", status, string(respBody)) + } + } + var result Text2SQLResponse + if err := json.Unmarshal(respBody, &result); err != nil { + return nil, fmt.Errorf("generating dq rule sql: decoding response: %w", err) + } + return &result, nil +} diff --git a/pkg/clients/dq_rule_search_client.go b/pkg/clients/dq_rule_search_client.go new file mode 100644 index 0000000..614e6ff --- /dev/null +++ b/pkg/clients/dq_rule_search_client.go @@ -0,0 +1,82 @@ +package clients + +import ( + "context" + "encoding/json" + "fmt" + "net/http" +) + +// DQMonitorFilter is one filter clause for the monitors dashboard query. Field +// is a MonitorFilterableField (e.g. JOB_NAME, COLUMN_NAME, MONITOR_NAME); +// Operator is a FilterOperator (e.g. EQUALS, CONTAINS). +type DQMonitorFilter struct { + Field string `json:"field"` + Operator string `json:"operator"` + Values []string `json:"values"` +} + +// dqDashboardRequest is the request body for the monitors dashboard query. When +// FilterFormula is empty the service ANDs all filters together. +type dqDashboardRequest struct { + Filters []DQMonitorFilter `json:"filters"` + SortField string `json:"sortField,omitempty"` + SortOrder string `json:"sortOrder,omitempty"` + Offset int `json:"offset"` + Limit int `json:"limit"` +} + +// DQMonitorSummary is one rule (monitor) row from the dashboard query — enough to +// recognize an existing/duplicate rule on a column. +type DQMonitorSummary struct { + MonitorName string `json:"monitorName"` + MonitorType string `json:"monitorType"` + MonitorStatus string `json:"monitorStatus"` + ColumnName string `json:"columnName"` + JobName string `json:"jobName"` + ConnectionName string `json:"connectionName"` + SchemaName string `json:"schemaName"` + TableName string `json:"tableName"` + Dimensions []string `json:"dimensions"` + RuleQuery string `json:"ruleQuery"` + FilterQuery string `json:"filterQuery"` + Tolerance string `json:"tolerance"` +} + +type dqDashboardResponse struct { + Results []DQMonitorSummary `json:"results"` + Total int64 `json:"total"` + Offset int64 `json:"offset"` + Limit int64 `json:"limit"` +} + +// DQMonitorSearchResult is the paginated set of matching rules. +type DQMonitorSearchResult struct { + Results []DQMonitorSummary + Total int64 + Offset int64 + Limit int64 +} + +// FindDQRules searches existing rules (monitors) across jobs via the monitoring +// dashboard — POST /rest/dq/internal/v1/monitoring/monitors/dashboard. Pass +// filters (e.g. JOB_NAME + COLUMN_NAME EQUALS) to find rules on a specific +// column, for duplicate detection. Filters are ANDed together. +func FindDQRules(ctx context.Context, client *http.Client, filters []DQMonitorFilter, offset, limit int) (*DQMonitorSearchResult, error) { + req := dqDashboardRequest{Filters: filters, Offset: offset, Limit: limit} + respBody, status, err := dqDo(ctx, client, http.MethodPost, "/rest/dq/internal/v1/monitoring/monitors/dashboard", req) + if err != nil { + return nil, fmt.Errorf("finding dq rules: %w", err) + } + if status != http.StatusOK { + if status == http.StatusBadRequest { + return nil, fmt.Errorf("finding dq rules: bad request (invalid filter): %s", string(respBody)) + } + return nil, fmt.Errorf("finding dq rules: unexpected status %d: %s", status, string(respBody)) + } + var resp dqDashboardResponse + if err := json.Unmarshal(respBody, &resp); err != nil { + return nil, fmt.Errorf("finding dq rules: decoding response: %w", err) + } + return &DQMonitorSearchResult{Results: resp.Results, Total: resp.Total, Offset: resp.Offset, Limit: resp.Limit}, nil +} diff --git a/pkg/clients/dq_rules_client.go b/pkg/clients/dq_rules_client.go new file mode 100644 index 0000000..3512505 --- /dev/null +++ b/pkg/clients/dq_rules_client.go @@ -0,0 +1,246 @@ +package clients + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" +) + +// CreateDQRuleRequest is the request body for +// POST /rest/dq/internal/v1/monitoring/monitor. The field shapes mirror the +// DQ `Monitor` DTO so the wire format matches what the DQ service expects. +type CreateDQRuleRequest struct { + JobName string `json:"jobName"` + MonitorName string `json:"monitorName"` + MonitorType string `json:"monitorType"` + MonitorValue string `json:"monitorValue"` + FilterQuery string `json:"filterQuery,omitempty"` + ColumnName string `json:"columnName,omitempty"` + Description string `json:"description,omitempty"` + Dimensions []string `json:"dimensions,omitempty"` + Tolerance int `json:"tolerance"` + IsActive int `json:"isActive"` + IsSuppressed bool `json:"isSuppressed"` + TemplateID string `json:"templateId,omitempty"` +} + +// CreateDQRuleResponse is the response from +// POST /rest/dq/internal/v1/monitoring/monitor. +type CreateDQRuleResponse struct { + JobName string `json:"jobName"` + MonitorName string `json:"monitorName"` +} + +// CreateDQRule creates a data quality rule (monitor) on an existing DQ job. +func CreateDQRule(ctx context.Context, client *http.Client, request CreateDQRuleRequest) (*CreateDQRuleResponse, error) { + respBody, status, err := dqDo(ctx, client, http.MethodPost, "/rest/dq/internal/v1/monitoring/monitor", request) + if err != nil { + return nil, fmt.Errorf("creating dq rule: %w", err) + } + if status != http.StatusOK { + switch status { + case http.StatusBadRequest: + return nil, fmt.Errorf("creating dq rule: bad request (invalid rule definition): %s", string(respBody)) + case http.StatusForbidden: + return nil, fmt.Errorf("creating dq rule: missing permission to create rules on this job: %s", string(respBody)) + case http.StatusNotFound: + return nil, fmt.Errorf("creating dq rule: job or template not found: %s", string(respBody)) + case http.StatusUnprocessableEntity: + return nil, fmt.Errorf("creating dq rule: rule creation not allowed for this job (e.g. dataset is not of type PUSHDOWN): %s", string(respBody)) + default: + return nil, fmt.Errorf("creating dq rule: unexpected status %d: %s", status, string(respBody)) + } + } + + var result CreateDQRuleResponse + if err := json.Unmarshal(respBody, &result); err != nil { + return nil, fmt.Errorf("creating dq rule: decoding response: %w", err) + } + return &result, nil +} + +// dqDo executes a DQ API request against the Collibra client and returns the raw +// response body and status code. It marshals body (when non-nil) as JSON and only +// returns a non-nil error for transport/encoding failures — callers inspect the +// returned status code so they can map DQ error statuses to descriptive messages. +func dqDo(ctx context.Context, client *http.Client, method, path string, body any) ([]byte, int, error) { + var bodyReader io.Reader + if body != nil { + raw, err := json.Marshal(body) + if err != nil { + return nil, 0, fmt.Errorf("marshaling request: %w", err) + } + bodyReader = bytes.NewReader(raw) + } + + req, err := http.NewRequestWithContext(ctx, method, path, bodyReader) + if err != nil { + return nil, 0, fmt.Errorf("building request: %w", err) + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + req.Header.Set("Accept", "application/json") + + resp, err := client.Do(req) + if err != nil { + return nil, 0, fmt.Errorf("sending request: %w", err) + } + defer func() { _ = resp.Body.Close() }() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, resp.StatusCode, fmt.Errorf("reading response: %w", err) + } + return respBody, resp.StatusCode, nil +} + +// DQRule is the full monitor (rule) definition returned by GetDQRule. Its shape +// mirrors the DQ `Monitor` DTO. +type DQRule struct { + JobName string `json:"jobName"` + MonitorName string `json:"monitorName"` + MonitorType string `json:"monitorType"` + MonitorValue string `json:"monitorValue"` + FilterQuery string `json:"filterQuery,omitempty"` + ColumnName string `json:"columnName,omitempty"` + Description string `json:"description,omitempty"` + Dimensions []string `json:"dimensions,omitempty"` + Tolerance int `json:"tolerance"` + IsActive int `json:"isActive"` + IsSuppressed bool `json:"isSuppressed"` + TemplateID string `json:"templateId,omitempty"` +} + +// GetDQRule fetches a single rule (monitor) on a job by name — +// GET /rest/dq/internal/v1/jobs/{jobName}/monitors/rules/{monitorName}. +func GetDQRule(ctx context.Context, client *http.Client, jobName, monitorName string) (*DQRule, error) { + path := "/rest/dq/internal/v1/jobs/" + url.PathEscape(jobName) + "/monitors/rules/" + url.PathEscape(monitorName) + respBody, status, err := dqDo(ctx, client, http.MethodGet, path, nil) + if err != nil { + return nil, fmt.Errorf("getting dq rule: %w", err) + } + if status != http.StatusOK { + if status == http.StatusNotFound { + return nil, fmt.Errorf("getting dq rule: rule %q not found on job %q: %s", monitorName, jobName, string(respBody)) + } + return nil, fmt.Errorf("getting dq rule: unexpected status %d: %s", status, string(respBody)) + } + var result DQRule + if err := json.Unmarshal(respBody, &result); err != nil { + return nil, fmt.Errorf("getting dq rule: decoding response: %w", err) + } + return &result, nil +} + +// DQRuleResultEntry is one per-run result for a rule (a RuleDetailsEntry): the +// score, break counts and status for a single job run. +type DQRuleResultEntry struct { + Exception string `json:"exception,omitempty"` + RunDate int64 `json:"runDate"` + Score int `json:"score"` + BreakMsg string `json:"breakMsg,omitempty"` + RuleCondition string `json:"ruleCondition,omitempty"` + TotalCount float64 `json:"totalCount"` + BreakingRecords float64 `json:"breakingRecords"` + PassingRecords float64 `json:"passingRecords"` + RuleStatus string `json:"ruleStatus,omitempty"` + PassFail bool `json:"passFail"` + BreakingPerc float64 `json:"breakingPerc"` + PassingPerc float64 `json:"passingPerc"` +} + +// DQRuleResults is the paginated result set for a rule (a RuleDetails response): +// the rule's definition summary plus its per-run result entries. +type DQRuleResults struct { + Dataset string `json:"dataset"` + RuleName string `json:"ruleName"` + RuleType string `json:"ruleType"` + RuleValue string `json:"ruleValue"` + RuleValueBuilder string `json:"ruleValueBuilder,omitempty"` + FilterQuery string `json:"filterQuery,omitempty"` + Tolerance int `json:"tolerance"` + IsActive int `json:"isActive"` + Results []DQRuleResultEntry `json:"results"` + Total int64 `json:"total"` + Offset int64 `json:"offset"` + Limit int64 `json:"limit"` +} + +// GetDQRuleResults reads a rule's per-run results / breaking records — +// GET /rest/dq/internal/v1/monitoring/rules/{jobName}/{ruleName}. offset/limit +// paginate; sortOrder is "ASC" or "DESC" (defaults to DESC when empty). +func GetDQRuleResults(ctx context.Context, client *http.Client, jobName, ruleName string, offset, limit int, sortOrder string) (*DQRuleResults, error) { + q := url.Values{ + "offset": {fmt.Sprintf("%d", offset)}, + "limit": {fmt.Sprintf("%d", limit)}, + } + if sortOrder != "" { + q.Set("sortOrder", sortOrder) + } + path := "/rest/dq/internal/v1/monitoring/rules/" + url.PathEscape(jobName) + "/" + url.PathEscape(ruleName) + "?" + q.Encode() + respBody, status, err := dqDo(ctx, client, http.MethodGet, path, nil) + if err != nil { + return nil, fmt.Errorf("getting dq rule results: %w", err) + } + if status != http.StatusOK { + if status == http.StatusNotFound { + return nil, fmt.Errorf("getting dq rule results: rule %q not found on job %q: %s", ruleName, jobName, string(respBody)) + } + return nil, fmt.Errorf("getting dq rule results: unexpected status %d: %s", status, string(respBody)) + } + var result DQRuleResults + if err := json.Unmarshal(respBody, &result); err != nil { + return nil, fmt.Errorf("getting dq rule results: decoding response: %w", err) + } + return &result, nil +} + +// PreviewRuleRequest is the request body for both rule validation and SQL preview +// (POST /rest/dq/internal/v1/rules/validate and .../rules/previewRule). It mirrors +// the DQ `PreviewRuleRequest` DTO. +type PreviewRuleRequest struct { + EdgeSiteID string `json:"edgeSiteId"` + ConnectionID string `json:"connectionId"` + SchemaName string `json:"schemaName"` + JobName string `json:"jobName"` + PreviewRule string `json:"previewRule"` + FilterQuery string `json:"filterQuery,omitempty"` + RowLimit int `json:"rowLimit"` +} + +// ValidateDQRuleResponse is the validation verdict from +// POST /rest/dq/internal/v1/rules/validate. A rule that fails validation still +// returns HTTP 200 with IsValid=false and a Message explaining why. +type ValidateDQRuleResponse struct { + IsValid bool `json:"isValid"` + Message string `json:"message"` +} + +// ValidateDQRule checks that a rule's SQL/definition is valid before it is saved +// or run — POST /rest/dq/internal/v1/rules/validate. +func ValidateDQRule(ctx context.Context, client *http.Client, request PreviewRuleRequest) (*ValidateDQRuleResponse, error) { + respBody, status, err := dqDo(ctx, client, http.MethodPost, "/rest/dq/internal/v1/rules/validate", request) + if err != nil { + return nil, fmt.Errorf("validating dq rule: %w", err) + } + if status != http.StatusOK { + switch status { + case http.StatusForbidden: + return nil, fmt.Errorf("validating dq rule: missing permission: %s", string(respBody)) + case http.StatusNotFound: + return nil, fmt.Errorf("validating dq rule: connection or job not found: %s", string(respBody)) + default: + return nil, fmt.Errorf("validating dq rule: unexpected status %d: %s", status, string(respBody)) + } + } + var result ValidateDQRuleResponse + if err := json.Unmarshal(respBody, &result); err != nil { + return nil, fmt.Errorf("validating dq rule: decoding response: %w", err) + } + return &result, nil +} diff --git a/pkg/clients/dq_templates_client.go b/pkg/clients/dq_templates_client.go new file mode 100644 index 0000000..46a5c49 --- /dev/null +++ b/pkg/clients/dq_templates_client.go @@ -0,0 +1,192 @@ +package clients + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strconv" +) + +// DQRuleTemplate is a data quality rule template (a "RuleTemplate") — a +// parameterized SQL pattern (with a {{column}} placeholder) that can be deployed +// as concrete rules across many columns/jobs. IsSystem marks the built-in +// (system) templates; custom templates are user-defined. Field names follow the +// public /rest/dq/1.0/ruleTemplates API. +type DQRuleTemplate struct { + ID string `json:"id"` + Name string `json:"ruleTemplateName"` + Description string `json:"description,omitempty"` + SQL string `json:"sql,omitempty"` + Dialect string `json:"dialect,omitempty"` + Dimensions []string `json:"dimensions,omitempty"` + Tolerance *int `json:"tolerance,omitempty"` + IsSystem bool `json:"isSystem"` + DeployedRuleCount int64 `json:"deployedRuleCount"` +} + +// dqRuleTemplateListResponse is the paginated list envelope +// (RuleTemplatePaginated). +type dqRuleTemplateListResponse struct { + Results []DQRuleTemplate `json:"results"` + Total int64 `json:"total"` + Offset int64 `json:"offset"` + Limit int64 `json:"limit"` +} + +// DQRuleTemplateList is the returned page of templates plus pagination metadata. +type DQRuleTemplateList struct { + Results []DQRuleTemplate + Total int64 + Offset int64 + Limit int64 +} + +// ListDQRuleTemplatesParams are the optional filters/pagination for listing +// templates. IsSystem is a tri-state: nil = all, true = system (built-in) only, +// false = custom only. +type ListDQRuleTemplatesParams struct { + Name string + Dimension string + CreatedBy string + IsSystem *bool + SortBy string + SortDir string + Offset int + Limit int +} + +// ListDQRuleTemplates lists rule templates (system and custom) — +// GET /rest/dq/1.0/ruleTemplates. +func ListDQRuleTemplates(ctx context.Context, client *http.Client, params ListDQRuleTemplatesParams) (*DQRuleTemplateList, error) { + q := url.Values{} + if params.Name != "" { + q.Set("name", params.Name) + } + if params.Dimension != "" { + q.Set("dimension", params.Dimension) + } + if params.CreatedBy != "" { + q.Set("createdBy", params.CreatedBy) + } + if params.IsSystem != nil { + q.Set("isSystem", strconv.FormatBool(*params.IsSystem)) + } + if params.SortBy != "" { + q.Set("sortBy", params.SortBy) + } + if params.SortDir != "" { + q.Set("sortDir", params.SortDir) + } + q.Set("offset", strconv.Itoa(params.Offset)) + if params.Limit > 0 { + q.Set("limit", strconv.Itoa(params.Limit)) + } + + path := "/rest/dq/1.0/ruleTemplates" + if enc := q.Encode(); enc != "" { + path += "?" + enc + } + respBody, status, err := dqDo(ctx, client, http.MethodGet, path, nil) + if err != nil { + return nil, fmt.Errorf("listing dq rule templates: %w", err) + } + if status != http.StatusOK { + if status == http.StatusForbidden { + return nil, fmt.Errorf("listing dq rule templates: missing permission to view templates: %s", string(respBody)) + } + return nil, fmt.Errorf("listing dq rule templates: unexpected status %d: %s", status, string(respBody)) + } + var resp dqRuleTemplateListResponse + if err := json.Unmarshal(respBody, &resp); err != nil { + return nil, fmt.Errorf("listing dq rule templates: decoding response: %w", err) + } + return &DQRuleTemplateList{Results: resp.Results, Total: resp.Total, Offset: resp.Offset, Limit: resp.Limit}, nil +} + +// GetDQRuleTemplate fetches a single rule template by name — +// GET /rest/dq/1.0/ruleTemplates/{ruleTemplateName}. +func GetDQRuleTemplate(ctx context.Context, client *http.Client, ruleTemplateName string) (*DQRuleTemplate, error) { + path := "/rest/dq/1.0/ruleTemplates/" + url.PathEscape(ruleTemplateName) + respBody, status, err := dqDo(ctx, client, http.MethodGet, path, nil) + if err != nil { + return nil, fmt.Errorf("getting dq rule template: %w", err) + } + if status != http.StatusOK { + switch status { + case http.StatusForbidden: + return nil, fmt.Errorf("getting dq rule template: missing permission to view templates: %s", string(respBody)) + case http.StatusNotFound: + return nil, fmt.Errorf("getting dq rule template: template %q not found: %s", ruleTemplateName, string(respBody)) + default: + return nil, fmt.Errorf("getting dq rule template: unexpected status %d: %s", status, string(respBody)) + } + } + var tmpl DQRuleTemplate + if err := json.Unmarshal(respBody, &tmpl); err != nil { + return nil, fmt.Errorf("getting dq rule template: decoding response: %w", err) + } + return &tmpl, nil +} + +// DQTemplateDeployTarget is one deployment target: the job and, for a +// column-level template, the column substituted for the {{column}} placeholder. +// ColumnName and ConnectionName are optional. +type DQTemplateDeployTarget struct { + JobName string `json:"jobName"` + ColumnName string `json:"columnName,omitempty"` + ConnectionName string `json:"connectionName,omitempty"` +} + +// dqTemplateDeployRequest is the request body for the deploy endpoint. +type dqTemplateDeployRequest struct { + Targets []DQTemplateDeployTarget `json:"targets"` +} + +// DQTemplateDeployOutcome is the per-target result of a deploy. Status reports +// whether the target was deployed or skipped; Reason explains skips/failures. +type DQTemplateDeployOutcome struct { + JobName string `json:"jobName"` + ColumnName string `json:"columnName,omitempty"` + DeployedRuleName string `json:"deployedRuleName,omitempty"` + Status string `json:"status"` + Reason string `json:"reason,omitempty"` +} + +// DQTemplateDeployResult is the deploy response (RuleTemplateDeployResult): a +// per-target outcome list. Deploy is partial-success — individual targets may be +// deployed or skipped independently. +type DQTemplateDeployResult struct { + Results []DQTemplateDeployOutcome `json:"results"` +} + +// DeployDQRuleTemplate instantiates a template as concrete rules on the given +// targets, using dialect-specific SQL resolved server-side — +// POST /rest/dq/1.0/ruleTemplates/{ruleTemplateName}/deploy. The deploy is +// partial-success: it returns HTTP 200 with a per-target outcome list even when +// some targets are skipped. +func DeployDQRuleTemplate(ctx context.Context, client *http.Client, ruleTemplateName string, targets []DQTemplateDeployTarget) (*DQTemplateDeployResult, error) { + path := "/rest/dq/1.0/ruleTemplates/" + url.PathEscape(ruleTemplateName) + "/deploy" + respBody, status, err := dqDo(ctx, client, http.MethodPost, path, dqTemplateDeployRequest{Targets: targets}) + if err != nil { + return nil, fmt.Errorf("deploying dq rule template: %w", err) + } + if status != http.StatusOK { + switch status { + case http.StatusBadRequest: + return nil, fmt.Errorf("deploying dq rule template: bad request (e.g. invalid targets or incompatible template): %s", string(respBody)) + case http.StatusForbidden: + return nil, fmt.Errorf("deploying dq rule template: missing permission to deploy templates: %s", string(respBody)) + case http.StatusNotFound: + return nil, fmt.Errorf("deploying dq rule template: template %q not found: %s", ruleTemplateName, string(respBody)) + default: + return nil, fmt.Errorf("deploying dq rule template: unexpected status %d: %s", status, string(respBody)) + } + } + var result DQTemplateDeployResult + if err := json.Unmarshal(respBody, &result); err != nil { + return nil, fmt.Errorf("deploying dq rule template: decoding response: %w", err) + } + return &result, nil +} diff --git a/pkg/skills/files/collibra/dq-rule-workbench/SKILL.md b/pkg/skills/files/collibra/dq-rule-workbench/SKILL.md new file mode 100644 index 0000000..36a03a3 --- /dev/null +++ b/pkg/skills/files/collibra/dq-rule-workbench/SKILL.md @@ -0,0 +1,178 @@ +--- +description: Author data quality rules at scale against catalog columns — target columns, check for duplicates, define rules via templates or plain-language SQL, assign or create a job, and deploy in bulk with a partial-success model. +related: collibra/dq-rules, collibra/discovery, collibra/asset-edit +--- + +# Data quality rule workbench + +A multi-turn flow for creating DQ rules against one or more **catalog columns** at +scale — from templates (bulk) or plain-language intent (Text2SQL) — without the +user writing SQL by hand. This skill orchestrates existing tools; it does not add +new API surface. + +Relationship to the other DQ skills: +- **`collibra/dq-rules`** — the mechanics of a single rule on an existing job + (validate → create → inspect → read results). This workbench reuses those + rules and defers to that skill for the per-rule detail. +- **Job creation** — when a column has no suitable job, this flow calls + `prepare_create_data_quality_job` + `create_data_quality_job` (see their tool descriptions). + +## Tools this flow orchestrates + +- **Target columns**: `search_catalog_columns` (metadata filters — description, + data type, data-steward role, and relations to a business term / business rule / + data element / data attribute; needs the Knowledge Graph API), plus + `search_asset_keyword` (domain/community/asset-type + free-text), + `discover_data_assets` (natural-language), `get_asset_details` (by UUID). +- **Resolve DQ location / job + detect PUSHDOWN**: `prepare_create_data_quality_job` + (resolves a catalog Table asset → connection / edge / job and reports the job + type). `create_data_quality_job` when none exists. +- **Duplicate detection**: `find_data_quality_rules` (filter by `jobName` + `columnName`). +- **Define rules** — two paths: + - Templates: `list_data_quality_rule_templates` / `get_data_quality_rule_template` → + `deploy_data_quality_rule_template` (bulk). + - Plain-language / AI: `generate_data_quality_rule_sql` (Text2SQL) → `create_data_quality_rule`. +- **Review SQL**: `validate_data_quality_rule`. +- **Read results**: `get_data_quality_rule_results` (per-run rule outcomes, once the job has run). +- **Catalog associations**: `edit_asset` (`add_relation`) to Business Rule, Data + Element, Data Attribute, or a catalog Data Quality Rule asset. + +## Hard rules + +1. **Confirm at every checkpoint before any write.** The user must explicitly + approve (a) the confirmed column list, (b) the job assignment per group, and + (c) the final rule set. Never create, deploy, or run without that confirmation. +2. **Cap at 25 columns per invocation.** If targeting yields more, present the + top 25, state how many were excluded, and ask the user to refine. +3. **Rules require a PUSHDOWN, DQ-connected table.** Resolve each column's table + with `prepare_create_data_quality_job` and confirm the job type is PUSHDOWN. Columns + whose table is not connected to a DQ Pushdown source are excluded — silently + for search results (report the count + reason), but a **hard error** if the + user named such a column explicitly. +4. **Check for duplicates before creating.** For each targeted column call + `find_data_quality_rules` with `jobName` + `columnName`. Show any existing rule (name, + job, SQL) and have the user confirm-to-proceed or skip that column. +5. **Validate every rule's SQL before saving.** Run `validate_data_quality_rule` on each + generated or edited SQL. Do not create a rule whose SQL is invalid. See + `collibra/dq-rules`. +6. **A newly created job must complete at least one run before rules can be + added.** When the job-assignment step creates a job via the sub-flow, tell the + user about this dependency and confirm before proceeding; the new job runs + once (create auto-runs) before its rules are added. +7. **Partial success.** A failure on one rule does not abort the batch. Continue, + then report counts of created / skipped (with reasons) / failed (with reasons) + and offer retry, modify, or skip for the failures. +8. **Permissions are per job.** If a `403` comes back for a job, exclude that + job's columns with a clear message and continue with the rest. +9. **Metadata search via `search_catalog_columns` (needs the Knowledge Graph + API).** It filters columns by description, data type, a data-steward role, and + relations to a business term / business rule / data element / data attribute + (by name), AND-combined — plus domain/community. Two caveats: **Classification + Tags is not supported** (no KG predicate), and the tool errors if the KG API + isn't enabled on the instance. When KG is unavailable, or for a + classification-tag filter, fall back to `search_asset_keyword` + (domain/community + free-text) or explicit column naming, and say so. A broad + lone substring filter can hit the KG query timeout — combine filters. + +## Conversational flow + +Progress through these states; the user may revise within a state before moving on. + +**1. Column targeting.** Two modes: + - *Search*: use `search_catalog_columns` for metadata filters (description, + data type, data-steward role, business term / rule / data element / data + attribute), or `search_asset_keyword` for domain/community + free-text, or + `discover_data_assets` for a natural-language ask. Apply rule 9 (KG + availability, classification tags). + - *Explicit*: the user names columns by qualified path (`schema.table.column`) + or catalog asset name; resolve each with `search_asset_keyword` / + `get_asset_details`. + Enforce the 25-cap (rule 2) and the DQ/PUSHDOWN exclusion (rule 3). + +**2. Column confirmation.** Present a summary per column: qualified path, parent + domain/community, data type, existing-rule count (from `find_data_quality_rules`), and any + duplicate flag. The user confirms or edits the list (remove columns / restart + targeting) before advancing. + +**3. Rule definition.** Pick one path for the whole confirmed list: + - *Template*: `list_data_quality_rule_templates` (filter by dimension / OOTB), show the + choices, let the user pick one. There is **no** template/data-type + compatibility API, so incompatible combinations can't be pre-flagged — they + surface as errors at deploy (step 7); warn the user of this. + - *Plain-language / AI*: the user gives one intent; call `generate_data_quality_rule_sql` + per column (it needs `edgeSiteId`/`connectionId` from step 5's resolution and + the column list). Note Text2SQL returns a single SQL string — there is no + separate filter/WHERE clause; if the intent implies scoping, author a + `filterQuery` yourself and confirm it. + +**4. SQL review & revision.** For every rule, show the SQL and let the user + approve, edit SQL, revise the intent and regenerate, or skip the column. Run + `validate_data_quality_rule` on each version before presenting (rule 5). Loop until the + user approves or skips. + +**5. Job assignment.** For each column resolve a job in priority order: + 1. *Exact match* — an existing job covering the column's table + connection. + 2. *Connection match* — a job on the same connection but not yet the table. + 3. *No match* — create one via `prepare_create_data_quality_job` + `create_data_quality_job` + (rule 6 applies). + Group columns by connection, resolve one job per group, and show the full + grouping. The user confirms all assignments before any rule is saved (rule 1). + +**6. Final confirmation.** Summarize what will be written, and present it by path — + do not dump a flat list of template×column pairs: + - *Template path*: show the chosen template **once** (its name and what it + checks), then, grouped by job, the list of columns it will be applied to and + the resulting rule name per column (`{template}_{column}`). One template + header, then its columns — not a row per template/column combination. + - *Plain-language path*: list one entry per rule — column, job, rule name, and + the approved SQL (+ any filter) — since each rule's SQL can differ. + Then get explicit approval. + +**7. Execution.** Create the rules. Both write tools have a **confirm checkpoint**: + call with `confirm` omitted to get a `preview` (nothing is written), then call + again with `confirm: true` only after the step-6 approval. The tools enforce + this — a `confirm=false` call never writes. + - Template path: `deploy_data_quality_rule_template` with `targets` = the confirmed + `{jobName, columnName}` list (bulk; rules named `{template}_{column}`). + - Plain-language path: `create_data_quality_rule` per column (validated SQL as + `monitorValue`; a meaningful `monitorName` — see `collibra/dq-rules`). + Apply the partial-success model (rule 7), and report outcomes with job links. + The rules are evaluated on each job's next (scheduled) run — this flow does not + trigger runs. Once a job has run, read outcomes with `get_data_quality_rule_results`. + +## Rule settings & catalog associations + +Which settings you can set depends on the path: + +- **Plain-language path (`create_data_quality_rule`)**: set `dimensions` (default none), + `tolerance` (default 0), and `description` (default auto-generated from the + intent) per rule, bulk-uniform unless the user asks for per-column values. + **`tolerance` is a count of breaking records allowed before the rule fails, not + a percentage** — despite the ticket's "Tolerance %", the API takes an integer + record count. Say it that way to the user. +- **Template path (`deploy_data_quality_rule_template`)**: rules inherit the template's + `dimensions`, `tolerance` and `description`. The deploy call takes only + `{jobName, columnName}` targets, so these are **not** set per deploy — choose + (or create) a template that already carries the settings you want. + +**Notifications are job-level, not per-rule.** `create_data_quality_rule` cannot attach +notifications to a rule; notification recipients/triggers are configured when the +job is created (`create_data_quality_job` `notify*` fields) and apply to the whole job. Do +not tell the user a rule carries its own notification settings. + +To link created rules to catalog assets — Business Rule, Data Element, Data +Attribute, or a Data Quality Rule asset — use `edit_asset` `add_relation` by role +name. If a Business Rule asset was used as a targeting filter, pre-populate that +association and confirm it with the user. + +## Known limitations (state these when relevant) + +- Column metadata search (`search_catalog_columns`) needs the Knowledge Graph API + enabled, and does not support Classification Tags — see rule 9. +- Template/data-type compatibility is not pre-flagged (no API); incompatible + deploys fail at execution. +- Text2SQL returns one SQL string, not a primary-SQL + filter split. +- Per-rule notifications are not supported — notifications are job-level only + (`create_data_quality_job`). The ticket lists notifications as a rule setting; the tools + do not. +- `tolerance` is a breaking-record count, not the ticket's "Tolerance %". diff --git a/pkg/skills/files/collibra/dq-rules/SKILL.md b/pkg/skills/files/collibra/dq-rules/SKILL.md new file mode 100644 index 0000000..a09e520 --- /dev/null +++ b/pkg/skills/files/collibra/dq-rules/SKILL.md @@ -0,0 +1,72 @@ +--- +description: Author and validate custom data quality rules (monitors) on an existing DQ job, inspect them, and read their per-run results, using the validate/create/get/results DQ tools. +related: collibra/discovery, collibra/dq-rule-workbench +--- + +# Data quality rules + +A data quality **rule** (a "monitor") is a check attached to an existing DQ **job** +(a dataset — a saved data-quality check on one database table). This skill covers +authoring a custom rule (validate → create), inspecting it, and reading its per-run +results. It does **not** cover creating the job itself, editing or deleting rules, +or triggering job runs — a new rule is evaluated on the job's next (scheduled) run. + +Rule tools: `validate_data_quality_rule`, `create_data_quality_rule`, `get_data_quality_rule`, `get_data_quality_rule_results`. + +## Hard rules + +1. **Validate before you create.** Always call `validate_data_quality_rule` on the rule SQL *before* + `create_data_quality_rule`. It checks the SQL against the source and returns `valid: true/false` + plus a message, so a malformed rule (e.g. a bad `SIMPLE_SQL`/SQLG predicate) is caught up + front. `validate_data_quality_rule` takes the raw SQL — the rule does **not** need to exist yet. If + `valid` is `false`, fix the SQL and re-validate; do **not** create the rule. + - **`create_data_quality_rule` has a confirm checkpoint.** Call it first with `confirm` omitted/false: + it returns a `preview` (the composed rule and its SQL) and creates nothing. Show that + preview to the user, then call again with `confirm: true` to actually create. The tool + enforces this — it will not write on a `confirm=false` call. +2. **`validate_data_quality_rule` needs discovery IDs.** It requires `edgeSiteId`, `connectionId` and + `schemaName`. Get them from `prepare_create_data_quality_job` for the target job — do not guess them. + (`create_data_quality_rule`, `get_data_quality_rule` and `get_data_quality_rule_results` take only names/ids and need no + discovery step.) +3. **`monitorType` is `FREEFORM_SQL` or `SIMPLE_SQL`.** `FREEFORM_SQL` is a full SQL query; + `SIMPLE_SQL` is a single-column predicate. Nothing else is valid. +4. **Always give the rule a meaningful name.** `monitorName` is required and is how the rule is + found and reported on later. Ask the user for a name; if they don't supply one, propose a + clear, descriptive name (e.g. `orders_amount_not_null`) and confirm it before creating — do + not invent an opaque name. Names allow only letters, digits, `-` and `_`. +5. **For `SIMPLE_SQL`, ask which column the check targets** and pass it as `columnName`. For + `FREEFORM_SQL` the column(s) live inside the SQL, so `columnName` is not needed. +6. **Rules require a PUSHDOWN job.** If `create_data_quality_rule` returns an error mentioning the dataset + is not PUSHDOWN (HTTP 422), rule creation is not allowed on that job — tell the user rather + than retrying. +7. **Read the `status` field in every response.** Branch on `success`, `validation_error`, or + `error`. For `validate_data_quality_rule`, `status: success` means validation *ran* — the verdict is + the separate `valid` field. +8. **Creating a rule does not run it.** A new rule is only evaluated on the job's next run + (runs happen via the job's schedule — this skill does not trigger them). Once a run has + happened, use `get_data_quality_rule_results` to see how the rule did. + +## Workflow: author a rule + +1. **Discover** — call `prepare_create_data_quality_job` for the job to get `edgeSiteId`, `connectionId` + and `schemaName`. (Skip if you already have them.) +2. **Validate** — call `validate_data_quality_rule` with those IDs, the `jobName`, and `previewRule` + (the SQL you intend to use as `monitorValue`). If `valid` is `false`, fix and re-validate. +3. **Create** — only once valid, call `create_data_quality_rule`. First make sure you have a meaningful + `monitorName` from the user (rule 4) and, for a `SIMPLE_SQL` rule, the target `columnName` + (rule 5). Call with `confirm` omitted to preview, then `confirm: true` to create. Pass + `jobName`, `monitorName`, `monitorType`, `monitorValue` (and optional `filterQuery`, + `columnName`, `dimensions`, `tolerance`, `active`, `suppressed`). + +## Inspecting a rule + +`get_data_quality_rule` returns a rule's current definition (type, SQL, filter, tolerance, +active/suppressed) by `jobName` + `monitorName`. + +## Reading results + +`get_data_quality_rule_results` is paginated (`offset`/`limit`, newest first by default). Each entry is +one job run: `ruleStatus` (PASSING / BREAKING / EXCEPTION), `passFail`, `score`, `totalCount`, +`breakingRecords`, `passingRecords`, and `exception` when the run errored. Use it to confirm a +rule behaved as intended after the job has run. Results appear only once the job has executed +at least once since the rule was created. diff --git a/pkg/skills/files/collibra/index/SKILL.md b/pkg/skills/files/collibra/index/SKILL.md index 6fa4490..d9e9313 100644 --- a/pkg/skills/files/collibra/index/SKILL.md +++ b/pkg/skills/files/collibra/index/SKILL.md @@ -1,6 +1,6 @@ --- description: Navigator for chip's Collibra skills. Start here when unsure which skill applies. -related: collibra/discovery, collibra/lineage, collibra/asset-create, collibra/asset-edit, collibra/data-product-create, collibra/context +related: collibra/discovery, collibra/lineage, collibra/asset-create, collibra/asset-edit, collibra/data-product-create, collibra/context, collibra/dq-rules, collibra/dq-rule-workbench --- # Collibra skills — navigator @@ -19,6 +19,8 @@ must be bridged to another, and which permissions are required. | Modify an existing asset's attributes, relations, tags, status, or owners | `collibra/asset-edit` | | Register a table (and its dimension tables) as a Collibra Data Product with ports | `collibra/data-product-create` | | Generate governed YAML context (semantic blueprints, metric definitions) for an asset | `collibra/context` | +| Author, validate, run or inspect data quality rules (monitors) on a DQ job | `collibra/dq-rules` | +| Create DQ rules at scale across many catalog columns (templates or plain-language), with job assignment | `collibra/dq-rule-workbench` | If a task is a single tool call with no chaining (e.g. `get_asset_details` by UUID, `list_asset_types`, `pull_data_contract_manifest`), no skill is needed — the tool's own diff --git a/pkg/tools/create_dq_rule/tool.go b/pkg/tools/create_dq_rule/tool.go new file mode 100644 index 0000000..b36dda7 --- /dev/null +++ b/pkg/tools/create_dq_rule/tool.go @@ -0,0 +1,210 @@ +// Package create_dq_rule implements the create_dq_rule MCP tool: it creates a +// data quality rule (a "monitor") on an existing DQ job (dataset) via the DQ +// monitoring API. The agent supplies the job name, a rule name, the rule type +// and its SQL; the tool validates the inputs and writes the rule. +package create_dq_rule + +import ( + "context" + "fmt" + "net/http" + "regexp" + "strings" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +// monitorNameRe is the server's rule-name constraint: letters, digits, '-' and +// '_', 1-256 characters. Enforced up front so a bad name is a cheap, +// self-correcting error rather than a downstream 400. +var monitorNameRe = regexp.MustCompile(`^[A-Za-z0-9_-]{1,256}$`) + +// maxDescriptionLen is the server's description length cap. +const maxDescriptionLen = 256 + +// OutputStatus is the overall outcome of a create_dq_rule call. +type OutputStatus string + +const ( + // StatusSuccess means the rule was created. + StatusSuccess OutputStatus = "success" + // StatusValidationError means the inputs failed validation before any + // write — empty required field or an unsupported monitorType. + StatusValidationError OutputStatus = "validation_error" + // StatusError means the rule could not be created due to a downstream + // DQ service error. + StatusError OutputStatus = "error" + // StatusPreview means confirm was not set: the tool returned the composed + // rule (including its SQL) for review and created nothing. + StatusPreview OutputStatus = "preview" +) + +// monitorType discriminators accepted by the DQ API. +const ( + monitorTypeFreeformSQL = "FREEFORM_SQL" + monitorTypeSimpleSQL = "SIMPLE_SQL" +) + +// Input is the tool's typed input. +type Input struct { + JobName string `json:"jobName" jsonschema:"Required. Name of the existing data quality job the rule is attached to (a job, also called a 'dataset', is a saved data-quality check on one database table), e.g. 'PUBLIC.SAMPLE_DATASET'."` + MonitorName string `json:"monitorName" jsonschema:"Required. Rule name. Only letters, digits, '-' and '_' are allowed; max 256 characters."` + MonitorType string `json:"monitorType" jsonschema:"Required. Rule type: 'FREEFORM_SQL' for a full SQL expression, or 'SIMPLE_SQL' for a single-column check."` + MonitorValue string `json:"monitorValue" jsonschema:"Required. The rule's SQL. For FREEFORM_SQL this is a full query (e.g. 'SELECT * FROM @PUBLIC.SAMPLE_DATASET WHERE NAME IS NULL'); for SIMPLE_SQL it is the column predicate."` + FilterQuery string `json:"filterQuery,omitempty" jsonschema:"Optional. Additional WHERE-clause filter applied to the rule (e.g. ' where NAME IS NULL')."` + ColumnName string `json:"columnName,omitempty" jsonschema:"The single column the check targets. Required for SIMPLE_SQL rules; not needed for FREEFORM_SQL (the column(s) live in the SQL)."` + Description string `json:"description,omitempty" jsonschema:"Optional. Human-readable description; max 256 characters."` + Dimensions []string `json:"dimensions,omitempty" jsonschema:"Optional. Data quality dimensions — categories such as Accuracy, Completeness, Validity — to associate with the rule (e.g. ['Accuracy','Completeness'])."` + Tolerance int `json:"tolerance,omitempty" jsonschema:"Optional. Number of failing ('breaking') records allowed before the rule is considered failed — a count, NOT a percentage. Defaults to 0."` + Active *bool `json:"active,omitempty" jsonschema:"Optional. Whether the rule is active. Defaults to true."` + Suppressed bool `json:"suppressed,omitempty" jsonschema:"Optional. Whether the rule is suppressed (kept but not scored). Defaults to false."` + TemplateID string `json:"templateId,omitempty" jsonschema:"Optional. UUID of a rule template to link this rule to so it appears under the template's 'Used In' tab."` + Confirm bool `json:"confirm,omitempty" jsonschema:"Safety checkpoint. false (default) returns a PREVIEW of the rule — including its SQL — WITHOUT creating it, so it can be reviewed with the user. Set true to actually create the rule after the user has approved."` +} + +// RulePreview is the composed rule echoed back for review when confirm is false. +// It mirrors every field that will be written, so the confirm checkpoint shows +// the complete rule. +type RulePreview struct { + JobName string `json:"jobName"` + MonitorName string `json:"monitorName"` + MonitorType string `json:"monitorType"` + MonitorValue string `json:"monitorValue" jsonschema:"The rule's SQL that will be saved — review this with the user before confirming."` + FilterQuery string `json:"filterQuery,omitempty"` + ColumnName string `json:"columnName,omitempty"` + Description string `json:"description,omitempty"` + Dimensions []string `json:"dimensions,omitempty"` + Tolerance int `json:"tolerance"` + Active bool `json:"active"` + Suppressed bool `json:"suppressed"` + TemplateID string `json:"templateId,omitempty"` +} + +// Output is the typed response. +type Output struct { + Status OutputStatus `json:"status" jsonschema:"'preview' when confirm was not set (nothing created — review the preview and call again with confirm=true); 'success' when the rule was created; 'validation_error' for bad inputs; 'error' for downstream DQ failures."` + Message string `json:"message" jsonschema:"Human-readable summary."` + Preview *RulePreview `json:"preview,omitempty" jsonschema:"The composed rule (with its SQL) returned when confirm=false; nothing was created."` + JobName string `json:"jobName,omitempty" jsonschema:"Job the rule was created on, on success."` + MonitorName string `json:"monitorName,omitempty" jsonschema:"Name of the created rule, on success."` +} + +// NewTool returns the registered tool. +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "create_data_quality_rule", + Title: "Create Data Quality Rule", + Description: "Create a data quality rule (a single data-quality check on a table's data; Collibra calls it a 'monitor') " + + "on an existing data quality job (a saved data-quality check on ONE database table that scans the table and runs its rules; also called a 'dataset'), identified by its job name. " + + "monitorType is 'FREEFORM_SQL' (a full SQL query) or 'SIMPLE_SQL' (a single-column check). " + + "The rule defaults to active and not suppressed (suppressed = kept but not scored). " + + "Built around a confirm checkpoint: confirm=false (default) returns a PREVIEW of the rule and its SQL without creating anything — review it with the user; confirm=true creates the rule. " + + "Returns the job name and rule name on success. " + + "Note: requires permission to create rules on the target job.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(false)}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if out := validate(input); out != nil { + return *out, nil + } + + request := clients.CreateDQRuleRequest{ + JobName: strings.TrimSpace(input.JobName), + MonitorName: strings.TrimSpace(input.MonitorName), + MonitorType: input.MonitorType, + MonitorValue: input.MonitorValue, + FilterQuery: input.FilterQuery, + ColumnName: input.ColumnName, + Description: input.Description, + Dimensions: input.Dimensions, + Tolerance: input.Tolerance, + IsActive: activeFlag(input.Active), + IsSuppressed: input.Suppressed, + TemplateID: strings.TrimSpace(input.TemplateID), + } + + // Confirm checkpoint: without confirm, return the composed rule (SQL + // included) for review and create nothing. + if !input.Confirm { + return Output{ + Status: StatusPreview, + Message: fmt.Sprintf("Preview only — nothing created. Will create rule %q on job %q with SQL: %s. "+ + "Review this with the user, then call again with confirm=true.", request.MonitorName, request.JobName, request.MonitorValue), + Preview: &RulePreview{ + JobName: request.JobName, + MonitorName: request.MonitorName, + MonitorType: request.MonitorType, + MonitorValue: request.MonitorValue, + FilterQuery: request.FilterQuery, + ColumnName: request.ColumnName, + Description: request.Description, + Dimensions: request.Dimensions, + Tolerance: request.Tolerance, + Active: request.IsActive == 1, + Suppressed: request.IsSuppressed, + TemplateID: request.TemplateID, + }, + }, nil + } + + resp, err := clients.CreateDQRule(ctx, collibraClient, request) + if err != nil { + return Output{Status: StatusError, Message: fmt.Sprintf("Could not create rule: %v", err)}, nil + } + + return Output{ + Status: StatusSuccess, + Message: fmt.Sprintf("Created rule %q on job %q.", resp.MonitorName, resp.JobName), + JobName: resp.JobName, + MonitorName: resp.MonitorName, + }, nil + } +} + +// validate enforces the required fields and the monitorType enum before any +// network call, so the agent gets a cheap, self-correcting error. +func validate(input Input) *Output { + if strings.TrimSpace(input.JobName) == "" { + return &Output{Status: StatusValidationError, Message: "jobName is required."} + } + if name := strings.TrimSpace(input.MonitorName); name == "" { + return &Output{Status: StatusValidationError, Message: "monitorName is required."} + } else if !monitorNameRe.MatchString(name) { + return &Output{Status: StatusValidationError, Message: "monitorName may contain only letters, digits, '-' and '_', and must be 1-256 characters."} + } + if strings.TrimSpace(input.MonitorValue) == "" { + return &Output{Status: StatusValidationError, Message: "monitorValue is required."} + } + if len(input.Description) > maxDescriptionLen { + return &Output{Status: StatusValidationError, Message: fmt.Sprintf("description must be at most %d characters.", maxDescriptionLen)} + } + switch input.MonitorType { + case monitorTypeFreeformSQL: + case monitorTypeSimpleSQL: + if strings.TrimSpace(input.ColumnName) == "" { + return &Output{Status: StatusValidationError, Message: "columnName is required for a SIMPLE_SQL rule (the single column the check targets)."} + } + default: + return &Output{ + Status: StatusValidationError, + Message: fmt.Sprintf("monitorType %q is invalid. Use %q or %q.", input.MonitorType, monitorTypeFreeformSQL, monitorTypeSimpleSQL), + } + } + return nil +} + +// activeFlag maps the optional `active` input to the DQ API's isActive int. +// A nil pointer (field omitted) defaults to active. +func activeFlag(active *bool) int { + if active == nil || *active { + return 1 + } + return 0 +} diff --git a/pkg/tools/create_dq_rule/tool_test.go b/pkg/tools/create_dq_rule/tool_test.go new file mode 100644 index 0000000..09d1a93 --- /dev/null +++ b/pkg/tools/create_dq_rule/tool_test.go @@ -0,0 +1,199 @@ +package create_dq_rule_test + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/create_dq_rule" + "github.com/collibra/chip/pkg/tools/testutil" +) + +// server boots an httptest server that captures the create-monitor request and +// echoes back a MonitorResponse. createCode overrides the success status. +func server(t *testing.T, createCode int, captured *clients.CreateDQRuleRequest) *http.Client { + mux := http.NewServeMux() + mux.HandleFunc("POST /rest/dq/internal/v1/monitoring/monitor", func(w http.ResponseWriter, r *http.Request) { + if err := json.NewDecoder(r.Body).Decode(captured); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + code := createCode + if code == 0 { + code = http.StatusOK + } + if code != http.StatusOK { + w.WriteHeader(code) + _, _ = w.Write([]byte(`{"message":"boom"}`)) + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(clients.CreateDQRuleResponse{ + JobName: captured.JobName, + MonitorName: captured.MonitorName, + }) + }) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + return testutil.NewClient(srv) +} + +func TestCreateDQRule_HappyPath_DefaultsActive(t *testing.T) { + var got clients.CreateDQRuleRequest + c := server(t, http.StatusOK, &got) + + out, err := create_dq_rule.NewTool(c).Handler(t.Context(), create_dq_rule.Input{ + JobName: "PUBLIC.SAMPLE_DATASET", + MonitorName: "Name_Not_Null", + MonitorType: "FREEFORM_SQL", + MonitorValue: "SELECT * FROM @PUBLIC.SAMPLE_DATASET WHERE NAME IS NULL", + Confirm: true, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if out.Status != create_dq_rule.StatusSuccess { + t.Fatalf("status = %q, want success (%s)", out.Status, out.Message) + } + if out.MonitorName != "Name_Not_Null" || out.JobName != "PUBLIC.SAMPLE_DATASET" { + t.Fatalf("unexpected output: %+v", out) + } + if got.IsActive != 1 { + t.Fatalf("isActive = %d, want 1 (default active)", got.IsActive) + } + if got.IsSuppressed { + t.Fatalf("isSuppressed = true, want false (default)") + } +} + +func TestCreateDQRule_InactiveMapsToZero(t *testing.T) { + var got clients.CreateDQRuleRequest + c := server(t, http.StatusOK, &got) + + inactive := false + _, err := create_dq_rule.NewTool(c).Handler(t.Context(), create_dq_rule.Input{ + JobName: "DS", + MonitorName: "R", + MonitorType: "SIMPLE_SQL", + MonitorValue: "NAME IS NOT NULL", + ColumnName: "NAME", + Active: &inactive, + Confirm: true, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got.IsActive != 0 { + t.Fatalf("isActive = %d, want 0", got.IsActive) + } +} + +func TestCreateDQRule_InvalidMonitorType(t *testing.T) { + var got clients.CreateDQRuleRequest + c := server(t, http.StatusOK, &got) + + out, _ := create_dq_rule.NewTool(c).Handler(t.Context(), create_dq_rule.Input{ + JobName: "DS", + MonitorName: "R", + MonitorType: "REGEX", + MonitorValue: "x", + }) + if out.Status != create_dq_rule.StatusValidationError { + t.Fatalf("status = %q, want validation_error", out.Status) + } +} + +func TestCreateDQRule_MissingRequiredFields(t *testing.T) { + var got clients.CreateDQRuleRequest + c := server(t, http.StatusOK, &got) + + out, _ := create_dq_rule.NewTool(c).Handler(t.Context(), create_dq_rule.Input{ + MonitorName: "R", + MonitorType: "FREEFORM_SQL", + MonitorValue: "x", + }) + if out.Status != create_dq_rule.StatusValidationError { + t.Fatalf("status = %q, want validation_error", out.Status) + } +} + +func TestCreateDQRule_DownstreamErrorSurfaces(t *testing.T) { + var got clients.CreateDQRuleRequest + c := server(t, http.StatusUnprocessableEntity, &got) + + out, _ := create_dq_rule.NewTool(c).Handler(t.Context(), create_dq_rule.Input{ + JobName: "DS", + MonitorName: "R", + MonitorType: "FREEFORM_SQL", + MonitorValue: "x", + Confirm: true, + }) + if out.Status != create_dq_rule.StatusError { + t.Fatalf("status = %q, want error", out.Status) + } +} + +func TestCreateDQRule_PreviewByDefault_CreatesNothing(t *testing.T) { + var got clients.CreateDQRuleRequest + c := server(t, http.StatusOK, &got) + + out, err := create_dq_rule.NewTool(c).Handler(t.Context(), create_dq_rule.Input{ + JobName: "PUBLIC.DS", + MonitorName: "Name_Not_Null", + MonitorType: "FREEFORM_SQL", + MonitorValue: "SELECT * FROM @PUBLIC.DS WHERE NAME IS NULL", + // Confirm omitted -> preview + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if out.Status != create_dq_rule.StatusPreview { + t.Fatalf("status = %q, want preview", out.Status) + } + if out.Preview == nil || out.Preview.MonitorValue == "" { + t.Fatalf("expected preview with SQL, got %+v", out.Preview) + } + // The DQ endpoint must not have been called (nothing written). + if got.MonitorName != "" { + t.Fatalf("expected no create request in preview mode, but server was called: %+v", got) + } +} + +func TestCreateDQRule_InvalidMonitorName(t *testing.T) { + var got clients.CreateDQRuleRequest + c := server(t, http.StatusOK, &got) + + out, _ := create_dq_rule.NewTool(c).Handler(t.Context(), create_dq_rule.Input{ + JobName: "DS", + MonitorName: "bad name!", // space and '!' are not allowed + MonitorType: "FREEFORM_SQL", + MonitorValue: "SELECT 1", + Confirm: true, + }) + if out.Status != create_dq_rule.StatusValidationError { + t.Fatalf("status = %q, want validation_error", out.Status) + } + if got.MonitorName != "" { + t.Fatalf("expected no request for an invalid name") + } +} + +func TestCreateDQRule_SimpleSQLRequiresColumn(t *testing.T) { + var got clients.CreateDQRuleRequest + c := server(t, http.StatusOK, &got) + + out, _ := create_dq_rule.NewTool(c).Handler(t.Context(), create_dq_rule.Input{ + JobName: "DS", + MonitorName: "R", + MonitorType: "SIMPLE_SQL", + MonitorValue: "NAME IS NOT NULL", + // columnName omitted + Confirm: true, + }) + if out.Status != create_dq_rule.StatusValidationError { + t.Fatalf("status = %q, want validation_error (SIMPLE_SQL needs columnName)", out.Status) + } +} diff --git a/pkg/tools/deploy_dq_rule_template/tool.go b/pkg/tools/deploy_dq_rule_template/tool.go new file mode 100644 index 0000000..578f5dc --- /dev/null +++ b/pkg/tools/deploy_dq_rule_template/tool.go @@ -0,0 +1,171 @@ +// Package deploy_dq_rule_template implements the deploy_dq_rule_template MCP tool: +// it instantiates a rule template as concrete rules across one or more job/column +// targets, using dialect-specific SQL resolved by the DQ service. +package deploy_dq_rule_template + +import ( + "context" + "fmt" + "net/http" + "strings" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +// OutputStatus is the overall outcome of a deploy_dq_rule_template call. +type OutputStatus string + +const ( + // StatusSuccess means every target was deployed. + StatusSuccess OutputStatus = "success" + // StatusPartial means the deploy ran but some targets were skipped/failed + // while others were deployed. + StatusPartial OutputStatus = "partial" + // StatusValidationError means the inputs failed validation before any write. + StatusValidationError OutputStatus = "validation_error" + // StatusError means the deployment failed due to a downstream DQ error. + StatusError OutputStatus = "error" + // StatusPreview means confirm was not set: the tool returned the template + + // target list for review and deployed nothing. + StatusPreview OutputStatus = "preview" +) + +// Target is one deployment target. +type Target struct { + JobName string `json:"jobName" jsonschema:"Required. Name of the existing data quality job to deploy the rule on (a job, also called a 'dataset', is a saved check on one database table), e.g. 'PUBLIC.CUSTOMERS'."` + ColumnName string `json:"columnName,omitempty" jsonschema:"Column substituted for the template's {{column}} placeholder. Required for column-level templates; omit for table-level templates."` +} + +// Input is the tool's typed input. +type Input struct { + RuleTemplateName string `json:"ruleTemplateName" jsonschema:"Required. Name of the rule template to deploy (from list_data_quality_rule_templates)."` + Targets []Target `json:"targets" jsonschema:"Required. One or more job/column targets. Each deployed rule is named {templateName}_{columnName} by the server."` + Confirm bool `json:"confirm,omitempty" jsonschema:"Safety checkpoint. false (default) returns a PREVIEW of the template and the target list WITHOUT deploying, so it can be reviewed with the user (inspect the template's SQL with get_data_quality_rule_template). Set true to actually deploy after the user has approved."` +} + +// Outcome is the per-target result of a deploy. +type Outcome struct { + JobName string `json:"jobName" jsonschema:"The target job."` + ColumnName string `json:"columnName,omitempty" jsonschema:"The target column, when set."` + DeployedRuleName string `json:"deployedRuleName,omitempty" jsonschema:"Name of the rule the server created for this target, when deployed."` + Status string `json:"status" jsonschema:"Per-target status reported by the DQ service (e.g. deployed or SKIPPED)."` + Reason string `json:"reason,omitempty" jsonschema:"Why a target was skipped or failed, when applicable."` +} + +// Output is the typed response. +type Output struct { + Status OutputStatus `json:"status" jsonschema:"'preview' when confirm was not set (nothing deployed — review and call again with confirm=true); 'success' when every target was deployed; 'partial' when some targets were skipped/failed; 'validation_error' for bad inputs; 'error' for downstream DQ failures."` + Message string `json:"message" jsonschema:"Human-readable summary, including deployed vs skipped counts."` + Preview *Preview `json:"preview,omitempty" jsonschema:"The template name and resolved targets returned when confirm=false; nothing was deployed."` + Outcomes []Outcome `json:"outcomes,omitempty" jsonschema:"Per-target deploy outcomes (partial-success): each target's status and, on skip/failure, the reason."` + Deployed int `json:"deployed,omitempty" jsonschema:"Number of targets successfully deployed."` + Skipped int `json:"skipped,omitempty" jsonschema:"Number of targets skipped or failed."` +} + +// Preview is the deployment plan echoed back for review when confirm is false. +type Preview struct { + RuleTemplateName string `json:"ruleTemplateName"` + Targets []Target `json:"targets"` + Count int `json:"count"` +} + +// NewTool returns the registered tool. +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "deploy_data_quality_rule_template", + Title: "Deploy Data Quality Rule Template", + Description: "Instantiate a rule template as concrete rules (checks on a table's data; Collibra calls them 'monitors') across one or more job/column targets " + + "(a job, also called a 'dataset', is a saved check on ONE database table). " + + "The DQ service resolves dialect-specific SQL and creates one rule per target, each named " + + "{templateName}_{columnName}. Provide a columnName per target for column-level templates. " + + "Built around a confirm checkpoint: confirm=false (default) returns a PREVIEW of the template + targets without deploying — review it with the user; confirm=true deploys. " + + "Requires permission to deploy templates and to create rules on the target jobs. " + + "The deploy is partial-success: each target is deployed or skipped independently, and the per-target outcomes (with skip reasons) are returned.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{DestructiveHint: chip.Ptr(false)}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if strings.TrimSpace(input.RuleTemplateName) == "" { + return Output{Status: StatusValidationError, Message: "ruleTemplateName is required."}, nil + } + if len(input.Targets) == 0 { + return Output{Status: StatusValidationError, Message: "at least one target is required."}, nil + } + + targets := make([]clients.DQTemplateDeployTarget, 0, len(input.Targets)) + for i, t := range input.Targets { + if strings.TrimSpace(t.JobName) == "" { + return Output{Status: StatusValidationError, Message: fmt.Sprintf("targets[%d].jobName is required.", i)}, nil + } + targets = append(targets, clients.DQTemplateDeployTarget{ + JobName: strings.TrimSpace(t.JobName), + ColumnName: strings.TrimSpace(t.ColumnName), + }) + } + + ruleTemplateName := strings.TrimSpace(input.RuleTemplateName) + + // Confirm checkpoint: without confirm, return the deployment plan for + // review and deploy nothing. + if !input.Confirm { + review := make([]Target, len(targets)) + for i, t := range targets { + review[i] = Target{JobName: t.JobName, ColumnName: t.ColumnName} + } + return Output{ + Status: StatusPreview, + Message: fmt.Sprintf("Preview only — nothing deployed. Will deploy template %q to %d target(s) (each rule named {template}_{column}). "+ + "Inspect the template's SQL with get_data_quality_rule_template, review the targets with the user, then call again with confirm=true.", ruleTemplateName, len(targets)), + Preview: &Preview{RuleTemplateName: ruleTemplateName, Targets: review, Count: len(targets)}, + }, nil + } + + result, err := clients.DeployDQRuleTemplate(ctx, collibraClient, ruleTemplateName, targets) + if err != nil { + return Output{Status: StatusError, Message: fmt.Sprintf("Could not deploy template: %v", err)}, nil + } + + // Partial-success: surface each target's outcome and tally deployed vs + // skipped. A target counts as deployed when the server assigned it a rule + // name; otherwise it was skipped/failed (with a reason). + outcomes := make([]Outcome, 0, len(result.Results)) + deployed, skipped := 0, 0 + for _, o := range result.Results { + outcomes = append(outcomes, Outcome{ + JobName: o.JobName, + ColumnName: o.ColumnName, + DeployedRuleName: o.DeployedRuleName, + Status: o.Status, + Reason: o.Reason, + }) + if o.DeployedRuleName != "" { + deployed++ + } else { + skipped++ + } + } + + status := StatusSuccess + if skipped > 0 { + if deployed == 0 { + status = StatusError + } else { + status = StatusPartial + } + } + + return Output{ + Status: status, + Message: fmt.Sprintf("Deployed %d of %d target(s); %d skipped.", deployed, len(outcomes), skipped), + Outcomes: outcomes, + Deployed: deployed, + Skipped: skipped, + }, nil + } +} diff --git a/pkg/tools/deploy_dq_rule_template/tool_test.go b/pkg/tools/deploy_dq_rule_template/tool_test.go new file mode 100644 index 0000000..ae4ce88 --- /dev/null +++ b/pkg/tools/deploy_dq_rule_template/tool_test.go @@ -0,0 +1,158 @@ +package deploy_dq_rule_template_test + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/tools/deploy_dq_rule_template" + "github.com/collibra/chip/pkg/tools/testutil" +) + +type capture struct { + path string + targets []map[string]string +} + +// server mocks the public deploy endpoint. When code is 200 (or 0) it returns +// the given per-target results array (RuleTemplateDeployResult); otherwise it +// returns an error body with the given status. +func server(t *testing.T, code int, results []map[string]any, rec *capture) *http.Client { + mux := http.NewServeMux() + mux.HandleFunc("POST /rest/dq/1.0/ruleTemplates/{ruleTemplateName}/deploy", func(w http.ResponseWriter, r *http.Request) { + if rec != nil { + rec.path = r.URL.Path + var body struct { + Targets []map[string]string `json:"targets"` + } + _ = json.NewDecoder(r.Body).Decode(&body) + rec.targets = body.Targets + } + if code == 0 { + code = http.StatusOK + } + if code != http.StatusOK { + w.WriteHeader(code) + _, _ = w.Write([]byte(`{"message":"boom"}`)) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"results": results}) + }) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + return testutil.NewClient(srv) +} + +func TestDeployDQRuleTemplate_HappyPath(t *testing.T) { + var rec capture + c := server(t, http.StatusOK, []map[string]any{ + {"jobName": "PUBLIC.CUSTOMERS", "columnName": "email", "deployedRuleName": "NotNull_email", "status": "deployed"}, + {"jobName": "PUBLIC.CUSTOMERS", "columnName": "name", "deployedRuleName": "NotNull_name", "status": "deployed"}, + }, &rec) + out, err := deploy_dq_rule_template.NewTool(c).Handler(t.Context(), deploy_dq_rule_template.Input{ + RuleTemplateName: "Not Null Check", + Targets: []deploy_dq_rule_template.Target{ + {JobName: "PUBLIC.CUSTOMERS", ColumnName: "email"}, + {JobName: "PUBLIC.CUSTOMERS", ColumnName: "name"}, + }, + Confirm: true, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if out.Status != deploy_dq_rule_template.StatusSuccess { + t.Fatalf("status = %q, want success (%s)", out.Status, out.Message) + } + if out.Deployed != 2 || out.Skipped != 0 || len(out.Outcomes) != 2 { + t.Fatalf("unexpected tally: deployed=%d skipped=%d outcomes=%+v", out.Deployed, out.Skipped, out.Outcomes) + } + if out.Outcomes[0].DeployedRuleName != "NotNull_email" { + t.Fatalf("unexpected outcome: %+v", out.Outcomes[0]) + } + if rec.path != "/rest/dq/1.0/ruleTemplates/Not Null Check/deploy" { + t.Fatalf("unexpected path: %s", rec.path) + } + if len(rec.targets) != 2 || rec.targets[0]["columnName"] != "email" { + t.Fatalf("unexpected targets: %+v", rec.targets) + } +} + +func TestDeployDQRuleTemplate_PartialSuccess(t *testing.T) { + c := server(t, http.StatusOK, []map[string]any{ + {"jobName": "PUBLIC.CUSTOMERS", "columnName": "email", "deployedRuleName": "NotNull_email", "status": "deployed"}, + {"jobName": "PUBLIC.CUSTOMERS", "columnName": "name", "status": "SKIPPED", "reason": "rule already exists"}, + }, nil) + out, _ := deploy_dq_rule_template.NewTool(c).Handler(t.Context(), deploy_dq_rule_template.Input{ + RuleTemplateName: "Not Null Check", + Targets: []deploy_dq_rule_template.Target{ + {JobName: "PUBLIC.CUSTOMERS", ColumnName: "email"}, + {JobName: "PUBLIC.CUSTOMERS", ColumnName: "name"}, + }, + Confirm: true, + }) + if out.Status != deploy_dq_rule_template.StatusPartial { + t.Fatalf("status = %q, want partial (%s)", out.Status, out.Message) + } + if out.Deployed != 1 || out.Skipped != 1 { + t.Fatalf("unexpected tally: deployed=%d skipped=%d", out.Deployed, out.Skipped) + } + if out.Outcomes[1].Reason != "rule already exists" { + t.Fatalf("expected skip reason surfaced, got %+v", out.Outcomes[1]) + } +} + +func TestDeployDQRuleTemplate_MissingTargets(t *testing.T) { + c := server(t, http.StatusOK, nil, nil) + out, _ := deploy_dq_rule_template.NewTool(c).Handler(t.Context(), deploy_dq_rule_template.Input{RuleTemplateName: "t1"}) + if out.Status != deploy_dq_rule_template.StatusValidationError { + t.Fatalf("status = %q, want validation_error", out.Status) + } +} + +func TestDeployDQRuleTemplate_MissingJobName(t *testing.T) { + c := server(t, http.StatusOK, nil, nil) + out, _ := deploy_dq_rule_template.NewTool(c).Handler(t.Context(), deploy_dq_rule_template.Input{ + RuleTemplateName: "t1", + Targets: []deploy_dq_rule_template.Target{{ColumnName: "email"}}, + }) + if out.Status != deploy_dq_rule_template.StatusValidationError { + t.Fatalf("status = %q, want validation_error", out.Status) + } +} + +func TestDeployDQRuleTemplate_DownstreamErrorSurfaces(t *testing.T) { + c := server(t, http.StatusNotFound, nil, nil) + out, _ := deploy_dq_rule_template.NewTool(c).Handler(t.Context(), deploy_dq_rule_template.Input{ + RuleTemplateName: "t1", + Targets: []deploy_dq_rule_template.Target{{JobName: "DS"}}, + Confirm: true, + }) + if out.Status != deploy_dq_rule_template.StatusError { + t.Fatalf("status = %q, want error", out.Status) + } +} + +func TestDeployDQRuleTemplate_PreviewByDefault_DeploysNothing(t *testing.T) { + var rec capture + c := server(t, http.StatusOK, nil, &rec) + out, err := deploy_dq_rule_template.NewTool(c).Handler(t.Context(), deploy_dq_rule_template.Input{ + RuleTemplateName: "t1", + Targets: []deploy_dq_rule_template.Target{{JobName: "PUBLIC.CUSTOMERS", ColumnName: "email"}}, + // Confirm omitted -> preview + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if out.Status != deploy_dq_rule_template.StatusPreview { + t.Fatalf("status = %q, want preview", out.Status) + } + if out.Preview == nil || out.Preview.Count != 1 { + t.Fatalf("expected preview with 1 target, got %+v", out.Preview) + } + // The deploy endpoint must not have been called. + if rec.path != "" { + t.Fatalf("expected no deploy call in preview mode, but server was hit: %s", rec.path) + } +} diff --git a/pkg/tools/find_dq_rules/tool.go b/pkg/tools/find_dq_rules/tool.go new file mode 100644 index 0000000..f8b519a --- /dev/null +++ b/pkg/tools/find_dq_rules/tool.go @@ -0,0 +1,138 @@ +// Package find_dq_rules implements the find_dq_rules MCP tool: it searches +// existing data quality rules (monitors) across jobs, primarily to detect rules +// already present on a target column before creating a new one. +package find_dq_rules + +import ( + "context" + "fmt" + "net/http" + "strings" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +// OutputStatus is the overall outcome of a find_dq_rules call. +type OutputStatus string + +const ( + // StatusSuccess means the search ran. + StatusSuccess OutputStatus = "success" + // StatusValidationError means the inputs failed validation before any read. + StatusValidationError OutputStatus = "validation_error" + // StatusError means the search failed due to a downstream DQ error. + StatusError OutputStatus = "error" +) + +const defaultLimit = 25 + +// DQ monitor filterable fields and operators used by this tool. +const ( + fieldJobName = "JOB_NAME" + fieldColumnName = "COLUMN_NAME" + fieldMonitorName = "MONITOR_NAME" + opEquals = "EQUALS" + opContains = "CONTAINS" +) + +// Input is the tool's typed input. At least one filter should be provided; for +// duplicate detection, set jobName and columnName. +type Input struct { + JobName string `json:"jobName,omitempty" jsonschema:"Optional. Exact job name to scope the search to (a job, also called a 'dataset', is a saved check on one database table)."` + ColumnName string `json:"columnName,omitempty" jsonschema:"Optional. Exact column name — combine with jobName to find existing rules on a specific column (duplicate detection)."` + NameContains string `json:"nameContains,omitempty" jsonschema:"Optional. Substring match on the rule name (the check; Collibra calls it a 'monitor')."` + Offset int `json:"offset,omitempty" jsonschema:"Optional. Pagination offset (min 0). Defaults to 0."` + Limit int `json:"limit,omitempty" jsonschema:"Optional. Max rules to return (1-100). Defaults to 25."` +} + +// Rule is one matching rule (monitor). +type Rule struct { + MonitorName string `json:"monitorName"` + JobName string `json:"jobName"` + ColumnName string `json:"columnName,omitempty"` + MonitorType string `json:"monitorType,omitempty"` + MonitorStatus string `json:"monitorStatus,omitempty"` + Dimensions []string `json:"dimensions,omitempty"` + RuleQuery string `json:"ruleQuery,omitempty"` + FilterQuery string `json:"filterQuery,omitempty"` +} + +// Output is the typed response. +type Output struct { + Status OutputStatus `json:"status" jsonschema:"'success' when the search ran; 'validation_error' for bad inputs; 'error' for downstream DQ failures."` + Message string `json:"message" jsonschema:"Human-readable summary."` + Rules []Rule `json:"rules,omitempty" jsonschema:"Matching rules."` + Total int64 `json:"total" jsonschema:"Total number of matching rules (for pagination)."` +} + +// NewTool returns the registered tool. +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "find_data_quality_rules", + Title: "Find Data Quality Rules", + Description: "Search existing data quality rules (checks on a table's data; Collibra calls them 'monitors') across data quality jobs " + + "(a job, also called a 'dataset', is a saved check on ONE database table). Filter by exact jobName and/or " + + "columnName (combine both to detect rules already on a target column before creating a new one), or by " + + "a rule-name substring. Returns each rule's job, column, type, status and SQL. Paginated (offset/limit).", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if input.Offset < 0 { + return Output{Status: StatusValidationError, Message: "offset must be >= 0."}, nil + } + limit := input.Limit + if limit == 0 { + limit = defaultLimit + } + if limit < 1 || limit > 100 { + return Output{Status: StatusValidationError, Message: "limit must be between 1 and 100."}, nil + } + + var filters []clients.DQMonitorFilter + if s := strings.TrimSpace(input.JobName); s != "" { + filters = append(filters, clients.DQMonitorFilter{Field: fieldJobName, Operator: opEquals, Values: []string{s}}) + } + if s := strings.TrimSpace(input.ColumnName); s != "" { + filters = append(filters, clients.DQMonitorFilter{Field: fieldColumnName, Operator: opEquals, Values: []string{s}}) + } + if s := strings.TrimSpace(input.NameContains); s != "" { + filters = append(filters, clients.DQMonitorFilter{Field: fieldMonitorName, Operator: opContains, Values: []string{s}}) + } + if len(filters) == 0 { + return Output{Status: StatusValidationError, Message: "Provide at least one filter (jobName, columnName, or nameContains) — an unfiltered search would page through every rule on the instance."}, nil + } + + res, err := clients.FindDQRules(ctx, collibraClient, filters, input.Offset, limit) + if err != nil { + return Output{Status: StatusError, Message: fmt.Sprintf("Could not search rules: %v", err)}, nil + } + + rules := make([]Rule, 0, len(res.Results)) + for _, m := range res.Results { + rules = append(rules, Rule{ + MonitorName: m.MonitorName, + JobName: m.JobName, + ColumnName: m.ColumnName, + MonitorType: m.MonitorType, + MonitorStatus: m.MonitorStatus, + Dimensions: m.Dimensions, + RuleQuery: m.RuleQuery, + FilterQuery: m.FilterQuery, + }) + } + + return Output{ + Status: StatusSuccess, + Message: fmt.Sprintf("Found %d of %d matching rule(s).", len(rules), res.Total), + Rules: rules, + Total: res.Total, + }, nil + } +} diff --git a/pkg/tools/find_dq_rules/tool_test.go b/pkg/tools/find_dq_rules/tool_test.go new file mode 100644 index 0000000..5c3b363 --- /dev/null +++ b/pkg/tools/find_dq_rules/tool_test.go @@ -0,0 +1,78 @@ +package find_dq_rules_test + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/find_dq_rules" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func server(t *testing.T, code int, results []clients.DQMonitorSummary, total int64, captured *[]clients.DQMonitorFilter) *http.Client { + mux := http.NewServeMux() + mux.HandleFunc("POST /rest/dq/internal/v1/monitoring/monitors/dashboard", func(w http.ResponseWriter, r *http.Request) { + if captured != nil { + var body struct { + Filters []clients.DQMonitorFilter `json:"filters"` + } + _ = json.NewDecoder(r.Body).Decode(&body) + *captured = body.Filters + } + if code != http.StatusOK { + w.WriteHeader(code) + _, _ = w.Write([]byte(`{"message":"boom"}`)) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"results": results, "total": total, "offset": 0, "limit": 25}) + }) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + return testutil.NewClient(srv) +} + +func TestFindDQRules_HappyPath_ColumnDuplicateCheck(t *testing.T) { + var filters []clients.DQMonitorFilter + c := server(t, http.StatusOK, []clients.DQMonitorSummary{ + {MonitorName: "email_not_null", JobName: "PUBLIC.USERS", ColumnName: "email", MonitorType: "SQLF", MonitorStatus: "PASSING"}, + }, 1, &filters) + + out, err := find_dq_rules.NewTool(c).Handler(t.Context(), find_dq_rules.Input{JobName: "PUBLIC.USERS", ColumnName: "email"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if out.Status != find_dq_rules.StatusSuccess { + t.Fatalf("status = %q, want success (%s)", out.Status, out.Message) + } + if len(out.Rules) != 1 || out.Rules[0].MonitorName != "email_not_null" { + t.Fatalf("unexpected rules: %+v", out.Rules) + } + // Two EQUALS filters (JOB_NAME + COLUMN_NAME) should be sent. + if len(filters) != 2 { + t.Fatalf("expected 2 filters, got %+v", filters) + } + for _, f := range filters { + if f.Operator != "EQUALS" { + t.Fatalf("expected EQUALS operator, got %+v", f) + } + } +} + +func TestFindDQRules_InvalidLimit(t *testing.T) { + c := server(t, http.StatusOK, nil, 0, nil) + out, _ := find_dq_rules.NewTool(c).Handler(t.Context(), find_dq_rules.Input{Limit: 1000}) + if out.Status != find_dq_rules.StatusValidationError { + t.Fatalf("status = %q, want validation_error", out.Status) + } +} + +func TestFindDQRules_DownstreamErrorSurfaces(t *testing.T) { + c := server(t, http.StatusBadRequest, nil, 0, nil) + out, _ := find_dq_rules.NewTool(c).Handler(t.Context(), find_dq_rules.Input{JobName: "DS"}) + if out.Status != find_dq_rules.StatusError { + t.Fatalf("status = %q, want error", out.Status) + } +} diff --git a/pkg/tools/generate_dq_rule_sql/tool.go b/pkg/tools/generate_dq_rule_sql/tool.go new file mode 100644 index 0000000..c71f579 --- /dev/null +++ b/pkg/tools/generate_dq_rule_sql/tool.go @@ -0,0 +1,115 @@ +// Package generate_dq_rule_sql implements the generate_dq_rule_sql MCP tool: it +// turns a plain-language description of a data quality check into rule SQL, so a +// rule can be authored without writing SQL by hand. +package generate_dq_rule_sql + +import ( + "context" + "fmt" + "net/http" + "strings" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +// OutputStatus is the overall outcome of a generate_dq_rule_sql call. +type OutputStatus string + +const ( + // StatusSuccess means SQL was generated. + StatusSuccess OutputStatus = "success" + // StatusValidationError means the inputs failed validation before any call. + StatusValidationError OutputStatus = "validation_error" + // StatusError means generation failed due to a downstream DQ error. + StatusError OutputStatus = "error" +) + +// Input is the tool's typed input. edgeSiteId/connectionId come from +// prepare_create_dq_job; the table is identified by jobName. +type Input struct { + EdgeSiteID string `json:"edgeSiteId" jsonschema:"Required. UUID of the Collibra Edge runtime/site that reaches the source database. From prepare_create_data_quality_job resolved.edgeSiteId."` + ConnectionID string `json:"connectionId" jsonschema:"Required. UUID of the specific database connection on that Edge site. From prepare_create_data_quality_job resolved.connectionId."` + JobName string `json:"jobName" jsonschema:"Required. Name of the data quality job whose table the rule runs against (a job, also called a 'dataset', is a saved check on one database table)."` + Columns []string `json:"columns" jsonschema:"Required. One or more column names giving the rule its context (e.g. the column(s) the check concerns)."` + Query string `json:"query" jsonschema:"Required. Plain-language description of the rule intent, e.g. 'email must not be null and must contain an @'."` +} + +// Output is the typed response. +type Output struct { + Status OutputStatus `json:"status" jsonschema:"'success' when SQL was generated; 'validation_error' for bad inputs; 'error' for downstream DQ failures."` + Message string `json:"message" jsonschema:"Human-readable summary."` + SQLQuery string `json:"sqlQuery,omitempty" jsonschema:"The generated rule SQL. Review/validate it (validate_data_quality_rule) before creating the rule; use it as monitorValue in create_data_quality_rule."` +} + +// NewTool returns the registered tool. +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "generate_data_quality_rule_sql", + Title: "Generate Data Quality Rule SQL (Text2SQL)", + Description: "Turn a plain-language description of a data quality check into rule SQL, so a rule (the check; Collibra calls it a 'monitor') can be " + + "authored without writing SQL by hand. Returns a single SQL string (no separate filter clause). " + + "Always review and validate the generated SQL (validate_data_quality_rule) before creating the rule. " + + "Requires edgeSiteId and connectionId — the connection to the source database (edgeSiteId = the Collibra Edge runtime/site that reaches the source, connectionId = the specific database connection), from prepare_create_data_quality_job — and uses Collibra DQ AI.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if out := validate(input); out != nil { + return *out, nil + } + + cols := make([]string, 0, len(input.Columns)) + for _, c := range input.Columns { + if s := strings.TrimSpace(c); s != "" { + cols = append(cols, s) + } + } + if len(cols) == 0 { + return Output{Status: StatusValidationError, Message: "columns must contain at least one non-empty column name."}, nil + } + + resp, err := clients.GenerateDQRuleSQL(ctx, collibraClient, clients.Text2SQLRequest{ + EdgeSiteID: strings.TrimSpace(input.EdgeSiteID), + ConnectionID: strings.TrimSpace(input.ConnectionID), + JobName: strings.TrimSpace(input.JobName), + Columns: cols, + Query: input.Query, + }) + if err != nil { + return Output{Status: StatusError, Message: fmt.Sprintf("Could not generate SQL: %v", err)}, nil + } + + return Output{ + Status: StatusSuccess, + Message: "Generated rule SQL. Review and validate it before creating the rule.", + SQLQuery: resp.SQLQuery, + }, nil + } +} + +func validate(input Input) *Output { + required := []struct { + name string + val string + }{ + {"edgeSiteId", input.EdgeSiteID}, + {"connectionId", input.ConnectionID}, + {"jobName", input.JobName}, + {"query", input.Query}, + } + for _, f := range required { + if strings.TrimSpace(f.val) == "" { + return &Output{Status: StatusValidationError, Message: f.name + " is required."} + } + } + if len(input.Columns) == 0 { + return &Output{Status: StatusValidationError, Message: "columns is required (at least one column)."} + } + return nil +} diff --git a/pkg/tools/generate_dq_rule_sql/tool_test.go b/pkg/tools/generate_dq_rule_sql/tool_test.go new file mode 100644 index 0000000..21a4a3a --- /dev/null +++ b/pkg/tools/generate_dq_rule_sql/tool_test.go @@ -0,0 +1,84 @@ +package generate_dq_rule_sql_test + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/generate_dq_rule_sql" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func server(t *testing.T, code int, resp clients.Text2SQLResponse, captured *clients.Text2SQLRequest) *http.Client { + mux := http.NewServeMux() + mux.HandleFunc("POST /rest/dq/internal/v1/ai/text2sql", func(w http.ResponseWriter, r *http.Request) { + if captured != nil { + _ = json.NewDecoder(r.Body).Decode(captured) + } + if code != http.StatusOK { + w.WriteHeader(code) + _, _ = w.Write([]byte(`{"message":"boom"}`)) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(resp) + }) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + return testutil.NewClient(srv) +} + +func input() generate_dq_rule_sql.Input { + return generate_dq_rule_sql.Input{ + EdgeSiteID: "site", ConnectionID: "conn", JobName: "PUBLIC.USERS", + Columns: []string{"email"}, Query: "email must not be null", + } +} + +func TestGenerateDQRuleSQL_HappyPath(t *testing.T) { + var got clients.Text2SQLRequest + c := server(t, http.StatusOK, clients.Text2SQLResponse{SQLQuery: "SELECT * FROM @dataset WHERE email IS NULL"}, &got) + out, err := generate_dq_rule_sql.NewTool(c).Handler(t.Context(), input()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if out.Status != generate_dq_rule_sql.StatusSuccess { + t.Fatalf("status = %q, want success (%s)", out.Status, out.Message) + } + if out.SQLQuery == "" { + t.Fatalf("expected generated SQL") + } + if len(got.Columns) != 1 || got.Columns[0] != "email" { + t.Fatalf("columns not forwarded: %+v", got.Columns) + } +} + +func TestGenerateDQRuleSQL_MissingInput(t *testing.T) { + c := server(t, http.StatusOK, clients.Text2SQLResponse{}, nil) + in := input() + in.Query = "" + out, _ := generate_dq_rule_sql.NewTool(c).Handler(t.Context(), in) + if out.Status != generate_dq_rule_sql.StatusValidationError { + t.Fatalf("status = %q, want validation_error", out.Status) + } +} + +func TestGenerateDQRuleSQL_NoColumns(t *testing.T) { + c := server(t, http.StatusOK, clients.Text2SQLResponse{}, nil) + in := input() + in.Columns = nil + out, _ := generate_dq_rule_sql.NewTool(c).Handler(t.Context(), in) + if out.Status != generate_dq_rule_sql.StatusValidationError { + t.Fatalf("status = %q, want validation_error", out.Status) + } +} + +func TestGenerateDQRuleSQL_BadRequestSurfaces(t *testing.T) { + c := server(t, http.StatusBadRequest, clients.Text2SQLResponse{}, nil) + out, _ := generate_dq_rule_sql.NewTool(c).Handler(t.Context(), input()) + if out.Status != generate_dq_rule_sql.StatusError { + t.Fatalf("status = %q, want error", out.Status) + } +} diff --git a/pkg/tools/get_dq_rule/tool.go b/pkg/tools/get_dq_rule/tool.go new file mode 100644 index 0000000..4fa714a --- /dev/null +++ b/pkg/tools/get_dq_rule/tool.go @@ -0,0 +1,103 @@ +// Package get_dq_rule implements the get_dq_rule MCP tool: it reads the +// definition of a single data quality rule (monitor) on an existing DQ job. +package get_dq_rule + +import ( + "context" + "fmt" + "net/http" + "strings" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +// OutputStatus is the overall outcome of a get_dq_rule call. +type OutputStatus string + +const ( + // StatusSuccess means the rule was found and returned. + StatusSuccess OutputStatus = "success" + // StatusValidationError means the inputs failed validation before any read. + StatusValidationError OutputStatus = "validation_error" + // StatusError means the rule could not be read due to a downstream DQ error. + StatusError OutputStatus = "error" +) + +// Input is the tool's typed input. +type Input struct { + JobName string `json:"jobName" jsonschema:"Required. Name of the data quality job the rule is attached to (a job, also called a 'dataset', is a saved check on one database table), e.g. 'PUBLIC.SAMPLE_DATASET'."` + MonitorName string `json:"monitorName" jsonschema:"Required. Name of the rule (the check; Collibra calls it a 'monitor') to read."` +} + +// Rule is the returned rule definition. +type Rule struct { + JobName string `json:"jobName"` + MonitorName string `json:"monitorName"` + MonitorType string `json:"monitorType"` + MonitorValue string `json:"monitorValue"` + FilterQuery string `json:"filterQuery,omitempty"` + ColumnName string `json:"columnName,omitempty"` + Description string `json:"description,omitempty"` + Dimensions []string `json:"dimensions,omitempty"` + Tolerance int `json:"tolerance"` + Active bool `json:"active"` + Suppressed bool `json:"suppressed"` + TemplateID string `json:"templateId,omitempty"` +} + +// Output is the typed response. +type Output struct { + Status OutputStatus `json:"status" jsonschema:"'success' when the rule was found; 'validation_error' for bad inputs; 'error' for downstream DQ failures."` + Message string `json:"message" jsonschema:"Human-readable summary."` + Rule *Rule `json:"rule,omitempty" jsonschema:"The rule definition, on success."` +} + +// NewTool returns the registered tool. +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "get_data_quality_rule", + Title: "Get Data Quality Rule", + Description: "Read the definition of a single data quality rule (a check on a table's data; Collibra calls it a 'monitor') on an existing data quality job (a saved check on ONE database table; also called a 'dataset'). " + + "Returns the rule's type, SQL, filter, tolerance (count of failing records allowed before it fails) and active/suppressed (kept but not scored) state.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if strings.TrimSpace(input.JobName) == "" { + return Output{Status: StatusValidationError, Message: "jobName is required."}, nil + } + if strings.TrimSpace(input.MonitorName) == "" { + return Output{Status: StatusValidationError, Message: "monitorName is required."}, nil + } + + r, err := clients.GetDQRule(ctx, collibraClient, strings.TrimSpace(input.JobName), strings.TrimSpace(input.MonitorName)) + if err != nil { + return Output{Status: StatusError, Message: fmt.Sprintf("Could not read rule: %v", err)}, nil + } + + return Output{ + Status: StatusSuccess, + Message: fmt.Sprintf("Found rule %q on job %q.", r.MonitorName, r.JobName), + Rule: &Rule{ + JobName: r.JobName, + MonitorName: r.MonitorName, + MonitorType: r.MonitorType, + MonitorValue: r.MonitorValue, + FilterQuery: r.FilterQuery, + ColumnName: r.ColumnName, + Description: r.Description, + Dimensions: r.Dimensions, + Tolerance: r.Tolerance, + Active: r.IsActive == 1, + Suppressed: r.IsSuppressed, + TemplateID: r.TemplateID, + }, + }, nil + } +} diff --git a/pkg/tools/get_dq_rule/tool_test.go b/pkg/tools/get_dq_rule/tool_test.go new file mode 100644 index 0000000..240ce33 --- /dev/null +++ b/pkg/tools/get_dq_rule/tool_test.go @@ -0,0 +1,63 @@ +package get_dq_rule_test + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/get_dq_rule" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func server(t *testing.T, code int, rule clients.DQRule) *http.Client { + mux := http.NewServeMux() + mux.HandleFunc("GET /rest/dq/internal/v1/jobs/{jobName}/monitors/rules/{monitorName}", func(w http.ResponseWriter, r *http.Request) { + if code != http.StatusOK { + w.WriteHeader(code) + _, _ = w.Write([]byte(`{"message":"boom"}`)) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(rule) + }) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + return testutil.NewClient(srv) +} + +func TestGetDQRule_HappyPath(t *testing.T) { + c := server(t, http.StatusOK, clients.DQRule{ + JobName: "PUBLIC.SAMPLE_DATASET", MonitorName: "Name_Not_Null", + MonitorType: "FREEFORM_SQL", MonitorValue: "SELECT 1", IsActive: 1, IsSuppressed: false, + }) + out, err := get_dq_rule.NewTool(c).Handler(t.Context(), get_dq_rule.Input{ + JobName: "PUBLIC.SAMPLE_DATASET", MonitorName: "Name_Not_Null", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if out.Status != get_dq_rule.StatusSuccess { + t.Fatalf("status = %q, want success (%s)", out.Status, out.Message) + } + if out.Rule == nil || out.Rule.MonitorName != "Name_Not_Null" || !out.Rule.Active { + t.Fatalf("unexpected rule: %+v", out.Rule) + } +} + +func TestGetDQRule_MissingInput(t *testing.T) { + c := server(t, http.StatusOK, clients.DQRule{}) + out, _ := get_dq_rule.NewTool(c).Handler(t.Context(), get_dq_rule.Input{JobName: "DS"}) + if out.Status != get_dq_rule.StatusValidationError { + t.Fatalf("status = %q, want validation_error", out.Status) + } +} + +func TestGetDQRule_NotFound(t *testing.T) { + c := server(t, http.StatusNotFound, clients.DQRule{}) + out, _ := get_dq_rule.NewTool(c).Handler(t.Context(), get_dq_rule.Input{JobName: "DS", MonitorName: "R"}) + if out.Status != get_dq_rule.StatusError { + t.Fatalf("status = %q, want error", out.Status) + } +} diff --git a/pkg/tools/get_dq_rule_results/tool.go b/pkg/tools/get_dq_rule_results/tool.go new file mode 100644 index 0000000..855ba33 --- /dev/null +++ b/pkg/tools/get_dq_rule_results/tool.go @@ -0,0 +1,148 @@ +// Package get_dq_rule_results implements the get_dq_rule_results MCP tool: it +// reads a rule's per-run results (scores, breaking-record counts, exceptions) +// after a job run. +package get_dq_rule_results + +import ( + "context" + "fmt" + "net/http" + "strings" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +// OutputStatus is the overall outcome of a get_dq_rule_results call. +type OutputStatus string + +const ( + // StatusSuccess means the results were returned. + StatusSuccess OutputStatus = "success" + // StatusValidationError means the inputs failed validation before any read. + StatusValidationError OutputStatus = "validation_error" + // StatusError means the results could not be read due to a downstream error. + StatusError OutputStatus = "error" +) + +const ( + sortOrderAsc = "ASC" + sortOrderDesc = "DESC" + + defaultLimit = 10 +) + +// Input is the tool's typed input. +type Input struct { + JobName string `json:"jobName" jsonschema:"Required. Name of the data quality job the rule is attached to (a job, also called a 'dataset', is a saved check on one database table)."` + RuleName string `json:"ruleName" jsonschema:"Required. Name of the rule (the check; Collibra calls it a 'monitor') whose results to read."` + Offset int `json:"offset,omitempty" jsonschema:"Optional. Pagination offset (min 0). Defaults to 0."` + Limit int `json:"limit,omitempty" jsonschema:"Optional. Max number of run-result entries to return (min 1). Defaults to 10."` + SortOrder string `json:"sortOrder,omitempty" jsonschema:"Optional. Order results by run date: 'DESC' (newest first, the default) or 'ASC'."` +} + +// ResultEntry is one per-run result for the rule. +type ResultEntry struct { + RunDate int64 `json:"runDate" jsonschema:"Run date as epoch milliseconds."` + RuleStatus string `json:"ruleStatus,omitempty" jsonschema:"Outcome for this run, e.g. PASSING, BREAKING, or EXCEPTION."` + PassFail bool `json:"passFail" jsonschema:"Whether the rule passed for this run."` + Score int `json:"score" jsonschema:"Rule score for this run (0-100)."` + TotalCount float64 `json:"totalCount" jsonschema:"Total records evaluated."` + BreakingRecords float64 `json:"breakingRecords" jsonschema:"Number of breaking (failing) records for this run."` + PassingRecords float64 `json:"passingRecords" jsonschema:"Number of passing records."` + BreakMsg string `json:"breakMsg,omitempty" jsonschema:"Break message, when the rule broke."` + Exception string `json:"exception,omitempty" jsonschema:"Exception message, when the run errored."` +} + +// Output is the typed response. +type Output struct { + Status OutputStatus `json:"status" jsonschema:"'success' when results were returned; 'validation_error' for bad inputs; 'error' for downstream DQ failures."` + Message string `json:"message" jsonschema:"Human-readable summary."` + RuleName string `json:"ruleName,omitempty" jsonschema:"The rule name, on success."` + RuleType string `json:"ruleType,omitempty" jsonschema:"The rule type, on success."` + Results []ResultEntry `json:"results,omitempty" jsonschema:"Per-run result entries, newest first by default."` + Total int64 `json:"total" jsonschema:"Total number of result entries available (for pagination)."` +} + +// NewTool returns the registered tool. +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "get_data_quality_rule_results", + Title: "Get Data Quality Rule Results", + Description: "Read a data quality rule's (a check on a table's data; Collibra calls it a 'monitor') results for each run of the job — the 0-100 score, the counts of breaking (failing) and passing records, " + + "pass/fail status and any exception for each run. Paginated (offset/limit), newest first by default.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if strings.TrimSpace(input.JobName) == "" { + return Output{Status: StatusValidationError, Message: "jobName is required."}, nil + } + if strings.TrimSpace(input.RuleName) == "" { + return Output{Status: StatusValidationError, Message: "ruleName is required."}, nil + } + sortOrder, out := resolveSortOrder(input.SortOrder) + if out != nil { + return *out, nil + } + if input.Offset < 0 { + return Output{Status: StatusValidationError, Message: "offset must be >= 0."}, nil + } + limit := input.Limit + if limit == 0 { + limit = defaultLimit + } + if limit < 1 { + return Output{Status: StatusValidationError, Message: "limit must be >= 1."}, nil + } + + res, err := clients.GetDQRuleResults(ctx, collibraClient, strings.TrimSpace(input.JobName), strings.TrimSpace(input.RuleName), input.Offset, limit, sortOrder) + if err != nil { + return Output{Status: StatusError, Message: fmt.Sprintf("Could not read rule results: %v", err)}, nil + } + + entries := make([]ResultEntry, 0, len(res.Results)) + for _, r := range res.Results { + entries = append(entries, ResultEntry{ + RunDate: r.RunDate, + RuleStatus: r.RuleStatus, + PassFail: r.PassFail, + Score: r.Score, + TotalCount: r.TotalCount, + BreakingRecords: r.BreakingRecords, + PassingRecords: r.PassingRecords, + BreakMsg: r.BreakMsg, + Exception: r.Exception, + }) + } + + return Output{ + Status: StatusSuccess, + Message: fmt.Sprintf("Returned %d of %d result(s) for rule %q on job %q.", len(entries), res.Total, res.RuleName, res.Dataset), + RuleName: res.RuleName, + RuleType: res.RuleType, + Results: entries, + Total: res.Total, + }, nil + } +} + +// resolveSortOrder defaults an empty sortOrder to DESC and rejects anything else. +func resolveSortOrder(sortOrder string) (string, *Output) { + switch strings.ToUpper(strings.TrimSpace(sortOrder)) { + case "", sortOrderDesc: + return sortOrderDesc, nil + case sortOrderAsc: + return sortOrderAsc, nil + default: + return "", &Output{ + Status: StatusValidationError, + Message: fmt.Sprintf("sortOrder %q is invalid. Use %q or %q.", sortOrder, sortOrderAsc, sortOrderDesc), + } + } +} diff --git a/pkg/tools/get_dq_rule_results/tool_test.go b/pkg/tools/get_dq_rule_results/tool_test.go new file mode 100644 index 0000000..ddd7074 --- /dev/null +++ b/pkg/tools/get_dq_rule_results/tool_test.go @@ -0,0 +1,68 @@ +package get_dq_rule_results_test + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/get_dq_rule_results" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func server(t *testing.T, code int, resp clients.DQRuleResults, sortOrder *string) *http.Client { + mux := http.NewServeMux() + mux.HandleFunc("GET /rest/dq/internal/v1/monitoring/rules/{jobName}/{ruleName}", func(w http.ResponseWriter, r *http.Request) { + if sortOrder != nil { + *sortOrder = r.URL.Query().Get("sortOrder") + } + if code != http.StatusOK { + w.WriteHeader(code) + _, _ = w.Write([]byte(`{"message":"boom"}`)) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(resp) + }) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + return testutil.NewClient(srv) +} + +func TestGetDQRuleResults_HappyPath_DefaultsToDesc(t *testing.T) { + var sortOrder string + c := server(t, http.StatusOK, clients.DQRuleResults{ + Dataset: "DS", RuleName: "R", RuleType: "SQLF", Total: 2, + Results: []clients.DQRuleResultEntry{{RunDate: 1, Score: 100, PassFail: true, RuleStatus: "PASSING"}}, + }, &sortOrder) + out, err := get_dq_rule_results.NewTool(c).Handler(t.Context(), get_dq_rule_results.Input{JobName: "DS", RuleName: "R"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if out.Status != get_dq_rule_results.StatusSuccess { + t.Fatalf("status = %q, want success (%s)", out.Status, out.Message) + } + if sortOrder != "DESC" { + t.Fatalf("sortOrder = %q, want DESC (default)", sortOrder) + } + if len(out.Results) != 1 || out.Total != 2 { + t.Fatalf("unexpected results: %+v total=%d", out.Results, out.Total) + } +} + +func TestGetDQRuleResults_InvalidSortOrder(t *testing.T) { + c := server(t, http.StatusOK, clients.DQRuleResults{}, nil) + out, _ := get_dq_rule_results.NewTool(c).Handler(t.Context(), get_dq_rule_results.Input{JobName: "DS", RuleName: "R", SortOrder: "SIDEWAYS"}) + if out.Status != get_dq_rule_results.StatusValidationError { + t.Fatalf("status = %q, want validation_error", out.Status) + } +} + +func TestGetDQRuleResults_MissingInput(t *testing.T) { + c := server(t, http.StatusOK, clients.DQRuleResults{}, nil) + out, _ := get_dq_rule_results.NewTool(c).Handler(t.Context(), get_dq_rule_results.Input{JobName: "DS"}) + if out.Status != get_dq_rule_results.StatusValidationError { + t.Fatalf("status = %q, want validation_error", out.Status) + } +} diff --git a/pkg/tools/get_dq_rule_template/tool.go b/pkg/tools/get_dq_rule_template/tool.go new file mode 100644 index 0000000..c710116 --- /dev/null +++ b/pkg/tools/get_dq_rule_template/tool.go @@ -0,0 +1,93 @@ +// Package get_dq_rule_template implements the get_dq_rule_template MCP tool: it +// reads a single data quality rule template by name. +package get_dq_rule_template + +import ( + "context" + "fmt" + "net/http" + "strings" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +// OutputStatus is the overall outcome of a get_dq_rule_template call. +type OutputStatus string + +const ( + // StatusSuccess means the template was found and returned. + StatusSuccess OutputStatus = "success" + // StatusValidationError means the inputs failed validation before any read. + StatusValidationError OutputStatus = "validation_error" + // StatusError means the template could not be read due to a downstream error. + StatusError OutputStatus = "error" +) + +// Input is the tool's typed input. +type Input struct { + RuleTemplateName string `json:"ruleTemplateName" jsonschema:"Required. Name of the rule template (from list_data_quality_rule_templates)."` +} + +// Template is the returned template definition. +type Template struct { + ID string `json:"id"` + Name string `json:"ruleTemplateName"` + Description string `json:"description,omitempty"` + SQL string `json:"sql,omitempty"` + Dialect string `json:"dialect,omitempty"` + Dimensions []string `json:"dimensions,omitempty"` + Tolerance *int `json:"tolerance,omitempty"` + IsSystem bool `json:"isSystem"` + DeployedRuleCount int64 `json:"deployedRuleCount"` +} + +// Output is the typed response. +type Output struct { + Status OutputStatus `json:"status" jsonschema:"'success' when the template was found; 'validation_error' for bad inputs; 'error' for downstream DQ failures."` + Message string `json:"message" jsonschema:"Human-readable summary."` + Template *Template `json:"template,omitempty" jsonschema:"The template definition, on success."` +} + +// NewTool returns the registered tool. +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "get_data_quality_rule_template", + Title: "Get Data Quality Rule Template", + Description: "Read a single data quality rule template by name — its parameterized SQL, dimensions (data-quality categories such as Accuracy or Completeness), " + + "default tolerance (number of failing records allowed before a rule fails; a count, not a percentage), whether it is built-in (system) vs custom, and how many rules have been deployed from it.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if strings.TrimSpace(input.RuleTemplateName) == "" { + return Output{Status: StatusValidationError, Message: "ruleTemplateName is required."}, nil + } + + t, err := clients.GetDQRuleTemplate(ctx, collibraClient, strings.TrimSpace(input.RuleTemplateName)) + if err != nil { + return Output{Status: StatusError, Message: fmt.Sprintf("Could not read template: %v", err)}, nil + } + + return Output{ + Status: StatusSuccess, + Message: fmt.Sprintf("Found template %q.", t.Name), + Template: &Template{ + ID: t.ID, + Name: t.Name, + Description: t.Description, + SQL: t.SQL, + Dialect: t.Dialect, + Dimensions: t.Dimensions, + Tolerance: t.Tolerance, + IsSystem: t.IsSystem, + DeployedRuleCount: t.DeployedRuleCount, + }, + }, nil + } +} diff --git a/pkg/tools/get_dq_rule_template/tool_test.go b/pkg/tools/get_dq_rule_template/tool_test.go new file mode 100644 index 0000000..fca1e7d --- /dev/null +++ b/pkg/tools/get_dq_rule_template/tool_test.go @@ -0,0 +1,58 @@ +package get_dq_rule_template_test + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/get_dq_rule_template" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func server(t *testing.T, code int, tmpl clients.DQRuleTemplate) *http.Client { + mux := http.NewServeMux() + mux.HandleFunc("GET /rest/dq/1.0/ruleTemplates/{ruleTemplateName}", func(w http.ResponseWriter, r *http.Request) { + if code != http.StatusOK { + w.WriteHeader(code) + _, _ = w.Write([]byte(`{"message":"boom"}`)) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(tmpl) + }) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + return testutil.NewClient(srv) +} + +func TestGetDQRuleTemplate_HappyPath(t *testing.T) { + c := server(t, http.StatusOK, clients.DQRuleTemplate{ID: "t1", Name: "Not Null Check", IsSystem: true, DeployedRuleCount: 3}) + out, err := get_dq_rule_template.NewTool(c).Handler(t.Context(), get_dq_rule_template.Input{RuleTemplateName: "Not Null Check"}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if out.Status != get_dq_rule_template.StatusSuccess { + t.Fatalf("status = %q, want success (%s)", out.Status, out.Message) + } + if out.Template == nil || out.Template.Name != "Not Null Check" || out.Template.DeployedRuleCount != 3 { + t.Fatalf("unexpected template: %+v", out.Template) + } +} + +func TestGetDQRuleTemplate_MissingInput(t *testing.T) { + c := server(t, http.StatusOK, clients.DQRuleTemplate{}) + out, _ := get_dq_rule_template.NewTool(c).Handler(t.Context(), get_dq_rule_template.Input{}) + if out.Status != get_dq_rule_template.StatusValidationError { + t.Fatalf("status = %q, want validation_error", out.Status) + } +} + +func TestGetDQRuleTemplate_NotFound(t *testing.T) { + c := server(t, http.StatusNotFound, clients.DQRuleTemplate{}) + out, _ := get_dq_rule_template.NewTool(c).Handler(t.Context(), get_dq_rule_template.Input{RuleTemplateName: "nope"}) + if out.Status != get_dq_rule_template.StatusError { + t.Fatalf("status = %q, want error", out.Status) + } +} diff --git a/pkg/tools/list_dq_rule_templates/tool.go b/pkg/tools/list_dq_rule_templates/tool.go new file mode 100644 index 0000000..3384903 --- /dev/null +++ b/pkg/tools/list_dq_rule_templates/tool.go @@ -0,0 +1,116 @@ +// Package list_dq_rule_templates implements the list_dq_rule_templates MCP tool: +// it lists the data quality rule templates (built-in and custom) available in the +// connected DQ environment, so they can be chosen for deployment. +package list_dq_rule_templates + +import ( + "context" + "fmt" + "net/http" + "strings" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +// OutputStatus is the overall outcome of a list_dq_rule_templates call. +type OutputStatus string + +const ( + // StatusSuccess means the templates were returned. + StatusSuccess OutputStatus = "success" + // StatusValidationError means the inputs failed validation before any read. + StatusValidationError OutputStatus = "validation_error" + // StatusError means the templates could not be read due to a downstream error. + StatusError OutputStatus = "error" +) + +const defaultLimit = 100 + +// Input is the tool's typed input. All fields are optional filters/pagination. +type Input struct { + Name string `json:"name,omitempty" jsonschema:"Optional. Partial-match filter on the template name."` + Dimension string `json:"dimension,omitempty" jsonschema:"Optional. Filter by data quality dimension — a category such as Accuracy, Completeness or Validity (e.g. 'Completeness', 'Validity')."` + IsSystem *bool `json:"isSystem,omitempty" jsonschema:"Optional. Filter by origin: true = built-in (system) templates only, false = custom user-defined templates only, omit = all."` + Offset int `json:"offset,omitempty" jsonschema:"Optional. Pagination offset (min 0). Defaults to 0."` + Limit int `json:"limit,omitempty" jsonschema:"Optional. Max templates to return (1-1000). Defaults to 100."` +} + +// Template is one returned rule template. +type Template struct { + Name string `json:"ruleTemplateName" jsonschema:"Template name — the key passed to get_data_quality_rule_template and deploy_data_quality_rule_template."` + Description string `json:"description,omitempty" jsonschema:"What the template checks."` + SQL string `json:"sql,omitempty" jsonschema:"Parameterized SQL pattern (uses a {{column}} placeholder)."` + Dimensions []string `json:"dimensions,omitempty" jsonschema:"Data quality dimensions (categories such as Accuracy, Completeness, Validity) the template covers."` + Tolerance *int `json:"tolerance,omitempty" jsonschema:"Default tolerance — number of failing ('breaking') records allowed before a rule fails; a count, not a percentage — when set."` + IsSystem bool `json:"isSystem" jsonschema:"True for built-in (system) templates, false for custom user-defined ones."` +} + +// Output is the typed response. +type Output struct { + Status OutputStatus `json:"status" jsonschema:"'success' when templates were returned; 'validation_error' for bad inputs; 'error' for downstream DQ failures."` + Message string `json:"message" jsonschema:"Human-readable summary."` + Templates []Template `json:"templates,omitempty" jsonschema:"The matching templates."` + Total int64 `json:"total" jsonschema:"Total number of matching templates (for pagination)."` +} + +// NewTool returns the registered tool. +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "list_data_quality_rule_templates", + Title: "List Data Quality Rule Templates", + Description: "List the data quality rule templates available in the connected DQ environment — built-in " + + "(OOTB, i.e. out-of-the-box) templates plus any custom user-defined ones. Each template is a parameterized SQL pattern that " + + "can be deployed as concrete rules (checks; Collibra calls them 'monitors') across columns via deploy_data_quality_rule_template. Optional filters: name, " + + "dimension (a data-quality category such as Accuracy or Completeness), and isSystem (built-in vs custom). Paginated (offset/limit).", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if input.Offset < 0 { + return Output{Status: StatusValidationError, Message: "offset must be >= 0."}, nil + } + limit := input.Limit + if limit == 0 { + limit = defaultLimit + } + if limit < 1 || limit > 1000 { + return Output{Status: StatusValidationError, Message: "limit must be between 1 and 1000."}, nil + } + + list, err := clients.ListDQRuleTemplates(ctx, collibraClient, clients.ListDQRuleTemplatesParams{ + Name: strings.TrimSpace(input.Name), + Dimension: strings.TrimSpace(input.Dimension), + IsSystem: input.IsSystem, + Offset: input.Offset, + Limit: limit, + }) + if err != nil { + return Output{Status: StatusError, Message: fmt.Sprintf("Could not list templates: %v", err)}, nil + } + + templates := make([]Template, 0, len(list.Results)) + for _, t := range list.Results { + templates = append(templates, Template{ + Name: t.Name, + Description: t.Description, + SQL: t.SQL, + Dimensions: t.Dimensions, + Tolerance: t.Tolerance, + IsSystem: t.IsSystem, + }) + } + + return Output{ + Status: StatusSuccess, + Message: fmt.Sprintf("Returned %d of %d template(s).", len(templates), list.Total), + Templates: templates, + Total: list.Total, + }, nil + } +} diff --git a/pkg/tools/list_dq_rule_templates/tool_test.go b/pkg/tools/list_dq_rule_templates/tool_test.go new file mode 100644 index 0000000..bbbc69d --- /dev/null +++ b/pkg/tools/list_dq_rule_templates/tool_test.go @@ -0,0 +1,74 @@ +package list_dq_rule_templates_test + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/list_dq_rule_templates" + "github.com/collibra/chip/pkg/tools/testutil" +) + +func server(t *testing.T, code int, body any, gotQuery *string) *http.Client { + mux := http.NewServeMux() + mux.HandleFunc("GET /rest/dq/1.0/ruleTemplates", func(w http.ResponseWriter, r *http.Request) { + if gotQuery != nil { + *gotQuery = r.URL.RawQuery + } + if code != http.StatusOK { + w.WriteHeader(code) + _, _ = w.Write([]byte(`{"message":"boom"}`)) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(body) + }) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + return testutil.NewClient(srv) +} + +func TestListDQRuleTemplates_HappyPath(t *testing.T) { + var query string + tol := 5 + c := server(t, http.StatusOK, map[string]any{ + "results": []clients.DQRuleTemplate{ + {ID: "t1", Name: "Not Null Check", Dimensions: []string{"Completeness"}, Tolerance: &tol, IsSystem: true}, + }, + "total": 1, "offset": 0, "limit": 100, + }, &query) + + isSystem := true + out, err := list_dq_rule_templates.NewTool(c).Handler(t.Context(), list_dq_rule_templates.Input{IsSystem: &isSystem}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if out.Status != list_dq_rule_templates.StatusSuccess { + t.Fatalf("status = %q, want success (%s)", out.Status, out.Message) + } + if len(out.Templates) != 1 || out.Templates[0].Name != "Not Null Check" || !out.Templates[0].IsSystem { + t.Fatalf("unexpected templates: %+v", out.Templates) + } + if !strings.Contains(query, "isSystem=true") { + t.Fatalf("expected isSystem=true in query, got %q", query) + } +} + +func TestListDQRuleTemplates_InvalidLimit(t *testing.T) { + c := server(t, http.StatusOK, map[string]any{}, nil) + out, _ := list_dq_rule_templates.NewTool(c).Handler(t.Context(), list_dq_rule_templates.Input{Limit: 5000}) + if out.Status != list_dq_rule_templates.StatusValidationError { + t.Fatalf("status = %q, want validation_error", out.Status) + } +} + +func TestListDQRuleTemplates_ForbiddenSurfaces(t *testing.T) { + c := server(t, http.StatusForbidden, nil, nil) + out, _ := list_dq_rule_templates.NewTool(c).Handler(t.Context(), list_dq_rule_templates.Input{}) + if out.Status != list_dq_rule_templates.StatusError { + t.Fatalf("status = %q, want error", out.Status) + } +} diff --git a/pkg/tools/register.go b/pkg/tools/register.go index 98df37e..862275b 100644 --- a/pkg/tools/register.go +++ b/pkg/tools/register.go @@ -10,16 +10,23 @@ import ( "github.com/collibra/chip/pkg/tools/create_assessment" "github.com/collibra/chip/pkg/tools/create_asset" "github.com/collibra/chip/pkg/tools/create_dq_job" + "github.com/collibra/chip/pkg/tools/create_dq_rule" + "github.com/collibra/chip/pkg/tools/deploy_dq_rule_template" "github.com/collibra/chip/pkg/tools/discover_business_glossary" "github.com/collibra/chip/pkg/tools/discover_data_assets" "github.com/collibra/chip/pkg/tools/edit_assessment" "github.com/collibra/chip/pkg/tools/edit_asset" + "github.com/collibra/chip/pkg/tools/find_dq_rules" + "github.com/collibra/chip/pkg/tools/generate_dq_rule_sql" "github.com/collibra/chip/pkg/tools/get_assessment" "github.com/collibra/chip/pkg/tools/get_asset_details" "github.com/collibra/chip/pkg/tools/get_business_term_data" "github.com/collibra/chip/pkg/tools/get_column_semantics" "github.com/collibra/chip/pkg/tools/get_context_specification" "github.com/collibra/chip/pkg/tools/get_debug_mcp_init_request" + "github.com/collibra/chip/pkg/tools/get_dq_rule" + "github.com/collibra/chip/pkg/tools/get_dq_rule_results" + "github.com/collibra/chip/pkg/tools/get_dq_rule_template" "github.com/collibra/chip/pkg/tools/get_lineage_downstream" "github.com/collibra/chip/pkg/tools/get_lineage_entity" "github.com/collibra/chip/pkg/tools/get_lineage_transformation" @@ -30,21 +37,30 @@ import ( "github.com/collibra/chip/pkg/tools/list_asset_types" "github.com/collibra/chip/pkg/tools/list_context_specifications" "github.com/collibra/chip/pkg/tools/list_data_contracts" + "github.com/collibra/chip/pkg/tools/list_dq_rule_templates" "github.com/collibra/chip/pkg/tools/prepare_create_asset" "github.com/collibra/chip/pkg/tools/pull_data_contract_manifest" "github.com/collibra/chip/pkg/tools/push_data_contract_manifest" "github.com/collibra/chip/pkg/tools/remove_data_classification_match" "github.com/collibra/chip/pkg/tools/search_asset_keyword" + "github.com/collibra/chip/pkg/tools/search_catalog_columns" "github.com/collibra/chip/pkg/tools/search_data_classes" "github.com/collibra/chip/pkg/tools/search_data_classification_matches" "github.com/collibra/chip/pkg/tools/search_lineage_entities" "github.com/collibra/chip/pkg/tools/search_lineage_transformations" + "github.com/collibra/chip/pkg/tools/validate_dq_rule" ) // ContextSpecificationsFeature is the experimental-feature identifier used to // gate the context specification tools. const ContextSpecificationsFeature = "context-specifications" +// DataQualityFeatureName gates the data-quality rule tools (create/validate/read rules, +// rule templates, Text2SQL and catalog column search) behind --experimental. Some WRITE to +// Collibra (create rules, deploy templates), so they stay opt-in until they graduate. Off by +// default. Shared with the data-quality job-creation tools. +const DataQualityFeatureName = "data-quality" + // CopilotToolNames lists tool names that are routed to the copilot service. // Used by chip-service to direct these requests to the copilot backend // instead of the standard DGC API. @@ -53,11 +69,6 @@ var CopilotToolNames = []string{ "discover_business_glossary", } -// DataQualityFeatureName gates the create_data_quality_job tool (discovery + preview + create in one) -// behind --experimental. It WRITES to Collibra (creates and queues jobs), so it stays opt-in — like the -// Data Product skills — until it graduates. Off by default. -const DataQualityFeatureName = "data-quality" - func RegisterAll(server *chip.Server, client *http.Client, toolConfig *chip.ServerToolConfig) error { toolRegister(server, toolConfig, discover_data_assets.NewTool(client)) toolRegister(server, toolConfig, discover_business_glossary.NewTool(client)) @@ -88,13 +99,23 @@ func RegisterAll(server *chip.Server, client *http.Client, toolConfig *chip.Serv toolRegister(server, toolConfig, get_assessment.NewTool(client)) toolRegister(server, toolConfig, create_assessment.NewTool(client)) toolRegister(server, toolConfig, edit_assessment.NewTool(client)) + if toolConfig.IsExperimentalEnabled(DataQualityFeatureName) { + toolRegister(server, toolConfig, create_dq_job.NewTool(client)) + toolRegister(server, toolConfig, create_dq_rule.NewTool(client)) + toolRegister(server, toolConfig, get_dq_rule.NewTool(client)) + toolRegister(server, toolConfig, get_dq_rule_results.NewTool(client)) + toolRegister(server, toolConfig, validate_dq_rule.NewTool(client)) + toolRegister(server, toolConfig, list_dq_rule_templates.NewTool(client)) + toolRegister(server, toolConfig, get_dq_rule_template.NewTool(client)) + toolRegister(server, toolConfig, deploy_dq_rule_template.NewTool(client)) + toolRegister(server, toolConfig, generate_dq_rule_sql.NewTool(client)) + toolRegister(server, toolConfig, find_dq_rules.NewTool(client)) + toolRegister(server, toolConfig, search_catalog_columns.NewTool(client)) + } if toolConfig.IsExperimentalEnabled(ContextSpecificationsFeature) { toolRegister(server, toolConfig, list_context_specifications.NewTool(client)) toolRegister(server, toolConfig, get_context_specification.NewTool(client)) } - if toolConfig.IsExperimentalEnabled(DataQualityFeatureName) { - toolRegister(server, toolConfig, create_dq_job.NewTool(client)) - } if toolConfig.EnableDebugTools { toolRegister(server, toolConfig, get_debug_mcp_init_request.NewTool(client)) diff --git a/pkg/tools/register_test.go b/pkg/tools/register_test.go index e2ff635..7ac3928 100644 --- a/pkg/tools/register_test.go +++ b/pkg/tools/register_test.go @@ -28,7 +28,19 @@ func TestRegisterAll_DebugToolVisibleWhenEnabled(t *testing.T) { } } -var dataQualityToolNames = []string{"create_data_quality_job"} +var dataQualityToolNames = []string{ + "create_data_quality_job", + "create_data_quality_rule", + "get_data_quality_rule", + "get_data_quality_rule_results", + "validate_data_quality_rule", + "list_data_quality_rule_templates", + "get_data_quality_rule_template", + "deploy_data_quality_rule_template", + "generate_data_quality_rule_sql", + "find_data_quality_rules", + "search_catalog_columns", +} func TestRegisterAll_DataQualityToolsHiddenByDefault(t *testing.T) { names := listToolNames(t, &chip.ServerToolConfig{}) diff --git a/pkg/tools/search_catalog_columns/tool.go b/pkg/tools/search_catalog_columns/tool.go new file mode 100644 index 0000000..a6f7e42 --- /dev/null +++ b/pkg/tools/search_catalog_columns/tool.go @@ -0,0 +1,134 @@ +// Package search_catalog_columns implements the search_catalog_columns MCP tool: +// it finds catalog Column assets by metadata the public REST search cannot filter +// on (attribute values, assigned roles, relations to other assets), via the DGC +// Knowledge Graph GraphQL API. Multiple filters are combined with AND. +package search_catalog_columns + +import ( + "context" + "fmt" + "net/http" + "strings" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +// OutputStatus is the overall outcome of a search_catalog_columns call. +type OutputStatus string + +const ( + // StatusSuccess means the search ran. + StatusSuccess OutputStatus = "success" + // StatusValidationError means the inputs failed validation before any call. + StatusValidationError OutputStatus = "validation_error" + // StatusError means the search failed (e.g. KG endpoint unavailable). + StatusError OutputStatus = "error" +) + +const defaultLimit = 25 + +// Input is the tool's typed input. All filters are optional but at least one +// must be set; they are combined with AND. The search is always scoped to Column +// assets. +type Input struct { + Domain string `json:"domain,omitempty" jsonschema:"Optional. Exact domain name the column lives in."` + Community string `json:"community,omitempty" jsonschema:"Optional. Exact community name (the column's domain's parent)."` + Description string `json:"description,omitempty" jsonschema:"Optional. Substring to match in the column's Description attribute."` + DataType string `json:"dataType,omitempty" jsonschema:"Optional. Substring to match in the column's Data Type attribute."` + DataStewardRole string `json:"dataStewardRole,omitempty" jsonschema:"Optional. Match columns that have a responsibility with this role name assigned (e.g. 'Data Steward'). Matches by role, not by a specific person."` + BusinessTerm string `json:"businessTerm,omitempty" jsonschema:"Optional. Exact display name of a Business Term the column represents."` + BusinessRule string `json:"businessRule,omitempty" jsonschema:"Optional. Exact display name of a Business Rule that governs the column."` + DataElement string `json:"dataElement,omitempty" jsonschema:"Optional. Exact display name of a Data Element the column targets (technical lineage)."` + DataAttribute string `json:"dataAttribute,omitempty" jsonschema:"Optional. Exact display name of a Data Attribute that represents the column."` + Limit int `json:"limit,omitempty" jsonschema:"Optional. Max columns to return. Defaults to 25."` + Offset int `json:"offset,omitempty" jsonschema:"Optional. Pagination offset (min 0). Defaults to 0."` +} + +// ColumnResult is one matching column. +type ColumnResult struct { + ID string `json:"id" jsonschema:"Column asset UUID."` + FullName string `json:"fullName" jsonschema:"Fully-qualified column name."` + DisplayName string `json:"displayName,omitempty" jsonschema:"Column display name."` + Domain string `json:"domain,omitempty" jsonschema:"The column's domain."` +} + +// Output is the typed response. +type Output struct { + Status OutputStatus `json:"status" jsonschema:"'success' when the search ran; 'validation_error' for bad inputs; 'error' for downstream failures (incl. KG endpoint unavailable)."` + Message string `json:"message" jsonschema:"Human-readable summary."` + Columns []ColumnResult `json:"columns,omitempty" jsonschema:"Matching columns."` + Count int `json:"count" jsonschema:"Number of columns returned."` +} + +// NewTool returns the registered tool. +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "search_catalog_columns", + Title: "Search Catalog Columns by Metadata", + Description: "Find catalog Column assets by metadata that keyword search cannot filter on — Description / Data Type " + + "(attribute values), a Data Steward role, or relations to a Business Term, Business Rule, Data Element or Data Attribute " + + "(by name). Filters are combined with AND. Useful for picking out columns to attach data-quality rules (checks; Collibra calls them 'monitors') to at scale. " + + "Requires the DGC Knowledge Graph API (Collibra's metadata graph query service) to be enabled on the instance; classification-tag filtering is not supported. " + + "A broad lone substring filter (e.g. description alone) can exceed the Knowledge Graph query timeout — combine with a domain or another selective filter.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if input.Offset < 0 { + return Output{Status: StatusValidationError, Message: "offset must be >= 0."}, nil + } + params := clients.CatalogColumnSearchParams{ + Domain: strings.TrimSpace(input.Domain), + Community: strings.TrimSpace(input.Community), + Description: strings.TrimSpace(input.Description), + DataType: strings.TrimSpace(input.DataType), + StewardRole: strings.TrimSpace(input.DataStewardRole), + BusinessTerm: strings.TrimSpace(input.BusinessTerm), + BusinessRule: strings.TrimSpace(input.BusinessRule), + DataElement: strings.TrimSpace(input.DataElement), + DataAttribute: strings.TrimSpace(input.DataAttribute), + Limit: input.Limit, + Offset: input.Offset, + } + if !hasAnyFilter(params) { + return Output{Status: StatusValidationError, Message: "Provide at least one filter (domain, community, description, dataType, dataStewardRole, businessTerm, businessRule, dataElement, or dataAttribute)."}, nil + } + if params.Limit == 0 { + params.Limit = defaultLimit + } + + cols, err := clients.SearchCatalogColumns(ctx, collibraClient, params) + if err != nil { + return Output{Status: StatusError, Message: fmt.Sprintf("Could not search columns: %v", err)}, nil + } + + results := make([]ColumnResult, 0, len(cols)) + for _, c := range cols { + results = append(results, ColumnResult{ + ID: c.ID, + FullName: c.FullName, + DisplayName: c.DisplayName, + Domain: c.Domain.Name, + }) + } + + return Output{ + Status: StatusSuccess, + Message: fmt.Sprintf("Found %d matching column(s).", len(results)), + Columns: results, + Count: len(results), + }, nil + } +} + +func hasAnyFilter(p clients.CatalogColumnSearchParams) bool { + return p.Domain != "" || p.Community != "" || p.Description != "" || p.DataType != "" || + p.StewardRole != "" || p.BusinessTerm != "" || p.BusinessRule != "" || + p.DataElement != "" || p.DataAttribute != "" +} diff --git a/pkg/tools/search_catalog_columns/tool_test.go b/pkg/tools/search_catalog_columns/tool_test.go new file mode 100644 index 0000000..8a5b0c0 --- /dev/null +++ b/pkg/tools/search_catalog_columns/tool_test.go @@ -0,0 +1,92 @@ +package search_catalog_columns_test + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/collibra/chip/pkg/tools/search_catalog_columns" + "github.com/collibra/chip/pkg/tools/testutil" +) + +// server mocks the KG GraphQL endpoint. It captures the raw request body and +// returns either the given assets or a graphql error. +func server(t *testing.T, body string, graphqlErr bool, captured *string) *http.Client { + mux := http.NewServeMux() + mux.HandleFunc("POST /graphql/knowledgeGraph/v1", func(w http.ResponseWriter, r *http.Request) { + raw, _ := io.ReadAll(r.Body) + if captured != nil { + *captured = string(raw) + } + w.Header().Set("Content-Type", "application/json") + if graphqlErr { + _, _ = w.Write([]byte(`{"errors":[{"message":"boom"}]}`)) + return + } + _, _ = w.Write([]byte(body)) + }) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + return testutil.NewClient(srv) +} + +func TestSearchCatalogColumns_HappyPath_BuildsColumnAndRelationFilter(t *testing.T) { + var reqBody string + resp := `{"data":{"assets":[ + {"id":"c1","fullName":"schema.table.email","displayName":"email","type":{"name":"Column"},"domain":{"name":"Sales"}} + ]}}` + c := server(t, resp, false, &reqBody) + + out, err := search_catalog_columns.NewTool(c).Handler(t.Context(), search_catalog_columns.Input{ + Domain: "Sales", + BusinessTerm: "Customer Email", + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if out.Status != search_catalog_columns.StatusSuccess { + t.Fatalf("status = %q, want success (%s)", out.Status, out.Message) + } + if out.Count != 1 || out.Columns[0].FullName != "schema.table.email" || out.Columns[0].Domain != "Sales" { + t.Fatalf("unexpected columns: %+v", out.Columns) + } + // The where clause must scope to Column and carry the Business Term relation + // public-id. + if !strings.Contains(reqBody, "\"Column\"") { + t.Fatalf("expected type=Column in where clause: %s", reqBody) + } + if !strings.Contains(reqBody, "BusinessAssetRepresentsDataAsset") { + t.Fatalf("expected business-term relation public id in where clause: %s", reqBody) + } + // Sanity: the request body must be valid JSON with a variables.where object. + var parsed struct { + Variables struct { + Where map[string]any `json:"where"` + } `json:"variables"` + } + if err := json.Unmarshal([]byte(reqBody), &parsed); err != nil { + t.Fatalf("request body not valid JSON: %v", err) + } + if parsed.Variables.Where == nil { + t.Fatalf("expected variables.where to be set") + } +} + +func TestSearchCatalogColumns_RequiresAtLeastOneFilter(t *testing.T) { + c := server(t, `{"data":{"assets":[]}}`, false, nil) + out, _ := search_catalog_columns.NewTool(c).Handler(t.Context(), search_catalog_columns.Input{}) + if out.Status != search_catalog_columns.StatusValidationError { + t.Fatalf("status = %q, want validation_error", out.Status) + } +} + +func TestSearchCatalogColumns_GraphqlErrorSurfaces(t *testing.T) { + c := server(t, "", true, nil) + out, _ := search_catalog_columns.NewTool(c).Handler(t.Context(), search_catalog_columns.Input{Description: "pii"}) + if out.Status != search_catalog_columns.StatusError { + t.Fatalf("status = %q, want error", out.Status) + } +} diff --git a/pkg/tools/validate_dq_rule/tool.go b/pkg/tools/validate_dq_rule/tool.go new file mode 100644 index 0000000..4186865 --- /dev/null +++ b/pkg/tools/validate_dq_rule/tool.go @@ -0,0 +1,109 @@ +// Package validate_dq_rule implements the validate_dq_rule MCP tool: it checks +// that a rule's SQL/definition is valid before it is saved or run, so a bad rule +// is caught up front rather than at run time. +package validate_dq_rule + +import ( + "context" + "fmt" + "net/http" + "strings" + + "github.com/collibra/chip/pkg/chip" + "github.com/collibra/chip/pkg/clients" + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +// OutputStatus is the overall outcome of a validate_dq_rule call. +type OutputStatus string + +const ( + // StatusSuccess means validation ran (see Valid for the verdict). + StatusSuccess OutputStatus = "success" + // StatusValidationError means the inputs failed validation before any call. + StatusValidationError OutputStatus = "validation_error" + // StatusError means validation could not run due to a downstream error. + StatusError OutputStatus = "error" +) + +// Input is the tool's typed input. edgeSiteId/connectionId/schemaName are the +// discovery fields from prepare_create_dq_job. +type Input struct { + EdgeSiteID string `json:"edgeSiteId" jsonschema:"Required. UUID of the Collibra Edge runtime/site that reaches the source database. From prepare_create_data_quality_job resolved.edgeSiteId."` + ConnectionID string `json:"connectionId" jsonschema:"Required. UUID of the specific database connection on that Edge site. From prepare_create_data_quality_job resolved.connectionId."` + SchemaName string `json:"schemaName" jsonschema:"Required. Schema name the rule runs against."` + JobName string `json:"jobName" jsonschema:"Required. Name of the data quality job the rule belongs to (a job, also called a 'dataset', is a saved check on one database table)."` + PreviewRule string `json:"previewRule" jsonschema:"Required. The rule SQL to validate (the same value you would pass as monitorValue)."` + FilterQuery string `json:"filterQuery,omitempty" jsonschema:"Optional. Additional WHERE-clause filter applied to the rule."` + RowLimit int `json:"rowLimit,omitempty" jsonschema:"Optional. Row limit for the underlying probe query. Defaults to 0 (service default)."` +} + +// Output is the typed response. +type Output struct { + Status OutputStatus `json:"status" jsonschema:"'success' when validation ran; 'validation_error' for bad inputs; 'error' for downstream DQ failures."` + Message string `json:"message" jsonschema:"Human-readable summary, including the validation message from the DQ engine."` + Valid bool `json:"valid" jsonschema:"True when the rule is valid; false when the DQ engine rejected it (see message)."` +} + +// NewTool returns the registered tool. +func NewTool(collibraClient *http.Client) *chip.Tool[Input, Output] { + return &chip.Tool[Input, Output]{ + Name: "validate_data_quality_rule", + Title: "Validate Data Quality Rule", + Description: "Validate a data quality rule's (a check on a table's data; Collibra calls it a 'monitor') SQL/definition against the source database before saving or running it, " + + "so a malformed rule is caught up front. Returns whether the rule is valid plus the engine's validation message. " + + "Requires edgeSiteId, connectionId and schemaName — the connection to the source database (edgeSiteId = the Collibra Edge runtime/site that reaches the source, connectionId = the specific database connection), all from prepare_create_data_quality_job.", + Handler: handler(collibraClient), + Permissions: []string{}, + Annotations: &mcp.ToolAnnotations{ReadOnlyHint: true}, + } +} + +func handler(collibraClient *http.Client) chip.ToolHandlerFunc[Input, Output] { + return func(ctx context.Context, input Input) (Output, error) { + if out := validate(input); out != nil { + return *out, nil + } + + resp, err := clients.ValidateDQRule(ctx, collibraClient, clients.PreviewRuleRequest{ + EdgeSiteID: strings.TrimSpace(input.EdgeSiteID), + ConnectionID: strings.TrimSpace(input.ConnectionID), + SchemaName: strings.TrimSpace(input.SchemaName), + JobName: strings.TrimSpace(input.JobName), + PreviewRule: input.PreviewRule, + FilterQuery: input.FilterQuery, + RowLimit: input.RowLimit, + }) + if err != nil { + return Output{Status: StatusError, Message: fmt.Sprintf("Could not validate rule: %v", err)}, nil + } + + msg := "Rule is valid." + if !resp.IsValid { + msg = "Rule is invalid." + } + if strings.TrimSpace(resp.Message) != "" { + msg = fmt.Sprintf("%s %s", msg, resp.Message) + } + return Output{Status: StatusSuccess, Message: msg, Valid: resp.IsValid}, nil + } +} + +func validate(input Input) *Output { + required := []struct { + name string + val string + }{ + {"edgeSiteId", input.EdgeSiteID}, + {"connectionId", input.ConnectionID}, + {"schemaName", input.SchemaName}, + {"jobName", input.JobName}, + {"previewRule", input.PreviewRule}, + } + for _, f := range required { + if strings.TrimSpace(f.val) == "" { + return &Output{Status: StatusValidationError, Message: f.name + " is required."} + } + } + return nil +} diff --git a/pkg/tools/validate_dq_rule/tool_test.go b/pkg/tools/validate_dq_rule/tool_test.go new file mode 100644 index 0000000..9abb4d3 --- /dev/null +++ b/pkg/tools/validate_dq_rule/tool_test.go @@ -0,0 +1,66 @@ +package validate_dq_rule_test + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/collibra/chip/pkg/clients" + "github.com/collibra/chip/pkg/tools/testutil" + "github.com/collibra/chip/pkg/tools/validate_dq_rule" +) + +func server(t *testing.T, code int, resp clients.ValidateDQRuleResponse) *http.Client { + mux := http.NewServeMux() + mux.HandleFunc("POST /rest/dq/internal/v1/rules/validate", func(w http.ResponseWriter, r *http.Request) { + if code != http.StatusOK { + w.WriteHeader(code) + _, _ = w.Write([]byte(`{"message":"boom"}`)) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(resp) + }) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + return testutil.NewClient(srv) +} + +func input() validate_dq_rule.Input { + return validate_dq_rule.Input{ + EdgeSiteID: "site", ConnectionID: "conn", SchemaName: "public", JobName: "DS", PreviewRule: "SELECT 1", + } +} + +func TestValidateDQRule_Valid(t *testing.T) { + c := server(t, http.StatusOK, clients.ValidateDQRuleResponse{IsValid: true, Message: "ok"}) + out, err := validate_dq_rule.NewTool(c).Handler(t.Context(), input()) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if out.Status != validate_dq_rule.StatusSuccess || !out.Valid { + t.Fatalf("expected success+valid, got status=%q valid=%v (%s)", out.Status, out.Valid, out.Message) + } +} + +func TestValidateDQRule_InvalidRuleStillSuccess(t *testing.T) { + c := server(t, http.StatusOK, clients.ValidateDQRuleResponse{IsValid: false, Message: "bad SQLG"}) + out, _ := validate_dq_rule.NewTool(c).Handler(t.Context(), input()) + if out.Status != validate_dq_rule.StatusSuccess { + t.Fatalf("status = %q, want success (validation ran)", out.Status) + } + if out.Valid { + t.Fatalf("expected valid=false") + } +} + +func TestValidateDQRule_MissingInput(t *testing.T) { + c := server(t, http.StatusOK, clients.ValidateDQRuleResponse{}) + in := input() + in.PreviewRule = "" + out, _ := validate_dq_rule.NewTool(c).Handler(t.Context(), in) + if out.Status != validate_dq_rule.StatusValidationError { + t.Fatalf("status = %q, want validation_error", out.Status) + } +}