From 1b7bc2e21dc272ccda087ec8c7c234316c8b0c45 Mon Sep 17 00:00:00 2001 From: Emanuele Tajariol Date: Thu, 8 Oct 2026 15:29:06 +0200 Subject: [PATCH] [Fixes #14705] Metadata validation: call validator from admin page Port of the 5.1.x work (14705_51x_validate_admin): extracts MetadataValidator out of the validate_metadata management command into geonode/metadata/validation/, adds a validate_metadata admin action (reports outcome as admin messages) and a download_validation_report action (full JSON report, uncapped) on DatasetAdmin, DocumentAdmin, MapAdmin and GeoAppAdmin, plus tests for both the validator and the admin actions. Includes the follow-up fix capping schema-error admin messages and pointing at the download action for the rest, instead of risking an unbounded loop against Django's cookie-based message storage. --- geonode/documents/admin.py | 2 + geonode/geoapps/admin.py | 2 + geonode/layers/admin.py | 2 + geonode/maps/admin.py | 2 + geonode/metadata/admin.py | 134 ++++++++ .../management/commands/validate_metadata.py | 306 ++---------------- geonode/metadata/tests/test_admin_actions.py | 200 ++++++++++++ .../metadata/tests/test_metadata_validator.py | 235 ++++++++++++++ .../metadata/tests/test_validate_metadata.py | 2 +- geonode/metadata/validation/__init__.py | 22 ++ geonode/metadata/validation/reporting.py | 67 ++++ geonode/metadata/validation/validator.py | 272 ++++++++++++++++ 12 files changed, 964 insertions(+), 282 deletions(-) create mode 100644 geonode/metadata/tests/test_admin_actions.py create mode 100644 geonode/metadata/tests/test_metadata_validator.py create mode 100644 geonode/metadata/validation/__init__.py create mode 100644 geonode/metadata/validation/reporting.py create mode 100644 geonode/metadata/validation/validator.py diff --git a/geonode/documents/admin.py b/geonode/documents/admin.py index 2f9cbc7b8be..acdec38be70 100644 --- a/geonode/documents/admin.py +++ b/geonode/documents/admin.py @@ -20,6 +20,7 @@ from django.contrib import admin from geonode.documents.models import Document +from geonode.metadata.admin import download_validation_report, validate_metadata from geonode.base.admin import ResourceBaseAdminForm, SparseInline, LinkInline @@ -33,6 +34,7 @@ class Meta(ResourceBaseAdminForm.Meta): class DocumentAdmin(admin.ModelAdmin): + actions = [validate_metadata, download_validation_report] inlines = ( LinkInline, SparseInline, diff --git a/geonode/geoapps/admin.py b/geonode/geoapps/admin.py index 3f11aa3512a..44abc2572f3 100644 --- a/geonode/geoapps/admin.py +++ b/geonode/geoapps/admin.py @@ -21,6 +21,7 @@ from geonode.geoapps.models import GeoApp from geonode.base.admin import ResourceBaseAdminForm +from geonode.metadata.admin import download_validation_report, validate_metadata class GeoAppAdminForm(ResourceBaseAdminForm): @@ -30,6 +31,7 @@ class Meta(ResourceBaseAdminForm.Meta): class GeoAppAdmin(admin.ModelAdmin): + actions = [validate_metadata, download_validation_report] exclude = ("ll_bbox_polygon", "bbox_polygon", "srid", "tkeywords") list_display_links = ("title",) list_display = ( diff --git a/geonode/layers/admin.py b/geonode/layers/admin.py index 5eea5ec88e4..974cdcef771 100644 --- a/geonode/layers/admin.py +++ b/geonode/layers/admin.py @@ -21,6 +21,7 @@ from geonode.base.admin import ResourceBaseAdminForm, SparseInline, LinkInline from geonode.layers.models import Dataset, Attribute, Style +from geonode.metadata.admin import download_validation_report, validate_metadata class AttributeInline(admin.TabularInline): @@ -34,6 +35,7 @@ class Meta(ResourceBaseAdminForm.Meta): class DatasetAdmin(admin.ModelAdmin): + actions = [validate_metadata, download_validation_report] exclude = ("ll_bbox_polygon", "bbox_polygon", "srid", "tkeywords") list_display = ( "id", diff --git a/geonode/maps/admin.py b/geonode/maps/admin.py index bafbacf78be..97a75519d62 100644 --- a/geonode/maps/admin.py +++ b/geonode/maps/admin.py @@ -21,6 +21,7 @@ from django.contrib import admin from geonode.maps.models import Map, MapLayer +from geonode.metadata.admin import download_validation_report, validate_metadata from geonode.base.admin import ResourceBaseAdminForm, SparseInline @@ -38,6 +39,7 @@ class Meta(ResourceBaseAdminForm.Meta): class MapAdmin(admin.ModelAdmin): + actions = [validate_metadata, download_validation_report] inlines = ( MapLayerInline, SparseInline, diff --git a/geonode/metadata/admin.py b/geonode/metadata/admin.py index e69de29bb2d..9a2104f7537 100644 --- a/geonode/metadata/admin.py +++ b/geonode/metadata/admin.py @@ -0,0 +1,134 @@ +######################################################################### +# +# Copyright (C) 2026 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import logging + +from django.contrib import messages +from django.http import HttpResponse +from django.utils.translation.trans_real import get_language_from_request + +from geonode.metadata.validation import MetadataValidator +from geonode.metadata.validation.reporting import write_report + +logger = logging.getLogger(__name__) + +# One admin message per line: report the first few only, and say how many were left out +MAX_REPORTED_RESOURCES = 10 +MAX_REPORTED_ERRORS = 5 + + +def validate_metadata(modeladmin, request, queryset): + """ + Validate the metadata of the selected resources against the metadata jsonschema, + reporting the outcome as admin messages + """ + try: + validator = MetadataValidator(lang=get_language_from_request(request)[:2]) + except ImportError as e: + modeladmin.message_user(request, str(e), messages.ERROR) + return + + # Admin messages go through Django's cookie-based message storage, which has a fixed size + # budget and no session fallback here: an uncapped loop risks losing messages to that limit + # instead of to this cap. Point at the download action for anything past it. + for message in validator.schema_errors[:MAX_REPORTED_RESOURCES]: + modeladmin.message_user(request, message, messages.WARNING) + if len(validator.schema_errors) > MAX_REPORTED_RESOURCES: + omitted = len(validator.schema_errors) - MAX_REPORTED_RESOURCES + modeladmin.message_user( + request, + f"... {omitted} more schema problem(s) not shown here: use 'Download validation report' " + f"for the full list", + messages.WARNING, + ) + + total = 0 + invalid = [] + for resource in queryset: + total += 1 + record = validator.validate_resource(resource, max_errors=MAX_REPORTED_ERRORS) + if not record["valid"]: + invalid.append(record) + + for record in invalid[:MAX_REPORTED_RESOURCES]: + modeladmin.message_user(request, describe(record), messages.ERROR) + + # An optional field the schema refuses to leave empty is not the resource's fault: say so + for field in sorted(validator.observed_defects): + modeladmin.message_user( + request, + f"The metadata schema does not allow the optional field '{field}' to be left empty: " + f"a problem of the schema itself, not of the selected resources", + messages.WARNING, + ) + + modeladmin.message_user(request, summarize(total, invalid), messages.ERROR if invalid else messages.SUCCESS) + + +validate_metadata.short_description = "Validate the metadata" + + +def download_validation_report(modeladmin, request, queryset): + """ + Validate the metadata of the selected resources and return the full report as a downloadable + JSON file: unlike validate_metadata, nothing here is capped for display as admin messages + """ + try: + validator = MetadataValidator(lang=get_language_from_request(request)[:2]) + except ImportError as e: + modeladmin.message_user(request, str(e), messages.ERROR) + return + + records = [validator.validate_resource(resource) for resource in queryset] + invalid = [record for record in records if not record["valid"]] + + summary = list(validator.schema_errors) + for field in sorted(validator.observed_defects): + summary.append( + f"The metadata schema does not allow the optional field '{field}' to be left empty: " + f"a problem of the schema itself, not of the selected resources" + ) + summary.append(summarize(len(records), invalid, capped=False)) + + response = HttpResponse(content_type="application/json") + response["Content-Disposition"] = 'attachment; filename="metadata_validation_report.json"' + write_report(summary, records, response, outformat="json") + return response + + +download_validation_report.short_description = "Download validation report" + + +def describe(record): + """One line per invalid resource: its name, how many errors it has, and the first few of them""" + details = "; ".join(f"{error['title'] or error['path']}: {error['message']}" for error in record["errors"]) + if record["error_count"] > len(record["errors"]): + details += f"; ... {record['error_count'] - len(record['errors'])} more" + + return f"'{record['title']}' (id {record['id']}): {record['error_count']} error(s) -- {details}" + + +def summarize(total, invalid, capped=True): + if not invalid: + return f"Validated {total} resource(s): all valid" + + summary = f"Validated {total} resource(s): {len(invalid)} invalid, {total - len(invalid)} valid" + if capped and len(invalid) > MAX_REPORTED_RESOURCES: + summary += f" (only the first {MAX_REPORTED_RESOURCES} are listed)" + return summary diff --git a/geonode/metadata/management/commands/validate_metadata.py b/geonode/metadata/management/commands/validate_metadata.py index f368698b891..fe8d0a4e10d 100644 --- a/geonode/metadata/management/commands/validate_metadata.py +++ b/geonode/metadata/management/commands/validate_metadata.py @@ -17,35 +17,24 @@ # ######################################################################### -import csv import json import logging import sys from django.core.management.base import BaseCommand, CommandError -from rest_framework.utils.encoders import JSONEncoder from geonode.base.management import command_utils from geonode.base.models import ResourceBase -from geonode.metadata.manager import metadata_manager +from geonode.metadata.validation import MetadataValidator +from geonode.metadata.validation.reporting import write_report logger = logging.getLogger(__name__) -# Max length of the offending instance value reported along with an error -MAX_VALUE_REPR = 200 - -# Max number of problems reported for the schema itself -MAX_SCHEMA_ERRORS = 20 - # Exit codes, combined: a run with both problems exits with 3 EXIT_INVALID_RESOURCES = 1 EXIT_INVALID_SCHEMA = 2 -BASE_FIELDS = ["id", "uuid", "resource_type", "title", "valid", "error_count"] -ERROR_FIELDS = ["error_path", "error_title", "error_validator", "error_message", "error_value"] - - class Command(BaseCommand): help = ( "Validate the metadata of the GeoNode resources: the jsonschema instance of each resource " @@ -57,7 +46,6 @@ class Command(BaseCommand): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.summary = [] # the lines to be repeated in the report - self.schema_errors = 0 def add_arguments(self, parser): parser.add_argument( @@ -149,7 +137,7 @@ def handle(self, *args, **options): if options["instance_files"] and (options["ids"] or options["resource_types"]): raise CommandError("--instance selects the instances by itself: --id and --resource-type do not apply") - validator = self.build_validator(options["lang"], options["check_formats"], options["schema_file"]) + validator = self.build_validator(options) records = [] total = 0 @@ -162,7 +150,7 @@ def handle(self, *args, **options): if not record["valid"] or options["include_valid"]: records.append(record) - self.report_schema_defects() + self.report_schema_defects(validator) self.notify( f"Validated {total} resource(s): {invalid_count} invalid, {total - invalid_count} valid", logging.INFO ) @@ -170,7 +158,7 @@ def handle(self, *args, **options): self.write_report(records, options) exit_code = (EXIT_INVALID_RESOURCES if invalid_count else 0) | ( - EXIT_INVALID_SCHEMA if self.schema_errors else 0 + EXIT_INVALID_SCHEMA if validator.schema_errors else 0 ) if exit_code: sys.exit(exit_code) @@ -178,53 +166,20 @@ def handle(self, *args, **options): # ------------------------------------------------------------------- # Setup # ------------------------------------------------------------------- - def build_validator(self, lang, check_formats, schema_file=None): + def build_validator(self, options): try: - from jsonschema import validators as js_validators - except ImportError: - raise CommandError("The 'jsonschema' package is needed by this command. Please install it") - - # A schema read from a file is plain JSON already, no django objects to render down - schema = self.read_json(schema_file) if schema_file else self.as_json_types(metadata_manager.get_schema(lang)) - validator_class = js_validators.validator_for(schema) - logger.debug(f"Using validator {validator_class.__name__}") - - # A broken schema is worth reporting, but the resources may still be validated against it. - # check_schema() would raise on the first problem only: we want to see them all at once - metaschema_validator = js_validators.validator_for(validator_class.META_SCHEMA, default=validator_class) - for count, error in enumerate(metaschema_validator(validator_class.META_SCHEMA).iter_errors(schema), 1): - self.schema_errors += 1 - if count > MAX_SCHEMA_ERRORS: - self.notify(f"More than {MAX_SCHEMA_ERRORS} problems in the metadata schema, skipping the rest") - break - path = "/" + "/".join(str(item) for item in error.absolute_path) - self.notify(f"The metadata schema itself is not valid: {path}: {error.message}") - - # A schema reported as invalid above may hold anything in place of a keyword: nothing - # derived from it can be taken on trust, starting with the shape of these two - self.schema = schema - required = schema.get("required", []) - self.required_fields = set(required) if isinstance(required, list) else set() - properties = schema.get("properties", {}) - - # Optional fields not accepting the null they get when left empty: a defect of the schema, - # reported once at the end of the run rather than against every resource. - # Probed with an actual null: a "oneOf" or an "enum" may reject it as well as "type" does - self.broken_optionals = { - name - for name, subschema in (properties.items() if isinstance(properties, dict) else ()) - if name not in self.required_fields and self.rejects_null(validator_class, subschema) - } - self.observed_defects = {} + validator = MetadataValidator( + lang=options["lang"], + check_formats=options["check_formats"], + schema=self.read_json(options["schema_file"]) if options["schema_file"] else None, + ) + except ImportError as e: + raise CommandError(str(e)) - format_checker = getattr(validator_class, "FORMAT_CHECKER", None) if check_formats else None - return validator_class(schema, format_checker=format_checker) + for message in validator.schema_errors: + self.notify(message) - def rejects_null(self, validator_class, subschema): - try: - return not validator_class(subschema).is_valid(None) - except Exception: # a malformed subschema: the metaschema check has reported it already - return False + return validator def read_json(self, path): try: @@ -233,13 +188,6 @@ def read_json(self, path): except (OSError, ValueError) as e: raise CommandError(f"Can not read the JSON file '{path}': {e}") - def as_json_types(self, data): - """ - Render the data down to plain JSON types, as the API does before sending it to the client: - a lazy translation is not a str, and jsonschema would reject it as "not of type 'string'" - """ - return json.loads(json.dumps(data, cls=JSONEncoder)) - def select_resources(self, ids, resource_types): queryset = ResourceBase.objects.all() @@ -287,7 +235,7 @@ def validate_all(self, validator, options): # Non polymorphic, exactly like the API read path: we validate what the client is served. # iterator() keeps the resources out of memory; the records being reported are not for resource in queryset.iterator(): - yield self.validate_resource(resource, validator, options["lang"], options["max_errors"]) + yield validator.validate_resource(resource, options["max_errors"]) def validate_file(self, path, validator, max_errors): instance = self.read_json(path) @@ -297,239 +245,35 @@ def validate_file(self, path, validator, max_errors): "resource_type": None, # not part of the schema "title": instance.get("title"), } - return self.validate_instance(record, instance, validator, max_errors) - - def validate_resource(self, resource, validator, lang, max_errors): - record = { - "id": resource.pk, - "uuid": resource.uuid, - "resource_type": resource.resource_type, - "title": resource.title, - } - logger.debug(f"Validating resource {resource.pk}: '{resource.title}'") - - try: - instance = metadata_manager.build_schema_instance(resource, lang) - instance = self.as_json_types(instance) - except Exception as e: - logger.error(f"Can not build the schema instance for resource {resource.pk}", exc_info=e) - return dict( - record, - valid=False, - error_count=1, - errors=[ - { - "path": "/", - "title": "", - "validator": "geonode:build_schema_instance", - "message": f"Can not build the schema instance: {e}", - "value": None, - } - ], - ) - - return self.validate_instance(record, instance, validator, max_errors) - - def validate_instance(self, record, instance, validator, max_errors): - # GeoNode injects this key in the instance for testing purposes only (resources titled "*error*") - instance.pop("extraErrors", None) - - try: - errors = self.drop_redundant(validator.iter_errors(instance)) - except Exception as e: - # A keyword the metaschema has already refused can blow up on a value of the right type - logger.error(f"Can not validate {record['id']} against the schema", exc_info=e) - return dict( - record, - valid=False, - error_count=1, - errors=[ - { - "path": "/", - "title": "", - "validator": "geonode:invalid_schema", - "message": f"Can not validate against the schema: {e}", - "value": None, - } - ], - ) - - errors = [error for error in errors if not self.is_schema_defect(error)] - errors = sorted(errors, key=lambda err: ([str(p) for p in err.absolute_path], str(err.validator))) - reported = errors[:max_errors] if max_errors > 0 else errors - - return dict( - record, - valid=not errors, - error_count=len(errors), - errors=[self.format_error(err) for err in reported], - ) - - def is_unset(self, error): - # Keyed on the value: depending on the subschema a null may trip "type", "oneOf", or both - return error.instance is None - - def is_schema_defect(self, error): - # An optional field left empty: blame the schema, not the resource - if not (self.is_unset(error) and len(error.absolute_path) == 1): - return False - - field = error.absolute_path[0] - if field not in self.broken_optionals: - return False - - self.observed_defects[field] = self.observed_defects.get(field, 0) + 1 - return True + return validator.validate_instance(record, instance, max_errors) + # ------------------------------------------------------------------- + # Reporting + # ------------------------------------------------------------------- def notify(self, message, level=logging.WARNING): # Logged, and repeated in the report: the report file is meant to travel on its own logger.log(level, message) self.summary.append(message) - def report_schema_defects(self): + def report_schema_defects(self, validator): # Only what some resource really ran into: a null never handed over is nobody's problem - for field, count in sorted(self.observed_defects.items()): + for field, count in sorted(validator.observed_defects.items()): self.notify( f"The metadata schema does not allow a null value on the optional field " f"'{field}', left empty by {count} resource(s)" ) - def drop_redundant(self, errors): - """ - A null fails every keyword declared for the field, all reporting the same problem: keep - one error per unset field, preferring "type" since it names the expected types - """ - errors = list(errors) - unset_paths = {tuple(error.absolute_path) for error in errors if self.is_unset(error)} - - kept = [error for error in errors if tuple(error.absolute_path) not in unset_paths] - - for path in unset_paths: - at_path = [error for error in errors if tuple(error.absolute_path) == path] - kept.append(next((e for e in at_path if e.validator == "type"), at_path[0])) - - return kept - - def path_title(self, path): - # The property names are ids: walk the schema alongside the path to pick up their titles - titles = [] - subschema = self.schema - for item in path: - # A malformed schema may hold anything in place of a subschema: fall back to the names - if isinstance(item, int): - subschema = self.as_dict(subschema).get("items") - titles.append(f"[{item}]") - else: - subschema = self.as_dict(self.as_dict(subschema).get("properties")).get(item) - titles.append(self.as_dict(subschema).get("title", item)) - return "/".join(titles) - - @staticmethod - def as_dict(value): - return value if isinstance(value, dict) else {} - - def missing_property(self, error): - # jsonschema names the missing property in the message only, and does not add it to the - # path: match the message back to the required list, so a reworded one just yields nothing - if error.validator != "required": - return None - return next((p for p in error.validator_value if f"{p!r} is a required property" == error.message), None) - - def format_error(self, error): - path = tuple(error.absolute_path) - message = error.message - - # A missing property is reported against the object holding it: point at the property - missing = self.missing_property(error) - if missing is not None: - path += (missing,) - - # The instance behind a container error is the container itself: useless noise - value = None - if path and missing is None: - value = json.dumps(error.instance, ensure_ascii=False, default=str) - if len(value) > MAX_VALUE_REPR: - value = f"{value[:MAX_VALUE_REPR]}…" - - # A top level field only gets reported when mandatory, no point in repeating it. Deeper - # down nothing is filtered, so there it is worth telling a mandatory field from the rest - if missing is not None: - message = "not set" if len(path) == 1 else "not set (required field)" - - # jsonschema puts it in python terms ("None is not of type 'object'"), reading as a type - # mismatch where the field is simply empty. Spell it out, and drop the pointless null - elif self.is_unset(error): - if len(path) > 1 and error.validator == "type": - expected = error.validator_value - expected = expected if isinstance(expected, list) else [expected] - message = f"not set (expected {' or '.join(expected)})" - else: - message = "not set" - value = None - - # jsonschema quotes the whole offending string in the message ("'xxxxx…' is too long"), - # which is exactly the value we are careful to truncate in the value column - elif error.validator == "maxLength": - message = f"too long: {len(error.instance)} chars (max {error.validator_value})" - - return { - "path": "/" + "/".join(str(item) for item in path), - "title": self.path_title(path), - "validator": str(error.validator), - "message": message, - "value": value, - } - - # ------------------------------------------------------------------- - # Reporting - # ------------------------------------------------------------------- def write_report(self, records, options): outpath = options["output"] if outpath: try: with open(outpath, "w", newline="", encoding="utf-8") as out: - self.dump(records, out, options) + write_report(self.summary, records, out, options["outformat"], options["error_details"]) except OSError as e: raise CommandError(f"Can not write the report to '{outpath}': {e}") logger.info(f"Report written to {outpath}") else: # OutputWrapper appends a newline to every write() call, which would break the report self.stdout.ending = "" - self.dump(records, self.stdout, options) - - def dump(self, records, out, options): - if not options["error_details"]: - records = [{k: v for k, v in record.items() if k != "errors"} for record in records] - - if options["outformat"] == "json": - json.dump({"summary": self.summary, "resources": records}, out, indent=2, ensure_ascii=False, default=str) - out.write("\n") - else: - # CSV has no comment syntax: "#" is only a widespread convention - for message in self.summary: - out.write(f"# {message}\n") - self.dump_csv(records, out, options["error_details"]) - - def dump_csv(self, records, out, error_details): - fields = BASE_FIELDS + (ERROR_FIELDS if error_details else []) - writer = csv.DictWriter(out, fieldnames=fields, extrasaction="ignore") - writer.writeheader() - - for record in records: - if not error_details or not record["errors"]: - writer.writerow(record) - continue - - # one row per error, the resource columns are repeated - for error in record["errors"]: - writer.writerow( - dict( - record, - error_path=error["path"], - error_title=error["title"], - error_validator=error["validator"], - error_message=error["message"], - error_value=error["value"], - ) - ) + write_report(self.summary, records, self.stdout, options["outformat"], options["error_details"]) diff --git a/geonode/metadata/tests/test_admin_actions.py b/geonode/metadata/tests/test_admin_actions.py new file mode 100644 index 00000000000..2d8c0175018 --- /dev/null +++ b/geonode/metadata/tests/test_admin_actions.py @@ -0,0 +1,200 @@ +######################################################################### +# +# Copyright (C) 2026 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +""" +The validate_metadata / download_validation_report admin actions, exercised directly: both only +need a modeladmin with message_user() and a request/queryset, no admin site or HTTP client involved. +""" +import json +import sys +from unittest.mock import MagicMock, patch +from uuid import uuid4 + +from django.contrib import messages +from django.contrib.auth import get_user_model +from django.test import RequestFactory, TestCase + +from geonode.base.models import ResourceBase +from geonode.metadata.admin import MAX_REPORTED_RESOURCES, download_validation_report, validate_metadata + +# "edition" is optional and rejects the null it gets when left empty: a schema defect, not a +# resource error. "title" has a tight maxLength so a long one is an easy resource error to trigger. +SCHEMA = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": ["title"], + "properties": { + "title": {"type": "string", "title": "Title", "maxLength": 20}, + "edition": {"type": "string", "title": "Edition"}, + }, +} + + +class MetadataManagerMixin: + """Resources plus a mocked metadata_manager feeding MetadataValidator a known schema/instances""" + + def setUp(self): + self.owner = get_user_model().objects.create_user("admin_action_user", "a@fakemail.com", "pwd") + self.valid = self.make_resource("Valid one") + self.invalid = self.make_resource("Invalid one") + self.defect = self.make_resource("Defect one") + + self.instances = { + self.valid.uuid: {"title": "fine"}, + self.invalid.uuid: {"title": "x" * 25}, # longer than the schema's maxLength: 20 + self.defect.uuid: {"title": "fine", "edition": None}, + } + + patcher = patch("geonode.metadata.validation.validator.metadata_manager") + self.manager = patcher.start() + self.addCleanup(patcher.stop) + self.manager.get_schema.return_value = SCHEMA + self.manager.build_schema_instance.side_effect = lambda resource, lang: dict(self.instances[resource.uuid]) + + self.modeladmin = MagicMock() + self.request = RequestFactory().get("/admin/") + + def make_resource(self, title): + return ResourceBase.objects.create(title=title, uuid=str(uuid4()), resource_type="dataset", owner=self.owner) + + def queryset(self, *resources): + return ResourceBase.objects.filter(pk__in=[r.pk for r in resources]) + + +class ValidateMetadataActionTests(MetadataManagerMixin, TestCase): + def messages(self): + return [(call.args[1], call.args[2]) for call in self.modeladmin.message_user.call_args_list] + + def test_all_valid_reports_success(self): + validate_metadata(self.modeladmin, self.request, self.queryset(self.valid)) + + text, level = self.messages()[-1] + self.assertEqual("Validated 1 resource(s): all valid", text) + self.assertEqual(messages.SUCCESS, level) + + def test_invalid_resource_is_described_and_summarized_as_an_error(self): + validate_metadata(self.modeladmin, self.request, self.queryset(self.valid, self.invalid)) + + calls = self.messages() + texts = [t for t, _ in calls] + described = next(t for t in texts if t.startswith("'Invalid one'")) + self.assertIn("too long", described) + self.assertEqual("Validated 2 resource(s): 1 invalid, 1 valid", calls[-1][0]) + self.assertEqual(messages.ERROR, calls[-1][1]) + + def test_schema_defect_is_blamed_on_the_schema_not_the_resource(self): + validate_metadata(self.modeladmin, self.request, self.queryset(self.defect)) + + calls = self.messages() + texts = [t for t, _ in calls] + self.assertTrue(any("a problem of the schema itself, not of the selected resources" in t for t in texts)) + # the resource itself is not to blame: it is reported as valid + self.assertEqual("Validated 1 resource(s): all valid", calls[-1][0]) + + def broken_schema(self, property_count): + return { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": {f"f{i}": {"maxLength": "nope"} for i in range(property_count)}, + } + + def test_schema_errors_within_the_cap_are_all_shown(self): + self.manager.get_schema.return_value = self.broken_schema(3) + + validate_metadata(self.modeladmin, self.request, self.queryset(self.valid)) + + warnings = [text for text, level in self.messages() if level == messages.WARNING] + self.assertEqual(3, len(warnings)) + self.assertFalse(any("more schema problem" in w for w in warnings)) + + def test_schema_errors_past_the_cap_point_at_the_download_action(self): + self.manager.get_schema.return_value = self.broken_schema(MAX_REPORTED_RESOURCES + 3) + + validate_metadata(self.modeladmin, self.request, self.queryset(self.valid)) + + warnings = [text for text, level in self.messages() if level == messages.WARNING] + self.assertEqual(MAX_REPORTED_RESOURCES + 1, len(warnings)) # capped, plus one omission notice + self.assertIn("3 more schema problem", warnings[-1]) + self.assertIn("Download validation report", warnings[-1]) + + def test_reported_resources_are_capped(self): + extra = [self.make_resource(f"Bad {i}") for i in range(MAX_REPORTED_RESOURCES)] + for resource in extra: + self.instances[resource.uuid] = {"title": "x" * 25} + + validate_metadata(self.modeladmin, self.request, self.queryset(self.invalid, *extra)) + + texts = [t for t, _ in self.messages()] + described = [t for t in texts if t.startswith("'Bad") or t.startswith("'Invalid one'")] + self.assertEqual(MAX_REPORTED_RESOURCES, len(described)) + self.assertIn("only the first", texts[-1]) + + def test_missing_jsonschema_package_is_reported_and_stops_processing(self): + with patch.dict(sys.modules, {"jsonschema": None}): + validate_metadata(self.modeladmin, self.request, self.queryset(self.valid)) + + calls = self.messages() + self.assertEqual(1, len(calls)) + self.assertIn("jsonschema", calls[0][0]) + self.assertEqual(messages.ERROR, calls[0][1]) + self.manager.build_schema_instance.assert_not_called() + + +class DownloadValidationReportActionTests(MetadataManagerMixin, TestCase): + def test_response_is_a_json_attachment(self): + response = download_validation_report(self.modeladmin, self.request, self.queryset(self.valid)) + + self.assertEqual("application/json", response["Content-Type"]) + self.assertEqual('attachment; filename="metadata_validation_report.json"', response["Content-Disposition"]) + + def test_report_holds_every_resource_with_full_error_details(self): + response = download_validation_report(self.modeladmin, self.request, self.queryset(self.valid, self.invalid)) + + report = json.loads(response.content) + by_id = {record["id"]: record for record in report["resources"]} + self.assertEqual({self.valid.pk, self.invalid.pk}, set(by_id)) + self.assertTrue(by_id[self.valid.pk]["valid"]) + self.assertFalse(by_id[self.invalid.pk]["valid"]) + self.assertIn("too long", by_id[self.invalid.pk]["errors"][0]["message"]) + + def test_nothing_is_capped_past_the_admin_message_limit(self): + extra = [self.make_resource(f"Bad {i}") for i in range(MAX_REPORTED_RESOURCES + 3)] + for resource in extra: + self.instances[resource.uuid] = {"title": "x" * 25} + + response = download_validation_report(self.modeladmin, self.request, self.queryset(self.invalid, *extra)) + + report = json.loads(response.content) + self.assertEqual(MAX_REPORTED_RESOURCES + 4, len(report["resources"])) + self.assertNotIn("only the first", report["summary"][-1]) + + def test_schema_defect_is_worded_the_same_way_as_the_message_action(self): + response = download_validation_report(self.modeladmin, self.request, self.queryset(self.defect)) + + report = json.loads(response.content) + self.assertTrue( + any("a problem of the schema itself, not of the selected resources" in m for m in report["summary"]) + ) + + def test_missing_jsonschema_package_messages_the_user_and_returns_none(self): + with patch.dict(sys.modules, {"jsonschema": None}): + result = download_validation_report(self.modeladmin, self.request, self.queryset(self.valid)) + + self.assertIsNone(result) + self.modeladmin.message_user.assert_called_once() + self.assertEqual(messages.ERROR, self.modeladmin.message_user.call_args.args[2]) diff --git a/geonode/metadata/tests/test_metadata_validator.py b/geonode/metadata/tests/test_metadata_validator.py new file mode 100644 index 00000000000..984befa1b5b --- /dev/null +++ b/geonode/metadata/tests/test_metadata_validator.py @@ -0,0 +1,235 @@ +######################################################################### +# +# Copyright (C) 2026 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### +""" +MetadataValidator exercised as a direct API: no management command, no CLI parsing. +geonode.metadata.tests.test_validate_metadata already covers the reporting contract +(valid/invalid, max_errors, value truncation, ...) through the command; this file is +about what is only reachable by calling the class itself. +""" +import sys +from unittest.mock import patch +from uuid import uuid4 + +from django.contrib.auth import get_user_model +from django.test import SimpleTestCase, TestCase +from django.utils.translation import gettext_lazy + +from geonode.base.models import ResourceBase +from geonode.metadata.validation import MAX_SCHEMA_ERRORS, MetadataValidator + +# Optional "edition" rejects the null it gets when left empty: a defect of the schema. +# Optional "abstract" accepts it: not a defect. +BASE_SCHEMA = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": ["title"], + "properties": { + "title": {"type": "string", "title": "Title"}, + "abstract": {"type": ["string", "null"], "title": "Abstract"}, + "edition": {"type": "string", "title": "Edition"}, + }, +} + + +class MetadataValidatorConstructionTests(SimpleTestCase): + def test_required_and_broken_optionals_are_computed_from_the_schema(self): + validator = MetadataValidator(schema=BASE_SCHEMA) + + self.assertEqual({"title"}, validator.required_fields) + self.assertEqual({"edition"}, validator.broken_optionals) + + def test_valid_schema_has_no_schema_errors(self): + validator = MetadataValidator(schema=BASE_SCHEMA) + + self.assertEqual([], validator.schema_errors) + + def test_broken_schema_is_reported_in_schema_errors(self): + broken = dict(BASE_SCHEMA, properties=dict(BASE_SCHEMA["properties"], edition={"maxLength": "nope"})) + + validator = MetadataValidator(schema=broken) + + self.assertEqual(1, len(validator.schema_errors)) + self.assertIn("/properties/edition/maxLength", validator.schema_errors[0]) + + def test_schema_errors_are_capped(self): + # one broken keyword per property, well past the cap + properties = {f"f{i}": {"maxLength": "nope"} for i in range(MAX_SCHEMA_ERRORS + 5)} + broken = dict(BASE_SCHEMA, properties=properties) + + validator = MetadataValidator(schema=broken) + + self.assertEqual(MAX_SCHEMA_ERRORS + 1, len(validator.schema_errors)) + self.assertIn("skipping the rest", validator.schema_errors[-1]) + + def test_missing_jsonschema_package_raises_import_error(self): + with patch.dict(sys.modules, {"jsonschema": None}): + with self.assertRaises(ImportError): + MetadataValidator(schema=BASE_SCHEMA) + + +class MetadataValidatorFormatCheckingTests(SimpleTestCase): + SCHEMA = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": {"reportDate": {"type": "string", "title": "Report date", "format": "date"}}, + } + + def test_bad_format_is_ignored_by_default(self): + validator = MetadataValidator(schema=self.SCHEMA) + + record = validator.validate_instance({"id": 1}, {"reportDate": "not-a-date"}) + + self.assertTrue(record["valid"]) + + def test_bad_format_is_reported_when_check_formats_is_enabled(self): + validator = MetadataValidator(schema=self.SCHEMA, check_formats=True) + + record = validator.validate_instance({"id": 1}, {"reportDate": "not-a-date"}) + + self.assertFalse(record["valid"]) + self.assertEqual("format", record["errors"][0]["validator"]) + self.assertEqual("/reportDate", record["errors"][0]["path"]) + + +class MetadataValidatorInstanceTests(SimpleTestCase): + def test_empty_optional_field_is_a_schema_defect_counted_and_excluded(self): + validator = MetadataValidator(schema=BASE_SCHEMA) + + first = validator.validate_instance({"id": 1}, {"title": "a", "edition": None}) + second = validator.validate_instance({"id": 2}, {"title": "b", "edition": None}) + + self.assertTrue(first["valid"]) + self.assertTrue(second["valid"]) + self.assertEqual({"edition": 2}, validator.observed_defects) + + def test_null_failing_several_keywords_is_reported_once_preferring_type(self): + # "code" is nested, so it is not eligible for the top-level-only schema-defect shortcut: + # the null genuinely trips both "type" and "enum", and drop_redundant must dedupe them + schema = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nested": { + "type": "object", + "title": "Nested", + "properties": {"code": {"type": "string", "enum": ["a", "b"], "title": "Code"}}, + } + }, + } + validator = MetadataValidator(schema=schema) + + record = validator.validate_instance({"id": 1}, {"nested": {"code": None}}) + + self.assertEqual(1, len(record["errors"])) + self.assertEqual("type", record["errors"][0]["validator"]) + + def test_missing_required_field_below_top_level_is_worded_as_required(self): + schema = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "nested": { + "type": "object", + "title": "Nested", + "required": ["code"], + "properties": {"code": {"type": "string", "title": "Code"}}, + } + }, + } + validator = MetadataValidator(schema=schema) + + record = validator.validate_instance({"id": 1}, {"nested": {}}) + + error = record["errors"][0] + self.assertEqual("/nested/code", error["path"]) + self.assertEqual("Nested/Code", error["title"]) + self.assertEqual("not set (required field)", error["message"]) + + def test_array_item_error_path_and_title_carry_the_index(self): + schema = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "contacts": { + "type": "array", + "title": "Contacts", + "items": { + "type": "object", + "title": "Contact", + "properties": {"email": {"type": "string", "title": "Email"}}, + }, + } + }, + } + validator = MetadataValidator(schema=schema) + + record = validator.validate_instance({"id": 1}, {"contacts": [{"email": 123}]}) + + error = record["errors"][0] + self.assertEqual("/contacts/0/email", error["path"]) + self.assertEqual("Contacts/[0]/Email", error["title"]) + + def test_extra_errors_testing_key_is_dropped_before_validation(self): + validator = MetadataValidator(schema=BASE_SCHEMA) + instance = {"title": "a", "extraErrors": "boom"} + + validator.validate_instance({"id": 1}, instance) + + self.assertNotIn("extraErrors", instance) + + def test_as_json_types_renders_lazy_strings_down_to_plain_str(self): + rendered = MetadataValidator.as_json_types({"title": gettext_lazy("hello")}) + + self.assertEqual({"title": "hello"}, rendered) + self.assertIs(str, type(rendered["title"])) + + +class MetadataValidatorResourceTests(TestCase): + """validate_resource(), the entry point that goes through the metadata manager""" + + def setUp(self): + owner = get_user_model().objects.create_user("validator_user", "v@fakemail.com", "pwd") + self.resource = ResourceBase.objects.create( + title="A dataset", uuid=str(uuid4()), resource_type="dataset", owner=owner + ) + self.validator = MetadataValidator(schema=BASE_SCHEMA) + + patcher = patch("geonode.metadata.validation.validator.metadata_manager") + self.manager = patcher.start() + self.addCleanup(patcher.stop) + + def test_instance_is_built_through_the_metadata_manager_and_then_validated(self): + self.manager.build_schema_instance.return_value = {"title": "fine"} + + record = self.validator.validate_resource(self.resource) + + self.manager.build_schema_instance.assert_called_once_with(self.resource, self.validator.lang) + self.assertEqual(self.resource.pk, record["id"]) + self.assertEqual(self.resource.uuid, record["uuid"]) + self.assertEqual("dataset", record["resource_type"]) + self.assertTrue(record["valid"]) + + def test_a_failing_handler_is_reported_against_the_resource_not_raised(self): + self.manager.build_schema_instance.side_effect = RuntimeError("handler exploded") + + record = self.validator.validate_resource(self.resource) + + self.assertFalse(record["valid"]) + self.assertEqual("geonode:build_schema_instance", record["errors"][0]["validator"]) + self.assertIn("handler exploded", record["errors"][0]["message"]) diff --git a/geonode/metadata/tests/test_validate_metadata.py b/geonode/metadata/tests/test_validate_metadata.py index 615a4c2ba31..cf3f5cc8bb6 100644 --- a/geonode/metadata/tests/test_validate_metadata.py +++ b/geonode/metadata/tests/test_validate_metadata.py @@ -240,7 +240,7 @@ def setUp(self): title="A document", uuid=str(uuid4()), resource_type="document", owner=owner ) - patcher = patch("geonode.metadata.management.commands.validate_metadata.metadata_manager") + patcher = patch("geonode.metadata.validation.validator.metadata_manager") self.manager = patcher.start() self.addCleanup(patcher.stop) # every resource yields the same valid instance: what is under test here is the selection diff --git a/geonode/metadata/validation/__init__.py b/geonode/metadata/validation/__init__.py new file mode 100644 index 00000000000..bf1681c63ac --- /dev/null +++ b/geonode/metadata/validation/__init__.py @@ -0,0 +1,22 @@ +######################################################################### +# +# Copyright (C) 2026 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from geonode.metadata.validation.validator import MAX_SCHEMA_ERRORS, MetadataValidator + +__all__ = ["MAX_SCHEMA_ERRORS", "MetadataValidator"] diff --git a/geonode/metadata/validation/reporting.py b/geonode/metadata/validation/reporting.py new file mode 100644 index 00000000000..df03d9f4296 --- /dev/null +++ b/geonode/metadata/validation/reporting.py @@ -0,0 +1,67 @@ +######################################################################### +# +# Copyright (C) 2026 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import csv +import json + +BASE_FIELDS = ["id", "uuid", "resource_type", "title", "valid", "error_count"] +ERROR_FIELDS = ["error_path", "error_title", "error_validator", "error_message", "error_value"] + + +def write_report(summary, records, out, outformat="json", error_details=True): + """ + Write a MetadataValidator report (the summary lines plus the per-resource records) to `out`, + in the given format. `out` only needs a `write(str)` method: an open file, Django's stdout + wrapper, an HttpResponse, ... + """ + if not error_details: + records = [{k: v for k, v in record.items() if k != "errors"} for record in records] + + if outformat == "json": + json.dump({"summary": summary, "resources": records}, out, indent=2, ensure_ascii=False, default=str) + out.write("\n") + else: + # CSV has no comment syntax: "#" is only a widespread convention + for message in summary: + out.write(f"# {message}\n") + write_csv(records, out, error_details) + + +def write_csv(records, out, error_details): + fields = BASE_FIELDS + (ERROR_FIELDS if error_details else []) + writer = csv.DictWriter(out, fieldnames=fields, extrasaction="ignore") + writer.writeheader() + + for record in records: + if not error_details or not record["errors"]: + writer.writerow(record) + continue + + # one row per error, the resource columns are repeated + for error in record["errors"]: + writer.writerow( + dict( + record, + error_path=error["path"], + error_title=error["title"], + error_validator=error["validator"], + error_message=error["message"], + error_value=error["value"], + ) + ) diff --git a/geonode/metadata/validation/validator.py b/geonode/metadata/validation/validator.py new file mode 100644 index 00000000000..38094b8cc43 --- /dev/null +++ b/geonode/metadata/validation/validator.py @@ -0,0 +1,272 @@ +######################################################################### +# +# Copyright (C) 2026 OSGeo +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import json +import logging + +from rest_framework.utils.encoders import JSONEncoder + +from geonode.metadata.manager import metadata_manager + +logger = logging.getLogger(__name__) + +# Max length of the offending instance value reported along with an error +MAX_VALUE_REPR = 200 + +# Max number of problems reported for the schema itself +MAX_SCHEMA_ERRORS = 20 + + +class MetadataValidator: + """ + Validate the metadata of the resources against the metadata jsonschema. + + A record is returned per resource, holding the resource identification, whether it is valid, + how many errors it has and the details of each of them. + + Two kinds of problem are not the resource's fault and are kept apart from its errors: + `schema_errors` lists what is wrong in the schema itself, and `observed_defects` counts, per + field, the optional fields the schema refuses to leave empty. Both are returned as data, for + the caller to word as it sees fit. + """ + + def __init__(self, lang=None, check_formats=False, schema=None): + try: + from jsonschema import validators as js_validators + except ImportError: + raise ImportError("The 'jsonschema' package is needed to validate the metadata. Please install it") + + # A schema handed over by the caller is plain JSON already: nothing to render down + self.lang = lang + self.schema = schema if schema is not None else self.as_json_types(metadata_manager.get_schema(lang)) + + validator_class = js_validators.validator_for(self.schema) + logger.debug(f"Using validator {validator_class.__name__}") + + self.schema_errors = self._check_schema(js_validators, validator_class) + + # A schema reported as invalid above may hold anything in place of a keyword: nothing + # derived from it can be taken on trust, starting with the shape of these two + required = self.schema.get("required", []) + self.required_fields = set(required) if isinstance(required, list) else set() + properties = self.schema.get("properties", {}) + + # Optional fields not accepting the null they get when left empty: a defect of the schema, + # to be kept out of the resource errors and counted here. + # Probed with an actual null: a "oneOf" or an "enum" may reject it as well as "type" does + self.broken_optionals = { + name + for name, subschema in (properties.items() if isinstance(properties, dict) else ()) + if name not in self.required_fields and self._rejects_null(validator_class, subschema) + } + self.observed_defects = {} + + format_checker = getattr(validator_class, "FORMAT_CHECKER", None) if check_formats else None + self.validator = validator_class(self.schema, format_checker=format_checker) + + # ------------------------------------------------------------------- + # Setup + # ------------------------------------------------------------------- + def _check_schema(self, js_validators, validator_class): + """ + Return the problems of the schema itself. A broken schema is worth reporting, but the + resources may still be validated against it: check_schema() would raise on the first + problem only, and we want to see them all at once + """ + metaschema_validator = js_validators.validator_for(validator_class.META_SCHEMA, default=validator_class) + + problems = [] + for count, error in enumerate(metaschema_validator(validator_class.META_SCHEMA).iter_errors(self.schema), 1): + if count > MAX_SCHEMA_ERRORS: + problems.append(f"More than {MAX_SCHEMA_ERRORS} problems in the metadata schema, skipping the rest") + break + path = "/" + "/".join(str(item) for item in error.absolute_path) + problems.append(f"The metadata schema itself is not valid: {path}: {error.message}") + return problems + + def _rejects_null(self, validator_class, subschema): + try: + return not validator_class(subschema).is_valid(None) + except Exception: # a malformed subschema: the metaschema check has reported it already + return False + + @staticmethod + def as_json_types(data): + """ + Render the data down to plain JSON types, as the API does before sending it to the client: + a lazy translation is not a str, and jsonschema would reject it as "not of type 'string'" + """ + return json.loads(json.dumps(data, cls=JSONEncoder)) + + # ------------------------------------------------------------------- + # Validation + # ------------------------------------------------------------------- + def validate_resource(self, resource, max_errors=0): + record = { + "id": resource.pk, + "uuid": resource.uuid, + "resource_type": resource.resource_type, + "title": resource.title, + } + logger.debug(f"Validating resource {resource.pk}: '{resource.title}'") + + try: + instance = metadata_manager.build_schema_instance(resource, self.lang) + instance = self.as_json_types(instance) + except Exception as e: + logger.error(f"Can not build the schema instance for resource {resource.pk}", exc_info=e) + return self._failed(record, "geonode:build_schema_instance", f"Can not build the schema instance: {e}") + + return self.validate_instance(record, instance, max_errors) + + def validate_instance(self, record, instance, max_errors=0): + # GeoNode injects this key in the instance for testing purposes only (resources titled "*error*") + instance.pop("extraErrors", None) + + try: + errors = self.drop_redundant(self.validator.iter_errors(instance)) + except Exception as e: + # A keyword the metaschema has already refused can blow up on a value of the right type + logger.error(f"Can not validate {record['id']} against the schema", exc_info=e) + return self._failed(record, "geonode:invalid_schema", f"Can not validate against the schema: {e}") + + errors = [error for error in errors if not self.is_schema_defect(error)] + errors = sorted(errors, key=lambda err: ([str(p) for p in err.absolute_path], str(err.validator))) + reported = errors[:max_errors] if max_errors > 0 else errors + + return dict( + record, + valid=not errors, + error_count=len(errors), + errors=[self.format_error(err) for err in reported], + ) + + @staticmethod + def _failed(record, validator, message): + return dict( + record, + valid=False, + error_count=1, + errors=[{"path": "/", "title": "", "validator": validator, "message": message, "value": None}], + ) + + def is_unset(self, error): + # Keyed on the value: depending on the subschema a null may trip "type", "oneOf", or both + return error.instance is None + + def is_schema_defect(self, error): + # An optional field left empty: blame the schema, not the resource + if not (self.is_unset(error) and len(error.absolute_path) == 1): + return False + + field = error.absolute_path[0] + if field not in self.broken_optionals: + return False + + self.observed_defects[field] = self.observed_defects.get(field, 0) + 1 + return True + + def drop_redundant(self, errors): + """ + A null fails every keyword declared for the field, all reporting the same problem: keep + one error per unset field, preferring "type" since it names the expected types + """ + errors = list(errors) + unset_paths = {tuple(error.absolute_path) for error in errors if self.is_unset(error)} + + kept = [error for error in errors if tuple(error.absolute_path) not in unset_paths] + + for path in unset_paths: + at_path = [error for error in errors if tuple(error.absolute_path) == path] + kept.append(next((e for e in at_path if e.validator == "type"), at_path[0])) + + return kept + + # ------------------------------------------------------------------- + # Error rendering + # ------------------------------------------------------------------- + def path_title(self, path): + # The property names are ids: walk the schema alongside the path to pick up their titles + titles = [] + subschema = self.schema + for item in path: + # A malformed schema may hold anything in place of a subschema: fall back to the names + if isinstance(item, int): + subschema = self.as_dict(subschema).get("items") + titles.append(f"[{item}]") + else: + subschema = self.as_dict(self.as_dict(subschema).get("properties")).get(item) + titles.append(self.as_dict(subschema).get("title", item)) + return "/".join(titles) + + @staticmethod + def as_dict(value): + return value if isinstance(value, dict) else {} + + def missing_property(self, error): + # jsonschema names the missing property in the message only, and does not add it to the + # path: match the message back to the required list, so a reworded one just yields nothing + if error.validator != "required": + return None + return next((p for p in error.validator_value if f"{p!r} is a required property" == error.message), None) + + def format_error(self, error): + path = tuple(error.absolute_path) + message = error.message + + # A missing property is reported against the object holding it: point at the property + missing = self.missing_property(error) + if missing is not None: + path += (missing,) + + # The instance behind a container error is the container itself: useless noise + value = None + if path and missing is None: + value = json.dumps(error.instance, ensure_ascii=False, default=str) + if len(value) > MAX_VALUE_REPR: + value = f"{value[:MAX_VALUE_REPR]}…" + + # A top level field only gets reported when mandatory, no point in repeating it. Deeper + # down nothing is filtered, so there it is worth telling a mandatory field from the rest + if missing is not None: + message = "not set" if len(path) == 1 else "not set (required field)" + + # jsonschema puts it in python terms ("None is not of type 'object'"), reading as a type + # mismatch where the field is simply empty. Spell it out, and drop the pointless null + elif self.is_unset(error): + if len(path) > 1 and error.validator == "type": + expected = error.validator_value + expected = expected if isinstance(expected, list) else [expected] + message = f"not set (expected {' or '.join(expected)})" + else: + message = "not set" + value = None + + # jsonschema quotes the whole offending string in the message ("'xxxxx…' is too long"), + # which is exactly the value we are careful to truncate in the value column + elif error.validator == "maxLength": + message = f"too long: {len(error.instance)} chars (max {error.validator_value})" + + return { + "path": "/" + "/".join(str(item) for item in path), + "title": self.path_title(path), + "validator": str(error.validator), + "message": message, + "value": value, + }