From ae6275aeca82d2bfd6a8e0275a902b3f2b44c8ab Mon Sep 17 00:00:00 2001 From: louispt1 Date: Wed, 17 Jun 2026 17:02:47 +0200 Subject: [PATCH 1/3] Add another spec to lib/graph_data_validation/spec/validation/emissions.rb to validate the totals queries align with actual csv data to ensure no queries are missed when adding aggregates --- .../spec/validation/emissions.rb | 38 ++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/lib/graph_data_validation/spec/validation/emissions.rb b/lib/graph_data_validation/spec/validation/emissions.rb index 8101c625f..d3330578c 100644 --- a/lib/graph_data_validation/spec/validation/emissions.rb +++ b/lib/graph_data_validation/spec/validation/emissions.rb @@ -18,5 +18,41 @@ end end end -end + # Reconcile the 1990 direct_emissions total queries against the dataset's + # emissions.csv (read from the local ETSource). + # + # Bunkers is excluded on both sides: its query scales the inventory by the + # bunkers_allocated_percentage_* inputs (default 0), so it cannot equal the raw + # CSV. Subtracting the bunkers query and the Bunkers CSV rows makes the check + # allocation-independent. + # + # LULUCF removals are stored as positive values in the CSV but represent a + # carbon sink; the queries subtract them (NEG), so the CSV sum negates them too. + emissions = Atlas::Dataset.find(dataset.scenario.area_code).emissions + + [ + { total: 'direct_emissions_co2_1990', bunkers: 'direct_emissions_bunkers_co2_1990', ghgs: %w[co2] }, + { total: 'direct_emissions_other_ghg_1990', bunkers: 'direct_emissions_bunkers_other_ghg_1990', ghgs: %w[other_ghg] }, + { total: 'direct_emissions_total_ghg_1990', bunkers: 'direct_emissions_bunkers_total_ghg_1990', ghgs: %w[co2 other_ghg] } + ].each do |check| + # Queries report Mt; the inventory CSV is in kton CO2-eq. Exclude bunkers. + queried_mt = dataset.query("present:Q(#{check[:total]})") - + dataset.query("present:Q(#{check[:bunkers]})") + emissions_csv_mt = emissions.table.sum do |row| + next 0.0 unless row[:year] == 1990 && check[:ghgs].include?(row[:ghg]) + next 0.0 if row[:etm_sector].to_s.casecmp?('bunkers') + + value = row[:value].to_f + removal = row[:etm_sector].to_s.casecmp?('lulucf') && + row[:etm_subsector].to_s.casecmp?('removals') + removal ? -value : value + end / 1000.0 + + context "1990 inventory reconciliation for #{check[:total]}" do + it 'matches emissions.csv (excl. bunkers)' do + expect(queried_mt).to be_within(0.001 * emissions_csv_mt + 1e-6).of(emissions_csv_mt) + end + end + end +end From b080ece474182160adc535b9718404affdd595cc Mon Sep 17 00:00:00 2001 From: louispt1 Date: Thu, 25 Jun 2026 12:24:14 +0200 Subject: [PATCH 2/3] define a small emissions_1990_validation dataset list in config.yml and abstract complex logic out of spec --- lib/graph_data_validation/config.yml | 6 ++++ lib/graph_data_validation/lib/datasets.rb | 1 + .../lib/emissions_csv_reconciler.rb | 24 +++++++++++++++ .../spec/dataset_spec.rb | 1 + .../spec/emissions_1990_spec.rb | 12 ++++++++ .../spec/validation/emissions.rb | 29 ++++--------------- 6 files changed, 50 insertions(+), 23 deletions(-) create mode 100644 lib/graph_data_validation/lib/emissions_csv_reconciler.rb create mode 100644 lib/graph_data_validation/spec/emissions_1990_spec.rb diff --git a/lib/graph_data_validation/config.yml b/lib/graph_data_validation/config.yml index f1b62c277..6a70305a7 100644 --- a/lib/graph_data_validation/config.yml +++ b/lib/graph_data_validation/config.yml @@ -1,4 +1,10 @@ dataset_collections: + emissions_1990_validation: + - nl2023 + - nl2019 + - DE_germany + - UKNI01_northern_ireland + countries: - AT_austria - BEGM11002_antwerpen diff --git a/lib/graph_data_validation/lib/datasets.rb b/lib/graph_data_validation/lib/datasets.rb index 0d89e71d2..c48701a92 100644 --- a/lib/graph_data_validation/lib/datasets.rb +++ b/lib/graph_data_validation/lib/datasets.rb @@ -10,6 +10,7 @@ def initialize(*area_codes, env: :dev) if env == :dev Settings.etsource_lazy_load_dataset = true Atlas.data_dir = Etsource::Base.clean_path(File.expand_path("../etsource")) + Etsource::Loader.instance.clear!(:gqueries) end # Silently reject unexisting areas diff --git a/lib/graph_data_validation/lib/emissions_csv_reconciler.rb b/lib/graph_data_validation/lib/emissions_csv_reconciler.rb new file mode 100644 index 000000000..669e32486 --- /dev/null +++ b/lib/graph_data_validation/lib/emissions_csv_reconciler.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module GraphDataValidation + class EmissionsCsvReconciler + def initialize(dataset) + @emissions = Atlas::Dataset.find(dataset.scenario.area_code).emissions + end + + # Returns the 1990 inventory total in Mt for the given GHG types, + # excluding bunkers rows and negating LULUCF removals (stored as positive + # values in the CSV but representing a carbon sink). + def total_mt(ghgs:) + @emissions.table.sum do |row| + next 0.0 unless row[:year] == 1990 && ghgs.include?(row[:ghg]) + next 0.0 if row[:etm_sector].to_s.casecmp?('bunkers') + + value = row[:value].to_f + removal = row[:etm_sector].to_s.casecmp?('lulucf') && + row[:etm_subsector].to_s.casecmp?('removals') + removal ? -value : value + end / 1000.0 + end + end +end diff --git a/lib/graph_data_validation/spec/dataset_spec.rb b/lib/graph_data_validation/spec/dataset_spec.rb index 8ef904222..b0befe307 100644 --- a/lib/graph_data_validation/spec/dataset_spec.rb +++ b/lib/graph_data_validation/spec/dataset_spec.rb @@ -9,3 +9,4 @@ end end end + diff --git a/lib/graph_data_validation/spec/emissions_1990_spec.rb b/lib/graph_data_validation/spec/emissions_1990_spec.rb new file mode 100644 index 000000000..2c5a0d693 --- /dev/null +++ b/lib/graph_data_validation/spec/emissions_1990_spec.rb @@ -0,0 +1,12 @@ +require_relative 'spec_helper' +require_relative '../lib/datasets' +require_relative '../lib/emissions_csv_reconciler' +require_relative 'validation/emissions' + +describe 'Validating 1990 emissions inventory' do + GraphDataValidation::Datasets.from_collection(:emissions_1990_validation).each do |dataset| + context "with area_code #{dataset.scenario.area_code}" do + it_behaves_like 'emissions_1990_reconciliation', dataset + end + end +end diff --git a/lib/graph_data_validation/spec/validation/emissions.rb b/lib/graph_data_validation/spec/validation/emissions.rb index d3330578c..d40eb7bde 100644 --- a/lib/graph_data_validation/spec/validation/emissions.rb +++ b/lib/graph_data_validation/spec/validation/emissions.rb @@ -18,40 +18,23 @@ end end end +end - # Reconcile the 1990 direct_emissions total queries against the dataset's - # emissions.csv (read from the local ETSource). - # - # Bunkers is excluded on both sides: its query scales the inventory by the - # bunkers_allocated_percentage_* inputs (default 0), so it cannot equal the raw - # CSV. Subtracting the bunkers query and the Bunkers CSV rows makes the check - # allocation-independent. - # - # LULUCF removals are stored as positive values in the CSV but represent a - # carbon sink; the queries subtract them (NEG), so the CSV sum negates them too. - emissions = Atlas::Dataset.find(dataset.scenario.area_code).emissions +RSpec.shared_examples('emissions_1990_reconciliation') do |dataset| + reconciler = GraphDataValidation::EmissionsCsvReconciler.new(dataset) [ { total: 'direct_emissions_co2_1990', bunkers: 'direct_emissions_bunkers_co2_1990', ghgs: %w[co2] }, { total: 'direct_emissions_other_ghg_1990', bunkers: 'direct_emissions_bunkers_other_ghg_1990', ghgs: %w[other_ghg] }, { total: 'direct_emissions_total_ghg_1990', bunkers: 'direct_emissions_bunkers_total_ghg_1990', ghgs: %w[co2 other_ghg] } ].each do |check| - # Queries report Mt; the inventory CSV is in kton CO2-eq. Exclude bunkers. queried_mt = dataset.query("present:Q(#{check[:total]})") - dataset.query("present:Q(#{check[:bunkers]})") - emissions_csv_mt = emissions.table.sum do |row| - next 0.0 unless row[:year] == 1990 && check[:ghgs].include?(row[:ghg]) - next 0.0 if row[:etm_sector].to_s.casecmp?('bunkers') - - value = row[:value].to_f - removal = row[:etm_sector].to_s.casecmp?('lulucf') && - row[:etm_subsector].to_s.casecmp?('removals') - removal ? -value : value - end / 1000.0 + expected_mt = reconciler.total_mt(ghgs: check[:ghgs]) - context "1990 inventory reconciliation for #{check[:total]}" do + context "1990 emissions results for #{check[:total]}" do it 'matches emissions.csv (excl. bunkers)' do - expect(queried_mt).to be_within(0.001 * emissions_csv_mt + 1e-6).of(emissions_csv_mt) + expect(queried_mt).to be_within(0.001 * expected_mt + 1e-6).of(expected_mt) end end end From 11fd98fdda9c37821993186d1544ca2619d8c623 Mon Sep 17 00:00:00 2001 From: louispt1 Date: Thu, 25 Jun 2026 14:30:31 +0200 Subject: [PATCH 3/3] Split examples up for clarity --- .../spec/validation/emissions.rb | 43 +++++++++++++------ 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/lib/graph_data_validation/spec/validation/emissions.rb b/lib/graph_data_validation/spec/validation/emissions.rb index d40eb7bde..c9fe978b2 100644 --- a/lib/graph_data_validation/spec/validation/emissions.rb +++ b/lib/graph_data_validation/spec/validation/emissions.rb @@ -23,19 +23,36 @@ RSpec.shared_examples('emissions_1990_reconciliation') do |dataset| reconciler = GraphDataValidation::EmissionsCsvReconciler.new(dataset) - [ - { total: 'direct_emissions_co2_1990', bunkers: 'direct_emissions_bunkers_co2_1990', ghgs: %w[co2] }, - { total: 'direct_emissions_other_ghg_1990', bunkers: 'direct_emissions_bunkers_other_ghg_1990', ghgs: %w[other_ghg] }, - { total: 'direct_emissions_total_ghg_1990', bunkers: 'direct_emissions_bunkers_total_ghg_1990', ghgs: %w[co2 other_ghg] } - ].each do |check| - queried_mt = dataset.query("present:Q(#{check[:total]})") - - dataset.query("present:Q(#{check[:bunkers]})") - expected_mt = reconciler.total_mt(ghgs: check[:ghgs]) - - context "1990 emissions results for #{check[:total]}" do - it 'matches emissions.csv (excl. bunkers)' do - expect(queried_mt).to be_within(0.001 * expected_mt + 1e-6).of(expected_mt) - end + context 'with 1990 CO2 emissions' do + queried_mt = + dataset.query('present:Q(direct_emissions_co2_1990)') - + dataset.query('present:Q(direct_emissions_bunkers_co2_1990)') + expected_mt = reconciler.total_mt(ghgs: %w[co2]) + + it 'matches emissions.csv (excluding bunkers)' do + expect(queried_mt).to be_within((0.001 * expected_mt) + 1e-6).of(expected_mt) + end + end + + context 'with 1990 other greenhouse gas emissions' do + queried_mt = + dataset.query('present:Q(direct_emissions_other_ghg_1990)') - + dataset.query('present:Q(direct_emissions_bunkers_other_ghg_1990)') + expected_mt = reconciler.total_mt(ghgs: %w[other_ghg]) + + it 'matches emissions.csv (excluding bunkers)' do + expect(queried_mt).to be_within((0.001 * expected_mt) + 1e-6).of(expected_mt) + end + end + + context 'with 1990 total greenhouse gas emissions' do + queried_mt = + dataset.query('present:Q(direct_emissions_total_ghg_1990)') - + dataset.query('present:Q(direct_emissions_bunkers_total_ghg_1990)') + expected_mt = reconciler.total_mt(ghgs: %w[co2 other_ghg]) + + it 'matches emissions.csv (excluding bunkers)' do + expect(queried_mt).to be_within((0.001 * expected_mt) + 1e-6).of(expected_mt) end end end