From bc1d5446e08f42d225437141d3c1dd022ef47d16 Mon Sep 17 00:00:00 2001 From: louispt1 Date: Wed, 3 Jun 2026 07:58:59 +0200 Subject: [PATCH 01/23] Implement EMISSIONS_MAP lookup and tests EMISSIONS_MAP('1.A.4.a') --> [, , ...] --- app/models/gql/runtime/functions/lookup.rb | 95 ++++++++++++ spec/fixtures/etsource/crt_mapping.csv | 5 + .../gql/runtime/functions/lookup_spec.rb | 136 ++++++++++++++++++ 3 files changed, 236 insertions(+) create mode 100644 spec/fixtures/etsource/crt_mapping.csv diff --git a/app/models/gql/runtime/functions/lookup.rb b/app/models/gql/runtime/functions/lookup.rb index 6c979bdad..2a11045a5 100644 --- a/app/models/gql/runtime/functions/lookup.rb +++ b/app/models/gql/runtime/functions/lookup.rb @@ -367,6 +367,101 @@ def EMISSIONS(*keys) # EMISSIONS(sector, use, ghg [, year]) -> return value scope.graph.emissions[keys.join('_').to_sym] end + + # Returns an Array of {Qernel::Node} matching a CRT/IPCC code. + # + # Looks up the CRT code in the generic crt_mapping.csv file to find the + # corresponding ETM sector, subsector, and use. Then returns all nodes + # (from both energy and molecule graphs) that match those attributes. + # + # No crt_code attribute is needed on nodes - matching is inferred from + # the mapping based on node sector, key (containing subsector), and use. + # + # crt_code - String or Symbol CRT code (e.g., '1.A.2', '2.B.10.a') + # + # Examples + # + # EMISSIONS_MAP('1.A.4.a') + # # => [, , ...] + # + # SUM(V(EMISSIONS_MAP('1.A.4.a'), demand)) + # # => Sum demand from all nodes matching CRT code 1.A.4.a + # + # UPDATE(EMISSIONS_MAP('1.A.4.a'), some_attribute, value) + # # => Update all matching nodes + # + # Returns an Array of Qernel::Nodes, or empty array if no mapping found. + def EMISSIONS_MAP(crt_code) + normalized_code = normalize_crt_code(crt_code) + mapping = load_crt_mapping(normalized_code) + + return [] unless mapping + + find_nodes_by_mapping(mapping) + end + + private + + # Internal: Normalizes a string for key matching. + # Converts to lowercase, replaces spaces/hyphens with underscores. + # + # Returns a String. + def normalize_key(str) + str.to_s.downcase.tr(' -', '_') + end + + # Internal: Normalizes CRT code for lookup. + # Converts dots/hyphens to underscores, lowercase. + # + # Returns a Symbol. + def normalize_crt_code(code) + code.to_s.downcase.tr('-.', '_').to_sym + end + + # Internal: Loads CRT mapping from dataset. + # + # Returns a CSV::Row or nil. + def load_crt_mapping(normalized_code) + dataset = Atlas::Dataset.find(scope.graph.area.area_code) + dataset.crt_mapping[normalized_code] + end + + # Internal: Finds all nodes matching the CRT mapping. + # + # Returns an Array of Qernel::Nodes. + def find_nodes_by_mapping(mapping) + sector = normalize_key(mapping[:etm_sector]) + subsector = normalize_key(mapping[:etm_subsector]) + use = normalize_key(mapping[:use]) + + all_nodes.select { |node| node_matches_mapping?(node, sector, subsector, use) } + end + + # Internal: Returns all nodes from both graphs. + # + # Returns an Array of Qernel::Nodes. + def all_nodes + scope.all_energy_nodes + scope.all_molecule_nodes + end + + # Internal: Checks if a node matches the mapping criteria. + # + # Returns Boolean. + def node_matches_mapping?(node, sector, subsector, use) + return false unless node.sector_key.to_s == sector + return false unless subsector_matches?(node, subsector) + return false unless node.use_key.to_s == use + + true + end + + # Internal: Checks if subsector matches node key. + # Expects the subsector to be included in the node key. + # + # Returns Boolean. + def subsector_matches?(node, subsector) + node.key.to_s.include?(subsector) + end end end end diff --git a/spec/fixtures/etsource/crt_mapping.csv b/spec/fixtures/etsource/crt_mapping.csv new file mode 100644 index 000000000..6f01b0f01 --- /dev/null +++ b/spec/fixtures/etsource/crt_mapping.csv @@ -0,0 +1,5 @@ +crt_code,etm_sector,etm_subsector,use +1.A.1,Energy,Electricity and heat production,energetic +1.A.2,Industry,Non-specified,energetic +1.A.4.a,Buildings,Non-specified,energetic +1.A.4.b,Households,Non-specified,energetic diff --git a/spec/models/gql/runtime/functions/lookup_spec.rb b/spec/models/gql/runtime/functions/lookup_spec.rb index 60fc6ec9e..07dcaa2af 100644 --- a/spec/models/gql/runtime/functions/lookup_spec.rb +++ b/spec/models/gql/runtime/functions/lookup_spec.rb @@ -67,5 +67,141 @@ module Gql::Runtime::Functions expect(result).to eq(7.0) end end + + # EMISSIONS_MAP + # ------------- + + describe 'EMISSIONS_MAP("1.A.1")' do + it 'returns an array of nodes matching the CRT code' do + expect(result).to be_an(Array) + expect(result).to all(be_a(Qernel::Node)) + end + end + + describe 'EMISSIONS_MAP("nonexistent")' do + it 'returns an empty array for non-existent CRT codes' do + expect(result).to eq([]) + end + end + + # Test various CRT code formats + describe 'EMISSIONS_MAP("1-A-1")' do + it 'handles CRT codes with hyphens' do + expect(result).to be_an(Array) + # Should normalize to same as "1.A.1" + end + end + + describe 'EMISSIONS_MAP("1_a_1")' do + it 'handles CRT codes with underscores and lowercase' do + expect(result).to be_an(Array) + # Should normalize to same as "1.A.1" + end + end + + # Test when mapping exists but no nodes match + context 'when CRT mapping exists but no nodes match' do + let(:mock_mapping) do + { + etm_sector: 'nonexistent_sector', + etm_subsector: 'nonexistent_subsector', + use: 'energetic' + } + end + + before do + dataset = Atlas::Dataset.find(gql.future_graph.area.area_code) + allow(dataset.crt_mapping).to receive(:[]).with(:test_code).and_return(mock_mapping) + end + + describe 'EMISSIONS_MAP("test_code")' do + it 'returns an empty array' do + expect(result).to eq([]) + end + end + end + + # Test non_specified subsector special case + context 'when subsector is non_specified' do + let(:mock_mapping) do + { + etm_sector: 'buildings', + etm_subsector: 'non_specified', + use: 'energetic' + } + end + + before do + dataset = Atlas::Dataset.find(gql.future_graph.area.area_code) + allow(dataset.crt_mapping).to receive(:[]).with(:test_nonspec).and_return(mock_mapping) + end + + describe 'EMISSIONS_MAP("test_nonspec")' do + it 'matches nodes without requiring subsector in key' do + expect(result).to be_an(Array) + # Should match all buildings sector nodes with energetic use + # regardless of whether key contains 'non_specified' + result.each do |node| + expect(node.sector_key).to eq(:buildings) + expect(node.use_key).to eq(:energetic) + end + end + end + end + + # Test matching across both energy and molecule graphs + context 'with nodes in both energy and molecule graphs' do + describe 'EMISSIONS_MAP("1.A.1")' do + it 'searches both energy and molecule graphs' do + # The function should search both graphs (implementation verified) + # Result may be empty if no matching nodes in test fixtures + expect(result).to be_an(Array) + + # If there are results, verify they come from one or both graphs + if result.any? + energy_nodes = result.select { |n| gql.future_graph.nodes.include?(n) } + molecule_nodes = result.select { |n| gql.future.molecule_graph.nodes.include?(n) } + + expect(energy_nodes.any? || molecule_nodes.any?).to be true + end + end + end + end + + # Test node attribute matching + context 'when nodes have matching attributes' do + describe 'EMISSIONS_MAP("1.A.1")' do + it 'matches nodes based on sector, subsector in key, and use' do + skip 'if no nodes match' if result.empty? + + result.each do |node| + # All returned nodes should have matching attributes + expect(node).to respond_to(:sector_key) + expect(node).to respond_to(:key) + expect(node).to respond_to(:use_key) + end + end + end + end + + # Integration test with real CRT code + context 'with real emissions data' do + describe 'EMISSIONS_MAP("1.A.4.a")' do + it 'returns nodes for commercial/institutional combustion' do + expect(result).to be_an(Array) + # Should match services sector nodes if they exist in the dataset + end + end + end + + # Edge case: nil/missing attributes + context 'when nodes might have missing attributes' do + describe 'EMISSIONS_MAP("1.A.1")' do + it 'handles nodes without sector gracefully' do + # Should not raise error, just skip nodes without required attributes + expect { result }.not_to raise_error + end + end + end end end From 2b30de0ed55595a63e2f2fbcf89ea1ba8b4e1c34 Mon Sep 17 00:00:00 2001 From: louispt1 Date: Mon, 8 Jun 2026 10:10:08 +0200 Subject: [PATCH 02/23] Handle sum to aggregate sectors based on prefixes, and hangle year emissions format --- app/models/gql/runtime/functions/lookup.rb | 9 +- app/models/qernel/emissions.rb | 130 ++++++++++++--- spec/fixtures/etsource/crt_etm_mapping.csv | 27 +++ .../etsource/datasets/nl/emissions.csv | 34 +++- .../gql/runtime/functions/lookup_spec.rb | 31 +++- spec/models/qernel/emissions_spec.rb | 155 +++++++++++++++--- 6 files changed, 323 insertions(+), 63 deletions(-) create mode 100644 spec/fixtures/etsource/crt_etm_mapping.csv diff --git a/app/models/gql/runtime/functions/lookup.rb b/app/models/gql/runtime/functions/lookup.rb index 2a11045a5..0de51bd83 100644 --- a/app/models/gql/runtime/functions/lookup.rb +++ b/app/models/gql/runtime/functions/lookup.rb @@ -352,8 +352,9 @@ def DATASET_CURVE(key) # # Examples # EMISSIONS(households, energetic, other_ghg) # => 12.0 (from emissions.csv) - # EMISSIONS(households, energetic, co2, 1990) # => value (from emissions_1990.csv) - # EMISSIONS(buildings_non_specified, energetic, other_ghg) # => 18.0 + # EMISSIONS(households, energetic, co2) # => aggregated value (sum of subsectors, default year) + # EMISSIONS(households, energetic, co2, 1990) # => aggregated value for 1990 + # EMISSIONS(buildings_non_specified, energetic, other_ghg, 2023) # => 18.0 # def EMISSIONS(*keys) return scope.graph.emissions if keys.empty? @@ -364,8 +365,8 @@ def EMISSIONS(*keys) # EMISSIONS(sector, use) -> return ScopedSector for UPDATE operations return scope.graph.emissions.scope(keys.join('_').to_sym) if keys.size == 2 - # EMISSIONS(sector, use, ghg [, year]) -> return value - scope.graph.emissions[keys.join('_').to_sym] + # EMISSIONS(sector, use, ghg [, year]) -> aggregate and return value + scope.graph.emissions.sum(*keys) end # Returns an Array of {Qernel::Node} matching a CRT/IPCC code. diff --git a/app/models/qernel/emissions.rb b/app/models/qernel/emissions.rb index e300c2b27..86aed5977 100644 --- a/app/models/qernel/emissions.rb +++ b/app/models/qernel/emissions.rb @@ -1,13 +1,29 @@ module Qernel - # Class for getting and setting emissions data + # Class for getting and setting emissions data. # Behaves much like Qernel::Area, can be seen as an extension of - # area attributes, scoped for emissions + # area attributes, scoped for emissions. + # + # == Data Structure # # Emissions data is loaded from CSV files in ETSource with structure: - # etm_sector, etm_subsector, use, ghg, unit, value + # etm_sector, etm_subsector, use, ghg, year, unit, value + # + # Keys are generated as: sector_subsector_use_ghg_year + # Examples: + # - buildings_non_specified_energetic_other_ghg_2023 + # - energy_electricity_and_heat_production_energetic_co2_1990 + # + # == Aggregation via sum() + # + # The sum() method aggregates emissions across subsectors for a given sector, + # use, ghg, and year combination. This allows queries like: + # EMISSIONS(energy, energetic, co2, 2023) + # to return the sum of all energy subsectors' energetic CO2 emissions for 2023. # - # Keys are generated as: sector_[subsector_]use_ghg[_year] - # Example: buildings_non_specified_energetic_co2 + # == Year Handling + # + # Year parameter defaults to the dataset's analysis_year when not specified. + # Multiple years can coexist in the same dataset (e.g., 1990 baseline, 2023 current). class Emissions include DatasetAttributes @@ -23,9 +39,10 @@ class Emissions # EMISSIONS(households, energetic) returns a ScopedSector # Then UPDATE can call: scoped.co2 = 100.0 class ScopedSector - def initialize(emissions, scope) + def initialize(emissions, scope, year = nil) @emissions = emissions @scope = scope + @year = year end def [](attr_name) @@ -41,27 +58,56 @@ def inspect end def scoped_method(method_name) - "#{@scope}_#{method_name}" + year = @year || @emissions.graph&.area&.analysis_year + "#{@scope}_#{method_name}_#{year}" end def respond_to_missing?(method_name, include_private = false) - data_key = scoped_method(method_name).split('=').first + # Remove '=' suffix if present before generating scoped key + clean_method = method_name.to_s.delete_suffix('=') + data_key = scoped_method(clean_method).to_sym - @emissions.respond_to?(data_key) || super + # Setters are allowed if the scope exists (at least one key with this scope prefix) + # Getters require the exact key to exist in the dataset + if method_name.to_s.end_with?('=') + scope_exists? + else + @emissions.dataset_attributes&.key?(data_key) || super + end end def method_missing(method_name, *args) - data_key = scoped_method(method_name).split('=').first.to_sym + # Remove '=' suffix if present before generating scoped key + clean_method = method_name.to_s.delete_suffix('=') + data_key = scoped_method(clean_method).to_sym - # Validate the key exists for both getters and setters - unless @emissions.respond_to?(data_key) - raise NoMethodError, "undefined method `#{method_name}' for #{inspect}" + # Setter if method name ended with '=' + if method_name.to_s.end_with?('=') + # Validate that the scope exists (at least one key with this scope prefix) + unless scope_exists? + raise NoMethodError, "undefined method `#{method_name}' for #{inspect}" + end + @emissions[data_key] = args.first + else + # Getter - validate the key exists + unless @emissions.dataset_attributes&.key?(data_key) + raise NoMethodError, "undefined method `#{method_name}' for #{inspect}" + end + @emissions[data_key] end + end - if data_key.to_s == scoped_method(method_name) - @emissions[data_key] - else - @emissions[data_key] = args.first + private + + # Check if at least one key exists with the current scope prefix + # (validates the scope exists in the dataset, allowing any GHG/year combination) + def scope_exists? + prefix = "#{@scope}_" + + return false unless @emissions.dataset_attributes + + @emissions.dataset_attributes.keys.any? do |key| + key.to_s.start_with?(prefix) end end end @@ -72,11 +118,57 @@ def initialize(graph = nil) @dataset_key = @key = :emissions_data end + # Public: Aggregates emissions across subsectors for a given sector, use, ghg, and year. + # + # This method sums all emissions entries that match the specified sector prefix, + # use type, GHG type, and year. It aggregates across all subsectors within the + # specified sector. + # + # == Examples + # + # # Sum all energy sector energetic CO2 emissions for 2023 + # # (aggregates electricity production, fuels production, etc.) + # emissions.sum(:energy, :energetic, :co2, 2023) + # # => 500.25 + # + # # Get single subsector (no aggregation needed, but still uses sum) + # emissions.sum(:buildings_non_specified, :energetic, :other_ghg, 2023) + # # => 55.64 + # + # # Use default year (analysis_year from dataset) + # emissions.sum(:agriculture, :non_energetic, :other_ghg) + # # => 18863.47 + # + # == Parameters + # + # sector - Sector name or full subsector key (e.g., :energy, :buildings_non_specified) + # Normalized: dashes/dots → underscores, lowercased + # use - Use type (:energetic or :non_energetic) + # ghg - GHG type (:co2 or :other_ghg) + # year - Optional year (Integer). Defaults to graph.area.analysis_year if not specified + # + # == Returns + # + # Float sum of all matching emissions. Returns 0 if no matches found. + # Includes any runtime UPDATE modifications to emission values. + def sum(sector, use, ghg, year = nil) + year ||= graph&.area&.analysis_year + prefix = sector.to_s.tr('-.', '_').downcase + suffix = "_#{use}_#{ghg}_#{year}" + + dataset_attributes.keys.select { |key| + key.to_s.start_with?(prefix) && key.to_s.end_with?(suffix) + }.sum { |key| dataset_get(key) || 0 } + end + # Public: define the sector scope for access to the hashed emission keys # + # sector - Sector identifier (e.g., :buildings_non_specified_energetic) + # year - Optional year (defaults to analysis_year). Used to target specific year for UPDATE operations. + # # Returns a scoped version of the emissions data - def scope(sector) - ScopedSector.new(self, sector) + def scope(sector, year = nil) + ScopedSector.new(self, sector, year) end end end diff --git a/spec/fixtures/etsource/crt_etm_mapping.csv b/spec/fixtures/etsource/crt_etm_mapping.csv new file mode 100644 index 000000000..ae601684e --- /dev/null +++ b/spec/fixtures/etsource/crt_etm_mapping.csv @@ -0,0 +1,27 @@ +crt_code,etm_sector,etm_subsector,use +1.A.1,Energy,Electricity and heat production,energetic +1.A.1.a,Energy,Electricity and heat production,energetic +1.A.1.b,Industry,Refineries,energetic +1.A.1.c,Energy,Fuels production,energetic +1.A.2,Industry,Non-specified,energetic +1.A.3.a,National transport,Non-specified,energetic +1.A.4.a,Buildings,Non-specified,energetic +1.A.4.b,Households,Non-specified,energetic +1.A.4.c.i,Agriculture,Non-specified,energetic +1.A.5,Other,Non-specified,energetic +1.B.1,Energy,Fugitive emissions,non_energetic +1.B.2.a.i,Energy,Fugitive emissions,non_energetic +1.C,Energy,CCUS,non_energetic +2.A,Industry,Other,non_energetic +2.B.1,Industry,Fertilizers,non_energetic +2.B.3,Industry,Chemicals,non_energetic +2.B.8.a,Energy,Methanol production,non_energetic +2.B.10.a,Energy,Hydrogen production,non_energetic +2.C.1,Industry,Steel,non_energetic +2.C.3,Industry,Aluminium,non_energetic +3,Agriculture,Non-specified,non_energetic +4,LULUCF,Non-specified,non_energetic +5,Waste,Non-specified,non_energetic +ind_CO2,Other,Indirect emissions,non_energetic +1.D.1.a,International transport,International aviation,energetic +1.D.1.b,International transport,International navigation,energetic diff --git a/spec/fixtures/etsource/datasets/nl/emissions.csv b/spec/fixtures/etsource/datasets/nl/emissions.csv index c4c204046..b1236ad06 100644 --- a/spec/fixtures/etsource/datasets/nl/emissions.csv +++ b/spec/fixtures/etsource/datasets/nl/emissions.csv @@ -1,9 +1,25 @@ -etm_sector,etm_subsector,use,ghg,unit,value -Energy,Electricity and heat production,energetic,other_ghg,kg CO2eq,18.0 -Energy,Fugitive emissions,non_energetic,co2,kg CO2eq,20.0 -Households,Non-specified,energetic,other_ghg,kg CO2eq,7.0 -Buildings,Non-specified,energetic,other_ghg,kg CO2eq,2796620.0 -Industry,Non-specified,energetic,other_ghg,kg CO2eq,100.0 -Agriculture,Non-specified,energetic,other_ghg,kg CO2eq,50.0 -Agriculture,Non-specified,non_energetic,co2,kg CO2eq,75.0 -Waste,Non-specified,non_energetic,co2,kg CO2eq,25.0 +etm_sector,etm_subsector,use,ghg,year,unit,value +Energy,Electricity and heat production,energetic,other_ghg,2023,kg CO2eq,18.0 +Energy,Electricity and heat production,energetic,other_ghg,2023,kg CO2eq,18.0 +Energy,Electricity and heat production,energetic,other_ghg,1990,kg CO2eq,25.0 +Energy,Fugitive emissions,non_energetic,co2,2023,kg CO2eq,20.0 +Energy,Fugitive emissions,non_energetic,co2,2023,kg CO2eq,20.0 +Energy,Fugitive emissions,non_energetic,co2,1990,kg CO2eq,30.0 +Households,Non-specified,energetic,other_ghg,2023,kg CO2eq,7.0 +Households,Non-specified,energetic,other_ghg,2023,kg CO2eq,7.0 +Households,Non-specified,energetic,other_ghg,1990,kg CO2eq,10.0 +Buildings,Non-specified,energetic,other_ghg,2023,kg CO2eq,2796620.0 +Buildings,Non-specified,energetic,other_ghg,2023,kg CO2eq,2796620.0 +Buildings,Non-specified,energetic,other_ghg,1990,kg CO2eq,3000000.0 +Industry,Non-specified,energetic,other_ghg,2023,kg CO2eq,100.0 +Industry,Non-specified,energetic,other_ghg,2023,kg CO2eq,100.0 +Industry,Non-specified,energetic,other_ghg,1990,kg CO2eq,150.0 +Agriculture,Non-specified,energetic,other_ghg,2023,kg CO2eq,50.0 +Agriculture,Non-specified,energetic,other_ghg,2023,kg CO2eq,50.0 +Agriculture,Non-specified,energetic,other_ghg,1990,kg CO2eq,75.0 +Agriculture,Non-specified,non_energetic,co2,2023,kg CO2eq,75.0 +Agriculture,Non-specified,non_energetic,co2,2023,kg CO2eq,75.0 +Agriculture,Non-specified,non_energetic,co2,1990,kg CO2eq,100.0 +Waste,Non-specified,non_energetic,co2,2023,kg CO2eq,25.0 +Waste,Non-specified,non_energetic,co2,2023,kg CO2eq,25.0 +Waste,Non-specified,non_energetic,co2,1990,kg CO2eq,35.0 diff --git a/spec/models/gql/runtime/functions/lookup_spec.rb b/spec/models/gql/runtime/functions/lookup_spec.rb index 07dcaa2af..280e4f3a0 100644 --- a/spec/models/gql/runtime/functions/lookup_spec.rb +++ b/spec/models/gql/runtime/functions/lookup_spec.rb @@ -50,24 +50,43 @@ module Gql::Runtime::Functions end end - describe "EMISSIONS(buildings_non_specified, energetic, other_ghg)" do - it 'returns the emission value' do + describe "EMISSIONS(buildings_non_specified, energetic, other_ghg, 2023)" do + it 'returns the emission value for the specified year' do expect(result).to eq(2796620.0) end end - describe "EMISSIONS(energy_electricity_and_heat_production, energetic, other_ghg)" do - it 'returns the emission value' do + describe "EMISSIONS(energy_electricity_and_heat_production, energetic, other_ghg, 2023)" do + it 'returns the emission value for the specified year' do expect(result).to eq(18.0) end end - describe 'EMISSIONS(households_non_specified, energetic, other_ghg)' do - it 'returns the emission value' do + describe 'EMISSIONS(households_non_specified, energetic, other_ghg, 2023)' do + it 'returns the emission value for the specified year' do expect(result).to eq(7.0) end end + describe 'EMISSIONS(energy, non_energetic, co2, 2023)' do + it 'aggregates across energy subsectors (Fugitive emissions)' do + expect(result).to eq(20.0) # Fugitive: 20.0 + end + end + + describe 'EMISSIONS(energy, energetic, other_ghg, 2023)' do + it 'aggregates multiple energy subsectors' do + # Electricity: 18.0 + expect(result).to eq(18.0) + end + end + + describe 'EMISSIONS(agriculture, energetic, other_ghg, 2023)' do + it 'sums single subsector' do + expect(result).to eq(50.0) + end + end + # EMISSIONS_MAP # ------------- diff --git a/spec/models/qernel/emissions_spec.rb b/spec/models/qernel/emissions_spec.rb index dc696a88b..53aeb74c7 100644 --- a/spec/models/qernel/emissions_spec.rb +++ b/spec/models/qernel/emissions_spec.rb @@ -45,12 +45,14 @@ module Qernel end describe Emissions::ScopedSector do - let(:emissions) { Emissions.new.with({}) } + let(:graph) { double('Graph', area: area) } + let(:area) { double('Area', analysis_year: 2023) } + let(:emissions) { Emissions.new(graph).with({}) } let(:scoped) { emissions.scope(:households_non_specified_energetic) } before do - emissions[:households_non_specified_energetic_other_ghg] = 50.0 - emissions[:agriculture_non_specified_energetic_other_ghg] = 25.0 + emissions[:households_non_specified_energetic_other_ghg_2023] = 50.0 + emissions[:agriculture_non_specified_energetic_other_ghg_2023] = 25.0 end describe '#method_missing' do @@ -66,13 +68,13 @@ module Qernel it 'delegates setter methods to emissions with scoped prefix' do scoped.other_ghg = 75.0 # Setters convert to symbol keys via dataset_set - expect(emissions.dataset_get(:households_non_specified_energetic_other_ghg)).to eq(75.0) + expect(emissions.dataset_get(:households_non_specified_energetic_other_ghg_2023)).to eq(75.0) end it 'delegates setter for a different scope' do other_scoped = emissions.scope(:agriculture_non_specified_energetic) other_scoped.other_ghg = 30.0 - expect(emissions.dataset_get(:agriculture_non_specified_energetic_other_ghg)).to eq(30.0) + expect(emissions.dataset_get(:agriculture_non_specified_energetic_other_ghg_2023)).to eq(30.0) end it 'raises NoMethodError for undefined getter methods' do @@ -80,15 +82,17 @@ module Qernel expect { scoped.invalid_emission }.to raise_error(NoMethodError) end - it 'raises NoMethodError for setter keys that do not exist in the dataset' do - expect { scoped.arbitrary_key = 100.0 }.to raise_error(NoMethodError) - expect { scoped.custom_emission_type = 200.0 }.to raise_error(NoMethodError) - expect { scoped.nonexistent = 300.0 }.to raise_error(NoMethodError) + it 'allows setters for any GHG type when scope exists in dataset' do + # Setters are allowed for any GHG type if the scope (subsector + use) exists + # This enables UPDATE operations to set runtime values + expect { scoped.arbitrary_key = 100.0 }.not_to raise_error + expect { scoped.custom_emission_type = 200.0 }.not_to raise_error + expect { scoped.co2 = 300.0 }.not_to raise_error end it 'allows setters for emission keys that exist in the dataset' do expect { scoped.other_ghg = 2.0 }.not_to raise_error - expect(emissions.dataset_get(:households_non_specified_energetic_other_ghg)).to eq(2.0) + expect(emissions.dataset_get(:households_non_specified_energetic_other_ghg_2023)).to eq(2.0) end end @@ -101,10 +105,12 @@ module Qernel expect(scoped.respond_to?(:other_ghg=)).to be true end - it 'returns false for setter methods where the key does not exist in dataset' do - expect(scoped.respond_to?(:invalid_key=)).to be false - expect(scoped.respond_to?(:co2=)).to be false # co2 doesn't exist for this scope - expect(scoped.respond_to?(:arbitrary_name=)).to be false + it 'returns true for setter methods when scope exists in dataset' do + # Setters are allowed for any GHG type if the scope exists + # This enables UPDATE operations to set runtime values + expect(scoped.respond_to?(:invalid_key=)).to be true + expect(scoped.respond_to?(:co2=)).to be true + expect(scoped.respond_to?(:arbitrary_name=)).to be true end it 'returns false for getter methods where the key does not exist in dataset' do @@ -116,7 +122,7 @@ module Qernel describe '[]' do before do - emissions[:agriculture_non_specified_energetic_other_ghg] = 123.45 + emissions[:agriculture_non_specified_energetic_other_ghg_2023] = 123.45 end let(:scoped) { emissions.scope(:agriculture_non_specified_energetic) } @@ -135,7 +141,7 @@ module Qernel it 'sets the value' do scoped[:other_ghg] = 999.0 - expect(emissions.dataset_get(:industry_non_specified_energetic_other_ghg)).to eq(999.0) + expect(emissions.dataset_get(:industry_non_specified_energetic_other_ghg_2023)).to eq(999.0) end end @@ -150,20 +156,22 @@ module Qernel let(:scoped) { emissions.scope(:energy_fugitive_emissions_non_energetic) } before do - emissions[:energy_fugitive_emissions_non_energetic_co2] = 0.0 - emissions[:energy_electricity_and_heat_production_energetic_other_ghg] = 0.0 - emissions[:buildings_non_specified_energetic_other_ghg] = 0.0 + emissions[:energy_fugitive_emissions_non_energetic_co2_2023] = 0.0 + emissions[:energy_electricity_and_heat_production_energetic_other_ghg_2023] = 0.0 + emissions[:buildings_non_specified_energetic_other_ghg_2023] = 0.0 + emissions[:agriculture_non_specified_energetic_other_ghg_2023] = 0.0 + emissions[:agriculture_non_specified_non_energetic_co2_2023] = 0.0 end it 'handles zero values' do scoped.co2 = 0.0 - expect(emissions.dataset_get(:energy_fugitive_emissions_non_energetic_co2)).to eq(0.0) + expect(emissions.dataset_get(:energy_fugitive_emissions_non_energetic_co2_2023)).to eq(0.0) end it 'handles large values' do buildings_scoped = emissions.scope(:buildings_non_specified_energetic) buildings_scoped.other_ghg = 9999999.0 - expect(emissions.dataset_get(:buildings_non_specified_energetic_other_ghg)).to eq(9999999.0) + expect(emissions.dataset_get(:buildings_non_specified_energetic_other_ghg_2023)).to eq(9999999.0) end it 'handles multi-word subsector scopes' do @@ -171,23 +179,120 @@ module Qernel # Scope: energy_electricity_and_heat_production_energetic multi_scoped = emissions.scope(:energy_electricity_and_heat_production_energetic) multi_scoped.other_ghg = 275.0 - expect(emissions.dataset_get(:energy_electricity_and_heat_production_energetic_other_ghg)).to eq(275.0) + expect(emissions.dataset_get(:energy_electricity_and_heat_production_energetic_other_ghg_2023)).to eq(275.0) end it 'works with multi-part keys from real dataset' do scoped.co2 = 100.0 - expect(emissions.dataset_get(:energy_fugitive_emissions_non_energetic_co2)).to eq(100.0) + expect(emissions.dataset_get(:energy_fugitive_emissions_non_energetic_co2_2023)).to eq(100.0) # Test agriculture keys that actually exist in default dataset ag_energetic = emissions.scope(:agriculture_non_specified_energetic) ag_energetic.other_ghg = 200.0 - expect(emissions.dataset_get(:agriculture_non_specified_energetic_other_ghg)).to eq(200.0) + expect(emissions.dataset_get(:agriculture_non_specified_energetic_other_ghg_2023)).to eq(200.0) ag_non_energetic = emissions.scope(:agriculture_non_specified_non_energetic) ag_non_energetic.co2 = 300.0 - expect(emissions.dataset_get(:agriculture_non_specified_non_energetic_co2)).to eq(300.0) + expect(emissions.dataset_get(:agriculture_non_specified_non_energetic_co2_2023)).to eq(300.0) end end end + + describe '#sum' do + let(:graph) { double('Graph', area: area) } + let(:area) { double('Area', analysis_year: 2023) } + let(:emissions) { Emissions.new(graph) } + + before do + emissions.with({ + energy_electricity_and_heat_production_energetic_other_ghg_2023: 18.0, + energy_electricity_and_heat_production_energetic_other_ghg_1990: 25.0, + energy_fugitive_emissions_non_energetic_co2_2023: 20.0, + energy_fugitive_emissions_non_energetic_co2_1990: 30.0, + agriculture_non_specified_energetic_other_ghg_2023: 50.0, + agriculture_non_specified_energetic_other_ghg_1990: 75.0, + agriculture_non_specified_non_energetic_co2_2023: 75.0, + agriculture_non_specified_non_energetic_co2_1990: 100.0, + industry_steel_non_energetic_co2_2023: 100.0, + industry_chemicals_non_energetic_co2_2023: 50.0, + buildings_non_specified_energetic_other_ghg_2023: 2796620.0, + buildings_non_specified_energetic_other_ghg_1990: 3000000.0 + }) + end + + it 'sums emissions across multiple subsectors for a sector' do + # Energy has two subsectors: electricity (18.0) + fugitive (20.0) for 2023 + # But they have different use types, so this tests non_energetic only + result = emissions.sum(:energy, :non_energetic, :co2, 2023) + expect(result).to eq(20.0) # Only fugitive has non_energetic co2 + end + + it 'aggregates energy sector with energetic other_ghg for default year' do + # Energy has electricity (18.0) for energetic other_ghg in 2023 + result = emissions.sum(:energy, :energetic, :other_ghg) + expect(result).to eq(18.0) + end + + it 'sums emissions for 1990 year when specified' do + # Energy fugitive: 30.0 (1990) for non_energetic co2 + result = emissions.sum(:energy, :non_energetic, :co2, 1990) + expect(result).to eq(30.0) + end + + it 'handles single subsector aggregation' do + # Agriculture only has one subsector (Non-specified) + result = emissions.sum(:agriculture, :energetic, :other_ghg, 2023) + expect(result).to eq(50.0) + end + + it 'sums multiple subsectors in industry' do + # Industry has steel (100.0) + chemicals (50.0) = 150.0 + result = emissions.sum(:industry, :non_energetic, :co2, 2023) + expect(result).to eq(150.0) + end + + it 'returns 0 when no matches are found' do + result = emissions.sum(:nonexistent_sector, :energetic, :co2, 2023) + expect(result).to eq(0) + end + + it 'normalizes sector names with dashes' do + # Test that dashes are converted to underscores + result = emissions.sum(:'energy', :non_energetic, :co2, 2023) + expect(result).to eq(20.0) + end + + it 'includes runtime UPDATE modifications in sum' do + # Modify a value via UPDATE + emissions[:energy_fugitive_emissions_non_energetic_co2_2023] = 50.0 + + result = emissions.sum(:energy, :non_energetic, :co2, 2023) + expect(result).to eq(50.0) # Modified value + end + + it 'uses analysis_year when year parameter is not provided' do + allow(area).to receive(:analysis_year).and_return(1990) + + result = emissions.sum(:energy, :energetic, :other_ghg) + expect(result).to eq(25.0) # 1990 value + end + + it 'handles nil values gracefully' do + # Set a value to nil + emissions[:buildings_non_specified_energetic_other_ghg_2023] = nil + + result = emissions.sum(:buildings, :energetic, :other_ghg, 2023) + expect(result).to eq(0) + end + + it 'sums multiple values correctly' do + # Agriculture has both energetic (50.0) and non_energetic (75.0) for co2/other_ghg + energetic = emissions.sum(:agriculture, :energetic, :other_ghg, 2023) + non_energetic = emissions.sum(:agriculture, :non_energetic, :co2, 2023) + + expect(energetic).to eq(50.0) + expect(non_energetic).to eq(75.0) + end + end end end From 15755f7507754faa91efe4e09421bf36c5db4882 Mon Sep 17 00:00:00 2001 From: louispt1 Date: Mon, 8 Jun 2026 10:50:31 +0200 Subject: [PATCH 03/23] Remove the crt mapping logic for now --- app/models/gql/runtime/functions/lookup.rb | 95 ------------ spec/fixtures/etsource/crt_etm_mapping.csv | 27 ---- spec/fixtures/etsource/crt_mapping.csv | 5 - .../gql/runtime/functions/lookup_spec.rb | 136 ------------------ 4 files changed, 263 deletions(-) delete mode 100644 spec/fixtures/etsource/crt_etm_mapping.csv delete mode 100644 spec/fixtures/etsource/crt_mapping.csv diff --git a/app/models/gql/runtime/functions/lookup.rb b/app/models/gql/runtime/functions/lookup.rb index 0de51bd83..09fd2b36c 100644 --- a/app/models/gql/runtime/functions/lookup.rb +++ b/app/models/gql/runtime/functions/lookup.rb @@ -368,101 +368,6 @@ def EMISSIONS(*keys) # EMISSIONS(sector, use, ghg [, year]) -> aggregate and return value scope.graph.emissions.sum(*keys) end - - # Returns an Array of {Qernel::Node} matching a CRT/IPCC code. - # - # Looks up the CRT code in the generic crt_mapping.csv file to find the - # corresponding ETM sector, subsector, and use. Then returns all nodes - # (from both energy and molecule graphs) that match those attributes. - # - # No crt_code attribute is needed on nodes - matching is inferred from - # the mapping based on node sector, key (containing subsector), and use. - # - # crt_code - String or Symbol CRT code (e.g., '1.A.2', '2.B.10.a') - # - # Examples - # - # EMISSIONS_MAP('1.A.4.a') - # # => [, , ...] - # - # SUM(V(EMISSIONS_MAP('1.A.4.a'), demand)) - # # => Sum demand from all nodes matching CRT code 1.A.4.a - # - # UPDATE(EMISSIONS_MAP('1.A.4.a'), some_attribute, value) - # # => Update all matching nodes - # - # Returns an Array of Qernel::Nodes, or empty array if no mapping found. - def EMISSIONS_MAP(crt_code) - normalized_code = normalize_crt_code(crt_code) - mapping = load_crt_mapping(normalized_code) - - return [] unless mapping - - find_nodes_by_mapping(mapping) - end - - private - - # Internal: Normalizes a string for key matching. - # Converts to lowercase, replaces spaces/hyphens with underscores. - # - # Returns a String. - def normalize_key(str) - str.to_s.downcase.tr(' -', '_') - end - - # Internal: Normalizes CRT code for lookup. - # Converts dots/hyphens to underscores, lowercase. - # - # Returns a Symbol. - def normalize_crt_code(code) - code.to_s.downcase.tr('-.', '_').to_sym - end - - # Internal: Loads CRT mapping from dataset. - # - # Returns a CSV::Row or nil. - def load_crt_mapping(normalized_code) - dataset = Atlas::Dataset.find(scope.graph.area.area_code) - dataset.crt_mapping[normalized_code] - end - - # Internal: Finds all nodes matching the CRT mapping. - # - # Returns an Array of Qernel::Nodes. - def find_nodes_by_mapping(mapping) - sector = normalize_key(mapping[:etm_sector]) - subsector = normalize_key(mapping[:etm_subsector]) - use = normalize_key(mapping[:use]) - - all_nodes.select { |node| node_matches_mapping?(node, sector, subsector, use) } - end - - # Internal: Returns all nodes from both graphs. - # - # Returns an Array of Qernel::Nodes. - def all_nodes - scope.all_energy_nodes + scope.all_molecule_nodes - end - - # Internal: Checks if a node matches the mapping criteria. - # - # Returns Boolean. - def node_matches_mapping?(node, sector, subsector, use) - return false unless node.sector_key.to_s == sector - return false unless subsector_matches?(node, subsector) - return false unless node.use_key.to_s == use - - true - end - - # Internal: Checks if subsector matches node key. - # Expects the subsector to be included in the node key. - # - # Returns Boolean. - def subsector_matches?(node, subsector) - node.key.to_s.include?(subsector) - end end end end diff --git a/spec/fixtures/etsource/crt_etm_mapping.csv b/spec/fixtures/etsource/crt_etm_mapping.csv deleted file mode 100644 index ae601684e..000000000 --- a/spec/fixtures/etsource/crt_etm_mapping.csv +++ /dev/null @@ -1,27 +0,0 @@ -crt_code,etm_sector,etm_subsector,use -1.A.1,Energy,Electricity and heat production,energetic -1.A.1.a,Energy,Electricity and heat production,energetic -1.A.1.b,Industry,Refineries,energetic -1.A.1.c,Energy,Fuels production,energetic -1.A.2,Industry,Non-specified,energetic -1.A.3.a,National transport,Non-specified,energetic -1.A.4.a,Buildings,Non-specified,energetic -1.A.4.b,Households,Non-specified,energetic -1.A.4.c.i,Agriculture,Non-specified,energetic -1.A.5,Other,Non-specified,energetic -1.B.1,Energy,Fugitive emissions,non_energetic -1.B.2.a.i,Energy,Fugitive emissions,non_energetic -1.C,Energy,CCUS,non_energetic -2.A,Industry,Other,non_energetic -2.B.1,Industry,Fertilizers,non_energetic -2.B.3,Industry,Chemicals,non_energetic -2.B.8.a,Energy,Methanol production,non_energetic -2.B.10.a,Energy,Hydrogen production,non_energetic -2.C.1,Industry,Steel,non_energetic -2.C.3,Industry,Aluminium,non_energetic -3,Agriculture,Non-specified,non_energetic -4,LULUCF,Non-specified,non_energetic -5,Waste,Non-specified,non_energetic -ind_CO2,Other,Indirect emissions,non_energetic -1.D.1.a,International transport,International aviation,energetic -1.D.1.b,International transport,International navigation,energetic diff --git a/spec/fixtures/etsource/crt_mapping.csv b/spec/fixtures/etsource/crt_mapping.csv deleted file mode 100644 index 6f01b0f01..000000000 --- a/spec/fixtures/etsource/crt_mapping.csv +++ /dev/null @@ -1,5 +0,0 @@ -crt_code,etm_sector,etm_subsector,use -1.A.1,Energy,Electricity and heat production,energetic -1.A.2,Industry,Non-specified,energetic -1.A.4.a,Buildings,Non-specified,energetic -1.A.4.b,Households,Non-specified,energetic diff --git a/spec/models/gql/runtime/functions/lookup_spec.rb b/spec/models/gql/runtime/functions/lookup_spec.rb index 280e4f3a0..be46f6f10 100644 --- a/spec/models/gql/runtime/functions/lookup_spec.rb +++ b/spec/models/gql/runtime/functions/lookup_spec.rb @@ -86,141 +86,5 @@ module Gql::Runtime::Functions expect(result).to eq(50.0) end end - - # EMISSIONS_MAP - # ------------- - - describe 'EMISSIONS_MAP("1.A.1")' do - it 'returns an array of nodes matching the CRT code' do - expect(result).to be_an(Array) - expect(result).to all(be_a(Qernel::Node)) - end - end - - describe 'EMISSIONS_MAP("nonexistent")' do - it 'returns an empty array for non-existent CRT codes' do - expect(result).to eq([]) - end - end - - # Test various CRT code formats - describe 'EMISSIONS_MAP("1-A-1")' do - it 'handles CRT codes with hyphens' do - expect(result).to be_an(Array) - # Should normalize to same as "1.A.1" - end - end - - describe 'EMISSIONS_MAP("1_a_1")' do - it 'handles CRT codes with underscores and lowercase' do - expect(result).to be_an(Array) - # Should normalize to same as "1.A.1" - end - end - - # Test when mapping exists but no nodes match - context 'when CRT mapping exists but no nodes match' do - let(:mock_mapping) do - { - etm_sector: 'nonexistent_sector', - etm_subsector: 'nonexistent_subsector', - use: 'energetic' - } - end - - before do - dataset = Atlas::Dataset.find(gql.future_graph.area.area_code) - allow(dataset.crt_mapping).to receive(:[]).with(:test_code).and_return(mock_mapping) - end - - describe 'EMISSIONS_MAP("test_code")' do - it 'returns an empty array' do - expect(result).to eq([]) - end - end - end - - # Test non_specified subsector special case - context 'when subsector is non_specified' do - let(:mock_mapping) do - { - etm_sector: 'buildings', - etm_subsector: 'non_specified', - use: 'energetic' - } - end - - before do - dataset = Atlas::Dataset.find(gql.future_graph.area.area_code) - allow(dataset.crt_mapping).to receive(:[]).with(:test_nonspec).and_return(mock_mapping) - end - - describe 'EMISSIONS_MAP("test_nonspec")' do - it 'matches nodes without requiring subsector in key' do - expect(result).to be_an(Array) - # Should match all buildings sector nodes with energetic use - # regardless of whether key contains 'non_specified' - result.each do |node| - expect(node.sector_key).to eq(:buildings) - expect(node.use_key).to eq(:energetic) - end - end - end - end - - # Test matching across both energy and molecule graphs - context 'with nodes in both energy and molecule graphs' do - describe 'EMISSIONS_MAP("1.A.1")' do - it 'searches both energy and molecule graphs' do - # The function should search both graphs (implementation verified) - # Result may be empty if no matching nodes in test fixtures - expect(result).to be_an(Array) - - # If there are results, verify they come from one or both graphs - if result.any? - energy_nodes = result.select { |n| gql.future_graph.nodes.include?(n) } - molecule_nodes = result.select { |n| gql.future.molecule_graph.nodes.include?(n) } - - expect(energy_nodes.any? || molecule_nodes.any?).to be true - end - end - end - end - - # Test node attribute matching - context 'when nodes have matching attributes' do - describe 'EMISSIONS_MAP("1.A.1")' do - it 'matches nodes based on sector, subsector in key, and use' do - skip 'if no nodes match' if result.empty? - - result.each do |node| - # All returned nodes should have matching attributes - expect(node).to respond_to(:sector_key) - expect(node).to respond_to(:key) - expect(node).to respond_to(:use_key) - end - end - end - end - - # Integration test with real CRT code - context 'with real emissions data' do - describe 'EMISSIONS_MAP("1.A.4.a")' do - it 'returns nodes for commercial/institutional combustion' do - expect(result).to be_an(Array) - # Should match services sector nodes if they exist in the dataset - end - end - end - - # Edge case: nil/missing attributes - context 'when nodes might have missing attributes' do - describe 'EMISSIONS_MAP("1.A.1")' do - it 'handles nodes without sector gracefully' do - # Should not raise error, just skip nodes without required attributes - expect { result }.not_to raise_error - end - end - end end end From 4b233c105838601f6d29627ed9b4c6acad2a165b Mon Sep 17 00:00:00 2001 From: louispt1 Date: Mon, 8 Jun 2026 12:00:49 +0200 Subject: [PATCH 04/23] Refine permissiveness of getters + setters for qernel::emissions --- app/models/qernel/emissions.rb | 27 ++++++++++++------- .../etsource/datasets/nl/emissions.csv | 8 ------ 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/app/models/qernel/emissions.rb b/app/models/qernel/emissions.rb index 86aed5977..671691564 100644 --- a/app/models/qernel/emissions.rb +++ b/app/models/qernel/emissions.rb @@ -67,8 +67,8 @@ def respond_to_missing?(method_name, include_private = false) clean_method = method_name.to_s.delete_suffix('=') data_key = scoped_method(clean_method).to_sym - # Setters are allowed if the scope exists (at least one key with this scope prefix) - # Getters require the exact key to exist in the dataset + # Setters: validate scope exists (allows runtime values for valid scopes) + # Getters: require exact key to exist (strict validation) if method_name.to_s.end_with?('=') scope_exists? else @@ -81,15 +81,14 @@ def method_missing(method_name, *args) clean_method = method_name.to_s.delete_suffix('=') data_key = scoped_method(clean_method).to_sym - # Setter if method name ended with '=' + # Setter: validate scope exists (allows runtime values for valid scopes) + # Getter: require exact key to exist (strict validation) if method_name.to_s.end_with?('=') - # Validate that the scope exists (at least one key with this scope prefix) unless scope_exists? raise NoMethodError, "undefined method `#{method_name}' for #{inspect}" end @emissions[data_key] = args.first else - # Getter - validate the key exists unless @emissions.dataset_attributes&.key?(data_key) raise NoMethodError, "undefined method `#{method_name}' for #{inspect}" end @@ -99,8 +98,11 @@ def method_missing(method_name, *args) private - # Check if at least one key exists with the current scope prefix - # (validates the scope exists in the dataset, allowing any GHG/year combination) + # Validates that the scope (sector_subsector_use) exists in the dataset. + # This allows UPDATE operations to set runtime values for valid scopes, + # while preventing completely invalid scopes like 'invalid_sector_energetic'. + # + # Returns true if at least one key matching the scope prefix exists. def scope_exists? prefix = "#{@scope}_" @@ -156,9 +158,14 @@ def sum(sector, use, ghg, year = nil) prefix = sector.to_s.tr('-.', '_').downcase suffix = "_#{use}_#{ghg}_#{year}" - dataset_attributes.keys.select { |key| - key.to_s.start_with?(prefix) && key.to_s.end_with?(suffix) - }.sum { |key| dataset_get(key) || 0 } + dataset_attributes.keys.reduce(0) do |total, key| + key_str = key.to_s + if key_str.start_with?(prefix) && key_str.end_with?(suffix) + total + (dataset_get(key) || 0) + else + total + end + end end # Public: define the sector scope for access to the hashed emission keys diff --git a/spec/fixtures/etsource/datasets/nl/emissions.csv b/spec/fixtures/etsource/datasets/nl/emissions.csv index b1236ad06..87c5ae941 100644 --- a/spec/fixtures/etsource/datasets/nl/emissions.csv +++ b/spec/fixtures/etsource/datasets/nl/emissions.csv @@ -1,25 +1,17 @@ etm_sector,etm_subsector,use,ghg,year,unit,value Energy,Electricity and heat production,energetic,other_ghg,2023,kg CO2eq,18.0 -Energy,Electricity and heat production,energetic,other_ghg,2023,kg CO2eq,18.0 Energy,Electricity and heat production,energetic,other_ghg,1990,kg CO2eq,25.0 Energy,Fugitive emissions,non_energetic,co2,2023,kg CO2eq,20.0 -Energy,Fugitive emissions,non_energetic,co2,2023,kg CO2eq,20.0 Energy,Fugitive emissions,non_energetic,co2,1990,kg CO2eq,30.0 Households,Non-specified,energetic,other_ghg,2023,kg CO2eq,7.0 -Households,Non-specified,energetic,other_ghg,2023,kg CO2eq,7.0 Households,Non-specified,energetic,other_ghg,1990,kg CO2eq,10.0 Buildings,Non-specified,energetic,other_ghg,2023,kg CO2eq,2796620.0 -Buildings,Non-specified,energetic,other_ghg,2023,kg CO2eq,2796620.0 Buildings,Non-specified,energetic,other_ghg,1990,kg CO2eq,3000000.0 Industry,Non-specified,energetic,other_ghg,2023,kg CO2eq,100.0 -Industry,Non-specified,energetic,other_ghg,2023,kg CO2eq,100.0 Industry,Non-specified,energetic,other_ghg,1990,kg CO2eq,150.0 Agriculture,Non-specified,energetic,other_ghg,2023,kg CO2eq,50.0 -Agriculture,Non-specified,energetic,other_ghg,2023,kg CO2eq,50.0 Agriculture,Non-specified,energetic,other_ghg,1990,kg CO2eq,75.0 Agriculture,Non-specified,non_energetic,co2,2023,kg CO2eq,75.0 -Agriculture,Non-specified,non_energetic,co2,2023,kg CO2eq,75.0 Agriculture,Non-specified,non_energetic,co2,1990,kg CO2eq,100.0 Waste,Non-specified,non_energetic,co2,2023,kg CO2eq,25.0 -Waste,Non-specified,non_energetic,co2,2023,kg CO2eq,25.0 Waste,Non-specified,non_energetic,co2,1990,kg CO2eq,35.0 From e1f18925b4ae4b99e825f754d41c6617622429a1 Mon Sep 17 00:00:00 2001 From: louispt1 Date: Mon, 8 Jun 2026 15:19:45 +0200 Subject: [PATCH 05/23] Minor improvements --- app/models/gql/runtime/functions/lookup.rb | 13 +++++++++++-- app/models/qernel/emissions.rb | 10 ++++++++-- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/app/models/gql/runtime/functions/lookup.rb b/app/models/gql/runtime/functions/lookup.rb index 09fd2b36c..b749fe7d1 100644 --- a/app/models/gql/runtime/functions/lookup.rb +++ b/app/models/gql/runtime/functions/lookup.rb @@ -365,8 +365,17 @@ def EMISSIONS(*keys) # EMISSIONS(sector, use) -> return ScopedSector for UPDATE operations return scope.graph.emissions.scope(keys.join('_').to_sym) if keys.size == 2 - # EMISSIONS(sector, use, ghg [, year]) -> aggregate and return value - scope.graph.emissions.sum(*keys) + sector, use, ghg, year = keys + year ||= scope.graph.area.analysis_year + + # Build expected key for direct lookup + full_key = "#{sector}_#{use}_#{ghg}_#{year}".to_sym + + # Try direct lookup first (common case: single subsector) + value = scope.graph.emissions[full_key] + + # If not found, aggregate across matching subsectors + value.nil? ? scope.graph.emissions.sum(*keys) : value end end end diff --git a/app/models/qernel/emissions.rb b/app/models/qernel/emissions.rb index 671691564..0c2c005d7 100644 --- a/app/models/qernel/emissions.rb +++ b/app/models/qernel/emissions.rb @@ -58,7 +58,7 @@ def inspect end def scoped_method(method_name) - year = @year || @emissions.graph&.area&.analysis_year + year = @year || @emissions.default_year "#{@scope}_#{method_name}_#{year}" end @@ -154,7 +154,7 @@ def initialize(graph = nil) # Float sum of all matching emissions. Returns 0 if no matches found. # Includes any runtime UPDATE modifications to emission values. def sum(sector, use, ghg, year = nil) - year ||= graph&.area&.analysis_year + year ||= default_year prefix = sector.to_s.tr('-.', '_').downcase suffix = "_#{use}_#{ghg}_#{year}" @@ -177,5 +177,11 @@ def sum(sector, use, ghg, year = nil) def scope(sector, year = nil) ScopedSector.new(self, sector, year) end + + # Returns the default year for emissions queries. + # Uses the area's analysis_year if graph is present, nil otherwise. + def default_year + graph&.area.analysis_year + end end end From 50d14d6e382229b8cdcbeb96803cad7702a66e35 Mon Sep 17 00:00:00 2001 From: louispt1 Date: Tue, 9 Jun 2026 17:45:59 +0200 Subject: [PATCH 06/23] Workaround for non_energetic vs energetic use in the sum function --- app/models/qernel/emissions.rb | 28 ++++++++++--------- .../etsource/datasets/nl/emissions.csv | 2 ++ .../gql/runtime/functions/lookup_spec.rb | 14 ++++++++++ spec/models/qernel/emissions_spec.rb | 25 +++++++++++++++++ 4 files changed, 56 insertions(+), 13 deletions(-) diff --git a/app/models/qernel/emissions.rb b/app/models/qernel/emissions.rb index 0c2c005d7..02a168883 100644 --- a/app/models/qernel/emissions.rb +++ b/app/models/qernel/emissions.rb @@ -153,20 +153,22 @@ def initialize(graph = nil) # # Float sum of all matching emissions. Returns 0 if no matches found. # Includes any runtime UPDATE modifications to emission values. - def sum(sector, use, ghg, year = nil) - year ||= default_year - prefix = sector.to_s.tr('-.', '_').downcase - suffix = "_#{use}_#{ghg}_#{year}" - - dataset_attributes.keys.reduce(0) do |total, key| - key_str = key.to_s - if key_str.start_with?(prefix) && key_str.end_with?(suffix) - total + (dataset_get(key) || 0) - else - total - end - end + def sum(sector, use, ghg, year = nil) + year ||= default_year + prefix = sector.to_s.tr('-.', '_').downcase + use_str = use.to_s + ghg_str = ghg.to_s + year_str = year.to_s + + # Build regex pattern to match keys with exact component boundaries. + # When searching for "energetic", we must not match "non_energetic". + # Negative lookbehind (? Date: Wed, 10 Jun 2026 10:22:53 +0200 Subject: [PATCH 07/23] Make regex lookbehind more explicit --- app/models/qernel/emissions.rb | 37 ++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/app/models/qernel/emissions.rb b/app/models/qernel/emissions.rb index 02a168883..6d429b17f 100644 --- a/app/models/qernel/emissions.rb +++ b/app/models/qernel/emissions.rb @@ -153,22 +153,29 @@ def initialize(graph = nil) # # Float sum of all matching emissions. Returns 0 if no matches found. # Includes any runtime UPDATE modifications to emission values. - def sum(sector, use, ghg, year = nil) - year ||= default_year - prefix = sector.to_s.tr('-.', '_').downcase - use_str = use.to_s - ghg_str = ghg.to_s - year_str = year.to_s - - # Build regex pattern to match keys with exact component boundaries. - # When searching for "energetic", we must not match "non_energetic". - # Negative lookbehind (? Date: Wed, 10 Jun 2026 11:28:08 +0200 Subject: [PATCH 08/23] builds a structured emissions_index alongside the flat value hash to preserve column boundaries whic are lost when normalising keys --- app/models/etsource/dataset.rb | 8 - app/models/etsource/dataset/import.rb | 30 ++- app/models/gql/runtime/functions/lookup.rb | 20 +- app/models/qernel/emissions.rb | 161 +++++++-------- spec/models/etsource/dataset/import_spec.rb | 37 ++++ .../gql/runtime/functions/lookup_spec.rb | 6 +- spec/models/qernel/emissions_spec.rb | 191 ++++++++++-------- 7 files changed, 251 insertions(+), 202 deletions(-) create mode 100644 spec/models/etsource/dataset/import_spec.rb diff --git a/app/models/etsource/dataset.rb b/app/models/etsource/dataset.rb index 54adb0335..20e3616ab 100644 --- a/app/models/etsource/dataset.rb +++ b/app/models/etsource/dataset.rb @@ -48,13 +48,5 @@ def self.region_codes(refresh: false) .map { |dataset| dataset.key.to_s } end end - - def self.emissions_keys - NastyCache.instance.fetch('emission_keys') do - Atlas::Dataset.find( - Etsource::Config.default_dataset_key - ).emissions.to_hash.keys - end - end end end diff --git a/app/models/etsource/dataset/import.rb b/app/models/etsource/dataset/import.rb index 9d39bd06f..5d92c5382 100644 --- a/app/models/etsource/dataset/import.rb +++ b/app/models/etsource/dataset/import.rb @@ -117,9 +117,35 @@ def load_region_data # Internal: Loads the emission data. # - # Returns a hash nested under emissions_data key for DatasetAttributes compatibility. + # Returns a hash nested under keys for DatasetAttributes compatibility. def load_emission_data - { emissions_data: @atlas_ds.emissions.to_hash } + { + emissions_data: @atlas_ds.emissions.to_hash, + emissions_index: build_emissions_index + } + end + + # Internal: Groups emission keys by their CSV column components. + # + # Returns a hash with :sectors (sector => subsector keys), :subsectors + # (subsector key => true), :scopes (sector_subsector_use => true) and + # :ghgs (ghg => true). + def build_emissions_index + index = { sectors: {}, subsectors: {}, scopes: {}, ghgs: {} } + + @atlas_ds.emissions.component_keys.each_value do |components| + *subsector_parts, use, ghg, _year = components + subsector = subsector_parts.join('_').to_sym + + sectors = (index[:sectors][components.first] ||= []) + sectors << subsector unless sectors.include?(subsector) + + index[:subsectors][subsector] = true + index[:scopes][:"#{subsector}_#{use}"] = true + index[:ghgs][ghg] = true + end + + index end # Internal: Loads the carrier data. diff --git a/app/models/gql/runtime/functions/lookup.rb b/app/models/gql/runtime/functions/lookup.rb index b749fe7d1..2c73b2840 100644 --- a/app/models/gql/runtime/functions/lookup.rb +++ b/app/models/gql/runtime/functions/lookup.rb @@ -321,16 +321,16 @@ def DATASET_CURVE(key) # Returns an attribute {Qernel::Emissions} or {Qernel::Emissions::ScopedSector} or a value. # # Emissions data is loaded from CSV files in ETSource with the following structure: - # etm_sector, etm_subsector, use, ghg, unit, value + # etm_sector, etm_subsector, use, ghg, year, unit, value # # Parameters: # - sector: ETM sector name (e.g., 'households', 'buildings_non_specified') # Dashes/dots in sector names are converted to underscores for key generation # - use: Emission use type (energetic, non_energetic) - REQUIRED when accessing values # - ghg: GHG type (co2, other_ghg) - optional - # - year: Year of emission (e.g., 1990) - optional, reads from emissions_YEAR.csv files + # - year: Year of emission (e.g., 1990) - optional, defaults to the analysis year # - # Key generation combines: sector_[subsector_]use_ghg[_year] + # Key generation combines: sector_[subsector_]use_ghg_year # Note: Unit column from CSV is not included in keys, blank values return nil # # EMISSIONS() without any keys returns {Qernel::Emissions} @@ -365,17 +365,9 @@ def EMISSIONS(*keys) # EMISSIONS(sector, use) -> return ScopedSector for UPDATE operations return scope.graph.emissions.scope(keys.join('_').to_sym) if keys.size == 2 - sector, use, ghg, year = keys - year ||= scope.graph.area.analysis_year - - # Build expected key for direct lookup - full_key = "#{sector}_#{use}_#{ghg}_#{year}".to_sym - - # Try direct lookup first (common case: single subsector) - value = scope.graph.emissions[full_key] - - # If not found, aggregate across matching subsectors - value.nil? ? scope.graph.emissions.sum(*keys) : value + # EMISSIONS(sector, use, ghg [, year]) -> value; a full subsector key + # sums a single entry, a sector key sums all of its subsectors. + scope.graph.emissions.sum(*keys) end end end diff --git a/app/models/qernel/emissions.rb b/app/models/qernel/emissions.rb index 6d429b17f..037d3b093 100644 --- a/app/models/qernel/emissions.rb +++ b/app/models/qernel/emissions.rb @@ -8,26 +8,25 @@ module Qernel # Emissions data is loaded from CSV files in ETSource with structure: # etm_sector, etm_subsector, use, ghg, year, unit, value # - # Keys are generated as: sector_subsector_use_ghg_year + # Values are stored flat, keyed as: sector_subsector_use_ghg_year # Examples: # - buildings_non_specified_energetic_other_ghg_2023 # - energy_electricity_and_heat_production_energetic_co2_1990 # - # == Aggregation via sum() - # - # The sum() method aggregates emissions across subsectors for a given sector, - # use, ghg, and year combination. This allows queries like: - # EMISSIONS(energy, energetic, co2, 2023) - # to return the sum of all energy subsectors' energetic CO2 emissions for 2023. + # Joining the CSV columns into one key loses the column boundaries + # (components such as "non_energetic" contain the separator), so the + # import also builds an index preserving the structure (see + # Etsource::Dataset::Import#build_emissions_index). Lookups construct + # exact keys from the index rather than parsing them. # # == Year Handling # - # Year parameter defaults to the dataset's analysis_year when not specified. - # Multiple years can coexist in the same dataset (e.g., 1990 baseline, 2023 current). + # Year parameter defaults to the dataset's analysis_year when not + # specified. Multiple years coexist in the same dataset (e.g., 1990 + # baseline, 2023 current). class Emissions include DatasetAttributes - dataset_accessors ::Etsource::Dataset.emissions_keys attr_accessor :graph # Queryable object that provides scoped access to emissions data @@ -36,10 +35,14 @@ class Emissions # statements in etsource # # Example: - # EMISSIONS(households, energetic) returns a ScopedSector + # EMISSIONS(households_non_specified, energetic) returns a ScopedSector # Then UPDATE can call: scoped.co2 = 100.0 class ScopedSector def initialize(emissions, scope, year = nil) + unless emissions.valid_scope?(scope) + raise ArgumentError, "unknown emissions scope: #{scope}" + end + @emissions = emissions @scope = scope @year = year @@ -62,55 +65,27 @@ def scoped_method(method_name) "#{@scope}_#{method_name}_#{year}" end - def respond_to_missing?(method_name, include_private = false) - # Remove '=' suffix if present before generating scoped key - clean_method = method_name.to_s.delete_suffix('=') - data_key = scoped_method(clean_method).to_sym + # Getters and setters for each GHG in the dataset (e.g. scoped.co2, + # scoped.co2 = 1.0), delegated to []/[]=. The GHG set comes from the + # emissions index, so a new gas in the CSV needs no code change here. + def method_missing(name, *args) + return super unless ghg?(name) - # Setters: validate scope exists (allows runtime values for valid scopes) - # Getters: require exact key to exist (strict validation) - if method_name.to_s.end_with?('=') - scope_exists? + if name.end_with?('=') + self[name.to_s.delete_suffix('=')] = args.first else - @emissions.dataset_attributes&.key?(data_key) || super + self[name] end end - def method_missing(method_name, *args) - # Remove '=' suffix if present before generating scoped key - clean_method = method_name.to_s.delete_suffix('=') - data_key = scoped_method(clean_method).to_sym - - # Setter: validate scope exists (allows runtime values for valid scopes) - # Getter: require exact key to exist (strict validation) - if method_name.to_s.end_with?('=') - unless scope_exists? - raise NoMethodError, "undefined method `#{method_name}' for #{inspect}" - end - @emissions[data_key] = args.first - else - unless @emissions.dataset_attributes&.key?(data_key) - raise NoMethodError, "undefined method `#{method_name}' for #{inspect}" - end - @emissions[data_key] - end + def respond_to_missing?(name, include_private = false) + ghg?(name) || super end private - # Validates that the scope (sector_subsector_use) exists in the dataset. - # This allows UPDATE operations to set runtime values for valid scopes, - # while preventing completely invalid scopes like 'invalid_sector_energetic'. - # - # Returns true if at least one key matching the scope prefix exists. - def scope_exists? - prefix = "#{@scope}_" - - return false unless @emissions.dataset_attributes - - @emissions.dataset_attributes.keys.any? do |key| - key.to_s.start_with?(prefix) - end + def ghg?(name) + @emissions.ghgs.key?(name.to_s.delete_suffix('=').to_sym) end end @@ -120,77 +95,79 @@ def initialize(graph = nil) @dataset_key = @key = :emissions_data end - # Public: Aggregates emissions across subsectors for a given sector, use, ghg, and year. + # Public: Aggregates emissions for a sector, use, ghg, and year. # - # This method sums all emissions entries that match the specified sector prefix, - # use type, GHG type, and year. It aggregates across all subsectors within the - # specified sector. + # Sums the value of every subsector belonging to the given sector. When + # given a full subsector key instead of a sector, returns its single + # value. Includes runtime UPDATE modifications. # # == Examples # # # Sum all energy sector energetic CO2 emissions for 2023 - # # (aggregates electricity production, fuels production, etc.) # emissions.sum(:energy, :energetic, :co2, 2023) # # => 500.25 # - # # Get single subsector (no aggregation needed, but still uses sum) + # # Single subsector # emissions.sum(:buildings_non_specified, :energetic, :other_ghg, 2023) # # => 55.64 # - # # Use default year (analysis_year from dataset) + # # Default year (analysis_year from dataset) # emissions.sum(:agriculture, :non_energetic, :other_ghg) # # => 18863.47 # - # == Parameters - # - # sector - Sector name or full subsector key (e.g., :energy, :buildings_non_specified) - # Normalized: dashes/dots → underscores, lowercased - # use - Use type (:energetic or :non_energetic) - # ghg - GHG type (:co2 or :other_ghg) - # year - Optional year (Integer). Defaults to graph.area.analysis_year if not specified - # - # == Returns - # - # Float sum of all matching emissions. Returns 0 if no matches found. - # Includes any runtime UPDATE modifications to emission values. + # Returns a Float; 0.0 when nothing matches. def sum(sector, use, ghg, year = nil) year ||= default_year - prefix = sector.to_s.tr('-.', '_').downcase - use_str = use.to_s - ghg_str = ghg.to_s - year_str = year.to_s - - # Match known emission types explicitly to avoid substring collisions. - # Use non-greedy match (.*?) and longest-first alternations to ensure - # "non_energetic" and "other_ghg" are matched correctly. - # This replaces the negative lookbehind with explicit enumeration of valid values. - pattern = /^#{Regexp.escape(prefix)}.*?_(non_energetic|energetic)_(other_ghg|co2)_#{Regexp.escape(year_str)}$/ - - dataset_attributes.keys.sum do |key| - match = key.to_s.match(pattern) - # Verify exact match on use and ghg components - if match && match[1] == use_str && match[2] == ghg_str - dataset_get(key) || 0 - else - 0 - end + + subsector_keys(sector).sum do |subsector| + self[:"#{subsector}_#{use}_#{ghg}_#{year}"] || 0.0 end end # Public: define the sector scope for access to the hashed emission keys # - # sector - Sector identifier (e.g., :buildings_non_specified_energetic) - # year - Optional year (defaults to analysis_year). Used to target specific year for UPDATE operations. + # sector - Scope identifier (e.g., :buildings_non_specified_energetic). + # Must exist in the dataset; raises ArgumentError otherwise. + # year - Optional year (defaults to analysis_year). Used to target a + # specific year for UPDATE operations. # # Returns a scoped version of the emissions data def scope(sector, year = nil) ScopedSector.new(self, sector, year) end + # Public: Whether the scope (sector_subsector_use) exists in the dataset. + def valid_scope?(scope) + index[:scopes].key?(scope.to_sym) + end + + # Public: The GHG types present in the dataset (ghg => true). + def ghgs + index[:ghgs] + end + # Returns the default year for emissions queries. # Uses the area's analysis_year if graph is present, nil otherwise. def default_year - graph&.area.analysis_year + graph&.area&.analysis_year + end + + EMPTY_INDEX = { sectors: {}, subsectors: {}, scopes: {}, ghgs: {} }.freeze + + # The structured emissions index built at import time. + # See Etsource::Dataset::Import#build_emissions_index. + def index + @index ||= + (dataset && dataset.data[:emissions][:emissions_index]) || EMPTY_INDEX + end + + private + + # An exact sector maps to its subsector keys; an exact subsector key + # maps to itself; anything else matches nothing. + def subsector_keys(sector) + key = sector.to_s.tr('-.', '_').downcase.to_sym + index[:sectors][key] || (index[:subsectors].key?(key) ? [key] : []) end end end diff --git a/spec/models/etsource/dataset/import_spec.rb b/spec/models/etsource/dataset/import_spec.rb new file mode 100644 index 000000000..7c340fd1a --- /dev/null +++ b/spec/models/etsource/dataset/import_spec.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Etsource::Dataset::Import, :etsource_fixture do + describe '#load_emission_data' do + let(:emissions) { described_class.new('nl').send(:load_emission_data) } + + it 'loads flat emission values keyed by joined CSV columns' do + expect(emissions[:emissions_data][:energy_fugitive_emissions_non_energetic_co2_2023]) + .to eq(20.0) + end + + it 'indexes sectors to their subsector keys' do + expect(emissions[:emissions_index][:sectors][:energy]).to contain_exactly( + :energy_electricity_and_heat_production, :energy_fugitive_emissions + ) + end + + it 'indexes subsector keys' do + expect(emissions[:emissions_index][:subsectors]) + .to include(:energy_electricity_and_heat_production, :buildings_non_specified) + end + + it 'indexes scopes including the use' do + expect(emissions[:emissions_index][:scopes]).to include( + :energy_fugitive_emissions_non_energetic, + :agriculture_non_specified_energetic, + :agriculture_non_specified_non_energetic + ) + end + + it 'indexes the GHG types' do + expect(emissions[:emissions_index][:ghgs]).to eq(co2: true, other_ghg: true) + end + end +end diff --git a/spec/models/gql/runtime/functions/lookup_spec.rb b/spec/models/gql/runtime/functions/lookup_spec.rb index 31f65dea7..af5c10cea 100644 --- a/spec/models/gql/runtime/functions/lookup_spec.rb +++ b/spec/models/gql/runtime/functions/lookup_spec.rb @@ -94,10 +94,8 @@ module Gql::Runtime::Functions end describe 'EMISSIONS(agriculture_non_, energetic, other_ghg, 2023)' do - it 'uses partial sector matching but exact use matching' do - # Partial sector "agriculture_non_" should match "agriculture_non_specified" - # But use "energetic" should NOT match "non_energetic" - expect(result).to eq(50.0) # Only energetic (50.0), not sum with non_energetic (100.0) + it 'does not match partial sector names' do + expect(result).to eq(0) end end end diff --git a/spec/models/qernel/emissions_spec.rb b/spec/models/qernel/emissions_spec.rb index e4b2d6f45..0e9790100 100644 --- a/spec/models/qernel/emissions_spec.rb +++ b/spec/models/qernel/emissions_spec.rb @@ -2,6 +2,48 @@ module Qernel describe Emissions do + # Builds an index in the shape produced by + # Etsource::Dataset::Import#build_emissions_index from + # [sector, subsector, use] rows. + def self.emissions_index(rows) + index = { sectors: {}, subsectors: {}, scopes: {}, ghgs: { co2: true, other_ghg: true } } + + rows.each do |sector, subsector, use| + key = :"#{sector}_#{subsector}" + sectors = (index[:sectors][sector.to_sym] ||= []) + sectors << key unless sectors.include?(key) + + index[:subsectors][key] = true + index[:scopes][:"#{key}_#{use}"] = true + end + + index + end + + INDEX = emissions_index([ + %w[energy electricity_and_heat_production energetic], + %w[energy fugitive_emissions non_energetic], + %w[households non_specified energetic], + %w[buildings non_specified energetic], + %w[industry steel non_energetic], + %w[industry chemicals non_energetic], + %w[industry non_specified energetic], + %w[agriculture non_specified energetic], + %w[agriculture non_specified non_energetic] + ]).freeze + + let(:area) { double('Area', analysis_year: 2023) } + + let(:dataset) do + Qernel::Dataset.new(1).tap do |ds| + ds.data[:emissions] = { emissions_data: data, emissions_index: INDEX } + end + end + + let(:graph) { double('Graph', area: area, dataset: dataset) } + let(:emissions) { Emissions.new(graph).tap(&:assign_dataset_attributes) } + let(:data) { {} } + describe '#initialize' do context 'with a graph' do let(:graph) { Qernel::Graph.new } @@ -31,23 +73,40 @@ module Qernel end describe '#scope' do - let(:emissions) { Emissions.new.with({}) } - it 'returns a ScopedSector instance' do - scoped = emissions.scope(:households_energetic) + scoped = emissions.scope(:households_non_specified_energetic) expect(scoped).to be_a(Emissions::ScopedSector) end it 'sets the correct scope' do - scoped = emissions.scope(:agriculture_energetic) - expect(scoped.instance_variable_get(:@scope)).to eq(:agriculture_energetic) + scoped = emissions.scope(:agriculture_non_specified_energetic) + expect(scoped.instance_variable_get(:@scope)).to eq(:agriculture_non_specified_energetic) + end + + it 'raises ArgumentError for a scope not present in the dataset' do + expect { emissions.scope(:invalid_sector_energetic) } + .to raise_error(ArgumentError, /unknown emissions scope/) + end + end + + describe '#valid_scope?' do + it 'returns true for a sector_subsector_use combination from the data' do + expect(emissions.valid_scope?(:energy_fugitive_emissions_non_energetic)).to be true + end + + it 'returns false for unknown combinations' do + expect(emissions.valid_scope?(:energy_fugitive_emissions_energetic)).to be false + expect(emissions.valid_scope?(:invalid_sector_energetic)).to be false + end + end + + describe '#ghgs' do + it 'returns the GHG types from the index' do + expect(emissions.ghgs.keys).to contain_exactly(:co2, :other_ghg) end end describe Emissions::ScopedSector do - let(:graph) { double('Graph', area: area) } - let(:area) { double('Area', analysis_year: 2023) } - let(:emissions) { Emissions.new(graph).with({}) } let(:scoped) { emissions.scope(:households_non_specified_energetic) } before do @@ -55,68 +114,58 @@ module Qernel emissions[:agriculture_non_specified_energetic_other_ghg_2023] = 25.0 end - describe '#method_missing' do - it 'delegates getter methods to emissions with scoped prefix' do + describe 'GHG accessors' do + it 'reads values with the scoped prefix' do expect(scoped.other_ghg).to eq(50.0) end - it 'delegates getter for a different scope' do + it 'reads values for a different scope' do other_scoped = emissions.scope(:agriculture_non_specified_energetic) expect(other_scoped.other_ghg).to eq(25.0) end - it 'delegates setter methods to emissions with scoped prefix' do + it 'writes values with the scoped prefix' do scoped.other_ghg = 75.0 - # Setters convert to symbol keys via dataset_set expect(emissions.dataset_get(:households_non_specified_energetic_other_ghg_2023)).to eq(75.0) end - it 'delegates setter for a different scope' do + it 'writes values for a different scope' do other_scoped = emissions.scope(:agriculture_non_specified_energetic) other_scoped.other_ghg = 30.0 expect(emissions.dataset_get(:agriculture_non_specified_energetic_other_ghg_2023)).to eq(30.0) end - it 'raises NoMethodError for undefined getter methods' do - expect { scoped.nonexistent_attribute }.to raise_error(NoMethodError) - expect { scoped.invalid_emission }.to raise_error(NoMethodError) + it 'returns nil when the scope has no value for the GHG' do + expect(scoped.co2).to be_nil end - it 'allows setters for any GHG type when scope exists in dataset' do - # Setters are allowed for any GHG type if the scope (subsector + use) exists - # This enables UPDATE operations to set runtime values - expect { scoped.arbitrary_key = 100.0 }.not_to raise_error - expect { scoped.custom_emission_type = 200.0 }.not_to raise_error - expect { scoped.co2 = 300.0 }.not_to raise_error + it 'allows setting a GHG which has no value yet (runtime UPDATE values)' do + scoped.co2 = 300.0 + expect(emissions.dataset_get(:households_non_specified_energetic_co2_2023)).to eq(300.0) end - it 'allows setters for emission keys that exist in the dataset' do - expect { scoped.other_ghg = 2.0 }.not_to raise_error - expect(emissions.dataset_get(:households_non_specified_energetic_other_ghg_2023)).to eq(2.0) + it 'raises NoMethodError for attributes which are not GHGs' do + expect { scoped.nonexistent_attribute }.to raise_error(NoMethodError) + expect { scoped.arbitrary_key = 100.0 }.to raise_error(NoMethodError) end - end - describe '#respond_to_missing?' do - it 'returns true for valid GHG types that exist' do - expect(scoped.respond_to?(:other_ghg)).to be true + it 'responds to the GHG getters and setters' do + expect(scoped).to respond_to(:co2, :co2=, :other_ghg, :other_ghg=) end - it 'returns true for setter methods where the key exists in dataset' do - expect(scoped.respond_to?(:other_ghg=)).to be true + it 'does not respond to attributes which are not GHGs' do + expect(scoped).not_to respond_to(:arbitrary_key) end + end - it 'returns true for setter methods when scope exists in dataset' do - # Setters are allowed for any GHG type if the scope exists - # This enables UPDATE operations to set runtime values - expect(scoped.respond_to?(:invalid_key=)).to be true - expect(scoped.respond_to?(:co2=)).to be true - expect(scoped.respond_to?(:arbitrary_name=)).to be true - end + describe 'year targeting' do + it 'reads and writes the requested year' do + scoped_1990 = emissions.scope(:households_non_specified_energetic, 1990) + scoped_1990.other_ghg = 12.0 - it 'returns false for getter methods where the key does not exist in dataset' do - expect(scoped.respond_to?(:invalid_key)).to be false - expect(scoped.respond_to?(:co2)).to be false # co2 doesn't exist for this scope - expect(scoped.respond_to?(:nonexistent_attribute)).to be false + expect(emissions.dataset_get(:households_non_specified_energetic_other_ghg_1990)).to eq(12.0) + expect(scoped_1990.other_ghg).to eq(12.0) + expect(scoped.other_ghg).to eq(50.0) end end @@ -147,8 +196,7 @@ module Qernel describe '#inspect' do it 'returns a readable string representation' do - scoped = emissions.scope(:households_energetic) - expect(scoped.inspect).to eq('') + expect(scoped.inspect).to eq('') end end @@ -186,7 +234,6 @@ module Qernel scoped.co2 = 100.0 expect(emissions.dataset_get(:energy_fugitive_emissions_non_energetic_co2_2023)).to eq(100.0) - # Test agriculture keys that actually exist in default dataset ag_energetic = emissions.scope(:agriculture_non_specified_energetic) ag_energetic.other_ghg = 200.0 expect(emissions.dataset_get(:agriculture_non_specified_energetic_other_ghg_2023)).to eq(200.0) @@ -199,12 +246,8 @@ module Qernel end describe '#sum' do - let(:graph) { double('Graph', area: area) } - let(:area) { double('Area', analysis_year: 2023) } - let(:emissions) { Emissions.new(graph) } - - before do - emissions.with({ + let(:data) do + { energy_electricity_and_heat_production_energetic_other_ghg_2023: 18.0, energy_electricity_and_heat_production_energetic_other_ghg_1990: 25.0, energy_fugitive_emissions_non_energetic_co2_2023: 20.0, @@ -217,30 +260,27 @@ module Qernel industry_chemicals_non_energetic_co2_2023: 50.0, buildings_non_specified_energetic_other_ghg_2023: 2796620.0, buildings_non_specified_energetic_other_ghg_1990: 3000000.0 - }) + } end it 'sums emissions across multiple subsectors for a sector' do - # Energy has two subsectors: electricity (18.0) + fugitive (20.0) for 2023 - # But they have different use types, so this tests non_energetic only + # Energy has two subsectors with different use types, so this matches + # non_energetic (fugitive) only. result = emissions.sum(:energy, :non_energetic, :co2, 2023) - expect(result).to eq(20.0) # Only fugitive has non_energetic co2 + expect(result).to eq(20.0) end it 'aggregates energy sector with energetic other_ghg for default year' do - # Energy has electricity (18.0) for energetic other_ghg in 2023 result = emissions.sum(:energy, :energetic, :other_ghg) expect(result).to eq(18.0) end it 'sums emissions for 1990 year when specified' do - # Energy fugitive: 30.0 (1990) for non_energetic co2 result = emissions.sum(:energy, :non_energetic, :co2, 1990) expect(result).to eq(30.0) end it 'handles single subsector aggregation' do - # Agriculture only has one subsector (Non-specified) result = emissions.sum(:agriculture, :energetic, :other_ghg, 2023) expect(result).to eq(50.0) end @@ -251,34 +291,36 @@ module Qernel expect(result).to eq(150.0) end + it 'accepts a full subsector key instead of a sector' do + result = emissions.sum(:buildings_non_specified, :energetic, :other_ghg, 2023) + expect(result).to eq(2796620.0) + end + it 'returns 0 when no matches are found' do result = emissions.sum(:nonexistent_sector, :energetic, :co2, 2023) expect(result).to eq(0) end - it 'normalizes sector names with dashes' do - # Test that dashes are converted to underscores - result = emissions.sum(:'energy', :non_energetic, :co2, 2023) - expect(result).to eq(20.0) + it 'normalizes sector names with dashes and mixed case' do + result = emissions.sum(:'Buildings-Non-Specified', :energetic, :other_ghg, 2023) + expect(result).to eq(2796620.0) end it 'includes runtime UPDATE modifications in sum' do - # Modify a value via UPDATE emissions[:energy_fugitive_emissions_non_energetic_co2_2023] = 50.0 result = emissions.sum(:energy, :non_energetic, :co2, 2023) - expect(result).to eq(50.0) # Modified value + expect(result).to eq(50.0) end it 'uses analysis_year when year parameter is not provided' do allow(area).to receive(:analysis_year).and_return(1990) result = emissions.sum(:energy, :energetic, :other_ghg) - expect(result).to eq(25.0) # 1990 value + expect(result).to eq(25.0) end it 'handles nil values gracefully' do - # Set a value to nil emissions[:buildings_non_specified_energetic_other_ghg_2023] = nil result = emissions.sum(:buildings, :energetic, :other_ghg, 2023) @@ -286,7 +328,6 @@ module Qernel end it 'sums multiple values correctly' do - # Agriculture has both energetic (50.0) and non_energetic (75.0) for co2/other_ghg energetic = emissions.sum(:agriculture, :energetic, :other_ghg, 2023) non_energetic = emissions.sum(:agriculture, :non_energetic, :co2, 2023) @@ -295,29 +336,15 @@ module Qernel end it 'does not match "energetic" substring in "non_energetic" (regression test)' do - # Add data with same sector+ghg but different use types - # This tests the bug where "energetic" would incorrectly match "non_energetic" emissions[:agriculture_non_specified_energetic_other_ghg_2023] = 1263.93948 emissions[:agriculture_non_specified_non_energetic_other_ghg_2023] = 18361.45468 - # Query for energetic should return ONLY 1263.93948, not 19625.39416 (sum of both) result = emissions.sum(:agriculture, :energetic, :other_ghg, 2023) expect(result).to eq(1263.93948) - # Query for non_energetic should return ONLY 18361.45468 result_non = emissions.sum(:agriculture, :non_energetic, :other_ghg, 2023) expect(result_non).to eq(18361.45468) end - - it 'handles partial sector matching with exact use matching' do - # Partial sector "agriculture_non_" should match "agriculture_non_specified" - # But use "energetic" should NOT match "non_energetic" - emissions[:agriculture_non_specified_energetic_other_ghg_2023] = 1263.93948 - emissions[:agriculture_non_specified_non_energetic_other_ghg_2023] = 18361.45468 - - result = emissions.sum(:agriculture_non_, :energetic, :other_ghg, 2023) - expect(result).to eq(1263.93948) # Only energetic, not 19625.39416 - end end end end From 407ff05c87361df7557d3ab09b64c312a20ad343 Mon Sep 17 00:00:00 2001 From: louispt1 Date: Thu, 11 Jun 2026 16:08:12 +0200 Subject: [PATCH 09/23] Add reporting methods for CO2 utilisation and biogenic emissions & spec --- .../qernel/node_api/direct_emissions.rb | 22 ++ .../qernel/node_api/direct_emissions_spec.rb | 189 ++++++++++++++++++ 2 files changed, 211 insertions(+) diff --git a/app/models/qernel/node_api/direct_emissions.rb b/app/models/qernel/node_api/direct_emissions.rb index e486c5ffc..b9091c9bc 100644 --- a/app/models/qernel/node_api/direct_emissions.rb +++ b/app/models/qernel/node_api/direct_emissions.rb @@ -183,6 +183,28 @@ def direct_reporting_emissions_total_ghg_emissions end end + # CO2 utilisation at this node. + # + # Biogenic CO2 emissions from production. + # + # @return [Float, nil] CO2 utilisation in kg, or nil if node is not in emissions group + def direct_reporting_emissions_co2_utilisation + with_emissions_node do + direct_co2_output_production_emissions_biogenic + end + end + + # Biogenic CO2 emissions at this node. + # + # Fossil CO2 input utilisation. + # + # @return [Float, nil] Biogenic CO2 emissions in kg, or nil if node is not in emissions group + def direct_reporting_emissions_biogenic_co2_emissions + with_emissions_node do + direct_co2_input_utilisation_fossil + end + end + private # Yields the given block only if the node belongs to the :emissions group. diff --git a/spec/models/qernel/node_api/direct_emissions_spec.rb b/spec/models/qernel/node_api/direct_emissions_spec.rb index fc0c06c5d..f047e7a7f 100644 --- a/spec/models/qernel/node_api/direct_emissions_spec.rb +++ b/spec/models/qernel/node_api/direct_emissions_spec.rb @@ -2462,4 +2462,193 @@ end end end + + describe '#direct_reporting_emissions_co2_utilisation' do + context 'with biogenic production emissions' do + # Create a graph: + # [Biogenic Waste Producer] -> [Biogenic Plant] -> [Terminus] + # Plant produces biogenic CO2 emissions during production + let(:builder) do + TestGraphBuilder.new.tap do |builder| + builder.add(:terminus, demand: 100) + builder.add(:biogenic_plant, groups: [:emissions]) + builder.add(:biogenic_waste_producer, groups: [:primary_energy_demand]) + + builder.connect(:biogenic_waste_producer, :biogenic_plant, :biogenic_waste, type: :share) + builder.connect(:biogenic_plant, :terminus, :heat, type: :share) + + builder.carrier_attrs(:biogenic_waste, potential_co2_conversion_per_mj: 0.05) + builder.carrier_attrs(:heat, potential_co2_conversion_per_mj: 0.0) + end + end + + let(:graph) { builder.to_qernel } + let(:biogenic_plant) { graph.node(:biogenic_plant) } + + it 'reports CO2 utilisation equal to biogenic production emissions' do + # Biogenic: 100 MJ * 0.05 kg/MJ = 5.0 kg CO2 + # CO2 Utilisation = Biogenic production emissions = 5.0 kg CO2 + expect(biogenic_plant).to have_query_value(:direct_reporting_emissions_co2_utilisation, 5.0) + end + + it 'equals direct_co2_output_production_emissions_biogenic' do + biogenic = biogenic_plant.query.direct_co2_output_production_emissions_biogenic + utilisation = biogenic_plant.query.direct_reporting_emissions_co2_utilisation + + expect(utilisation).to eq(biogenic) + end + end + + context 'with zero biogenic emissions' do + # Create a graph: + # [Fossil Producer] -> [Fossil Plant] -> [Terminus] + # Plant has no biogenic inputs + let(:builder) do + TestGraphBuilder.new.tap do |builder| + builder.add(:terminus, demand: 100) + builder.add(:fossil_plant, groups: [:emissions]) + builder.add(:fossil_producer, groups: [:primary_energy_demand], demand: 100) + + builder.connect(:fossil_producer, :fossil_plant, :natural_gas, type: :share) + builder.connect(:fossil_plant, :terminus, :electricity, type: :share) + + builder.carrier_attrs(:natural_gas, co2_conversion_per_mj: 0.0564) + builder.carrier_attrs(:electricity, co2_conversion_per_mj: 0.0) + end + end + + let(:graph) { builder.to_qernel } + let(:fossil_plant) { graph.node(:fossil_plant) } + + it 'reports zero CO2 utilisation' do + # No biogenic inputs = 0 kg CO2 utilisation + expect(fossil_plant).to have_query_value(:direct_reporting_emissions_co2_utilisation, 0.0) + end + + it 'equals direct_co2_output_production_emissions_biogenic (zero)' do + biogenic = fossil_plant.query.direct_co2_output_production_emissions_biogenic + utilisation = fossil_plant.query.direct_reporting_emissions_co2_utilisation + + expect(utilisation).to eq(biogenic) + expect(utilisation).to eq(0.0) + end + end + + context 'with a non-emissions node' do + let(:builder) do + TestGraphBuilder.new.tap do |builder| + builder.add(:terminus, demand: 100) + builder.add(:non_emissions_node) + builder.add(:producer, groups: [:primary_energy_demand]) + + builder.connect(:producer, :non_emissions_node, :natural_gas, type: :share) + builder.connect(:non_emissions_node, :terminus, :electricity, type: :share) + + builder.carrier_attrs(:natural_gas, co2_conversion_per_mj: 0.0564) + end + end + + let(:graph) { builder.to_qernel } + let(:non_emissions_node) { graph.node(:non_emissions_node) } + + it 'returns nil for non-emissions nodes' do + expect(non_emissions_node.query.direct_reporting_emissions_co2_utilisation).to be_nil + end + end + end + + describe '#direct_reporting_emissions_biogenic_co2_emissions' do + context 'with fossil CO2 input utilisation' do + # Create a graph: + # [CO2 Source] -> [CO2 Utilisation Plant] -> [Terminus] + # Plant uses fossil CO2 as input for production + let(:builder) do + TestGraphBuilder.new.tap do |builder| + builder.add(:terminus, demand: 100) + builder.add(:co2_utilisation_plant, groups: [:emissions], co2_utilisation_per_mj: 0.08) + builder.add(:co2_source, groups: [:primary_energy_demand]) + + builder.connect(:co2_source, :co2_utilisation_plant, :co2, type: :share) + builder.connect(:co2_utilisation_plant, :terminus, :electricity, type: :share) + + builder.carrier_attrs(:co2, co2_conversion_per_mj: 0.0) + builder.carrier_attrs(:electricity, co2_conversion_per_mj: 0.0) + end + end + + let(:graph) { builder.to_qernel } + let(:co2_plant) { graph.node(:co2_utilisation_plant) } + + it 'reports biogenic CO2 emissions equal to fossil input utilisation' do + # Total useful output: 100 MJ + # Fossil CO2 utilisation: 100 MJ * 0.08 kg/MJ = 8.0 kg CO2 + # Biogenic CO2 emissions (reporting) = 8.0 kg CO2 + expect(co2_plant).to have_query_value(:direct_reporting_emissions_biogenic_co2_emissions, 8.0) + end + + it 'equals direct_co2_input_utilisation_fossil' do + fossil_util = co2_plant.query.direct_co2_input_utilisation_fossil + biogenic_reporting = co2_plant.query.direct_reporting_emissions_biogenic_co2_emissions + + expect(biogenic_reporting).to eq(fossil_util) + end + end + + context 'with zero fossil CO2 utilisation' do + # Create a graph: + # [Producer] -> [Plant without CO2 utilisation] -> [Terminus] + # Plant has no CO2 utilisation attribute + let(:builder) do + TestGraphBuilder.new.tap do |builder| + builder.add(:terminus, demand: 100) + builder.add(:regular_plant, groups: [:emissions]) + builder.add(:producer, groups: [:primary_energy_demand], demand: 100) + + builder.connect(:producer, :regular_plant, :natural_gas, type: :share) + builder.connect(:regular_plant, :terminus, :electricity, type: :share) + + builder.carrier_attrs(:natural_gas, co2_conversion_per_mj: 0.0564) + builder.carrier_attrs(:electricity, co2_conversion_per_mj: 0.0) + end + end + + let(:graph) { builder.to_qernel } + let(:regular_plant) { graph.node(:regular_plant) } + + it 'reports zero biogenic CO2 emissions' do + # No CO2 utilisation = 0 kg CO2 + expect(regular_plant).to have_query_value(:direct_reporting_emissions_biogenic_co2_emissions, 0.0) + end + + it 'equals direct_co2_input_utilisation_fossil (zero)' do + fossil_util = regular_plant.query.direct_co2_input_utilisation_fossil + biogenic_reporting = regular_plant.query.direct_reporting_emissions_biogenic_co2_emissions + + expect(biogenic_reporting).to eq(fossil_util) + expect(biogenic_reporting).to eq(0.0) + end + end + + context 'with a non-emissions node' do + let(:builder) do + TestGraphBuilder.new.tap do |builder| + builder.add(:terminus, demand: 100) + builder.add(:non_emissions_node) + builder.add(:producer, groups: [:primary_energy_demand]) + + builder.connect(:producer, :non_emissions_node, :natural_gas, type: :share) + builder.connect(:non_emissions_node, :terminus, :electricity, type: :share) + + builder.carrier_attrs(:natural_gas, co2_conversion_per_mj: 0.0564) + end + end + + let(:graph) { builder.to_qernel } + let(:non_emissions_node) { graph.node(:non_emissions_node) } + + it 'returns nil for non-emissions nodes' do + expect(non_emissions_node.query.direct_reporting_emissions_biogenic_co2_emissions).to be_nil + end + end + end end From 63956d4ca8bb855d6ab05274ea8d61c09f042403 Mon Sep 17 00:00:00 2001 From: louispt1 Date: Thu, 11 Jun 2026 17:10:58 +0200 Subject: [PATCH 10/23] Swap arguments for reporting methods for CO2 utilisation and biogenic emissions --- .../qernel/node_api/direct_emissions.rb | 8 +- .../qernel/node_api/direct_emissions_spec.rb | 130 +++++++++--------- 2 files changed, 69 insertions(+), 69 deletions(-) diff --git a/app/models/qernel/node_api/direct_emissions.rb b/app/models/qernel/node_api/direct_emissions.rb index b9091c9bc..deb5c1a32 100644 --- a/app/models/qernel/node_api/direct_emissions.rb +++ b/app/models/qernel/node_api/direct_emissions.rb @@ -185,23 +185,23 @@ def direct_reporting_emissions_total_ghg_emissions # CO2 utilisation at this node. # - # Biogenic CO2 emissions from production. + # Fossil CO2 input utilisation. # # @return [Float, nil] CO2 utilisation in kg, or nil if node is not in emissions group def direct_reporting_emissions_co2_utilisation with_emissions_node do - direct_co2_output_production_emissions_biogenic + direct_co2_input_utilisation_fossil end end # Biogenic CO2 emissions at this node. # - # Fossil CO2 input utilisation. + # Biogenic CO2 emissions from production. # # @return [Float, nil] Biogenic CO2 emissions in kg, or nil if node is not in emissions group def direct_reporting_emissions_biogenic_co2_emissions with_emissions_node do - direct_co2_input_utilisation_fossil + direct_co2_output_production_emissions_biogenic end end diff --git a/spec/models/qernel/node_api/direct_emissions_spec.rb b/spec/models/qernel/node_api/direct_emissions_spec.rb index f047e7a7f..1d6b0a8c5 100644 --- a/spec/models/qernel/node_api/direct_emissions_spec.rb +++ b/spec/models/qernel/node_api/direct_emissions_spec.rb @@ -2464,53 +2464,54 @@ end describe '#direct_reporting_emissions_co2_utilisation' do - context 'with biogenic production emissions' do + context 'with fossil CO2 input utilisation' do # Create a graph: - # [Biogenic Waste Producer] -> [Biogenic Plant] -> [Terminus] - # Plant produces biogenic CO2 emissions during production + # [CO2 Source] -> [CO2 Utilisation Plant] -> [Terminus] + # Plant uses fossil CO2 as input for production let(:builder) do TestGraphBuilder.new.tap do |builder| builder.add(:terminus, demand: 100) - builder.add(:biogenic_plant, groups: [:emissions]) - builder.add(:biogenic_waste_producer, groups: [:primary_energy_demand]) + builder.add(:co2_utilisation_plant, groups: [:emissions], co2_utilisation_per_mj: 0.08) + builder.add(:co2_source, groups: [:primary_energy_demand]) - builder.connect(:biogenic_waste_producer, :biogenic_plant, :biogenic_waste, type: :share) - builder.connect(:biogenic_plant, :terminus, :heat, type: :share) + builder.connect(:co2_source, :co2_utilisation_plant, :co2, type: :share) + builder.connect(:co2_utilisation_plant, :terminus, :electricity, type: :share) - builder.carrier_attrs(:biogenic_waste, potential_co2_conversion_per_mj: 0.05) - builder.carrier_attrs(:heat, potential_co2_conversion_per_mj: 0.0) + builder.carrier_attrs(:co2, co2_conversion_per_mj: 0.0) + builder.carrier_attrs(:electricity, co2_conversion_per_mj: 0.0) end end let(:graph) { builder.to_qernel } - let(:biogenic_plant) { graph.node(:biogenic_plant) } + let(:co2_plant) { graph.node(:co2_utilisation_plant) } - it 'reports CO2 utilisation equal to biogenic production emissions' do - # Biogenic: 100 MJ * 0.05 kg/MJ = 5.0 kg CO2 - # CO2 Utilisation = Biogenic production emissions = 5.0 kg CO2 - expect(biogenic_plant).to have_query_value(:direct_reporting_emissions_co2_utilisation, 5.0) + it 'reports CO2 utilisation equal to fossil input utilisation' do + # Total useful output: 100 MJ + # Fossil CO2 utilisation: 100 MJ * 0.08 kg/MJ = 8.0 kg CO2 + # CO2 Utilisation (reporting) = 8.0 kg CO2 + expect(co2_plant).to have_query_value(:direct_reporting_emissions_co2_utilisation, 8.0) end - it 'equals direct_co2_output_production_emissions_biogenic' do - biogenic = biogenic_plant.query.direct_co2_output_production_emissions_biogenic - utilisation = biogenic_plant.query.direct_reporting_emissions_co2_utilisation + it 'equals direct_co2_input_utilisation_fossil' do + fossil_util = co2_plant.query.direct_co2_input_utilisation_fossil + utilisation = co2_plant.query.direct_reporting_emissions_co2_utilisation - expect(utilisation).to eq(biogenic) + expect(utilisation).to eq(fossil_util) end end - context 'with zero biogenic emissions' do + context 'with zero fossil CO2 utilisation' do # Create a graph: - # [Fossil Producer] -> [Fossil Plant] -> [Terminus] - # Plant has no biogenic inputs + # [Producer] -> [Plant without CO2 utilisation] -> [Terminus] + # Plant has no CO2 utilisation attribute let(:builder) do TestGraphBuilder.new.tap do |builder| builder.add(:terminus, demand: 100) - builder.add(:fossil_plant, groups: [:emissions]) - builder.add(:fossil_producer, groups: [:primary_energy_demand], demand: 100) + builder.add(:regular_plant, groups: [:emissions]) + builder.add(:producer, groups: [:primary_energy_demand]) - builder.connect(:fossil_producer, :fossil_plant, :natural_gas, type: :share) - builder.connect(:fossil_plant, :terminus, :electricity, type: :share) + builder.connect(:producer, :regular_plant, :natural_gas, type: :share) + builder.connect(:regular_plant, :terminus, :electricity, type: :share) builder.carrier_attrs(:natural_gas, co2_conversion_per_mj: 0.0564) builder.carrier_attrs(:electricity, co2_conversion_per_mj: 0.0) @@ -2518,18 +2519,18 @@ end let(:graph) { builder.to_qernel } - let(:fossil_plant) { graph.node(:fossil_plant) } + let(:regular_plant) { graph.node(:regular_plant) } it 'reports zero CO2 utilisation' do - # No biogenic inputs = 0 kg CO2 utilisation - expect(fossil_plant).to have_query_value(:direct_reporting_emissions_co2_utilisation, 0.0) + # No CO2 utilisation = 0 kg CO2 + expect(regular_plant).to have_query_value(:direct_reporting_emissions_co2_utilisation, 0.0) end - it 'equals direct_co2_output_production_emissions_biogenic (zero)' do - biogenic = fossil_plant.query.direct_co2_output_production_emissions_biogenic - utilisation = fossil_plant.query.direct_reporting_emissions_co2_utilisation + it 'equals direct_co2_input_utilisation_fossil (zero)' do + fossil_util = regular_plant.query.direct_co2_input_utilisation_fossil + utilisation = regular_plant.query.direct_reporting_emissions_co2_utilisation - expect(utilisation).to eq(biogenic) + expect(utilisation).to eq(fossil_util) expect(utilisation).to eq(0.0) end end @@ -2558,54 +2559,53 @@ end describe '#direct_reporting_emissions_biogenic_co2_emissions' do - context 'with fossil CO2 input utilisation' do + context 'with biogenic production emissions' do # Create a graph: - # [CO2 Source] -> [CO2 Utilisation Plant] -> [Terminus] - # Plant uses fossil CO2 as input for production + # [Biogenic Waste Producer] -> [Biogenic Plant] -> [Terminus] + # Plant produces biogenic CO2 emissions during production let(:builder) do TestGraphBuilder.new.tap do |builder| builder.add(:terminus, demand: 100) - builder.add(:co2_utilisation_plant, groups: [:emissions], co2_utilisation_per_mj: 0.08) - builder.add(:co2_source, groups: [:primary_energy_demand]) + builder.add(:biogenic_plant, groups: [:emissions]) + builder.add(:biogenic_waste_producer, groups: [:primary_energy_demand]) - builder.connect(:co2_source, :co2_utilisation_plant, :co2, type: :share) - builder.connect(:co2_utilisation_plant, :terminus, :electricity, type: :share) + builder.connect(:biogenic_waste_producer, :biogenic_plant, :biogenic_waste, type: :share) + builder.connect(:biogenic_plant, :terminus, :heat, type: :share) - builder.carrier_attrs(:co2, co2_conversion_per_mj: 0.0) - builder.carrier_attrs(:electricity, co2_conversion_per_mj: 0.0) + builder.carrier_attrs(:biogenic_waste, potential_co2_conversion_per_mj: 0.05) + builder.carrier_attrs(:heat, potential_co2_conversion_per_mj: 0.0) end end let(:graph) { builder.to_qernel } - let(:co2_plant) { graph.node(:co2_utilisation_plant) } + let(:biogenic_plant) { graph.node(:biogenic_plant) } - it 'reports biogenic CO2 emissions equal to fossil input utilisation' do - # Total useful output: 100 MJ - # Fossil CO2 utilisation: 100 MJ * 0.08 kg/MJ = 8.0 kg CO2 - # Biogenic CO2 emissions (reporting) = 8.0 kg CO2 - expect(co2_plant).to have_query_value(:direct_reporting_emissions_biogenic_co2_emissions, 8.0) + it 'reports biogenic CO2 emissions equal to biogenic production emissions' do + # Biogenic: 100 MJ * 0.05 kg/MJ = 5.0 kg CO2 + # Biogenic CO2 emissions (reporting) = 5.0 kg CO2 + expect(biogenic_plant).to have_query_value(:direct_reporting_emissions_biogenic_co2_emissions, 5.0) end - it 'equals direct_co2_input_utilisation_fossil' do - fossil_util = co2_plant.query.direct_co2_input_utilisation_fossil - biogenic_reporting = co2_plant.query.direct_reporting_emissions_biogenic_co2_emissions + it 'equals direct_co2_output_production_emissions_biogenic' do + biogenic = biogenic_plant.query.direct_co2_output_production_emissions_biogenic + biogenic_reporting = biogenic_plant.query.direct_reporting_emissions_biogenic_co2_emissions - expect(biogenic_reporting).to eq(fossil_util) + expect(biogenic_reporting).to eq(biogenic) end end - context 'with zero fossil CO2 utilisation' do + context 'with zero biogenic emissions' do # Create a graph: - # [Producer] -> [Plant without CO2 utilisation] -> [Terminus] - # Plant has no CO2 utilisation attribute + # [Fossil Producer] -> [Fossil Plant] -> [Terminus] + # Plant has no biogenic inputs let(:builder) do TestGraphBuilder.new.tap do |builder| builder.add(:terminus, demand: 100) - builder.add(:regular_plant, groups: [:emissions]) - builder.add(:producer, groups: [:primary_energy_demand], demand: 100) + builder.add(:fossil_plant, groups: [:emissions]) + builder.add(:fossil_producer, groups: [:primary_energy_demand]) - builder.connect(:producer, :regular_plant, :natural_gas, type: :share) - builder.connect(:regular_plant, :terminus, :electricity, type: :share) + builder.connect(:fossil_producer, :fossil_plant, :natural_gas, type: :share) + builder.connect(:fossil_plant, :terminus, :electricity, type: :share) builder.carrier_attrs(:natural_gas, co2_conversion_per_mj: 0.0564) builder.carrier_attrs(:electricity, co2_conversion_per_mj: 0.0) @@ -2613,18 +2613,18 @@ end let(:graph) { builder.to_qernel } - let(:regular_plant) { graph.node(:regular_plant) } + let(:fossil_plant) { graph.node(:fossil_plant) } it 'reports zero biogenic CO2 emissions' do - # No CO2 utilisation = 0 kg CO2 - expect(regular_plant).to have_query_value(:direct_reporting_emissions_biogenic_co2_emissions, 0.0) + # No biogenic inputs = 0 kg CO2 + expect(fossil_plant).to have_query_value(:direct_reporting_emissions_biogenic_co2_emissions, 0.0) end - it 'equals direct_co2_input_utilisation_fossil (zero)' do - fossil_util = regular_plant.query.direct_co2_input_utilisation_fossil - biogenic_reporting = regular_plant.query.direct_reporting_emissions_biogenic_co2_emissions + it 'equals direct_co2_output_production_emissions_biogenic (zero)' do + biogenic = fossil_plant.query.direct_co2_output_production_emissions_biogenic + biogenic_reporting = fossil_plant.query.direct_reporting_emissions_biogenic_co2_emissions - expect(biogenic_reporting).to eq(fossil_util) + expect(biogenic_reporting).to eq(biogenic) expect(biogenic_reporting).to eq(0.0) end end From 3637361e3062bef852b7c4a2d3daf0e1c668d50c Mon Sep 17 00:00:00 2001 From: louispt1 Date: Tue, 16 Jun 2026 13:54:45 +0200 Subject: [PATCH 11/23] Simplify 1990 changes --- app/models/etsource/dataset.rb | 9 + app/models/etsource/dataset/import.rb | 30 +--- app/models/gql/runtime/functions/lookup.rb | 11 +- app/models/qernel/emissions.rb | 98 ++-------- spec/models/etsource/dataset/import_spec.rb | 23 --- .../gql/runtime/functions/lookup_spec.rb | 31 ---- .../gql/runtime/functions/update_spec.rb | 7 - spec/models/qernel/emissions_spec.rb | 169 +----------------- 8 files changed, 31 insertions(+), 347 deletions(-) diff --git a/app/models/etsource/dataset.rb b/app/models/etsource/dataset.rb index 20e3616ab..111df87f0 100644 --- a/app/models/etsource/dataset.rb +++ b/app/models/etsource/dataset.rb @@ -48,5 +48,14 @@ def self.region_codes(refresh: false) .map { |dataset| dataset.key.to_s } end end + + + def self.emissions_keys + NastyCache.instance.fetch('emission_keys') do + Atlas::Dataset.find( + Etsource::Config.default_dataset_key + ).emissions.to_hash.keys + end + end end end diff --git a/app/models/etsource/dataset/import.rb b/app/models/etsource/dataset/import.rb index 5d92c5382..99d1dbae5 100644 --- a/app/models/etsource/dataset/import.rb +++ b/app/models/etsource/dataset/import.rb @@ -117,35 +117,9 @@ def load_region_data # Internal: Loads the emission data. # - # Returns a hash nested under keys for DatasetAttributes compatibility. + # Returns a hash nested under emisisons_data key for DatasetAttributes compatibility. def load_emission_data - { - emissions_data: @atlas_ds.emissions.to_hash, - emissions_index: build_emissions_index - } - end - - # Internal: Groups emission keys by their CSV column components. - # - # Returns a hash with :sectors (sector => subsector keys), :subsectors - # (subsector key => true), :scopes (sector_subsector_use => true) and - # :ghgs (ghg => true). - def build_emissions_index - index = { sectors: {}, subsectors: {}, scopes: {}, ghgs: {} } - - @atlas_ds.emissions.component_keys.each_value do |components| - *subsector_parts, use, ghg, _year = components - subsector = subsector_parts.join('_').to_sym - - sectors = (index[:sectors][components.first] ||= []) - sectors << subsector unless sectors.include?(subsector) - - index[:subsectors][subsector] = true - index[:scopes][:"#{subsector}_#{use}"] = true - index[:ghgs][ghg] = true - end - - index + { emissions_data: @atlas_ds.emissions.to_hash } end # Internal: Loads the carrier data. diff --git a/app/models/gql/runtime/functions/lookup.rb b/app/models/gql/runtime/functions/lookup.rb index 2c73b2840..83b1f2f83 100644 --- a/app/models/gql/runtime/functions/lookup.rb +++ b/app/models/gql/runtime/functions/lookup.rb @@ -352,8 +352,8 @@ def DATASET_CURVE(key) # # Examples # EMISSIONS(households, energetic, other_ghg) # => 12.0 (from emissions.csv) - # EMISSIONS(households, energetic, co2) # => aggregated value (sum of subsectors, default year) - # EMISSIONS(households, energetic, co2, 1990) # => aggregated value for 1990 + # EMISSIONS(households, energetic, co2) # => value for default year + # EMISSIONS(households, energetic, co2, 1990) # => value for 1990 # EMISSIONS(buildings_non_specified, energetic, other_ghg, 2023) # => 18.0 # def EMISSIONS(*keys) @@ -365,9 +365,10 @@ def EMISSIONS(*keys) # EMISSIONS(sector, use) -> return ScopedSector for UPDATE operations return scope.graph.emissions.scope(keys.join('_').to_sym) if keys.size == 2 - # EMISSIONS(sector, use, ghg [, year]) -> value; a full subsector key - # sums a single entry, a sector key sums all of its subsectors. - scope.graph.emissions.sum(*keys) + # EMISSIONS(sector, use, ghg [, year]) -> return value + year = keys[3] || scope.graph.area.analysis_year + full_key = "#{keys[0]}_#{keys[1]}_#{keys[2]}_#{year}".to_sym + scope.graph.emissions[full_key] end end end diff --git a/app/models/qernel/emissions.rb b/app/models/qernel/emissions.rb index 037d3b093..53cc9f2f1 100644 --- a/app/models/qernel/emissions.rb +++ b/app/models/qernel/emissions.rb @@ -13,12 +13,6 @@ module Qernel # - buildings_non_specified_energetic_other_ghg_2023 # - energy_electricity_and_heat_production_energetic_co2_1990 # - # Joining the CSV columns into one key loses the column boundaries - # (components such as "non_energetic" contain the separator), so the - # import also builds an index preserving the structure (see - # Etsource::Dataset::Import#build_emissions_index). Lookups construct - # exact keys from the index rather than parsing them. - # # == Year Handling # # Year parameter defaults to the dataset's analysis_year when not @@ -27,6 +21,7 @@ module Qernel class Emissions include DatasetAttributes + dataset_accessors ::Etsource::Dataset.emissions_keys attr_accessor :graph # Queryable object that provides scoped access to emissions data @@ -39,10 +34,6 @@ class Emissions # Then UPDATE can call: scoped.co2 = 100.0 class ScopedSector def initialize(emissions, scope, year = nil) - unless emissions.valid_scope?(scope) - raise ArgumentError, "unknown emissions scope: #{scope}" - end - @emissions = emissions @scope = scope @year = year @@ -65,27 +56,22 @@ def scoped_method(method_name) "#{@scope}_#{method_name}_#{year}" end - # Getters and setters for each GHG in the dataset (e.g. scoped.co2, - # scoped.co2 = 1.0), delegated to []/[]=. The GHG set comes from the - # emissions index, so a new gas in the CSV needs no code change here. - def method_missing(name, *args) - return super unless ghg?(name) - - if name.end_with?('=') - self[name.to_s.delete_suffix('=')] = args.first - else - self[name] - end - end + def respond_to_missing?(method_name, include_private = false) + attr_name = method_name.to_s.delete_suffix('=') + data_key = scoped_method(attr_name) - def respond_to_missing?(name, include_private = false) - ghg?(name) || super + @emissions.respond_to?(data_key) || super end - private + def method_missing(method_name, *args) + attr_name = method_name.to_s.delete_suffix('=') + data_key = scoped_method(attr_name).to_sym - def ghg?(name) - @emissions.ghgs.key?(name.to_s.delete_suffix('=').to_sym) + if method_name.to_s.end_with?('=') + @emissions[data_key] = args.first + else + @emissions[data_key] + end end end @@ -95,39 +81,9 @@ def initialize(graph = nil) @dataset_key = @key = :emissions_data end - # Public: Aggregates emissions for a sector, use, ghg, and year. - # - # Sums the value of every subsector belonging to the given sector. When - # given a full subsector key instead of a sector, returns its single - # value. Includes runtime UPDATE modifications. - # - # == Examples - # - # # Sum all energy sector energetic CO2 emissions for 2023 - # emissions.sum(:energy, :energetic, :co2, 2023) - # # => 500.25 - # - # # Single subsector - # emissions.sum(:buildings_non_specified, :energetic, :other_ghg, 2023) - # # => 55.64 - # - # # Default year (analysis_year from dataset) - # emissions.sum(:agriculture, :non_energetic, :other_ghg) - # # => 18863.47 - # - # Returns a Float; 0.0 when nothing matches. - def sum(sector, use, ghg, year = nil) - year ||= default_year - - subsector_keys(sector).sum do |subsector| - self[:"#{subsector}_#{use}_#{ghg}_#{year}"] || 0.0 - end - end - # Public: define the sector scope for access to the hashed emission keys # # sector - Scope identifier (e.g., :buildings_non_specified_energetic). - # Must exist in the dataset; raises ArgumentError otherwise. # year - Optional year (defaults to analysis_year). Used to target a # specific year for UPDATE operations. # @@ -136,38 +92,10 @@ def scope(sector, year = nil) ScopedSector.new(self, sector, year) end - # Public: Whether the scope (sector_subsector_use) exists in the dataset. - def valid_scope?(scope) - index[:scopes].key?(scope.to_sym) - end - - # Public: The GHG types present in the dataset (ghg => true). - def ghgs - index[:ghgs] - end - # Returns the default year for emissions queries. # Uses the area's analysis_year if graph is present, nil otherwise. def default_year graph&.area&.analysis_year end - - EMPTY_INDEX = { sectors: {}, subsectors: {}, scopes: {}, ghgs: {} }.freeze - - # The structured emissions index built at import time. - # See Etsource::Dataset::Import#build_emissions_index. - def index - @index ||= - (dataset && dataset.data[:emissions][:emissions_index]) || EMPTY_INDEX - end - - private - - # An exact sector maps to its subsector keys; an exact subsector key - # maps to itself; anything else matches nothing. - def subsector_keys(sector) - key = sector.to_s.tr('-.', '_').downcase.to_sym - index[:sectors][key] || (index[:subsectors].key?(key) ? [key] : []) - end end end diff --git a/spec/models/etsource/dataset/import_spec.rb b/spec/models/etsource/dataset/import_spec.rb index 7c340fd1a..2630de70b 100644 --- a/spec/models/etsource/dataset/import_spec.rb +++ b/spec/models/etsource/dataset/import_spec.rb @@ -10,28 +10,5 @@ expect(emissions[:emissions_data][:energy_fugitive_emissions_non_energetic_co2_2023]) .to eq(20.0) end - - it 'indexes sectors to their subsector keys' do - expect(emissions[:emissions_index][:sectors][:energy]).to contain_exactly( - :energy_electricity_and_heat_production, :energy_fugitive_emissions - ) - end - - it 'indexes subsector keys' do - expect(emissions[:emissions_index][:subsectors]) - .to include(:energy_electricity_and_heat_production, :buildings_non_specified) - end - - it 'indexes scopes including the use' do - expect(emissions[:emissions_index][:scopes]).to include( - :energy_fugitive_emissions_non_energetic, - :agriculture_non_specified_energetic, - :agriculture_non_specified_non_energetic - ) - end - - it 'indexes the GHG types' do - expect(emissions[:emissions_index][:ghgs]).to eq(co2: true, other_ghg: true) - end end end diff --git a/spec/models/gql/runtime/functions/lookup_spec.rb b/spec/models/gql/runtime/functions/lookup_spec.rb index af5c10cea..b801bd1b1 100644 --- a/spec/models/gql/runtime/functions/lookup_spec.rb +++ b/spec/models/gql/runtime/functions/lookup_spec.rb @@ -67,36 +67,5 @@ module Gql::Runtime::Functions expect(result).to eq(7.0) end end - - describe 'EMISSIONS(energy, non_energetic, co2, 2023)' do - it 'aggregates across energy subsectors (Fugitive emissions)' do - expect(result).to eq(20.0) # Fugitive: 20.0 - end - end - - describe 'EMISSIONS(energy, energetic, other_ghg, 2023)' do - it 'aggregates multiple energy subsectors' do - # Electricity: 18.0 - expect(result).to eq(18.0) - end - end - - describe 'EMISSIONS(agriculture, energetic, other_ghg, 2023)' do - it 'sums single subsector' do - expect(result).to eq(50.0) - end - end - - describe 'EMISSIONS(agriculture, non_energetic, other_ghg, 2023)' do - it 'returns only non_energetic value, not including energetic' do - expect(result).to eq(100.0) - end - end - - describe 'EMISSIONS(agriculture_non_, energetic, other_ghg, 2023)' do - it 'does not match partial sector names' do - expect(result).to eq(0) - end - end end end diff --git a/spec/models/gql/runtime/functions/update_spec.rb b/spec/models/gql/runtime/functions/update_spec.rb index aace84d43..09d848d27 100644 --- a/spec/models/gql/runtime/functions/update_spec.rb +++ b/spec/models/gql/runtime/functions/update_spec.rb @@ -69,13 +69,6 @@ end end - describe 'UPDATE(EMISSIONS(invalid_sector, energetic), co2, 100.0)' do - it 'raises an error when the emission key does not exist in the dataset' do - # The full key 'invalid_sector_energetic_co2' does not exist in the dataset - expect { result }.to raise_error(Gql::CommandError) - end - end - describe 'UPDATE(EMISSIONS(energy_fugitive_emissions, non_energetic), co2, 0.0)' do before { result } diff --git a/spec/models/qernel/emissions_spec.rb b/spec/models/qernel/emissions_spec.rb index 0e9790100..6d1362d65 100644 --- a/spec/models/qernel/emissions_spec.rb +++ b/spec/models/qernel/emissions_spec.rb @@ -2,41 +2,11 @@ module Qernel describe Emissions do - # Builds an index in the shape produced by - # Etsource::Dataset::Import#build_emissions_index from - # [sector, subsector, use] rows. - def self.emissions_index(rows) - index = { sectors: {}, subsectors: {}, scopes: {}, ghgs: { co2: true, other_ghg: true } } - - rows.each do |sector, subsector, use| - key = :"#{sector}_#{subsector}" - sectors = (index[:sectors][sector.to_sym] ||= []) - sectors << key unless sectors.include?(key) - - index[:subsectors][key] = true - index[:scopes][:"#{key}_#{use}"] = true - end - - index - end - - INDEX = emissions_index([ - %w[energy electricity_and_heat_production energetic], - %w[energy fugitive_emissions non_energetic], - %w[households non_specified energetic], - %w[buildings non_specified energetic], - %w[industry steel non_energetic], - %w[industry chemicals non_energetic], - %w[industry non_specified energetic], - %w[agriculture non_specified energetic], - %w[agriculture non_specified non_energetic] - ]).freeze - let(:area) { double('Area', analysis_year: 2023) } let(:dataset) do Qernel::Dataset.new(1).tap do |ds| - ds.data[:emissions] = { emissions_data: data, emissions_index: INDEX } + ds.data[:emissions] = { emissions_data: data } end end @@ -82,28 +52,6 @@ def self.emissions_index(rows) scoped = emissions.scope(:agriculture_non_specified_energetic) expect(scoped.instance_variable_get(:@scope)).to eq(:agriculture_non_specified_energetic) end - - it 'raises ArgumentError for a scope not present in the dataset' do - expect { emissions.scope(:invalid_sector_energetic) } - .to raise_error(ArgumentError, /unknown emissions scope/) - end - end - - describe '#valid_scope?' do - it 'returns true for a sector_subsector_use combination from the data' do - expect(emissions.valid_scope?(:energy_fugitive_emissions_non_energetic)).to be true - end - - it 'returns false for unknown combinations' do - expect(emissions.valid_scope?(:energy_fugitive_emissions_energetic)).to be false - expect(emissions.valid_scope?(:invalid_sector_energetic)).to be false - end - end - - describe '#ghgs' do - it 'returns the GHG types from the index' do - expect(emissions.ghgs.keys).to contain_exactly(:co2, :other_ghg) - end end describe Emissions::ScopedSector do @@ -143,19 +91,6 @@ def self.emissions_index(rows) scoped.co2 = 300.0 expect(emissions.dataset_get(:households_non_specified_energetic_co2_2023)).to eq(300.0) end - - it 'raises NoMethodError for attributes which are not GHGs' do - expect { scoped.nonexistent_attribute }.to raise_error(NoMethodError) - expect { scoped.arbitrary_key = 100.0 }.to raise_error(NoMethodError) - end - - it 'responds to the GHG getters and setters' do - expect(scoped).to respond_to(:co2, :co2=, :other_ghg, :other_ghg=) - end - - it 'does not respond to attributes which are not GHGs' do - expect(scoped).not_to respond_to(:arbitrary_key) - end end describe 'year targeting' do @@ -244,107 +179,5 @@ def self.emissions_index(rows) end end end - - describe '#sum' do - let(:data) do - { - energy_electricity_and_heat_production_energetic_other_ghg_2023: 18.0, - energy_electricity_and_heat_production_energetic_other_ghg_1990: 25.0, - energy_fugitive_emissions_non_energetic_co2_2023: 20.0, - energy_fugitive_emissions_non_energetic_co2_1990: 30.0, - agriculture_non_specified_energetic_other_ghg_2023: 50.0, - agriculture_non_specified_energetic_other_ghg_1990: 75.0, - agriculture_non_specified_non_energetic_co2_2023: 75.0, - agriculture_non_specified_non_energetic_co2_1990: 100.0, - industry_steel_non_energetic_co2_2023: 100.0, - industry_chemicals_non_energetic_co2_2023: 50.0, - buildings_non_specified_energetic_other_ghg_2023: 2796620.0, - buildings_non_specified_energetic_other_ghg_1990: 3000000.0 - } - end - - it 'sums emissions across multiple subsectors for a sector' do - # Energy has two subsectors with different use types, so this matches - # non_energetic (fugitive) only. - result = emissions.sum(:energy, :non_energetic, :co2, 2023) - expect(result).to eq(20.0) - end - - it 'aggregates energy sector with energetic other_ghg for default year' do - result = emissions.sum(:energy, :energetic, :other_ghg) - expect(result).to eq(18.0) - end - - it 'sums emissions for 1990 year when specified' do - result = emissions.sum(:energy, :non_energetic, :co2, 1990) - expect(result).to eq(30.0) - end - - it 'handles single subsector aggregation' do - result = emissions.sum(:agriculture, :energetic, :other_ghg, 2023) - expect(result).to eq(50.0) - end - - it 'sums multiple subsectors in industry' do - # Industry has steel (100.0) + chemicals (50.0) = 150.0 - result = emissions.sum(:industry, :non_energetic, :co2, 2023) - expect(result).to eq(150.0) - end - - it 'accepts a full subsector key instead of a sector' do - result = emissions.sum(:buildings_non_specified, :energetic, :other_ghg, 2023) - expect(result).to eq(2796620.0) - end - - it 'returns 0 when no matches are found' do - result = emissions.sum(:nonexistent_sector, :energetic, :co2, 2023) - expect(result).to eq(0) - end - - it 'normalizes sector names with dashes and mixed case' do - result = emissions.sum(:'Buildings-Non-Specified', :energetic, :other_ghg, 2023) - expect(result).to eq(2796620.0) - end - - it 'includes runtime UPDATE modifications in sum' do - emissions[:energy_fugitive_emissions_non_energetic_co2_2023] = 50.0 - - result = emissions.sum(:energy, :non_energetic, :co2, 2023) - expect(result).to eq(50.0) - end - - it 'uses analysis_year when year parameter is not provided' do - allow(area).to receive(:analysis_year).and_return(1990) - - result = emissions.sum(:energy, :energetic, :other_ghg) - expect(result).to eq(25.0) - end - - it 'handles nil values gracefully' do - emissions[:buildings_non_specified_energetic_other_ghg_2023] = nil - - result = emissions.sum(:buildings, :energetic, :other_ghg, 2023) - expect(result).to eq(0) - end - - it 'sums multiple values correctly' do - energetic = emissions.sum(:agriculture, :energetic, :other_ghg, 2023) - non_energetic = emissions.sum(:agriculture, :non_energetic, :co2, 2023) - - expect(energetic).to eq(50.0) - expect(non_energetic).to eq(75.0) - end - - it 'does not match "energetic" substring in "non_energetic" (regression test)' do - emissions[:agriculture_non_specified_energetic_other_ghg_2023] = 1263.93948 - emissions[:agriculture_non_specified_non_energetic_other_ghg_2023] = 18361.45468 - - result = emissions.sum(:agriculture, :energetic, :other_ghg, 2023) - expect(result).to eq(1263.93948) - - result_non = emissions.sum(:agriculture, :non_energetic, :other_ghg, 2023) - expect(result_non).to eq(18361.45468) - end - end end end From cc2e39251173f00ba31581783424b2c7cdf32ddf Mon Sep 17 00:00:00 2001 From: louispt1 Date: Tue, 16 Jun 2026 14:18:02 +0200 Subject: [PATCH 12/23] Add reporting methods to molecule nodes that return 0.0 (and spec) --- .../qernel/node_api/molecule_emissions.rb | 22 ++++++++++++ .../node_api/molecule_emissions_spec.rb | 34 +++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/app/models/qernel/node_api/molecule_emissions.rb b/app/models/qernel/node_api/molecule_emissions.rb index ed6ba446e..e8bb48976 100644 --- a/app/models/qernel/node_api/molecule_emissions.rb +++ b/app/models/qernel/node_api/molecule_emissions.rb @@ -75,6 +75,28 @@ def direct_reporting_emissions_total_ghg_emissions end end + # CO2 utilisation at this node. + # + # Fossil CO2 input utilisation. + # + # @return [Float, nil] CO2 utilisation in kg, or nil if node is not in emissions group + def direct_reporting_emissions_co2_utilisation + with_emissions_node do + 0.0 + end + end + + # Biogenic CO2 emissions at this node. + # + # Biogenic CO2 emissions from production. + # + # @return [Float, nil] Biogenic CO2 emissions in kg, or nil if node is not in emissions group + def direct_reporting_emissions_biogenic_co2_emissions + with_emissions_node do + 0.0 + end + end + private # Yields the given block only if the node belongs to the :emissions group. diff --git a/spec/models/qernel/node_api/molecule_emissions_spec.rb b/spec/models/qernel/node_api/molecule_emissions_spec.rb index ec8ce1518..f1d6de08c 100644 --- a/spec/models/qernel/node_api/molecule_emissions_spec.rb +++ b/spec/models/qernel/node_api/molecule_emissions_spec.rb @@ -190,6 +190,38 @@ end end + describe '#direct_reporting_emissions_co2_utilisation' do + context 'node in emissions group' do + it 'returns 0.0' do + expect(node.query.direct_reporting_emissions_co2_utilisation).to eq(0.0) + end + end + + context 'node not in emissions group' do + let(:node_groups) { [] } + + it 'returns nil' do + expect(node.query.direct_reporting_emissions_co2_utilisation).to be_nil + end + end + end + + describe '#direct_reporting_emissions_biogenic_co2_emissions' do + context 'node in emissions group' do + it 'returns 0.0' do + expect(node.query.direct_reporting_emissions_biogenic_co2_emissions).to eq(0.0) + end + end + + context 'node not in emissions group' do + let(:node_groups) { [] } + + it 'returns nil' do + expect(node.query.direct_reporting_emissions_biogenic_co2_emissions).to be_nil + end + end + end + describe '#ghg_carrier' do context 'with a co2 input slot' do before do @@ -222,6 +254,8 @@ expect(node.query.direct_reporting_emissions_co2_capture).to be_nil expect(node.query.direct_reporting_emissions_other_ghg_emissions).to be_nil expect(node.query.direct_reporting_emissions_total_ghg_emissions).to be_nil + expect(node.query.direct_reporting_emissions_co2_utilisation).to be_nil + expect(node.query.direct_reporting_emissions_biogenic_co2_emissions).to be_nil end end end From 1a16194ce9b873276e5e36d193ba874ea1b2065e Mon Sep 17 00:00:00 2001 From: louispt1 Date: Tue, 16 Jun 2026 14:40:31 +0200 Subject: [PATCH 13/23] Improve spec and cover missing methods --- app/models/etsource/dataset.rb | 1 - app/models/etsource/dataset/import.rb | 2 +- app/models/qernel/emissions.rb | 16 +++++++- spec/models/qernel/emissions_spec.rb | 57 +++++++++++++++++++++++++++ 4 files changed, 73 insertions(+), 3 deletions(-) diff --git a/app/models/etsource/dataset.rb b/app/models/etsource/dataset.rb index 111df87f0..54adb0335 100644 --- a/app/models/etsource/dataset.rb +++ b/app/models/etsource/dataset.rb @@ -49,7 +49,6 @@ def self.region_codes(refresh: false) end end - def self.emissions_keys NastyCache.instance.fetch('emission_keys') do Atlas::Dataset.find( diff --git a/app/models/etsource/dataset/import.rb b/app/models/etsource/dataset/import.rb index 99d1dbae5..9d39bd06f 100644 --- a/app/models/etsource/dataset/import.rb +++ b/app/models/etsource/dataset/import.rb @@ -117,7 +117,7 @@ def load_region_data # Internal: Loads the emission data. # - # Returns a hash nested under emisisons_data key for DatasetAttributes compatibility. + # Returns a hash nested under emissions_data key for DatasetAttributes compatibility. def load_emission_data { emissions_data: @atlas_ds.emissions.to_hash } end diff --git a/app/models/qernel/emissions.rb b/app/models/qernel/emissions.rb index 53cc9f2f1..731fa1497 100644 --- a/app/models/qernel/emissions.rb +++ b/app/models/qernel/emissions.rb @@ -60,7 +60,11 @@ def respond_to_missing?(method_name, include_private = false) attr_name = method_name.to_s.delete_suffix('=') data_key = scoped_method(attr_name) - @emissions.respond_to?(data_key) || super + if method_name.to_s.end_with?('=') + scope_exists? + else + @emissions.respond_to?(data_key) || super + end end def method_missing(method_name, *args) @@ -68,11 +72,21 @@ def method_missing(method_name, *args) data_key = scoped_method(attr_name).to_sym if method_name.to_s.end_with?('=') + unless scope_exists? + raise NoMethodError, "undefined method `#{method_name}' for #{inspect}" + end @emissions[data_key] = args.first else @emissions[data_key] end end + + private + + def scope_exists? + prefix = "#{@scope}_" + @emissions.dataset_attributes.keys.any? { |key| key.to_s.start_with?(prefix) } + end end def initialize(graph = nil) diff --git a/spec/models/qernel/emissions_spec.rb b/spec/models/qernel/emissions_spec.rb index 6d1362d65..cab627ef5 100644 --- a/spec/models/qernel/emissions_spec.rb +++ b/spec/models/qernel/emissions_spec.rb @@ -54,6 +54,34 @@ module Qernel end end + describe 'scope validation' do + context 'with valid scope' do + it 'allows creating a ScopedSector instance' do + expect { emissions.scope(:households_non_specified_energetic) }.not_to raise_error + end + end + + context 'with invalid scope (does not exist in dataset)' do + it 'allows creating the ScopedSector instance' do + # Note: Validation happens on setter, not on scope() call + expect { emissions.scope(:invalid_sector_energetic) }.not_to raise_error + end + + it 'raises NoMethodError when trying to set a value' do + invalid_scope = emissions.scope(:invalid_sector_energetic) + expect { invalid_scope.co2 = 100.0 }.to raise_error( + NoMethodError, + /undefined method `co2=' for / + ) + end + + it 'returns nil when trying to get a value (no validation on getter)' do + invalid_scope = emissions.scope(:invalid_sector_energetic) + expect(invalid_scope.co2).to be_nil + end + end + end + describe Emissions::ScopedSector do let(:scoped) { emissions.scope(:households_non_specified_energetic) } @@ -104,6 +132,35 @@ module Qernel end end + describe 'scope existence validation' do + context 'with scope that does not exist in dataset' do + before do + # Create a scope that has no matching keys in dataset + @invalid_scoped = emissions.scope(:nonexistent_sector_energetic) + end + + it 'raises NoMethodError when setting any GHG value' do + expect { @invalid_scoped.co2 = 100.0 }.to raise_error(NoMethodError) + expect { @invalid_scoped.other_ghg = 50.0 }.to raise_error(NoMethodError) + end + + it 'returns nil when getting any GHG value (no validation)' do + expect(@invalid_scoped.co2).to be_nil + expect(@invalid_scoped.other_ghg).to be_nil + end + + it 'does not respond to setter methods' do + expect(@invalid_scoped.respond_to?(:co2=)).to be false + expect(@invalid_scoped.respond_to?(:other_ghg=)).to be false + end + + it 'does not respond to getter methods' do + expect(@invalid_scoped.respond_to?(:co2)).to be false + expect(@invalid_scoped.respond_to?(:other_ghg)).to be false + end + end + end + describe '[]' do before do emissions[:agriculture_non_specified_energetic_other_ghg_2023] = 123.45 From 0ffd5df4a8a081f5987da2f438a9a48cbed48b56 Mon Sep 17 00:00:00 2001 From: louispt1 Date: Thu, 2 Jul 2026 16:41:18 +0200 Subject: [PATCH 14/23] Remove passing the present year around to further simplify 1990 changes --- Gemfile | 2 +- Gemfile.lock | 4 +- app/models/gql/runtime/functions/lookup.rb | 21 +++++----- app/models/qernel/emissions.rb | 37 +++++++---------- spec/models/etsource/dataset/import_spec.rb | 2 +- .../gql/runtime/functions/lookup_spec.rb | 19 ++++++--- spec/models/qernel/emissions_spec.rb | 40 +++++++++---------- 7 files changed, 60 insertions(+), 65 deletions(-) diff --git a/Gemfile b/Gemfile index d83e9429d..8e73e6328 100644 --- a/Gemfile +++ b/Gemfile @@ -73,7 +73,7 @@ gem 'ruby-progressbar' # own gems gem 'quintel_merit', ref: 'aae77e0', github: 'quintel/merit' -gem 'atlas', ref: '72dbea1', github: 'quintel/atlas' #TODO: Update to latest merge commit once Atlas branch is merged into master +gem 'atlas', ref: '5504fac', github: 'quintel/atlas' gem 'fever', ref: '2afebd1', github: 'quintel/fever' gem 'refinery', ref: '36b8e34', github: 'quintel/refinery' gem 'rubel', ref: '9fe7010', github: 'quintel/rubel' diff --git a/Gemfile.lock b/Gemfile.lock index 3ffc5dd64..36acecd4b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ GIT remote: https://github.com/quintel/atlas.git - revision: 72dbea148d09aaae4b38361203f6e98bb29a9efb - ref: 72dbea1 + revision: 5504fac349b1b25fda2ef285b044b025fbfbdb1e + ref: 5504fac specs: atlas (1.0.0) activemodel (>= 7) diff --git a/app/models/gql/runtime/functions/lookup.rb b/app/models/gql/runtime/functions/lookup.rb index 83b1f2f83..501ae566f 100644 --- a/app/models/gql/runtime/functions/lookup.rb +++ b/app/models/gql/runtime/functions/lookup.rb @@ -133,8 +133,6 @@ def MSECTOR(*keys) # Returns an Array with {Qernel::Node} for given energy use. # - # See Qernel::Node::USES - # # Examples # # USE(energetic) @@ -328,9 +326,10 @@ def DATASET_CURVE(key) # Dashes/dots in sector names are converted to underscores for key generation # - use: Emission use type (energetic, non_energetic) - REQUIRED when accessing values # - ghg: GHG type (co2, other_ghg) - optional - # - year: Year of emission (e.g., 1990) - optional, defaults to the analysis year + # - year: Pass 1990 to read the historic baseline; the present year is + # implicit and needs no argument # - # Key generation combines: sector_[subsector_]use_ghg_year + # Key generation combines: sector_[subsector_]use_ghg[_1990] # Note: Unit column from CSV is not included in keys, blank values return nil # # EMISSIONS() without any keys returns {Qernel::Emissions} @@ -351,10 +350,9 @@ def DATASET_CURVE(key) # EMISSIONS(sector, use, ghg) or EMISSIONS(sector, use, ghg, year) returns an emission value # # Examples - # EMISSIONS(households, energetic, other_ghg) # => 12.0 (from emissions.csv) - # EMISSIONS(households, energetic, co2) # => value for default year + # EMISSIONS(households, energetic, other_ghg) # => 12.0 (present year) # EMISSIONS(households, energetic, co2, 1990) # => value for 1990 - # EMISSIONS(buildings_non_specified, energetic, other_ghg, 2023) # => 18.0 + # EMISSIONS(buildings_non_specified, energetic, other_ghg) # => 18.0 # def EMISSIONS(*keys) return scope.graph.emissions if keys.empty? @@ -365,10 +363,11 @@ def EMISSIONS(*keys) # EMISSIONS(sector, use) -> return ScopedSector for UPDATE operations return scope.graph.emissions.scope(keys.join('_').to_sym) if keys.size == 2 - # EMISSIONS(sector, use, ghg [, year]) -> return value - year = keys[3] || scope.graph.area.analysis_year - full_key = "#{keys[0]}_#{keys[1]}_#{keys[2]}_#{year}".to_sym - scope.graph.emissions[full_key] + # EMISSIONS(sector, use, ghg [, year]) -> return value. + # The present year is implicit; only 1990 is suffixed with a year. + scope.graph.emissions[ + [*keys.first(3), (1990 if keys[3].to_i == 1990)].compact.join('_').to_sym + ] end end end diff --git a/app/models/qernel/emissions.rb b/app/models/qernel/emissions.rb index 731fa1497..84696c7be 100644 --- a/app/models/qernel/emissions.rb +++ b/app/models/qernel/emissions.rb @@ -8,16 +8,16 @@ module Qernel # Emissions data is loaded from CSV files in ETSource with structure: # etm_sector, etm_subsector, use, ghg, year, unit, value # - # Values are stored flat, keyed as: sector_subsector_use_ghg_year + # Values are stored flat, keyed as: sector_subsector_use_ghg[_1990] # Examples: - # - buildings_non_specified_energetic_other_ghg_2023 + # - buildings_non_specified_energetic_other_ghg # - energy_electricity_and_heat_production_energetic_co2_1990 # # == Year Handling # - # Year parameter defaults to the dataset's analysis_year when not - # specified. Multiple years coexist in the same dataset (e.g., 1990 - # baseline, 2023 current). + # The present year is implicit and carries no suffix. Only the historic + # 1990 baseline is suffixed with its year, so both can coexist in the + # same dataset. class Emissions include DatasetAttributes @@ -52,37 +52,35 @@ def inspect end def scoped_method(method_name) - year = @year || @emissions.default_year - "#{@scope}_#{method_name}_#{year}" + attr = method_name.to_s.delete_suffix('=') + @year.to_i == 1990 ? "#{@scope}_#{attr}_1990" : "#{@scope}_#{attr}" end def respond_to_missing?(method_name, include_private = false) - attr_name = method_name.to_s.delete_suffix('=') - data_key = scoped_method(attr_name) - if method_name.to_s.end_with?('=') scope_exists? else - @emissions.respond_to?(data_key) || super + @emissions.respond_to?(scoped_method(method_name)) || super end end def method_missing(method_name, *args) - attr_name = method_name.to_s.delete_suffix('=') - data_key = scoped_method(attr_name).to_sym + key = scoped_method(method_name).to_sym if method_name.to_s.end_with?('=') unless scope_exists? raise NoMethodError, "undefined method `#{method_name}' for #{inspect}" end - @emissions[data_key] = args.first + + @emissions[key] = args.first else - @emissions[data_key] + @emissions[key] end end private + # Scoped writes work for any region that has the scope. def scope_exists? prefix = "#{@scope}_" @emissions.dataset_attributes.keys.any? { |key| key.to_s.start_with?(prefix) } @@ -98,18 +96,11 @@ def initialize(graph = nil) # Public: define the sector scope for access to the hashed emission keys # # sector - Scope identifier (e.g., :buildings_non_specified_energetic). - # year - Optional year (defaults to analysis_year). Used to target a - # specific year for UPDATE operations. + # year - Optional year. Pass 1990 to target the historic baseline. # # Returns a scoped version of the emissions data def scope(sector, year = nil) ScopedSector.new(self, sector, year) end - - # Returns the default year for emissions queries. - # Uses the area's analysis_year if graph is present, nil otherwise. - def default_year - graph&.area&.analysis_year - end end end diff --git a/spec/models/etsource/dataset/import_spec.rb b/spec/models/etsource/dataset/import_spec.rb index 2630de70b..c9e9db180 100644 --- a/spec/models/etsource/dataset/import_spec.rb +++ b/spec/models/etsource/dataset/import_spec.rb @@ -7,7 +7,7 @@ let(:emissions) { described_class.new('nl').send(:load_emission_data) } it 'loads flat emission values keyed by joined CSV columns' do - expect(emissions[:emissions_data][:energy_fugitive_emissions_non_energetic_co2_2023]) + expect(emissions[:emissions_data][:energy_fugitive_emissions_non_energetic_co2]) .to eq(20.0) end end diff --git a/spec/models/gql/runtime/functions/lookup_spec.rb b/spec/models/gql/runtime/functions/lookup_spec.rb index b801bd1b1..22a5c8e97 100644 --- a/spec/models/gql/runtime/functions/lookup_spec.rb +++ b/spec/models/gql/runtime/functions/lookup_spec.rb @@ -3,6 +3,7 @@ module Gql::Runtime::Functions describe Lookup, :etsource_fixture do let(:gql) { Scenario.default.gql(prepare: true) } + let(:molecule_graph) { gql.future.molecules } let(:result) do |example| gql.query_future(example.metadata[:example_group][:description]) @@ -50,22 +51,28 @@ module Gql::Runtime::Functions end end - describe "EMISSIONS(buildings_non_specified, energetic, other_ghg, 2023)" do - it 'returns the emission value for the specified year' do + describe "EMISSIONS(buildings_non_specified, energetic, other_ghg)" do + it 'returns the present-year emission value' do expect(result).to eq(2796620.0) end end - describe "EMISSIONS(energy_electricity_and_heat_production, energetic, other_ghg, 2023)" do - it 'returns the emission value for the specified year' do + describe "EMISSIONS(energy_electricity_and_heat_production, energetic, other_ghg)" do + it 'returns the present-year emission value' do expect(result).to eq(18.0) end end - describe 'EMISSIONS(households_non_specified, energetic, other_ghg, 2023)' do - it 'returns the emission value for the specified year' do + describe 'EMISSIONS(households_non_specified, energetic, other_ghg)' do + it 'returns the present-year emission value' do expect(result).to eq(7.0) end end + + describe 'EMISSIONS(households_non_specified, energetic, other_ghg, 1990)' do + it 'returns the 1990 baseline emission value' do + expect(result).to eq(10.0) + end + end end end diff --git a/spec/models/qernel/emissions_spec.rb b/spec/models/qernel/emissions_spec.rb index cab627ef5..0e8751453 100644 --- a/spec/models/qernel/emissions_spec.rb +++ b/spec/models/qernel/emissions_spec.rb @@ -2,15 +2,13 @@ module Qernel describe Emissions do - let(:area) { double('Area', analysis_year: 2023) } - let(:dataset) do Qernel::Dataset.new(1).tap do |ds| ds.data[:emissions] = { emissions_data: data } end end - let(:graph) { double('Graph', area: area, dataset: dataset) } + let(:graph) { double('Graph', dataset: dataset) } let(:emissions) { Emissions.new(graph).tap(&:assign_dataset_attributes) } let(:data) { {} } @@ -86,8 +84,8 @@ module Qernel let(:scoped) { emissions.scope(:households_non_specified_energetic) } before do - emissions[:households_non_specified_energetic_other_ghg_2023] = 50.0 - emissions[:agriculture_non_specified_energetic_other_ghg_2023] = 25.0 + emissions[:households_non_specified_energetic_other_ghg] = 50.0 + emissions[:agriculture_non_specified_energetic_other_ghg] = 25.0 end describe 'GHG accessors' do @@ -102,13 +100,13 @@ module Qernel it 'writes values with the scoped prefix' do scoped.other_ghg = 75.0 - expect(emissions.dataset_get(:households_non_specified_energetic_other_ghg_2023)).to eq(75.0) + expect(emissions.dataset_get(:households_non_specified_energetic_other_ghg)).to eq(75.0) end it 'writes values for a different scope' do other_scoped = emissions.scope(:agriculture_non_specified_energetic) other_scoped.other_ghg = 30.0 - expect(emissions.dataset_get(:agriculture_non_specified_energetic_other_ghg_2023)).to eq(30.0) + expect(emissions.dataset_get(:agriculture_non_specified_energetic_other_ghg)).to eq(30.0) end it 'returns nil when the scope has no value for the GHG' do @@ -117,7 +115,7 @@ module Qernel it 'allows setting a GHG which has no value yet (runtime UPDATE values)' do scoped.co2 = 300.0 - expect(emissions.dataset_get(:households_non_specified_energetic_co2_2023)).to eq(300.0) + expect(emissions.dataset_get(:households_non_specified_energetic_co2)).to eq(300.0) end end @@ -163,7 +161,7 @@ module Qernel describe '[]' do before do - emissions[:agriculture_non_specified_energetic_other_ghg_2023] = 123.45 + emissions[:agriculture_non_specified_energetic_other_ghg] = 123.45 end let(:scoped) { emissions.scope(:agriculture_non_specified_energetic) } @@ -182,7 +180,7 @@ module Qernel it 'sets the value' do scoped[:other_ghg] = 999.0 - expect(emissions.dataset_get(:industry_non_specified_energetic_other_ghg_2023)).to eq(999.0) + expect(emissions.dataset_get(:industry_non_specified_energetic_other_ghg)).to eq(999.0) end end @@ -196,22 +194,22 @@ module Qernel let(:scoped) { emissions.scope(:energy_fugitive_emissions_non_energetic) } before do - emissions[:energy_fugitive_emissions_non_energetic_co2_2023] = 0.0 - emissions[:energy_electricity_and_heat_production_energetic_other_ghg_2023] = 0.0 - emissions[:buildings_non_specified_energetic_other_ghg_2023] = 0.0 - emissions[:agriculture_non_specified_energetic_other_ghg_2023] = 0.0 - emissions[:agriculture_non_specified_non_energetic_co2_2023] = 0.0 + emissions[:energy_fugitive_emissions_non_energetic_co2] = 0.0 + emissions[:energy_electricity_and_heat_production_energetic_other_ghg] = 0.0 + emissions[:buildings_non_specified_energetic_other_ghg] = 0.0 + emissions[:agriculture_non_specified_energetic_other_ghg] = 0.0 + emissions[:agriculture_non_specified_non_energetic_co2] = 0.0 end it 'handles zero values' do scoped.co2 = 0.0 - expect(emissions.dataset_get(:energy_fugitive_emissions_non_energetic_co2_2023)).to eq(0.0) + expect(emissions.dataset_get(:energy_fugitive_emissions_non_energetic_co2)).to eq(0.0) end it 'handles large values' do buildings_scoped = emissions.scope(:buildings_non_specified_energetic) buildings_scoped.other_ghg = 9999999.0 - expect(emissions.dataset_get(:buildings_non_specified_energetic_other_ghg_2023)).to eq(9999999.0) + expect(emissions.dataset_get(:buildings_non_specified_energetic_other_ghg)).to eq(9999999.0) end it 'handles multi-word subsector scopes' do @@ -219,20 +217,20 @@ module Qernel # Scope: energy_electricity_and_heat_production_energetic multi_scoped = emissions.scope(:energy_electricity_and_heat_production_energetic) multi_scoped.other_ghg = 275.0 - expect(emissions.dataset_get(:energy_electricity_and_heat_production_energetic_other_ghg_2023)).to eq(275.0) + expect(emissions.dataset_get(:energy_electricity_and_heat_production_energetic_other_ghg)).to eq(275.0) end it 'works with multi-part keys from real dataset' do scoped.co2 = 100.0 - expect(emissions.dataset_get(:energy_fugitive_emissions_non_energetic_co2_2023)).to eq(100.0) + expect(emissions.dataset_get(:energy_fugitive_emissions_non_energetic_co2)).to eq(100.0) ag_energetic = emissions.scope(:agriculture_non_specified_energetic) ag_energetic.other_ghg = 200.0 - expect(emissions.dataset_get(:agriculture_non_specified_energetic_other_ghg_2023)).to eq(200.0) + expect(emissions.dataset_get(:agriculture_non_specified_energetic_other_ghg)).to eq(200.0) ag_non_energetic = emissions.scope(:agriculture_non_specified_non_energetic) ag_non_energetic.co2 = 300.0 - expect(emissions.dataset_get(:agriculture_non_specified_non_energetic_co2_2023)).to eq(300.0) + expect(emissions.dataset_get(:agriculture_non_specified_non_energetic_co2)).to eq(300.0) end end end From 2ea553295454f8ece9349877b1f4e943edd26ebd Mon Sep 17 00:00:00 2001 From: louispt1 Date: Thu, 2 Jul 2026 16:41:48 +0200 Subject: [PATCH 15/23] Add MUSE method and spec --- app/models/gql/query_interface/lookup.rb | 4 +++ app/models/gql/runtime/functions/lookup.rb | 5 ++++ .../graphs/molecules/nodes/m_right_one.ad | 1 + .../graphs/molecules/nodes/m_right_two.ad | 1 + .../gql/runtime/functions/lookup_spec.rb | 30 +++++++++++++++++++ 5 files changed, 41 insertions(+) diff --git a/app/models/gql/query_interface/lookup.rb b/app/models/gql/query_interface/lookup.rb index 48f4aa88d..62c20c12d 100644 --- a/app/models/gql/query_interface/lookup.rb +++ b/app/models/gql/query_interface/lookup.rb @@ -68,6 +68,10 @@ def energy_use_nodes(keys) energy_graph_helper.use_nodes(keys) end + def molecule_use_nodes(keys) + molecule_graph_helper.use_nodes(keys) + end + def group_energy_nodes(keys) energy_graph_helper.group_nodes(keys) end diff --git a/app/models/gql/runtime/functions/lookup.rb b/app/models/gql/runtime/functions/lookup.rb index 501ae566f..f15c80ed4 100644 --- a/app/models/gql/runtime/functions/lookup.rb +++ b/app/models/gql/runtime/functions/lookup.rb @@ -143,6 +143,11 @@ def USE(*keys) scope.energy_use_nodes(keys) end + # Returns an Array of {Qernel::Node} for given molecule use. See USE. + def MUSE(*keys) + scope.molecule_use_nodes(keys) + end + # Returns an Array of {Qernel::Carrier} for given key(s). Returns carriers belonging to the # energy graph. # diff --git a/spec/fixtures/etsource/graphs/molecules/nodes/m_right_one.ad b/spec/fixtures/etsource/graphs/molecules/nodes/m_right_one.ad index e9fdc0737..b2102dcac 100644 --- a/spec/fixtures/etsource/graphs/molecules/nodes/m_right_one.ad +++ b/spec/fixtures/etsource/graphs/molecules/nodes/m_right_one.ad @@ -1,2 +1,3 @@ - output.co2 = 1.0 +- use = energetic ~ demand = 0.75 diff --git a/spec/fixtures/etsource/graphs/molecules/nodes/m_right_two.ad b/spec/fixtures/etsource/graphs/molecules/nodes/m_right_two.ad index 9067dc98a..211a9f6fa 100644 --- a/spec/fixtures/etsource/graphs/molecules/nodes/m_right_two.ad +++ b/spec/fixtures/etsource/graphs/molecules/nodes/m_right_two.ad @@ -1,2 +1,3 @@ - output.co2 = 1.0 +- use = non_energetic ~ demand = 0.25 diff --git a/spec/models/gql/runtime/functions/lookup_spec.rb b/spec/models/gql/runtime/functions/lookup_spec.rb index 22a5c8e97..af3eff0e2 100644 --- a/spec/models/gql/runtime/functions/lookup_spec.rb +++ b/spec/models/gql/runtime/functions/lookup_spec.rb @@ -74,5 +74,35 @@ module Gql::Runtime::Functions expect(result).to eq(10.0) end end + + # MUSE + # ---- + + describe 'MUSE(energetic)' do + it 'returns [Node(m_right_one)]' do + expect(result).to eq([molecule_graph.node(:m_right_one)]) + end + end + + describe 'MUSE(non_energetic)' do + it 'returns [Node(m_right_two)]' do + expect(result).to eq([molecule_graph.node(:m_right_two)]) + end + end + + describe 'MUSE(energetic, non_energetic)' do + it 'returns [Node(m_right_one), Node(m_right_two)]' do + expect(result).to eq([ + molecule_graph.node(:m_right_one), + molecule_graph.node(:m_right_two) + ]) + end + end + + describe 'MUSE(undefined)' do + it 'returns []' do + expect(result).to eq([]) + end + end end end From d3211d38656e8901b7c50792d5601cbfd86c1ad6 Mon Sep 17 00:00:00 2001 From: louispt1 Date: Wed, 8 Jul 2026 17:00:17 +0200 Subject: [PATCH 16/23] Add sector mapping handling and rspec --- app/models/etsource/sectors.rb | 50 ++++++ app/models/gql/gql_error.rb | 17 ++ app/models/gql/query_interface/lookup.rb | 16 ++ app/models/gql/runtime/functions/lookup.rb | 57 +++++- app/models/qernel/graph.rb | 12 ++ app/models/qernel/sectors.rb | 73 ++++++++ .../etsource/config/sector_mapping.csv | 12 ++ .../buildings_space_heating_demand.ad | 2 + .../graphs/energy/nodes/nosector/bar.ad | 2 + .../graphs/energy/nodes/nosector/baz.ad | 2 + .../graphs/energy/nodes/nosector/foo.ad | 2 + .../graphs/energy/nodes/nosector/lft.ad | 4 + .../etsource/graphs/molecules/nodes/m_left.ad | 2 + .../gql/runtime/functions/lookup_spec.rb | 163 ++++++++++++++++++ 14 files changed, 408 insertions(+), 6 deletions(-) create mode 100644 app/models/etsource/sectors.rb create mode 100644 app/models/qernel/sectors.rb create mode 100644 spec/fixtures/etsource/config/sector_mapping.csv diff --git a/app/models/etsource/sectors.rb b/app/models/etsource/sectors.rb new file mode 100644 index 000000000..b8cecea26 --- /dev/null +++ b/app/models/etsource/sectors.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Etsource + # Reads the sector mapping and node labels from Atlas and caches the two + # structures GQL needs to resolve `SECTOR(scheme, value)` style queries. + # + # Mirrors the merit order import precedent: the expensive Atlas read happens + # once and the result is memoized in the Rails cache, so per-request cost is a + # couple of hash lookups. + # + # * mapping - {scheme => {value => [[label, use], ...]}} + # * node_index(g) - {[label, use] => [node_key, ...]} for one graph type, + # with `use` baked in at import so it is never consulted + # per query. + class Sectors + def initialize(etsource = Etsource::Base.instance) + @etsource = etsource + end + + # Public: The inverted mapping index, keyed by scheme then normalized value. + def mapping + Rails.cache.fetch('sector_mapping_hash') do + Atlas::SectorMapping.load.to_h + end + end + + # Public: The (label, use) -> node keys index for the given graph type + # (:energy or :molecules). + def node_index(graph_type) + Rails.cache.fetch("sector_node_index_#{graph_type}") do + build_node_index(node_class_for(graph_type)) + end + end + + private + + def build_node_index(node_class) + node_class.all.each_with_object({}) do |node, index| + next if node.sector_label.nil? + + pair = [node.sector_label, Atlas::SectorMapping.normalize(node.use)] + (index[pair] ||= []) << node.key + end + end + + def node_class_for(graph_type) + graph_type.to_sym == :molecules ? Atlas::MoleculeNode : Atlas::EnergyNode + end + end +end diff --git a/app/models/gql/gql_error.rb b/app/models/gql/gql_error.rb index 462b1a5de..d016999cf 100644 --- a/app/models/gql/gql_error.rb +++ b/app/models/gql/gql_error.rb @@ -23,4 +23,21 @@ def initialize(curve, attribute = nil) end end + # Raised when a SECTOR/MSECTOR/EMISSIONS scheme-form query names a + # classification scheme which does not exist in the sector mapping. + class UnknownSectorSchemeError < GqlError + def initialize(scheme, valid_schemes) + super("Unknown sector mapping scheme #{scheme.inspect}. " \ + "Valid schemes: #{valid_schemes.map(&:inspect).join(', ')}.") + end + end + + # Raised when a SECTOR/MSECTOR/EMISSIONS scheme-form query names a value + # which does not appear in that scheme's column. + class UnknownSectorValueError < GqlError + def initialize(scheme, value) + super("Unknown value #{value.inspect} for sector mapping scheme " \ + "#{scheme.inspect}.") + end + end end diff --git a/app/models/gql/query_interface/lookup.rb b/app/models/gql/query_interface/lookup.rb index 62c20c12d..ca372b175 100644 --- a/app/models/gql/query_interface/lookup.rb +++ b/app/models/gql/query_interface/lookup.rb @@ -64,6 +64,22 @@ def molecule_sector_nodes(keys) molecule_graph_helper.sector_nodes(keys) end + # Scheme-based sector mapping lookups. `energy`/`molecule` select the + # graph whose live nodes are returned; the mapping itself is shared. + def energy_sector_map(scheme, values) + graph.sector_map.lookup(scheme, values) + end + + def molecule_sector_map(scheme, values) + molecules.sector_map.lookup(scheme, values) + end + + # The energy graph's resolver, used for EMISSIONS scheme dispatch and + # pair resolution (the mapping is graph-independent). + def sector_resolver + graph.sector_map + end + def energy_use_nodes(keys) energy_graph_helper.use_nodes(keys) end diff --git a/app/models/gql/runtime/functions/lookup.rb b/app/models/gql/runtime/functions/lookup.rb index f15c80ed4..d66e384bd 100644 --- a/app/models/gql/runtime/functions/lookup.rb +++ b/app/models/gql/runtime/functions/lookup.rb @@ -116,19 +116,36 @@ def MEDGE_GROUP(*keys) end alias MEG EDGE_GROUP - # Returns an Array of {Qernel::Node} for given energy sector. + # Returns an Array of {Qernel::Node} for an energy sector. # - # Examples + # Dispatches on arity: # - # SECTOR(households) + # * One argument - namespace-sector filter. + # SECTOR(households) + # * Two or more - a classification scheme followed by one or more + # values; returns the union of matching nodes. + # SECTOR(klimaattafel, 'Industrie') + # SECTOR(ipcc_crt_code_agg, '1.A.1', '1.A.2') # + # Unknown scheme or value raises; a valid value with no labelled nodes + # returns an empty array. def SECTOR(*keys) - scope.energy_sector_nodes(keys) + if keys.size <= 1 + scope.energy_sector_nodes(keys) + else + scope.energy_sector_map(keys.first, keys.drop(1)) + end end - # Returns an Array of {Qernel::Node} for given molecule sector. See SECTOR. + # Returns an Array of {Qernel::Node} for a molecule sector. See SECTOR; + # identical dispatch, normalization and error contract, resolved against + # the molecule graph. def MSECTOR(*keys) - scope.molecule_sector_nodes(keys) + if keys.size <= 1 + scope.molecule_sector_nodes(keys) + else + scope.molecule_sector_map(keys.first, keys.drop(1)) + end end # Returns an Array with {Qernel::Node} for given energy use. @@ -362,6 +379,12 @@ def DATASET_CURVE(key) def EMISSIONS(*keys) return scope.graph.emissions if keys.empty? + # Mapped form: EMISSIONS(scheme, value, ghg[, 1990]). Dispatches on the + # first argument being a known classification scheme, because the legacy + # arities overlap. Read-only: sums the store over the mapping's resolved + # (sector label, use) pairs; never returns a scoped-sector handle. + return mapped_emissions(keys) if scope.sector_resolver.scheme?(keys.first) + # Convert dashes/dots to underscores in the first key (sector) keys[0] = keys.first.to_s.tr('-.', '_').to_sym @@ -374,6 +397,28 @@ def EMISSIONS(*keys) [*keys.first(3), (1990 if keys[3].to_i == 1990)].compact.join('_').to_sym ] end + + private + + # Internal: Sums the emissions store over every (sector label, use) pair + # the mapping resolves for `scheme`/`value`. Present year is implicit; + # only 1990 is suffixed. Missing store keys count as zero. + def mapped_emissions(keys) + scheme, value, ghg, year = keys + + if ghg.nil? + raise Gql::GqlError, "EMISSIONS(#{scheme.inspect}, ...) needs a GHG argument, e.g. " \ + "EMISSIONS(#{scheme.inspect}, #{value.inspect}, co2)" + end + + suffix = ('1990' if year.to_i == 1990) + emissions = scope.graph.emissions + + scope.sector_resolver.pairs(scheme, value).sum do |label, use| + key = [label, use, ghg, suffix].compact.join('_').to_sym + emissions[key] || 0.0 + end + end end end end diff --git a/app/models/qernel/graph.rb b/app/models/qernel/graph.rb index 208a3605e..756d70d79 100644 --- a/app/models/qernel/graph.rb +++ b/app/models/qernel/graph.rb @@ -264,6 +264,18 @@ def sectors nodes.map(&:sector_key).uniq.compact end + # Resolver for scheme-based sector mapping queries (SECTOR(scheme, value)). + # + # Memoized per graph instance. + # + # @return [Qernel::Sectors] + def sector_map + @sector_map ||= begin + sectors = Etsource::Sectors.new + Qernel::Sectors.new(self, sectors.mapping, sectors.node_index(@name)) + end + end + # Return all nodes in the given sector. # # @param sector_key [String,Symbol] sector identifier diff --git a/app/models/qernel/sectors.rb b/app/models/qernel/sectors.rb new file mode 100644 index 000000000..630895d5e --- /dev/null +++ b/app/models/qernel/sectors.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module Qernel + # Per-graph-instance resolver for scheme-based sector queries. + # + # Present and future graphs (and the energy and molecule graphs) are separate + # instances holding separate node objects, so each gets its own resolver via + # the graph's `sector_map` accessor. The mapping and the (label, use) -> node + # key index are read once from {Etsource::Sectors}; resolution turns node keys + # into the live nodes of this graph. + # + # Dispatch, arity and aggregation live in the GQL layer; this class answers + # two questions: which (label, use) pairs, and which live nodes, belong to + # (scheme, values)? + class Sectors + def initialize(graph, mapping, node_index) + @graph = graph + @mapping = mapping + @node_index = node_index + @node_cache = {} + @pair_cache = {} + end + + # Public: Whether `scheme`` names a classification scheme in the mapping. + # Used by the EMISSIONS first-argument dispatch. + def scheme?(scheme) + @mapping.key?(normalize(scheme)) + end + + # Public: The unique (label, use) pairs falling under any of `values` of + # `scheme`. Reads the mapping only, so it is independent of node labelling; + # the EMISSIONS mapped form sums the store over these pairs. + # + # Raises Gql::UnknownSectorSchemeError / Gql::UnknownSectorValueError. + def pairs(scheme, values) + value_list = Array(values) + key = [normalize(scheme), value_list.map { |value| normalize(value) }] + + @pair_cache[key] ||= resolve_pairs(scheme, value_list) + end + + # Public: The live nodes of this graph whose (label, use) pair falls under + # any of `values` of 'scheme` (the union). A value which resolves to pairs + # with no labelled nodes contributes nothing. + def lookup(scheme, values) + value_list = Array(values) + key = [normalize(scheme), value_list.map { |value| normalize(value) }] + + @node_cache[key] ||= + pairs(scheme, value_list).flat_map { |pair| nodes_for(pair) }.uniq + end + + private + + def resolve_pairs(scheme, value_list) + value_map = @mapping[normalize(scheme)] + raise Gql::UnknownSectorSchemeError.new(scheme, @mapping.keys) if value_map.nil? + + value_list.flat_map do |value| + value_map[normalize(value)] || + raise(Gql::UnknownSectorValueError.new(scheme, value)) + end.uniq + end + + def nodes_for(pair) + Array(@node_index[pair]).filter_map { |key| @graph.node(key) } + end + + def normalize(value) + Atlas::SectorMapping.normalize(value) + end + end +end diff --git a/spec/fixtures/etsource/config/sector_mapping.csv b/spec/fixtures/etsource/config/sector_mapping.csv new file mode 100644 index 000000000..eb2e1e87e --- /dev/null +++ b/spec/fixtures/etsource/config/sector_mapping.csv @@ -0,0 +1,12 @@ +sector_label,use,ipcc_crt_code_agg,klimaattafel +energy_electricity_and_heat_production,energetic,1.A.1,Elektriciteit +industry_refineries,energetic,1.A.1,Industrie +industry_refineries,non_energetic,1.B,Industrie +industry_non_specified,energetic,1.A.2,Industrie +buildings_non_specified,energetic,1.A.4.a,Gebouwde omgeving +households_non_specified,energetic,1.A.4.b,Gebouwde omgeving +agriculture_non_specified,energetic,1.A.4.c,Landbouw +agriculture_non_specified,non_energetic,3,Landbouw +waste_non_specified,non_energetic,5,Elektriciteit +energy_fugitive_emissions,non_energetic,1.B,Elektriciteit +other_heating,energetic,-,Mobiliteit diff --git a/spec/fixtures/etsource/graphs/energy/nodes/buildings/buildings_space_heating_demand.ad b/spec/fixtures/etsource/graphs/energy/nodes/buildings/buildings_space_heating_demand.ad index ef7605b8d..70ed26212 100644 --- a/spec/fixtures/etsource/graphs/energy/nodes/buildings/buildings_space_heating_demand.ad +++ b/spec/fixtures/etsource/graphs/energy/nodes/buildings/buildings_space_heating_demand.ad @@ -6,3 +6,5 @@ - demand = 100.0 - preset_demand = 100.0 - number_of_units = 200.0 +- sector_label = buildings_non_specified +- use = energetic diff --git a/spec/fixtures/etsource/graphs/energy/nodes/nosector/bar.ad b/spec/fixtures/etsource/graphs/energy/nodes/nosector/bar.ad index 1b98c7d08..7d0e985d0 100644 --- a/spec/fixtures/etsource/graphs/energy/nodes/nosector/bar.ad +++ b/spec/fixtures/etsource/graphs/energy/nodes/nosector/bar.ad @@ -1 +1,3 @@ - groups = [application_group] +- sector_label = industry_refineries +- use = energetic diff --git a/spec/fixtures/etsource/graphs/energy/nodes/nosector/baz.ad b/spec/fixtures/etsource/graphs/energy/nodes/nosector/baz.ad index d2f2e56cb..150b40384 100644 --- a/spec/fixtures/etsource/graphs/energy/nodes/nosector/baz.ad +++ b/spec/fixtures/etsource/graphs/energy/nodes/nosector/baz.ad @@ -1,2 +1,4 @@ - groups = [] - graph_methods = [demand_setter] +- sector_label = industry_refineries +- use = non_energetic diff --git a/spec/fixtures/etsource/graphs/energy/nodes/nosector/foo.ad b/spec/fixtures/etsource/graphs/energy/nodes/nosector/foo.ad index 636979b5d..e158cc9e9 100644 --- a/spec/fixtures/etsource/graphs/energy/nodes/nosector/foo.ad +++ b/spec/fixtures/etsource/graphs/energy/nodes/nosector/foo.ad @@ -3,3 +3,5 @@ - merit_order.type = dispatchable - merit_order.group = solar_pv - graph_methods = [demand_setter] +- sector_label = industry_non_specified +- use = energetic diff --git a/spec/fixtures/etsource/graphs/energy/nodes/nosector/lft.ad b/spec/fixtures/etsource/graphs/energy/nodes/nosector/lft.ad index 27c3b9df6..1254ea0ec 100644 --- a/spec/fixtures/etsource/graphs/energy/nodes/nosector/lft.ad +++ b/spec/fixtures/etsource/graphs/energy/nodes/nosector/lft.ad @@ -1,2 +1,6 @@ - groups = [] - preset_demand = 0.0 + +# Sector mapping test assignment (blank-cell case: "-" ipcc, reachable via klimaattafel) +- sector_label = other_heating +- use = energetic diff --git a/spec/fixtures/etsource/graphs/molecules/nodes/m_left.ad b/spec/fixtures/etsource/graphs/molecules/nodes/m_left.ad index 89d1c5a8a..cb14c83bb 100644 --- a/spec/fixtures/etsource/graphs/molecules/nodes/m_left.ad +++ b/spec/fixtures/etsource/graphs/molecules/nodes/m_left.ad @@ -1,3 +1,5 @@ - groups = [emissions] - from_energy.source = molecule_source - input.co2 = 1.0 +- sector_label = waste_non_specified +- use = non_energetic diff --git a/spec/models/gql/runtime/functions/lookup_spec.rb b/spec/models/gql/runtime/functions/lookup_spec.rb index af3eff0e2..693f7e367 100644 --- a/spec/models/gql/runtime/functions/lookup_spec.rb +++ b/spec/models/gql/runtime/functions/lookup_spec.rb @@ -104,5 +104,168 @@ module Gql::Runtime::Functions expect(result).to eq([]) end end + + # SECTOR - scheme-based mapping lookups (energy graph) + # --------------------------------------------------- + + def keys_of(nodes) + nodes.map(&:key).sort + end + + describe 'legacy one-argument SECTOR' do + it 'is unchanged: returns the namespace-sector nodes' do + legacy = gql.query_future('SECTOR(nosector)') + direct = gql.future_graph.sector_nodes(:nosector) + + expect(keys_of(legacy)).to eq(keys_of(direct)) + end + end + + describe "SECTOR(klimaattafel, 'Industrie')" do + it 'returns the labelled energy nodes in that klimaattafel' do + expect(keys_of(result)).to eq(%i[bar baz foo]) + end + end + + describe "SECTOR(ipcc_crt_code_agg, '1.A.1')" do + it 'returns only the energetic-use node for that IPCC category' do + expect(keys_of(result)).to eq(%i[bar]) + end + end + + describe "SECTOR(ipcc_crt_code_agg, '1.B')" do + it 'returns only the non-energetic-use node (pair correctness)' do + expect(keys_of(result)).to eq(%i[baz]) + end + end + + describe "SECTOR(ipcc_crt_code_agg, '1.A.1', '1.A.2')" do + it 'returns the union over several values' do + expect(keys_of(result)).to eq(%i[bar foo]) + end + end + + describe 'SECTOR(sector_label, industry_refineries)' do + it 'resolves the canonical key directly (both uses)' do + expect(keys_of(result)).to eq(%i[bar baz]) + end + end + + describe "SECTOR(klimaattafel, 'Landbouw')" do + it 'returns [] for a valid value with no labelled nodes' do + expect(result).to eq([]) + end + end + + describe "SECTOR(impossible, 'x')" do + it 'raises naming the valid schemes' do + expect { result }.to raise_error(/Valid schemes.*klimaattafel/m) + end + end + + describe "SECTOR(klimaattafel, 'Nonexistent')" do + it 'raises for an unknown value' do + expect { result }.to raise_error(/Unknown value "Nonexistent"/) + end + end + + # Blank / "-" cells. The `other_heating,energetic` row has a "-" ipcc cell + # (fixture) and the `lft` node is labelled to it. + + describe "SECTOR(klimaattafel, 'Mobiliteit')" do + it 'reaches a "-"-celled row through a populated scheme' do + expect(keys_of(result)).to eq(%i[lft]) + end + end + + describe 'SECTOR(sector_label, other_heating)' do + it 'reaches the "-"-celled row by its canonical key' do + expect(keys_of(result)).to eq(%i[lft]) + end + end + + describe "SECTOR(ipcc_crt_code_agg, '-')" do + it 'raises: a "-" cell is unqueryable, not a category' do + expect { result }.to raise_error(/Unknown value/) + end + end + + describe "EMISSIONS(ipcc_crt_code_agg, '-', co2)" do + it 'raises: the mapped form rejects a blank value too' do + expect { result }.to raise_error(/Unknown value/) + end + end + + # MSECTOR - scheme-based mapping lookups (molecule graph) + # ------------------------------------------------------ + + describe "MSECTOR(ipcc_crt_code_agg, '5')" do + it 'returns the labelled molecule node' do + expect(keys_of(result)).to eq(%i[m_left]) + end + end + + describe "MSECTOR(klimaattafel, 'Elektriciteit')" do + it 'returns only molecule nodes (disjoint from the energy graph)' do + expect(keys_of(result)).to eq(%i[m_left]) + end + end + + describe "SECTOR(ipcc_crt_code_agg, '5')" do + it 'returns [] on the energy graph for a molecule-only pair' do + expect(result).to eq([]) + end + end + + describe 'legacy one-argument MSECTOR' do + it 'is unchanged: routes to the namespace-sector molecule filter' do + expect(gql.query_future('MSECTOR(anything)')) + .to eq(molecule_graph.sector_nodes(:anything)) + end + end + + # EMISSIONS - mapped scheme form + # ------------------------------ + + describe "EMISSIONS(ipcc_crt_code_agg, '1.A.1', other_ghg)" do + it 'sums the store over the resolved pairs (missing keys as zero)' do + # energy_electricity_and_heat_production_energetic_other_ghg = 18.0 + # industry_refineries_energetic_other_ghg is absent -> 0.0 + expect(result).to eq(18.0) + end + end + + describe "EMISSIONS(ipcc_crt_code_agg, '1.A.1', other_ghg, 1990)" do + it 'reads the 1990 baseline keys' do + expect(result).to eq(25.0) + end + end + + describe "EMISSIONS(ipcc_crt_code_agg, '3', co2)" do + it 'sums only the non-energetic rows the mapping lists' do + # agriculture_non_specified_non_energetic_co2 = 75.0; the energetic + # agriculture row (also Landbouw) is not under IPCC 3. + expect(result).to eq(75.0) + end + end + + describe "EMISSIONS(klimaattafel, 'Landbouw', other_ghg)" do + it 'unions energetic and non-energetic rows of the value' do + # agriculture energetic (50.0) + agriculture non_energetic (100.0) + expect(result).to eq(150.0) + end + end + + describe "EMISSIONS(ipcc_crt_code_agg, '1.A.1')" do + it 'raises without a GHG argument' do + expect { result }.to raise_error(Gql::GqlError, /needs a GHG argument/) + end + end + + it 'rejects UPDATE on a mapped EMISSIONS expression' do + expect do + gql.query_future("UPDATE(EMISSIONS(ipcc_crt_code_agg, '1.A.1', co2), co2, 1.0)") + end.to raise_error(StandardError) + end end end From 510cb141708a6b5879b3eed6ef37788323ef9079 Mon Sep 17 00:00:00 2001 From: louispt1 Date: Wed, 8 Jul 2026 17:22:50 +0200 Subject: [PATCH 17/23] New fixture to avoid conflicting with MUSE spec --- .../etsource/graphs/molecules/nodes/m_left.ad | 2 -- .../graphs/molecules/nodes/nosector/m_waste.ad | 3 +++ spec/models/gql/runtime/functions/lookup_spec.rb | 16 ++++++++++------ 3 files changed, 13 insertions(+), 8 deletions(-) create mode 100644 spec/fixtures/etsource/graphs/molecules/nodes/nosector/m_waste.ad diff --git a/spec/fixtures/etsource/graphs/molecules/nodes/m_left.ad b/spec/fixtures/etsource/graphs/molecules/nodes/m_left.ad index cb14c83bb..89d1c5a8a 100644 --- a/spec/fixtures/etsource/graphs/molecules/nodes/m_left.ad +++ b/spec/fixtures/etsource/graphs/molecules/nodes/m_left.ad @@ -1,5 +1,3 @@ - groups = [emissions] - from_energy.source = molecule_source - input.co2 = 1.0 -- sector_label = waste_non_specified -- use = non_energetic diff --git a/spec/fixtures/etsource/graphs/molecules/nodes/nosector/m_waste.ad b/spec/fixtures/etsource/graphs/molecules/nodes/nosector/m_waste.ad new file mode 100644 index 000000000..8ce379f39 --- /dev/null +++ b/spec/fixtures/etsource/graphs/molecules/nodes/nosector/m_waste.ad @@ -0,0 +1,3 @@ +- groups = [] +- sector_label = waste_non_specified +- use = non_energetic diff --git a/spec/models/gql/runtime/functions/lookup_spec.rb b/spec/models/gql/runtime/functions/lookup_spec.rb index 693f7e367..acf7c72b8 100644 --- a/spec/models/gql/runtime/functions/lookup_spec.rb +++ b/spec/models/gql/runtime/functions/lookup_spec.rb @@ -85,16 +85,20 @@ module Gql::Runtime::Functions end describe 'MUSE(non_energetic)' do - it 'returns [Node(m_right_two)]' do - expect(result).to eq([molecule_graph.node(:m_right_two)]) + it 'returns [Node(m_right_two), Node(m_waste)]' do + expect(result).to eq([ + molecule_graph.node(:m_right_two), + molecule_graph.node(:m_waste) + ]) end end describe 'MUSE(energetic, non_energetic)' do - it 'returns [Node(m_right_one), Node(m_right_two)]' do + it 'returns [Node(m_right_one), Node(m_right_two), Node(m_waste)]' do expect(result).to eq([ molecule_graph.node(:m_right_one), - molecule_graph.node(:m_right_two) + molecule_graph.node(:m_right_two), + molecule_graph.node(:m_waste) ]) end end @@ -201,13 +205,13 @@ def keys_of(nodes) describe "MSECTOR(ipcc_crt_code_agg, '5')" do it 'returns the labelled molecule node' do - expect(keys_of(result)).to eq(%i[m_left]) + expect(keys_of(result)).to eq(%i[m_waste]) end end describe "MSECTOR(klimaattafel, 'Elektriciteit')" do it 'returns only molecule nodes (disjoint from the energy graph)' do - expect(keys_of(result)).to eq(%i[m_left]) + expect(keys_of(result)).to eq(%i[m_waste]) end end From f0319c17b5f8475ca0db1ed64ca92f954b44377c Mon Sep 17 00:00:00 2001 From: louispt1 Date: Tue, 14 Jul 2026 11:01:42 +0200 Subject: [PATCH 18/23] Use nastycache for csv mapping, rename methods and move summing to the emissions class --- app/models/etsource/sectors.rb | 7 +-- app/models/gql/command.rb | 9 ++-- app/models/gql/query_interface/lookup.rb | 2 +- app/models/gql/runtime/functions/lookup.rb | 23 +++----- app/models/qernel/emissions.rb | 29 ++++++++-- app/models/qernel/sectors.rb | 6 +-- spec/models/gql/command_spec.rb | 35 ++++++++++++ .../gql/runtime/functions/lookup_spec.rb | 6 +++ spec/models/qernel/emissions_spec.rb | 54 +++++++++++++++++++ 9 files changed, 139 insertions(+), 32 deletions(-) create mode 100644 spec/models/gql/command_spec.rb diff --git a/app/models/etsource/sectors.rb b/app/models/etsource/sectors.rb index b8cecea26..0967f002a 100644 --- a/app/models/etsource/sectors.rb +++ b/app/models/etsource/sectors.rb @@ -4,9 +4,6 @@ module Etsource # Reads the sector mapping and node labels from Atlas and caches the two # structures GQL needs to resolve `SECTOR(scheme, value)` style queries. # - # Mirrors the merit order import precedent: the expensive Atlas read happens - # once and the result is memoized in the Rails cache, so per-request cost is a - # couple of hash lookups. # # * mapping - {scheme => {value => [[label, use], ...]}} # * node_index(g) - {[label, use] => [node_key, ...]} for one graph type, @@ -19,7 +16,7 @@ def initialize(etsource = Etsource::Base.instance) # Public: The inverted mapping index, keyed by scheme then normalized value. def mapping - Rails.cache.fetch('sector_mapping_hash') do + NastyCache.instance.fetch('sector_mapping_hash') do Atlas::SectorMapping.load.to_h end end @@ -27,7 +24,7 @@ def mapping # Public: The (label, use) -> node keys index for the given graph type # (:energy or :molecules). def node_index(graph_type) - Rails.cache.fetch("sector_node_index_#{graph_type}") do + NastyCache.instance.fetch("sector_node_index_#{graph_type}") do build_node_index(node_class_for(graph_type)) end end diff --git a/app/models/gql/command.rb b/app/models/gql/command.rb index 653333df1..73546f37d 100644 --- a/app/models/gql/command.rb +++ b/app/models/gql/command.rb @@ -108,12 +108,15 @@ def truncate(string, length = 40) string.length > (length - 3) ? "#{ string[0..length] }..." : string end - # Internal: Removes extra "artifacts" from the source. + # Internal: Removes extra "artifacts" from the source. Whitespace is + # insignificant in GQL and stripped, except inside string literals, where + # it may be meaningful (e.g. SECTOR scheme values such as 'Fuels + # production'). def clean(string) cleaned = (string || '').dup - cleaned.gsub!(/[\n\s\t]/, '') - cleaned.gsub!(/^[a-z]+\:/,'') + cleaned.gsub!(/('[^']*'|"[^"]*")|\s+/) { Regexp.last_match(1) || '' } + cleaned.sub!(/\A[a-z]+\:/, '') cleaned end diff --git a/app/models/gql/query_interface/lookup.rb b/app/models/gql/query_interface/lookup.rb index ca372b175..2985cf9ef 100644 --- a/app/models/gql/query_interface/lookup.rb +++ b/app/models/gql/query_interface/lookup.rb @@ -66,7 +66,7 @@ def molecule_sector_nodes(keys) # Scheme-based sector mapping lookups. `energy`/`molecule` select the # graph whose live nodes are returned; the mapping itself is shared. - def energy_sector_map(scheme, values) + def energy_sector_node_map(scheme, values) graph.sector_map.lookup(scheme, values) end diff --git a/app/models/gql/runtime/functions/lookup.rb b/app/models/gql/runtime/functions/lookup.rb index d66e384bd..36d331357 100644 --- a/app/models/gql/runtime/functions/lookup.rb +++ b/app/models/gql/runtime/functions/lookup.rb @@ -124,7 +124,7 @@ def MEDGE_GROUP(*keys) # SECTOR(households) # * Two or more - a classification scheme followed by one or more # values; returns the union of matching nodes. - # SECTOR(klimaattafel, 'Industrie') + # SECTOR(emissions_subsector, 'Electricity and heat production') # SECTOR(ipcc_crt_code_agg, '1.A.1', '1.A.2') # # Unknown scheme or value raises; a valid value with no labelled nodes @@ -133,7 +133,7 @@ def SECTOR(*keys) if keys.size <= 1 scope.energy_sector_nodes(keys) else - scope.energy_sector_map(keys.first, keys.drop(1)) + scope.energy_sector_node_map(keys.first, keys.drop(1)) end end @@ -393,16 +393,13 @@ def EMISSIONS(*keys) # EMISSIONS(sector, use, ghg [, year]) -> return value. # The present year is implicit; only 1990 is suffixed with a year. - scope.graph.emissions[ - [*keys.first(3), (1990 if keys[3].to_i == 1990)].compact.join('_').to_sym - ] + scope.graph.emissions.value_for(*keys.first(3), year: keys[3]) end private - # Internal: Sums the emissions store over every (sector label, use) pair - # the mapping resolves for `scheme`/`value`. Present year is implicit; - # only 1990 is suffixed. Missing store keys count as zero. + # Internal: The mapped EMISSIONS form. Resolves `scheme`/`value` to + # (sector label, use) pairs and lets the emissions store sum over them. def mapped_emissions(keys) scheme, value, ghg, year = keys @@ -411,13 +408,9 @@ def mapped_emissions(keys) "EMISSIONS(#{scheme.inspect}, #{value.inspect}, co2)" end - suffix = ('1990' if year.to_i == 1990) - emissions = scope.graph.emissions - - scope.sector_resolver.pairs(scheme, value).sum do |label, use| - key = [label, use, ghg, suffix].compact.join('_').to_sym - emissions[key] || 0.0 - end + scope.graph.emissions.sum_pairs( + scope.sector_resolver.pairs(scheme, value), ghg, year: year + ) end end end diff --git a/app/models/qernel/emissions.rb b/app/models/qernel/emissions.rb index 84696c7be..3a04c67f9 100644 --- a/app/models/qernel/emissions.rb +++ b/app/models/qernel/emissions.rb @@ -24,6 +24,14 @@ class Emissions dataset_accessors ::Etsource::Dataset.emissions_keys attr_accessor :graph + # Public: The flat store key for the given parts, e.g. (sector, use, ghg). + # The single place the `parts_joined_by_underscores[_1990]` schema lives. + # + # Returns a Symbol. + def self.key_for(*parts, year: nil) + [*parts, (1990 if year.to_i == 1990)].compact.join('_').to_sym + end + # Queryable object that provides scoped access to emissions data # # GQL uses this to scope the sector for easier queries and input/update @@ -40,11 +48,11 @@ def initialize(emissions, scope, year = nil) end def [](attr_name) - @emissions[scoped_method(attr_name).to_sym] + @emissions[scoped_method(attr_name)] end def []=(attr_name, value) - @emissions[scoped_method(attr_name).to_sym] = value + @emissions[scoped_method(attr_name)] = value end def inspect @@ -52,8 +60,7 @@ def inspect end def scoped_method(method_name) - attr = method_name.to_s.delete_suffix('=') - @year.to_i == 1990 ? "#{@scope}_#{attr}_1990" : "#{@scope}_#{attr}" + Emissions.key_for(@scope, method_name.to_s.delete_suffix('='), year: @year) end def respond_to_missing?(method_name, include_private = false) @@ -65,7 +72,7 @@ def respond_to_missing?(method_name, include_private = false) end def method_missing(method_name, *args) - key = scoped_method(method_name).to_sym + key = scoped_method(method_name) if method_name.to_s.end_with?('=') unless scope_exists? @@ -102,5 +109,17 @@ def initialize(graph = nil) def scope(sector, year = nil) ScopedSector.new(self, sector, year) end + + # Public: The stored value for (sector, use, ghg), or nil when the store + # has no such key. Pass year: 1990 to read the historic baseline. + def value_for(sector, use, ghg, year: nil) + self[self.class.key_for(sector, use, ghg, year: year)] + end + + # Public: Sums the store over (sector, use) pairs for one GHG. Pairs + # without a stored value count as zero. + def sum_pairs(pairs, ghg, year: nil) + pairs.sum { |sector, use| value_for(sector, use, ghg, year: year) || 0.0 } + end end end diff --git a/app/models/qernel/sectors.rb b/app/models/qernel/sectors.rb index 630895d5e..d56476afd 100644 --- a/app/models/qernel/sectors.rb +++ b/app/models/qernel/sectors.rb @@ -9,9 +9,9 @@ module Qernel # key index are read once from {Etsource::Sectors}; resolution turns node keys # into the live nodes of this graph. # - # Dispatch, arity and aggregation live in the GQL layer; this class answers - # two questions: which (label, use) pairs, and which live nodes, belong to - # (scheme, values)? + # Dispatch and arity live in the GQL layer; aggregation lives on + # {Qernel::Emissions}. This class answers two questions: which (label, use) + # pairs, and which live nodes, belong to (scheme, values)? class Sectors def initialize(graph, mapping, node_index) @graph = graph diff --git a/spec/models/gql/command_spec.rb b/spec/models/gql/command_spec.rb new file mode 100644 index 000000000..8b3cd4c19 --- /dev/null +++ b/spec/models/gql/command_spec.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'spec_helper' + +module Gql + describe Command do + describe 'cleaning the source' do + it 'strips whitespace outside string literals' do + expect(described_class.new("SUM(\n 1,\t2\n)").source).to eq('SUM(1,2)') + end + + it 'preserves whitespace inside single-quoted strings' do + expect(described_class.new("SECTOR(emissions_subsector, 'Fuels production')").source) + .to eq("SECTOR(emissions_subsector,'Fuels production')") + end + + it 'preserves whitespace inside double-quoted strings' do + expect(described_class.new('V(foo, "demand * conversion")').source) + .to eq('V(foo,"demand * conversion")') + end + + it 'removes a label prefix' do + expect(described_class.new('future:SUM(1, 2)').source).to eq('SUM(1,2)') + end + + it 'does not remove label-like text inside a string literal' do + expect(described_class.new("'future:keep me'").source).to eq("'future:keep me'") + end + + it 'cleans a nil source to an empty string' do + expect(described_class.new(nil).source).to eq('') + end + end + end +end diff --git a/spec/models/gql/runtime/functions/lookup_spec.rb b/spec/models/gql/runtime/functions/lookup_spec.rb index acf7c72b8..a9c20fb40 100644 --- a/spec/models/gql/runtime/functions/lookup_spec.rb +++ b/spec/models/gql/runtime/functions/lookup_spec.rb @@ -155,6 +155,12 @@ def keys_of(nodes) end end + describe "SECTOR(klimaattafel, 'Gebouwde omgeving')" do + it 'resolves a quoted value containing a space' do + expect(keys_of(result)).to eq(%i[buildings_space_heating_demand]) + end + end + describe "SECTOR(klimaattafel, 'Landbouw')" do it 'returns [] for a valid value with no labelled nodes' do expect(result).to eq([]) diff --git a/spec/models/qernel/emissions_spec.rb b/spec/models/qernel/emissions_spec.rb index 0e8751453..3d29bcc2f 100644 --- a/spec/models/qernel/emissions_spec.rb +++ b/spec/models/qernel/emissions_spec.rb @@ -40,6 +40,60 @@ module Qernel end end + describe '.key_for' do + it 'joins the parts with underscores' do + expect(described_class.key_for(:households, :energetic, :co2)) + .to eq(:households_energetic_co2) + end + + it 'suffixes 1990 for the historic baseline' do + expect(described_class.key_for(:households, :energetic, :co2, year: 1990)) + .to eq(:households_energetic_co2_1990) + end + + it 'ignores any other year (the present year is implicit)' do + expect(described_class.key_for(:households, :energetic, :co2, year: 2019)) + .to eq(:households_energetic_co2) + end + end + + describe '#value_for' do + before do + emissions[:households_non_specified_energetic_co2] = 40.0 + emissions[:households_non_specified_energetic_co2_1990] = 60.0 + end + + it 'reads the present-year value' do + expect(emissions.value_for(:households_non_specified, :energetic, :co2)).to eq(40.0) + end + + it 'reads the 1990 baseline when asked' do + expect(emissions.value_for(:households_non_specified, :energetic, :co2, year: 1990)) + .to eq(60.0) + end + + it 'returns nil for a missing key' do + expect(emissions.value_for(:industry_steel, :energetic, :co2)).to be_nil + end + end + + describe '#sum_pairs' do + before do + emissions[:households_non_specified_energetic_co2] = 40.0 + emissions[:agriculture_non_specified_energetic_co2] = 2.0 + end + + it 'sums the stored values over the pairs' do + pairs = [%i[households_non_specified energetic], %i[agriculture_non_specified energetic]] + expect(emissions.sum_pairs(pairs, :co2)).to eq(42.0) + end + + it 'counts pairs without a stored value as zero' do + pairs = [%i[households_non_specified energetic], %i[industry_steel energetic]] + expect(emissions.sum_pairs(pairs, :co2)).to eq(40.0) + end + end + describe '#scope' do it 'returns a ScopedSector instance' do scoped = emissions.scope(:households_non_specified_energetic) From b29e82ef8b7f6fc8895fc4b44caf343dc242d24e Mon Sep 17 00:00:00 2001 From: louispt1 Date: Tue, 14 Jul 2026 12:08:02 +0200 Subject: [PATCH 19/23] Update serializer to match the sector mapping configuration --- app/models/etsource/sectors.rb | 18 +- .../export/configured_csv_serializer.rb | 280 ++++++++++------ .../api/v3/exports_controller_spec.rb | 49 ++- .../etsource/config/direct_emissions_csv.yml | 13 +- .../etsource/config/sector_mapping.csv | 24 +- .../molecules/nodes/nosector/m_waste.ad | 2 +- .../export/configured_csv_serializer_spec.rb | 317 ++++++++++++++---- 7 files changed, 508 insertions(+), 195 deletions(-) diff --git a/app/models/etsource/sectors.rb b/app/models/etsource/sectors.rb index 0967f002a..caa968eb3 100644 --- a/app/models/etsource/sectors.rb +++ b/app/models/etsource/sectors.rb @@ -1,14 +1,16 @@ # frozen_string_literal: true module Etsource - # Reads the sector mapping and node labels from Atlas and caches the two - # structures GQL needs to resolve `SECTOR(scheme, value)` style queries. + # Reads the sector mapping and node labels from Atlas and caches the + # structures GQL and the configured CSV exports need. # - # - # * mapping - {scheme => {value => [[label, use], ...]}} + # * mapping - {scheme => {value => [[label, use], ...]}}, for + # `SECTOR(scheme, value)` style GQL queries. # * node_index(g) - {[label, use] => [node_key, ...]} for one graph type, # with `use` baked in at import so it is never consulted # per query. + # * raw_rows - mapping rows in file order with raw display values, + # for mapping-driven CSV exports. class Sectors def initialize(etsource = Etsource::Base.instance) @etsource = etsource @@ -29,6 +31,14 @@ def node_index(graph_type) end end + # Public: Every mapping row in file order, as an Atlas::SectorMapping::RawRow + # (the (label, use) pair plus the raw display value of every scheme cell). + def raw_rows + NastyCache.instance.fetch('sector_raw_rows') do + Atlas::SectorMapping.load.raw_rows + end + end + private def build_node_index(node_class) diff --git a/app/serializers/export/configured_csv_serializer.rb b/app/serializers/export/configured_csv_serializer.rb index 2d3e0e4ef..31eb294fe 100644 --- a/app/serializers/export/configured_csv_serializer.rb +++ b/app/serializers/export/configured_csv_serializer.rb @@ -11,10 +11,11 @@ # # - schema: An array of hashes describing each column in the CSV. Each hash should contain a `name` # key and optionally a `type` key. -# - rows: An array of hashes, each matching the schema. Any keys contained in a hash which are not -# present in the schema are ignored. +# - rows: Either an array of hashes, each matching the schema (query-driven mode; any keys not +# present in the schema are ignored), or a hash `{ require: , order_by: +# }` (mapping-driven mode, see below). `order_by` is optional. # -# For example: +# Query-driven example: # # { # schema: [ @@ -36,131 +37,206 @@ # - "future": The value will be the result of evaluating the query in the future. # - "unit": The value will be the unit of the specified query. # - "query": This expands into three columns: `present`, `future` and `unit` for specified query. -# - "node_group": Hidden column. Its value names a node group; the row is expanded into one row per -# node in that group. Requires a `period:` to be set on the serializer. +# +# Mapping-driven mode: `rows: { require: }` selects every (sector label, use) pair +# in the sector mapping (see Atlas::SectorMapping) whose cell in `require`'s column has a value, in +# mapping-file order by default. An optional `order_by: ` instead orders pairs by the +# raw display value of that column (ascending string comparison), regardless of whether the column is +# included in `schema:`; a pair whose `order_by` cell is blank/`-` sorts last, and pairs tied on +# `order_by` keep their relative mapping-file order. Each pair expands to the energy and molecule +# nodes whose own `sector_label` and `use` match the pair AND which belong to the node's `:emissions` +# group (key-sorted). A pair with zero labelled nodes, or whose only matching nodes aren't in the +# `:emissions` group, legally yields zero rows. Requires a `period:` to be set on the serializer. Two +# extra column types apply: +# # - "node_attribute": The value will be the result of calling the attribute named by `value:` in the -# schema on node_api for each expanded node. Requires `node_group` column in schema. -# `value:` may be any Ruby expression evaluated on node_api via instance_eval. -# Supports an optional `transform:` Ruby expression evaluated with `value` bound -# to the result of `value:`. For example: +# schema on node_api for each expanded node. `value:` may be any Ruby expression +# evaluated on node_api via instance_eval. A nil result (e.g. a node not in the +# :emissions group) renders as an empty string without evaluating `transform`. +# Otherwise, an optional `transform:` Ruby expression is evaluated with `value` +# bound to the result of `value:`. For example: # transform: "value * 10e-6" # transform: "value ? :other_ghg : :co2" -class Export::ConfiguredCSVSerializer - # Represents the schema for a column in the CSV file. - class Column - attr_reader :name, :type, :label, :value, :transform - - def initialize(name, type, label: name, value: nil, transform: nil) - @name = name - @type = type || 'literal' - @label = label || name - @value = value - @transform = transform +# - "sector_mapping": The value will be the raw display value (never a normalized slug) of the +# mapping column named by `value:`, for the row's pair. A `-`/blank mapping cell +# renders as an empty string. +# +# An unknown mapping column named by `require:`, `order_by:`, or a `sector_mapping` column's `value:` +# raises Export::ConfiguredCSVSerializer::UnknownMappingColumnError at construction, naming the valid +# columns. +module Export + class ConfiguredCSVSerializer # rubocop:disable Style/Documentation + class UnknownMappingColumnError < StandardError end - end - # Creates a serializer using an ETSource config. - # - # period - optional :present or :future; when set, node_group rows are expanded using that graph - # and node_attribute columns are evaluated against it. Existing present/future/unit column - # types continue to work regardless of this setting. - def initialize(config, gql, period: nil) - @config = config.symbolize_keys - @config[:schema] = @config[:schema].map(&:symbolize_keys) - - @columns = @config[:schema].flat_map { |c| create_columns(c) } - @columns.delete(@node_group_column) if unpack_nodes? - @gql = gql - @period = period - end + # Represents the schema for a column in the CSV file. + class Column + attr_reader :name, :type, :label, :value, :transform - def data - rows = [@columns.map(&:label)] - @config[:rows].each { |row| serialize_row(row) { |csv_row| rows << csv_row } } - rows - end + def initialize(name, type, label: name, value: nil, transform: nil) + @name = name + @type = type || 'literal' + @label = label || name + @value = value + @transform = transform + end + end + + # Creates a serializer using an ETSource config. + # + # period - optional :present or :future; required when `rows:` is mapping-driven. + def initialize(config, gql, period: nil) + @config = config.symbolize_keys + @config[:schema] = @config[:schema].map(&:symbolize_keys) + @config[:rows] = @config[:rows].symbolize_keys if @config[:rows].is_a?(Hash) + + @columns = @config[:schema].flat_map { |c| create_columns(c) } + @gql = gql + @period = period - def as_csv - CSV.generate do |csv| - data.each { |row| csv << row } + validate_mapping_references! if mapping_driven? end - end - private + def data + rows = [@columns.map(&:label)] - def serialize_row(row) - if unpack_nodes? - group_name = row[@node_group_column.name] - nodes = graph_interface.group_energy_nodes(group_name) + - graph_interface.group_molecule_nodes(group_name) - nodes.each do |node| - yield @columns.map { |column| serialize_node_column(column, row, node) } + if mapping_driven? + serialize_mapping_rows { |row| rows << row } + else + @config[:rows].each { |row| rows << @columns.map { |column| serialize_column(column, row) } } end - else - yield @columns.map { |column| serialize_column(column, row) } + + rows end - end - def serialize_column(column, row) - value = row[column.name] + def as_csv + CSV.generate do |csv| + data.each { |row| csv << row } + end + end - return '' if value.blank? + private - case column.type - when 'future' then @gql.future.subquery(value).to_s - when 'present' then @gql.present.subquery(value).to_s - when 'unit' then Gquery.get(value).unit.to_s - else value + def mapping_driven? + @config[:rows].is_a?(Hash) end - end - def serialize_node_column(column, row, node) - case column.type - when 'node_attribute' - value = node.node_api.instance_eval(column.value) - value = eval(column.transform) if column.transform - value.to_s - else serialize_column(column, row) + def membership_scheme + @config[:rows][:require].to_sym end - end - def unpack_nodes? - node_group_column.present? - end + def order_scheme + @config[:rows][:order_by]&.to_sym + end - def node_group_column - @node_group_column ||= @columns.find { |c| c.type == 'node_group' } - end + def serialize_mapping_rows + eligible_rows.each do |raw_row| + nodes_for_pair(raw_row.pair).each do |node| + yield @columns.map { |column| serialize_mapping_column(column, raw_row, node) } + end + end + end - def graph_interface - @gql.public_send(@period) - end + # Rows whose `require:` cell has a value, in mapping-file order. When `order_by:` is set, sorted + # by that column's raw value instead (blank/`-` last), with ties broken by mapping-file order. + def eligible_rows + rows = sectors.raw_rows.select { |raw_row| raw_row.cells[membership_scheme] } + return rows unless order_scheme - def create_columns(column) - if column[:type] != 'query' - return Column.new( - column[:name], - column[:type], - label: column[:label], - value: column[:value], - transform: column[:transform] - ) + rows.each_with_index.sort_by { |raw_row, index| [raw_row.cells[order_scheme].nil? ? 1 : 0, raw_row.cells[order_scheme].to_s, index] } + .map(&:first) end - %w[present future unit].map do |subtype| - Column.new( - column[:name], - subtype, - label: column[:"#{subtype}_label"] || default_label_for(subtype, column[:name]) - ) + def serialize_mapping_column(column, raw_row, node) + case column.type + when 'node_attribute' + value = node.node_api.instance_eval(column.value) + return '' if value.nil? + + value = eval(column.transform) if column.transform + value.to_s + when 'sector_mapping' + raw_row.cells[column.value.to_sym].to_s + else + '' + end end - end - def default_label_for(subtype, column_name) - if subtype == 'unit' - "#{column_name} Unit" - else - "#{subtype.capitalize} #{column_name}" + def nodes_for_pair(pair) + nodes = %i[energy molecules].flat_map do |graph_type| + sectors.node_index(graph_type).fetch(pair, []).filter_map do |key| + node_for(graph_type, key) + end + end + nodes.select(&:emissions?).sort_by { |node| node.node_api.key.to_s } + end + + def node_for(graph_type, key) + graph_type == :molecules ? graph_interface.molecules.node(key) : graph_interface.graph.node(key) + end + + def graph_interface + @gql.public_send(@period) + end + + def sectors + @sectors ||= Etsource::Sectors.new + end + + def validate_mapping_references! + valid = sectors.mapping.keys + references = [@config[:rows][:require], @config[:rows][:order_by]] + @columns.select { |c| + c.type == 'sector_mapping' + }.map(&:value) + + references.compact.each do |reference| + next if valid.include?(reference.to_sym) + + raise UnknownMappingColumnError, + "Unknown sector mapping column #{reference.inspect}. " \ + "Valid columns: #{valid.map(&:inspect).join(', ')}." + end + end + + def serialize_column(column, row) + value = row[column.name] + + return '' if value.blank? + + case column.type + when 'future' then @gql.future.subquery(value).to_s + when 'present' then @gql.present.subquery(value).to_s + when 'unit' then Gquery.get(value).unit.to_s + else value + end + end + + def create_columns(column) + if column[:type] != 'query' + return Column.new( + column[:name], + column[:type], + label: column[:label], + value: column[:value], + transform: column[:transform] + ) + end + + %w[present future unit].map do |subtype| + Column.new( + column[:name], + subtype, + label: column[:"#{subtype}_label"] || default_label_for(subtype, column[:name]) + ) + end + end + + def default_label_for(subtype, column_name) + if subtype == 'unit' + "#{column_name} Unit" + else + "#{subtype.capitalize} #{column_name}" + end end end end diff --git a/spec/controllers/api/v3/exports_controller_spec.rb b/spec/controllers/api/v3/exports_controller_spec.rb index 374da0fa6..c88612a6d 100644 --- a/spec/controllers/api/v3/exports_controller_spec.rb +++ b/spec/controllers/api/v3/exports_controller_spec.rb @@ -78,12 +78,7 @@ end end - describe 'GET direct_emissions_present.csv' do - before do - request.headers.merge!(headers) - get :direct_emissions_present, params: { id: scenario.id }, format: :csv - end - + shared_examples 'a mapping-driven direct emissions export' do it 'is successful' do expect(response).to be_ok end @@ -92,6 +87,40 @@ expect(response.media_type).to eq('text/csv') end + it 'includes the eight legacy columns, byte-identical to the pre-rework export' do + expect(CSV.parse(response.body).first).to eq( + [ + 'Sector', 'Subsector', 'Key', 'GHG', + 'CO2 production [kton CO2-eq]', 'CO2 capture [kton CO2-eq]', + 'Other GHG emissions [kton CO2-eq]', 'Total GHG emissions [kton CO2-eq]' + ] + ) + end + + it 'renders Sector/Subsector as mapping lookups for a labelled, :emissions-group node' do + row = CSV.parse(response.body).find { |r| r[2] == 'm_waste' } + expect(row).to eq(%w[Waste Non-specified m_waste co2 0.0 0.0 0.0 0.0]) + end + + it 'excludes a labelled node whose pair has no value in the require column' do + expect(CSV.parse(response.body).flatten).not_to include('foo') + end + + it 'excludes a labelled node that is not in the :emissions group' do + # `bar`/`baz`/`lft`/`buildings_space_heating_demand` all carry a sector_label/use that + # matches an exported mapping row, but none is in the :emissions node group. + expect(CSV.parse(response.body).flatten).not_to include('bar', 'baz', 'lft', 'buildings_space_heating_demand') + end + end + + describe 'GET direct_emissions_present.csv' do + before do + request.headers.merge!(headers) + get :direct_emissions_present, params: { id: scenario.id }, format: :csv + end + + include_examples 'a mapping-driven direct emissions export' + it 'sets the CSV filename' do expect(response.headers['Content-Disposition']).to include("direct_emissions_present.#{scenario.id}.csv") end @@ -103,13 +132,7 @@ get :direct_emissions_future, params: { id: scenario.id }, format: :csv end - it 'is successful' do - expect(response).to be_ok - end - - it 'sets the content type to text/csv' do - expect(response.media_type).to eq('text/csv') - end + include_examples 'a mapping-driven direct emissions export' it 'sets the CSV filename' do expect(response.headers['Content-Disposition']).to include("direct_emissions_future.#{scenario.id}.csv") diff --git a/spec/fixtures/etsource/config/direct_emissions_csv.yml b/spec/fixtures/etsource/config/direct_emissions_csv.yml index 0a15463be..9abee5d06 100644 --- a/spec/fixtures/etsource/config/direct_emissions_csv.yml +++ b/spec/fixtures/etsource/config/direct_emissions_csv.yml @@ -1,8 +1,10 @@ schema: - - name: Group - type: node_group - name: Sector + type: sector_mapping + value: emissions_sector - name: Subsector + type: sector_mapping + value: emissions_subsector - name: Key type: node_attribute value: key @@ -25,12 +27,11 @@ schema: value: direct_reporting_emissions_other_ghg_emissions transform: "(value * 1e-6).round(6)" - name: Total_GHG_emissions - label: "Total GHG emissions[kton CO2-eq]" + label: "Total GHG emissions [kton CO2-eq]" type: node_attribute value: direct_reporting_emissions_total_ghg_emissions transform: "(value * 1e-6).round(6)" rows: - - Group: emissions_industry_chemicals - Sector: Industry - Subsector: Chemicals + require: emissions_sector + order_by: ipcc_crt_code diff --git a/spec/fixtures/etsource/config/sector_mapping.csv b/spec/fixtures/etsource/config/sector_mapping.csv index eb2e1e87e..59db0709d 100644 --- a/spec/fixtures/etsource/config/sector_mapping.csv +++ b/spec/fixtures/etsource/config/sector_mapping.csv @@ -1,12 +1,12 @@ -sector_label,use,ipcc_crt_code_agg,klimaattafel -energy_electricity_and_heat_production,energetic,1.A.1,Elektriciteit -industry_refineries,energetic,1.A.1,Industrie -industry_refineries,non_energetic,1.B,Industrie -industry_non_specified,energetic,1.A.2,Industrie -buildings_non_specified,energetic,1.A.4.a,Gebouwde omgeving -households_non_specified,energetic,1.A.4.b,Gebouwde omgeving -agriculture_non_specified,energetic,1.A.4.c,Landbouw -agriculture_non_specified,non_energetic,3,Landbouw -waste_non_specified,non_energetic,5,Elektriciteit -energy_fugitive_emissions,non_energetic,1.B,Elektriciteit -other_heating,energetic,-,Mobiliteit +sector_label,use,emissions_sector,emissions_subsector,ipcc_crt_code_agg,ipcc_crt_code,klimaattafel +energy_electricity_and_heat_production,energetic,Energy,Electricity and heat production,1.A.1,1.A.1.a,Elektriciteit +industry_refineries,energetic,Industry,Refineries,1.A.1,1.A.1.b,Industrie +industry_refineries,non_energetic,Industry,Refineries,1.B,1.B.2.a.iv,Industrie +industry_non_specified,energetic,-,-,1.A.2,1.A.2,Industrie +buildings_non_specified,energetic,Buildings,Non-specified,1.A.4.a,1.A.4.a,Gebouwde omgeving +households_non_specified,energetic,Households,Non-specified,1.A.4.b,1.A.4.b,Gebouwde omgeving +agriculture_non_specified,energetic,Agriculture,Non-specified,1.A.4.c,1.A.4.c,Landbouw +agriculture_non_specified,non_energetic,Agriculture,Non-specified,3,3,Landbouw +waste_non_specified,non_energetic,Waste,Non-specified,5,5,Elektriciteit +energy_fugitive_emissions,non_energetic,Energy,Fugitive emissions,1.B,1.B excl. 1.B.2.a.iv,Elektriciteit +other_heating,energetic,Other,Heating,-,-,Mobiliteit diff --git a/spec/fixtures/etsource/graphs/molecules/nodes/nosector/m_waste.ad b/spec/fixtures/etsource/graphs/molecules/nodes/nosector/m_waste.ad index 8ce379f39..1bf13b204 100644 --- a/spec/fixtures/etsource/graphs/molecules/nodes/nosector/m_waste.ad +++ b/spec/fixtures/etsource/graphs/molecules/nodes/nosector/m_waste.ad @@ -1,3 +1,3 @@ -- groups = [] +- groups = [emissions] - sector_label = waste_non_specified - use = non_energetic diff --git a/spec/serializers/export/configured_csv_serializer_spec.rb b/spec/serializers/export/configured_csv_serializer_spec.rb index 99b0056c0..b5d36e9d5 100644 --- a/spec/serializers/export/configured_csv_serializer_spec.rb +++ b/spec/serializers/export/configured_csv_serializer_spec.rb @@ -122,108 +122,311 @@ end end - context 'when given a "node_group" column' do - let(:node_api_1) { instance_double('Qernel::NodeApi::EnergyApi', key: 'node_a') } - let(:node_api_2) { instance_double('Qernel::NodeApi::EnergyApi', key: 'node_b') } - let(:node_1) { instance_double('Qernel::Node', node_api: node_api_1) } - let(:node_2) { instance_double('Qernel::Node', node_api: node_api_2) } + context 'when given mapping-driven rows' do + # Real pairs from spec/fixtures/etsource/config/sector_mapping.csv, joined against the real + # labelled fixture nodes (spec/fixtures/etsource/graphs). In mapping-file order: + # + # energy_electricity_and_heat_production/energetic -> Energy, emissions_sector set, 0 nodes + # industry_refineries/energetic -> Industry, node `bar` (dual-use label) + # industry_refineries/non_energetic -> Industry, node `baz` (dual-use label) + # industry_non_specified/energetic -> emissions_sector blank -> excluded + # (node `foo` exists but must not appear) + # buildings_non_specified/energetic -> Buildings, node `buildings_space_heating_demand` + # households_non_specified/energetic -> 0 nodes + # agriculture_non_specified/energetic -> 0 nodes + # agriculture_non_specified/non_energetic -> 0 nodes + # waste_non_specified/non_energetic -> Waste, node `m_waste` (molecule graph) + # energy_fugitive_emissions/non_energetic -> 0 nodes + # other_heating/energetic -> Other, node `lft`, blank ipcc cell let(:config) do { schema: [ - { name: 'Group', type: 'node_group' }, - { name: 'Sector' }, - { name: 'Node', type: 'node_attribute', value: 'key' } + { name: 'Sector', type: 'sector_mapping', value: 'emissions_sector' }, + { name: 'Key', type: 'node_attribute', value: 'key' }, + { name: 'IPCC', type: 'sector_mapping', value: 'ipcc_crt_code_agg' } ], - rows: [ - { 'Group' => 'some_group', 'Sector' => 'Industry' } - ] + rows: { require: 'emissions_sector' } } end - let(:node_api_3) { instance_double('Qernel::NodeApi::MoleculeApi', key: 'node_c') } - let(:node_3) { instance_double('Qernel::Node', node_api: node_api_3) } - - let(:gql) { Scenario.default.gql } + let(:gql) { Scenario.default.gql } let(:serializer) { described_class.new(config, gql, period: :future) } - before do - allow(gql.future).to receive(:group_energy_nodes).with('some_group').and_return([node_1, node_2]) - allow(gql.future).to receive(:group_molecule_nodes).with('some_group').and_return([node_3]) + let(:node_bar) do + instance_double( + 'Qernel::Node', + emissions?: true, + node_api: instance_double( + 'Qernel::NodeApi::EnergyApi', key: :bar, direct_reporting_emissions_co2_production: nil + ) + ) end - - it 'excludes the node_group column from the header' do - expect(serializer.data[0]).to eq(%w[Sector Node]) + let(:node_baz) do + instance_double('Qernel::Node', emissions?: true, node_api: instance_double('Qernel::NodeApi::EnergyApi', key: :baz)) + end + let(:node_foo) do + instance_double('Qernel::Node', emissions?: true, node_api: instance_double('Qernel::NodeApi::EnergyApi', key: :foo)) + end + let(:node_buildings) do + instance_double( + 'Qernel::Node', emissions?: true, + node_api: instance_double('Qernel::NodeApi::EnergyApi', key: :buildings_space_heating_demand) + ) + end + let(:node_lft) do + instance_double('Qernel::Node', emissions?: true, node_api: instance_double('Qernel::NodeApi::EnergyApi', key: :lft)) + end + let(:node_waste) do + instance_double('Qernel::Node', emissions?: true, node_api: instance_double('Qernel::NodeApi::MoleculeApi', key: :m_waste)) end - it 'expands the row into one row per node from both graphs' do - expect(serializer.data.length).to eq(4) # header + 2 energy + 1 molecule + let(:energy_nodes) do + { bar: node_bar, baz: node_baz, foo: node_foo, buildings_space_heating_demand: node_buildings, lft: node_lft } end - it 'includes the literal column value on each expanded row' do - expect(serializer.data[1][0]).to eq('Industry') - expect(serializer.data[2][0]).to eq('Industry') - expect(serializer.data[3][0]).to eq('Industry') + before do + allow(gql.future.graph).to receive(:node) { |key| energy_nodes[key.to_sym] } + allow(gql.future).to receive(:molecules) + .and_return(instance_double('Qernel::Graph').tap { |g| allow(g).to receive(:node).with(:m_waste).and_return(node_waste) }) end - it 'includes energy node_attribute values' do - expect(serializer.data[1][1]).to eq('node_a') - expect(serializer.data[2][1]).to eq('node_b') + it 'includes the CSV headers' do + expect(serializer.data[0]).to eq(%w[Sector Key IPCC]) end - it 'includes molecule node_attribute values' do - expect(serializer.data[3][1]).to eq('node_c') + it 'emits rows in mapping-file order, skipping pairs with zero labelled nodes' do + expect(serializer.data[1..].map { |row| row[1] }).to eq( + %w[bar baz buildings_space_heating_demand m_waste lft] + ) end - context 'with a transform for numeric conversion' do - let(:node_api_1) { instance_double('Qernel::NodeApi::EnergyApi', key: 'node_a', demand: 1_000_000.0) } + it 'excludes a pair whose require column cell is blank, even though it has a labelled node' do + expect(serializer.data.flatten).not_to include('foo') + end + context 'with an "order_by" rule' do let(:config) do { - schema: [ - { name: 'Group', type: 'node_group' }, - { name: 'Demand', type: 'node_attribute', value: 'demand', transform: 'value * 1e-6' } - ], - rows: [{ 'Group' => 'some_group' }] + schema: [{ name: 'Key', type: 'node_attribute', value: 'key' }], + rows: { require: 'emissions_sector', order_by: 'ipcc_crt_code' } } end + it 'orders rows by the raw value of the order_by column, not the mapping-file order' do + # ipcc_crt_code: bar 1.A.1.b, buildings 1.A.4.a, baz 1.B.2.a.iv, m_waste 5, lft blank. + expect(serializer.data[1..].flatten).to eq( + %w[bar buildings_space_heating_demand baz m_waste lft] + ) + end + + it 'sorts a pair with a blank order_by cell last' do + expect(serializer.data.last).to eq(['lft']) + end + end + + context 'when a labelled node is not in the :emissions group' do + before { allow(node_baz).to receive(:emissions?).and_return(false) } + + it 'excludes it, even though its pair matches an exported row' do + expect(serializer.data.flatten).not_to include('baz') + end + + it 'still includes the other node under the same dual-use label' do + expect(serializer.data.flatten).to include('bar') + end + end + + it 'renders the raw display value of the require column (not a slug)' do + expect(serializer.data[1][0]).to eq('Industry') + expect(serializer.data[4][0]).to eq('Waste') + end + + it 'joins a dual-use label by the node\'s own (label, use) pair, not by name alone' do + # `bar` and `baz` share sector_label industry_refineries but differ in `use`, and land under + # different IPCC codes as a result. + expect(serializer.data[1]).to eq(%w[Industry bar 1.A.1]) + expect(serializer.data[2]).to eq(%w[Industry baz 1.B]) + end + + it 'includes molecule-graph nodes alongside energy nodes' do + expect(serializer.data[4]).to eq(%w[Waste m_waste 5]) + end + + it 'renders a raw value with punctuation unchanged' do + expect(serializer.data[1][2]).to eq('1.A.1') + end + + it 'renders a blank mapping cell as an empty string' do + expect(serializer.data[5]).to eq(['Other', 'lft', '']) + end + + context 'with an isolated pair (stubbed Etsource::Sectors)' do + let(:sectors) { instance_double(Etsource::Sectors) } + let(:pair) { %i[industry_refineries energetic] } + let(:raw_row) { Atlas::SectorMapping::RawRow.new(pair, { emissions_sector: 'Industry' }) } + before do - allow(gql.future).to receive(:group_energy_nodes).with('some_group').and_return([node_1]) - allow(gql.future).to receive(:group_molecule_nodes).with('some_group').and_return([]) + allow(Etsource::Sectors).to receive(:new).and_return(sectors) + allow(sectors).to receive(:mapping).and_return({ emissions_sector: {} }) + allow(sectors).to receive(:raw_rows).and_return([raw_row]) + allow(sectors).to receive(:node_index).with(:molecules).and_return({}) + end + + context 'sorting nodes within a pair' do + let(:config) do + { + schema: [{ name: 'Key', type: 'node_attribute', value: 'key' }], + rows: { require: 'emissions_sector' } + } + end + + let(:node_z) do + instance_double('Qernel::Node', emissions?: true, node_api: instance_double('Qernel::NodeApi::EnergyApi', key: :z_node)) + end + let(:node_a) do + instance_double('Qernel::Node', emissions?: true, node_api: instance_double('Qernel::NodeApi::EnergyApi', key: :a_node)) + end + + before do + allow(sectors).to receive(:node_index).with(:energy).and_return({ pair => %i[z_node a_node] }) + allow(gql.future.graph).to receive(:node).with(:z_node).and_return(node_z) + allow(gql.future.graph).to receive(:node).with(:a_node).and_return(node_a) + end + + it 'sorts the expanded nodes by key, regardless of node_index order' do + expect(serializer.data[1..].map { |row| row[0] }).to eq(%w[a_node z_node]) + end end - it 'applies the transform to the value' do - expect(serializer.data[1][0]).to eq('1.0') + context 'with an "order_by" rule and rows tied on that column' do + let(:pair_a) { %i[pair_a energetic] } + let(:pair_b) { %i[pair_b energetic] } + let(:pair_c) { %i[pair_c energetic] } + + let(:raw_row_a) { Atlas::SectorMapping::RawRow.new(pair_a, { emissions_sector: 'A', ipcc_crt_code: '1' }) } + let(:raw_row_b) { Atlas::SectorMapping::RawRow.new(pair_b, { emissions_sector: 'B', ipcc_crt_code: '1' }) } + let(:raw_row_c) { Atlas::SectorMapping::RawRow.new(pair_c, { emissions_sector: 'C', ipcc_crt_code: '0' }) } + + let(:config) do + { + schema: [{ name: 'Key', type: 'node_attribute', value: 'key' }], + rows: { require: 'emissions_sector', order_by: 'ipcc_crt_code' } + } + end + + let(:node_a) do + instance_double('Qernel::Node', emissions?: true, node_api: instance_double('Qernel::NodeApi::EnergyApi', key: :node_a)) + end + let(:node_b) do + instance_double('Qernel::Node', emissions?: true, node_api: instance_double('Qernel::NodeApi::EnergyApi', key: :node_b)) + end + let(:node_c) do + instance_double('Qernel::Node', emissions?: true, node_api: instance_double('Qernel::NodeApi::EnergyApi', key: :node_c)) + end + + before do + allow(sectors).to receive(:mapping).and_return({ emissions_sector: {}, ipcc_crt_code: {} }) + # File order: A, B, C. A and B tie on ipcc_crt_code ("1"); C has a lower code ("0"). + allow(sectors).to receive(:raw_rows).and_return([raw_row_a, raw_row_b, raw_row_c]) + allow(sectors).to receive(:node_index).with(:energy).and_return( + pair_a => [:node_a], pair_b => [:node_b], pair_c => [:node_c] + ) + allow(gql.future.graph).to receive(:node).with(:node_a).and_return(node_a) + allow(gql.future.graph).to receive(:node).with(:node_b).and_return(node_b) + allow(gql.future.graph).to receive(:node).with(:node_c).and_return(node_c) + end + + it 'sorts by the order_by value first, breaking ties by mapping-file order' do + expect(serializer.data[1..].flatten).to eq(%w[node_c node_a node_b]) + end + end + + context 'when a node_attribute value is nil (node not in the :emissions group)' do + let(:config) do + { + schema: [ + { name: 'Key', type: 'node_attribute', value: 'key' }, + { name: 'CO2', type: 'node_attribute', value: 'direct_reporting_emissions_co2_production', + transform: 'value * 1e-6' } + ], + rows: { require: 'emissions_sector' } + } + end + + let(:node_bar) do + instance_double( + 'Qernel::Node', + emissions?: true, + node_api: instance_double( + 'Qernel::NodeApi::EnergyApi', key: :bar, direct_reporting_emissions_co2_production: nil + ) + ) + end + + before do + allow(sectors).to receive(:node_index).with(:energy).and_return({ pair => [:bar] }) + allow(gql.future.graph).to receive(:node).with(:bar).and_return(node_bar) + end + + it 'renders an empty string without evaluating the transform' do + expect(serializer.data[1]).to eq(['bar', '']) + end end end + end - context 'with a transform for conditional mapping' do - let(:node_api_1) { double('node_api', key: 'node_a', is_special: true) } - let(:node_api_2) { double('node_api', key: 'node_b', is_special: false) } + context 'when given an unknown sector mapping column' do + let(:gql) { Scenario.default.gql } + let(:serializer) { described_class.new(config, gql, period: :future) } + + context 'as a "rows: require:" reference' do + let(:config) do + { + schema: [{ name: 'Key', type: 'node_attribute', value: 'key' }], + rows: { require: 'impossible' } + } + end + + it 'raises at construction, naming the invalid reference and the valid columns' do + expect { serializer }.to raise_error( + Export::ConfiguredCSVSerializer::UnknownMappingColumnError, + /impossible.*Valid columns.*emissions_sector/m + ) + end + end + context 'as a "sector_mapping" column value' do let(:config) do { schema: [ - { name: 'Group', type: 'node_group' }, - { name: 'Type', type: 'node_attribute', value: 'is_special', - transform: "value ? 'other_ghg' : 'co2'" } + { name: 'Sector', type: 'sector_mapping', value: 'impossible' } ], - rows: [{ 'Group' => 'some_group' }] + rows: { require: 'emissions_sector' } } end - before do - allow(gql.future).to receive(:group_energy_nodes).with('some_group').and_return([node_1, node_2]) - allow(gql.future).to receive(:group_molecule_nodes).with('some_group').and_return([]) + it 'raises at construction, naming the invalid reference and the valid columns' do + expect { serializer }.to raise_error( + Export::ConfiguredCSVSerializer::UnknownMappingColumnError, + /impossible.*Valid columns.*emissions_sector/m + ) end + end - it 'maps a truthy result via transform' do - expect(serializer.data[1][0]).to eq('other_ghg') + context 'as a "rows: order_by:" reference' do + let(:config) do + { + schema: [{ name: 'Key', type: 'node_attribute', value: 'key' }], + rows: { require: 'emissions_sector', order_by: 'impossible' } + } end - it 'maps a falsy result via transform' do - expect(serializer.data[2][0]).to eq('co2') + it 'raises at construction, naming the invalid reference and the valid columns' do + expect { serializer }.to raise_error( + Export::ConfiguredCSVSerializer::UnknownMappingColumnError, + /impossible.*Valid columns.*emissions_sector/m + ) end end end From 69426930f472e166022b80333ec70ad8e0a5b6bf Mon Sep 17 00:00:00 2001 From: louispt1 Date: Wed, 15 Jul 2026 09:07:44 +0200 Subject: [PATCH 20/23] Refine sectors and serializer implementation based on new mapping approach. --- Gemfile | 2 +- Gemfile.lock | 4 +- app/models/etsource/sectors.rb | 4 - app/models/gql/runtime/functions/lookup.rb | 13 ++- app/models/qernel/sectors.rb | 17 +-- .../export/configured_csv_serializer.rb | 102 +++++++++++------- .../gql/runtime/functions/lookup_spec.rb | 6 ++ .../export/configured_csv_serializer_spec.rb | 58 +++++++++- 8 files changed, 150 insertions(+), 56 deletions(-) diff --git a/Gemfile b/Gemfile index 8e73e6328..4c12d8b39 100644 --- a/Gemfile +++ b/Gemfile @@ -73,7 +73,7 @@ gem 'ruby-progressbar' # own gems gem 'quintel_merit', ref: 'aae77e0', github: 'quintel/merit' -gem 'atlas', ref: '5504fac', github: 'quintel/atlas' +gem 'atlas', ref: '5d2e5b2', github: 'quintel/atlas' gem 'fever', ref: '2afebd1', github: 'quintel/fever' gem 'refinery', ref: '36b8e34', github: 'quintel/refinery' gem 'rubel', ref: '9fe7010', github: 'quintel/rubel' diff --git a/Gemfile.lock b/Gemfile.lock index 36acecd4b..8227f48d4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ GIT remote: https://github.com/quintel/atlas.git - revision: 5504fac349b1b25fda2ef285b044b025fbfbdb1e - ref: 5504fac + revision: 5d2e5b20e3d397b764f688180e3987bd43161612 + ref: 5d2e5b2 specs: atlas (1.0.0) activemodel (>= 7) diff --git a/app/models/etsource/sectors.rb b/app/models/etsource/sectors.rb index caa968eb3..72b590914 100644 --- a/app/models/etsource/sectors.rb +++ b/app/models/etsource/sectors.rb @@ -12,10 +12,6 @@ module Etsource # * raw_rows - mapping rows in file order with raw display values, # for mapping-driven CSV exports. class Sectors - def initialize(etsource = Etsource::Base.instance) - @etsource = etsource - end - # Public: The inverted mapping index, keyed by scheme then normalized value. def mapping NastyCache.instance.fetch('sector_mapping_hash') do diff --git a/app/models/gql/runtime/functions/lookup.rb b/app/models/gql/runtime/functions/lookup.rb index 36d331357..d878350df 100644 --- a/app/models/gql/runtime/functions/lookup.rb +++ b/app/models/gql/runtime/functions/lookup.rb @@ -91,13 +91,13 @@ def MALL def GROUP(*keys) scope.group_energy_nodes(keys) end - alias G GROUP + alias_method :G, :GROUP # Returns an Array of {Qernel::Node} for given molecule group. See GROUP. def MGROUP(*keys) scope.group_molecule_nodes(keys) end - alias MG MGROUP + alias_method :MG, :MGROUP # Returns an Array of {Qernel::Edges} for given energy group. # @@ -108,13 +108,13 @@ def MGROUP(*keys) def EDGE_GROUP(*keys) scope.group_energy_edges(keys) end - alias EG EDGE_GROUP + alias_method :EG, :EDGE_GROUP # Returns an Array of {Qernel::Edges} for given molecule group. See EDGE_GROUP. def MEDGE_GROUP(*keys) scope.group_molecule_edges(keys) end - alias MEG EDGE_GROUP + alias_method :MEG, :MEDGE_GROUP # Returns an Array of {Qernel::Node} for an energy sector. # @@ -401,6 +401,11 @@ def EMISSIONS(*keys) # Internal: The mapped EMISSIONS form. Resolves `scheme`/`value` to # (sector label, use) pairs and lets the emissions store sum over them. def mapped_emissions(keys) + if keys.size > 4 + raise Gql::GqlError, "EMISSIONS(#{keys.first.inspect}, ...) accepts a single value: " \ + 'EMISSIONS(scheme, value, ghg[, 1990]).' + end + scheme, value, ghg, year = keys if ghg.nil? diff --git a/app/models/qernel/sectors.rb b/app/models/qernel/sectors.rb index d56476afd..465171747 100644 --- a/app/models/qernel/sectors.rb +++ b/app/models/qernel/sectors.rb @@ -21,7 +21,7 @@ def initialize(graph, mapping, node_index) @pair_cache = {} end - # Public: Whether `scheme`` names a classification scheme in the mapping. + # Public: Whether `scheme` names a classification scheme in the mapping. # Used by the EMISSIONS first-argument dispatch. def scheme?(scheme) @mapping.key?(normalize(scheme)) @@ -40,14 +40,21 @@ def pairs(scheme, values) end # Public: The live nodes of this graph whose (label, use) pair falls under - # any of `values` of 'scheme` (the union). A value which resolves to pairs + # any of `values` of `scheme` (the union). A value which resolves to pairs # with no labelled nodes contributes nothing. def lookup(scheme, values) value_list = Array(values) key = [normalize(scheme), value_list.map { |value| normalize(value) }] @node_cache[key] ||= - pairs(scheme, value_list).flat_map { |pair| nodes_for(pair) }.uniq + pairs(scheme, value_list).flat_map { |pair| nodes_for_pair(pair) }.uniq + end + + # Public: The live nodes of this graph whose (label, use) pair equals + # `pair` (an already-normalized [label, use]). Used by mapping-driven CSV + # exports as well as {#lookup}. + def nodes_for_pair(pair) + Array(@node_index[pair]).filter_map { |key| @graph.node(key) } end private @@ -62,10 +69,6 @@ def resolve_pairs(scheme, value_list) end.uniq end - def nodes_for(pair) - Array(@node_index[pair]).filter_map { |key| @graph.node(key) } - end - def normalize(value) Atlas::SectorMapping.normalize(value) end diff --git a/app/serializers/export/configured_csv_serializer.rb b/app/serializers/export/configured_csv_serializer.rb index 31eb294fe..7ade8414e 100644 --- a/app/serializers/export/configured_csv_serializer.rb +++ b/app/serializers/export/configured_csv_serializer.rb @@ -46,8 +46,9 @@ # `order_by` keep their relative mapping-file order. Each pair expands to the energy and molecule # nodes whose own `sector_label` and `use` match the pair AND which belong to the node's `:emissions` # group (key-sorted). A pair with zero labelled nodes, or whose only matching nodes aren't in the -# `:emissions` group, legally yields zero rows. Requires a `period:` to be set on the serializer. Two -# extra column types apply: +# `:emissions` group, legally yields zero rows. Mapping-driven mode requires a `period:` (raises +# ArgumentError at construction otherwise) and permits only the two column types below (any other +# type raises UnsupportedColumnTypeError at construction): # # - "node_attribute": The value will be the result of calling the attribute named by `value:` in the # schema on node_api for each expanded node. `value:` may be any Ruby expression @@ -57,9 +58,8 @@ # bound to the result of `value:`. For example: # transform: "value * 10e-6" # transform: "value ? :other_ghg : :co2" -# - "sector_mapping": The value will be the raw display value (never a normalized slug) of the -# mapping column named by `value:`, for the row's pair. A `-`/blank mapping cell -# renders as an empty string. +# - "sector_mapping": The value will be the raw display value of the mapping column named by `value:`, +# for the row's pair. A `-`/blank mapping cell renders as an empty string. # # An unknown mapping column named by `require:`, `order_by:`, or a `sector_mapping` column's `value:` # raises Export::ConfiguredCSVSerializer::UnknownMappingColumnError at construction, naming the valid @@ -69,6 +69,12 @@ class ConfiguredCSVSerializer # rubocop:disable Style/Documentation class UnknownMappingColumnError < StandardError end + class UnsupportedColumnTypeError < StandardError + end + + # Column types which may appear in a mapping-driven schema. + MAPPING_COLUMN_TYPES = %w[node_attribute sector_mapping].freeze + # Represents the schema for a column in the CSV file. class Column attr_reader :name, :type, :label, :value, :transform @@ -78,7 +84,19 @@ def initialize(name, type, label: name, value: nil, transform: nil) @type = type || 'literal' @label = label || name @value = value - @transform = transform + @transform = transform && compile_transform(transform) + end + + private + + # Internal: Compiles the `transform:` expression to a lambda once, rather + # than eval-ing the string for every exported cell. `value` is the + # column's evaluated node attribute. + def compile_transform(expression) + eval( + "->(value) { #{expression} }", # ->(value) { (value * 1e-6).round(6) } + binding, __FILE__, __LINE__ - 1 + ) end end @@ -94,7 +112,11 @@ def initialize(config, gql, period: nil) @gql = gql @period = period - validate_mapping_references! if mapping_driven? + return unless mapping_driven? + + @membership_scheme = @config[:rows][:require]&.to_sym + @order_scheme = @config[:rows][:order_by]&.to_sym + validate_mapping_config! end def data @@ -121,14 +143,6 @@ def mapping_driven? @config[:rows].is_a?(Hash) end - def membership_scheme - @config[:rows][:require].to_sym - end - - def order_scheme - @config[:rows][:order_by]&.to_sym - end - def serialize_mapping_rows eligible_rows.each do |raw_row| nodes_for_pair(raw_row.pair).each do |node| @@ -140,39 +154,36 @@ def serialize_mapping_rows # Rows whose `require:` cell has a value, in mapping-file order. When `order_by:` is set, sorted # by that column's raw value instead (blank/`-` last), with ties broken by mapping-file order. def eligible_rows - rows = sectors.raw_rows.select { |raw_row| raw_row.cells[membership_scheme] } - return rows unless order_scheme + rows = sectors.raw_rows.select { |raw_row| raw_row.cells[@membership_scheme] } + return rows unless @order_scheme - rows.each_with_index.sort_by { |raw_row, index| [raw_row.cells[order_scheme].nil? ? 1 : 0, raw_row.cells[order_scheme].to_s, index] } - .map(&:first) + rows.sort_by.with_index { |raw_row, index| order_key(raw_row, index) } + end + + def order_key(raw_row, index) + cell = raw_row.cells[@order_scheme] + [cell.nil? ? 1 : 0, cell.to_s, index] end def serialize_mapping_column(column, raw_row, node) - case column.type - when 'node_attribute' + if column.type == 'node_attribute' value = node.node_api.instance_eval(column.value) return '' if value.nil? - value = eval(column.transform) if column.transform + value = column.transform.call(value) if column.transform value.to_s - when 'sector_mapping' + else # 'sector_mapping'; the only other type validate_mapping_config! admits raw_row.cells[column.value.to_sym].to_s - else - '' end end + # The `:emissions`-group nodes of both live graphs whose (sector_label, use) matches `pair`. def nodes_for_pair(pair) - nodes = %i[energy molecules].flat_map do |graph_type| - sectors.node_index(graph_type).fetch(pair, []).filter_map do |key| - node_for(graph_type, key) - end + nodes = [graph_interface.graph, graph_interface.molecules].flat_map do |graph| + graph.sector_map.nodes_for_pair(pair) end - nodes.select(&:emissions?).sort_by { |node| node.node_api.key.to_s } - end - def node_for(graph_type, key) - graph_type == :molecules ? graph_interface.molecules.node(key) : graph_interface.graph.node(key) + nodes.select(&:emissions?).sort_by { |node| node.node_api.key.to_s } end def graph_interface @@ -183,11 +194,30 @@ def sectors @sectors ||= Etsource::Sectors.new end + def validate_mapping_config! + if @period.nil? + raise ArgumentError, 'Mapping-driven rows require a period: of :present or :future.' + end + + if @membership_scheme.nil? + raise ArgumentError, 'Mapping-driven rows require a require: mapping column.' + end + + unsupported = @columns.reject { |column| MAPPING_COLUMN_TYPES.include?(column.type) } + + if unsupported.any? + raise UnsupportedColumnTypeError, + "Column types #{unsupported.map(&:type).uniq.inspect} are not supported with " \ + "mapping-driven rows. Supported types: #{MAPPING_COLUMN_TYPES.inspect}." + end + + validate_mapping_references! + end + def validate_mapping_references! valid = sectors.mapping.keys - references = [@config[:rows][:require], @config[:rows][:order_by]] + @columns.select { |c| - c.type == 'sector_mapping' - }.map(&:value) + sector_mapping_columns = @columns.select { |column| column.type == 'sector_mapping' } + references = [@membership_scheme, @order_scheme, *sector_mapping_columns.map(&:value)] references.compact.each do |reference| next if valid.include?(reference.to_sym) diff --git a/spec/models/gql/runtime/functions/lookup_spec.rb b/spec/models/gql/runtime/functions/lookup_spec.rb index a9c20fb40..2c65f1eae 100644 --- a/spec/models/gql/runtime/functions/lookup_spec.rb +++ b/spec/models/gql/runtime/functions/lookup_spec.rb @@ -272,6 +272,12 @@ def keys_of(nodes) end end + describe "EMISSIONS(ipcc_crt_code_agg, '1.A.1', '3', co2, 1990)" do + it 'raises on a multi-value call instead of misreading a value as the GHG' do + expect { result }.to raise_error(Gql::GqlError, /single value/) + end + end + it 'rejects UPDATE on a mapped EMISSIONS expression' do expect do gql.query_future("UPDATE(EMISSIONS(ipcc_crt_code_agg, '1.A.1', co2), co2, 1.0)") diff --git a/spec/serializers/export/configured_csv_serializer_spec.rb b/spec/serializers/export/configured_csv_serializer_spec.rb index b5d36e9d5..c858fec43 100644 --- a/spec/serializers/export/configured_csv_serializer_spec.rb +++ b/spec/serializers/export/configured_csv_serializer_spec.rb @@ -185,10 +185,17 @@ { bar: node_bar, baz: node_baz, foo: node_foo, buildings_space_heating_demand: node_buildings, lft: node_lft } end + let(:molecule_graph) { instance_double('Qernel::Graph') } + before do allow(gql.future.graph).to receive(:node) { |key| energy_nodes[key.to_sym] } - allow(gql.future).to receive(:molecules) - .and_return(instance_double('Qernel::Graph').tap { |g| allow(g).to receive(:node).with(:m_waste).and_return(node_waste) }) + allow(gql.future).to receive(:molecules).and_return(molecule_graph) + allow(molecule_graph).to receive(:node).with(:m_waste).and_return(node_waste) + + allow(molecule_graph).to receive(:sector_map) do + sectors = Etsource::Sectors.new + Qernel::Sectors.new(molecule_graph, sectors.mapping, sectors.node_index(:molecules)) + end end it 'includes the CSV headers' do @@ -376,6 +383,53 @@ end end + context 'when a mapping-driven config is invalid' do + let(:gql) { Scenario.default.gql } + let(:mapping_rows) { { require: 'emissions_sector' } } + let(:schema) { [{ name: 'Key', type: 'node_attribute', value: 'key' }] } + + context 'with no period' do + let(:serializer) { described_class.new({ schema: schema, rows: mapping_rows }, gql) } + + it 'raises at construction' do + expect { serializer }.to raise_error(ArgumentError, /period/) + end + end + + context 'with no require: column' do + let(:serializer) do + described_class.new({ schema: schema, rows: { order_by: 'ipcc_crt_code' } }, gql, period: :future) + end + + it 'raises at construction' do + expect { serializer }.to raise_error(ArgumentError, /require/) + end + end + + context 'with a column type only valid for query-driven rows' do + let(:schema) { super() + [{ name: 'Total', type: 'query' }] } + let(:serializer) { described_class.new({ schema: schema, rows: mapping_rows }, gql, period: :future) } + + it 'raises at construction instead of rendering silent blank cells' do + expect { serializer }.to raise_error( + Export::ConfiguredCSVSerializer::UnsupportedColumnTypeError, + /node_attribute.*sector_mapping/ + ) + end + end + + context 'with an untyped (literal) column' do + let(:schema) { super() + [{ name: 'Sector' }] } + let(:serializer) { described_class.new({ schema: schema, rows: mapping_rows }, gql, period: :future) } + + it 'raises at construction instead of rendering silent blank cells' do + expect { serializer }.to raise_error( + Export::ConfiguredCSVSerializer::UnsupportedColumnTypeError, /literal/ + ) + end + end + end + context 'when given an unknown sector mapping column' do let(:gql) { Scenario.default.gql } let(:serializer) { described_class.new(config, gql, period: :future) } From a2c24decf168ea4623ffec82ca3dbd946df92957 Mon Sep 17 00:00:00 2001 From: louispt1 Date: Wed, 15 Jul 2026 15:02:39 +0200 Subject: [PATCH 21/23] Fix whitespace issue in gqueries controller for inspect gqueries --- app/controllers/inspect/gqueries_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/inspect/gqueries_controller.rb b/app/controllers/inspect/gqueries_controller.rb index 6dffce086..85981933f 100644 --- a/app/controllers/inspect/gqueries_controller.rb +++ b/app/controllers/inspect/gqueries_controller.rb @@ -18,7 +18,7 @@ def test redirect_to inspect_debug_gql_path(gquery: params[:query]) elsif params[:query].present? begin - @result = @gql.query(params[:query].gsub(/\s/,''), nil, true) + @result = @gql.query(params[:query], nil, true) rescue Gql::CommandError => ex @error = ex end From ff8d4aeb82adcc367dcaf431d5c445b4145279aa Mon Sep 17 00:00:00 2001 From: louispt1 Date: Thu, 16 Jul 2026 13:12:09 +0200 Subject: [PATCH 22/23] Bump atlas SHA and update gemfile.lock --- Gemfile | 2 +- Gemfile.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index 4c12d8b39..4e0aaa4d8 100644 --- a/Gemfile +++ b/Gemfile @@ -73,7 +73,7 @@ gem 'ruby-progressbar' # own gems gem 'quintel_merit', ref: 'aae77e0', github: 'quintel/merit' -gem 'atlas', ref: '5d2e5b2', github: 'quintel/atlas' +gem 'atlas', ref: '732edc0', github: 'quintel/atlas' gem 'fever', ref: '2afebd1', github: 'quintel/fever' gem 'refinery', ref: '36b8e34', github: 'quintel/refinery' gem 'rubel', ref: '9fe7010', github: 'quintel/rubel' diff --git a/Gemfile.lock b/Gemfile.lock index 8227f48d4..2ef2f40d3 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ GIT remote: https://github.com/quintel/atlas.git - revision: 5d2e5b20e3d397b764f688180e3987bd43161612 - ref: 5d2e5b2 + revision: 732edc0301c7fb4a663c586cc213bae7178eb7b5 + ref: 732edc0 specs: atlas (1.0.0) activemodel (>= 7) From d3347f9a7ca19a8498c7d3ab853779b689abf2e5 Mon Sep 17 00:00:00 2001 From: Kyra de Haan Date: Fri, 17 Jul 2026 16:08:58 +0200 Subject: [PATCH 23/23] Bump Atlas to quintel/atlas@b2d47a1 --- Gemfile | 2 +- Gemfile.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index 4e0aaa4d8..3f9023901 100644 --- a/Gemfile +++ b/Gemfile @@ -73,7 +73,7 @@ gem 'ruby-progressbar' # own gems gem 'quintel_merit', ref: 'aae77e0', github: 'quintel/merit' -gem 'atlas', ref: '732edc0', github: 'quintel/atlas' +gem 'atlas', ref: 'b2d47a1', github: 'quintel/atlas' gem 'fever', ref: '2afebd1', github: 'quintel/fever' gem 'refinery', ref: '36b8e34', github: 'quintel/refinery' gem 'rubel', ref: '9fe7010', github: 'quintel/rubel' diff --git a/Gemfile.lock b/Gemfile.lock index 2ef2f40d3..1d8757f4d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ GIT remote: https://github.com/quintel/atlas.git - revision: 732edc0301c7fb4a663c586cc213bae7178eb7b5 - ref: 732edc0 + revision: b2d47a1495f85638398d89acc17c6a2da7d2acac + ref: b2d47a1 specs: atlas (1.0.0) activemodel (>= 7)