From b954b5c65f12e230233a8344b26296affd3fcaa7 Mon Sep 17 00:00:00 2001 From: louispt1 Date: Wed, 8 Jul 2026 13:28:09 +0200 Subject: [PATCH 1/7] Bump atlas SHA and update gemfile.lock --- Gemfile | 2 +- Gemfile.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile b/Gemfile index 8e73e6328..423f50229 100644 --- a/Gemfile +++ b/Gemfile @@ -73,7 +73,7 @@ gem 'ruby-progressbar' # own gems gem 'quintel_merit', ref: 'aae77e0', github: 'quintel/merit' -gem 'atlas', ref: '5504fac', github: 'quintel/atlas' +gem 'atlas', ref: '78bc9d9', github: 'quintel/atlas' gem 'fever', ref: '2afebd1', github: 'quintel/fever' gem 'refinery', ref: '36b8e34', github: 'quintel/refinery' gem 'rubel', ref: '9fe7010', github: 'quintel/rubel' diff --git a/Gemfile.lock b/Gemfile.lock index 36acecd4b..97e0643e9 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ GIT remote: https://github.com/quintel/atlas.git - revision: 5504fac349b1b25fda2ef285b044b025fbfbdb1e - ref: 5504fac + revision: 78bc9d961c5fccbc30fc7a39a98fccedefc7e2ef + ref: 78bc9d9 specs: atlas (1.0.0) activemodel (>= 7) From 2561960178e6af064b3f6a4c9cefd38b161cd33b Mon Sep 17 00:00:00 2001 From: louispt1 Date: Wed, 8 Jul 2026 17:00:17 +0200 Subject: [PATCH 2/7] Add sector mapping handling and rspec --- app/models/etsource/sectors.rb | 50 ++++++ app/models/gql/gql_error.rb | 17 ++ app/models/gql/query_interface/lookup.rb | 16 ++ app/models/gql/runtime/functions/lookup.rb | 57 +++++- app/models/qernel/graph.rb | 12 ++ app/models/qernel/sectors.rb | 73 ++++++++ .../etsource/config/sector_mapping.csv | 12 ++ .../buildings_space_heating_demand.ad | 2 + .../graphs/energy/nodes/nosector/bar.ad | 2 + .../graphs/energy/nodes/nosector/baz.ad | 2 + .../graphs/energy/nodes/nosector/foo.ad | 2 + .../graphs/energy/nodes/nosector/lft.ad | 4 + .../etsource/graphs/molecules/nodes/m_left.ad | 2 + .../gql/runtime/functions/lookup_spec.rb | 163 ++++++++++++++++++ 14 files changed, 408 insertions(+), 6 deletions(-) create mode 100644 app/models/etsource/sectors.rb create mode 100644 app/models/qernel/sectors.rb create mode 100644 spec/fixtures/etsource/config/sector_mapping.csv diff --git a/app/models/etsource/sectors.rb b/app/models/etsource/sectors.rb new file mode 100644 index 000000000..b8cecea26 --- /dev/null +++ b/app/models/etsource/sectors.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Etsource + # Reads the sector mapping and node labels from Atlas and caches the two + # structures GQL needs to resolve `SECTOR(scheme, value)` style queries. + # + # Mirrors the merit order import precedent: the expensive Atlas read happens + # once and the result is memoized in the Rails cache, so per-request cost is a + # couple of hash lookups. + # + # * mapping - {scheme => {value => [[label, use], ...]}} + # * node_index(g) - {[label, use] => [node_key, ...]} for one graph type, + # with `use` baked in at import so it is never consulted + # per query. + class Sectors + def initialize(etsource = Etsource::Base.instance) + @etsource = etsource + end + + # Public: The inverted mapping index, keyed by scheme then normalized value. + def mapping + Rails.cache.fetch('sector_mapping_hash') do + Atlas::SectorMapping.load.to_h + end + end + + # Public: The (label, use) -> node keys index for the given graph type + # (:energy or :molecules). + def node_index(graph_type) + Rails.cache.fetch("sector_node_index_#{graph_type}") do + build_node_index(node_class_for(graph_type)) + end + end + + private + + def build_node_index(node_class) + node_class.all.each_with_object({}) do |node, index| + next if node.sector_label.nil? + + pair = [node.sector_label, Atlas::SectorMapping.normalize(node.use)] + (index[pair] ||= []) << node.key + end + end + + def node_class_for(graph_type) + graph_type.to_sym == :molecules ? Atlas::MoleculeNode : Atlas::EnergyNode + end + end +end diff --git a/app/models/gql/gql_error.rb b/app/models/gql/gql_error.rb index 462b1a5de..d016999cf 100644 --- a/app/models/gql/gql_error.rb +++ b/app/models/gql/gql_error.rb @@ -23,4 +23,21 @@ def initialize(curve, attribute = nil) end end + # Raised when a SECTOR/MSECTOR/EMISSIONS scheme-form query names a + # classification scheme which does not exist in the sector mapping. + class UnknownSectorSchemeError < GqlError + def initialize(scheme, valid_schemes) + super("Unknown sector mapping scheme #{scheme.inspect}. " \ + "Valid schemes: #{valid_schemes.map(&:inspect).join(', ')}.") + end + end + + # Raised when a SECTOR/MSECTOR/EMISSIONS scheme-form query names a value + # which does not appear in that scheme's column. + class UnknownSectorValueError < GqlError + def initialize(scheme, value) + super("Unknown value #{value.inspect} for sector mapping scheme " \ + "#{scheme.inspect}.") + end + end end diff --git a/app/models/gql/query_interface/lookup.rb b/app/models/gql/query_interface/lookup.rb index 62c20c12d..ca372b175 100644 --- a/app/models/gql/query_interface/lookup.rb +++ b/app/models/gql/query_interface/lookup.rb @@ -64,6 +64,22 @@ def molecule_sector_nodes(keys) molecule_graph_helper.sector_nodes(keys) end + # Scheme-based sector mapping lookups. `energy`/`molecule` select the + # graph whose live nodes are returned; the mapping itself is shared. + def energy_sector_map(scheme, values) + graph.sector_map.lookup(scheme, values) + end + + def molecule_sector_map(scheme, values) + molecules.sector_map.lookup(scheme, values) + end + + # The energy graph's resolver, used for EMISSIONS scheme dispatch and + # pair resolution (the mapping is graph-independent). + def sector_resolver + graph.sector_map + end + def energy_use_nodes(keys) energy_graph_helper.use_nodes(keys) end diff --git a/app/models/gql/runtime/functions/lookup.rb b/app/models/gql/runtime/functions/lookup.rb index f15c80ed4..d66e384bd 100644 --- a/app/models/gql/runtime/functions/lookup.rb +++ b/app/models/gql/runtime/functions/lookup.rb @@ -116,19 +116,36 @@ def MEDGE_GROUP(*keys) end alias MEG EDGE_GROUP - # Returns an Array of {Qernel::Node} for given energy sector. + # Returns an Array of {Qernel::Node} for an energy sector. # - # Examples + # Dispatches on arity: # - # SECTOR(households) + # * One argument - namespace-sector filter. + # SECTOR(households) + # * Two or more - a classification scheme followed by one or more + # values; returns the union of matching nodes. + # SECTOR(klimaattafel, 'Industrie') + # SECTOR(ipcc_crt_code_agg, '1.A.1', '1.A.2') # + # Unknown scheme or value raises; a valid value with no labelled nodes + # returns an empty array. def SECTOR(*keys) - scope.energy_sector_nodes(keys) + if keys.size <= 1 + scope.energy_sector_nodes(keys) + else + scope.energy_sector_map(keys.first, keys.drop(1)) + end end - # Returns an Array of {Qernel::Node} for given molecule sector. See SECTOR. + # Returns an Array of {Qernel::Node} for a molecule sector. See SECTOR; + # identical dispatch, normalization and error contract, resolved against + # the molecule graph. def MSECTOR(*keys) - scope.molecule_sector_nodes(keys) + if keys.size <= 1 + scope.molecule_sector_nodes(keys) + else + scope.molecule_sector_map(keys.first, keys.drop(1)) + end end # Returns an Array with {Qernel::Node} for given energy use. @@ -362,6 +379,12 @@ def DATASET_CURVE(key) def EMISSIONS(*keys) return scope.graph.emissions if keys.empty? + # Mapped form: EMISSIONS(scheme, value, ghg[, 1990]). Dispatches on the + # first argument being a known classification scheme, because the legacy + # arities overlap. Read-only: sums the store over the mapping's resolved + # (sector label, use) pairs; never returns a scoped-sector handle. + return mapped_emissions(keys) if scope.sector_resolver.scheme?(keys.first) + # Convert dashes/dots to underscores in the first key (sector) keys[0] = keys.first.to_s.tr('-.', '_').to_sym @@ -374,6 +397,28 @@ def EMISSIONS(*keys) [*keys.first(3), (1990 if keys[3].to_i == 1990)].compact.join('_').to_sym ] end + + private + + # Internal: Sums the emissions store over every (sector label, use) pair + # the mapping resolves for `scheme`/`value`. Present year is implicit; + # only 1990 is suffixed. Missing store keys count as zero. + def mapped_emissions(keys) + scheme, value, ghg, year = keys + + if ghg.nil? + raise Gql::GqlError, "EMISSIONS(#{scheme.inspect}, ...) needs a GHG argument, e.g. " \ + "EMISSIONS(#{scheme.inspect}, #{value.inspect}, co2)" + end + + suffix = ('1990' if year.to_i == 1990) + emissions = scope.graph.emissions + + scope.sector_resolver.pairs(scheme, value).sum do |label, use| + key = [label, use, ghg, suffix].compact.join('_').to_sym + emissions[key] || 0.0 + end + end end end end diff --git a/app/models/qernel/graph.rb b/app/models/qernel/graph.rb index 208a3605e..756d70d79 100644 --- a/app/models/qernel/graph.rb +++ b/app/models/qernel/graph.rb @@ -264,6 +264,18 @@ def sectors nodes.map(&:sector_key).uniq.compact end + # Resolver for scheme-based sector mapping queries (SECTOR(scheme, value)). + # + # Memoized per graph instance. + # + # @return [Qernel::Sectors] + def sector_map + @sector_map ||= begin + sectors = Etsource::Sectors.new + Qernel::Sectors.new(self, sectors.mapping, sectors.node_index(@name)) + end + end + # Return all nodes in the given sector. # # @param sector_key [String,Symbol] sector identifier diff --git a/app/models/qernel/sectors.rb b/app/models/qernel/sectors.rb new file mode 100644 index 000000000..630895d5e --- /dev/null +++ b/app/models/qernel/sectors.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module Qernel + # Per-graph-instance resolver for scheme-based sector queries. + # + # Present and future graphs (and the energy and molecule graphs) are separate + # instances holding separate node objects, so each gets its own resolver via + # the graph's `sector_map` accessor. The mapping and the (label, use) -> node + # key index are read once from {Etsource::Sectors}; resolution turns node keys + # into the live nodes of this graph. + # + # Dispatch, arity and aggregation live in the GQL layer; this class answers + # two questions: which (label, use) pairs, and which live nodes, belong to + # (scheme, values)? + class Sectors + def initialize(graph, mapping, node_index) + @graph = graph + @mapping = mapping + @node_index = node_index + @node_cache = {} + @pair_cache = {} + end + + # Public: Whether `scheme`` names a classification scheme in the mapping. + # Used by the EMISSIONS first-argument dispatch. + def scheme?(scheme) + @mapping.key?(normalize(scheme)) + end + + # Public: The unique (label, use) pairs falling under any of `values` of + # `scheme`. Reads the mapping only, so it is independent of node labelling; + # the EMISSIONS mapped form sums the store over these pairs. + # + # Raises Gql::UnknownSectorSchemeError / Gql::UnknownSectorValueError. + def pairs(scheme, values) + value_list = Array(values) + key = [normalize(scheme), value_list.map { |value| normalize(value) }] + + @pair_cache[key] ||= resolve_pairs(scheme, value_list) + end + + # Public: The live nodes of this graph whose (label, use) pair falls under + # any of `values` of 'scheme` (the union). A value which resolves to pairs + # with no labelled nodes contributes nothing. + def lookup(scheme, values) + value_list = Array(values) + key = [normalize(scheme), value_list.map { |value| normalize(value) }] + + @node_cache[key] ||= + pairs(scheme, value_list).flat_map { |pair| nodes_for(pair) }.uniq + end + + private + + def resolve_pairs(scheme, value_list) + value_map = @mapping[normalize(scheme)] + raise Gql::UnknownSectorSchemeError.new(scheme, @mapping.keys) if value_map.nil? + + value_list.flat_map do |value| + value_map[normalize(value)] || + raise(Gql::UnknownSectorValueError.new(scheme, value)) + end.uniq + end + + def nodes_for(pair) + Array(@node_index[pair]).filter_map { |key| @graph.node(key) } + end + + def normalize(value) + Atlas::SectorMapping.normalize(value) + end + end +end diff --git a/spec/fixtures/etsource/config/sector_mapping.csv b/spec/fixtures/etsource/config/sector_mapping.csv new file mode 100644 index 000000000..eb2e1e87e --- /dev/null +++ b/spec/fixtures/etsource/config/sector_mapping.csv @@ -0,0 +1,12 @@ +sector_label,use,ipcc_crt_code_agg,klimaattafel +energy_electricity_and_heat_production,energetic,1.A.1,Elektriciteit +industry_refineries,energetic,1.A.1,Industrie +industry_refineries,non_energetic,1.B,Industrie +industry_non_specified,energetic,1.A.2,Industrie +buildings_non_specified,energetic,1.A.4.a,Gebouwde omgeving +households_non_specified,energetic,1.A.4.b,Gebouwde omgeving +agriculture_non_specified,energetic,1.A.4.c,Landbouw +agriculture_non_specified,non_energetic,3,Landbouw +waste_non_specified,non_energetic,5,Elektriciteit +energy_fugitive_emissions,non_energetic,1.B,Elektriciteit +other_heating,energetic,-,Mobiliteit diff --git a/spec/fixtures/etsource/graphs/energy/nodes/buildings/buildings_space_heating_demand.ad b/spec/fixtures/etsource/graphs/energy/nodes/buildings/buildings_space_heating_demand.ad index ef7605b8d..70ed26212 100644 --- a/spec/fixtures/etsource/graphs/energy/nodes/buildings/buildings_space_heating_demand.ad +++ b/spec/fixtures/etsource/graphs/energy/nodes/buildings/buildings_space_heating_demand.ad @@ -6,3 +6,5 @@ - demand = 100.0 - preset_demand = 100.0 - number_of_units = 200.0 +- sector_label = buildings_non_specified +- use = energetic diff --git a/spec/fixtures/etsource/graphs/energy/nodes/nosector/bar.ad b/spec/fixtures/etsource/graphs/energy/nodes/nosector/bar.ad index 1b98c7d08..7d0e985d0 100644 --- a/spec/fixtures/etsource/graphs/energy/nodes/nosector/bar.ad +++ b/spec/fixtures/etsource/graphs/energy/nodes/nosector/bar.ad @@ -1 +1,3 @@ - groups = [application_group] +- sector_label = industry_refineries +- use = energetic diff --git a/spec/fixtures/etsource/graphs/energy/nodes/nosector/baz.ad b/spec/fixtures/etsource/graphs/energy/nodes/nosector/baz.ad index d2f2e56cb..150b40384 100644 --- a/spec/fixtures/etsource/graphs/energy/nodes/nosector/baz.ad +++ b/spec/fixtures/etsource/graphs/energy/nodes/nosector/baz.ad @@ -1,2 +1,4 @@ - groups = [] - graph_methods = [demand_setter] +- sector_label = industry_refineries +- use = non_energetic diff --git a/spec/fixtures/etsource/graphs/energy/nodes/nosector/foo.ad b/spec/fixtures/etsource/graphs/energy/nodes/nosector/foo.ad index 636979b5d..e158cc9e9 100644 --- a/spec/fixtures/etsource/graphs/energy/nodes/nosector/foo.ad +++ b/spec/fixtures/etsource/graphs/energy/nodes/nosector/foo.ad @@ -3,3 +3,5 @@ - merit_order.type = dispatchable - merit_order.group = solar_pv - graph_methods = [demand_setter] +- sector_label = industry_non_specified +- use = energetic diff --git a/spec/fixtures/etsource/graphs/energy/nodes/nosector/lft.ad b/spec/fixtures/etsource/graphs/energy/nodes/nosector/lft.ad index 27c3b9df6..1254ea0ec 100644 --- a/spec/fixtures/etsource/graphs/energy/nodes/nosector/lft.ad +++ b/spec/fixtures/etsource/graphs/energy/nodes/nosector/lft.ad @@ -1,2 +1,6 @@ - groups = [] - preset_demand = 0.0 + +# Sector mapping test assignment (blank-cell case: "-" ipcc, reachable via klimaattafel) +- sector_label = other_heating +- use = energetic diff --git a/spec/fixtures/etsource/graphs/molecules/nodes/m_left.ad b/spec/fixtures/etsource/graphs/molecules/nodes/m_left.ad index 89d1c5a8a..cb14c83bb 100644 --- a/spec/fixtures/etsource/graphs/molecules/nodes/m_left.ad +++ b/spec/fixtures/etsource/graphs/molecules/nodes/m_left.ad @@ -1,3 +1,5 @@ - groups = [emissions] - from_energy.source = molecule_source - input.co2 = 1.0 +- sector_label = waste_non_specified +- use = non_energetic diff --git a/spec/models/gql/runtime/functions/lookup_spec.rb b/spec/models/gql/runtime/functions/lookup_spec.rb index af3eff0e2..693f7e367 100644 --- a/spec/models/gql/runtime/functions/lookup_spec.rb +++ b/spec/models/gql/runtime/functions/lookup_spec.rb @@ -104,5 +104,168 @@ module Gql::Runtime::Functions expect(result).to eq([]) end end + + # SECTOR - scheme-based mapping lookups (energy graph) + # --------------------------------------------------- + + def keys_of(nodes) + nodes.map(&:key).sort + end + + describe 'legacy one-argument SECTOR' do + it 'is unchanged: returns the namespace-sector nodes' do + legacy = gql.query_future('SECTOR(nosector)') + direct = gql.future_graph.sector_nodes(:nosector) + + expect(keys_of(legacy)).to eq(keys_of(direct)) + end + end + + describe "SECTOR(klimaattafel, 'Industrie')" do + it 'returns the labelled energy nodes in that klimaattafel' do + expect(keys_of(result)).to eq(%i[bar baz foo]) + end + end + + describe "SECTOR(ipcc_crt_code_agg, '1.A.1')" do + it 'returns only the energetic-use node for that IPCC category' do + expect(keys_of(result)).to eq(%i[bar]) + end + end + + describe "SECTOR(ipcc_crt_code_agg, '1.B')" do + it 'returns only the non-energetic-use node (pair correctness)' do + expect(keys_of(result)).to eq(%i[baz]) + end + end + + describe "SECTOR(ipcc_crt_code_agg, '1.A.1', '1.A.2')" do + it 'returns the union over several values' do + expect(keys_of(result)).to eq(%i[bar foo]) + end + end + + describe 'SECTOR(sector_label, industry_refineries)' do + it 'resolves the canonical key directly (both uses)' do + expect(keys_of(result)).to eq(%i[bar baz]) + end + end + + describe "SECTOR(klimaattafel, 'Landbouw')" do + it 'returns [] for a valid value with no labelled nodes' do + expect(result).to eq([]) + end + end + + describe "SECTOR(impossible, 'x')" do + it 'raises naming the valid schemes' do + expect { result }.to raise_error(/Valid schemes.*klimaattafel/m) + end + end + + describe "SECTOR(klimaattafel, 'Nonexistent')" do + it 'raises for an unknown value' do + expect { result }.to raise_error(/Unknown value "Nonexistent"/) + end + end + + # Blank / "-" cells. The `other_heating,energetic` row has a "-" ipcc cell + # (fixture) and the `lft` node is labelled to it. + + describe "SECTOR(klimaattafel, 'Mobiliteit')" do + it 'reaches a "-"-celled row through a populated scheme' do + expect(keys_of(result)).to eq(%i[lft]) + end + end + + describe 'SECTOR(sector_label, other_heating)' do + it 'reaches the "-"-celled row by its canonical key' do + expect(keys_of(result)).to eq(%i[lft]) + end + end + + describe "SECTOR(ipcc_crt_code_agg, '-')" do + it 'raises: a "-" cell is unqueryable, not a category' do + expect { result }.to raise_error(/Unknown value/) + end + end + + describe "EMISSIONS(ipcc_crt_code_agg, '-', co2)" do + it 'raises: the mapped form rejects a blank value too' do + expect { result }.to raise_error(/Unknown value/) + end + end + + # MSECTOR - scheme-based mapping lookups (molecule graph) + # ------------------------------------------------------ + + describe "MSECTOR(ipcc_crt_code_agg, '5')" do + it 'returns the labelled molecule node' do + expect(keys_of(result)).to eq(%i[m_left]) + end + end + + describe "MSECTOR(klimaattafel, 'Elektriciteit')" do + it 'returns only molecule nodes (disjoint from the energy graph)' do + expect(keys_of(result)).to eq(%i[m_left]) + end + end + + describe "SECTOR(ipcc_crt_code_agg, '5')" do + it 'returns [] on the energy graph for a molecule-only pair' do + expect(result).to eq([]) + end + end + + describe 'legacy one-argument MSECTOR' do + it 'is unchanged: routes to the namespace-sector molecule filter' do + expect(gql.query_future('MSECTOR(anything)')) + .to eq(molecule_graph.sector_nodes(:anything)) + end + end + + # EMISSIONS - mapped scheme form + # ------------------------------ + + describe "EMISSIONS(ipcc_crt_code_agg, '1.A.1', other_ghg)" do + it 'sums the store over the resolved pairs (missing keys as zero)' do + # energy_electricity_and_heat_production_energetic_other_ghg = 18.0 + # industry_refineries_energetic_other_ghg is absent -> 0.0 + expect(result).to eq(18.0) + end + end + + describe "EMISSIONS(ipcc_crt_code_agg, '1.A.1', other_ghg, 1990)" do + it 'reads the 1990 baseline keys' do + expect(result).to eq(25.0) + end + end + + describe "EMISSIONS(ipcc_crt_code_agg, '3', co2)" do + it 'sums only the non-energetic rows the mapping lists' do + # agriculture_non_specified_non_energetic_co2 = 75.0; the energetic + # agriculture row (also Landbouw) is not under IPCC 3. + expect(result).to eq(75.0) + end + end + + describe "EMISSIONS(klimaattafel, 'Landbouw', other_ghg)" do + it 'unions energetic and non-energetic rows of the value' do + # agriculture energetic (50.0) + agriculture non_energetic (100.0) + expect(result).to eq(150.0) + end + end + + describe "EMISSIONS(ipcc_crt_code_agg, '1.A.1')" do + it 'raises without a GHG argument' do + expect { result }.to raise_error(Gql::GqlError, /needs a GHG argument/) + end + end + + it 'rejects UPDATE on a mapped EMISSIONS expression' do + expect do + gql.query_future("UPDATE(EMISSIONS(ipcc_crt_code_agg, '1.A.1', co2), co2, 1.0)") + end.to raise_error(StandardError) + end end end From 6bfd63d5cb7d32522cc664266f179c8f73ca755a Mon Sep 17 00:00:00 2001 From: louispt1 Date: Wed, 8 Jul 2026 17:22:50 +0200 Subject: [PATCH 3/7] New fixture to avoid conflicting with MUSE spec --- .../etsource/graphs/molecules/nodes/m_left.ad | 2 -- .../graphs/molecules/nodes/nosector/m_waste.ad | 3 +++ spec/models/gql/runtime/functions/lookup_spec.rb | 16 ++++++++++------ 3 files changed, 13 insertions(+), 8 deletions(-) create mode 100644 spec/fixtures/etsource/graphs/molecules/nodes/nosector/m_waste.ad diff --git a/spec/fixtures/etsource/graphs/molecules/nodes/m_left.ad b/spec/fixtures/etsource/graphs/molecules/nodes/m_left.ad index cb14c83bb..89d1c5a8a 100644 --- a/spec/fixtures/etsource/graphs/molecules/nodes/m_left.ad +++ b/spec/fixtures/etsource/graphs/molecules/nodes/m_left.ad @@ -1,5 +1,3 @@ - groups = [emissions] - from_energy.source = molecule_source - input.co2 = 1.0 -- sector_label = waste_non_specified -- use = non_energetic diff --git a/spec/fixtures/etsource/graphs/molecules/nodes/nosector/m_waste.ad b/spec/fixtures/etsource/graphs/molecules/nodes/nosector/m_waste.ad new file mode 100644 index 000000000..8ce379f39 --- /dev/null +++ b/spec/fixtures/etsource/graphs/molecules/nodes/nosector/m_waste.ad @@ -0,0 +1,3 @@ +- groups = [] +- sector_label = waste_non_specified +- use = non_energetic diff --git a/spec/models/gql/runtime/functions/lookup_spec.rb b/spec/models/gql/runtime/functions/lookup_spec.rb index 693f7e367..acf7c72b8 100644 --- a/spec/models/gql/runtime/functions/lookup_spec.rb +++ b/spec/models/gql/runtime/functions/lookup_spec.rb @@ -85,16 +85,20 @@ module Gql::Runtime::Functions end describe 'MUSE(non_energetic)' do - it 'returns [Node(m_right_two)]' do - expect(result).to eq([molecule_graph.node(:m_right_two)]) + it 'returns [Node(m_right_two), Node(m_waste)]' do + expect(result).to eq([ + molecule_graph.node(:m_right_two), + molecule_graph.node(:m_waste) + ]) end end describe 'MUSE(energetic, non_energetic)' do - it 'returns [Node(m_right_one), Node(m_right_two)]' do + it 'returns [Node(m_right_one), Node(m_right_two), Node(m_waste)]' do expect(result).to eq([ molecule_graph.node(:m_right_one), - molecule_graph.node(:m_right_two) + molecule_graph.node(:m_right_two), + molecule_graph.node(:m_waste) ]) end end @@ -201,13 +205,13 @@ def keys_of(nodes) describe "MSECTOR(ipcc_crt_code_agg, '5')" do it 'returns the labelled molecule node' do - expect(keys_of(result)).to eq(%i[m_left]) + expect(keys_of(result)).to eq(%i[m_waste]) end end describe "MSECTOR(klimaattafel, 'Elektriciteit')" do it 'returns only molecule nodes (disjoint from the energy graph)' do - expect(keys_of(result)).to eq(%i[m_left]) + expect(keys_of(result)).to eq(%i[m_waste]) end end From 8db898cf67d639ac0b865064d1432fb5242c6f9c Mon Sep 17 00:00:00 2001 From: louispt1 Date: Tue, 14 Jul 2026 11:01:42 +0200 Subject: [PATCH 4/7] Use nastycache for csv mapping, rename methods and move summing to the emissions class --- app/models/etsource/sectors.rb | 7 +-- app/models/gql/command.rb | 9 ++-- app/models/gql/query_interface/lookup.rb | 2 +- app/models/gql/runtime/functions/lookup.rb | 23 +++----- app/models/qernel/emissions.rb | 29 ++++++++-- app/models/qernel/sectors.rb | 6 +-- spec/models/gql/command_spec.rb | 35 ++++++++++++ .../gql/runtime/functions/lookup_spec.rb | 6 +++ spec/models/qernel/emissions_spec.rb | 54 +++++++++++++++++++ 9 files changed, 139 insertions(+), 32 deletions(-) create mode 100644 spec/models/gql/command_spec.rb diff --git a/app/models/etsource/sectors.rb b/app/models/etsource/sectors.rb index b8cecea26..0967f002a 100644 --- a/app/models/etsource/sectors.rb +++ b/app/models/etsource/sectors.rb @@ -4,9 +4,6 @@ module Etsource # Reads the sector mapping and node labels from Atlas and caches the two # structures GQL needs to resolve `SECTOR(scheme, value)` style queries. # - # Mirrors the merit order import precedent: the expensive Atlas read happens - # once and the result is memoized in the Rails cache, so per-request cost is a - # couple of hash lookups. # # * mapping - {scheme => {value => [[label, use], ...]}} # * node_index(g) - {[label, use] => [node_key, ...]} for one graph type, @@ -19,7 +16,7 @@ def initialize(etsource = Etsource::Base.instance) # Public: The inverted mapping index, keyed by scheme then normalized value. def mapping - Rails.cache.fetch('sector_mapping_hash') do + NastyCache.instance.fetch('sector_mapping_hash') do Atlas::SectorMapping.load.to_h end end @@ -27,7 +24,7 @@ def mapping # Public: The (label, use) -> node keys index for the given graph type # (:energy or :molecules). def node_index(graph_type) - Rails.cache.fetch("sector_node_index_#{graph_type}") do + NastyCache.instance.fetch("sector_node_index_#{graph_type}") do build_node_index(node_class_for(graph_type)) end end diff --git a/app/models/gql/command.rb b/app/models/gql/command.rb index 653333df1..73546f37d 100644 --- a/app/models/gql/command.rb +++ b/app/models/gql/command.rb @@ -108,12 +108,15 @@ def truncate(string, length = 40) string.length > (length - 3) ? "#{ string[0..length] }..." : string end - # Internal: Removes extra "artifacts" from the source. + # Internal: Removes extra "artifacts" from the source. Whitespace is + # insignificant in GQL and stripped, except inside string literals, where + # it may be meaningful (e.g. SECTOR scheme values such as 'Fuels + # production'). def clean(string) cleaned = (string || '').dup - cleaned.gsub!(/[\n\s\t]/, '') - cleaned.gsub!(/^[a-z]+\:/,'') + cleaned.gsub!(/('[^']*'|"[^"]*")|\s+/) { Regexp.last_match(1) || '' } + cleaned.sub!(/\A[a-z]+\:/, '') cleaned end diff --git a/app/models/gql/query_interface/lookup.rb b/app/models/gql/query_interface/lookup.rb index ca372b175..2985cf9ef 100644 --- a/app/models/gql/query_interface/lookup.rb +++ b/app/models/gql/query_interface/lookup.rb @@ -66,7 +66,7 @@ def molecule_sector_nodes(keys) # Scheme-based sector mapping lookups. `energy`/`molecule` select the # graph whose live nodes are returned; the mapping itself is shared. - def energy_sector_map(scheme, values) + def energy_sector_node_map(scheme, values) graph.sector_map.lookup(scheme, values) end diff --git a/app/models/gql/runtime/functions/lookup.rb b/app/models/gql/runtime/functions/lookup.rb index d66e384bd..36d331357 100644 --- a/app/models/gql/runtime/functions/lookup.rb +++ b/app/models/gql/runtime/functions/lookup.rb @@ -124,7 +124,7 @@ def MEDGE_GROUP(*keys) # SECTOR(households) # * Two or more - a classification scheme followed by one or more # values; returns the union of matching nodes. - # SECTOR(klimaattafel, 'Industrie') + # SECTOR(emissions_subsector, 'Electricity and heat production') # SECTOR(ipcc_crt_code_agg, '1.A.1', '1.A.2') # # Unknown scheme or value raises; a valid value with no labelled nodes @@ -133,7 +133,7 @@ def SECTOR(*keys) if keys.size <= 1 scope.energy_sector_nodes(keys) else - scope.energy_sector_map(keys.first, keys.drop(1)) + scope.energy_sector_node_map(keys.first, keys.drop(1)) end end @@ -393,16 +393,13 @@ def EMISSIONS(*keys) # EMISSIONS(sector, use, ghg [, year]) -> return value. # The present year is implicit; only 1990 is suffixed with a year. - scope.graph.emissions[ - [*keys.first(3), (1990 if keys[3].to_i == 1990)].compact.join('_').to_sym - ] + scope.graph.emissions.value_for(*keys.first(3), year: keys[3]) end private - # Internal: Sums the emissions store over every (sector label, use) pair - # the mapping resolves for `scheme`/`value`. Present year is implicit; - # only 1990 is suffixed. Missing store keys count as zero. + # Internal: The mapped EMISSIONS form. Resolves `scheme`/`value` to + # (sector label, use) pairs and lets the emissions store sum over them. def mapped_emissions(keys) scheme, value, ghg, year = keys @@ -411,13 +408,9 @@ def mapped_emissions(keys) "EMISSIONS(#{scheme.inspect}, #{value.inspect}, co2)" end - suffix = ('1990' if year.to_i == 1990) - emissions = scope.graph.emissions - - scope.sector_resolver.pairs(scheme, value).sum do |label, use| - key = [label, use, ghg, suffix].compact.join('_').to_sym - emissions[key] || 0.0 - end + scope.graph.emissions.sum_pairs( + scope.sector_resolver.pairs(scheme, value), ghg, year: year + ) end end end diff --git a/app/models/qernel/emissions.rb b/app/models/qernel/emissions.rb index 84696c7be..3a04c67f9 100644 --- a/app/models/qernel/emissions.rb +++ b/app/models/qernel/emissions.rb @@ -24,6 +24,14 @@ class Emissions dataset_accessors ::Etsource::Dataset.emissions_keys attr_accessor :graph + # Public: The flat store key for the given parts, e.g. (sector, use, ghg). + # The single place the `parts_joined_by_underscores[_1990]` schema lives. + # + # Returns a Symbol. + def self.key_for(*parts, year: nil) + [*parts, (1990 if year.to_i == 1990)].compact.join('_').to_sym + end + # Queryable object that provides scoped access to emissions data # # GQL uses this to scope the sector for easier queries and input/update @@ -40,11 +48,11 @@ def initialize(emissions, scope, year = nil) end def [](attr_name) - @emissions[scoped_method(attr_name).to_sym] + @emissions[scoped_method(attr_name)] end def []=(attr_name, value) - @emissions[scoped_method(attr_name).to_sym] = value + @emissions[scoped_method(attr_name)] = value end def inspect @@ -52,8 +60,7 @@ def inspect end def scoped_method(method_name) - attr = method_name.to_s.delete_suffix('=') - @year.to_i == 1990 ? "#{@scope}_#{attr}_1990" : "#{@scope}_#{attr}" + Emissions.key_for(@scope, method_name.to_s.delete_suffix('='), year: @year) end def respond_to_missing?(method_name, include_private = false) @@ -65,7 +72,7 @@ def respond_to_missing?(method_name, include_private = false) end def method_missing(method_name, *args) - key = scoped_method(method_name).to_sym + key = scoped_method(method_name) if method_name.to_s.end_with?('=') unless scope_exists? @@ -102,5 +109,17 @@ def initialize(graph = nil) def scope(sector, year = nil) ScopedSector.new(self, sector, year) end + + # Public: The stored value for (sector, use, ghg), or nil when the store + # has no such key. Pass year: 1990 to read the historic baseline. + def value_for(sector, use, ghg, year: nil) + self[self.class.key_for(sector, use, ghg, year: year)] + end + + # Public: Sums the store over (sector, use) pairs for one GHG. Pairs + # without a stored value count as zero. + def sum_pairs(pairs, ghg, year: nil) + pairs.sum { |sector, use| value_for(sector, use, ghg, year: year) || 0.0 } + end end end diff --git a/app/models/qernel/sectors.rb b/app/models/qernel/sectors.rb index 630895d5e..d56476afd 100644 --- a/app/models/qernel/sectors.rb +++ b/app/models/qernel/sectors.rb @@ -9,9 +9,9 @@ module Qernel # key index are read once from {Etsource::Sectors}; resolution turns node keys # into the live nodes of this graph. # - # Dispatch, arity and aggregation live in the GQL layer; this class answers - # two questions: which (label, use) pairs, and which live nodes, belong to - # (scheme, values)? + # Dispatch and arity live in the GQL layer; aggregation lives on + # {Qernel::Emissions}. This class answers two questions: which (label, use) + # pairs, and which live nodes, belong to (scheme, values)? class Sectors def initialize(graph, mapping, node_index) @graph = graph diff --git a/spec/models/gql/command_spec.rb b/spec/models/gql/command_spec.rb new file mode 100644 index 000000000..8b3cd4c19 --- /dev/null +++ b/spec/models/gql/command_spec.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'spec_helper' + +module Gql + describe Command do + describe 'cleaning the source' do + it 'strips whitespace outside string literals' do + expect(described_class.new("SUM(\n 1,\t2\n)").source).to eq('SUM(1,2)') + end + + it 'preserves whitespace inside single-quoted strings' do + expect(described_class.new("SECTOR(emissions_subsector, 'Fuels production')").source) + .to eq("SECTOR(emissions_subsector,'Fuels production')") + end + + it 'preserves whitespace inside double-quoted strings' do + expect(described_class.new('V(foo, "demand * conversion")').source) + .to eq('V(foo,"demand * conversion")') + end + + it 'removes a label prefix' do + expect(described_class.new('future:SUM(1, 2)').source).to eq('SUM(1,2)') + end + + it 'does not remove label-like text inside a string literal' do + expect(described_class.new("'future:keep me'").source).to eq("'future:keep me'") + end + + it 'cleans a nil source to an empty string' do + expect(described_class.new(nil).source).to eq('') + end + end + end +end diff --git a/spec/models/gql/runtime/functions/lookup_spec.rb b/spec/models/gql/runtime/functions/lookup_spec.rb index acf7c72b8..a9c20fb40 100644 --- a/spec/models/gql/runtime/functions/lookup_spec.rb +++ b/spec/models/gql/runtime/functions/lookup_spec.rb @@ -155,6 +155,12 @@ def keys_of(nodes) end end + describe "SECTOR(klimaattafel, 'Gebouwde omgeving')" do + it 'resolves a quoted value containing a space' do + expect(keys_of(result)).to eq(%i[buildings_space_heating_demand]) + end + end + describe "SECTOR(klimaattafel, 'Landbouw')" do it 'returns [] for a valid value with no labelled nodes' do expect(result).to eq([]) diff --git a/spec/models/qernel/emissions_spec.rb b/spec/models/qernel/emissions_spec.rb index 0e8751453..3d29bcc2f 100644 --- a/spec/models/qernel/emissions_spec.rb +++ b/spec/models/qernel/emissions_spec.rb @@ -40,6 +40,60 @@ module Qernel end end + describe '.key_for' do + it 'joins the parts with underscores' do + expect(described_class.key_for(:households, :energetic, :co2)) + .to eq(:households_energetic_co2) + end + + it 'suffixes 1990 for the historic baseline' do + expect(described_class.key_for(:households, :energetic, :co2, year: 1990)) + .to eq(:households_energetic_co2_1990) + end + + it 'ignores any other year (the present year is implicit)' do + expect(described_class.key_for(:households, :energetic, :co2, year: 2019)) + .to eq(:households_energetic_co2) + end + end + + describe '#value_for' do + before do + emissions[:households_non_specified_energetic_co2] = 40.0 + emissions[:households_non_specified_energetic_co2_1990] = 60.0 + end + + it 'reads the present-year value' do + expect(emissions.value_for(:households_non_specified, :energetic, :co2)).to eq(40.0) + end + + it 'reads the 1990 baseline when asked' do + expect(emissions.value_for(:households_non_specified, :energetic, :co2, year: 1990)) + .to eq(60.0) + end + + it 'returns nil for a missing key' do + expect(emissions.value_for(:industry_steel, :energetic, :co2)).to be_nil + end + end + + describe '#sum_pairs' do + before do + emissions[:households_non_specified_energetic_co2] = 40.0 + emissions[:agriculture_non_specified_energetic_co2] = 2.0 + end + + it 'sums the stored values over the pairs' do + pairs = [%i[households_non_specified energetic], %i[agriculture_non_specified energetic]] + expect(emissions.sum_pairs(pairs, :co2)).to eq(42.0) + end + + it 'counts pairs without a stored value as zero' do + pairs = [%i[households_non_specified energetic], %i[industry_steel energetic]] + expect(emissions.sum_pairs(pairs, :co2)).to eq(40.0) + end + end + describe '#scope' do it 'returns a ScopedSector instance' do scoped = emissions.scope(:households_non_specified_energetic) From c5d47dae7fdc94b69f720b0afc8b9894055b4603 Mon Sep 17 00:00:00 2001 From: louispt1 Date: Tue, 14 Jul 2026 12:08:02 +0200 Subject: [PATCH 5/7] Update serializer to match the sector mapping configuration --- Gemfile | 2 +- Gemfile.lock | 4 +- app/models/etsource/sectors.rb | 18 +- .../export/configured_csv_serializer.rb | 280 ++++++++++------ .../api/v3/exports_controller_spec.rb | 49 ++- .../etsource/config/direct_emissions_csv.yml | 13 +- .../etsource/config/sector_mapping.csv | 24 +- .../molecules/nodes/nosector/m_waste.ad | 2 +- .../export/configured_csv_serializer_spec.rb | 317 ++++++++++++++---- 9 files changed, 511 insertions(+), 198 deletions(-) diff --git a/Gemfile b/Gemfile index 423f50229..fd945c569 100644 --- a/Gemfile +++ b/Gemfile @@ -73,7 +73,7 @@ gem 'ruby-progressbar' # own gems gem 'quintel_merit', ref: 'aae77e0', github: 'quintel/merit' -gem 'atlas', ref: '78bc9d9', github: 'quintel/atlas' +gem 'atlas', ref: '727707b', github: 'quintel/atlas' gem 'fever', ref: '2afebd1', github: 'quintel/fever' gem 'refinery', ref: '36b8e34', github: 'quintel/refinery' gem 'rubel', ref: '9fe7010', github: 'quintel/rubel' diff --git a/Gemfile.lock b/Gemfile.lock index 97e0643e9..4f5f6763e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ GIT remote: https://github.com/quintel/atlas.git - revision: 78bc9d961c5fccbc30fc7a39a98fccedefc7e2ef - ref: 78bc9d9 + revision: 727707b515122d5d81906de2000d268ec8923bc5 + ref: 727707b specs: atlas (1.0.0) activemodel (>= 7) diff --git a/app/models/etsource/sectors.rb b/app/models/etsource/sectors.rb index 0967f002a..caa968eb3 100644 --- a/app/models/etsource/sectors.rb +++ b/app/models/etsource/sectors.rb @@ -1,14 +1,16 @@ # frozen_string_literal: true module Etsource - # Reads the sector mapping and node labels from Atlas and caches the two - # structures GQL needs to resolve `SECTOR(scheme, value)` style queries. + # Reads the sector mapping and node labels from Atlas and caches the + # structures GQL and the configured CSV exports need. # - # - # * mapping - {scheme => {value => [[label, use], ...]}} + # * mapping - {scheme => {value => [[label, use], ...]}}, for + # `SECTOR(scheme, value)` style GQL queries. # * node_index(g) - {[label, use] => [node_key, ...]} for one graph type, # with `use` baked in at import so it is never consulted # per query. + # * raw_rows - mapping rows in file order with raw display values, + # for mapping-driven CSV exports. class Sectors def initialize(etsource = Etsource::Base.instance) @etsource = etsource @@ -29,6 +31,14 @@ def node_index(graph_type) end end + # Public: Every mapping row in file order, as an Atlas::SectorMapping::RawRow + # (the (label, use) pair plus the raw display value of every scheme cell). + def raw_rows + NastyCache.instance.fetch('sector_raw_rows') do + Atlas::SectorMapping.load.raw_rows + end + end + private def build_node_index(node_class) diff --git a/app/serializers/export/configured_csv_serializer.rb b/app/serializers/export/configured_csv_serializer.rb index 2d3e0e4ef..31eb294fe 100644 --- a/app/serializers/export/configured_csv_serializer.rb +++ b/app/serializers/export/configured_csv_serializer.rb @@ -11,10 +11,11 @@ # # - schema: An array of hashes describing each column in the CSV. Each hash should contain a `name` # key and optionally a `type` key. -# - rows: An array of hashes, each matching the schema. Any keys contained in a hash which are not -# present in the schema are ignored. +# - rows: Either an array of hashes, each matching the schema (query-driven mode; any keys not +# present in the schema are ignored), or a hash `{ require: , order_by: +# }` (mapping-driven mode, see below). `order_by` is optional. # -# For example: +# Query-driven example: # # { # schema: [ @@ -36,131 +37,206 @@ # - "future": The value will be the result of evaluating the query in the future. # - "unit": The value will be the unit of the specified query. # - "query": This expands into three columns: `present`, `future` and `unit` for specified query. -# - "node_group": Hidden column. Its value names a node group; the row is expanded into one row per -# node in that group. Requires a `period:` to be set on the serializer. +# +# Mapping-driven mode: `rows: { require: }` selects every (sector label, use) pair +# in the sector mapping (see Atlas::SectorMapping) whose cell in `require`'s column has a value, in +# mapping-file order by default. An optional `order_by: ` instead orders pairs by the +# raw display value of that column (ascending string comparison), regardless of whether the column is +# included in `schema:`; a pair whose `order_by` cell is blank/`-` sorts last, and pairs tied on +# `order_by` keep their relative mapping-file order. Each pair expands to the energy and molecule +# nodes whose own `sector_label` and `use` match the pair AND which belong to the node's `:emissions` +# group (key-sorted). A pair with zero labelled nodes, or whose only matching nodes aren't in the +# `:emissions` group, legally yields zero rows. Requires a `period:` to be set on the serializer. Two +# extra column types apply: +# # - "node_attribute": The value will be the result of calling the attribute named by `value:` in the -# schema on node_api for each expanded node. Requires `node_group` column in schema. -# `value:` may be any Ruby expression evaluated on node_api via instance_eval. -# Supports an optional `transform:` Ruby expression evaluated with `value` bound -# to the result of `value:`. For example: +# schema on node_api for each expanded node. `value:` may be any Ruby expression +# evaluated on node_api via instance_eval. A nil result (e.g. a node not in the +# :emissions group) renders as an empty string without evaluating `transform`. +# Otherwise, an optional `transform:` Ruby expression is evaluated with `value` +# bound to the result of `value:`. For example: # transform: "value * 10e-6" # transform: "value ? :other_ghg : :co2" -class Export::ConfiguredCSVSerializer - # Represents the schema for a column in the CSV file. - class Column - attr_reader :name, :type, :label, :value, :transform - - def initialize(name, type, label: name, value: nil, transform: nil) - @name = name - @type = type || 'literal' - @label = label || name - @value = value - @transform = transform +# - "sector_mapping": The value will be the raw display value (never a normalized slug) of the +# mapping column named by `value:`, for the row's pair. A `-`/blank mapping cell +# renders as an empty string. +# +# An unknown mapping column named by `require:`, `order_by:`, or a `sector_mapping` column's `value:` +# raises Export::ConfiguredCSVSerializer::UnknownMappingColumnError at construction, naming the valid +# columns. +module Export + class ConfiguredCSVSerializer # rubocop:disable Style/Documentation + class UnknownMappingColumnError < StandardError end - end - # Creates a serializer using an ETSource config. - # - # period - optional :present or :future; when set, node_group rows are expanded using that graph - # and node_attribute columns are evaluated against it. Existing present/future/unit column - # types continue to work regardless of this setting. - def initialize(config, gql, period: nil) - @config = config.symbolize_keys - @config[:schema] = @config[:schema].map(&:symbolize_keys) - - @columns = @config[:schema].flat_map { |c| create_columns(c) } - @columns.delete(@node_group_column) if unpack_nodes? - @gql = gql - @period = period - end + # Represents the schema for a column in the CSV file. + class Column + attr_reader :name, :type, :label, :value, :transform - def data - rows = [@columns.map(&:label)] - @config[:rows].each { |row| serialize_row(row) { |csv_row| rows << csv_row } } - rows - end + def initialize(name, type, label: name, value: nil, transform: nil) + @name = name + @type = type || 'literal' + @label = label || name + @value = value + @transform = transform + end + end + + # Creates a serializer using an ETSource config. + # + # period - optional :present or :future; required when `rows:` is mapping-driven. + def initialize(config, gql, period: nil) + @config = config.symbolize_keys + @config[:schema] = @config[:schema].map(&:symbolize_keys) + @config[:rows] = @config[:rows].symbolize_keys if @config[:rows].is_a?(Hash) + + @columns = @config[:schema].flat_map { |c| create_columns(c) } + @gql = gql + @period = period - def as_csv - CSV.generate do |csv| - data.each { |row| csv << row } + validate_mapping_references! if mapping_driven? end - end - private + def data + rows = [@columns.map(&:label)] - def serialize_row(row) - if unpack_nodes? - group_name = row[@node_group_column.name] - nodes = graph_interface.group_energy_nodes(group_name) + - graph_interface.group_molecule_nodes(group_name) - nodes.each do |node| - yield @columns.map { |column| serialize_node_column(column, row, node) } + if mapping_driven? + serialize_mapping_rows { |row| rows << row } + else + @config[:rows].each { |row| rows << @columns.map { |column| serialize_column(column, row) } } end - else - yield @columns.map { |column| serialize_column(column, row) } + + rows end - end - def serialize_column(column, row) - value = row[column.name] + def as_csv + CSV.generate do |csv| + data.each { |row| csv << row } + end + end - return '' if value.blank? + private - case column.type - when 'future' then @gql.future.subquery(value).to_s - when 'present' then @gql.present.subquery(value).to_s - when 'unit' then Gquery.get(value).unit.to_s - else value + def mapping_driven? + @config[:rows].is_a?(Hash) end - end - def serialize_node_column(column, row, node) - case column.type - when 'node_attribute' - value = node.node_api.instance_eval(column.value) - value = eval(column.transform) if column.transform - value.to_s - else serialize_column(column, row) + def membership_scheme + @config[:rows][:require].to_sym end - end - def unpack_nodes? - node_group_column.present? - end + def order_scheme + @config[:rows][:order_by]&.to_sym + end - def node_group_column - @node_group_column ||= @columns.find { |c| c.type == 'node_group' } - end + def serialize_mapping_rows + eligible_rows.each do |raw_row| + nodes_for_pair(raw_row.pair).each do |node| + yield @columns.map { |column| serialize_mapping_column(column, raw_row, node) } + end + end + end - def graph_interface - @gql.public_send(@period) - end + # Rows whose `require:` cell has a value, in mapping-file order. When `order_by:` is set, sorted + # by that column's raw value instead (blank/`-` last), with ties broken by mapping-file order. + def eligible_rows + rows = sectors.raw_rows.select { |raw_row| raw_row.cells[membership_scheme] } + return rows unless order_scheme - def create_columns(column) - if column[:type] != 'query' - return Column.new( - column[:name], - column[:type], - label: column[:label], - value: column[:value], - transform: column[:transform] - ) + rows.each_with_index.sort_by { |raw_row, index| [raw_row.cells[order_scheme].nil? ? 1 : 0, raw_row.cells[order_scheme].to_s, index] } + .map(&:first) end - %w[present future unit].map do |subtype| - Column.new( - column[:name], - subtype, - label: column[:"#{subtype}_label"] || default_label_for(subtype, column[:name]) - ) + def serialize_mapping_column(column, raw_row, node) + case column.type + when 'node_attribute' + value = node.node_api.instance_eval(column.value) + return '' if value.nil? + + value = eval(column.transform) if column.transform + value.to_s + when 'sector_mapping' + raw_row.cells[column.value.to_sym].to_s + else + '' + end end - end - def default_label_for(subtype, column_name) - if subtype == 'unit' - "#{column_name} Unit" - else - "#{subtype.capitalize} #{column_name}" + def nodes_for_pair(pair) + nodes = %i[energy molecules].flat_map do |graph_type| + sectors.node_index(graph_type).fetch(pair, []).filter_map do |key| + node_for(graph_type, key) + end + end + nodes.select(&:emissions?).sort_by { |node| node.node_api.key.to_s } + end + + def node_for(graph_type, key) + graph_type == :molecules ? graph_interface.molecules.node(key) : graph_interface.graph.node(key) + end + + def graph_interface + @gql.public_send(@period) + end + + def sectors + @sectors ||= Etsource::Sectors.new + end + + def validate_mapping_references! + valid = sectors.mapping.keys + references = [@config[:rows][:require], @config[:rows][:order_by]] + @columns.select { |c| + c.type == 'sector_mapping' + }.map(&:value) + + references.compact.each do |reference| + next if valid.include?(reference.to_sym) + + raise UnknownMappingColumnError, + "Unknown sector mapping column #{reference.inspect}. " \ + "Valid columns: #{valid.map(&:inspect).join(', ')}." + end + end + + def serialize_column(column, row) + value = row[column.name] + + return '' if value.blank? + + case column.type + when 'future' then @gql.future.subquery(value).to_s + when 'present' then @gql.present.subquery(value).to_s + when 'unit' then Gquery.get(value).unit.to_s + else value + end + end + + def create_columns(column) + if column[:type] != 'query' + return Column.new( + column[:name], + column[:type], + label: column[:label], + value: column[:value], + transform: column[:transform] + ) + end + + %w[present future unit].map do |subtype| + Column.new( + column[:name], + subtype, + label: column[:"#{subtype}_label"] || default_label_for(subtype, column[:name]) + ) + end + end + + def default_label_for(subtype, column_name) + if subtype == 'unit' + "#{column_name} Unit" + else + "#{subtype.capitalize} #{column_name}" + end end end end diff --git a/spec/controllers/api/v3/exports_controller_spec.rb b/spec/controllers/api/v3/exports_controller_spec.rb index 374da0fa6..c88612a6d 100644 --- a/spec/controllers/api/v3/exports_controller_spec.rb +++ b/spec/controllers/api/v3/exports_controller_spec.rb @@ -78,12 +78,7 @@ end end - describe 'GET direct_emissions_present.csv' do - before do - request.headers.merge!(headers) - get :direct_emissions_present, params: { id: scenario.id }, format: :csv - end - + shared_examples 'a mapping-driven direct emissions export' do it 'is successful' do expect(response).to be_ok end @@ -92,6 +87,40 @@ expect(response.media_type).to eq('text/csv') end + it 'includes the eight legacy columns, byte-identical to the pre-rework export' do + expect(CSV.parse(response.body).first).to eq( + [ + 'Sector', 'Subsector', 'Key', 'GHG', + 'CO2 production [kton CO2-eq]', 'CO2 capture [kton CO2-eq]', + 'Other GHG emissions [kton CO2-eq]', 'Total GHG emissions [kton CO2-eq]' + ] + ) + end + + it 'renders Sector/Subsector as mapping lookups for a labelled, :emissions-group node' do + row = CSV.parse(response.body).find { |r| r[2] == 'm_waste' } + expect(row).to eq(%w[Waste Non-specified m_waste co2 0.0 0.0 0.0 0.0]) + end + + it 'excludes a labelled node whose pair has no value in the require column' do + expect(CSV.parse(response.body).flatten).not_to include('foo') + end + + it 'excludes a labelled node that is not in the :emissions group' do + # `bar`/`baz`/`lft`/`buildings_space_heating_demand` all carry a sector_label/use that + # matches an exported mapping row, but none is in the :emissions node group. + expect(CSV.parse(response.body).flatten).not_to include('bar', 'baz', 'lft', 'buildings_space_heating_demand') + end + end + + describe 'GET direct_emissions_present.csv' do + before do + request.headers.merge!(headers) + get :direct_emissions_present, params: { id: scenario.id }, format: :csv + end + + include_examples 'a mapping-driven direct emissions export' + it 'sets the CSV filename' do expect(response.headers['Content-Disposition']).to include("direct_emissions_present.#{scenario.id}.csv") end @@ -103,13 +132,7 @@ get :direct_emissions_future, params: { id: scenario.id }, format: :csv end - it 'is successful' do - expect(response).to be_ok - end - - it 'sets the content type to text/csv' do - expect(response.media_type).to eq('text/csv') - end + include_examples 'a mapping-driven direct emissions export' it 'sets the CSV filename' do expect(response.headers['Content-Disposition']).to include("direct_emissions_future.#{scenario.id}.csv") diff --git a/spec/fixtures/etsource/config/direct_emissions_csv.yml b/spec/fixtures/etsource/config/direct_emissions_csv.yml index 0a15463be..9abee5d06 100644 --- a/spec/fixtures/etsource/config/direct_emissions_csv.yml +++ b/spec/fixtures/etsource/config/direct_emissions_csv.yml @@ -1,8 +1,10 @@ schema: - - name: Group - type: node_group - name: Sector + type: sector_mapping + value: emissions_sector - name: Subsector + type: sector_mapping + value: emissions_subsector - name: Key type: node_attribute value: key @@ -25,12 +27,11 @@ schema: value: direct_reporting_emissions_other_ghg_emissions transform: "(value * 1e-6).round(6)" - name: Total_GHG_emissions - label: "Total GHG emissions[kton CO2-eq]" + label: "Total GHG emissions [kton CO2-eq]" type: node_attribute value: direct_reporting_emissions_total_ghg_emissions transform: "(value * 1e-6).round(6)" rows: - - Group: emissions_industry_chemicals - Sector: Industry - Subsector: Chemicals + require: emissions_sector + order_by: ipcc_crt_code diff --git a/spec/fixtures/etsource/config/sector_mapping.csv b/spec/fixtures/etsource/config/sector_mapping.csv index eb2e1e87e..59db0709d 100644 --- a/spec/fixtures/etsource/config/sector_mapping.csv +++ b/spec/fixtures/etsource/config/sector_mapping.csv @@ -1,12 +1,12 @@ -sector_label,use,ipcc_crt_code_agg,klimaattafel -energy_electricity_and_heat_production,energetic,1.A.1,Elektriciteit -industry_refineries,energetic,1.A.1,Industrie -industry_refineries,non_energetic,1.B,Industrie -industry_non_specified,energetic,1.A.2,Industrie -buildings_non_specified,energetic,1.A.4.a,Gebouwde omgeving -households_non_specified,energetic,1.A.4.b,Gebouwde omgeving -agriculture_non_specified,energetic,1.A.4.c,Landbouw -agriculture_non_specified,non_energetic,3,Landbouw -waste_non_specified,non_energetic,5,Elektriciteit -energy_fugitive_emissions,non_energetic,1.B,Elektriciteit -other_heating,energetic,-,Mobiliteit +sector_label,use,emissions_sector,emissions_subsector,ipcc_crt_code_agg,ipcc_crt_code,klimaattafel +energy_electricity_and_heat_production,energetic,Energy,Electricity and heat production,1.A.1,1.A.1.a,Elektriciteit +industry_refineries,energetic,Industry,Refineries,1.A.1,1.A.1.b,Industrie +industry_refineries,non_energetic,Industry,Refineries,1.B,1.B.2.a.iv,Industrie +industry_non_specified,energetic,-,-,1.A.2,1.A.2,Industrie +buildings_non_specified,energetic,Buildings,Non-specified,1.A.4.a,1.A.4.a,Gebouwde omgeving +households_non_specified,energetic,Households,Non-specified,1.A.4.b,1.A.4.b,Gebouwde omgeving +agriculture_non_specified,energetic,Agriculture,Non-specified,1.A.4.c,1.A.4.c,Landbouw +agriculture_non_specified,non_energetic,Agriculture,Non-specified,3,3,Landbouw +waste_non_specified,non_energetic,Waste,Non-specified,5,5,Elektriciteit +energy_fugitive_emissions,non_energetic,Energy,Fugitive emissions,1.B,1.B excl. 1.B.2.a.iv,Elektriciteit +other_heating,energetic,Other,Heating,-,-,Mobiliteit diff --git a/spec/fixtures/etsource/graphs/molecules/nodes/nosector/m_waste.ad b/spec/fixtures/etsource/graphs/molecules/nodes/nosector/m_waste.ad index 8ce379f39..1bf13b204 100644 --- a/spec/fixtures/etsource/graphs/molecules/nodes/nosector/m_waste.ad +++ b/spec/fixtures/etsource/graphs/molecules/nodes/nosector/m_waste.ad @@ -1,3 +1,3 @@ -- groups = [] +- groups = [emissions] - sector_label = waste_non_specified - use = non_energetic diff --git a/spec/serializers/export/configured_csv_serializer_spec.rb b/spec/serializers/export/configured_csv_serializer_spec.rb index 99b0056c0..b5d36e9d5 100644 --- a/spec/serializers/export/configured_csv_serializer_spec.rb +++ b/spec/serializers/export/configured_csv_serializer_spec.rb @@ -122,108 +122,311 @@ end end - context 'when given a "node_group" column' do - let(:node_api_1) { instance_double('Qernel::NodeApi::EnergyApi', key: 'node_a') } - let(:node_api_2) { instance_double('Qernel::NodeApi::EnergyApi', key: 'node_b') } - let(:node_1) { instance_double('Qernel::Node', node_api: node_api_1) } - let(:node_2) { instance_double('Qernel::Node', node_api: node_api_2) } + context 'when given mapping-driven rows' do + # Real pairs from spec/fixtures/etsource/config/sector_mapping.csv, joined against the real + # labelled fixture nodes (spec/fixtures/etsource/graphs). In mapping-file order: + # + # energy_electricity_and_heat_production/energetic -> Energy, emissions_sector set, 0 nodes + # industry_refineries/energetic -> Industry, node `bar` (dual-use label) + # industry_refineries/non_energetic -> Industry, node `baz` (dual-use label) + # industry_non_specified/energetic -> emissions_sector blank -> excluded + # (node `foo` exists but must not appear) + # buildings_non_specified/energetic -> Buildings, node `buildings_space_heating_demand` + # households_non_specified/energetic -> 0 nodes + # agriculture_non_specified/energetic -> 0 nodes + # agriculture_non_specified/non_energetic -> 0 nodes + # waste_non_specified/non_energetic -> Waste, node `m_waste` (molecule graph) + # energy_fugitive_emissions/non_energetic -> 0 nodes + # other_heating/energetic -> Other, node `lft`, blank ipcc cell let(:config) do { schema: [ - { name: 'Group', type: 'node_group' }, - { name: 'Sector' }, - { name: 'Node', type: 'node_attribute', value: 'key' } + { name: 'Sector', type: 'sector_mapping', value: 'emissions_sector' }, + { name: 'Key', type: 'node_attribute', value: 'key' }, + { name: 'IPCC', type: 'sector_mapping', value: 'ipcc_crt_code_agg' } ], - rows: [ - { 'Group' => 'some_group', 'Sector' => 'Industry' } - ] + rows: { require: 'emissions_sector' } } end - let(:node_api_3) { instance_double('Qernel::NodeApi::MoleculeApi', key: 'node_c') } - let(:node_3) { instance_double('Qernel::Node', node_api: node_api_3) } - - let(:gql) { Scenario.default.gql } + let(:gql) { Scenario.default.gql } let(:serializer) { described_class.new(config, gql, period: :future) } - before do - allow(gql.future).to receive(:group_energy_nodes).with('some_group').and_return([node_1, node_2]) - allow(gql.future).to receive(:group_molecule_nodes).with('some_group').and_return([node_3]) + let(:node_bar) do + instance_double( + 'Qernel::Node', + emissions?: true, + node_api: instance_double( + 'Qernel::NodeApi::EnergyApi', key: :bar, direct_reporting_emissions_co2_production: nil + ) + ) end - - it 'excludes the node_group column from the header' do - expect(serializer.data[0]).to eq(%w[Sector Node]) + let(:node_baz) do + instance_double('Qernel::Node', emissions?: true, node_api: instance_double('Qernel::NodeApi::EnergyApi', key: :baz)) + end + let(:node_foo) do + instance_double('Qernel::Node', emissions?: true, node_api: instance_double('Qernel::NodeApi::EnergyApi', key: :foo)) + end + let(:node_buildings) do + instance_double( + 'Qernel::Node', emissions?: true, + node_api: instance_double('Qernel::NodeApi::EnergyApi', key: :buildings_space_heating_demand) + ) + end + let(:node_lft) do + instance_double('Qernel::Node', emissions?: true, node_api: instance_double('Qernel::NodeApi::EnergyApi', key: :lft)) + end + let(:node_waste) do + instance_double('Qernel::Node', emissions?: true, node_api: instance_double('Qernel::NodeApi::MoleculeApi', key: :m_waste)) end - it 'expands the row into one row per node from both graphs' do - expect(serializer.data.length).to eq(4) # header + 2 energy + 1 molecule + let(:energy_nodes) do + { bar: node_bar, baz: node_baz, foo: node_foo, buildings_space_heating_demand: node_buildings, lft: node_lft } end - it 'includes the literal column value on each expanded row' do - expect(serializer.data[1][0]).to eq('Industry') - expect(serializer.data[2][0]).to eq('Industry') - expect(serializer.data[3][0]).to eq('Industry') + before do + allow(gql.future.graph).to receive(:node) { |key| energy_nodes[key.to_sym] } + allow(gql.future).to receive(:molecules) + .and_return(instance_double('Qernel::Graph').tap { |g| allow(g).to receive(:node).with(:m_waste).and_return(node_waste) }) end - it 'includes energy node_attribute values' do - expect(serializer.data[1][1]).to eq('node_a') - expect(serializer.data[2][1]).to eq('node_b') + it 'includes the CSV headers' do + expect(serializer.data[0]).to eq(%w[Sector Key IPCC]) end - it 'includes molecule node_attribute values' do - expect(serializer.data[3][1]).to eq('node_c') + it 'emits rows in mapping-file order, skipping pairs with zero labelled nodes' do + expect(serializer.data[1..].map { |row| row[1] }).to eq( + %w[bar baz buildings_space_heating_demand m_waste lft] + ) end - context 'with a transform for numeric conversion' do - let(:node_api_1) { instance_double('Qernel::NodeApi::EnergyApi', key: 'node_a', demand: 1_000_000.0) } + it 'excludes a pair whose require column cell is blank, even though it has a labelled node' do + expect(serializer.data.flatten).not_to include('foo') + end + context 'with an "order_by" rule' do let(:config) do { - schema: [ - { name: 'Group', type: 'node_group' }, - { name: 'Demand', type: 'node_attribute', value: 'demand', transform: 'value * 1e-6' } - ], - rows: [{ 'Group' => 'some_group' }] + schema: [{ name: 'Key', type: 'node_attribute', value: 'key' }], + rows: { require: 'emissions_sector', order_by: 'ipcc_crt_code' } } end + it 'orders rows by the raw value of the order_by column, not the mapping-file order' do + # ipcc_crt_code: bar 1.A.1.b, buildings 1.A.4.a, baz 1.B.2.a.iv, m_waste 5, lft blank. + expect(serializer.data[1..].flatten).to eq( + %w[bar buildings_space_heating_demand baz m_waste lft] + ) + end + + it 'sorts a pair with a blank order_by cell last' do + expect(serializer.data.last).to eq(['lft']) + end + end + + context 'when a labelled node is not in the :emissions group' do + before { allow(node_baz).to receive(:emissions?).and_return(false) } + + it 'excludes it, even though its pair matches an exported row' do + expect(serializer.data.flatten).not_to include('baz') + end + + it 'still includes the other node under the same dual-use label' do + expect(serializer.data.flatten).to include('bar') + end + end + + it 'renders the raw display value of the require column (not a slug)' do + expect(serializer.data[1][0]).to eq('Industry') + expect(serializer.data[4][0]).to eq('Waste') + end + + it 'joins a dual-use label by the node\'s own (label, use) pair, not by name alone' do + # `bar` and `baz` share sector_label industry_refineries but differ in `use`, and land under + # different IPCC codes as a result. + expect(serializer.data[1]).to eq(%w[Industry bar 1.A.1]) + expect(serializer.data[2]).to eq(%w[Industry baz 1.B]) + end + + it 'includes molecule-graph nodes alongside energy nodes' do + expect(serializer.data[4]).to eq(%w[Waste m_waste 5]) + end + + it 'renders a raw value with punctuation unchanged' do + expect(serializer.data[1][2]).to eq('1.A.1') + end + + it 'renders a blank mapping cell as an empty string' do + expect(serializer.data[5]).to eq(['Other', 'lft', '']) + end + + context 'with an isolated pair (stubbed Etsource::Sectors)' do + let(:sectors) { instance_double(Etsource::Sectors) } + let(:pair) { %i[industry_refineries energetic] } + let(:raw_row) { Atlas::SectorMapping::RawRow.new(pair, { emissions_sector: 'Industry' }) } + before do - allow(gql.future).to receive(:group_energy_nodes).with('some_group').and_return([node_1]) - allow(gql.future).to receive(:group_molecule_nodes).with('some_group').and_return([]) + allow(Etsource::Sectors).to receive(:new).and_return(sectors) + allow(sectors).to receive(:mapping).and_return({ emissions_sector: {} }) + allow(sectors).to receive(:raw_rows).and_return([raw_row]) + allow(sectors).to receive(:node_index).with(:molecules).and_return({}) + end + + context 'sorting nodes within a pair' do + let(:config) do + { + schema: [{ name: 'Key', type: 'node_attribute', value: 'key' }], + rows: { require: 'emissions_sector' } + } + end + + let(:node_z) do + instance_double('Qernel::Node', emissions?: true, node_api: instance_double('Qernel::NodeApi::EnergyApi', key: :z_node)) + end + let(:node_a) do + instance_double('Qernel::Node', emissions?: true, node_api: instance_double('Qernel::NodeApi::EnergyApi', key: :a_node)) + end + + before do + allow(sectors).to receive(:node_index).with(:energy).and_return({ pair => %i[z_node a_node] }) + allow(gql.future.graph).to receive(:node).with(:z_node).and_return(node_z) + allow(gql.future.graph).to receive(:node).with(:a_node).and_return(node_a) + end + + it 'sorts the expanded nodes by key, regardless of node_index order' do + expect(serializer.data[1..].map { |row| row[0] }).to eq(%w[a_node z_node]) + end end - it 'applies the transform to the value' do - expect(serializer.data[1][0]).to eq('1.0') + context 'with an "order_by" rule and rows tied on that column' do + let(:pair_a) { %i[pair_a energetic] } + let(:pair_b) { %i[pair_b energetic] } + let(:pair_c) { %i[pair_c energetic] } + + let(:raw_row_a) { Atlas::SectorMapping::RawRow.new(pair_a, { emissions_sector: 'A', ipcc_crt_code: '1' }) } + let(:raw_row_b) { Atlas::SectorMapping::RawRow.new(pair_b, { emissions_sector: 'B', ipcc_crt_code: '1' }) } + let(:raw_row_c) { Atlas::SectorMapping::RawRow.new(pair_c, { emissions_sector: 'C', ipcc_crt_code: '0' }) } + + let(:config) do + { + schema: [{ name: 'Key', type: 'node_attribute', value: 'key' }], + rows: { require: 'emissions_sector', order_by: 'ipcc_crt_code' } + } + end + + let(:node_a) do + instance_double('Qernel::Node', emissions?: true, node_api: instance_double('Qernel::NodeApi::EnergyApi', key: :node_a)) + end + let(:node_b) do + instance_double('Qernel::Node', emissions?: true, node_api: instance_double('Qernel::NodeApi::EnergyApi', key: :node_b)) + end + let(:node_c) do + instance_double('Qernel::Node', emissions?: true, node_api: instance_double('Qernel::NodeApi::EnergyApi', key: :node_c)) + end + + before do + allow(sectors).to receive(:mapping).and_return({ emissions_sector: {}, ipcc_crt_code: {} }) + # File order: A, B, C. A and B tie on ipcc_crt_code ("1"); C has a lower code ("0"). + allow(sectors).to receive(:raw_rows).and_return([raw_row_a, raw_row_b, raw_row_c]) + allow(sectors).to receive(:node_index).with(:energy).and_return( + pair_a => [:node_a], pair_b => [:node_b], pair_c => [:node_c] + ) + allow(gql.future.graph).to receive(:node).with(:node_a).and_return(node_a) + allow(gql.future.graph).to receive(:node).with(:node_b).and_return(node_b) + allow(gql.future.graph).to receive(:node).with(:node_c).and_return(node_c) + end + + it 'sorts by the order_by value first, breaking ties by mapping-file order' do + expect(serializer.data[1..].flatten).to eq(%w[node_c node_a node_b]) + end + end + + context 'when a node_attribute value is nil (node not in the :emissions group)' do + let(:config) do + { + schema: [ + { name: 'Key', type: 'node_attribute', value: 'key' }, + { name: 'CO2', type: 'node_attribute', value: 'direct_reporting_emissions_co2_production', + transform: 'value * 1e-6' } + ], + rows: { require: 'emissions_sector' } + } + end + + let(:node_bar) do + instance_double( + 'Qernel::Node', + emissions?: true, + node_api: instance_double( + 'Qernel::NodeApi::EnergyApi', key: :bar, direct_reporting_emissions_co2_production: nil + ) + ) + end + + before do + allow(sectors).to receive(:node_index).with(:energy).and_return({ pair => [:bar] }) + allow(gql.future.graph).to receive(:node).with(:bar).and_return(node_bar) + end + + it 'renders an empty string without evaluating the transform' do + expect(serializer.data[1]).to eq(['bar', '']) + end end end + end - context 'with a transform for conditional mapping' do - let(:node_api_1) { double('node_api', key: 'node_a', is_special: true) } - let(:node_api_2) { double('node_api', key: 'node_b', is_special: false) } + context 'when given an unknown sector mapping column' do + let(:gql) { Scenario.default.gql } + let(:serializer) { described_class.new(config, gql, period: :future) } + + context 'as a "rows: require:" reference' do + let(:config) do + { + schema: [{ name: 'Key', type: 'node_attribute', value: 'key' }], + rows: { require: 'impossible' } + } + end + + it 'raises at construction, naming the invalid reference and the valid columns' do + expect { serializer }.to raise_error( + Export::ConfiguredCSVSerializer::UnknownMappingColumnError, + /impossible.*Valid columns.*emissions_sector/m + ) + end + end + context 'as a "sector_mapping" column value' do let(:config) do { schema: [ - { name: 'Group', type: 'node_group' }, - { name: 'Type', type: 'node_attribute', value: 'is_special', - transform: "value ? 'other_ghg' : 'co2'" } + { name: 'Sector', type: 'sector_mapping', value: 'impossible' } ], - rows: [{ 'Group' => 'some_group' }] + rows: { require: 'emissions_sector' } } end - before do - allow(gql.future).to receive(:group_energy_nodes).with('some_group').and_return([node_1, node_2]) - allow(gql.future).to receive(:group_molecule_nodes).with('some_group').and_return([]) + it 'raises at construction, naming the invalid reference and the valid columns' do + expect { serializer }.to raise_error( + Export::ConfiguredCSVSerializer::UnknownMappingColumnError, + /impossible.*Valid columns.*emissions_sector/m + ) end + end - it 'maps a truthy result via transform' do - expect(serializer.data[1][0]).to eq('other_ghg') + context 'as a "rows: order_by:" reference' do + let(:config) do + { + schema: [{ name: 'Key', type: 'node_attribute', value: 'key' }], + rows: { require: 'emissions_sector', order_by: 'impossible' } + } end - it 'maps a falsy result via transform' do - expect(serializer.data[2][0]).to eq('co2') + it 'raises at construction, naming the invalid reference and the valid columns' do + expect { serializer }.to raise_error( + Export::ConfiguredCSVSerializer::UnknownMappingColumnError, + /impossible.*Valid columns.*emissions_sector/m + ) end end end From fbf7788fdfd4e9c85c15cd1421de7e850a0b69e8 Mon Sep 17 00:00:00 2001 From: louispt1 Date: Wed, 15 Jul 2026 09:07:44 +0200 Subject: [PATCH 6/7] Refine sectors and serializer implementation based on new mapping approach. --- Gemfile | 2 +- Gemfile.lock | 4 +- app/models/etsource/sectors.rb | 4 - app/models/gql/runtime/functions/lookup.rb | 13 ++- app/models/qernel/sectors.rb | 17 +-- .../export/configured_csv_serializer.rb | 102 +++++++++++------- .../gql/runtime/functions/lookup_spec.rb | 6 ++ .../export/configured_csv_serializer_spec.rb | 58 +++++++++- 8 files changed, 150 insertions(+), 56 deletions(-) diff --git a/Gemfile b/Gemfile index fd945c569..4c12d8b39 100644 --- a/Gemfile +++ b/Gemfile @@ -73,7 +73,7 @@ gem 'ruby-progressbar' # own gems gem 'quintel_merit', ref: 'aae77e0', github: 'quintel/merit' -gem 'atlas', ref: '727707b', github: 'quintel/atlas' +gem 'atlas', ref: '5d2e5b2', github: 'quintel/atlas' gem 'fever', ref: '2afebd1', github: 'quintel/fever' gem 'refinery', ref: '36b8e34', github: 'quintel/refinery' gem 'rubel', ref: '9fe7010', github: 'quintel/rubel' diff --git a/Gemfile.lock b/Gemfile.lock index 4f5f6763e..8227f48d4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ GIT remote: https://github.com/quintel/atlas.git - revision: 727707b515122d5d81906de2000d268ec8923bc5 - ref: 727707b + revision: 5d2e5b20e3d397b764f688180e3987bd43161612 + ref: 5d2e5b2 specs: atlas (1.0.0) activemodel (>= 7) diff --git a/app/models/etsource/sectors.rb b/app/models/etsource/sectors.rb index caa968eb3..72b590914 100644 --- a/app/models/etsource/sectors.rb +++ b/app/models/etsource/sectors.rb @@ -12,10 +12,6 @@ module Etsource # * raw_rows - mapping rows in file order with raw display values, # for mapping-driven CSV exports. class Sectors - def initialize(etsource = Etsource::Base.instance) - @etsource = etsource - end - # Public: The inverted mapping index, keyed by scheme then normalized value. def mapping NastyCache.instance.fetch('sector_mapping_hash') do diff --git a/app/models/gql/runtime/functions/lookup.rb b/app/models/gql/runtime/functions/lookup.rb index 36d331357..d878350df 100644 --- a/app/models/gql/runtime/functions/lookup.rb +++ b/app/models/gql/runtime/functions/lookup.rb @@ -91,13 +91,13 @@ def MALL def GROUP(*keys) scope.group_energy_nodes(keys) end - alias G GROUP + alias_method :G, :GROUP # Returns an Array of {Qernel::Node} for given molecule group. See GROUP. def MGROUP(*keys) scope.group_molecule_nodes(keys) end - alias MG MGROUP + alias_method :MG, :MGROUP # Returns an Array of {Qernel::Edges} for given energy group. # @@ -108,13 +108,13 @@ def MGROUP(*keys) def EDGE_GROUP(*keys) scope.group_energy_edges(keys) end - alias EG EDGE_GROUP + alias_method :EG, :EDGE_GROUP # Returns an Array of {Qernel::Edges} for given molecule group. See EDGE_GROUP. def MEDGE_GROUP(*keys) scope.group_molecule_edges(keys) end - alias MEG EDGE_GROUP + alias_method :MEG, :MEDGE_GROUP # Returns an Array of {Qernel::Node} for an energy sector. # @@ -401,6 +401,11 @@ def EMISSIONS(*keys) # Internal: The mapped EMISSIONS form. Resolves `scheme`/`value` to # (sector label, use) pairs and lets the emissions store sum over them. def mapped_emissions(keys) + if keys.size > 4 + raise Gql::GqlError, "EMISSIONS(#{keys.first.inspect}, ...) accepts a single value: " \ + 'EMISSIONS(scheme, value, ghg[, 1990]).' + end + scheme, value, ghg, year = keys if ghg.nil? diff --git a/app/models/qernel/sectors.rb b/app/models/qernel/sectors.rb index d56476afd..465171747 100644 --- a/app/models/qernel/sectors.rb +++ b/app/models/qernel/sectors.rb @@ -21,7 +21,7 @@ def initialize(graph, mapping, node_index) @pair_cache = {} end - # Public: Whether `scheme`` names a classification scheme in the mapping. + # Public: Whether `scheme` names a classification scheme in the mapping. # Used by the EMISSIONS first-argument dispatch. def scheme?(scheme) @mapping.key?(normalize(scheme)) @@ -40,14 +40,21 @@ def pairs(scheme, values) end # Public: The live nodes of this graph whose (label, use) pair falls under - # any of `values` of 'scheme` (the union). A value which resolves to pairs + # any of `values` of `scheme` (the union). A value which resolves to pairs # with no labelled nodes contributes nothing. def lookup(scheme, values) value_list = Array(values) key = [normalize(scheme), value_list.map { |value| normalize(value) }] @node_cache[key] ||= - pairs(scheme, value_list).flat_map { |pair| nodes_for(pair) }.uniq + pairs(scheme, value_list).flat_map { |pair| nodes_for_pair(pair) }.uniq + end + + # Public: The live nodes of this graph whose (label, use) pair equals + # `pair` (an already-normalized [label, use]). Used by mapping-driven CSV + # exports as well as {#lookup}. + def nodes_for_pair(pair) + Array(@node_index[pair]).filter_map { |key| @graph.node(key) } end private @@ -62,10 +69,6 @@ def resolve_pairs(scheme, value_list) end.uniq end - def nodes_for(pair) - Array(@node_index[pair]).filter_map { |key| @graph.node(key) } - end - def normalize(value) Atlas::SectorMapping.normalize(value) end diff --git a/app/serializers/export/configured_csv_serializer.rb b/app/serializers/export/configured_csv_serializer.rb index 31eb294fe..7ade8414e 100644 --- a/app/serializers/export/configured_csv_serializer.rb +++ b/app/serializers/export/configured_csv_serializer.rb @@ -46,8 +46,9 @@ # `order_by` keep their relative mapping-file order. Each pair expands to the energy and molecule # nodes whose own `sector_label` and `use` match the pair AND which belong to the node's `:emissions` # group (key-sorted). A pair with zero labelled nodes, or whose only matching nodes aren't in the -# `:emissions` group, legally yields zero rows. Requires a `period:` to be set on the serializer. Two -# extra column types apply: +# `:emissions` group, legally yields zero rows. Mapping-driven mode requires a `period:` (raises +# ArgumentError at construction otherwise) and permits only the two column types below (any other +# type raises UnsupportedColumnTypeError at construction): # # - "node_attribute": The value will be the result of calling the attribute named by `value:` in the # schema on node_api for each expanded node. `value:` may be any Ruby expression @@ -57,9 +58,8 @@ # bound to the result of `value:`. For example: # transform: "value * 10e-6" # transform: "value ? :other_ghg : :co2" -# - "sector_mapping": The value will be the raw display value (never a normalized slug) of the -# mapping column named by `value:`, for the row's pair. A `-`/blank mapping cell -# renders as an empty string. +# - "sector_mapping": The value will be the raw display value of the mapping column named by `value:`, +# for the row's pair. A `-`/blank mapping cell renders as an empty string. # # An unknown mapping column named by `require:`, `order_by:`, or a `sector_mapping` column's `value:` # raises Export::ConfiguredCSVSerializer::UnknownMappingColumnError at construction, naming the valid @@ -69,6 +69,12 @@ class ConfiguredCSVSerializer # rubocop:disable Style/Documentation class UnknownMappingColumnError < StandardError end + class UnsupportedColumnTypeError < StandardError + end + + # Column types which may appear in a mapping-driven schema. + MAPPING_COLUMN_TYPES = %w[node_attribute sector_mapping].freeze + # Represents the schema for a column in the CSV file. class Column attr_reader :name, :type, :label, :value, :transform @@ -78,7 +84,19 @@ def initialize(name, type, label: name, value: nil, transform: nil) @type = type || 'literal' @label = label || name @value = value - @transform = transform + @transform = transform && compile_transform(transform) + end + + private + + # Internal: Compiles the `transform:` expression to a lambda once, rather + # than eval-ing the string for every exported cell. `value` is the + # column's evaluated node attribute. + def compile_transform(expression) + eval( + "->(value) { #{expression} }", # ->(value) { (value * 1e-6).round(6) } + binding, __FILE__, __LINE__ - 1 + ) end end @@ -94,7 +112,11 @@ def initialize(config, gql, period: nil) @gql = gql @period = period - validate_mapping_references! if mapping_driven? + return unless mapping_driven? + + @membership_scheme = @config[:rows][:require]&.to_sym + @order_scheme = @config[:rows][:order_by]&.to_sym + validate_mapping_config! end def data @@ -121,14 +143,6 @@ def mapping_driven? @config[:rows].is_a?(Hash) end - def membership_scheme - @config[:rows][:require].to_sym - end - - def order_scheme - @config[:rows][:order_by]&.to_sym - end - def serialize_mapping_rows eligible_rows.each do |raw_row| nodes_for_pair(raw_row.pair).each do |node| @@ -140,39 +154,36 @@ def serialize_mapping_rows # Rows whose `require:` cell has a value, in mapping-file order. When `order_by:` is set, sorted # by that column's raw value instead (blank/`-` last), with ties broken by mapping-file order. def eligible_rows - rows = sectors.raw_rows.select { |raw_row| raw_row.cells[membership_scheme] } - return rows unless order_scheme + rows = sectors.raw_rows.select { |raw_row| raw_row.cells[@membership_scheme] } + return rows unless @order_scheme - rows.each_with_index.sort_by { |raw_row, index| [raw_row.cells[order_scheme].nil? ? 1 : 0, raw_row.cells[order_scheme].to_s, index] } - .map(&:first) + rows.sort_by.with_index { |raw_row, index| order_key(raw_row, index) } + end + + def order_key(raw_row, index) + cell = raw_row.cells[@order_scheme] + [cell.nil? ? 1 : 0, cell.to_s, index] end def serialize_mapping_column(column, raw_row, node) - case column.type - when 'node_attribute' + if column.type == 'node_attribute' value = node.node_api.instance_eval(column.value) return '' if value.nil? - value = eval(column.transform) if column.transform + value = column.transform.call(value) if column.transform value.to_s - when 'sector_mapping' + else # 'sector_mapping'; the only other type validate_mapping_config! admits raw_row.cells[column.value.to_sym].to_s - else - '' end end + # The `:emissions`-group nodes of both live graphs whose (sector_label, use) matches `pair`. def nodes_for_pair(pair) - nodes = %i[energy molecules].flat_map do |graph_type| - sectors.node_index(graph_type).fetch(pair, []).filter_map do |key| - node_for(graph_type, key) - end + nodes = [graph_interface.graph, graph_interface.molecules].flat_map do |graph| + graph.sector_map.nodes_for_pair(pair) end - nodes.select(&:emissions?).sort_by { |node| node.node_api.key.to_s } - end - def node_for(graph_type, key) - graph_type == :molecules ? graph_interface.molecules.node(key) : graph_interface.graph.node(key) + nodes.select(&:emissions?).sort_by { |node| node.node_api.key.to_s } end def graph_interface @@ -183,11 +194,30 @@ def sectors @sectors ||= Etsource::Sectors.new end + def validate_mapping_config! + if @period.nil? + raise ArgumentError, 'Mapping-driven rows require a period: of :present or :future.' + end + + if @membership_scheme.nil? + raise ArgumentError, 'Mapping-driven rows require a require: mapping column.' + end + + unsupported = @columns.reject { |column| MAPPING_COLUMN_TYPES.include?(column.type) } + + if unsupported.any? + raise UnsupportedColumnTypeError, + "Column types #{unsupported.map(&:type).uniq.inspect} are not supported with " \ + "mapping-driven rows. Supported types: #{MAPPING_COLUMN_TYPES.inspect}." + end + + validate_mapping_references! + end + def validate_mapping_references! valid = sectors.mapping.keys - references = [@config[:rows][:require], @config[:rows][:order_by]] + @columns.select { |c| - c.type == 'sector_mapping' - }.map(&:value) + sector_mapping_columns = @columns.select { |column| column.type == 'sector_mapping' } + references = [@membership_scheme, @order_scheme, *sector_mapping_columns.map(&:value)] references.compact.each do |reference| next if valid.include?(reference.to_sym) diff --git a/spec/models/gql/runtime/functions/lookup_spec.rb b/spec/models/gql/runtime/functions/lookup_spec.rb index a9c20fb40..2c65f1eae 100644 --- a/spec/models/gql/runtime/functions/lookup_spec.rb +++ b/spec/models/gql/runtime/functions/lookup_spec.rb @@ -272,6 +272,12 @@ def keys_of(nodes) end end + describe "EMISSIONS(ipcc_crt_code_agg, '1.A.1', '3', co2, 1990)" do + it 'raises on a multi-value call instead of misreading a value as the GHG' do + expect { result }.to raise_error(Gql::GqlError, /single value/) + end + end + it 'rejects UPDATE on a mapped EMISSIONS expression' do expect do gql.query_future("UPDATE(EMISSIONS(ipcc_crt_code_agg, '1.A.1', co2), co2, 1.0)") diff --git a/spec/serializers/export/configured_csv_serializer_spec.rb b/spec/serializers/export/configured_csv_serializer_spec.rb index b5d36e9d5..c858fec43 100644 --- a/spec/serializers/export/configured_csv_serializer_spec.rb +++ b/spec/serializers/export/configured_csv_serializer_spec.rb @@ -185,10 +185,17 @@ { bar: node_bar, baz: node_baz, foo: node_foo, buildings_space_heating_demand: node_buildings, lft: node_lft } end + let(:molecule_graph) { instance_double('Qernel::Graph') } + before do allow(gql.future.graph).to receive(:node) { |key| energy_nodes[key.to_sym] } - allow(gql.future).to receive(:molecules) - .and_return(instance_double('Qernel::Graph').tap { |g| allow(g).to receive(:node).with(:m_waste).and_return(node_waste) }) + allow(gql.future).to receive(:molecules).and_return(molecule_graph) + allow(molecule_graph).to receive(:node).with(:m_waste).and_return(node_waste) + + allow(molecule_graph).to receive(:sector_map) do + sectors = Etsource::Sectors.new + Qernel::Sectors.new(molecule_graph, sectors.mapping, sectors.node_index(:molecules)) + end end it 'includes the CSV headers' do @@ -376,6 +383,53 @@ end end + context 'when a mapping-driven config is invalid' do + let(:gql) { Scenario.default.gql } + let(:mapping_rows) { { require: 'emissions_sector' } } + let(:schema) { [{ name: 'Key', type: 'node_attribute', value: 'key' }] } + + context 'with no period' do + let(:serializer) { described_class.new({ schema: schema, rows: mapping_rows }, gql) } + + it 'raises at construction' do + expect { serializer }.to raise_error(ArgumentError, /period/) + end + end + + context 'with no require: column' do + let(:serializer) do + described_class.new({ schema: schema, rows: { order_by: 'ipcc_crt_code' } }, gql, period: :future) + end + + it 'raises at construction' do + expect { serializer }.to raise_error(ArgumentError, /require/) + end + end + + context 'with a column type only valid for query-driven rows' do + let(:schema) { super() + [{ name: 'Total', type: 'query' }] } + let(:serializer) { described_class.new({ schema: schema, rows: mapping_rows }, gql, period: :future) } + + it 'raises at construction instead of rendering silent blank cells' do + expect { serializer }.to raise_error( + Export::ConfiguredCSVSerializer::UnsupportedColumnTypeError, + /node_attribute.*sector_mapping/ + ) + end + end + + context 'with an untyped (literal) column' do + let(:schema) { super() + [{ name: 'Sector' }] } + let(:serializer) { described_class.new({ schema: schema, rows: mapping_rows }, gql, period: :future) } + + it 'raises at construction instead of rendering silent blank cells' do + expect { serializer }.to raise_error( + Export::ConfiguredCSVSerializer::UnsupportedColumnTypeError, /literal/ + ) + end + end + end + context 'when given an unknown sector mapping column' do let(:gql) { Scenario.default.gql } let(:serializer) { described_class.new(config, gql, period: :future) } From 3b6381018ca9f17a9194bc8b6f916dbd67b78103 Mon Sep 17 00:00:00 2001 From: louispt1 Date: Wed, 15 Jul 2026 15:02:39 +0200 Subject: [PATCH 7/7] Fix whitespace issue in gqueries controller for inspect gqueries --- app/controllers/inspect/gqueries_controller.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/inspect/gqueries_controller.rb b/app/controllers/inspect/gqueries_controller.rb index 6dffce086..85981933f 100644 --- a/app/controllers/inspect/gqueries_controller.rb +++ b/app/controllers/inspect/gqueries_controller.rb @@ -18,7 +18,7 @@ def test redirect_to inspect_debug_gql_path(gquery: params[:query]) elsif params[:query].present? begin - @result = @gql.query(params[:query].gsub(/\s/,''), nil, true) + @result = @gql.query(params[:query], nil, true) rescue Gql::CommandError => ex @error = ex end