Emissions

Gemfile

@@ -73,7 +73,7 @@ gem 'ruby-progressbar'
 
 # own gems
 gem 'quintel_merit', ref: 'aae77e0', github: 'quintel/merit'
-gem 'atlas',         ref: '72dbea1', github: 'quintel/atlas' #TODO: Update to latest merge commit once Atlas branch is merged into master
+gem 'atlas',         ref: 'b2d47a1', github: 'quintel/atlas'
 gem 'fever',         ref: '2afebd1', github: 'quintel/fever'
 gem 'refinery',      ref: '36b8e34', github: 'quintel/refinery'
 gem 'rubel',         ref: '9fe7010', github: 'quintel/rubel'

Gemfile.lock

@@ -1,7 +1,7 @@
 GIT
   remote: https://github.com/quintel/atlas.git
-  revision: 72dbea148d09aaae4b38361203f6e98bb29a9efb
-  ref: 72dbea1
+  revision: b2d47a1495f85638398d89acc17c6a2da7d2acac
+  ref: b2d47a1
   specs:
     atlas (1.0.0)
       activemodel (>= 7)

app/controllers/inspect/gqueries_controller.rb

@@ -18,7 +18,7 @@ def test
       redirect_to inspect_debug_gql_path(gquery: params[:query])
     elsif params[:query].present?
       begin
-        @result = @gql.query(params[:query].gsub(/\s/,''), nil, true)
+        @result = @gql.query(params[:query], nil, true)
       rescue Gql::CommandError => ex
         @error = ex
       end

app/models/etsource/sectors.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Etsource
+  # Reads the sector mapping and node labels from Atlas and caches the
+  # structures GQL and the configured CSV exports need.
+  #
+  #   * mapping        - {scheme => {value => [[label, use], ...]}}, for
+  #                      `SECTOR(scheme, value)` style GQL queries.
+  #   * node_index(g)  - {[label, use] => [node_key, ...]} for one graph type,
+  #                      with `use` baked in at import so it is never consulted
+  #                      per query.
+  #   * raw_rows       - mapping rows in file order with raw display values,
+  #                      for mapping-driven CSV exports.
+  class Sectors
+    # Public: The inverted mapping index, keyed by scheme then normalized value.
+    def mapping
+      NastyCache.instance.fetch('sector_mapping_hash') do
+        Atlas::SectorMapping.load.to_h
+      end
+    end
+
+    # Public: The (label, use) -> node keys index for the given graph type
+    # (:energy or :molecules).
+    def node_index(graph_type)
+      NastyCache.instance.fetch("sector_node_index_#{graph_type}") do
+        build_node_index(node_class_for(graph_type))
+      end
+    end
+
+    # Public: Every mapping row in file order, as an Atlas::SectorMapping::RawRow
+    # (the (label, use) pair plus the raw display value of every scheme cell).
+    def raw_rows
+      NastyCache.instance.fetch('sector_raw_rows') do
+        Atlas::SectorMapping.load.raw_rows
+      end
+    end
+
+    private
+
+    def build_node_index(node_class)
+      node_class.all.each_with_object({}) do |node, index|
+        next if node.sector_label.nil?
+
+        pair = [node.sector_label, Atlas::SectorMapping.normalize(node.use)]
+        (index[pair] ||= []) << node.key
+      end
+    end
+
+    def node_class_for(graph_type)
+      graph_type.to_sym == :molecules ? Atlas::MoleculeNode : Atlas::EnergyNode
+    end
+  end
+end

app/models/gql/command.rb

@@ -108,12 +108,15 @@ def truncate(string, length = 40)
       string.length > (length - 3) ? "#{ string[0..length] }..." : string
     end
 
-    # Internal: Removes extra "artifacts" from the source.
+    # Internal: Removes extra "artifacts" from the source. Whitespace is
+    # insignificant in GQL and stripped, except inside string literals, where
+    # it may be meaningful (e.g. SECTOR scheme values such as 'Fuels
+    # production').
     def clean(string)
       cleaned = (string || '').dup
 
-      cleaned.gsub!(/[\n\s\t]/, '')
-      cleaned.gsub!(/^[a-z]+\:/,'')
+      cleaned.gsub!(/('[^']*'|"[^"]*")|\s+/) { Regexp.last_match(1) || '' }
+      cleaned.sub!(/\A[a-z]+\:/, '')
 
       cleaned
     end

app/models/gql/gql_error.rb

@@ -23,4 +23,21 @@ def initialize(curve, attribute = nil)
     end
   end
 
+  # Raised when a SECTOR/MSECTOR/EMISSIONS scheme-form query names a
+  # classification scheme which does not exist in the sector mapping.
+  class UnknownSectorSchemeError < GqlError
+    def initialize(scheme, valid_schemes)
+      super("Unknown sector mapping scheme #{scheme.inspect}. " \
+            "Valid schemes: #{valid_schemes.map(&:inspect).join(', ')}.")
+    end
+  end
+
+  # Raised when a SECTOR/MSECTOR/EMISSIONS scheme-form query names a value
+  # which does not appear in that scheme's column.
+  class UnknownSectorValueError < GqlError
+    def initialize(scheme, value)
+      super("Unknown value #{value.inspect} for sector mapping scheme " \
+            "#{scheme.inspect}.")
+    end
+  end
 end

app/models/gql/query_interface/lookup.rb

@@ -64,10 +64,30 @@ def molecule_sector_nodes(keys)
         molecule_graph_helper.sector_nodes(keys)
       end
 
+      # Scheme-based sector mapping lookups. `energy`/`molecule` select the
+      # graph whose live nodes are returned; the mapping itself is shared.
+      def energy_sector_node_map(scheme, values)
+        graph.sector_map.lookup(scheme, values)
+      end
+
+      def molecule_sector_map(scheme, values)
+        molecules.sector_map.lookup(scheme, values)
+      end
+
+      # The energy graph's resolver, used for EMISSIONS scheme dispatch and
+      # pair resolution (the mapping is graph-independent).
+      def sector_resolver
+        graph.sector_map
+      end
+
       def energy_use_nodes(keys)
         energy_graph_helper.use_nodes(keys)
       end
 
+      def molecule_use_nodes(keys)
+        molecule_graph_helper.use_nodes(keys)
+      end
+
       def group_energy_nodes(keys)
         energy_graph_helper.group_nodes(keys)
       end

app/models/gql/runtime/functions/lookup.rb

@@ -91,13 +91,13 @@ def MALL
       def GROUP(*keys)
         scope.group_energy_nodes(keys)
       end
-      alias G GROUP
+      alias_method :G, :GROUP
 
       # Returns an Array of {Qernel::Node} for given molecule group. See GROUP.
       def MGROUP(*keys)
         scope.group_molecule_nodes(keys)
       end
-      alias MG MGROUP
+      alias_method :MG, :MGROUP
 
       # Returns an Array of {Qernel::Edges} for given energy group.
       #
@@ -108,33 +108,48 @@ def MGROUP(*keys)
       def EDGE_GROUP(*keys)
         scope.group_energy_edges(keys)
       end
-      alias EG EDGE_GROUP
+      alias_method :EG, :EDGE_GROUP
 
       # Returns an Array of {Qernel::Edges} for given molecule group. See EDGE_GROUP.
       def MEDGE_GROUP(*keys)
         scope.group_molecule_edges(keys)
       end
-      alias MEG EDGE_GROUP
+      alias_method :MEG, :MEDGE_GROUP
 
-      # Returns an Array of {Qernel::Node} for given energy sector.
+      # Returns an Array of {Qernel::Node} for an energy sector.
       #
-      # Examples
+      # Dispatches on arity:
       #
-      #   SECTOR(households)
+      #   * One argument  - namespace-sector filter.
+      #       SECTOR(households)
+      #   * Two or more   - a classification scheme followed by one or more
+      #                     values; returns the union of matching nodes.
+      #       SECTOR(emissions_subsector, 'Electricity and heat production')
+      #       SECTOR(ipcc_crt_code_agg, '1.A.1', '1.A.2')
       #
+      # Unknown scheme or value raises; a valid value with no labelled nodes
+      # returns an empty array.
       def SECTOR(*keys)
-        scope.energy_sector_nodes(keys)
+        if keys.size <= 1
+          scope.energy_sector_nodes(keys)
+        else
+          scope.energy_sector_node_map(keys.first, keys.drop(1))
+        end
       end
 
-      # Returns an Array of {Qernel::Node} for given molecule sector. See SECTOR.
+      # Returns an Array of {Qernel::Node} for a molecule sector. See SECTOR;
+      # identical dispatch, normalization and error contract, resolved against
+      # the molecule graph.
       def MSECTOR(*keys)
-        scope.molecule_sector_nodes(keys)
+        if keys.size <= 1
+          scope.molecule_sector_nodes(keys)
+        else
+          scope.molecule_sector_map(keys.first, keys.drop(1))
+        end
       end
 
       # Returns an Array with {Qernel::Node} for given energy use.
       #
-      # See Qernel::Node::USES
-      #
       # Examples
       #
       #   USE(energetic)
@@ -145,6 +160,11 @@ def USE(*keys)
         scope.energy_use_nodes(keys)
       end
 
+      # Returns an Array of {Qernel::Node} for given molecule use. See USE.
+      def MUSE(*keys)
+        scope.molecule_use_nodes(keys)
+      end
+
       # Returns an Array of {Qernel::Carrier} for given key(s). Returns carriers belonging to the
       # energy graph.
       #
@@ -321,16 +341,17 @@ def DATASET_CURVE(key)
       # Returns an attribute {Qernel::Emissions} or {Qernel::Emissions::ScopedSector} or a value.
       #
       # Emissions data is loaded from CSV files in ETSource with the following structure:
-      #   etm_sector, etm_subsector, use, ghg, unit, value
+      #   etm_sector, etm_subsector, use, ghg, year, unit, value
       #
       # Parameters:
       #   - sector: ETM sector name (e.g., 'households', 'buildings_non_specified')
       #             Dashes/dots in sector names are converted to underscores for key generation
       #   - use: Emission use type (energetic, non_energetic) - REQUIRED when accessing values
       #   - ghg: GHG type (co2, other_ghg) - optional
-      #   - year: Year of emission (e.g., 1990) - optional, reads from emissions_YEAR.csv files
+      #   - year: Pass 1990 to read the historic baseline; the present year is
+      #           implicit and needs no argument
       #
-      # Key generation combines: sector_[subsector_]use_ghg[_year]
+      # Key generation combines: sector_[subsector_]use_ghg[_1990]
       # Note: Unit column from CSV is not included in keys, blank values return nil
       #
       # EMISSIONS() without any keys returns {Qernel::Emissions}
@@ -351,21 +372,50 @@ def DATASET_CURVE(key)
       # EMISSIONS(sector, use, ghg) or EMISSIONS(sector, use, ghg, year) returns an emission value
       #
       # Examples
-      #   EMISSIONS(households, energetic, other_ghg) # => 12.0 (from emissions.csv)
-      #   EMISSIONS(households, energetic, co2, 1990) # => value (from emissions_1990.csv)
+      #   EMISSIONS(households, energetic, other_ghg) # => 12.0 (present year)
+      #   EMISSIONS(households, energetic, co2, 1990) # => value for 1990
       #   EMISSIONS(buildings_non_specified, energetic, other_ghg) # => 18.0
       #
       def EMISSIONS(*keys)
         return scope.graph.emissions if keys.empty?
 
+        # Mapped form: EMISSIONS(scheme, value, ghg[, 1990]). Dispatches on the
+        # first argument being a known classification scheme, because the legacy
+        # arities overlap. Read-only: sums the store over the mapping's resolved
+        # (sector label, use) pairs; never returns a scoped-sector handle.
+        return mapped_emissions(keys) if scope.sector_resolver.scheme?(keys.first)
+
         # Convert dashes/dots to underscores in the first key (sector)
         keys[0] = keys.first.to_s.tr('-.', '_').to_sym
 
         # EMISSIONS(sector, use) -> return ScopedSector for UPDATE operations
         return scope.graph.emissions.scope(keys.join('_').to_sym) if keys.size == 2
 
-        # EMISSIONS(sector, use, ghg [, year]) -> return value
-        scope.graph.emissions[keys.join('_').to_sym]
+        # EMISSIONS(sector, use, ghg [, year]) -> return value.
+        # The present year is implicit; only 1990 is suffixed with a year.
+        scope.graph.emissions.value_for(*keys.first(3), year: keys[3])
+      end
+
+      private
+
+      # Internal: The mapped EMISSIONS form. Resolves `scheme`/`value` to
+      # (sector label, use) pairs and lets the emissions store sum over them.
+      def mapped_emissions(keys)
+        if keys.size > 4
+          raise Gql::GqlError, "EMISSIONS(#{keys.first.inspect}, ...) accepts a single value: " \
+                               'EMISSIONS(scheme, value, ghg[, 1990]).'
+        end
+
+        scheme, value, ghg, year = keys
+
+        if ghg.nil?
+          raise Gql::GqlError, "EMISSIONS(#{scheme.inspect}, ...) needs a GHG argument, e.g. " \
+                               "EMISSIONS(#{scheme.inspect}, #{value.inspect}, co2)"
+        end
+
+        scope.graph.emissions.sum_pairs(
+          scope.sector_resolver.pairs(scheme, value), ghg, year: year
+        )
       end
     end
   end