Add scheme-based SECTOR/MSECTOR lookups and mapped EMISSIONS form

Gemfile

@@ -73,7 +73,7 @@ gem 'ruby-progressbar'
 
 # own gems
 gem 'quintel_merit', ref: 'aae77e0', github: 'quintel/merit'
-gem 'atlas',         ref: '5504fac', github: 'quintel/atlas'
+gem 'atlas',         ref: '5d2e5b2', github: 'quintel/atlas'
 gem 'fever',         ref: '2afebd1', github: 'quintel/fever'
 gem 'refinery',      ref: '36b8e34', github: 'quintel/refinery'
 gem 'rubel',         ref: '9fe7010', github: 'quintel/rubel'

Gemfile.lock

@@ -1,7 +1,7 @@
 GIT
   remote: https://github.com/quintel/atlas.git
-  revision: 5504fac349b1b25fda2ef285b044b025fbfbdb1e
-  ref: 5504fac
+  revision: 5d2e5b20e3d397b764f688180e3987bd43161612
+  ref: 5d2e5b2
   specs:
     atlas (1.0.0)
       activemodel (>= 7)

app/controllers/inspect/gqueries_controller.rb

@@ -18,7 +18,7 @@ def test
       redirect_to inspect_debug_gql_path(gquery: params[:query])
     elsif params[:query].present?
       begin
-        @result = @gql.query(params[:query].gsub(/\s/,''), nil, true)
+        @result = @gql.query(params[:query], nil, true)
       rescue Gql::CommandError => ex
         @error = ex
       end

app/models/etsource/sectors.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Etsource
+  # Reads the sector mapping and node labels from Atlas and caches the
+  # structures GQL and the configured CSV exports need.
+  #
+  #   * mapping        - {scheme => {value => [[label, use], ...]}}, for
+  #                      `SECTOR(scheme, value)` style GQL queries.
+  #   * node_index(g)  - {[label, use] => [node_key, ...]} for one graph type,
+  #                      with `use` baked in at import so it is never consulted
+  #                      per query.
+  #   * raw_rows       - mapping rows in file order with raw display values,
+  #                      for mapping-driven CSV exports.
+  class Sectors
+    # Public: The inverted mapping index, keyed by scheme then normalized value.
+    def mapping
+      NastyCache.instance.fetch('sector_mapping_hash') do
+        Atlas::SectorMapping.load.to_h
+      end
+    end
+
+    # Public: The (label, use) -> node keys index for the given graph type
+    # (:energy or :molecules).
+    def node_index(graph_type)
+      NastyCache.instance.fetch("sector_node_index_#{graph_type}") do
+        build_node_index(node_class_for(graph_type))
+      end
+    end
+
+    # Public: Every mapping row in file order, as an Atlas::SectorMapping::RawRow
+    # (the (label, use) pair plus the raw display value of every scheme cell).
+    def raw_rows
+      NastyCache.instance.fetch('sector_raw_rows') do
+        Atlas::SectorMapping.load.raw_rows
+      end
+    end
+
+    private
+
+    def build_node_index(node_class)
+      node_class.all.each_with_object({}) do |node, index|
+        next if node.sector_label.nil?
+
+        pair = [node.sector_label, Atlas::SectorMapping.normalize(node.use)]
+        (index[pair] ||= []) << node.key
+      end
+    end
+
+    def node_class_for(graph_type)
+      graph_type.to_sym == :molecules ? Atlas::MoleculeNode : Atlas::EnergyNode
+    end
+  end
+end

app/models/gql/command.rb

@@ -108,12 +108,15 @@ def truncate(string, length = 40)
       string.length > (length - 3) ? "#{ string[0..length] }..." : string
     end
 
-    # Internal: Removes extra "artifacts" from the source.
+    # Internal: Removes extra "artifacts" from the source. Whitespace is
+    # insignificant in GQL and stripped, except inside string literals, where
+    # it may be meaningful (e.g. SECTOR scheme values such as 'Fuels
+    # production').
     def clean(string)
       cleaned = (string || '').dup
 
-      cleaned.gsub!(/[\n\s\t]/, '')
-      cleaned.gsub!(/^[a-z]+\:/,'')
+      cleaned.gsub!(/('[^']*'|"[^"]*")|\s+/) { Regexp.last_match(1) || '' }
+      cleaned.sub!(/\A[a-z]+\:/, '')
 
       cleaned
     end

app/models/gql/gql_error.rb

@@ -23,4 +23,21 @@ def initialize(curve, attribute = nil)
     end
   end
 
+  # Raised when a SECTOR/MSECTOR/EMISSIONS scheme-form query names a
+  # classification scheme which does not exist in the sector mapping.
+  class UnknownSectorSchemeError < GqlError
+    def initialize(scheme, valid_schemes)
+      super("Unknown sector mapping scheme #{scheme.inspect}. " \
+            "Valid schemes: #{valid_schemes.map(&:inspect).join(', ')}.")
+    end
+  end
+
+  # Raised when a SECTOR/MSECTOR/EMISSIONS scheme-form query names a value
+  # which does not appear in that scheme's column.
+  class UnknownSectorValueError < GqlError
+    def initialize(scheme, value)
+      super("Unknown value #{value.inspect} for sector mapping scheme " \
+            "#{scheme.inspect}.")
+    end
+  end
 end

app/models/gql/query_interface/lookup.rb

@@ -64,6 +64,22 @@ def molecule_sector_nodes(keys)
         molecule_graph_helper.sector_nodes(keys)
       end
 
+      # Scheme-based sector mapping lookups. `energy`/`molecule` select the
+      # graph whose live nodes are returned; the mapping itself is shared.
+      def energy_sector_node_map(scheme, values)
+        graph.sector_map.lookup(scheme, values)
+      end
+
+      def molecule_sector_map(scheme, values)
+        molecules.sector_map.lookup(scheme, values)
+      end
+
+      # The energy graph's resolver, used for EMISSIONS scheme dispatch and
+      # pair resolution (the mapping is graph-independent).
+      def sector_resolver
+        graph.sector_map
+      end
+
       def energy_use_nodes(keys)
         energy_graph_helper.use_nodes(keys)
       end

app/models/gql/runtime/functions/lookup.rb

@@ -91,13 +91,13 @@ def MALL
       def GROUP(*keys)
         scope.group_energy_nodes(keys)
       end
-      alias G GROUP
+      alias_method :G, :GROUP
 
       # Returns an Array of {Qernel::Node} for given molecule group. See GROUP.
       def MGROUP(*keys)
         scope.group_molecule_nodes(keys)
       end
-      alias MG MGROUP
+      alias_method :MG, :MGROUP
 
       # Returns an Array of {Qernel::Edges} for given energy group.
       #
@@ -108,27 +108,44 @@ def MGROUP(*keys)
       def EDGE_GROUP(*keys)
         scope.group_energy_edges(keys)
       end
-      alias EG EDGE_GROUP
+      alias_method :EG, :EDGE_GROUP
 
       # Returns an Array of {Qernel::Edges} for given molecule group. See EDGE_GROUP.
       def MEDGE_GROUP(*keys)
         scope.group_molecule_edges(keys)
       end
-      alias MEG EDGE_GROUP
+      alias_method :MEG, :MEDGE_GROUP
 
-      # Returns an Array of {Qernel::Node} for given energy sector.
+      # Returns an Array of {Qernel::Node} for an energy sector.
       #
-      # Examples
+      # Dispatches on arity:
       #
-      #   SECTOR(households)
+      #   * One argument  - namespace-sector filter.
+      #       SECTOR(households)
+      #   * Two or more   - a classification scheme followed by one or more
+      #                     values; returns the union of matching nodes.
+      #       SECTOR(emissions_subsector, 'Electricity and heat production')
+      #       SECTOR(ipcc_crt_code_agg, '1.A.1', '1.A.2')
       #
+      # Unknown scheme or value raises; a valid value with no labelled nodes
+      # returns an empty array.
       def SECTOR(*keys)
-        scope.energy_sector_nodes(keys)
+        if keys.size <= 1
+          scope.energy_sector_nodes(keys)
+        else
+          scope.energy_sector_node_map(keys.first, keys.drop(1))
+        end
       end
 
-      # Returns an Array of {Qernel::Node} for given molecule sector. See SECTOR.
+      # Returns an Array of {Qernel::Node} for a molecule sector. See SECTOR;
+      # identical dispatch, normalization and error contract, resolved against
+      # the molecule graph.
       def MSECTOR(*keys)
-        scope.molecule_sector_nodes(keys)
+        if keys.size <= 1
+          scope.molecule_sector_nodes(keys)
+        else
+          scope.molecule_sector_map(keys.first, keys.drop(1))
+        end
       end
 
       # Returns an Array with {Qernel::Node} for given energy use.
@@ -362,6 +379,12 @@ def DATASET_CURVE(key)
       def EMISSIONS(*keys)
         return scope.graph.emissions if keys.empty?
 
+        # Mapped form: EMISSIONS(scheme, value, ghg[, 1990]). Dispatches on the
+        # first argument being a known classification scheme, because the legacy
+        # arities overlap. Read-only: sums the store over the mapping's resolved
+        # (sector label, use) pairs; never returns a scoped-sector handle.
+        return mapped_emissions(keys) if scope.sector_resolver.scheme?(keys.first)
+
         # Convert dashes/dots to underscores in the first key (sector)
         keys[0] = keys.first.to_s.tr('-.', '_').to_sym
 
@@ -370,9 +393,29 @@ def EMISSIONS(*keys)
 
         # EMISSIONS(sector, use, ghg [, year]) -> return value.
         # The present year is implicit; only 1990 is suffixed with a year.
-        scope.graph.emissions[
-          [*keys.first(3), (1990 if keys[3].to_i == 1990)].compact.join('_').to_sym
-        ]
+        scope.graph.emissions.value_for(*keys.first(3), year: keys[3])
+      end
+
+      private
+
+      # Internal: The mapped EMISSIONS form. Resolves `scheme`/`value` to
+      # (sector label, use) pairs and lets the emissions store sum over them.
+      def mapped_emissions(keys)
+        if keys.size > 4
+          raise Gql::GqlError, "EMISSIONS(#{keys.first.inspect}, ...) accepts a single value: " \
+                               'EMISSIONS(scheme, value, ghg[, 1990]).'
+        end
+
+        scheme, value, ghg, year = keys
+
+        if ghg.nil?
+          raise Gql::GqlError, "EMISSIONS(#{scheme.inspect}, ...) needs a GHG argument, e.g. " \
+                               "EMISSIONS(#{scheme.inspect}, #{value.inspect}, co2)"
+        end
+
+        scope.graph.emissions.sum_pairs(
+          scope.sector_resolver.pairs(scheme, value), ghg, year: year
+        )
       end
     end
   end

app/models/qernel/emissions.rb

@@ -24,6 +24,14 @@ class Emissions
     dataset_accessors ::Etsource::Dataset.emissions_keys
     attr_accessor :graph
 
+    # Public: The flat store key for the given parts, e.g. (sector, use, ghg).
+    # The single place the `parts_joined_by_underscores[_1990]` schema lives.
+    #
+    # Returns a Symbol.
+    def self.key_for(*parts, year: nil)
+      [*parts, (1990 if year.to_i == 1990)].compact.join('_').to_sym
+    end
+
     # Queryable object that provides scoped access to emissions data
     #
     # GQL uses this to scope the sector for easier queries and input/update
@@ -40,20 +48,19 @@ def initialize(emissions, scope, year = nil)
       end
 
       def [](attr_name)
-        @emissions[scoped_method(attr_name).to_sym]
+        @emissions[scoped_method(attr_name)]
       end
 
       def []=(attr_name, value)
-        @emissions[scoped_method(attr_name).to_sym] = value
+        @emissions[scoped_method(attr_name)] = value
       end
 
       def inspect
         "<Qernel::Emissions::ScopedSector #{@scope}>"
       end
 
       def scoped_method(method_name)
-        attr = method_name.to_s.delete_suffix('=')
-        @year.to_i == 1990 ? "#{@scope}_#{attr}_1990" : "#{@scope}_#{attr}"
+        Emissions.key_for(@scope, method_name.to_s.delete_suffix('='), year: @year)
       end
 
       def respond_to_missing?(method_name, include_private = false)
@@ -65,7 +72,7 @@ def respond_to_missing?(method_name, include_private = false)
       end
 
       def method_missing(method_name, *args)
-        key = scoped_method(method_name).to_sym
+        key = scoped_method(method_name)
 
         if method_name.to_s.end_with?('=')
           unless scope_exists?
@@ -102,5 +109,17 @@ def initialize(graph = nil)
     def scope(sector, year = nil)
       ScopedSector.new(self, sector, year)
     end
+
+    # Public: The stored value for (sector, use, ghg), or nil when the store
+    # has no such key. Pass year: 1990 to read the historic baseline.
+    def value_for(sector, use, ghg, year: nil)
+      self[self.class.key_for(sector, use, ghg, year: year)]
+    end
+
+    # Public: Sums the store over (sector, use) pairs for one GHG. Pairs
+    # without a stored value count as zero.
+    def sum_pairs(pairs, ghg, year: nil)
+      pairs.sum { |sector, use| value_for(sector, use, ghg, year: year) || 0.0 }
+    end
   end
 end

app/models/qernel/graph.rb

@@ -264,6 +264,18 @@ def sectors
     nodes.map(&:sector_key).uniq.compact
   end
 
+  # Resolver for scheme-based sector mapping queries (SECTOR(scheme, value)).
+  #
+  # Memoized per graph instance.
+  #
+  # @return [Qernel::Sectors]
+  def sector_map
+    @sector_map ||= begin
+      sectors = Etsource::Sectors.new
+      Qernel::Sectors.new(self, sectors.mapping, sectors.node_index(@name))
+    end
+  end
+
   # Return all nodes in the given sector.
   #
   # @param sector_key [String,Symbol] sector identifier