Allow subscriptions to declare their real bar period

Algorithm/QCAlgorithm.Indicators.cs

@@ -4247,7 +4247,10 @@ private IDataConsolidator CreateConsolidator(Symbol symbol, Func<DateTime, Calen
 
             // verify this consolidator will give reasonable results, if someone asks for second consolidation but we have minute
             // data we won't be able to do anything good, we'll call it second, but it would really just be minute!
-            if (period.HasValue && period.Value < subscription.Increment || resolution.HasValue && resolution.Value < subscription.Resolution)
+            // Both operands are compared as durations so that subscriptions declaring an explicit bar period are
+            // validated against the real period rather than the one implied by the resolution.
+            if (period.HasValue && period.Value < subscription.Increment ||
+                resolution.HasValue && resolution.Value.ToTimeSpan() < subscription.Increment)
             {
                 throw new ArgumentException($"Unable to create {symbol} consolidator because {symbol} is registered for " +
                     Invariant($"{subscription.Resolution.ToStringInvariant()} data. Consolidators require higher resolution data to produce lower resolution data.")
@@ -4268,7 +4271,8 @@ private IDataConsolidator CreateConsolidator(Symbol symbol, Func<DateTime, Calen
                     period = subscription.Increment;
                 }
 
-                if (period.HasValue && period.Value == subscription.Increment || resolution.HasValue && resolution.Value == subscription.Resolution)
+                if (period.HasValue && period.Value == subscription.Increment ||
+                    resolution.HasValue && resolution.Value.ToTimeSpan() == subscription.Increment)
                 {
                     consolidator = CreateIdentityConsolidator(subscription.Type);
                 }

AlgorithmFactory/Python/Wrappers/AlgorithmPythonWrapper.cs

@@ -612,8 +612,8 @@ public Security AddSecurity(SecurityType securityType, string symbol, Resolution
         /// For example, 0 (default) will use the front month, 1 will use the back month contract</param>
         /// <returns>The new Security that was added to the algorithm</returns>
         public Security AddSecurity(Symbol symbol, Resolution? resolution = null, bool? fillForward = null, decimal leverage = Security.NullLeverage, bool? extendedMarketHours = null,
-            DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null, int contractDepthOffset = 0)
-            => _baseAlgorithm.AddSecurity(symbol, resolution, fillForward, leverage, extendedMarketHours, dataMappingMode, dataNormalizationMode, contractDepthOffset);
+            DataMappingMode? dataMappingMode = null, DataNormalizationMode? dataNormalizationMode = null, int contractDepthOffset = 0, TimeSpan? barPeriod = null)
+            => _baseAlgorithm.AddSecurity(symbol, resolution, fillForward, leverage, extendedMarketHours, dataMappingMode, dataNormalizationMode, contractDepthOffset, barPeriod);
 
         /// <summary>
         /// Creates and adds a new single <see cref="Future"/> contract to the algorithm
@@ -623,6 +623,8 @@ public Security AddSecurity(Symbol symbol, Resolution? resolution = null, bool?
         /// <param name="fillForward">If true, returns the last available data even if none in that timeslice. Default is <value>true</value></param>
         /// <param name="leverage">The requested leverage for this equity. Default is set by <see cref="SecurityInitializer"/></param>
         /// <param name="extendedMarketHours">Use extended market hours data</param>
+        /// <param name="barPeriod">The explicitly declared length of a single bar, when the stored data period
+        /// differs from the nominal period implied by <paramref name="resolution"/></param>
         /// <returns>The new <see cref="Future"/> security</returns>
         public Future AddFutureContract(Symbol symbol, Resolution? resolution = null, bool fillForward = true, decimal leverage = 0m,
             bool extendedMarketHours = false)
@@ -637,8 +639,9 @@ public Future AddFutureContract(Symbol symbol, Resolution? resolution = null, bo
         /// <param name="leverage">The requested leverage for this equity. Default is set by <see cref="SecurityInitializer"/></param>
         /// <param name="extendedMarketHours">Use extended market hours data</param>
         /// <returns>The new <see cref="Option"/> security</returns>
-        public Option AddOptionContract(Symbol symbol, Resolution? resolution = null, bool fillForward = true, decimal leverage = 0m, bool extendedMarketHours = false)
-            => _baseAlgorithm.AddOptionContract(symbol, resolution, fillForward, leverage, extendedMarketHours);
+        public Option AddOptionContract(Symbol symbol, Resolution? resolution = null, bool fillForward = true, decimal leverage = 0m,
+            bool extendedMarketHours = false, TimeSpan? barPeriod = null)
+            => _baseAlgorithm.AddOptionContract(symbol, resolution, fillForward, leverage, extendedMarketHours, barPeriod);
 
         /// <summary>
         /// Invoked at the end of every time step. This allows the algorithm

Common/Data/HistoryRequest.cs

@@ -40,6 +40,17 @@ public class HistoryRequest : BaseDataRequest
         /// </summary>
         public Resolution Resolution { get; set; }
 
+        /// <summary>
+        /// The explicitly declared length of a single bar, when the stored data period differs from the
+        /// nominal period implied by <see cref="Resolution"/>. Null derives it from the resolution
+        /// </summary>
+        public TimeSpan? BarPeriod { get; set; }
+
+        /// <summary>
+        /// The length of a single bar for this request
+        /// </summary>
+        public TimeSpan BarSpan => BarPeriod ?? Resolution.ToTimeSpan();
+
         /// <summary>
         /// Gets the requested fill forward resolution, set to null for no fill forward behavior.
         /// Will always return null when Resolution is set to Tick.
@@ -163,6 +174,7 @@ public HistoryRequest(SubscriptionDataConfig config, SecurityExchangeHours hours
                 hours, config.DataTimeZone, config.FillDataForward ? config.Resolution : (Resolution?)null,
                 config.ExtendedMarketHours, config.IsCustomData, config.DataNormalizationMode, config.TickType, config.DataMappingMode, config.ContractDepthOffset)
         {
+            BarPeriod = config.BarPeriod;
         }
 
         /// <summary>
@@ -174,6 +186,8 @@ public HistoryRequest(SubscriptionDataConfig config, SecurityExchangeHours hours
         public HistoryRequest(HistoryRequest request, Symbol newSymbol, DateTime newStartTimeUtc, DateTime newEndTimeUtc)
             : this (newStartTimeUtc, newEndTimeUtc, request.DataType, newSymbol, request.Resolution, request.ExchangeHours, request.DataTimeZone, request.FillForwardResolution,
                   request.IncludeExtendedMarketHours, request.IsCustomData, request.DataNormalizationMode, request.TickType, request.DataMappingMode, request.ContractDepthOffset)
-        { }
+        {
+            BarPeriod = request.BarPeriod;
+        }
     }
 }

Common/Data/HistoryRequestFactory.cs

@@ -89,7 +89,9 @@ public HistoryRequest CreateHistoryRequest(SubscriptionDataConfig subscription,
                 DataType = dataType,
                 Resolution = resolution.Value,
                 FillForwardResolution = fillForwardResolution,
-                TickType = subscription.TickType
+                TickType = subscription.TickType,
+                // a declared bar period describes the data of a specific resolution, drop it if the resolution changes
+                BarPeriod = resolution.Value == subscription.Resolution ? subscription.BarPeriod : null
             };
 
             if (extendedMarketHours != null)
@@ -136,9 +138,10 @@ public DateTime GetStartTimeAlgoTz(
             SecurityExchangeHours exchange,
             DateTimeZone dataTimeZone,
             Type dataType,
-            bool? extendedMarketHours = null)
+            bool? extendedMarketHours = null,
+            TimeSpan? barPeriod = null)
         {
-            return GetStartTimeAlgoTz(_algorithm.UtcTime, symbol, periods, resolution, exchange, dataTimeZone, dataType, extendedMarketHours);
+            return GetStartTimeAlgoTz(_algorithm.UtcTime, symbol, periods, resolution, exchange, dataTimeZone, dataType, extendedMarketHours, barPeriod);
         }
 
         /// <summary>
@@ -164,7 +167,8 @@ public DateTime GetStartTimeAlgoTz(
             SecurityExchangeHours exchange,
             DateTimeZone dataTimeZone,
             Type dataType,
-            bool? extendedMarketHours = null)
+            bool? extendedMarketHours = null,
+            TimeSpan? barPeriod = null)
         {
             var isExtendedMarketHours = false;
             // hour and daily resolution does no have extended market hours data. Same for chain universes
@@ -183,7 +187,7 @@ public DateTime GetStartTimeAlgoTz(
                 }
             }
 
-            var timeSpan = resolution.ToTimeSpan();
+            var timeSpan = barPeriod ?? resolution.ToTimeSpan();
             // make this a minimum of one second
             timeSpan = timeSpan < Time.OneSecond ? Time.OneSecond : timeSpan;
 

Common/Data/SubscriptionDataConfig.cs

@@ -66,6 +66,16 @@ public class SubscriptionDataConfig : IEquatable<SubscriptionDataConfig>
         /// </summary>
         public TimeSpan Increment { get; }
 
+        /// <summary>
+        /// The explicitly declared length of a single bar for this subscription, when the stored data
+        /// period differs from the nominal period implied by <see cref="Resolution"/>.
+        /// </summary>
+        /// <remarks>Null means the period is derived from <see cref="Resolution"/>, which is the default
+        /// behavior. This allows data whose true bar period is not one of the <see cref="Resolution"/>
+        /// values to be described accurately, so that bar end times, fill forward and consolidation all
+        /// operate on the real period instead of an assumed one.</remarks>
+        public TimeSpan? BarPeriod { get; }
+
         /// <summary>
         /// True if wish to send old data when time gaps in data feed.
         /// </summary>
@@ -197,6 +207,8 @@ public string MappedSymbol
         /// <param name="dataMappingMode">The contract mapping mode to use for the security</param>
         /// <param name="contractDepthOffset">The continuous contract desired offset from the current front month.
         /// For example, 0 (default) will use the front month, 1 will use the back month contract</param>
+        /// <param name="barPeriod">The explicitly declared length of a single bar, when the stored data period
+        /// differs from the nominal period implied by <paramref name="resolution"/>. Null derives it from the resolution</param>
         public SubscriptionDataConfig(Type objectType,
             Symbol symbol,
             Resolution resolution,
@@ -211,12 +223,21 @@ public SubscriptionDataConfig(Type objectType,
             DataNormalizationMode dataNormalizationMode = DataNormalizationMode.Adjusted,
             DataMappingMode dataMappingMode = DataMappingMode.OpenInterest,
             uint contractDepthOffset = 0,
-            bool mappedConfig = false)
+            bool mappedConfig = false,
+            TimeSpan? barPeriod = null)
         {
             if (objectType == null) throw new ArgumentNullException(nameof(objectType));
             if (symbol == null) throw new ArgumentNullException(nameof(symbol));
             if (dataTimeZone == null) throw new ArgumentNullException(nameof(dataTimeZone));
             if (exchangeTimeZone == null) throw new ArgumentNullException(nameof(exchangeTimeZone));
+            if (barPeriod.HasValue && barPeriod.Value <= TimeSpan.Zero)
+            {
+                throw new ArgumentOutOfRangeException(nameof(barPeriod), barPeriod, "Bar period must be positive");
+            }
+            if (barPeriod.HasValue && resolution == Resolution.Tick)
+            {
+                throw new ArgumentException("Bar period is not supported for tick resolution", nameof(barPeriod));
+            }
 
             Type = objectType;
             Resolution = resolution;
@@ -237,7 +258,8 @@ public SubscriptionDataConfig(Type objectType,
 
             TickType = tickType ?? LeanData.GetCommonTickTypeForCommonDataTypes(objectType, SecurityType);
 
-            Increment = resolution.ToTimeSpan();
+            Increment = barPeriod ?? resolution.ToTimeSpan();
+            BarPeriod = barPeriod;
             //Ticks are individual sales and fillforward doesn't apply.
             FillDataForward = resolution == Resolution.Tick ? false : fillForward;
         }
@@ -265,6 +287,9 @@ public SubscriptionDataConfig(Type objectType,
         /// For example, 0 (default) will use the front month, 1 will use the back month contract</param>
         /// <param name="mappedConfig">True if this is created as a mapped config. This is useful for continuous contract at live trading
         /// where we subscribe to the mapped symbol but want to preserve uniqueness</param>
+        /// <param name="barPeriod">The explicitly declared length of a single bar. When not provided, the source config's
+        /// value is inherited, unless <paramref name="resolution"/> changes the resolution, in which case the declared
+        /// period no longer describes the requested data and is dropped</param>
         public SubscriptionDataConfig(SubscriptionDataConfig config,
             Type objectType = null,
             Symbol symbol = null,
@@ -280,7 +305,8 @@ public SubscriptionDataConfig(SubscriptionDataConfig config,
             DataNormalizationMode? dataNormalizationMode = null,
             DataMappingMode? dataMappingMode = null,
             uint? contractDepthOffset = null,
-            bool? mappedConfig = null)
+            bool? mappedConfig = null,
+            TimeSpan? barPeriod = null)
             : this(
             objectType ?? config.Type,
             symbol ?? config.Symbol,
@@ -296,14 +322,28 @@ public SubscriptionDataConfig(SubscriptionDataConfig config,
             dataNormalizationMode ?? config.DataNormalizationMode,
             dataMappingMode ?? config.DataMappingMode,
             contractDepthOffset ?? config.ContractDepthOffset,
-            mappedConfig ?? false
+            mappedConfig ?? false,
+            barPeriod ?? GetInheritedBarPeriod(config, resolution)
             )
         {
             PriceScaleFactor = config.PriceScaleFactor;
             SumOfDividends = config.SumOfDividends;
             Consolidators = config.Consolidators;
         }
 
+        /// <summary>
+        /// Helper for the copy constructor. A declared bar period describes the data of a specific resolution,
+        /// so it is only inherited when the copy keeps the source resolution.
+        /// </summary>
+        private static TimeSpan? GetInheritedBarPeriod(SubscriptionDataConfig config, Resolution? resolution)
+        {
+            if (resolution.HasValue && resolution.Value != config.Resolution)
+            {
+                return null;
+            }
+            return config.BarPeriod;
+        }
+
         /// <summary>
         /// Indicates whether the current object is equal to another object of the same type.
         /// </summary>
@@ -327,6 +367,7 @@ public bool Equals(SubscriptionDataConfig other)
                 && ExchangeTimeZone.Equals(other.ExchangeTimeZone)
                 && ContractDepthOffset == other.ContractDepthOffset
                 && IsFilteredSubscription == other.IsFilteredSubscription
+                && BarPeriod == other.BarPeriod
                 && _mappedConfig == other._mappedConfig;
         }
 
@@ -368,6 +409,7 @@ public override int GetHashCode()
                 hashCode = (hashCode*397) ^ ExchangeTimeZone.Id.GetHashCode();// timezone hash is expensive, use id instead
                 hashCode = (hashCode*397) ^ ContractDepthOffset.GetHashCode();
                 hashCode = (hashCode*397) ^ IsFilteredSubscription.GetHashCode();
+                hashCode = (hashCode*397) ^ BarPeriod.GetHashCode();
                 hashCode = (hashCode*397) ^ _mappedConfig.GetHashCode();
                 return hashCode;
             }

Common/Data/SubscriptionDataConfigExtensions.cs

@@ -13,6 +13,7 @@
  * limitations under the License.
 */
 
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using QuantConnect.Util;
@@ -41,6 +42,26 @@ public static Resolution GetHighestResolution(
                 .Min();
         }
 
+        /// <summary>
+        /// Extension method used to obtain the shortest bar period for a given set of
+        /// <see cref="SubscriptionDataConfig"/>, honoring any explicitly declared
+        /// <see cref="SubscriptionDataConfig.BarPeriod"/>
+        /// </summary>
+        /// <param name="subscriptionDataConfigs"></param>
+        /// <returns>The shortest bar period, <see cref="Resolution.Daily"/>'s period if there
+        /// are no subscriptions</returns>
+        /// <remarks>Equivalent to <see cref="GetHighestResolution"/> followed by
+        /// <see cref="Extensions.ToTimeSpan"/> for any subscription that does not declare a bar period,
+        /// since <see cref="Extensions.ToTimeSpan"/> is monotonic in the <see cref="Resolution"/> order</remarks>
+        public static TimeSpan GetHighestResolutionSpan(
+            this IEnumerable<SubscriptionDataConfig> subscriptionDataConfigs)
+        {
+            return subscriptionDataConfigs
+                .Select(x => x.Increment)
+                .DefaultIfEmpty(Resolution.Daily.ToTimeSpan())
+                .Min();
+        }
+
         /// <summary>
         /// Extension method used to determine if FillForward is enabled
         /// for a given set of <see cref="SubscriptionDataConfig"/>

Common/Data/UniverseSelection/Universe.cs

@@ -294,7 +294,8 @@ public virtual IEnumerable<SubscriptionRequest> GetSubscriptionRequests(Security
                 dataNormalizationMode: UniverseSettings.DataNormalizationMode,
                 subscriptionDataTypes: UniverseSettings.SubscriptionDataTypes,
                 dataMappingMode: UniverseSettings.DataMappingMode,
-                contractDepthOffset: (uint)Math.Abs(UniverseSettings.ContractDepthOffset));
+                contractDepthOffset: (uint)Math.Abs(UniverseSettings.ContractDepthOffset),
+                barPeriod: UniverseSettings.BarPeriod);
             return result.Select(config => new SubscriptionRequest(isUniverseSubscription: false,
                 universe: this,
                 security: security,

Common/Data/UniverseSelection/UniverseSettings.cs

@@ -39,6 +39,12 @@ public class UniverseSettings
         /// </summary>
         public bool FillForward { get; set; }
 
+        /// <summary>
+        /// The explicitly declared length of a single bar, when the stored data period differs from the
+        /// nominal period implied by <see cref="Resolution"/>. Null derives it from the resolution
+        /// </summary>
+        public TimeSpan? BarPeriod { get; set; }
+
         /// <summary>
         /// If configured, will be used to determine universe selection schedule and filter or skip selection data
         /// that does not fit the schedule
@@ -127,6 +133,7 @@ public UniverseSettings(UniverseSettings universeSettings)
             Resolution = universeSettings.Resolution;
             Leverage = universeSettings.Leverage;
             FillForward = universeSettings.FillForward;
+            BarPeriod = universeSettings.BarPeriod;
             DataMappingMode = universeSettings.DataMappingMode;
             ContractDepthOffset = universeSettings.ContractDepthOffset;
             ExtendedMarketHours = universeSettings.ExtendedMarketHours;