Merge pull request #5 from LouisSzeto/master

New api endpoint, `BaseDataCollection` abstraction, demo algos

Author: Martin-Molinero

.github/workflows/build.yml

@@ -21,4 +21,4 @@ jobs:
         run: dotnet build ./tests/Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1
 
       - name: Run Tests
-        run: dotnet test ./tests/bin/Release/net5.0/Tests.dll
+        run: dotnet test ./tests/bin/Release/net6.0/Tests.dll

QuantConnect.DataSource.csproj

@@ -1,27 +1,25 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net5.0</TargetFramework>
+    <TargetFramework>net6.0</TargetFramework>
     <RootNamespace>QuantConnect.DataSource</RootNamespace>
     <AssemblyName>QuantConnect.DataSource.RegalyticsArticles</AssemblyName>
     <OutputPath>bin\$(Configuration)</OutputPath>
     <DocumentationFile>$(OutputPath)\QuantConnect.DataSource.RegalyticsArticles.xml</DocumentationFile>
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="QuantConnect.Common" Version="2.5.*" />
-    <PackageReference Include="protobuf-net" Version="3.0.29" />
     <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
   </ItemGroup>
 
   <ItemGroup>
     <Compile Remove="tests\**" />
     <EmbeddedResource Remove="tests\**" />
     <None Remove="tests\**" />
-    <Compile Remove="Demonstration.cs" />
+    <Compile Remove="RegalyticsRegulatoryArticlesDataAlgorithm.cs" />
+    <None Remove="RegalyticsRegulatoryArticlesDataAlgorithm.py" />
     <None Remove=".gitignore" />
-    <None Remove="process.sample.ipynb" />
-    <None Remove="process.sample.py" />
-    <None Remove="process.sample.sh" />
+    <None Remove="process.py" />
   </ItemGroup>
 
 </Project>

RegalyticsRegulatoryArticle.cs

@@ -15,24 +15,19 @@
 */
 
 using System;
-using NodaTime;
-using ProtoBuf;
-using System.IO;
-using QuantConnect.Data;
 using System.Collections.Generic;
 using Newtonsoft.Json;
-using DateTimeUtilityFunctions = QuantConnect.Securities.Future.FuturesExpiryUtilityFunctions;
+using NodaTime;
+using QuantConnect.Data;
+using QuantConnect.Util;
 
 namespace QuantConnect.DataSource
 {
     /// <summary>
     /// Regalytics Regulatory articles
     /// </summary>
-    [ProtoContract(SkipConstructor = true)]
     public class RegalyticsRegulatoryArticle : BaseData
     {
-        private DateTime _endTime;
-
         /// <summary>
         /// Data source ID
         /// </summary>
@@ -92,34 +87,11 @@ public class RegalyticsRegulatoryArticle : BaseData
         [JsonProperty(PropertyName = "pdf_url")]
         public string AnnouncementUrl { get; set; }
 
-        public override DateTime EndTime
-        {
-            get { return _endTime; }
-            set { _endTime = value; }
-        }
+        [JsonProperty(PropertyName = "created_at")]
+        [JsonConverter(typeof(DateTimeJsonConverter), "yyyy-MM-dd'T'HH:mm:ss.ffffff")]
+        public DateTime CreatedAt { get; set; }
 
-        /// <summary>
-        /// Return the URL string source of the file. This will be converted to a stream
-        /// </summary>
-        /// <param name="config">Configuration object</param>
-        /// <param name="date">Date of this source file</param>
-        /// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param>
-        /// <returns>String URL of source file.</returns>
-        public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
-        {
-            var publicationDate = DateTimeUtilityFunctions.AddBusinessDays(date.Date, -1, false);
-
-            return new SubscriptionDataSource(
-                Path.Combine(
-                    Globals.DataFolder,
-                    "alternative",
-                    "regalytics",
-                    "articles",
-                    $"{publicationDate:yyyyMMdd}.json"
-                ),
-                SubscriptionTransportMedium.LocalFile
-            );
-        }
+        public override DateTime EndTime => Time.AddDays(1);
 
         /// <summary>
         /// Parses the data from the line provided and loads it into LEAN
@@ -139,10 +111,7 @@ public override BaseData Reader(SubscriptionDataConfig config, string line, Date
             // us around 08:00:00 Eastern Time.
 
             article.Symbol = config.Symbol;
-            article.Time = article.LatestUpdate;
-            article.EndTime = DateTimeUtilityFunctions.AddBusinessDays(article.LatestUpdate.Date, 1, false)
-                .AddHours(8)
-                .ConvertTo(TimeZones.NewYork, config.ExchangeTimeZone);
+            article.Time = article.CreatedAt.Date;
 
             return article;
         }
@@ -177,6 +146,7 @@ public override BaseData Clone()
                 States = States,
                 Agencies = Agencies,
                 AnnouncementUrl = AnnouncementUrl,
+                CreatedAt = CreatedAt
             };
         }
 

RegalyticsRegulatoryArticles.cs

@@ -0,0 +1,138 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using NodaTime;
+using QuantConnect.Data;
+using QuantConnect.Data.UniverseSelection;
+using QuantConnect.Util;
+
+namespace QuantConnect.DataSource
+{
+    /// <summary>
+    /// Regalytics Regulatory articles collection
+    /// </summary>
+    public class RegalyticsRegulatoryArticles : BaseDataCollection
+    {
+        private static readonly RegalyticsRegulatoryArticle _factory = new();
+        
+        /// <summary>
+        /// Return the URL string source of the file. This will be converted to a stream
+        /// </summary>
+        /// <param name="config">Configuration object</param>
+        /// <param name="date">Date of this source file</param>
+        /// <param name="isLiveMode">true if we're in live mode, false for backtesting mode</param>
+        /// <returns>String URL of source file.</returns>
+        public override SubscriptionDataSource GetSource(SubscriptionDataConfig config, DateTime date, bool isLiveMode)
+        {
+            return new SubscriptionDataSource(
+                Path.Combine(
+                    Globals.DataFolder,
+                    "alternative",
+                    "regalytics",
+                    "articles",
+                    $"{date:yyyyMMdd}.json"
+                ),
+                SubscriptionTransportMedium.LocalFile,
+                FileFormat.FoldingCollection
+            );
+        }
+
+        /// <summary>
+        /// Parses the data from the line provided and loads it into LEAN
+        /// </summary>
+        /// <param name="config">Subscription configuration</param>
+        /// <param name="line">Line of data</param>
+        /// <param name="date">Date</param>
+        /// <param name="isLiveMode">Is live mode</param>
+        /// <returns>New instance</returns>
+        public override BaseData Reader(SubscriptionDataConfig config, string line, DateTime date, bool isLiveMode)
+        {
+            return _factory.Reader(config, line, date, isLiveMode);
+        }
+        
+        /// <summary>
+        /// Clones the data
+        /// </summary>
+        /// <returns>A clone of the object</returns>
+        public override BaseData Clone()
+        {
+            return new RegalyticsRegulatoryArticles
+            {
+                Symbol = Symbol,
+                Time = Time,
+                EndTime = EndTime,
+                Data = Data?.ToList(point => point.Clone())
+            };
+        }
+
+        /// <summary>
+        /// Formats a string with QuiverCNBC data
+        /// </summary>
+        /// <returns>string containing QuiverCNBC information</returns>
+        public override string ToString()
+        {
+            return $"[{string.Join(",", Data.Select(data => data.ToString()))}]";
+        }
+
+        /// <summary>
+        /// Indicates whether the data source is tied to an underlying symbol and requires that corporate events be applied to it as well, such as renames and delistings
+        /// </summary>
+        /// <returns>false</returns>
+        public override bool RequiresMapping()
+        {
+            return _factory.RequiresMapping();
+        }
+
+        /// <summary>
+        /// Indicates whether the data is sparse.
+        /// If true, we disable logging for missing files
+        /// </summary>
+        /// <returns>true</returns>
+        public override bool IsSparseData()
+        {
+            return _factory.IsSparseData();
+        }
+
+        /// <summary>
+        /// Gets the default resolution for this data and security type
+        /// </summary>
+        public override Resolution DefaultResolution()
+        {
+            return _factory.DefaultResolution();
+        }
+
+        /// <summary>
+        /// Gets the supported resolution for this data and security type
+        /// </summary>
+        public override List<Resolution> SupportedResolutions()
+        {
+            return _factory.SupportedResolutions();
+        }
+
+        /// <summary>
+        /// Specifies the data time zone for this data type. This is useful for custom data types
+        /// </summary>
+        /// <returns>The <see cref="T:NodaTime.DateTimeZone" /> of this data type</returns>
+        public override DateTimeZone DataTimeZone()
+        {
+            return _factory.DataTimeZone();
+        }
+    }
+}

RegalyticsRegulatoryArticlesDataAlgorithm.cs

@@ -0,0 +1,71 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System.Linq;
+using QuantConnect.Algorithm;
+using QuantConnect.Data;
+using QuantConnect.DataSource;
+using QuantConnect.Util;
+
+namespace QuantConnect.DataLibrary.Tests
+{
+    public class RegalyticsDataAlgorithm : QCAlgorithm
+    {
+        private readonly string[] _negativeSentimentPhrases = new [] {"emergency rule", "proposed rule change", "development of rulemaking"};
+        private Symbol _symbol;
+        private Symbol _regalyticsSymbol; 
+        
+        public override void Initialize()
+        {
+            SetStartDate(2022, 7, 10);
+            SetEndDate(2022, 7, 15);
+            SetCash(100000);
+            
+            _symbol = AddEquity("SPY", Resolution.Daily).Symbol;
+            
+            // Requesting data
+            _regalyticsSymbol = AddData<RegalyticsRegulatoryArticles>("REG").Symbol;
+            
+            // Historical data
+            var history = History<RegalyticsRegulatoryArticles>(_regalyticsSymbol, 7, Resolution.Daily);
+            Debug($"We got {history.Count()} items from our history request");
+        }
+
+        public override void OnData(Slice slice)
+        {
+            var data = slice.Get<RegalyticsRegulatoryArticles>();
+            if (!data.IsNullOrEmpty())
+            {
+                foreach (var articles in data.Values)
+                {
+                    Log($"{Time} {articles.ToString()}");
+                    foreach (RegalyticsRegulatoryArticle article in articles)
+                    {
+                        // based on the custom data property we will buy or short the underlying equity
+                        if (_negativeSentimentPhrases.Any(p => article.Title.ToLower().Contains(p)))
+                        {
+                            SetHoldings(_symbol, -1);
+                        }
+                        else
+                        {
+                            SetHoldings(_symbol, 1);
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

RegalyticsRegulatoryArticlesDataAlgorithm.py

@@ -0,0 +1,53 @@
+#  QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+#  Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+# 
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+# 
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+from AlgorithmImports import *
+
+### <summary>
+### Example algorithm using the Regalytics data as a source of alpha
+### </summary>
+class RegalyticsDataAlgorithm(QCAlgorithm): 
+    
+    negative_sentiment_phrases = ['restrict', 'barrier', 'tariff']
+    
+    def Initialize(self):
+        ''' Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.'''
+        self.SetStartDate(2022, 7, 10)
+        self.SetEndDate(2022, 7, 15)
+        self.SetCash(100000);
+        
+        self.symbol = self.AddEquity("SPY", Resolution.Daily).Symbol
+            
+        # Requesting data
+        self.regalyticsSymbol = self.AddData(RegalyticsRegulatoryArticles, "REG").Symbol
+            
+        # Historical data
+        history = self.History(self.regalyticsSymbol, 7, Resolution.Daily)
+        self.Debug(f"We got {len(history)} items from our history request")
+
+    def OnData(self, slice):
+        ''' OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.
+        :param Slice slice: Slice object keyed by symbol containing the stock data
+        '''
+        data = slice.Get(RegalyticsRegulatoryArticles)
+        if not data: return
+        
+        for articles in data.values():
+            self.Log(articles.ToString())
+            
+            for article in articles:
+                # based on the custom data property we will buy or short the underlying equity
+                if any([p in article.Title.lower() for p in self.negative_sentiment_phrases]):
+                    self.SetHoldings(self.symbol, -1)
+                else:
+                    self.SetHoldings(self.symbol, 1)