Updates to V3 (#10)

The new endpoint, https://api.regalytics.ai/api/v3, changes the format:
'articles' to 'results', 'states' to 'state' and 'countries' to 'country'. The 'id' from 'int' to 'string'.

Author: AlexCatarino

.github/workflows/build.yml

@@ -8,11 +8,18 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-20.04
-    container:
-        image: quantconnect/lean:foundation
+    runs-on: ubuntu-20.04    
     steps:
-      - uses: actions/checkout@v2
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Free space
+        run: df -h && rm -rf /opt/hostedtoolcache* && df -h
+
+      - name: Pull Foundation Image
+        uses: addnab/docker-run-action@v3
+        with:
+          image: quantconnect/lean:foundation
 
       - name: BuildDataSource
         run: dotnet build ./QuantConnect.DataSource.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1

QuantConnect.DataSource.csproj

@@ -9,7 +9,7 @@
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="QuantConnect.Common" Version="2.5.*" />
-    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
   </ItemGroup>
 
   <ItemGroup>

RegalyticsRegulatoryArticle.cs

@@ -34,7 +34,7 @@ public class RegalyticsRegulatoryArticle : BaseData
         public static int DataSourceId { get; } = 2030;
 
         [JsonProperty(PropertyName = "id")]
-        public int Id { get; set; }
+        public string Id { get; set; }
 
         [JsonProperty(PropertyName = "title")]
         public string Title { get; set; }

process.py

@@ -1,25 +1,22 @@
-import json
-import pathlib
+from json import dumps
+from pathlib import Path
 from datetime import datetime, timezone
-import os
-import requests
+from os import environ
+from requests import post
 
-URL = os.environ["REGALYTICS_API_BASE_URL"]
-HEADERS = {
-    'Content-Type': 'application/json'
-}
-ARTICLE_PATH = pathlib.Path('/temp-output-directory/alternative/regalytics/articles')
+URL = environ.get("REGALYTICS_API_BASE_URL", "https://api.regalytics.ai/api/v3")
+API_KEY = environ.get("REGALYTICS_API_KEY", "")
+DEPLOYMENT_DATE = environ.get('QC_DATAFLEET_DEPLOYMENT_DATE', f'{datetime.now():%Y%m%d}')
 
 # objectives:# download data from API -> temp folder or in memory. Output processed datat  to /temp-output-directory/alternative/regalytics/articles/yyyyMMdd.json
+ARTICLE_PATH = Path('/temp-output-directory/alternative/regalytics/articles')
 ARTICLE_PATH.mkdir(parents=True, exist_ok=True)
 articles_by_date = {}
 
-process_datetime = datetime.strptime(os.environ['QC_DATAFLEET_DEPLOYMENT_DATE'], '%Y%m%d').date()
-process_date = process_datetime.strftime('%Y-%m-%d')
+process_date = datetime.strptime(DEPLOYMENT_DATE, '%Y%m%d').strftime('%Y-%m-%d')
 
-url = f"{URL}/search"
-payload = json.dumps({
-    "apikey": os.environ["REGALYTICS_API_KEY"],
+payload = dumps({
+    "apikey": API_KEY,
     "search_options": {
         "created_at": {
             "start": process_date,
@@ -28,18 +25,17 @@
     }
 })
 
-response = requests.post(url, headers=HEADERS, data=payload).json()
-articles = response['articles']
+response = post(f"{URL}/search", headers={ 'Content-Type': 'application/json' }, data=payload).json()
 
 # "agencies": [
 #     {
 #         "name": "Iowa Department of Human Services",
-#         "states": [
+#         "state": [
 #             {
 #                 "name": "Iowa"
 #             }
 #         ],
-#         "countries": [
+#         "country": [
 #             {
 #                 "name": "United States"
 #             }
@@ -51,26 +47,18 @@
 # 1. query all data, -> /api/v2/.../get-all; 2. look at latest_update, add delta of 1/2 days;
 # 3. write data to date of latest_update + delta. This date must be on the date we published the article on Regalytics
 
-for article in articles:
+for article in response.get('results', []):
     article['in_federal_register'] = 'yes' in article['in_federal_register'].lower()
     # State -> Dictionary<string, List<string>>
     states = {}
-    for agency in article['agencies']:
-        state = agency['states']
-        
-        if 'states' not in agency or state is None:
+    for agency in article.get('agencies', []):
+        state = agency.get('state')
+        if not state:
             continue
 
-        if 'countries' not in agency:
-            continue
-
-        countries = agency['countries']
-        if countries is None:
-            continue
-        
-        for country in countries:
+        for country in agency.get('country', []):
             name = country['name']
-            
+
             if not name in states:
                 country_states = []
                 states[name] = country_states
@@ -88,7 +76,7 @@
     article['created_at'] = article['created_at'][:-3] + article['created_at'][-2:]       # %z only accepts `-0400` instead of `-04:00` in Python3.6
     created_at = datetime.strptime(article['created_at'], '%Y-%m-%dT%H:%M:%S.%f%z').astimezone(timezone.utc)
     article['created_at'] = created_at.strftime('%Y-%m-%dT%H:%M:%S.%f')
-    date_key = created_at.date().strftime('%Y%m%d')
+    date_key = created_at.strftime('%Y%m%d')
 
     if date_key not in articles_by_date:
         date_articles = []
@@ -99,11 +87,6 @@
     date_articles.append(article)
 
 for date, articles in articles_by_date.items():
-    lines = []
-    for article in articles:
-        lines.append(json.dumps(article, indent=None))
-
-    article_lines = '\n'.join(lines)
-
     with open(ARTICLE_PATH / f'{date}.json', 'w') as article_file:
+        article_lines = '\n'.join([dumps(article, indent=None) for article in articles])
         article_file.write(article_lines)

tests/RegalyticsRegulatoryArticleTests.cs

@@ -55,7 +55,18 @@ public void CloneCollection()
 
             AssertAreEqual(expected, result);
         }
-        
+
+        [Test]
+        public void BackwardCompatibilityToV2()
+        {
+            // The Id is a number in v2
+            var line = "{\"id\": 2051381, \"title\": \"House of Representatives Study Bill HSB692: A bill for an act relating to school security, including by modifying provisions related to the issuance of school bonds, requiring schools to conduct school safety reviews and have access to the statewide interoperable communications system, establishing the school emergency radio access grant program and the firearm detection software grant program within the department of homeland security and emergency management, requiring the department of public (I)\", \"summary\": \"Introduced on 2024-02-12. House Study Bill 692 is a piece of legislation in Iowa that focuses on school security. It includes provisions related to the issuance of school bonds, requires schools to conduct safety reviews and have access to a statewide communications system, establishes grant programs for school emergency radio access and firearm detection software, and requires the Department of Public Safety to convene a task force on school safety standards. The bill also appropriates funds for these programs and specifies that the state cost of compliance with the legislation will be paid by school districts from state school foundation aid. The bill takes effect upon enactment. (99IA202320242022HSB692)\", \"status\": \"New\", \"classification\": \"State\", \"filing_type\": \"Single\", \"in_federal_register\": false, \"federal_register_number\": null, \"regalytics_alert_id\": \"99IA2022HSB69225120240212\", \"proposed_comments_due_date\": null, \"original_publication_date\": \"2024-02-12\", \"federal_register_publication_date\": null, \"rule_effective_date\": null, \"latest_update\": \"2024-02-12\", \"alert_type\": \"Study Bill\", \"docket_file_number\": \"\", \"order_notice\": \"\", \"sec_release_number\": \"\", \"agencies\": [\"Iowa House of Representatives\"], \"sector_type\": [{\"name\": \"Financial\"}], \"tags\": [{\"name\": \"All State and Federal Legislatures\"}, {\"name\": \"Introduced Bill\"}], \"subtype_classification\": [{\"name\": \"House of Representatives Study Bill\", \"higher_order_alert_classification\": {\"name\": \"Rule\"}}], \"pdf_url\": \"https://www.legis.iowa.gov/legislation/BillBook?ga=90&ba=HSB692\", \"created_at\": \"2024-02-12T22:31:40.567008\", \"states\": {\"United States\": [\"Iowa\"]}}";
+            var instance = new RegalyticsRegulatoryArticle();
+            var config = new SubscriptionDataConfig(instance.GetType(), Symbol.None, Resolution.Daily, TimeZones.Utc, TimeZones.Utc, false, false, false);
+            var data = instance.Reader(config, line, new DateTime(2024, 2, 12), false) as RegalyticsRegulatoryArticle;
+            Assert.AreEqual("2051381", data.Id);
+        }
+
         private void AssertAreEqual(object expected, object result, bool filterByCustomAttributes = false)
         {
             foreach (var propertyInfo in expected.GetType().GetProperties())
@@ -80,7 +91,7 @@ private BaseData CreateNewInstance()
                 Time = DateTime.Today,
                 DataType = MarketDataType.Base,
 
-                Id = 0,
+                Id = "0",
                 Title = "string",
                 Summary = "string",
                 Status = "string",
@@ -111,7 +122,7 @@ private BaseData CreateNewCollectionInstance()
                     Time = DateTime.Today,
                     DataType = MarketDataType.Base,
 
-                    Id = 0,
+                    Id = "0",
                     Title = "string",
                     Summary = "string",
                     Status = "string",
@@ -136,7 +147,7 @@ private BaseData CreateNewCollectionInstance()
                     Time = DateTime.Today,
                     DataType = MarketDataType.Base,
 
-                    Id = 0,
+                    Id = "0",
                     Title = "string",
                     Summary = "string",
                     Status = "string",