Skip to content

Commit 9684df3

Browse files
jhonabreuljhonabreul
authored
Initial options implementation (#2)
* First version of data processor * Data processing * Add symbol mapper implementation * Fix option chain implementation and add unit tests * Improve and fix history provider and add unit tests * Implemented data downloader and add unit tests * Improvements on option chain provider. This allows to overcome the factset beta api timeouts in some of their endpoints * Move options details batch requests to api * Fixes * Minor api improvements * Address peer review * Address peer review * Address peer review * Fixes and improvements. Raw data storage improvements * Minor fixes in raw data storage * Minor fixes * Add Lean Tests project to solution * Minor fix * Minor fix * GH actions fix * Address self review * Minor fix * Address self review * fix: tests dll path in GH actions feat: get factset auth config either from json object or file path * feat: support quote historical data requests * feat: quote data support in data processort * Improvements * Minor improvement * minor fix: dispose of options details blocking collection
1 parent 6ae7f21 commit 9684df3

40 files changed

+2816
-1687
lines changed

.github/workflows/build.yml renamed to .github/workflows/gh-actions.yml

Lines changed: 14 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -3,23 +3,17 @@ name: Build & Test
33
on:
44
push:
55
branches: ['*']
6-
pull_request:
7-
branches: [master]
86

97
jobs:
108
build:
119
runs-on: ubuntu-20.04
1210

1311
steps:
14-
- uses: actions/checkout@v2
12+
- name: Checkout
13+
uses: actions/checkout@v2
1514

1615
- name: Free space
17-
run: df -h && rm -rf /opt/hostedtoolcache* && df -h
18-
19-
- name: Pull Foundation Image
20-
uses: addnab/docker-run-action@v3
21-
with:
22-
image: quantconnect/lean:foundation
16+
run: df -h && sudo rm -rf /usr/local/lib/android && sudo rm -rf /opt/ghc && rm -rf /opt/hostedtoolcache* && df -h
2317

2418
- name: Checkout Lean Same Branch
2519
id: lean-same-branch
@@ -40,11 +34,14 @@ jobs:
4034
- name: Move Lean
4135
run: mv Lean ../Lean
4236

43-
- name: BuildDataSource
44-
run: dotnet build ./QuantConnect.DataSource.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1
45-
46-
- name: BuildTests
47-
run: dotnet build ./tests/Tests.csproj /p:Configuration=Release /v:quiet /p:WarningLevel=1
48-
49-
- name: Run Tests
50-
run: dotnet test ./tests/bin/Release/net6.0/Tests.dll
37+
- name: Run Image
38+
uses: addnab/docker-run-action@v3
39+
with:
40+
image: quantconnect/lean:foundation
41+
options: -v /home/runner/work:/__w --workdir /__w/Lean.DataSource.FactSet/Lean.DataSource.FactSet
42+
shell: bash
43+
run: |
44+
# Build
45+
dotnet build /p:Configuration=Release /v:quiet /p:WarningLevel=1 QuantConnect.DataSource.FactSet.sln
46+
# Run Tests
47+
dotnet test ./tests/bin/Release/net6.0/QuantConnect.DataLibrary.Tests.dll

‎DataProcessing/DataProcessing.csproj‎

Lines changed: 16 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,17 @@
1-
<Project Sdk="Microsoft.NET.Sdk">
2-
<PropertyGroup>
3-
<OutputType>Exe</OutputType>
4-
<TargetFramework>net6.0</TargetFramework>
5-
<AssemblyName>process</AssemblyName>
6-
<CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
7-
</PropertyGroup>
8-
<ItemGroup>
9-
<PackageReference Include="QuantConnect.Lean.Engine" Version="2.5.*"/>
10-
<PackageReference Include="QuantConnect.Research" Version="2.5.*"/>
11-
<PackageReference Include="QuantConnect.Messaging" Version="2.5.*"/>
12-
<PackageReference Include="QuantConnect.pythonnet" Version="2.0.*" />
13-
<PackageReference Include="fasterflect" Version="3.0.0" />
14-
</ItemGroup>
15-
<ItemGroup>
16-
<ProjectReference Include="..\QuantConnect.DataSource.csproj" />
17-
</ItemGroup>
18-
<ItemGroup>
19-
<None Remove="CLRImports.py" />
20-
<Content Include="CLRImports.py">
21-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
22-
</Content>
23-
</ItemGroup>
24-
<ItemGroup>
25-
<None Remove="config.json" />
26-
<Content Include="config.json">
27-
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
28-
</Content>
29-
</ItemGroup>
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
<PropertyGroup>
3+
<OutputType>Exe</OutputType>
4+
<TargetFramework>net6.0</TargetFramework>
5+
<AssemblyName>process</AssemblyName>
6+
<CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
7+
</PropertyGroup>
8+
<ItemGroup>
9+
<ProjectReference Include="..\QuantConnect.DataSource.csproj" />
10+
</ItemGroup>
11+
<ItemGroup>
12+
<None Remove="config.json" />
13+
<Content Include="config.json">
14+
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
15+
</Content>
16+
</ItemGroup>
3017
</Project>

‎DataProcessing/FactSetDataProcessingDataDownloader.cs‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
/*
2+
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
3+
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software
10+
* distributed under the License is distributed on an "AS IS" BASIS,
11+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
* See the License for the specific language governing permissions and
13+
* limitations under the License.
14+
*/
15+
16+
using QuantConnect.Lean.DataSource.FactSet;
17+
using System.Collections.Generic;
18+
using System;
19+
20+
namespace QuantConnect.DataProcessing
21+
{
22+
/// <summary>
23+
/// The only purpose of this class is to provide a way to access the <see cref="FactSetDataDownloader"/> class and use its constructor
24+
/// that takes the raw data folder as a parameter.
25+
///
26+
/// This is done this way in order to keep the public interface of the <see cref="FactSetDataDownloader"/> clean and simple,
27+
/// without exposing its additional capabilities of storing the raw data downloaded from FactSet, which is only useful
28+
/// for the Data Processing program.
29+
/// </summary>
30+
internal class FactSetDataProcessingDataDownloader : FactSetDataDownloader
31+
{
32+
/// <summary>
33+
/// Initializes a new instance of the <see cref="FactSetDataProcessingDataDownloader"/>
34+
/// </summary>
35+
public FactSetDataProcessingDataDownloader(FactSet.SDK.Utils.Authentication.Configuration factSetAuthConfiguration, string rawDataFolder)
36+
: base(new FactSetDataProcessingDataProvider(factSetAuthConfiguration, rawDataFolder))
37+
{
38+
}
39+
40+
/// <summary>
41+
/// Get the option chains for the specified symbol and time range.
42+
/// Exposed to be used by the Data Processing program.
43+
/// </summary>
44+
public IEnumerable<Symbol> GetOptionChains(Symbol symbol, DateTime startUtc, DateTime endUtc)
45+
{
46+
return base.GetOptions(symbol, startUtc, endUtc);
47+
}
48+
}
49+
}

‎DataProcessing/FactSetDataProcessingDataProvider.cs‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
/*
2+
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
3+
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software
10+
* distributed under the License is distributed on an "AS IS" BASIS,
11+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
* See the License for the specific language governing permissions and
13+
* limitations under the License.
14+
*/
15+
16+
using QuantConnect.Lean.DataSource.FactSet;
17+
18+
namespace QuantConnect.DataProcessing
19+
{
20+
/// <summary>
21+
/// The only purpose of this class is to provide a way to access the <see cref="FactSetDataProvider"/> class and use its constructor
22+
/// that takes the raw data folder as a parameter.
23+
///
24+
/// This is done this way in order to keep the public interface of the <see cref="FactSetDataProvider"/> clean and simple,
25+
/// without exposing its additional capabilities of storing the raw data downloaded from FactSet, which is only useful
26+
/// for the Data Processing program.
27+
/// </summary>
28+
internal class FactSetDataProcessingDataProvider : FactSetDataProvider
29+
{
30+
/// <summary>
31+
/// Initializes a new instance of the <see cref="FactSetDataProcessingDataProvider"/>
32+
/// </summary>
33+
public FactSetDataProcessingDataProvider(FactSet.SDK.Utils.Authentication.Configuration factSetAuthConfig, string rawDataFolder)
34+
: base(factSetAuthConfig, rawDataFolder)
35+
{
36+
}
37+
}
38+
}

‎DataProcessing/FactSetDataProcessor.cs‎

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
/*
2+
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
3+
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software
10+
* distributed under the License is distributed on an "AS IS" BASIS,
11+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
* See the License for the specific language governing permissions and
13+
* limitations under the License.
14+
*/
15+
16+
using System;
17+
using System.Collections.Generic;
18+
using System.Diagnostics;
19+
using System.IO;
20+
using System.Linq;
21+
using System.Threading;
22+
using System.Threading.Tasks;
23+
using QuantConnect.Data;
24+
using QuantConnect.Logging;
25+
using QuantConnect.Util;
26+
27+
namespace QuantConnect.DataProcessing
28+
{
29+
/// <summary>
30+
/// Helper class to download, process and store FactSet data
31+
/// </summary>
32+
public class FactSetDataProcessor : IDisposable
33+
{
34+
private readonly string _destinationFolder;
35+
private readonly string _rawDataFolder;
36+
private readonly List<string> _tickerWhitelist;
37+
private readonly FactSet.SDK.Utils.Authentication.Configuration _factSetAuthConfig;
38+
private readonly FactSetDataProcessingDataDownloader _downloader;
39+
40+
private List<Symbol> _symbols;
41+
private Resolution _resolution;
42+
private DateTime _startDate;
43+
private DateTime _endDate;
44+
45+
/// <summary>
46+
/// Creates a new instance of the <see cref="FactSetDataProcessor"/> class.
47+
/// </summary>
48+
/// <param name="factSetAuthConfig">The FactSet authentication configuration</param>
49+
/// <param name="symbols">The symbols to download data for</param>
50+
/// <param name="resolution">The resolution of the data to download</param>
51+
/// <param name="startDate">The start date of the data to download</param>
52+
/// <param name="endDate">The end date of the data to download</param>
53+
/// <param name="destinationFolder">The destination folder to save the data</param>
54+
/// <param name="rawDataFolder">The raw data folder</param>
55+
/// <param name="tickerWhitelist">A list of supported tickers</param>
56+
public FactSetDataProcessor(FactSet.SDK.Utils.Authentication.Configuration factSetAuthConfig, List<Symbol> symbols, Resolution resolution,
57+
DateTime startDate, DateTime endDate, string destinationFolder, string rawDataFolder, List<string> tickerWhitelist = null)
58+
{
59+
_factSetAuthConfig = factSetAuthConfig;
60+
_symbols = symbols;
61+
_resolution = resolution;
62+
_startDate = startDate;
63+
_endDate = endDate;
64+
_destinationFolder = destinationFolder;
65+
_rawDataFolder = rawDataFolder;
66+
_tickerWhitelist = tickerWhitelist ?? new List<string>();
67+
68+
if (_symbols.Any(symbol => !symbol.SecurityType.IsOption() || !symbol.IsCanonical() || !_tickerWhitelist.Contains(symbol.Underlying.Value)))
69+
{
70+
throw new ArgumentException("The symbols must be canonical option symbols and the underlying must be in the whitelist.");
71+
}
72+
73+
_downloader = new FactSetDataProcessingDataDownloader(_factSetAuthConfig, _rawDataFolder);
74+
}
75+
76+
/// <summary>
77+
/// Disposes of the resources
78+
/// </summary>
79+
public void Dispose()
80+
{
81+
_downloader.DisposeSafely();
82+
}
83+
84+
/// <summary>
85+
/// Runs the instance of the object.
86+
/// </summary>
87+
/// <returns>True if process all downloads successfully</returns>
88+
public bool Run()
89+
{
90+
var stopwatch = Stopwatch.StartNew();
91+
92+
// Let's get the options ourselves so the data downloader doesn't have to do it for each tick type
93+
var symbolsStr = string.Join(", ", _symbols.Select(symbol => symbol.Value));
94+
Log.Trace($"FactSetDataProcessor.Run(): Fetching options for {symbolsStr}.");
95+
var options = _symbols.Select(symbol => _downloader.GetOptionChains(symbol, _startDate, _startDate)).SelectMany(x => x).ToList();
96+
97+
Log.Trace($"FactSetDataProcessor.Run(): Found {options.Count} options.");
98+
Log.Trace($"FactSetDataProcessor.Run(): Start downloading/processing {symbolsStr} {_resolution} data.");
99+
100+
var tickTypes = new[] { TickType.Trade, TickType.Quote, TickType.OpenInterest };
101+
var source = options.Select(option => tickTypes.Select(tickType => (option, tickType))).SelectMany(x => x);
102+
103+
var result = Parallel.ForEach(source, new ParallelOptions() { MaxDegreeOfParallelism = 16 }, (t, loopState) =>
104+
{
105+
var option = t.option;
106+
var tickType = t.tickType;
107+
108+
var data = _downloader.Get(new DataDownloaderGetParameters(option, _resolution, _startDate, _endDate, tickType));
109+
if (data == null)
110+
{
111+
Log.Trace($"FactSetDataProcessor.Run(): No {tickType} data found for {symbolsStr}.");
112+
loopState.Stop();
113+
return;
114+
}
115+
116+
var tradesWriter = new LeanDataWriter(_resolution, option, _destinationFolder, tickType);
117+
tradesWriter.Write(data);
118+
});
119+
120+
if (!result.IsCompleted)
121+
{
122+
Log.Error($"FactSetDataProcessor.Run(): Failed to download/processing {symbolsStr} {_resolution} data.");
123+
return false;
124+
}
125+
126+
Log.Trace($"FactSetDataProcessor.Run(): Finished in {stopwatch.Elapsed.ToStringInvariant(null)}");
127+
return true;
128+
}
129+
}
130+
}

0 commit comments

Comments
 (0)