Skip to content

Commit b79ecad

Browse files
RomazesRomazes
authored
Feature: support equity index security types for history requests (#6)
* feat: supported SecurityType's HashSet in SymbolMapper * feat: OHLC model and Json convert * feat: extension to generate DateRange by day interval * refactor: history request process Equity and EquityOption test:feat: history Equity test * test:refactor: helper class * feat: Price model and Json converter * feat: support history Index test:feat: history Index * feat: additional Log Warning in several cases test:feat: several new wrong history test cases * feat: handle response.StatusCode 472 * remove: validate StatusCode on 0 cuz duplication with StatusCode == Ok * feat: skip "Connected" msg in WS handle Message * feat: support of Index in GetLean SymbolMapper * feat: stop all previous subscription * feat: Un/S-ubscription process of Equity and Index * feat: new properties in models of WS responses * test:refactor: DataQueueHandler tests * refactor: OpenInterest resolution warning * feat: validate get distinct history data * feat: add OrderBook for HandleQuoteUpdates * fix: type of RequestId in WebSocketHeader entity * remove: Log Warning of OpenInterest * refactor: stop previous streaming subscriptions * fix: variable name in WebSocketTrade entity * feat: use ExchangeTimeZone for Symbols to get right update of ticks * update: json config with support new Security types * feat: new Wraps The Close() WS connection feat: new Send() msg to WS with increment process feat: call WS.Close() in Dispose() * REMOVE: new keyword in Close() rename: Close() to Close WSConnection() feat: add Log.Debug when we have send Stop previous subs * feat: use NewYork.TimeZone in all application * feat: Implement SetJob pattern * refactor: Base REST API Url to support different version and host uri * fix: duplication of Api version in BaseURL * refactor: blow up period in GetIndexHistory() * refactor: TryGetExchangeOrDefault() * feat: flag that exchange doesn't find log msg only once * remove: default param in GetMessageOption * refactor: create orderBook for symbol before we call Subscription
1 parent 32cf5e0 commit b79ecad

18 files changed

Lines changed: 1191 additions & 183 deletions

‎QuantConnect.ThetaData.Tests/TestHelpers.cs‎

Lines changed: 31 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -29,30 +29,41 @@ namespace QuantConnect.Lean.DataSource.ThetaData.Tests
2929
{
3030
public static class TestHelpers
3131
{
32+
/// <summary>
33+
/// Represents the time zone used by ThetaData, which returns time in the New York (EST) Time Zone with daylight savings time.
34+
/// </summary>
35+
/// <remarks>
36+
/// <see href="https://http-docs.thetadata.us/docs/theta-data-rest-api-v2/ke230k18g7fld-trading-hours"/>
37+
/// </remarks>
38+
private static DateTimeZone TimeZoneThetaData = TimeZones.NewYork;
39+
3240
public static void ValidateHistoricalBaseData(IEnumerable<BaseData> history, Resolution resolution, TickType tickType, DateTime startDate, DateTime endDate, Symbol requestedSymbol = null)
3341
{
3442
Assert.IsNotNull(history);
3543
Assert.IsNotEmpty(history);
3644

3745
if (resolution < Resolution.Daily)
3846
{
39-
Assert.That(history.First().Time.Date, Is.EqualTo(startDate.ConvertFromUtc(TimeZones.EasternStandard).Date));
40-
Assert.That(history.Last().Time.Date, Is.EqualTo(endDate.ConvertFromUtc(TimeZones.EasternStandard).Date));
47+
Assert.That(history.First().Time.Date, Is.EqualTo(startDate.ConvertFromUtc(TimeZoneThetaData).Date));
48+
Assert.That(history.Last().Time.Date, Is.EqualTo(endDate.ConvertFromUtc(TimeZoneThetaData).Date));
4149
}
4250
else
4351
{
44-
Assert.That(history.First().Time.Date, Is.GreaterThanOrEqualTo(startDate.ConvertFromUtc(TimeZones.EasternStandard).Date));
45-
Assert.That(history.Last().Time.Date, Is.LessThanOrEqualTo(endDate.ConvertFromUtc(TimeZones.EasternStandard).Date));
52+
Assert.That(history.First().Time.Date, Is.GreaterThanOrEqualTo(startDate.ConvertFromUtc(TimeZoneThetaData).Date));
53+
Assert.That(history.Last().Time.Date, Is.LessThanOrEqualTo(endDate.ConvertFromUtc(TimeZoneThetaData).Date));
4654
}
4755

4856
switch (tickType)
4957
{
5058
case TickType.Trade when resolution != Resolution.Tick:
5159
AssertTradeBars(history.Select(x => x as TradeBar), requestedSymbol, resolution.ToTimeSpan());
5260
break;
53-
case TickType.Trade:
61+
case TickType.Trade when requestedSymbol.SecurityType != SecurityType.Index:
5462
AssertTradeTickBars(history.Select(x => x as Tick), requestedSymbol);
5563
break;
64+
case TickType.Trade when requestedSymbol.SecurityType == SecurityType.Index:
65+
AssertIndexTradeTickBars(history.Select(x => x as Tick), requestedSymbol);
66+
break;
5667
case TickType.Quote when resolution == Resolution.Tick:
5768
AssertQuoteTickBars(history.Select(x => x as Tick), requestedSymbol);
5869
break;
@@ -77,6 +88,19 @@ public static void AssertTradeTickBars(IEnumerable<Tick> ticks, Symbol symbol =
7788
}
7889
}
7990

91+
public static void AssertIndexTradeTickBars(IEnumerable<Tick> ticks, Symbol symbol = null)
92+
{
93+
foreach (var tick in ticks)
94+
{
95+
if (symbol != null)
96+
{
97+
Assert.That(tick.Symbol, Is.EqualTo(symbol));
98+
}
99+
100+
Assert.That(tick.Price, Is.GreaterThan(0));
101+
}
102+
}
103+
80104
public static void AssertQuoteTickBars(IEnumerable<Tick> ticks, Symbol symbol = null)
81105
{
82106
foreach (var tick in ticks)
@@ -143,7 +167,7 @@ public static void AssertTradeBars(IEnumerable<TradeBar> tradeBars, Symbol symbo
143167
Assert.That(tradeBar.Low, Is.GreaterThan(0));
144168
Assert.That(tradeBar.Close, Is.GreaterThan(0));
145169
Assert.That(tradeBar.Price, Is.GreaterThan(0));
146-
Assert.That(tradeBar.Volume, Is.GreaterThan(0));
170+
Assert.That(tradeBar.Volume, Is.GreaterThanOrEqualTo(0));
147171
Assert.That(tradeBar.Time, Is.GreaterThan(default(DateTime)));
148172
Assert.That(tradeBar.EndTime, Is.GreaterThan(default(DateTime)));
149173
}
@@ -174,7 +198,7 @@ public static HistoryRequest CreateHistoryRequest(Symbol symbol, Resolution reso
174198
null,
175199
true,
176200
false,
177-
DataNormalizationMode.Adjusted,
201+
DataNormalizationMode.Raw,
178202
tickType
179203
);
180204
}

‎QuantConnect.ThetaData.Tests/ThetaDataHistoryProviderTests..cs‎

Lines changed: 57 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,12 @@ public class ThetaDataHistoryProviderTests
2525
{
2626
ThetaDataProvider _thetaDataProvider = new();
2727

28-
29-
[TestCase("AAPL", SecurityType.Option, Resolution.Hour, TickType.OpenInterest, "2024/03/18", "2024/03/28", Description = "Wrong Resolution for OpenInterest")]
3028
[TestCase("AAPL", SecurityType.Option, Resolution.Hour, TickType.OpenInterest, "2024/03/28", "2024/03/18", Description = "StartDate > EndDate")]
31-
[TestCase("AAPL", SecurityType.Equity, Resolution.Hour, TickType.OpenInterest, "2024/03/28", "2024/03/18", Description = "Wrong SecurityType")]
29+
[TestCase("AAPL", SecurityType.Equity, Resolution.Hour, TickType.OpenInterest, "2024/03/28", "2024/04/02", Description = "Wrong TickType")]
30+
[TestCase("AAPL", SecurityType.Equity, Resolution.Daily, TickType.Trade, "2024/07/07", "2024/07/08", Description = "Use Weekend data, No data return")]
3231
[TestCase("AAPL", SecurityType.FutureOption, Resolution.Hour, TickType.Trade, "2024/03/28", "2024/03/18", Description = "Wrong SecurityType")]
32+
[TestCase("SPY", SecurityType.Index, Resolution.Hour, TickType.OpenInterest, "2024/03/28", "2024/04/02", Description = "Wrong TickType, Index support only Trade")]
33+
[TestCase("SPY", SecurityType.Index, Resolution.Minute, TickType.Quote, "2024/03/28", "2024/04/02", Description = "Wrong TickType, Index support only Trade")]
3334
public void TryGetHistoryDataWithInvalidRequestedParameters(string ticker, SecurityType securityType, Resolution resolution, TickType tickType, DateTime startDate, DateTime endDate)
3435
{
3536
var symbol = TestHelpers.CreateSymbol(ticker, securityType, OptionRight.Call, 170, new DateTime(2024, 03, 28));
@@ -60,5 +61,58 @@ public void GetHistoryOptionData(string ticker, OptionRight optionRight, decimal
6061

6162
TestHelpers.ValidateHistoricalBaseData(history, resolution, tickType, startDate, endDate, symbol);
6263
}
64+
65+
[TestCase("AAPL", Resolution.Tick, TickType.Trade, "2024/07/02", "2024/07/30", Explicit = true, Description = "Skipped: Long execution time")]
66+
[TestCase("AAPL", Resolution.Tick, TickType.Quote, "2024/07/26", "2024/07/30", Explicit = true, Description = "Skipped: Long execution time")]
67+
[TestCase("AAPL", Resolution.Tick, TickType.Trade, "2024/07/26", "2024/07/30")]
68+
[TestCase("AAPL", Resolution.Second, TickType.Trade, "2024/07/02", "2024/07/30")]
69+
[TestCase("AAPL", Resolution.Second, TickType.Quote, "2024/07/02", "2024/07/30")]
70+
[TestCase("AAPL", Resolution.Minute, TickType.Trade, "2024/07/02", "2024/07/30")]
71+
[TestCase("AAPL", Resolution.Minute, TickType.Quote, "2024/07/02", "2024/07/30")]
72+
[TestCase("AAPL", Resolution.Hour, TickType.Trade, "2024/07/26", "2024/07/30")]
73+
[TestCase("AAPL", Resolution.Hour, TickType.Trade, "2024/07/02", "2024/07/30")]
74+
[TestCase("AAPL", Resolution.Hour, TickType.Quote, "2024/07/02", "2024/07/30")]
75+
[TestCase("AAPL", Resolution.Daily, TickType.Trade, "2024/07/01", "2024/07/30")]
76+
[TestCase("AAPL", Resolution.Daily, TickType.Quote, "2024/07/01", "2024/07/30")]
77+
public void GetHistoryEquityData(string ticker, Resolution resolution, TickType tickType, DateTime startDate, DateTime endDate)
78+
{
79+
var symbol = TestHelpers.CreateSymbol(ticker, SecurityType.Equity);
80+
81+
var historyRequest = TestHelpers.CreateHistoryRequest(symbol, resolution, tickType, startDate, endDate);
82+
83+
var history = _thetaDataProvider.GetHistory(historyRequest).ToList();
84+
85+
TestHelpers.ValidateHistoricalBaseData(history, resolution, tickType, startDate, endDate, symbol);
86+
}
87+
88+
[TestCase("SPX", Resolution.Tick, TickType.Trade, "2024/07/02", "2024/07/30")]
89+
[TestCase("SPX", Resolution.Second, TickType.Trade, "2024/07/02", "2024/07/30")]
90+
[TestCase("SPX", Resolution.Minute, TickType.Trade, "2024/07/02", "2024/07/30")]
91+
[TestCase("SPX", Resolution.Hour, TickType.Trade, "2024/07/02", "2024/07/30")]
92+
[TestCase("SPX", Resolution.Daily, TickType.Trade, "2024/07/01", "2024/07/30")]
93+
public void GetHistoryIndexData(string ticker, Resolution resolution, TickType tickType, DateTime startDate, DateTime endDate)
94+
{
95+
var symbol = TestHelpers.CreateSymbol(ticker, SecurityType.Index);
96+
97+
var historyRequest = TestHelpers.CreateHistoryRequest(symbol, resolution, tickType, startDate, endDate);
98+
99+
var history = _thetaDataProvider.GetHistory(historyRequest).ToList();
100+
101+
TestHelpers.ValidateHistoricalBaseData(history, resolution, tickType, startDate, endDate, symbol);
102+
}
103+
104+
[TestCase("AAPL", Resolution.Tick, TickType.Trade, "2024/07/24", "2024/07/26", Explicit = true, Description = "Skipped: Long execution time")]
105+
public void GetHistoryTickTradeValidateOnDistinctData(string ticker, Resolution resolution, TickType tickType, DateTime startDate, DateTime endDate)
106+
{
107+
var symbol = TestHelpers.CreateSymbol(ticker, SecurityType.Equity);
108+
109+
var historyRequest = TestHelpers.CreateHistoryRequest(symbol, resolution, tickType, startDate, endDate);
110+
111+
var history = _thetaDataProvider.GetHistory(historyRequest).ToList();
112+
113+
var distinctHistory = history.Distinct().ToList();
114+
115+
Assert.That(history.Count, Is.EqualTo(distinctHistory.Count));
116+
}
63117
}
64118
}

‎QuantConnect.ThetaData.Tests/ThetaDataProviderTests.cs‎

Lines changed: 24 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -54,31 +54,12 @@ public void TearDown()
5454
}
5555
}
5656

57-
[TestCase("AAPL", SecurityType.Equity)]
58-
[TestCase("VIX", SecurityType.Index)]
59-
public void SubscribeWithWrongInputParameters(string ticker, SecurityType securityType)
60-
{
61-
var symbol = TestHelpers.CreateSymbol(ticker, securityType);
62-
var configs = GetSubscriptionDataConfigs(symbol, Resolution.Minute).ToList();
63-
64-
var isNotSubscribed = new List<bool>();
65-
foreach (var config in configs)
66-
{
67-
if (_thetaDataProvider.Subscribe(config, (sender, args) => { }) == null)
68-
{
69-
isNotSubscribed.Add(false);
70-
}
71-
}
72-
73-
Assert.That(configs.Count, Is.EqualTo(isNotSubscribed.Count));
74-
Assert.IsFalse(isNotSubscribed.Contains(true), "One of config is subscribed successfully.");
75-
}
76-
77-
[TestCase("AAPL", Resolution.Second, 170, "2024/04/19")]
78-
[TestCase("NVDA", Resolution.Second, 890, "2024/04/12")]
79-
public void CanSubscribeAndUnsubscribeOnSecondResolution(string ticker, Resolution resolution, decimal strikePrice, DateTime expiryDate)
57+
[TestCase("AAPL", SecurityType.Equity, Resolution.Second, 0, "2024/08/16")]
58+
[TestCase("AAPL", SecurityType.Option, Resolution.Second, 215, "2024/08/16")]
59+
[TestCase("VIX", SecurityType.Index, Resolution.Second, 0, "2024/08/16")]
60+
public void CanSubscribeAndUnsubscribeOnSecondResolution(string ticker, SecurityType securityType, Resolution resolution, decimal strikePrice, DateTime expiryDate = default)
8061
{
81-
var configs = GetSubscriptionDataConfigs(ticker, resolution, strikePrice, expiryDate);
62+
var configs = GetSubscriptionDataConfigs(ticker, securityType, resolution, strikePrice, expiryDate);
8263

8364
Assert.That(configs, Is.Not.Empty);
8465

@@ -111,7 +92,7 @@ public void CanSubscribeAndUnsubscribeOnSecondResolution(string ticker, Resoluti
11192
}), _cancellationTokenSource.Token, callback: callback);
11293
}
11394

114-
Thread.Sleep(TimeSpan.FromSeconds(10));
95+
Thread.Sleep(TimeSpan.FromSeconds(25));
11596

11697
Log.Trace("Unsubscribing symbols");
11798
foreach (var config in configs)
@@ -121,21 +102,34 @@ public void CanSubscribeAndUnsubscribeOnSecondResolution(string ticker, Resoluti
121102

122103
Thread.Sleep(TimeSpan.FromSeconds(5));
123104

124-
Assert.Greater(dataFromEnumerator[typeof(QuoteBar)], 0);
105+
_cancellationTokenSource.Cancel();
106+
107+
Log.Trace($"{nameof(ThetaDataProviderTests)}.{nameof(CanSubscribeAndUnsubscribeOnSecondResolution)}: ***** Summary *****");
108+
Log.Trace($"Input parameters: ticker:{ticker} | securityType:{securityType} | resolution:{resolution}");
109+
110+
foreach (var data in dataFromEnumerator)
111+
{
112+
Log.Trace($"[{data.Key}] = {data.Value}");
113+
}
114+
115+
if (securityType != SecurityType.Index)
116+
{
117+
Assert.Greater(dataFromEnumerator[typeof(QuoteBar)], 0);
118+
}
125119
// The ThetaData returns TradeBar seldom. Perhaps should find more relevant ticker.
126120
Assert.GreaterOrEqual(dataFromEnumerator[typeof(TradeBar)], 0);
127121
}
128122

129123
[TestCase("AAPL", SecurityType.Equity)]
130-
[TestCase("VIX", SecurityType.Index)]
124+
[TestCase("SPX", SecurityType.Index)]
131125
public void MultipleSubscriptionOnOptionContractsTickResolution(string ticker, SecurityType securityType)
132126
{
133127
var minReturnResponse = 5;
134128
var obj = new object();
135129
var cancellationTokenSource = new CancellationTokenSource();
136130
var resetEvent = new AutoResetEvent(false);
137131
var underlyingSymbol = TestHelpers.CreateSymbol(ticker, securityType);
138-
var configs = _thetaDataProvider.LookupSymbols(underlyingSymbol, false).SelectMany(x => GetSubscriptionTickDataConfigs(x)).Take(500).ToList();
132+
var configs = _thetaDataProvider.LookupSymbols(underlyingSymbol, false).SelectMany(x => GetSubscriptionTickDataConfigs(x)).Take(250).ToList();
139133

140134
var incomingSymbolDataByTickType = new ConcurrentDictionary<(Symbol, TickType), int>();
141135

@@ -196,10 +190,10 @@ public void MultipleSubscriptionOnOptionContractsTickResolution(string ticker, S
196190
cancellationTokenSource.Cancel();
197191
}
198192

199-
private static IEnumerable<SubscriptionDataConfig> GetSubscriptionDataConfigs(string ticker, Resolution resolution, decimal strikePrice, DateTime expiry,
193+
private static IEnumerable<SubscriptionDataConfig> GetSubscriptionDataConfigs(string ticker, SecurityType securityType, Resolution resolution, decimal strikePrice, DateTime expiry,
200194
OptionRight optionRight = OptionRight.Call, string market = Market.USA)
201195
{
202-
var symbol = TestHelpers.CreateSymbol(ticker, SecurityType.Option, optionRight, strikePrice, expiry, market);
196+
var symbol = TestHelpers.CreateSymbol(ticker, securityType, optionRight, strikePrice, expiry, market);
203197
foreach (var subscription in GetSubscriptionDataConfigs(symbol, resolution))
204198
{
205199
yield return subscription;

‎QuantConnect.ThetaData/Converters/ThetaDataOpenHighLowCloseConverter.cs‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
/*
2+
* QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
3+
* Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software
10+
* distributed under the License is distributed on an "AS IS" BASIS,
11+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
* See the License for the specific language governing permissions and
13+
* limitations under the License.
14+
*/
15+
16+
using Newtonsoft.Json;
17+
using Newtonsoft.Json.Linq;
18+
using QuantConnect.Lean.DataSource.ThetaData.Models.Rest;
19+
using System.Globalization;
20+
21+
namespace QuantConnect.Lean.DataSource.ThetaData.Converters;
22+
23+
/// <summary>
24+
/// JSON converter to convert ThetaData OpenHighLowClose
25+
/// </summary>
26+
public class ThetaDataOpenHighLowCloseConverter : JsonConverter<OpenHighLowCloseResponse>
27+
{
28+
/// <summary>
29+
/// Gets a value indicating whether this <see cref="JsonConverter"/> can write JSON.
30+
/// </summary>
31+
/// <value><c>true</c> if this <see cref="JsonConverter"/> can write JSON; otherwise, <c>false</c>.</value>
32+
public override bool CanWrite => false;
33+
34+
/// <summary>
35+
/// Gets a value indicating whether this <see cref="JsonConverter"/> can read JSON.
36+
/// </summary>
37+
/// <value><c>true</c> if this <see cref="JsonConverter"/> can read JSON; otherwise, <c>false</c>.</value>
38+
public override bool CanRead => true;
39+
40+
/// <summary>
41+
/// Writes the JSON representation of the object.
42+
/// </summary>
43+
/// <param name="writer">The <see cref="JsonWriter"/> to write to.</param>
44+
/// <param name="value">The value.</param>
45+
/// <param name="serializer">The calling serializer.</param>
46+
public override void WriteJson(JsonWriter writer, OpenHighLowCloseResponse value, JsonSerializer serializer)
47+
{
48+
throw new NotSupportedException();
49+
}
50+
51+
/// <summary>
52+
/// Reads the JSON representation of the object.
53+
/// </summary>
54+
/// <param name="reader">The <see cref="JsonReader"/> to read from.</param>
55+
/// <param name="objectType">Type of the object.</param>
56+
/// <param name="existingValue">The existing value of object being read.</param>
57+
/// <param name="hasExistingValue">The existing value has a value.</param>
58+
/// <param name="serializer">The calling serializer.</param>
59+
/// <returns>The object value.</returns>
60+
public override OpenHighLowCloseResponse ReadJson(JsonReader reader, Type objectType, OpenHighLowCloseResponse existingValue, bool hasExistingValue, JsonSerializer serializer)
61+
{
62+
var token = JToken.Load(reader);
63+
if (token.Type != JTokenType.Array || token.Count() != 8) throw new Exception($"{nameof(ThetaDataQuoteConverter)}.{nameof(ReadJson)}: Invalid token type or count. Expected a JSON array with exactly four elements.");
64+
65+
return new OpenHighLowCloseResponse(
66+
timeMilliseconds: token[0]!.Value<uint>(),
67+
open: token[1]!.Value<decimal>(),
68+
high: token[2]!.Value<decimal>(),
69+
low: token[3]!.Value<decimal>(),
70+
close: token[4]!.Value<decimal>(),
71+
volume: token[5]!.Value<decimal>(),
72+
count: token[6]!.Value<uint>(),
73+
date: DateTime.ParseExact(token[7]!.ToString(), "yyyyMMdd", CultureInfo.InvariantCulture));
74+
}
75+
}

0 commit comments

Comments
 (0)