diff --git a/AcquiredDisposedCode.cs b/AcquiredDisposedCode.cs new file mode 100644 index 0000000..7046cf2 --- /dev/null +++ b/AcquiredDisposedCode.cs @@ -0,0 +1,46 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System.Runtime.Serialization; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; + +namespace QuantConnect.DataSource.QuiverQuant +{ + /// + /// SEC Form 4 indicator of whether the transaction was an acquisition or a disposal + /// + [JsonConverter(typeof(StringEnumConverter))] + public enum AcquiredDisposedCode + { + /// + /// Default value used when no acquired/disposed flag is provided or the value is unrecognized + /// + [EnumMember(Value = "")] + Unknown, + + /// + /// A - Share acquisition + /// + [EnumMember(Value = "A")] + Acquired, + + /// + /// D - Share disposal + /// + [EnumMember(Value = "D")] + Disposed, + } +} diff --git a/DataProcessing/Program.cs b/DataProcessing/Program.cs index 52fb61e..0582238 100644 --- a/DataProcessing/Program.cs +++ b/DataProcessing/Program.cs @@ -26,40 +26,52 @@ namespace QuantConnect.DataProcessing /// public class Program { + private static readonly string VendorName = QuiverDataDownloader.VendorName; /// /// Entrypoint of the program /// /// Exit code. 0 equals successful, and any other value indicates the downloader/converter failed. - public static void Main(string[] args) + public static void Main() { - var dataset = args[0]; + var dataset = Config.Get("vendor-data-name", "cnbc").Trim().ToLowerInvariant(); var destinationDirectory = Path.Combine( Config.Get("temp-output-directory", "/temp-output-directory"), "alternative"); var processedDataDirectory = Path.Combine( Config.Get("processed-data-directory", Globals.DataFolder), "alternative"); - var processingDateValue = Config.Get("processing-date", Environment.GetEnvironmentVariable("QC_DATAFLEET_DEPLOYMENT_DATE")); + var processingDateValue = Config.Get("processing-date", Environment.GetEnvironmentVariable("QC_DATAFLEET_DEPLOYMENT_DATE")) + ?? DateTime.UtcNow.AddDays(-1).ToString("yyyyMMdd"); + var processingDate = Parse.DateTimeExact(processingDateValue, "yyyyMMdd"); + var processingDateLookback = Config.GetInt("processing-date-lookback", 0); + var processingStartDate = processingDate.AddDays(-processingDateLookback); - switch (dataset.ToLowerInvariant()) + switch (dataset) { - case "cnbc": + case QuiverCNBCDataDownloader.VendorDataName: { - var processingDate = Parse.DateTimeExact(processingDateValue, "yyyyMMdd"); RunDownloader( - QuiverCNBCDataDownloader.VendorName, QuiverCNBCDataDownloader.VendorDataName, () => new QuiverCNBCDataDownloader(destinationDirectory, processedDataDirectory), - instance => instance.Run(processingDate)); + instance => + { + for (var date = processingStartDate; date <= processingDate; date = date.AddDays(1)) + { + if (!instance.Run(date)) + { + Log.Error($"QuantConnect.DataProcessing.Program.Main(): Failed to download/process " + + $"{VendorName} {QuiverCNBCDataDownloader.VendorDataName} data for date: {date:yyyy-MM-dd}"); + } + } + instance.Flush(); + instance.ProcessUniverse(); + return true; + }); break; } - case "governmentcontract": + case QuiverGovernmentContractDownloader.VendorDataName: { - var processingDate = string.IsNullOrWhiteSpace(processingDateValue) - ? DateTime.UtcNow.AddDays(-1) - : Parse.DateTimeExact(processingDateValue, "yyyyMMdd"); - var datasetStartDate = new DateTime(2022, 4, 21); if (processingDate < datasetStartDate) { @@ -68,7 +80,6 @@ public static void Main(string[] args) } RunDownloader( - QuiverGovernmentContractDownloader.VendorName, QuiverGovernmentContractDownloader.VendorDataName, () => new QuiverGovernmentContractDownloader(), instance => @@ -77,7 +88,7 @@ public static void Main(string[] args) if (!success) { Log.Error($"QuantConnect.DataProcessing.Program.Main(): Failed to download/process " + - $"{QuiverGovernmentContractDownloader.VendorName} {QuiverGovernmentContractDownloader.VendorDataName} data for date: {processingDate:yyyy-MM-dd}"); + $"{VendorName} {QuiverGovernmentContractDownloader.VendorDataName} data for date: {processingDate:yyyy-MM-dd}"); } instance.ProcessUniverse(); return true; @@ -85,60 +96,73 @@ public static void Main(string[] args) break; } - case "lobbying": + case QuiverLobbyingDataDownloader.VendorDataName: { - var processingDate = Parse.DateTimeExact(processingDateValue, "yyyyMMdd"); RunDownloader( - QuiverLobbyingDataDownloader.VendorName, QuiverLobbyingDataDownloader.VendorDataName, () => new QuiverLobbyingDataDownloader(destinationDirectory, processedDataDirectory), instance => instance.Run(processingDate)); break; } - case "congresstrading": + case QuiverCongressDataDownloader.VendorDataName: { - var congressDestination = Path.Combine(destinationDirectory, "quiver"); RunDownloader( - QuiverCongressDataDownloader.VendorName, QuiverCongressDataDownloader.VendorDataName, - () => new QuiverCongressDataDownloader(congressDestination), + () => new QuiverCongressDataDownloader(destinationDirectory), instance => instance.Run()); break; } - case "wallstreetbets": + case QuiverWallStreetBetsDataDownloader.VendorDataName: { - var tempOutput = Config.Get("temp-output-directory", "/temp-output-directory"); RunDownloader( - QuiverWallStreetBetsDataDownloader.VendorName, QuiverWallStreetBetsDataDownloader.VendorDataName, - () => new QuiverWallStreetBetsDataDownloader(tempOutput), + () => new QuiverWallStreetBetsDataDownloader(destinationDirectory), instance => instance.Run()); break; } - case "insidertrading": + case QuiverInsiderTradingDataDownloader.VendorDataName: { - var processingStartDate = GetDateConfig("processing-start-date"); - var processingEndDate = GetDateConfig("processing-end-date"); RunDownloader( - QuiverInsiderTradingDataDownloader.VendorName, QuiverInsiderTradingDataDownloader.VendorDataName, () => new QuiverInsiderTradingDataDownloader(destinationDirectory, processedDataDirectory), - instance => instance.Run(processingStartDate, processingEndDate)); + instance => + { + for (var date = processingStartDate; date <= processingDate; date = date.AddDays(1)) + { + if (!instance.Run(date)) + { + Log.Error($"QuantConnect.DataProcessing.Program.Main(): Failed to download/process " + + $"{VendorName} {QuiverInsiderTradingDataDownloader.VendorDataName} data for date: {date:yyyy-MM-dd}"); + } + } + instance.Flush(); + instance.ProcessUniverse(); + return true; + }); break; } default: - Log.Error($"Unknown dataset '{dataset}'"); + var validDatasets = string.Join(", ", new[] + { + QuiverCNBCDataDownloader.VendorDataName, + QuiverGovernmentContractDownloader.VendorDataName, + QuiverLobbyingDataDownloader.VendorDataName, + QuiverCongressDataDownloader.VendorDataName, + QuiverWallStreetBetsDataDownloader.VendorDataName, + QuiverInsiderTradingDataDownloader.VendorDataName, + }); + Log.Error($"Unknown dataset '{dataset}'. Valid options: {validDatasets}"); break; } Environment.Exit(0); } - private static void RunDownloader(string vendorName, string vendorDataName, Func factory, Func run) where T : class, IDisposable + private static void RunDownloader(string vendorDataName, Func factory, Func run) where T : class, IDisposable { T instance = null; try @@ -147,7 +171,7 @@ private static void RunDownloader(string vendorName, string vendorDataName, F } catch (Exception err) { - Log.Error(err, $"QuantConnect.DataProcessing.Program.Main(): The downloader/converter for {vendorName} {vendorDataName} data failed to be constructed"); + Log.Error(err, $"QuantConnect.DataProcessing.Program.Main(): The downloader/converter for {VendorName} {vendorDataName} data failed to be constructed"); Environment.Exit(1); } @@ -155,13 +179,13 @@ private static void RunDownloader(string vendorName, string vendorDataName, F { if (!run(instance)) { - Log.Error($"QuantConnect.DataProcessing.Program.Main(): Failed to download/process {vendorName} {vendorDataName} data"); + Log.Error($"QuantConnect.DataProcessing.Program.Main(): Failed to download/process {VendorName} {vendorDataName} data"); Environment.Exit(1); } } catch (Exception err) { - Log.Error(err, $"QuantConnect.DataProcessing.Program.Main(): The downloader/converter for {vendorName} {vendorDataName} data exited unexpectedly"); + Log.Error(err, $"QuantConnect.DataProcessing.Program.Main(): The downloader/converter for {VendorName} {vendorDataName} data exited unexpectedly"); Environment.Exit(1); } finally @@ -169,12 +193,5 @@ private static void RunDownloader(string vendorName, string vendorDataName, F instance.DisposeSafely(); } } - - private static DateTime GetDateConfig(string configKey) - { - var value = Config.Get(configKey, Environment.GetEnvironmentVariable("QC_DATAFLEET_DEPLOYMENT_DATE")) - ?? DateTime.Today.ToString("yyyyMMdd"); - return Parse.DateTimeExact(value, "yyyyMMdd"); - } } } \ No newline at end of file diff --git a/DataProcessing/QuiverCNBCDataDownloader.cs b/DataProcessing/QuiverCNBCDataDownloader.cs index 6a58e04..e40032b 100644 --- a/DataProcessing/QuiverCNBCDataDownloader.cs +++ b/DataProcessing/QuiverCNBCDataDownloader.cs @@ -23,6 +23,7 @@ using Newtonsoft.Json; using QuantConnect.Data.Auxiliary; using QuantConnect.DataSource; +using QuantConnect.DataSource.QuiverQuant; using QuantConnect.Lean.Engine.DataFeeds; using QuantConnect.Logging; using QuantConnect.Util; @@ -39,6 +40,7 @@ public class QuiverCNBCDataDownloader : QuiverDataDownloader private readonly string _destinationFolder; private readonly string _universeFolder; private readonly string _processedDataDirectory; + private readonly Dictionary> _cnbcByTicker = []; /// /// Creates a new instance of @@ -84,16 +86,10 @@ public bool Run(DateTime processDate) var cnbcByDate = JsonConvert.DeserializeObject>(quiverCnbcData, _jsonSerializerSettings); - var cnbcByTicker = new Dictionary>(); - var universeCsvContents = new List(); - - var mapFileProvider = new LocalZipMapFileProvider(); - mapFileProvider.Initialize(new DefaultDataProvider()); - foreach (var cnbc in cnbcByDate) { var ticker = cnbc.Ticker; - if (ticker == null) + if (ticker == null) { Log.Error($"QuiverCNBCDataDownloader.Run(): Null value for Ticker on {processDate:yyyyMMdd}"); continue; @@ -111,29 +107,113 @@ public bool Run(DateTime processDate) continue; } - if (!cnbcByTicker.TryGetValue(ticker, out var _)) + var note = SanitizeCsv(cnbc.Notes); + var traders = SanitizeCsv(cnbc.Traders); + var curRow = $"{cnbc.Direction.ToCsv()},{traders},{note}"; + var uploadDate = cnbc.UploadDate?.Date ?? processDate; + // csv[0] is always the uploadDate. csv[1] (adviceDate) is omitted when it equals uploadDate; + // Reader falls back to uploadedDate in that case. + var adviceDateCol = uploadDate == processDate + ? string.Empty + : $"{processDate:yyyyMMdd}"; + var line = $"{uploadDate:yyyyMMdd},{adviceDateCol},{curRow}"; + + if (!_cnbcByTicker.TryGetValue(ticker, out var lines)) { - cnbcByTicker.Add(ticker, new List()); + _cnbcByTicker[ticker] = lines = []; } + lines.Add(line); + } + } + catch (Exception e) + { + Log.Error(e); + return false; + } - var note = cnbc.Notes != null ? cnbc.Notes.Replace(Environment.NewLine, string.Empty).Trim() : null; - var curRow = $"{note},{cnbc.Direction},{cnbc.Traders.Trim()}"; - cnbcByTicker[ticker].Add($"{processDate:yyyyMMdd},{curRow}"); + Log.Trace($"QuiverCNBCDataDownloader.Run(): Finished in {stopwatch.Elapsed.ToStringInvariant(null)}"); + return true; + } - var sid = SecurityIdentifier.GenerateEquity(ticker, Market.USA, true, mapFileProvider, processDate); - universeCsvContents.Add($"{sid},{ticker},{curRow}"); + /// + /// Writes every accumulated per-ticker batch to disk, merging with any pre-existing file. + /// + /// True on success + public bool Flush() + { + try + { + foreach (var kvp in _cnbcByTicker) + { + SaveContentToFile(_destinationFolder, kvp.Key, kvp.Value); } + } + catch (Exception e) + { + Log.Error(e); + return false; + } + return true; + } + + /// + /// Regenerates the universe files keyed by upload date by reading every per-ticker file. + /// + /// True if the universe files were regenerated successfully + public bool ProcessUniverse() + { + if (!_canCreateUniverseFiles) + { + Log.Trace($"QuiverCNBCDataDownloader.ProcessUniverse(): Map files not available, skipping universe generation"); + return false; + } + + var stopwatch = Stopwatch.StartNew(); + Log.Trace($"QuiverCNBCDataDownloader.ProcessUniverse(): Start regenerating universe files by upload date"); + + try + { + var mapFileProvider = new LocalZipMapFileProvider(); + mapFileProvider.Initialize(new DefaultDataProvider()); + + Dictionary> dataByUploadDate = []; - if (!_canCreateUniverseFiles) + void processFile(string filePath) { - return false; + var ticker = Path.GetFileNameWithoutExtension(filePath).ToUpperInvariant(); + foreach (var line in File.ReadAllLines(filePath)) + { + var firstComma = line.IndexOf(','); + if (firstComma <= 0) continue; + + var uploadDateStr = line[..firstComma]; + if (!DateTime.TryParseExact(uploadDateStr, "yyyyMMdd", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal, out var uploadDate)) + { + continue; + } + var rest = line[(firstComma + 1)..]; + + if (!dataByUploadDate.TryGetValue(uploadDate, out var data)) + { + dataByUploadDate[uploadDate] = data = []; + } + + var sid = SecurityIdentifier.GenerateEquity(ticker, Market.USA, true, mapFileProvider, uploadDate); + data.Add($"{sid},{ticker},{rest}"); + } } - else if (universeCsvContents.Any()) + + if (Directory.Exists(_processedDataDirectory)) { - SaveContentToFile(_universeFolder, $"{processDate:yyyyMMdd}", universeCsvContents); + Directory.EnumerateFiles(_processedDataDirectory, "*.csv").DoForEach(processFile); } + Directory.EnumerateFiles(_destinationFolder, "*.csv").DoForEach(processFile); - cnbcByTicker.DoForEach(kvp => SaveContentToFile(_destinationFolder, kvp.Key, kvp.Value)); + dataByUploadDate.DoForEach(kvp => + { + var filePath = Path.Combine(_universeFolder, $"{kvp.Key:yyyyMMdd}.csv"); + File.WriteAllLines(filePath, kvp.Value.OrderBy(x => x)); + }); } catch (Exception e) { @@ -141,46 +221,45 @@ public bool Run(DateTime processDate) return false; } - Log.Trace($"QuiverCNBCDataDownloader.Run(): Finished in {stopwatch.Elapsed.ToStringInvariant(null)}"); + Log.Trace($"QuiverCNBCDataDownloader.ProcessUniverse(): Finished in {stopwatch.Elapsed.ToStringInvariant(null)}"); return true; } /// - /// Saves contents to disk, deleting existing zip files + /// Saves per-ticker contents to disk, merging with any pre-existing file /// /// Final destination of the data /// file name /// Contents to write - private void SaveContentToFile(string destinationFolder, string name, IEnumerable contents) + private static string SanitizeCsv(string value) { - name = name.ToLowerInvariant(); - var finalPath = Path.Combine(destinationFolder, $"{name}.csv"); - string filePath; - - if (destinationFolder.Contains("universe")) + if (string.IsNullOrEmpty(value)) { - filePath = Path.Combine(_processedDataDirectory, "universe", $"{name}.csv"); - } - else - { - filePath = Path.Combine(_processedDataDirectory, $"{name}.csv"); + return string.Empty; } + return value.Replace(",", string.Empty).Replace("\r", string.Empty).Replace("\n", string.Empty).Trim(); + } - var finalFileExists = File.Exists(filePath); + private void SaveContentToFile(string destinationFolder, string name, IEnumerable contents) + { + name = name.ToLowerInvariant(); + var finalPath = Path.Combine(destinationFolder, $"{name}.csv"); + var filePath = Path.Combine(_processedDataDirectory, $"{name}.csv"); - var lines = new HashSet(contents); - if (finalFileExists) + HashSet lines = [.. contents]; + foreach (var path in new[] { filePath, finalPath }) { - foreach (var line in File.ReadAllLines(filePath)) + if (File.Exists(path)) { - lines.Add(line); + foreach (var line in File.ReadAllLines(path)) + { + lines.Add(line); + } } } - var finalLines = destinationFolder.Contains("universe") ? - lines.OrderBy(x => x.Split(',').First()).ToList() : - lines - .OrderBy(x => DateTime.ParseExact(x.Split(',').First(), "yyyyMMdd", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal)) + var finalLines = lines + .OrderBy(x => DateTime.ParseExact(x.Split(',')[0], "yyyyMMdd", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal)) .ToList(); File.WriteAllLines(finalPath, finalLines); @@ -200,6 +279,13 @@ private class RawCNBC: QuiverCNBC /// [JsonProperty(PropertyName = "Ticker")] public string Ticker { get; set; } + + /// + /// The date this data was uploaded to QuiverQuant's database + /// + [JsonProperty(PropertyName = "upload_time")] + [JsonConverter(typeof(DateTimeJsonConverter), "yyyy-MM-dd")] + public DateTime? UploadDate { get; set; } } } diff --git a/DataProcessing/QuiverCongressDataDownloader.cs b/DataProcessing/QuiverCongressDataDownloader.cs index 87ca5db..7323e4c 100644 --- a/DataProcessing/QuiverCongressDataDownloader.cs +++ b/DataProcessing/QuiverCongressDataDownloader.cs @@ -57,7 +57,7 @@ public class QuiverCongressDataDownloader : QuiverDataDownloader public QuiverCongressDataDownloader(string destinationFolder, string apiKey = null) : base(100, TimeSpan.FromSeconds(60), apiKey) { - _destinationFolder = Directory.CreateDirectory(Path.Combine(destinationFolder, VendorDataName)).FullName; + _destinationFolder = Directory.CreateDirectory(Path.Combine(destinationFolder, VendorName, VendorDataName)).FullName; _universeFolder = Directory.CreateDirectory(Path.Combine(_destinationFolder, "universe")).FullName; } diff --git a/DataProcessing/QuiverInsiderTradingDataDownloader.cs b/DataProcessing/QuiverInsiderTradingDataDownloader.cs index 95d6dea..f091c33 100644 --- a/DataProcessing/QuiverInsiderTradingDataDownloader.cs +++ b/DataProcessing/QuiverInsiderTradingDataDownloader.cs @@ -19,9 +19,11 @@ using System.Globalization; using System.IO; using System.Linq; +using System.Text.RegularExpressions; using Newtonsoft.Json; using QuantConnect.Data.Auxiliary; using QuantConnect.DataSource; +using QuantConnect.DataSource.QuiverQuant; using QuantConnect.Lean.Engine.DataFeeds; using QuantConnect.Logging; using QuantConnect.Util; @@ -38,6 +40,8 @@ public class QuiverInsiderTradingDataDownloader : QuiverDataDownloader private readonly string _destinationFolder; private readonly string _universeFolder; private readonly string _processedDataDirectory; + private readonly Dictionary> _insiderTradingByTicker = []; + private static readonly List _defunctDelimiters = new() { '-', @@ -69,33 +73,14 @@ public QuiverInsiderTradingDataDownloader() } /// - /// Runs the instance of the object with a given date. - /// - /// First date of data to be fetched and processed - /// Last date of data to be fetched and processed - /// True if process last downloads successfully - public bool Run(DateTime processingStartDate, DateTime processingEndDate) - { - var success = false; - - for (var processDate= processingStartDate; processDate<= processingEndDate; processDate = processDate.AddDays(1)) - { - success = Run(processDate); - } - - return success; - } - - /// - /// Runs the instance of the object with a given date. + /// Fetches a single day of insider trading data and accumulates it per-ticker in memory. /// - /// The date of data to be fetched and processed - /// True if process all downloads successfully + /// The date of data to be fetched + /// True if the day was fetched and parsed successfully public bool Run(DateTime processDate) { - var symbolsProcessed = new List(); var stopwatch = Stopwatch.StartNew(); - Log.Trace($"QuiverInsiderTradingDataDownloader.Run(): Start downloading/processing QuiverQuant Insider Trading data"); + Log.Trace($"QuiverInsiderTradingDataDownloader.Run(): Start downloading QuiverQuant Insider Trading data for {processDate:yyyy-MM-dd}"); var today = DateTime.UtcNow.Date; try @@ -105,74 +90,167 @@ public bool Run(DateTime processDate) Log.Trace($"Encountered data from invalid date: {processDate:yyyy-MM-dd} - Skipping"); return false; } - + var quiverInsiderTradingData = HttpRequester($"live/insiders?date={processDate:yyyyMMdd}").SynchronouslyAwaitTaskResult(); if (string.IsNullOrWhiteSpace(quiverInsiderTradingData)) { - // We've already logged inside HttpRequester return false; } var insiderTradingByDate = JsonConvert.DeserializeObject>(quiverInsiderTradingData, _jsonSerializerSettings); - var insiderTradingByTicker = new Dictionary>(); - var universeCsvContents = new List(); - - var mapFileProvider = new LocalZipMapFileProvider(); - mapFileProvider.Initialize(new DefaultDataProvider()); - foreach (var insiderTrade in insiderTradingByDate) { var quiverTicker = insiderTrade.Ticker; if (quiverTicker == null) continue; + if (insiderTrade.Uploaded == null) + { + Log.Trace($"QuiverInsiderTradingDataDownloader.Run(): Skipping row with null Uploaded for ticker {quiverTicker} on {processDate:yyyyMMdd}"); + continue; + } + if (!TryNormalizeDefunctTicker(quiverTicker, out var tickerList)) { - Log.Error( - $"QuiverInsiderTradingDataDownloader.Run(): Defunct ticker {quiverTicker} is unable to be parsed. Continuing..."); + Log.Error($"QuiverInsiderTradingDataDownloader.Run(): Defunct ticker {quiverTicker} is unable to be parsed. Continuing..."); continue; } - foreach (var ticker in tickerList) + var uploadedDate = insiderTrade.Uploaded.Value.Date; + // Omit fileDate when its calendar day matches uploaded. Reader falls back to uploadedDate, + // preserving the day but dropping intraday precision (acceptable trade-off for storage). + var fileDate = insiderTrade.FileDate?.Date == uploadedDate + ? string.Empty + : insiderTrade.FileDate?.ToString("yyyyMMddHHmmss", CultureInfo.InvariantCulture) ?? string.Empty; + var transactionDate = insiderTrade.Date?.ToString("yyyyMMdd", CultureInfo.InvariantCulture) ?? string.Empty; + var officerTitle = SanitizeCsv(insiderTrade.OfficerTitle); + var transactionCode = insiderTrade.TransactionCode.ToCsv(); + var ownership = insiderTrade.DirectOrIndirectOwnership.ToCsv(); + var acquiredDisposed = insiderTrade.AcquiredDisposedCode.ToCsv(); + + var line = $"{uploadedDate:yyyyMMdd},{fileDate},{transactionDate}," + + $"{transactionCode},{insiderTrade.PricePerShare},{insiderTrade.Shares},{insiderTrade.SharesOwnedFollowing}," + + $"{acquiredDisposed},{ownership},{officerTitle}," + + $"{insiderTrade.IsDirector.ToCsv()},{insiderTrade.IsOfficer.ToCsv()},{insiderTrade.IsTenPercentOwner.ToCsv()},{insiderTrade.IsOther.ToCsv()}"; + + foreach (var rawTicker in tickerList) { - var sid = default(SecurityIdentifier); - try + var ticker = Regex.Replace(rawTicker, @"[^A-Z0-9.]", string.Empty).Trim('.'); + if (!Regex.IsMatch(ticker, @"^[A-Z0-9][A-Z0-9.]*[A-Z0-9]$") && !Regex.IsMatch(ticker, @"^[A-Z0-9]$")) { - sid = SecurityIdentifier.GenerateEquity(ticker, Market.USA, true, mapFileProvider, processDate); + Log.Trace($"QuiverInsiderTradingDataDownloader.Run(): Skipping invalid ticker '{rawTicker}' on {processDate:yyyyMMdd}"); + continue; } - catch (Exception) + if (!_insiderTradingByTicker.TryGetValue(ticker, out var lines)) + { + _insiderTradingByTicker[ticker] = lines = []; + } + lines.Add(line); + } + } + } + catch (Exception e) + { + Log.Error(e); + return false; + } + + Log.Trace($"QuiverInsiderTradingDataDownloader.Run(): Finished in {stopwatch.Elapsed.ToStringInvariant(null)}"); + return true; + } + + /// + /// Writes every accumulated per-ticker batch to disk, merging with any pre-existing file. + /// + /// True on success + public bool Flush() + { + var failed = 0; + foreach (var kvp in _insiderTradingByTicker) + { + try + { + SaveContentToFile(kvp.Key, kvp.Value); + } + catch (Exception e) + { + failed++; + Log.Error(e, $"QuiverInsiderTradingDataDownloader.Flush(): Failed to write data for ticker '{kvp.Key}'"); + } + } + return failed == 0; + } + + /// + /// Regenerates the universe files keyed by upload date by reading every per-ticker file. + /// + /// True if the universe files were regenerated successfully + public bool ProcessUniverse() + { + if (!_canCreateUniverseFiles) + { + Log.Trace("QuiverInsiderTradingDataDownloader.ProcessUniverse(): Map files not available, skipping universe generation"); + return false; + } + + var stopwatch = Stopwatch.StartNew(); + Log.Trace("QuiverInsiderTradingDataDownloader.ProcessUniverse(): Start regenerating universe files by upload date"); + + try + { + var mapFileProvider = new LocalZipMapFileProvider(); + mapFileProvider.Initialize(new DefaultDataProvider()); + + Dictionary> dataByUploadDate = []; + + void processFile(string filePath) + { + var ticker = Path.GetFileNameWithoutExtension(filePath).ToUpperInvariant(); + foreach (var line in File.ReadAllLines(filePath)) + { + var firstComma = line.IndexOf(','); + if (firstComma <= 0) continue; + + var uploadDateStr = line[..firstComma]; + if (!DateTime.TryParseExact(uploadDateStr, "yyyyMMdd", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal, out var uploadDate)) { - Log.Error($"QuiverInsiderTradingDataDownloader.Run(): Invalid ticker {ticker}. Continuing..."); continue; } + var rest = line[(firstComma + 1)..]; - symbolsProcessed.Add(ticker); - - if (sid.Date == SecurityIdentifier.DefaultDate || sid.ToString().Contains(" 2T")) continue; + if (!dataByUploadDate.TryGetValue(uploadDate, out var data)) + { + dataByUploadDate[uploadDate] = data = []; + } - if (!insiderTradingByTicker.TryGetValue(ticker, out var _)) + SecurityIdentifier sid; + try { - insiderTradingByTicker.Add(ticker, new List()); + sid = SecurityIdentifier.GenerateEquity(ticker, Market.USA, true, mapFileProvider, uploadDate); + } + catch (Exception) + { + Log.Error($"QuiverInsiderTradingDataDownloader.ProcessUniverse(): Invalid ticker {ticker} on {uploadDate:yyyyMMdd}. Skipping line."); + continue; } - var curRow = $"{insiderTrade.Name.Replace(",", string.Empty).Trim().ToLower()},{insiderTrade.Shares},{insiderTrade.PricePerShare},{insiderTrade.SharesOwnedFollowing}"; - insiderTradingByTicker[ticker].Add($"{processDate:yyyyMMdd},{curRow}"); + if (sid.Date == SecurityIdentifier.DefaultDate || sid.ToString().Contains(" 2T")) continue; - universeCsvContents.Add($"{sid},{ticker},{curRow}"); + data.Add($"{sid},{ticker},{rest}"); } } - if (!_canCreateUniverseFiles) - { - return false; - } - if (universeCsvContents.Any()) + if (Directory.Exists(_processedDataDirectory)) { - SaveContentToFile(_universeFolder, $"{processDate:yyyyMMdd}", universeCsvContents); + Directory.EnumerateFiles(_processedDataDirectory, "*.csv").DoForEach(processFile); } + Directory.EnumerateFiles(_destinationFolder, "*.csv").DoForEach(processFile); - insiderTradingByTicker.DoForEach(kvp => SaveContentToFile(_destinationFolder, kvp.Key, kvp.Value)); - Log.Trace($"QuiverInsiderTradingDataDownloader.Run(): Processed tickers for {processDate:yyyyMMdd} - {String.Join(", ", symbolsProcessed)}"); + dataByUploadDate.DoForEach(kvp => + { + var filePath = Path.Combine(_universeFolder, $"{kvp.Key:yyyyMMdd}.csv"); + File.WriteAllLines(filePath, kvp.Value.OrderBy(x => x)); + }); } catch (Exception e) { @@ -180,49 +258,49 @@ public bool Run(DateTime processDate) return false; } - Log.Trace($"QuiverInsiderTradingDataDownloader.Run(): Finished in {stopwatch.Elapsed.ToStringInvariant(null)}"); + Log.Trace($"QuiverInsiderTradingDataDownloader.ProcessUniverse(): Finished in {stopwatch.Elapsed.ToStringInvariant(null)}"); return true; } /// - /// Saves contents to disk, deleting existing zip files + /// Saves per-ticker contents to disk, merging with any pre-existing file. /// - /// Final destination of the data - /// File name + /// File name (ticker) /// Contents to write - private void SaveContentToFile(string destinationFolder, string name, IEnumerable contents) + private void SaveContentToFile(string name, IEnumerable contents) { - var finalPath = Path.Combine(destinationFolder, $"{name.ToLowerInvariant()}.csv"); - string filePath; + name = name.ToLowerInvariant(); + var finalPath = Path.Combine(_destinationFolder, $"{name}.csv"); + var existingPath = Path.Combine(_processedDataDirectory, $"{name}.csv"); - if (destinationFolder.Contains("universe")) - { - filePath = Path.Combine(_processedDataDirectory, "universe", $"{name}.csv"); - } - else + HashSet lines = [.. contents]; + foreach (var path in new[] { existingPath, finalPath }) { - filePath = Path.Combine(_processedDataDirectory, $"{name.ToLowerInvariant()}.csv"); - } - - var finalFileExists = File.Exists(filePath); - - var lines = new HashSet(contents); - if (finalFileExists) - { - foreach (var line in File.ReadAllLines(filePath)) + if (File.Exists(path)) { - lines.Add(line); + foreach (var line in File.ReadAllLines(path)) + { + lines.Add(line); + } } } - var finalLines = destinationFolder.Contains("universe") - ? lines.OrderBy(x => x) - : lines.OrderBy(x => DateTime.ParseExact(x.Split(',').First(), "yyyyMMdd", - CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal)); + var finalLines = lines + .OrderBy(x => DateTime.ParseExact(x.Split(',')[0], "yyyyMMdd", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal)) + .ToList(); File.WriteAllLines(finalPath, finalLines); } + private static string SanitizeCsv(string value) + { + if (string.IsNullOrEmpty(value)) + { + return string.Empty; + } + return value.Replace(",", string.Empty).Replace("\r", string.Empty).Replace("\n", string.Empty).Trim(); + } + /// /// Tries to normalize a potentially defunct ticker into a normal ticker. /// @@ -245,25 +323,18 @@ protected static bool TryNormalizeDefunctTicker(string rawTicker, out string[] t tickerList = ticker.Substring(0, length).Trim().Split(' '); return true; } - + tickerList = ticker.Split(' '); return true; } private class RawInsiderTrading : QuiverInsiderTrading { - /// - /// The time the data point ends at and becomes available to the algorithm - /// - [JsonProperty(PropertyName = "Date")] - [JsonConverter(typeof(DateTimeJsonConverter), "yyyy-MM-dd")] - public DateTime Date { get; set; } - - /// - /// The ticker/symbol for the company - /// [JsonProperty(PropertyName = "Ticker")] public string Ticker { get; set; } = null!; + + [JsonProperty(PropertyName = "uploaded")] + public DateTime? Uploaded { get; set; } } } diff --git a/DataProcessing/QuiverWallStreetBetsDataDownloader.cs b/DataProcessing/QuiverWallStreetBetsDataDownloader.cs index 97e1844..8b1b3b5 100644 --- a/DataProcessing/QuiverWallStreetBetsDataDownloader.cs +++ b/DataProcessing/QuiverWallStreetBetsDataDownloader.cs @@ -46,7 +46,7 @@ public class QuiverWallStreetBetsDataDownloader : QuiverDataDownloader public QuiverWallStreetBetsDataDownloader(string destinationFolder, string apiKey = null) : base(10, TimeSpan.FromSeconds(1.1), apiKey) { - _destinationFolder = Path.Combine(destinationFolder, "alternative", VendorName, VendorDataName); + _destinationFolder = Path.Combine(destinationFolder, VendorName, VendorDataName); _universeFolder = Path.Combine(_destinationFolder, "universe"); Directory.CreateDirectory(_destinationFolder); diff --git a/DataProcessing/config.json b/DataProcessing/config.json index 65f5a10..fdb03d4 100644 --- a/DataProcessing/config.json +++ b/DataProcessing/config.json @@ -1,4 +1,6 @@ { "data-folder": "../../Lean/Data/", - "quiver-auth-token": "" + "quiver-auth-token": "", + "processing-date-lookback": 0, + "vendor-data-name": "cnbc" } diff --git a/OwnershipType.cs b/OwnershipType.cs new file mode 100644 index 0000000..1a383a3 --- /dev/null +++ b/OwnershipType.cs @@ -0,0 +1,46 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System.Runtime.Serialization; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; + +namespace QuantConnect.DataSource.QuiverQuant +{ + /// + /// SEC Form 4 direct or indirect ownership classification + /// + [JsonConverter(typeof(StringEnumConverter))] + public enum OwnershipType + { + /// + /// Default value used when no ownership flag is provided or the value is unrecognized + /// + [EnumMember(Value = "")] + Unknown, + + /// + /// D - Direct ownership of the security by the reporting person + /// + [EnumMember(Value = "D")] + Direct, + + /// + /// I - Indirect ownership of the security (e.g., through a trust or family member) + /// + [EnumMember(Value = "I")] + Indirect, + } +} diff --git a/QuiverCNBC.cs b/QuiverCNBC.cs index 5555681..c453afd 100644 --- a/QuiverCNBC.cs +++ b/QuiverCNBC.cs @@ -28,8 +28,6 @@ namespace QuantConnect.DataSource /// public class QuiverCNBC : BaseData { - private static readonly TimeSpan _period = TimeSpan.FromDays(1); - /// /// Contract description /// @@ -49,10 +47,15 @@ public class QuiverCNBC : BaseData [JsonProperty(PropertyName = "Traders")] public string Traders { get; set; } + /// + /// Date the trader issued the stock advice on CNBC + /// + public DateTime AdviceDate { get; set; } + /// /// Time the data became available /// - public override DateTime EndTime => Time + _period; + public override DateTime EndTime => Time.AddDays(1); /// /// Parses the data from the line provided and loads it into LEAN @@ -66,16 +69,16 @@ public override BaseData Reader(SubscriptionDataConfig config, string line, Date { var csv = line.Split(','); - var parsedDate = Parse.DateTimeExact(csv[0], "yyyyMMdd"); + var uploadedDate = Parse.DateTimeExact(csv[0], "yyyyMMdd"); return new QuiverCNBC { Symbol = config.Symbol, - Notes = csv[1], - Direction = (OrderDirection)Enum.Parse(typeof(OrderDirection), csv[2], true), + Time = uploadedDate.AddDays(-1), + AdviceDate = (csv[1].IfNotNullOrEmpty(s => Parse.DateTimeExact(s, "yyyyMMdd")) ?? uploadedDate).AddDays(-1), + Direction = QuiverQuant.QuiverQuantCsvExtensions.ToOrderDirection(csv[2]), Traders = csv[3], - - Time = parsedDate + Notes = csv.Length > 4 ? csv[4] : string.Empty, }; } @@ -89,6 +92,7 @@ public override BaseData Clone() { Symbol = Symbol, Time = Time, + AdviceDate = AdviceDate, Notes = Notes, Direction = Direction, Traders = Traders, diff --git a/QuiverCNBCsUniverse.cs b/QuiverCNBCsUniverse.cs index 691d124..5dcc211 100644 --- a/QuiverCNBCsUniverse.cs +++ b/QuiverCNBCsUniverse.cs @@ -25,17 +25,15 @@ namespace QuantConnect.DataSource { /// - /// Universe Selection helper class for QuiverQuant Congress dataset + /// Universe Selection helper class for QuiverQuant CNBC dataset /// public class QuiverCNBCsUniverse : BaseDataCollection { - private static readonly TimeSpan _period = TimeSpan.FromDays(1); - /// /// Extra Information /// public string Notes { get; set; } - + /// /// Direction of trade /// @@ -46,10 +44,15 @@ public class QuiverCNBCsUniverse : BaseDataCollection /// public string Traders { get; set; } + /// + /// Date the trader issued the stock advice on CNBC + /// + public DateTime AdviceDate { get; set; } + /// /// Time the data became available /// - public override DateTime EndTime => Time + _period; + public override DateTime EndTime => Time.AddDays(1); /// /// Return the URL string source of the file. This will be converted to a stream @@ -89,10 +92,11 @@ public override BaseData Reader(SubscriptionDataConfig config, string line, Date return new QuiverCNBCsUniverse { Symbol = new Symbol(SecurityIdentifier.Parse(csv[0]), csv[1]), - Time = date, - Notes = csv[2], - Direction = (OrderDirection)Enum.Parse(typeof(OrderDirection), csv[3], true), + Time = date.AddDays(-1), + AdviceDate = (csv[2].IfNotNullOrEmpty(s => Parse.DateTimeExact(s, "yyyyMMdd")) ?? date).AddDays(-1), + Direction = QuiverQuant.QuiverQuantCsvExtensions.ToOrderDirection(csv[3]), Traders = csv[4], + Notes = csv.Length > 5 ? csv[5] : string.Empty, }; } @@ -150,6 +154,7 @@ public override BaseData Clone() Time = Time, Data = Data, + AdviceDate = AdviceDate, Notes = Notes, Direction = Direction, Traders = Traders diff --git a/QuiverInsiderTrading.cs b/QuiverInsiderTrading.cs index 25ba803..6d1ab07 100644 --- a/QuiverInsiderTrading.cs +++ b/QuiverInsiderTrading.cs @@ -1,4 +1,4 @@ -/* +/* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. * @@ -18,6 +18,8 @@ using NodaTime; using QuantConnect.Data; using QuantConnect.Data.UniverseSelection; +using QuantConnect.DataSource.QuiverQuant; +using QuantConnect.Util; using System; using System.Collections.Generic; using System.Globalization; @@ -31,36 +33,90 @@ namespace QuantConnect.DataSource [JsonObject] public class QuiverInsiderTrading : BaseDataCollection { - private static readonly TimeSpan _period = TimeSpan.FromDays(1); + /// + /// Transaction date as reported on SEC Form 4 + /// + [JsonProperty(PropertyName = "Date")] + [JsonConverter(typeof(DateTimeJsonConverter), "yyyy-MM-dd")] + public DateTime? Date { get; set; } /// - /// Name + /// Time the transaction was filed and became publicly available /// - [JsonProperty(PropertyName = "Name")] - public string Name { get; set; } + [JsonProperty(PropertyName = "fileDate")] + public DateTime? FileDate { get; set; } /// - /// Shares amount in transaction + /// Type of transaction (see SEC Form 4 codes: + /// https://www.sec.gov/files/forms-3-4-5.pdf) /// - [JsonProperty(PropertyName = "Shares")] - public decimal? Shares { get; set; } + [JsonProperty(PropertyName = "TransactionCode")] + public TransactionCode TransactionCode { get; set; } /// - /// PricePerShare of transaction + /// Reported price per share transacted /// [JsonProperty(PropertyName = "PricePerShare")] public decimal? PricePerShare { get; set; } /// - /// Shares Owned after transcation + /// Number of shares transacted + /// + [JsonProperty(PropertyName = "Shares")] + public decimal? Shares { get; set; } + + /// + /// Number of shares owned by insider following the transaction /// [JsonProperty(PropertyName = "SharesOwnedFollowing")] public decimal? SharesOwnedFollowing { get; set; } + /// + /// Indicates whether transaction was share acquisition or disposal + /// + [JsonProperty(PropertyName = "AcquiredDisposedCode")] + public AcquiredDisposedCode AcquiredDisposedCode { get; set; } + + /// + /// Whether the security is held directly or indirectly by the reporting person + /// + [JsonProperty(PropertyName = "directOrIndirectOwnership")] + public OwnershipType DirectOrIndirectOwnership { get; set; } + + /// + /// Corporate title of the transactor + /// + [JsonProperty(PropertyName = "officerTitle")] + public string OfficerTitle { get; set; } + + /// + /// Whether the transactor is a director of the company + /// + [JsonProperty(PropertyName = "isDirector")] + public bool? IsDirector { get; set; } + + /// + /// Whether the transactor is an officer of the company + /// + [JsonProperty(PropertyName = "isOfficer")] + public bool? IsOfficer { get; set; } + + /// + /// Whether the transactor is a 10% owner of the company + /// + [JsonProperty(PropertyName = "isTenPercentOwner")] + public bool? IsTenPercentOwner { get; set; } + + /// + /// Whether the transactor is not a director, officer, or 10% owner + /// + [JsonProperty(PropertyName = "isOther")] + public bool? IsOther { get; set; } + /// /// The time the data point ends at and becomes available to the algorithm /// - public override DateTime EndTime => Time + _period; + public override DateTime EndTime => Time.AddDays(1); /// /// Return the URL string source of the file. This will be converted to a stream @@ -96,16 +152,25 @@ public override BaseData Reader(SubscriptionDataConfig config, string line, Date { var csv = line.Split(','); - var parsedDate = Parse.DateTimeExact(csv[0], "yyyyMMdd");//, "'yyyy-MM-dd\'T\'HH:mm:ss.SSS\'Z\''" + var uploadedDate = Parse.DateTimeExact(csv[0], "yyyyMMdd"); return new QuiverInsiderTrading { - Name = csv[1], - Shares = csv[2].IfNotNullOrEmpty(s => decimal.Parse(s, NumberStyles.Any, CultureInfo.InvariantCulture)), - PricePerShare = csv[3].IfNotNullOrEmpty(s => decimal.Parse(s, NumberStyles.Any, CultureInfo.InvariantCulture)), - SharesOwnedFollowing = csv[4].IfNotNullOrEmpty(s => decimal.Parse(s, NumberStyles.Any, CultureInfo.InvariantCulture)), - Time = parsedDate, - Symbol = config.Symbol + Time = uploadedDate.AddDays(-1), + Symbol = config.Symbol, + FileDate = (csv[1].IfNotNullOrEmpty(s => Parse.DateTimeExact(s, "yyyyMMddHHmmss")) ?? uploadedDate).AddDays(-1), + Date = csv[2].IfNotNullOrEmpty(s => Parse.DateTimeExact(s, "yyyyMMdd")), + TransactionCode = QuiverQuantCsvExtensions.ToTransactionCode(csv[3]), + PricePerShare = csv[4].IfNotNullOrEmpty(s => decimal.Parse(s, NumberStyles.Any, CultureInfo.InvariantCulture)), + Shares = csv[5].IfNotNullOrEmpty(s => decimal.Parse(s, NumberStyles.Any, CultureInfo.InvariantCulture)), + SharesOwnedFollowing = csv[6].IfNotNullOrEmpty(s => decimal.Parse(s, NumberStyles.Any, CultureInfo.InvariantCulture)), + AcquiredDisposedCode = QuiverQuantCsvExtensions.ToAcquiredDisposedCode(csv[7]), + DirectOrIndirectOwnership = QuiverQuantCsvExtensions.ToOwnershipType(csv[8]), + OfficerTitle = csv[9], + IsDirector = QuiverQuantCsvExtensions.ToNullableBool(csv[10]), + IsOfficer = QuiverQuantCsvExtensions.ToNullableBool(csv[11]), + IsTenPercentOwner = QuiverQuantCsvExtensions.ToNullableBool(csv[12]), + IsOther = QuiverQuantCsvExtensions.ToNullableBool(csv[13]), }; } @@ -119,7 +184,9 @@ public override string ToString() // we are the wrapper instance return $"{Symbol} - Data Points {Data.Count}"; } - return $"{Symbol} - {Name} - {Shares} - {PricePerShare} - {SharesOwnedFollowing}"; + return $"{Symbol} ({OfficerTitle}) - {TransactionCode}/{AcquiredDisposedCode} - " + + $"{Shares} @ {PricePerShare} - SharesOwnedFollowing: {SharesOwnedFollowing} - " + + $"Ownership: {DirectOrIndirectOwnership} - Date: {Date} - Filed: {FileDate}"; } /// @@ -138,10 +205,19 @@ public override BaseData Clone() { return new QuiverInsiderTrading() { - Name = Name, - Shares = Shares, + Date = Date, + FileDate = FileDate, + TransactionCode = TransactionCode, PricePerShare = PricePerShare, + Shares = Shares, SharesOwnedFollowing = SharesOwnedFollowing, + AcquiredDisposedCode = AcquiredDisposedCode, + DirectOrIndirectOwnership = DirectOrIndirectOwnership, + OfficerTitle = OfficerTitle, + IsDirector = IsDirector, + IsOfficer = IsOfficer, + IsTenPercentOwner = IsTenPercentOwner, + IsOther = IsOther, Data = Data, Symbol = Symbol, Time = Time, @@ -180,7 +256,7 @@ public override List SupportedResolutions() /// The of this data type public override DateTimeZone DataTimeZone() { - return DateTimeZone.Utc; + return TimeZones.Utc; } } } diff --git a/QuiverInsiderTradingUniverse.cs b/QuiverInsiderTradingUniverse.cs index bfb3301..57e8228 100644 --- a/QuiverInsiderTradingUniverse.cs +++ b/QuiverInsiderTradingUniverse.cs @@ -21,6 +21,7 @@ using NodaTime; using QuantConnect.Data; using QuantConnect.Data.UniverseSelection; +using QuantConnect.DataSource.QuiverQuant; using static QuantConnect.StringExtensions; namespace QuantConnect.DataSource @@ -30,32 +31,75 @@ namespace QuantConnect.DataSource /// public class QuiverInsiderTradingUniverse : BaseDataCollection { - private static readonly TimeSpan _period = TimeSpan.FromDays(1); + /// + /// Transaction date as reported on SEC Form 4 + /// + public DateTime? Date { get; set; } /// - /// Name + /// Time the transaction was filed and became publicly available /// - public string Name { get; set; } + public DateTime? FileDate { get; set; } /// - /// Shares amount in transaction + /// Type of transaction (SEC Form 4 code) /// - public decimal? Shares { get; set; } + public TransactionCode TransactionCode { get; set; } /// - /// PricePerShare of transaction + /// Reported price per share transacted /// public decimal? PricePerShare { get; set; } /// - /// Shares Owned after transcation + /// Number of shares transacted + /// + public decimal? Shares { get; set; } + + /// + /// Number of shares owned by insider following the transaction /// public decimal? SharesOwnedFollowing { get; set; } /// - /// Time the data became available + /// Indicates whether transaction was share acquisition or disposal + /// + public AcquiredDisposedCode AcquiredDisposedCode { get; set; } + + /// + /// Whether the security is held directly or indirectly + /// + public OwnershipType DirectOrIndirectOwnership { get; set; } + + /// + /// Corporate title of the transactor + /// + public string OfficerTitle { get; set; } + + /// + /// Whether the transactor is a director of the company + /// + public bool? IsDirector { get; set; } + + /// + /// Whether the transactor is an officer of the company + /// + public bool? IsOfficer { get; set; } + + /// + /// Whether the transactor is a 10% owner of the company + /// + public bool? IsTenPercentOwner { get; set; } + + /// + /// Whether the transactor is not a director, officer, or 10% owner + /// + public bool? IsOther { get; set; } + + /// + /// Time the data becomes available to the algorithm /// - public override DateTime EndTime => Time + _period; + public override DateTime EndTime => Time.AddDays(1); /// /// Return the URL string source of the file. This will be converted to a stream @@ -92,19 +136,25 @@ public override BaseData Reader(SubscriptionDataConfig config, string line, Date { var csv = line.Split(','); - var shares = csv[3].IfNotNullOrEmpty(s => decimal.Parse(s, NumberStyles.Any, CultureInfo.InvariantCulture)); - var price = csv[4].IfNotNullOrEmpty(s => decimal.Parse(s, NumberStyles.Any, CultureInfo.InvariantCulture)); - var sharesAfter = csv[5].IfNotNullOrEmpty(s => decimal.Parse(s, NumberStyles.Any, CultureInfo.InvariantCulture)); - + var price = csv[5].IfNotNullOrEmpty(s => decimal.Parse(s, NumberStyles.Any, CultureInfo.InvariantCulture)); + return new QuiverInsiderTradingUniverse { - Time = date, - Name = csv[2], - Shares = shares, - PricePerShare = price, - SharesOwnedFollowing = sharesAfter, - + Time = date.AddDays(-1), Symbol = new Symbol(SecurityIdentifier.Parse(csv[0]), csv[1]), + FileDate = (csv[2].IfNotNullOrEmpty(s => Parse.DateTimeExact(s, "yyyyMMddHHmmss")) ?? date).AddDays(-1), + Date = csv[3].IfNotNullOrEmpty(s => Parse.DateTimeExact(s, "yyyyMMdd")), + TransactionCode = QuiverQuantCsvExtensions.ToTransactionCode(csv[4]), + PricePerShare = price, + Shares = csv[6].IfNotNullOrEmpty(s => decimal.Parse(s, NumberStyles.Any, CultureInfo.InvariantCulture)), + SharesOwnedFollowing = csv[7].IfNotNullOrEmpty(s => decimal.Parse(s, NumberStyles.Any, CultureInfo.InvariantCulture)), + AcquiredDisposedCode = QuiverQuantCsvExtensions.ToAcquiredDisposedCode(csv[8]), + DirectOrIndirectOwnership = QuiverQuantCsvExtensions.ToOwnershipType(csv[9]), + OfficerTitle = csv[10], + IsDirector = QuiverQuantCsvExtensions.ToNullableBool(csv[11]), + IsOfficer = QuiverQuantCsvExtensions.ToNullableBool(csv[12]), + IsTenPercentOwner = QuiverQuantCsvExtensions.ToNullableBool(csv[13]), + IsOther = QuiverQuantCsvExtensions.ToNullableBool(csv[14]), Value = price ?? 0 }; } @@ -115,10 +165,19 @@ public override BaseData Reader(SubscriptionDataConfig config, string line, Date public override string ToString() { return Invariant($"{Symbol}({Time}) :: ") + - Invariant($"Name: {string.Join(';', Name)} ") + - Invariant($"Shares: {Shares} ") + + Invariant($"Date: {Date} ") + + Invariant($"FileDate: {FileDate} ") + + Invariant($"TransactionCode: {TransactionCode} ") + Invariant($"PricePerShare: {PricePerShare} ") + - Invariant($"SharesOwnedFollowing: {SharesOwnedFollowing}"); + Invariant($"Shares: {Shares} ") + + Invariant($"SharesOwnedFollowing: {SharesOwnedFollowing} ") + + Invariant($"AcquiredDisposedCode: {AcquiredDisposedCode} ") + + Invariant($"DirectOrIndirectOwnership: {DirectOrIndirectOwnership} ") + + Invariant($"OfficerTitle: {OfficerTitle} ") + + Invariant($"IsDirector: {IsDirector} ") + + Invariant($"IsOfficer: {IsOfficer} ") + + Invariant($"IsTenPercentOwner: {IsTenPercentOwner} ") + + Invariant($"IsOther: {IsOther}"); } /// @@ -128,10 +187,19 @@ public override BaseData Clone() { return new QuiverInsiderTradingUniverse() { - Name = Name, - Shares = Shares, + Date = Date, + FileDate = FileDate, + TransactionCode = TransactionCode, PricePerShare = PricePerShare, + Shares = Shares, SharesOwnedFollowing = SharesOwnedFollowing, + AcquiredDisposedCode = AcquiredDisposedCode, + DirectOrIndirectOwnership = DirectOrIndirectOwnership, + OfficerTitle = OfficerTitle, + IsDirector = IsDirector, + IsOfficer = IsOfficer, + IsTenPercentOwner = IsTenPercentOwner, + IsOther = IsOther, Data = Data, Symbol = Symbol, Time = Time, @@ -160,7 +228,7 @@ public override List SupportedResolutions() /// The of this data type public override DateTimeZone DataTimeZone() { - return TimeZones.Chicago; + return TimeZones.Utc; } } -} \ No newline at end of file +} diff --git a/QuiverQuantCsvExtensions.cs b/QuiverQuantCsvExtensions.cs new file mode 100644 index 0000000..1259d7a --- /dev/null +++ b/QuiverQuantCsvExtensions.cs @@ -0,0 +1,128 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using QuantConnect.Orders; + +namespace QuantConnect.DataSource.QuiverQuant +{ + /// + /// Compact CSV serialization helpers for Quiver enums and primitives. Keeps the + /// on-disk format short (single SEC letters, 0/1 booleans, -1/0/1 trade direction) + /// while preserving full enum names in code. + /// + public static class QuiverQuantCsvExtensions + { + public static string ToCsv(this TransactionCode value) => value switch + { + TransactionCode.Sale => "S", + TransactionCode.Purchase => "P", + TransactionCode.VoluntaryReport => "V", + TransactionCode.GrantOrAward => "A", + TransactionCode.DispositionToIssuer => "D", + TransactionCode.ExercisePaymentWithSecurities => "F", + TransactionCode.DiscretionaryTransaction => "I", + TransactionCode.ExerciseOrConversionExempt => "M", + TransactionCode.ConversionOfDerivative => "C", + TransactionCode.ShortDerivativeExpiration => "E", + TransactionCode.LongDerivativeExpirationWithValue => "H", + TransactionCode.OutOfMoneyExercise => "O", + TransactionCode.InMoneyExercise => "X", + TransactionCode.Gift => "G", + TransactionCode.SmallAcquisition => "L", + TransactionCode.AcquisitionByWill => "W", + TransactionCode.VotingTrustDeposit => "Z", + TransactionCode.Other => "J", + TransactionCode.EquitySwap => "K", + TransactionCode.TenderDisposition => "U", + _ => string.Empty, + }; + + public static TransactionCode ToTransactionCode(string value) => value switch + { + "S" => TransactionCode.Sale, + "P" => TransactionCode.Purchase, + "V" => TransactionCode.VoluntaryReport, + "A" => TransactionCode.GrantOrAward, + "D" => TransactionCode.DispositionToIssuer, + "F" => TransactionCode.ExercisePaymentWithSecurities, + "I" => TransactionCode.DiscretionaryTransaction, + "M" => TransactionCode.ExerciseOrConversionExempt, + "C" => TransactionCode.ConversionOfDerivative, + "E" => TransactionCode.ShortDerivativeExpiration, + "H" => TransactionCode.LongDerivativeExpirationWithValue, + "O" => TransactionCode.OutOfMoneyExercise, + "X" => TransactionCode.InMoneyExercise, + "G" => TransactionCode.Gift, + "L" => TransactionCode.SmallAcquisition, + "W" => TransactionCode.AcquisitionByWill, + "Z" => TransactionCode.VotingTrustDeposit, + "J" => TransactionCode.Other, + "K" => TransactionCode.EquitySwap, + "U" => TransactionCode.TenderDisposition, + _ => TransactionCode.Other, + }; + + public static string ToCsv(this OwnershipType value) => value switch + { + OwnershipType.Direct => "D", + OwnershipType.Indirect => "I", + _ => string.Empty, + }; + + public static OwnershipType ToOwnershipType(string value) => value switch + { + "D" => OwnershipType.Direct, + "I" => OwnershipType.Indirect, + _ => OwnershipType.Unknown, + }; + + public static string ToCsv(this AcquiredDisposedCode value) => value switch + { + AcquiredDisposedCode.Acquired => "A", + AcquiredDisposedCode.Disposed => "D", + _ => string.Empty, + }; + + public static AcquiredDisposedCode ToAcquiredDisposedCode(string value) => value switch + { + "A" => AcquiredDisposedCode.Acquired, + "D" => AcquiredDisposedCode.Disposed, + _ => AcquiredDisposedCode.Unknown, + }; + + public static string ToCsv(this bool? value) => value switch + { + true => "T", + false => "F", + null => string.Empty, + }; + + public static bool? ToNullableBool(string value) => value switch + { + "T" => true, + "F" => false, + _ => null, + }; + + public static string ToCsv(this OrderDirection value) => ((int)value).ToString(System.Globalization.CultureInfo.InvariantCulture); + + public static OrderDirection ToOrderDirection(string value) + { + return int.TryParse(value, System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture, out var parsed) + ? (OrderDirection)parsed + : OrderDirection.Hold; + } + } +} diff --git a/TransactionCode.cs b/TransactionCode.cs new file mode 100644 index 0000000..ddf95a6 --- /dev/null +++ b/TransactionCode.cs @@ -0,0 +1,150 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using System.Runtime.Serialization; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; + +namespace QuantConnect.DataSource.QuiverQuant +{ + /// + /// SEC Form 4 transaction codes (see https://www.sec.gov/files/forms-3-4-5.pdf) + /// + [JsonConverter(typeof(StringEnumConverter))] + public enum TransactionCode + { + /// + /// S - Open market or private sale of non-derivative or derivative security + /// + [EnumMember(Value = "S")] + Sale = -1, + + /// + /// J - Other acquisition or disposition (describe transaction). Also used as the + /// default value when no transaction code is provided. + /// + [EnumMember(Value = "J")] + Other, + + /// + /// P - Open market or private purchase of non-derivative or derivative security + /// + [EnumMember(Value = "P")] + Purchase, + + /// + /// V - Transaction voluntarily reported earlier than required + /// + [EnumMember(Value = "V")] + VoluntaryReport, + + /// + /// A - Grant, award, or other acquisition pursuant to Rule 16b-3(d) + /// + [EnumMember(Value = "A")] + GrantOrAward, + + /// + /// D - Disposition to the issuer of issuer equity securities pursuant to Rule 16b-3(e) + /// + [EnumMember(Value = "D")] + DispositionToIssuer, + + /// + /// F - Payment of exercise price or tax liability by delivering or withholding securities + /// incident to the receipt, exercise, or vesting of a security issued in accordance with Rule 16b-3 + /// + [EnumMember(Value = "F")] + ExercisePaymentWithSecurities, + + /// + /// I - Discretionary transaction in accordance with Rule 16b-3(f) + /// + [EnumMember(Value = "I")] + DiscretionaryTransaction, + + /// + /// M - Exercise or conversion of derivative security exempted pursuant to Rule 16b-3 + /// + [EnumMember(Value = "M")] + ExerciseOrConversionExempt, + + /// + /// C - Conversion of derivative security + /// + [EnumMember(Value = "C")] + ConversionOfDerivative, + + /// + /// E - Expiration of short derivative position + /// + [EnumMember(Value = "E")] + ShortDerivativeExpiration, + + /// + /// H - Expiration (or cancellation) of long derivative position with value received + /// + [EnumMember(Value = "H")] + LongDerivativeExpirationWithValue, + + /// + /// O - Exercise of out-of-the-money derivative security + /// + [EnumMember(Value = "O")] + OutOfMoneyExercise, + + /// + /// X - Exercise of in-the-money or at-the-money derivative security + /// + [EnumMember(Value = "X")] + InMoneyExercise, + + /// + /// G - Bona fide gift + /// + [EnumMember(Value = "G")] + Gift, + + /// + /// L - Small acquisition under Rule 16a-6 + /// + [EnumMember(Value = "L")] + SmallAcquisition, + + /// + /// W - Acquisition or disposition by will or the laws of descent and distribution + /// + [EnumMember(Value = "W")] + AcquisitionByWill, + + /// + /// Z - Deposit into or withdrawal from voting trust + /// + [EnumMember(Value = "Z")] + VotingTrustDeposit, + + /// + /// K - Transaction in equity swap or instrument with similar characteristics + /// + [EnumMember(Value = "K")] + EquitySwap, + + /// + /// U - Disposition pursuant to a tender of shares in a change of control transaction + /// + [EnumMember(Value = "U")] + TenderDisposition, + } +} diff --git a/tests/QuiverCNBCTests.cs b/tests/QuiverCNBCTests.cs index a458529..95083a8 100644 --- a/tests/QuiverCNBCTests.cs +++ b/tests/QuiverCNBCTests.cs @@ -17,6 +17,7 @@ using System; using System.Linq; using Newtonsoft.Json; +using NodaTime; using NUnit.Framework; using QuantConnect.Data; using QuantConnect.DataSource; @@ -57,6 +58,82 @@ public void CloneCollection() AssertAreEqual(expected, result); } + [Test] + public void Reader_ParsesCompactFormat() + { + var symbol = new Symbol(SecurityIdentifier.Parse("AAPL R735QTJ8XC9X"), "AAPL"); + var config = CreateConfig(symbol); + var factory = new QuiverCNBC(); + // csv: uploadDate, adviceDate, direction(0=Buy/1=Sell/2=Hold), traders, notes + var line = "20260508,20260507,0,Jim Cramer,catalyst"; + + var result = (QuiverCNBC)factory.Reader(config, line, new DateTime(2026, 5, 8), false); + + Assert.AreEqual(symbol, result.Symbol); + Assert.AreEqual(new DateTime(2026, 5, 7), result.Time); + Assert.AreEqual(new DateTime(2026, 5, 8), result.EndTime); + Assert.AreEqual(new DateTime(2026, 5, 6), result.AdviceDate); + Assert.AreEqual(OrderDirection.Buy, result.Direction); + Assert.AreEqual("Jim Cramer", result.Traders); + Assert.AreEqual("catalyst", result.Notes); + } + + [Test] + public void Reader_EmptyAdviceDateFallsBackToUploadedMinusOne() + { + var symbol = new Symbol(SecurityIdentifier.Parse("AAPL R735QTJ8XC9X"), "AAPL"); + var config = CreateConfig(symbol); + var factory = new QuiverCNBC(); + var line = "20260508,,1,Steve Weiss,"; + + var result = (QuiverCNBC)factory.Reader(config, line, new DateTime(2026, 5, 8), false); + + Assert.AreEqual(new DateTime(2026, 5, 7), result.Time); + Assert.AreEqual(new DateTime(2026, 5, 7), result.AdviceDate); + Assert.AreEqual(OrderDirection.Sell, result.Direction); + Assert.AreEqual(string.Empty, result.Notes); + } + + [Test] + public void Reader_MissingTrailingNotesDefaultsToEmpty() + { + var symbol = new Symbol(SecurityIdentifier.Parse("AAPL R735QTJ8XC9X"), "AAPL"); + var config = CreateConfig(symbol); + var factory = new QuiverCNBC(); + // 4 columns only — trailing notes column missing entirely + var line = "20260508,20260507,2,Rob Sechan"; + + var result = (QuiverCNBC)factory.Reader(config, line, new DateTime(2026, 5, 8), false); + + Assert.AreEqual(OrderDirection.Hold, result.Direction); + Assert.AreEqual("Rob Sechan", result.Traders); + Assert.AreEqual(string.Empty, result.Notes); + } + + [Test] + public void UniverseReader_ParsesCompactFormat() + { + var factory = new QuiverCNBCsUniverse(); + // csv: sid, ticker, adviceDate, direction, traders, notes + var line = "AAPL R735QTJ8XC9X,AAPL,20260507,0,Jim Cramer,catalyst"; + + var result = (QuiverCNBCsUniverse)factory.Reader(null, line, new DateTime(2026, 5, 8), false); + + Assert.AreEqual("AAPL", result.Symbol.Value); + Assert.AreEqual(new DateTime(2026, 5, 7), result.Time); + Assert.AreEqual(new DateTime(2026, 5, 6), result.AdviceDate); + Assert.AreEqual(OrderDirection.Buy, result.Direction); + Assert.AreEqual("Jim Cramer", result.Traders); + Assert.AreEqual("catalyst", result.Notes); + } + + private static SubscriptionDataConfig CreateConfig(Symbol symbol) + { + return new SubscriptionDataConfig( + typeof(QuiverCNBC), symbol, Resolution.Daily, + DateTimeZone.Utc, DateTimeZone.Utc, false, false, false); + } + private void AssertAreEqual(object expected, object result, bool filterByCustomAttributes = false) { foreach (var propertyInfo in expected.GetType().GetProperties()) diff --git a/tests/QuiverInsiderTradingTests.cs b/tests/QuiverInsiderTradingTests.cs index 44370a9..104fa14 100644 --- a/tests/QuiverInsiderTradingTests.cs +++ b/tests/QuiverInsiderTradingTests.cs @@ -17,10 +17,12 @@ using System; using System.Linq; using Newtonsoft.Json; +using NodaTime; using NUnit.Framework; using QuantConnect.Data; using QuantConnect.DataProcessing; using QuantConnect.DataSource; +using QuantConnect.DataSource.QuiverQuant; namespace QuantConnect.DataLibrary.Tests { @@ -47,6 +49,98 @@ public void Clone() AssertAreEqual(expected, result); } + [Test] + public void Reader_ParsesCompactFormat() + { + var symbol = new Symbol(SecurityIdentifier.Parse("AAPL R735QTJ8XC9X"), "AAPL"); + var config = CreateConfig(symbol); + var factory = new QuiverInsiderTrading(); + var line = "20260508,20260507093000,20260507,P,150.25,100,500,A,D,CEO,T,T,F,"; + + var result = (QuiverInsiderTrading)factory.Reader(config, line, new DateTime(2026, 5, 8), false); + + Assert.AreEqual(symbol, result.Symbol); + Assert.AreEqual(new DateTime(2026, 5, 7), result.Time); + Assert.AreEqual(new DateTime(2026, 5, 8), result.EndTime); + Assert.AreEqual(new DateTime(2026, 5, 7), result.Date); + Assert.AreEqual(new DateTime(2026, 5, 6, 9, 30, 0), result.FileDate); + Assert.AreEqual(TransactionCode.Purchase, result.TransactionCode); + Assert.AreEqual(150.25m, result.PricePerShare); + Assert.AreEqual(100m, result.Shares); + Assert.AreEqual(500m, result.SharesOwnedFollowing); + Assert.AreEqual(AcquiredDisposedCode.Acquired, result.AcquiredDisposedCode); + Assert.AreEqual(OwnershipType.Direct, result.DirectOrIndirectOwnership); + Assert.AreEqual("CEO", result.OfficerTitle); + Assert.AreEqual(true, result.IsDirector); + Assert.AreEqual(true, result.IsOfficer); + Assert.AreEqual(false, result.IsTenPercentOwner); + Assert.IsNull(result.IsOther); + } + + [Test] + public void Reader_EmptyFileDateFallsBackToUploadedMinusOne() + { + var symbol = new Symbol(SecurityIdentifier.Parse("AAPL R735QTJ8XC9X"), "AAPL"); + var config = CreateConfig(symbol); + var factory = new QuiverInsiderTrading(); + // csv[1] (fileDate) is empty — Reader uses uploadedDate.AddDays(-1) + var line = "20260508,,20260507,S,275,1534,13366,D,D,CFO,,T,,"; + + var result = (QuiverInsiderTrading)factory.Reader(config, line, new DateTime(2026, 5, 8), false); + + Assert.AreEqual(new DateTime(2026, 5, 7), result.FileDate); + Assert.AreEqual(new DateTime(2026, 5, 7), result.Time); + } + + [Test] + public void Reader_EmptyOptionalFieldsAreNull() + { + var symbol = new Symbol(SecurityIdentifier.Parse("AAPL R735QTJ8XC9X"), "AAPL"); + var config = CreateConfig(symbol); + var factory = new QuiverInsiderTrading(); + // All optional numerics/booleans empty + var line = "20260508,,20260507,M,,1717,40879,A,D,,,,,"; + + var result = (QuiverInsiderTrading)factory.Reader(config, line, new DateTime(2026, 5, 8), false); + + Assert.AreEqual(TransactionCode.ExerciseOrConversionExempt, result.TransactionCode); + Assert.IsNull(result.PricePerShare); + Assert.AreEqual(1717m, result.Shares); + Assert.AreEqual(40879m, result.SharesOwnedFollowing); + Assert.AreEqual(string.Empty, result.OfficerTitle); + Assert.IsNull(result.IsDirector); + Assert.IsNull(result.IsOfficer); + Assert.IsNull(result.IsTenPercentOwner); + Assert.IsNull(result.IsOther); + } + + [Test] + public void UniverseReader_ParsesCompactFormat() + { + var factory = new QuiverInsiderTradingUniverse(); + // csv[0]=sid, csv[1]=ticker, csv[2]=fileDate(empty -> fallback), csv[3]=Date, csv[4]=TransactionCode, ... + var line = "AAPL R735QTJ8XC9X,AAPL,,20260507,P,150.25,100,500,A,D,CEO,T,T,F,"; + + var result = (QuiverInsiderTradingUniverse)factory.Reader(null, line, new DateTime(2026, 5, 8), false); + + Assert.AreEqual("AAPL", result.Symbol.Value); + Assert.AreEqual(new DateTime(2026, 5, 7), result.Time); + Assert.AreEqual(new DateTime(2026, 5, 7), result.FileDate); + Assert.AreEqual(new DateTime(2026, 5, 7), result.Date); + Assert.AreEqual(TransactionCode.Purchase, result.TransactionCode); + Assert.AreEqual(150.25m, result.PricePerShare); + Assert.AreEqual(AcquiredDisposedCode.Acquired, result.AcquiredDisposedCode); + Assert.AreEqual(OwnershipType.Direct, result.DirectOrIndirectOwnership); + Assert.AreEqual(150.25m, result.Value); + } + + private static SubscriptionDataConfig CreateConfig(Symbol symbol) + { + return new SubscriptionDataConfig( + typeof(QuiverInsiderTrading), symbol, Resolution.Daily, + DateTimeZone.Utc, DateTimeZone.Utc, false, false, false); + } + [TestCase("abc123:msft\"", ExpectedResult = new string[] {"MSFT"})] [TestCase("AAPL+", ExpectedResult = new string[] {"AAPL"})] [TestCase("AAPL-", ExpectedResult = new string[] {"AAPL"})] @@ -84,10 +178,19 @@ private BaseData CreateNewInstance() Symbol = Symbol.Empty, Time = DateTime.Today, DataType = MarketDataType.Base, - Name = "Institution name", + Date = DateTime.Today, + FileDate = DateTime.Today, + TransactionCode = TransactionCode.Purchase, Shares = 0.0m, PricePerShare = 0.0m, - SharesOwnedFollowing = 0.0m + SharesOwnedFollowing = 0.0m, + AcquiredDisposedCode = AcquiredDisposedCode.Acquired, + DirectOrIndirectOwnership = OwnershipType.Direct, + OfficerTitle = "CEO", + IsDirector = false, + IsOfficer = true, + IsTenPercentOwner = false, + IsOther = false, }; } diff --git a/tests/QuiverQuantCsvExtensionsTests.cs b/tests/QuiverQuantCsvExtensionsTests.cs new file mode 100644 index 0000000..3749123 --- /dev/null +++ b/tests/QuiverQuantCsvExtensionsTests.cs @@ -0,0 +1,120 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +using NUnit.Framework; +using QuantConnect.DataSource.QuiverQuant; +using QuantConnect.Orders; + +namespace QuantConnect.DataLibrary.Tests +{ + [TestFixture] + public class QuiverQuantCsvExtensionsTests + { + [TestCase(TransactionCode.Sale, "S")] + [TestCase(TransactionCode.Purchase, "P")] + [TestCase(TransactionCode.VoluntaryReport, "V")] + [TestCase(TransactionCode.GrantOrAward, "A")] + [TestCase(TransactionCode.DispositionToIssuer, "D")] + [TestCase(TransactionCode.ExercisePaymentWithSecurities, "F")] + [TestCase(TransactionCode.DiscretionaryTransaction, "I")] + [TestCase(TransactionCode.ExerciseOrConversionExempt, "M")] + [TestCase(TransactionCode.ConversionOfDerivative, "C")] + [TestCase(TransactionCode.ShortDerivativeExpiration, "E")] + [TestCase(TransactionCode.LongDerivativeExpirationWithValue, "H")] + [TestCase(TransactionCode.OutOfMoneyExercise, "O")] + [TestCase(TransactionCode.InMoneyExercise, "X")] + [TestCase(TransactionCode.Gift, "G")] + [TestCase(TransactionCode.SmallAcquisition, "L")] + [TestCase(TransactionCode.AcquisitionByWill, "W")] + [TestCase(TransactionCode.VotingTrustDeposit, "Z")] + [TestCase(TransactionCode.Other, "J")] + [TestCase(TransactionCode.EquitySwap, "K")] + [TestCase(TransactionCode.TenderDisposition, "U")] + public void TransactionCode_RoundTrip(TransactionCode value, string expectedLetter) + { + Assert.AreEqual(expectedLetter, value.ToCsv()); + Assert.AreEqual(value, QuiverQuantCsvExtensions.ToTransactionCode(expectedLetter)); + } + + [TestCase("", TransactionCode.Other)] + [TestCase("?", TransactionCode.Other)] + [TestCase("unknown", TransactionCode.Other)] + public void TransactionCode_UnknownInputFallsBackToOther(string input, TransactionCode expected) + { + Assert.AreEqual(expected, QuiverQuantCsvExtensions.ToTransactionCode(input)); + } + + [TestCase(OwnershipType.Direct, "D")] + [TestCase(OwnershipType.Indirect, "I")] + [TestCase(OwnershipType.Unknown, "")] + public void OwnershipType_RoundTrip(OwnershipType value, string expectedLetter) + { + Assert.AreEqual(expectedLetter, value.ToCsv()); + Assert.AreEqual(value, QuiverQuantCsvExtensions.ToOwnershipType(expectedLetter)); + } + + [TestCase("?", OwnershipType.Unknown)] + public void OwnershipType_UnknownInputFallsBackToUnknown(string input, OwnershipType expected) + { + Assert.AreEqual(expected, QuiverQuantCsvExtensions.ToOwnershipType(input)); + } + + [TestCase(AcquiredDisposedCode.Acquired, "A")] + [TestCase(AcquiredDisposedCode.Disposed, "D")] + [TestCase(AcquiredDisposedCode.Unknown, "")] + public void AcquiredDisposedCode_RoundTrip(AcquiredDisposedCode value, string expectedLetter) + { + Assert.AreEqual(expectedLetter, value.ToCsv()); + Assert.AreEqual(value, QuiverQuantCsvExtensions.ToAcquiredDisposedCode(expectedLetter)); + } + + [TestCase("?", AcquiredDisposedCode.Unknown)] + public void AcquiredDisposedCode_UnknownInputFallsBackToUnknown(string input, AcquiredDisposedCode expected) + { + Assert.AreEqual(expected, QuiverQuantCsvExtensions.ToAcquiredDisposedCode(input)); + } + + [TestCase(true, "T")] + [TestCase(false, "F")] + [TestCase(null, "")] + public void NullableBool_RoundTrip(bool? value, string expected) + { + Assert.AreEqual(expected, value.ToCsv()); + Assert.AreEqual(value, QuiverQuantCsvExtensions.ToNullableBool(expected)); + } + + [TestCase("anything", null)] + public void NullableBool_UnknownInputFallsBackToNull(string input, bool? expected) + { + Assert.AreEqual(expected, QuiverQuantCsvExtensions.ToNullableBool(input)); + } + + [TestCase(OrderDirection.Buy, "0")] + [TestCase(OrderDirection.Sell, "1")] + [TestCase(OrderDirection.Hold, "2")] + public void OrderDirection_RoundTrip(OrderDirection value, string expected) + { + Assert.AreEqual(expected, value.ToCsv()); + Assert.AreEqual(value, QuiverQuantCsvExtensions.ToOrderDirection(expected)); + } + + [TestCase("", OrderDirection.Hold)] + [TestCase("xyz", OrderDirection.Hold)] + public void OrderDirection_UnparsableInputFallsBackToHold(string input, OrderDirection expected) + { + Assert.AreEqual(expected, QuiverQuantCsvExtensions.ToOrderDirection(input)); + } + } +}