From 437fc8b44cc2b0eca434964d15fcd42fe92a6eb6 Mon Sep 17 00:00:00 2001 From: AymericNoel Date: Mon, 21 Dec 2020 23:22:13 +0100 Subject: [PATCH 1/2] Add Trakx.utils package --- .github/workflows/dotnet-core.yml | 9 +++ .../Converters/DateTimeOffsetConverter.cs | 24 -------- .../Converters/DictionaryConverter.cs | 60 ------------------- .../Converters/StringLongConverter.cs | 46 -------------- .../Converters/StringNullableEnumConverter.cs | 47 --------------- .../Trakx.CryptoCompare.ApiClient.csproj | 1 + .../WebSocket/DTOs/Inbound/Ohlc.cs | 2 +- .../WebSocket/DTOs/Inbound/Ticker.cs | 2 +- .../WebSocket/DTOs/Inbound/Trade.cs | 2 +- 9 files changed, 13 insertions(+), 180 deletions(-) delete mode 100644 src/Trakx.CryptoCompare.ApiClient/Serialisation/Converters/DateTimeOffsetConverter.cs delete mode 100644 src/Trakx.CryptoCompare.ApiClient/Serialisation/Converters/DictionaryConverter.cs delete mode 100644 src/Trakx.CryptoCompare.ApiClient/Serialisation/Converters/StringLongConverter.cs delete mode 100644 src/Trakx.CryptoCompare.ApiClient/Serialisation/Converters/StringNullableEnumConverter.cs diff --git a/.github/workflows/dotnet-core.yml b/.github/workflows/dotnet-core.yml index 1b30e36..823e6be 100644 --- a/.github/workflows/dotnet-core.yml +++ b/.github/workflows/dotnet-core.yml @@ -20,9 +20,18 @@ jobs: uses: actions/setup-dotnet@v1 with: dotnet-version: 5.0.x + + - name: Add github nuget source + run: dotnet nuget add source "https://nuget.pkg.github.com/trakx/index.json" --name "github" --username "trakx-bot" --password ${{secrets.TRAKX_BOT_READONLY_PAT}} --store-password-in-clear-text + - name: Install dependencies run: dotnet restore ${{env.SOLUTION_PATH}} + + - name: Remove github source + run: dotnet nuget remove source "github" + - name: Build run: dotnet build ${{env.SOLUTION_PATH}} --configuration Release --no-restore + - name: Test run: dotnet test ${{env.SOLUTION_PATH}} --no-restore --logger GitHubActions --verbosity normal diff --git a/src/Trakx.CryptoCompare.ApiClient/Serialisation/Converters/DateTimeOffsetConverter.cs b/src/Trakx.CryptoCompare.ApiClient/Serialisation/Converters/DateTimeOffsetConverter.cs deleted file mode 100644 index 6efc107..0000000 --- a/src/Trakx.CryptoCompare.ApiClient/Serialisation/Converters/DateTimeOffsetConverter.cs +++ /dev/null @@ -1,24 +0,0 @@ -using System; -using System.Diagnostics; -using System.Text.Json; -using System.Text.Json.Serialization; - -namespace Trakx.CryptoCompare.ApiClient.Serialisation.Converters -{ - public class DateTimeOffsetConverter : JsonConverter - { - public override DateTimeOffset? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) - { - Debug.Assert(typeToConvert == typeof(DateTimeOffset?)); - var valueRead = reader.GetString(); - if (string.IsNullOrWhiteSpace(valueRead) || valueRead.Equals("null", StringComparison.InvariantCultureIgnoreCase)) - return null; - return DateTimeOffset.Parse(valueRead); - } - - public override void Write(Utf8JsonWriter writer, DateTimeOffset? value, JsonSerializerOptions options) - { - writer.WriteStringValue(value?.ToString() ?? ""); - } - } -} diff --git a/src/Trakx.CryptoCompare.ApiClient/Serialisation/Converters/DictionaryConverter.cs b/src/Trakx.CryptoCompare.ApiClient/Serialisation/Converters/DictionaryConverter.cs deleted file mode 100644 index 28065ae..0000000 --- a/src/Trakx.CryptoCompare.ApiClient/Serialisation/Converters/DictionaryConverter.cs +++ /dev/null @@ -1,60 +0,0 @@ -using System; -using System.Collections; -using System.Collections.Generic; -using System.Globalization; -using System.Reflection; -using System.Text.Json; -using System.Text.Json.Serialization; - -namespace Trakx.CryptoCompare.ApiClient.Serialisation.Converters -{ - /// - /// Credits to https://github.com/dotnet/runtime/issues/30524#issuecomment-524619972 - /// - public class JsonNonStringKeyDictionaryConverter : JsonConverter> where TKey : notnull - { - public override IDictionary Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) - { - var convertedType = typeof(Dictionary<,>) - .MakeGenericType(typeof(string), typeToConvert.GenericTypeArguments[1]); - var value = JsonSerializer.Deserialize(ref reader, convertedType, options); - var instance = (Dictionary)Activator.CreateInstance( - typeof(Dictionary), - BindingFlags.Instance | BindingFlags.Public, - null, - null, - CultureInfo.CurrentCulture)!; - var enumerator = (IEnumerator)convertedType.GetMethod("GetEnumerator")!.Invoke(value, null)!; - var parse = typeof(TKey).GetMethod("Parse", 0, BindingFlags.Public | BindingFlags.Static, null, CallingConventions.Any, new[] { typeof(string) }, null); - if (parse == null) throw new NotSupportedException($"{typeof(TKey)} as TKey in IDictionary is not supported."); - while (enumerator.MoveNext()) - { - var element = (KeyValuePair)enumerator.Current!; - instance.Add((TKey)parse.Invoke(null, new [] { element.Key })!, element.Value); - } - return instance; - } - - public override void Write(Utf8JsonWriter writer, IDictionary value, JsonSerializerOptions options) - { - if (value == null) throw new ArgumentNullException(nameof(value)); - var convertedDictionary = new Dictionary(value.Count); - foreach (var (k, v) in value) if(k != null) convertedDictionary[k.ToString()!] = v; - - JsonSerializer.Serialize(writer, convertedDictionary, options); - convertedDictionary.Clear(); - } - } - - public sealed class JsonDateTimeKeyDictionaryConverter : JsonNonStringKeyDictionaryConverter - { - public override void Write(Utf8JsonWriter writer, IDictionary value, JsonSerializerOptions options) - { - var convertedDictionary = new Dictionary(value.Count); - foreach (var (k, v) in value) convertedDictionary[k.ToString("yyyy-MM-ddTHH:mm:ssK")] = v; - - JsonSerializer.Serialize(writer, convertedDictionary, options); - convertedDictionary.Clear(); - } - } -} \ No newline at end of file diff --git a/src/Trakx.CryptoCompare.ApiClient/Serialisation/Converters/StringLongConverter.cs b/src/Trakx.CryptoCompare.ApiClient/Serialisation/Converters/StringLongConverter.cs deleted file mode 100644 index 47f34ad..0000000 --- a/src/Trakx.CryptoCompare.ApiClient/Serialisation/Converters/StringLongConverter.cs +++ /dev/null @@ -1,46 +0,0 @@ -using System; -using System.Buffers; -using System.Buffers.Text; -using System.Text.Json; -using System.Text.Json.Serialization; - -namespace Trakx.CryptoCompare.ApiClient.Serialisation.Converters -{ - public class LongStringConverter : JsonConverter - { - public override long Read(ref Utf8JsonReader reader, Type type, JsonSerializerOptions options) - { - if (reader.TokenType != JsonTokenType.String) return reader.GetInt64(); - var span = reader.HasValueSequence ? reader.ValueSequence.ToArray() : reader.ValueSpan; - return Utf8Parser.TryParse(span, out long number, out var bytesConsumed) && span.Length == bytesConsumed - ? number - : long.TryParse(reader.GetString(), out number) - ? number - : reader.GetInt64(); - } - - public override void Write(Utf8JsonWriter writer, long value, JsonSerializerOptions options) - { - writer.WriteStringValue(value.ToString()); - } - } - - public class ULongOrStringConverter : JsonConverter - { - public override ulong Read(ref Utf8JsonReader reader, Type type, JsonSerializerOptions options) - { - if (reader.TokenType != JsonTokenType.String) return reader.GetUInt64(); - var span = reader.HasValueSequence ? reader.ValueSequence.ToArray() : reader.ValueSpan; - return Utf8Parser.TryParse(span, out ulong number, out var bytesConsumed) && span.Length == bytesConsumed - ? number - : ulong.TryParse(reader.GetString(), out number) - ? number - : reader.GetUInt64(); - } - - public override void Write(Utf8JsonWriter writer, ulong value, JsonSerializerOptions options) - { - writer.WriteStringValue(value.ToString()); - } - } -} diff --git a/src/Trakx.CryptoCompare.ApiClient/Serialisation/Converters/StringNullableEnumConverter.cs b/src/Trakx.CryptoCompare.ApiClient/Serialisation/Converters/StringNullableEnumConverter.cs deleted file mode 100644 index 2dc13f2..0000000 --- a/src/Trakx.CryptoCompare.ApiClient/Serialisation/Converters/StringNullableEnumConverter.cs +++ /dev/null @@ -1,47 +0,0 @@ -using System; -using System.Text.Json; -using System.Text.Json.Serialization; - -namespace Trakx.CryptoCompare.ApiClient.Serialisation.Converters -{ - /// - /// Thanks to https://github.com/dotnet/corefx/issues/41307#issuecomment-562845257 - /// - /// - public class StringNullableEnumConverter : JsonConverter - { - private readonly Type _underlyingType; - - public StringNullableEnumConverter() - { - _underlyingType = Nullable.GetUnderlyingType(typeof(T)); - } - - public override bool CanConvert(Type typeToConvert) - { - return typeof(T).IsAssignableFrom(typeToConvert); - //return true; - } - - public override T Read(ref Utf8JsonReader reader, - Type typeToConvert, - JsonSerializerOptions options) - { - var value = reader.GetString(); - if (string.IsNullOrEmpty(value)) return default; - if (!Enum.TryParse(_underlyingType, value, ignoreCase: false, out object result) && - !Enum.TryParse(_underlyingType, value, ignoreCase: true, out result)) - { - throw new JsonException($"Unable to convert \"{value}\" to Enum \"{_underlyingType}\"."); - } - return (T)result; - } - - public override void Write(Utf8JsonWriter writer, - T value, - JsonSerializerOptions options) - { - writer.WriteStringValue(value?.ToString()); - } - } -} diff --git a/src/Trakx.CryptoCompare.ApiClient/Trakx.CryptoCompare.ApiClient.csproj b/src/Trakx.CryptoCompare.ApiClient/Trakx.CryptoCompare.ApiClient.csproj index 814b3e2..055a734 100644 --- a/src/Trakx.CryptoCompare.ApiClient/Trakx.CryptoCompare.ApiClient.csproj +++ b/src/Trakx.CryptoCompare.ApiClient/Trakx.CryptoCompare.ApiClient.csproj @@ -21,6 +21,7 @@ + diff --git a/src/Trakx.CryptoCompare.ApiClient/WebSocket/DTOs/Inbound/Ohlc.cs b/src/Trakx.CryptoCompare.ApiClient/WebSocket/DTOs/Inbound/Ohlc.cs index e296928..84eafb8 100644 --- a/src/Trakx.CryptoCompare.ApiClient/WebSocket/DTOs/Inbound/Ohlc.cs +++ b/src/Trakx.CryptoCompare.ApiClient/WebSocket/DTOs/Inbound/Ohlc.cs @@ -1,5 +1,5 @@ using System.Text.Json.Serialization; -using Trakx.CryptoCompare.ApiClient.Serialisation.Converters; +using Trakx.Utils.Serialization.Converters; namespace Trakx.CryptoCompare.ApiClient.WebSocket.DTOs.Inbound { diff --git a/src/Trakx.CryptoCompare.ApiClient/WebSocket/DTOs/Inbound/Ticker.cs b/src/Trakx.CryptoCompare.ApiClient/WebSocket/DTOs/Inbound/Ticker.cs index fa014ab..326928a 100644 --- a/src/Trakx.CryptoCompare.ApiClient/WebSocket/DTOs/Inbound/Ticker.cs +++ b/src/Trakx.CryptoCompare.ApiClient/WebSocket/DTOs/Inbound/Ticker.cs @@ -1,5 +1,5 @@ using System.Text.Json.Serialization; -using Trakx.CryptoCompare.ApiClient.Serialisation.Converters; +using Trakx.Utils.Serialization.Converters; namespace Trakx.CryptoCompare.ApiClient.WebSocket.DTOs.Inbound { diff --git a/src/Trakx.CryptoCompare.ApiClient/WebSocket/DTOs/Inbound/Trade.cs b/src/Trakx.CryptoCompare.ApiClient/WebSocket/DTOs/Inbound/Trade.cs index a99eb13..84879ac 100644 --- a/src/Trakx.CryptoCompare.ApiClient/WebSocket/DTOs/Inbound/Trade.cs +++ b/src/Trakx.CryptoCompare.ApiClient/WebSocket/DTOs/Inbound/Trade.cs @@ -1,5 +1,5 @@ using System.Text.Json.Serialization; -using Trakx.CryptoCompare.ApiClient.Serialisation.Converters; +using Trakx.Utils.Serialization.Converters; namespace Trakx.CryptoCompare.ApiClient.WebSocket.DTOs.Inbound { From 4a604be2843e1be8971f51e64ac2ab93dc389218 Mon Sep 17 00:00:00 2001 From: monsieurleberre Date: Wed, 13 Jan 2021 09:37:23 +0000 Subject: [PATCH 2/2] create groups of symbols of less than 300 chars to fix issue #8 --- .../Rest/Clients/PriceClientTests.cs | 17 ++++++++ .../Rest/Clients/PriceClient.cs | 43 ++++++++++++++++++- 2 files changed, 58 insertions(+), 2 deletions(-) diff --git a/src/Trakx.CryptoCompare.ApiClient.Tests/Integration/Rest/Clients/PriceClientTests.cs b/src/Trakx.CryptoCompare.ApiClient.Tests/Integration/Rest/Clients/PriceClientTests.cs index bfc4eba..fe6d3ef 100644 --- a/src/Trakx.CryptoCompare.ApiClient.Tests/Integration/Rest/Clients/PriceClientTests.cs +++ b/src/Trakx.CryptoCompare.ApiClient.Tests/Integration/Rest/Clients/PriceClientTests.cs @@ -1,4 +1,5 @@ using System.Threading.Tasks; +using FluentAssertions; using Xunit; namespace Trakx.CryptoCompare.ApiClient.Tests.Integration.Rest.Clients @@ -36,5 +37,21 @@ public async Task CanCallGenerateCustomAverageEndpoint() var result = await CryptoCompareClient.Prices.GenerateCustomAverageAsync("BTC", "USD", new[] { "Kraken" }); Assert.NotNull(result); } + + [Fact] public async Task CanCallWithLargeNumberOfFSymbols() + { + var symbols = new[] + { + "bix", "bz", "edo", "ftt", "ht", "kcs", "leo", "okb", "qash", "zb", "ast", "bnt", "dgtx", "hot", "idex", + "knc", "lrc", "nec", "oax", "xin", "zrx", "lnd", "akro", "bcpt", "lba", "lend", "aave", "mkr", "nexo", + "ppt", "rcn", "salt", "comp", "cel", "wbtc", "weth", "link", "seele", "bat", "bal", "omg", "usdc", + "paxg", "uma", "yfi", "snx", "band", "mft", "hedg", "theta", "ren", "cvt", "chz", "rsr", "btc", "eth", + "xrp", "bch", "bnb", "dot", "ltc", "ada", "eos", "waves", "xlm", "sushi", "zil", "xem", "uni", "rune", + "srm", "cake", "luna" + }; + var result = await CryptoCompareClient.Prices.MultipleSymbolsPriceAsync(symbols, new[] { "USD", "EUR" }); + result.Count.Should().Be(symbols.Length); + Assert.NotNull(result); + } } } diff --git a/src/Trakx.CryptoCompare.ApiClient/Rest/Clients/PriceClient.cs b/src/Trakx.CryptoCompare.ApiClient/Rest/Clients/PriceClient.cs index c479ffc..e6ba584 100644 --- a/src/Trakx.CryptoCompare.ApiClient/Rest/Clients/PriceClient.cs +++ b/src/Trakx.CryptoCompare.ApiClient/Rest/Clients/PriceClient.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.Linq; using System.Net.Http; using System.Threading.Tasks; using JetBrains.Annotations; @@ -64,8 +65,46 @@ public async Task MultipleSymbolsPriceAsync( Check.NotEmpty(toSymbols, nameof(toSymbols)); Check.NotEmpty(fromSymbols, nameof(fromSymbols)); - return await this.GetAsync( - ApiUrls.PriceMulti(fromSymbols, toSymbols, tryConversion, exchangeName)).ConfigureAwait(false); + var groupsOfSymbols = GroupSymbolsByListsOfMaxCsvCharacters(fromSymbols.ToList()); + + var fetchTasks = groupsOfSymbols.Select(s => this.GetAsync( + ApiUrls.PriceMulti(s, toSymbols, tryConversion, exchangeName))) + .ToArray(); + + await Task.WhenAll(fetchTasks); + + var results = fetchTasks + .SelectMany(t => t.Result) + .ToDictionary(p => p.Key, p => p.Value); + var mergedResults = new PriceMultiResponse(results); + + return mergedResults; + } + + private static List> GroupSymbolsByListsOfMaxCsvCharacters(IReadOnlyList fromSymbolList, int maxLength = 300) + { + var i = 0; + var groupsOfSymbols = new List>(); + + do + { + var includedFromSymbols = new List(); + var length = 0; + do + { + if (length + fromSymbolList[i].Length + 1 < maxLength) + { + length += fromSymbolList[i].Length + 1; + includedFromSymbols.Add(fromSymbolList[i]); + i++; + } + else length = maxLength; + } while (length < maxLength && i < fromSymbolList.Count); + + groupsOfSymbols.Add(includedFromSymbols); + } while (i < fromSymbolList.Count); + + return groupsOfSymbols; } ///