Feature/rate limits

.github/ISSUE_TEMPLATE/bug_report.yml

@@ -11,7 +11,7 @@ body:
         This form is only for submitting bug reports. If you have a usage question
         or are unsure if this is really a bug, make sure to:
 
-        - Read the [docs](https://helldivers-2.fly.dev/swagger-ui.html)
+        - Read the [docs](https://helldivers-2.github.io/api/)
         - Ask on [Discord Chat](https://discord.gg/E8UUfWYmf9)
         - Ask on [GitHub Discussions](https://github.com/helldivers-2/api/discussions)
 

README.md

@@ -18,9 +18,9 @@ a [SwaggerUI](https://helldivers-2.github.io/api/docs/openapi/swagger-ui.html) (
 internally, however we strongly encourage you to use the `/raw` endpoints of the community wrapper instead of accessing
 the ArrowHead API directly, as it puts additional load on their servers (besides, we have shinier endpoints, I swear!).
 
-The root URL of the API is available here: https://helldivers-2-dotnet.fly.dev/
+The root URL of the API is available here: https://api.helldivers2.dev
 > [!WARNING]
-> Note that it might change as we are transitioning from the Elixir version to a new version!
+> The root domain of the API recently changed, it's recommended you use the domain above to avoid problems in the future
 
 We also ask that you send us a `User-Agent` header when making requests (if accessing directly from the browser,
 the headers sent by those should suffice and you don't need to add anything special).

docs/containers.md

@@ -62,3 +62,26 @@ docker run -p 8080:8080 -e "Helldivers__Synchronization__IntervalSeconds=10" hel
 ```
 
 You can read more about using environment variables to override configuration [here](https://learn.microsoft.com/en-us/aspnet/core/fundamentals/configuration/?view=aspnetcore-8.0#naming-of-environment-variables)
+
+### Overriding rate limits
+You can override the rate limits by overriding the following configuration:
+```json
+{
+  "Helldivers": {
+    "API": {
+      "RateLimit": 5,
+      "RateLimitWindow": 10
+    }
+  }
+}
+```
+
+The `RateLimit` (overridable with `-e Helldivers__API__RateLimit`) is how many requests can be made in the timeframe,
+the `RateLimitWindow` (overridable with `-e Helldivers__API__RateLimitWindow`) is how many seconds before the `RateLimit`
+resets again.
+
+Increasing the `RateLimit`, decreasing the `RateLimitWindow` or both will effectively increase how many requests you can
+make to the application.
+
+Alternatively, if you use the hosted versions you can request an API key that allows for higher rate limits
+by sponsoring this project! (if you self host you can generate your own keys too!).

src/Helldivers-2-API/Configuration/ApiConfiguration.cs

@@ -0,0 +1,22 @@
+namespace Helldivers.API.Configuration;
+
+/// <summary>
+/// Contains configuration of the API.
+/// </summary>
+public sealed class ApiConfiguration
+{
+    /// <summary>
+    /// The amount of requests that can be made within the time limit.
+    /// </summary>
+    public int RateLimit { get; set; }
+
+    /// <summary>
+    /// The time before the rate limit resets (in seconds).
+    /// </summary>
+    public int RateLimitWindow { get; set; }
+
+    /// <summary>
+    /// Contains the <see cref="AuthenticationConfiguration" /> for the API.
+    /// </summary>
+    public AuthenticationConfiguration Authentication { get; set; } = null!;
+}

src/Helldivers-2-API/Configuration/AuthenticationConfiguration.cs

@@ -0,0 +1,22 @@
+namespace Helldivers.API.Configuration;
+
+/// <summary>
+/// Contains configuration for the authentication functionality of the API.
+/// </summary>
+public sealed class AuthenticationConfiguration
+{
+    /// <summary>
+    /// A list of valid issuers of authentication tokens.
+    /// </summary>
+    public List<string> ValidIssuers { get; set; } = [];
+
+    /// <summary>
+    /// A list of valid audiences for said authentication tokens.
+    /// </summary>
+    public List<string> ValidAudiences { get; set; } = [];
+
+    /// <summary>
+    /// A string containing a base64 encoded secret used for signing and verifying authentication tokens.
+    /// </summary>
+    public string SigningKey { get; set; } = null!;
+}

src/Helldivers-2-API/Controllers/DevelopmentController.cs

@@ -0,0 +1,40 @@
+using Helldivers.API.Configuration;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Options;
+using Microsoft.IdentityModel.Tokens;
+using System.IdentityModel.Tokens.Jwt;
+using System.Security.Claims;
+
+namespace Helldivers.API.Controllers;
+
+/// <summary>
+/// Controller class that's only available in development, contains local debugging endpoints etc.
+/// </summary>
+public static class DevelopmentController
+{
+    private static readonly JwtSecurityTokenHandler TokenHandler = new();
+
+    /// <summary>
+    /// Creates a JWT token for the given <paramref name="name" /> and <paramref name="limit" />.
+    /// </summary>
+    public static IResult CreateToken([FromQuery] string name, [FromQuery] int limit, [FromServices] IOptions<ApiConfiguration> options)
+    {
+        var key = new SymmetricSecurityKey(Convert.FromBase64String(options.Value.Authentication.SigningKey));
+        var token = new JwtSecurityToken(
+            issuer: options.Value.Authentication.ValidIssuers.First(),
+            audience: options.Value.Authentication.ValidAudiences.First(),
+            claims: [
+                new Claim("sub", name),
+                new Claim("nbf", $"{DateTime.UtcNow.Subtract(DateTime.UnixEpoch).TotalSeconds:0}"),
+                new Claim("iat", $"{DateTime.UtcNow.Subtract(DateTime.UnixEpoch).TotalSeconds:0}"),
+                new Claim("exp", $"{DateTime.UtcNow.AddDays(30).Subtract(DateTime.UnixEpoch).TotalSeconds:0}"),
+                new Claim("RateLimit", $"{limit}")
+            ],
+            expires: DateTime.UtcNow.AddDays(30),
+            signingCredentials: new SigningCredentials(key, SecurityAlgorithms.HmacSha256)
+        );
+
+        var jwt = TokenHandler.WriteToken(token);
+        return Results.Ok(jwt);
+    }
+}

src/Helldivers-2-API/Extensions/ClaimsPrincipalExtensions.cs

@@ -0,0 +1,24 @@
+using System.Security.Claims;
+
+namespace Helldivers.API.Extensions;
+
+/// <summary>
+/// Contains extension methods for working with <see cref="ClaimsPrincipal" />s.
+/// </summary>
+public static class ClaimsPrincipalExtensions
+{
+    /// <summary>
+    /// Attempts to retrieve the integer value of a <see cref="Claim" /> with type <paramref name="type" />.
+    /// </summary>
+    /// <exception cref="InvalidOperationException">Thrown if the claim could not be found, or is not a valid integer.</exception>
+    public static int GetIntClaim(this ClaimsPrincipal user, string type)
+    {
+        var claim = user.Claims.FirstOrDefault(c =>
+            string.Equals(c.Type, type, StringComparison.InvariantCultureIgnoreCase));
+
+        if (claim is { Value: var str } && int.TryParse(str, out var result))
+            return result;
+
+        throw new InvalidOperationException($"Cannot fetch {type} or it is not a valid number");
+    }
+}

src/Helldivers-2-API/Helldivers-2-API.csproj

@@ -6,7 +6,7 @@
         <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
         <IncludeOpenAPIAnalyzers>true</IncludeOpenAPIAnalyzers>
         <EnableConfigurationBindingGenerator>true</EnableConfigurationBindingGenerator>
-    </PropertyGroup>
+  </PropertyGroup>
 
     <!-- Only generate OpenAPI docs for DEBUG builds -->
     <PropertyGroup Condition="'$(Configuration)' == 'Debug'">
@@ -23,23 +23,24 @@
 
     <!-- Dependencies for all build configurations -->
     <ItemGroup>
-        <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="8.0.1"/>
-        <ProjectReference Include="..\Helldivers-2-Models\Helldivers-2-Models.csproj"/>
-        <ProjectReference Include="..\Helldivers-2-Core\Helldivers-2-Core.csproj"/>
-        <ProjectReference Include="..\Helldivers-2-Sync\Helldivers-2-Sync.csproj"/>
-        <TrimmerRootAssembly Include="Helldivers-2-Models"/>
-        <TrimmerRootAssembly Include="Helldivers-2-Core"/>
-        <TrimmerRootAssembly Include="Helldivers-2-Sync"/>
+        <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="8.0.4" />
+        <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="8.0.1" />
+        <ProjectReference Include="..\Helldivers-2-Models\Helldivers-2-Models.csproj" />
+        <ProjectReference Include="..\Helldivers-2-Core\Helldivers-2-Core.csproj" />
+        <ProjectReference Include="..\Helldivers-2-Sync\Helldivers-2-Sync.csproj" />
+        <TrimmerRootAssembly Include="Helldivers-2-Models" />
+        <TrimmerRootAssembly Include="Helldivers-2-Core" />
+        <TrimmerRootAssembly Include="Helldivers-2-Sync" />
     </ItemGroup>
 
     <!-- Only include swagger dependencies in DEBUG builds -->
     <ItemGroup Condition="'$(Configuration)' == 'Debug'">
-        <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="8.0.3"/>
+        <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="8.0.3" />
         <PackageReference Include="Microsoft.Extensions.ApiDescription.Server" Version="8.0.3">
             <PrivateAssets>all</PrivateAssets>
             <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
         </PackageReference>
-        <PackageReference Include="NSwag.AspNetCore" Version="14.0.4"/>
+        <PackageReference Include="NSwag.AspNetCore" Version="14.0.4" />
     </ItemGroup>
 
 </Project>

src/Helldivers-2-API/Middlewares/RateLimitMiddleware.cs

@@ -1,29 +1,36 @@
-using Microsoft.Extensions.Caching.Memory;
+using Helldivers.API.Configuration;
+using Helldivers.API.Extensions;
+using Microsoft.Extensions.Caching.Memory;
+using Microsoft.Extensions.Options;
 using System.Net;
+using System.Security.Claims;
 using System.Threading.RateLimiting;
 
 namespace Helldivers.API.Middlewares;
 
 /// <summary>
 /// Handles applying rate limit logic to the API's requests.
 /// </summary>
-public sealed partial class RateLimitMiddleware(ILogger<RateLimitMiddleware> logger, IMemoryCache cache) : IMiddleware
+public sealed partial class RateLimitMiddleware(
+    ILogger<RateLimitMiddleware> logger,
+    IOptions<ApiConfiguration> options,
+    IMemoryCache cache
+) : IMiddleware
 {
-    // TODO: move to configurable policies.
-    private const int DefaultRequestLimit = 5;
-    private const int DefaultRequestWindow = 10;
-
-    [LoggerMessage(Level = LogLevel.Information, Message = "Retrieving rate limiter for {Key}")]
+    [LoggerMessage(Level = LogLevel.Debug, Message = "Retrieving rate limiter for {Key}")]
     private static partial void LogRateLimitKey(ILogger logger, IPAddress key);
 
+    [LoggerMessage(Level = LogLevel.Information, Message = "Retrieving rate limit for {Name} ({Limit})")]
+    private static partial void LogRateLimitForUser(ILogger logger, string name, int limit);
+
     /// <inheritdoc />
     public async Task InvokeAsync(HttpContext context, RequestDelegate next)
     {
         var limiter = GetRateLimiter(context);
         using var lease = await limiter.AcquireAsync(permitCount: 1, context.RequestAborted);
         if (limiter.GetStatistics() is { } statistics)
         {
-            context.Response.Headers["X-RateLimit-Limit"] = $"{DefaultRequestLimit}";
+            context.Response.Headers["X-RateLimit-Limit"] = $"{options.Value.RateLimit}";
             context.Response.Headers["X-RateLimit-Remaining"] = $"{statistics.CurrentAvailablePermits}";
             if (lease.TryGetMetadata(MetadataName.RetryAfter, out var retryAfter))
                 context.Response.Headers["Retry-After"] = $"{retryAfter.Seconds}";
@@ -40,20 +47,43 @@ public async Task InvokeAsync(HttpContext context, RequestDelegate next)
 
     private RateLimiter GetRateLimiter(HttpContext http)
     {
+        if (http.User.Identity?.IsAuthenticated ?? false)
+            return GetRateLimiterForUser(http.User);
+
         var key = http.Connection.RemoteIpAddress ?? IPAddress.Loopback;
         LogRateLimitKey(logger, key);
 
         return cache.GetOrCreate(key, entry =>
         {
-            entry.SlidingExpiration = TimeSpan.FromSeconds(DefaultRequestWindow);
+            entry.SlidingExpiration = TimeSpan.FromSeconds(options.Value.RateLimitWindow);
             return new TokenBucketRateLimiter(new()
             {
                 AutoReplenishment = true,
-                TokenLimit = DefaultRequestLimit,
-                TokensPerPeriod = DefaultRequestLimit,
+                TokenLimit = options.Value.RateLimit,
+                TokensPerPeriod = options.Value.RateLimit,
                 QueueLimit = 0,
-                ReplenishmentPeriod = TimeSpan.FromSeconds(DefaultRequestWindow)
+                ReplenishmentPeriod = TimeSpan.FromSeconds(options.Value.RateLimitWindow)
             });
         }) ?? throw new InvalidOperationException($"Creating rate limiter failed for {key}");
     }
+
+    private RateLimiter GetRateLimiterForUser(ClaimsPrincipal user)
+    {
+        var name = user.Identity?.Name!;
+        var limit = user.GetIntClaim("RateLimit");
+
+        LogRateLimitForUser(logger, name, limit);
+        return cache.GetOrCreate(name, entry =>
+        {
+            entry.SlidingExpiration = TimeSpan.FromSeconds(options.Value.RateLimitWindow);
+            return new TokenBucketRateLimiter(new()
+            {
+                AutoReplenishment = true,
+                TokenLimit = limit,
+                TokensPerPeriod = limit,
+                QueueLimit = 0,
+                ReplenishmentPeriod = TimeSpan.FromSeconds(options.Value.RateLimitWindow)
+            });
+        }) ?? throw new InvalidOperationException($"Creating rate limiter failed for {name}");
+    }
 }

src/Helldivers-2-API/OpenApi/DocumentProcessors/HelldiversDocumentProcessor.cs

@@ -10,7 +10,7 @@ namespace Helldivers.API.OpenApi.DocumentProcessors;
 /// </summary>
 public class HelldiversDocumentProcessor : IDocumentProcessor
 {
-    private const string HelldiversFlyServer = "https://helldivers-2-dotnet.fly.dev/";
+    private const string HelldiversFlyServer = "https://api.helldivers2.dev/";
     private const string LocalServer = "/";
 
     /// <inheritdoc />

src/Helldivers-2-API/Program.cs

@@ -1,3 +1,4 @@
+using Helldivers.API.Configuration;
 using Helldivers.API.Controllers;
 using Helldivers.API.Controllers.V1;
 using Helldivers.API.Middlewares;
@@ -6,9 +7,12 @@
 using Helldivers.Models.Domain.Localization;
 using Helldivers.Sync.Configuration;
 using Helldivers.Sync.Extensions;
+using Microsoft.AspNetCore.Authentication.JwtBearer;
 using Microsoft.AspNetCore.Http.Timeouts;
 using Microsoft.AspNetCore.HttpOverrides;
 using Microsoft.AspNetCore.Localization;
+using Microsoft.IdentityModel.Logging;
+using Microsoft.IdentityModel.Tokens;
 using System.Globalization;
 using System.Net;
 using System.Text.Json.Serialization;
@@ -81,6 +85,7 @@
 });
 
 // This configuration is bound here so that source generators kick in.
+builder.Services.Configure<ApiConfiguration>(builder.Configuration.GetSection("Helldivers:API"));
 builder.Services.Configure<HelldiversSyncConfiguration>(builder.Configuration.GetSection("Helldivers:Synchronization"));
 
 // If a request takes over 10s to complete, abort it.
@@ -103,6 +108,27 @@
     options.SerializerOptions.ReferenceHandler = ReferenceHandler.IgnoreCycles;
 });
 
+#if DEBUG
+IdentityModelEventSource.ShowPII = true;
+#endif
+builder.Services.AddAuthentication(JwtBearerDefaults.AuthenticationScheme).AddJwtBearer(options =>
+{
+    var config = new AuthenticationConfiguration();
+    builder.Configuration.GetSection("Helldivers:API:Authentication").Bind(config);
+
+    options.TokenValidationParameters = new()
+    {
+        ValidIssuers = config.ValidIssuers,
+        ValidAudiences = config.ValidAudiences,
+        IssuerSigningKey = new SymmetricSecurityKey(Convert.FromBase64String(config.SigningKey)),
+        ValidateIssuer = true,
+        ValidateAudience = true,
+        ValidateLifetime = true,
+        ValidateIssuerSigningKey = true
+    };
+});
+builder.Services.AddAuthorization();
+
 // Swagger is generated at compile time, so we don't include Swagger dependencies in Release builds.
 #if DEBUG
 // Only add OpenApi dependencies when generating 
@@ -167,6 +193,20 @@
 // Add middleware to timeout requests if they take too long.
 app.UseRequestTimeouts();
 
+#region API dev
+#if DEBUG
+
+var dev = app
+    .MapGroup("/dev")
+    .WithGroupName("development")
+    .WithTags("dev")
+    .ExcludeFromDescription();
+
+dev.MapGet("/token", DevelopmentController.CreateToken);
+
+#endif
+#endregion
+
 #region ArrowHead API endpoints ('raw' API)
 
 var raw = app

src/Helldivers-2-API/appsettings.Development.json

@@ -8,6 +8,11 @@
     }
   },
   "Helldivers": {
+    "API": {
+      "Authentication": {
+        "SigningKey": "I4eGmsXbDXfxlRo5N+w0ToRGN8aWSIaYWbZ2zMFqqnI="
+      }
+    },
     "Synchronization": {
       "IntervalSeconds": 300,
       "DefaultLanguage": "en-US",