From f5d9427d86c2168e226a745da9c935cb0c88d11b Mon Sep 17 00:00:00 2001 From: Friedrich von Never Date: Mon, 2 May 2022 14:17:17 +0700 Subject: [PATCH 01/16] (#147) Web: add and enable a web project template --- .../Controllers/WeatherForecastController.fs | 35 +++++++++++++++++++ Emulsion.Web/Emulsion.Web.fsproj | 12 +++++++ Emulsion.Web/WeatherForecast.fs | 11 ++++++ Emulsion.Web/WebServer.fs | 20 +++++++++++ Emulsion.sln | 6 ++++ Emulsion/Emulsion.fsproj | 1 + Emulsion/Program.fs | 25 ++++++++++--- 7 files changed, 105 insertions(+), 5 deletions(-) create mode 100644 Emulsion.Web/Controllers/WeatherForecastController.fs create mode 100644 Emulsion.Web/Emulsion.Web.fsproj create mode 100644 Emulsion.Web/WeatherForecast.fs create mode 100644 Emulsion.Web/WebServer.fs diff --git a/Emulsion.Web/Controllers/WeatherForecastController.fs b/Emulsion.Web/Controllers/WeatherForecastController.fs new file mode 100644 index 00000000..1d6e327f --- /dev/null +++ b/Emulsion.Web/Controllers/WeatherForecastController.fs @@ -0,0 +1,35 @@ +namespace Emulsion.Web.Controllers + +open System +open Microsoft.AspNetCore.Mvc +open Microsoft.Extensions.Logging +open Emulsion.Web + +[] +[] +type WeatherForecastController (logger : ILogger) = + inherit ControllerBase() + + let summaries = + [| + "Freezing" + "Bracing" + "Chilly" + "Cool" + "Mild" + "Warm" + "Balmy" + "Hot" + "Sweltering" + "Scorching" + |] + + [] + member _.Get() = + let rng = System.Random() + [| + for index in 0..4 -> + { Date = DateTime.Now.AddDays(float index) + TemperatureC = rng.Next(-20,55) + Summary = summaries.[rng.Next(summaries.Length)] } + |] diff --git a/Emulsion.Web/Emulsion.Web.fsproj b/Emulsion.Web/Emulsion.Web.fsproj new file mode 100644 index 00000000..cbce0ef0 --- /dev/null +++ b/Emulsion.Web/Emulsion.Web.fsproj @@ -0,0 +1,12 @@ + + + Library + net6.0 + + + + + + + + diff --git a/Emulsion.Web/WeatherForecast.fs b/Emulsion.Web/WeatherForecast.fs new file mode 100644 index 00000000..53494860 --- /dev/null +++ b/Emulsion.Web/WeatherForecast.fs @@ -0,0 +1,11 @@ +namespace Emulsion.Web + +open System + +type WeatherForecast = + { Date: DateTime + TemperatureC: int + Summary: string } + + member this.TemperatureF = + 32.0 + (float this.TemperatureC / 0.5556) diff --git a/Emulsion.Web/WebServer.fs b/Emulsion.Web/WebServer.fs new file mode 100644 index 00000000..c21afd15 --- /dev/null +++ b/Emulsion.Web/WebServer.fs @@ -0,0 +1,20 @@ +module Emulsion.Web.WebServer + +open System +open System.Threading.Tasks + +open Emulsion.Web.Controllers +open Microsoft.AspNetCore.Builder +open Microsoft.Extensions.DependencyInjection + +let run(baseUri: Uri): Task = + // TODO: Pass baseUri + let builder = WebApplication.CreateBuilder() + builder.Services + .AddControllers() + .AddApplicationPart(typeof.Assembly) + |> ignore + + let app = builder.Build() + app.MapControllers() |> ignore + app.RunAsync() diff --git a/Emulsion.sln b/Emulsion.sln index 0235560d..01c76983 100644 --- a/Emulsion.sln +++ b/Emulsion.sln @@ -40,6 +40,8 @@ EndProjectSection EndProject Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Emulsion.ContentProxy", "Emulsion.ContentProxy\Emulsion.ContentProxy.fsproj", "{A520FD41-A1CB-4062-AFBC-62A8BED12E81}" EndProject +Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Emulsion.Web", "Emulsion.Web\Emulsion.Web.fsproj", "{8EFD8F6C-43D3-4CE0-ADB8-63401E4A64FE}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -65,6 +67,10 @@ Global {A520FD41-A1CB-4062-AFBC-62A8BED12E81}.Debug|Any CPU.Build.0 = Debug|Any CPU {A520FD41-A1CB-4062-AFBC-62A8BED12E81}.Release|Any CPU.ActiveCfg = Release|Any CPU {A520FD41-A1CB-4062-AFBC-62A8BED12E81}.Release|Any CPU.Build.0 = Release|Any CPU + {8EFD8F6C-43D3-4CE0-ADB8-63401E4A64FE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8EFD8F6C-43D3-4CE0-ADB8-63401E4A64FE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8EFD8F6C-43D3-4CE0-ADB8-63401E4A64FE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8EFD8F6C-43D3-4CE0-ADB8-63401E4A64FE}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(NestedProjects) = preSolution {7D1ADF47-BF1C-4007-BB9B-08C283044467} = {973131E1-E645-4A50-A0D2-1886A1A8F0C6} diff --git a/Emulsion/Emulsion.fsproj b/Emulsion/Emulsion.fsproj index b1e52c65..d66ff988 100644 --- a/Emulsion/Emulsion.fsproj +++ b/Emulsion/Emulsion.fsproj @@ -46,5 +46,6 @@ + \ No newline at end of file diff --git a/Emulsion/Program.fs b/Emulsion/Program.fs index 52284261..652ae643 100644 --- a/Emulsion/Program.fs +++ b/Emulsion/Program.fs @@ -11,6 +11,7 @@ open Emulsion.Actors open Emulsion.Database open Emulsion.MessageSystem open Emulsion.Settings +open Emulsion.Web open Emulsion.Xmpp let private getConfiguration directory (fileName: string) = @@ -50,6 +51,13 @@ let private startApp config = | Some dbSettings -> do! migrateDatabase logger dbSettings | None -> () + let webServerTask = + config.Hosting + |> Option.map (fun hosting -> + logger.Information "Initializing web server…" + WebServer.run hosting.BaseUri + ) + logger.Information "Actor system preparation…" use system = ActorSystem.Create("emulsion") logger.Information "Clients preparation…" @@ -73,11 +81,18 @@ let private startApp config = let! xmppSystem = startMessageSystem logger xmpp core.Tell logger.Information "System ready" - logger.Information "Waiting for actor system termination…" - do! Async.AwaitTask system.WhenTerminated - logger.Information "Waiting for message systems termination…" - do! telegramSystem - do! xmppSystem + logger.Information "Waiting for the systems to terminate…" + let! _ = Async.Parallel(seq { + yield Async.AwaitTask system.WhenTerminated + yield telegramSystem + yield xmppSystem + + match webServerTask with + | Some task -> yield Async.AwaitTask task + | None -> () + }) + + logger.Information "Terminated successfully." with | error -> logger.Fatal(error, "General application failure") From 9e7305ec5ecce885b947b4780f3c994ad7373ea6 Mon Sep 17 00:00:00 2001 From: Friedrich von Never Date: Tue, 31 May 2022 23:28:15 +0700 Subject: [PATCH 02/16] (#147) Web: initial controller implementation --- Emulsion.ContentProxy/ContentStorage.fs | 4 +- Emulsion.Settings/Emulsion.Settings.fsproj | 16 +++++++ {Emulsion => Emulsion.Settings}/Settings.fs | 0 Emulsion.Tests/Telegram/LinkGeneratorTests.fs | 1 + Emulsion.Web/Controllers/ContentController.fs | 47 +++++++++++++++++++ .../Controllers/WeatherForecastController.fs | 35 -------------- Emulsion.Web/Emulsion.Web.fsproj | 10 ++-- Emulsion.Web/WeatherForecast.fs | 11 ----- Emulsion.Web/WebServer.fs | 5 +- Emulsion.sln | 6 +++ Emulsion/Emulsion.fsproj | 1 - 11 files changed, 82 insertions(+), 54 deletions(-) create mode 100644 Emulsion.Settings/Emulsion.Settings.fsproj rename {Emulsion => Emulsion.Settings}/Settings.fs (100%) create mode 100644 Emulsion.Web/Controllers/ContentController.fs delete mode 100644 Emulsion.Web/Controllers/WeatherForecastController.fs delete mode 100644 Emulsion.Web/WeatherForecast.fs diff --git a/Emulsion.ContentProxy/ContentStorage.fs b/Emulsion.ContentProxy/ContentStorage.fs index 8fc76717..11604e17 100644 --- a/Emulsion.ContentProxy/ContentStorage.fs +++ b/Emulsion.ContentProxy/ContentStorage.fs @@ -32,9 +32,9 @@ let getOrCreateMessageRecord (context: EmulsionDbContext) (id: MessageContentIde | Some item -> return item } -let getById (context: EmulsionDbContext) (id: int64): Async = async { +let getById (context: EmulsionDbContext) (id: int64): Async = async { return! query { for content in context.TelegramContents do where (content.Id = id) - } |> exactlyOneAsync + } |> tryExactlyOneAsync } diff --git a/Emulsion.Settings/Emulsion.Settings.fsproj b/Emulsion.Settings/Emulsion.Settings.fsproj new file mode 100644 index 00000000..bc1dee39 --- /dev/null +++ b/Emulsion.Settings/Emulsion.Settings.fsproj @@ -0,0 +1,16 @@ + + + + net6.0 + true + + + + + + + + + + + diff --git a/Emulsion/Settings.fs b/Emulsion.Settings/Settings.fs similarity index 100% rename from Emulsion/Settings.fs rename to Emulsion.Settings/Settings.fs diff --git a/Emulsion.Tests/Telegram/LinkGeneratorTests.fs b/Emulsion.Tests/Telegram/LinkGeneratorTests.fs index 04cbfcdf..1a04f3da 100644 --- a/Emulsion.Tests/Telegram/LinkGeneratorTests.fs +++ b/Emulsion.Tests/Telegram/LinkGeneratorTests.fs @@ -155,6 +155,7 @@ let private doDatabaseLinksTest (fileIds: string[]) message = let! content = DataStorage.transaction databaseSettings (fun context -> ContentStorage.getById context (Proxy.decodeHashId hostingSettings.HashIdSalt id) ) + let content = Option.get content Assert.Equal(message.MessageId, content.MessageId) Assert.Equal(message.Chat.Username, Some content.ChatUserName) diff --git a/Emulsion.Web/Controllers/ContentController.fs b/Emulsion.Web/Controllers/ContentController.fs new file mode 100644 index 00000000..df8b16bf --- /dev/null +++ b/Emulsion.Web/Controllers/ContentController.fs @@ -0,0 +1,47 @@ +namespace Emulsion.Web.Controllers + +open System.Globalization +open System.Threading.Tasks + +open Microsoft.AspNetCore.Mvc +open Microsoft.Extensions.Logging + +open Emulsion.ContentProxy +open Emulsion.Database +open Emulsion.Settings + +[] +[] +type ContentController(logger: ILogger, + configuration: HostingSettings, + context: EmulsionDbContext) = + inherit ControllerBase() + + let decodeHashId hashId = + try + Some <| Proxy.decodeHashId configuration.HashIdSalt hashId + with + | ex -> + logger.LogWarning(ex, "Error during hashId deserializing") + None + + let produceRedirect contentId: Async = async { + let! content = ContentStorage.getById context contentId + return + content + |> Option.map(fun c -> + let url = $"https://t.me/{c.ChatUserName}/{c.MessageId.ToString(CultureInfo.InvariantCulture)}" + RedirectResult url + ) + } + + + [] + member this.Get(hashId: string): Task = task { + match decodeHashId hashId with + | None -> return this.BadRequest() + | Some contentId -> + match! produceRedirect contentId with + | None -> return this.NotFound() :> IActionResult + | Some redirect -> return redirect + } diff --git a/Emulsion.Web/Controllers/WeatherForecastController.fs b/Emulsion.Web/Controllers/WeatherForecastController.fs deleted file mode 100644 index 1d6e327f..00000000 --- a/Emulsion.Web/Controllers/WeatherForecastController.fs +++ /dev/null @@ -1,35 +0,0 @@ -namespace Emulsion.Web.Controllers - -open System -open Microsoft.AspNetCore.Mvc -open Microsoft.Extensions.Logging -open Emulsion.Web - -[] -[] -type WeatherForecastController (logger : ILogger) = - inherit ControllerBase() - - let summaries = - [| - "Freezing" - "Bracing" - "Chilly" - "Cool" - "Mild" - "Warm" - "Balmy" - "Hot" - "Sweltering" - "Scorching" - |] - - [] - member _.Get() = - let rng = System.Random() - [| - for index in 0..4 -> - { Date = DateTime.Now.AddDays(float index) - TemperatureC = rng.Next(-20,55) - Summary = summaries.[rng.Next(summaries.Length)] } - |] diff --git a/Emulsion.Web/Emulsion.Web.fsproj b/Emulsion.Web/Emulsion.Web.fsproj index cbce0ef0..eaa95791 100644 --- a/Emulsion.Web/Emulsion.Web.fsproj +++ b/Emulsion.Web/Emulsion.Web.fsproj @@ -5,8 +5,12 @@ - - - + + + + + + + diff --git a/Emulsion.Web/WeatherForecast.fs b/Emulsion.Web/WeatherForecast.fs deleted file mode 100644 index 53494860..00000000 --- a/Emulsion.Web/WeatherForecast.fs +++ /dev/null @@ -1,11 +0,0 @@ -namespace Emulsion.Web - -open System - -type WeatherForecast = - { Date: DateTime - TemperatureC: int - Summary: string } - - member this.TemperatureF = - 32.0 + (float this.TemperatureC / 0.5556) diff --git a/Emulsion.Web/WebServer.fs b/Emulsion.Web/WebServer.fs index c21afd15..18b094c7 100644 --- a/Emulsion.Web/WebServer.fs +++ b/Emulsion.Web/WebServer.fs @@ -3,16 +3,17 @@ module Emulsion.Web.WebServer open System open System.Threading.Tasks -open Emulsion.Web.Controllers open Microsoft.AspNetCore.Builder open Microsoft.Extensions.DependencyInjection +open Emulsion.Web.Controllers + let run(baseUri: Uri): Task = // TODO: Pass baseUri let builder = WebApplication.CreateBuilder() builder.Services .AddControllers() - .AddApplicationPart(typeof.Assembly) + .AddApplicationPart(typeof.Assembly) |> ignore let app = builder.Build() diff --git a/Emulsion.sln b/Emulsion.sln index 01c76983..1c4da5bd 100644 --- a/Emulsion.sln +++ b/Emulsion.sln @@ -42,6 +42,8 @@ Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Emulsion.ContentProxy", "Em EndProject Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Emulsion.Web", "Emulsion.Web\Emulsion.Web.fsproj", "{8EFD8F6C-43D3-4CE0-ADB8-63401E4A64FE}" EndProject +Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Emulsion.Settings", "Emulsion.Settings\Emulsion.Settings.fsproj", "{88331E40-EF70-4AF1-99CA-8A849208CAB7}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -71,6 +73,10 @@ Global {8EFD8F6C-43D3-4CE0-ADB8-63401E4A64FE}.Debug|Any CPU.Build.0 = Debug|Any CPU {8EFD8F6C-43D3-4CE0-ADB8-63401E4A64FE}.Release|Any CPU.ActiveCfg = Release|Any CPU {8EFD8F6C-43D3-4CE0-ADB8-63401E4A64FE}.Release|Any CPU.Build.0 = Release|Any CPU + {88331E40-EF70-4AF1-99CA-8A849208CAB7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {88331E40-EF70-4AF1-99CA-8A849208CAB7}.Debug|Any CPU.Build.0 = Debug|Any CPU + {88331E40-EF70-4AF1-99CA-8A849208CAB7}.Release|Any CPU.ActiveCfg = Release|Any CPU + {88331E40-EF70-4AF1-99CA-8A849208CAB7}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(NestedProjects) = preSolution {7D1ADF47-BF1C-4007-BB9B-08C283044467} = {973131E1-E645-4A50-A0D2-1886A1A8F0C6} diff --git a/Emulsion/Emulsion.fsproj b/Emulsion/Emulsion.fsproj index d66ff988..ef4ab4b0 100644 --- a/Emulsion/Emulsion.fsproj +++ b/Emulsion/Emulsion.fsproj @@ -7,7 +7,6 @@ - From e80e7c5dc1ba1409f2ee29da3ee7ef0c79b802f8 Mon Sep 17 00:00:00 2001 From: Friedrich von Never Date: Sat, 2 Jul 2022 16:31:07 +0700 Subject: [PATCH 03/16] (#147) Web: add tests --- Emulsion.Tests/Emulsion.Tests.fsproj | 2 +- Emulsion.Tests/Web/ContentControllerTests.fs | 79 +++++++++++++++++++ .../{Controllers => }/ContentController.fs | 6 +- Emulsion.Web/Emulsion.Web.fsproj | 6 +- Emulsion.Web/WebServer.fs | 2 - Emulsion/Emulsion.fsproj | 2 +- 6 files changed, 88 insertions(+), 9 deletions(-) create mode 100644 Emulsion.Tests/Web/ContentControllerTests.fs rename Emulsion.Web/{Controllers => }/ContentController.fs (87%) diff --git a/Emulsion.Tests/Emulsion.Tests.fsproj b/Emulsion.Tests/Emulsion.Tests.fsproj index 724056e0..bcb0b36e 100644 --- a/Emulsion.Tests/Emulsion.Tests.fsproj +++ b/Emulsion.Tests/Emulsion.Tests.fsproj @@ -21,7 +21,6 @@ - @@ -33,6 +32,7 @@ + diff --git a/Emulsion.Tests/Web/ContentControllerTests.fs b/Emulsion.Tests/Web/ContentControllerTests.fs new file mode 100644 index 00000000..fa622df7 --- /dev/null +++ b/Emulsion.Tests/Web/ContentControllerTests.fs @@ -0,0 +1,79 @@ +namespace Emulsion.Tests.Web + +open System + +open System.Threading.Tasks +open Emulsion.ContentProxy +open Microsoft.AspNetCore.Mvc +open Microsoft.Extensions.Logging +open Serilog.Extensions.Logging +open Xunit +open Xunit.Abstractions + +open Emulsion.Database +open Emulsion.Database.Entities +open Emulsion.Settings +open Emulsion.Tests.TestUtils +open Emulsion.Tests.TestUtils.Logging +open Emulsion.Web + +type ContentControllerTests(output: ITestOutputHelper) = + + let hostingSettings = { + BaseUri = Uri "https://example.com/emulsion" + HashIdSalt = "test_salt" + } + + let logger = xunitLogger output + + let performTestWithPreparation prepareAction testAction = Async.StartAsTask(async { + return! TestDataStorage.doWithDatabase(fun databaseSettings -> async { + do! prepareAction databaseSettings + + use loggerFactory = new SerilogLoggerFactory(logger) + let logger = loggerFactory.CreateLogger() + use context = new EmulsionDbContext(databaseSettings.ContextOptions) + let controller = ContentController(logger, hostingSettings, context) + return! testAction controller + }) + }) + + let performTest = performTestWithPreparation(fun _ -> async.Return()) + + [] + member _.``ContentController returns BadRequest on hashId deserialization error``(): Task = + performTest (fun controller -> async { + let hashId = "z-z-z-z-z" + let! result = Async.AwaitTask <| controller.Get hashId + Assert.IsType result |> ignore + }) + + [] + member _.``ContentController returns NotFound if the content doesn't exist``(): Task = + performTest (fun controller -> async { + let hashId = Proxy.encodeHashId hostingSettings.HashIdSalt 667L + let! result = Async.AwaitTask <| controller.Get hashId + Assert.IsType result |> ignore + }) + + [] + member _.``ContentController returns a correct result``(): Task = + let contentId = 343L + let chatUserName = "MySuperExampleChat" + let messageId = 777L + performTestWithPreparation (fun databaseOptions -> async { + use context = new EmulsionDbContext(databaseOptions.ContextOptions) + let content = { + Id = contentId + ChatUserName = chatUserName + MessageId = messageId + FileId = "foobar" + } + do! DataStorage.addAsync context.TelegramContents content + return! Async.Ignore <| Async.AwaitTask(context.SaveChangesAsync()) + }) (fun controller -> async { + let hashId = Proxy.encodeHashId hostingSettings.HashIdSalt contentId + let! result = Async.AwaitTask <| controller.Get hashId + let redirect = Assert.IsType result + Assert.Equal($"https://t.me/{chatUserName}/{string messageId}", redirect.Url) + }) diff --git a/Emulsion.Web/Controllers/ContentController.fs b/Emulsion.Web/ContentController.fs similarity index 87% rename from Emulsion.Web/Controllers/ContentController.fs rename to Emulsion.Web/ContentController.fs index df8b16bf..4deacec6 100644 --- a/Emulsion.Web/Controllers/ContentController.fs +++ b/Emulsion.Web/ContentController.fs @@ -1,6 +1,5 @@ -namespace Emulsion.Web.Controllers +namespace Emulsion.Web -open System.Globalization open System.Threading.Tasks open Microsoft.AspNetCore.Mvc @@ -30,12 +29,11 @@ type ContentController(logger: ILogger, return content |> Option.map(fun c -> - let url = $"https://t.me/{c.ChatUserName}/{c.MessageId.ToString(CultureInfo.InvariantCulture)}" + let url = $"https://t.me/{c.ChatUserName}/{string c.MessageId}" RedirectResult url ) } - [] member this.Get(hashId: string): Task = task { match decodeHashId hashId with diff --git a/Emulsion.Web/Emulsion.Web.fsproj b/Emulsion.Web/Emulsion.Web.fsproj index eaa95791..0a4a0d3a 100644 --- a/Emulsion.Web/Emulsion.Web.fsproj +++ b/Emulsion.Web/Emulsion.Web.fsproj @@ -5,7 +5,7 @@ - + @@ -13,4 +13,8 @@ + + + + diff --git a/Emulsion.Web/WebServer.fs b/Emulsion.Web/WebServer.fs index 18b094c7..9eccb015 100644 --- a/Emulsion.Web/WebServer.fs +++ b/Emulsion.Web/WebServer.fs @@ -6,8 +6,6 @@ open System.Threading.Tasks open Microsoft.AspNetCore.Builder open Microsoft.Extensions.DependencyInjection -open Emulsion.Web.Controllers - let run(baseUri: Uri): Task = // TODO: Pass baseUri let builder = WebApplication.CreateBuilder() diff --git a/Emulsion/Emulsion.fsproj b/Emulsion/Emulsion.fsproj index ef4ab4b0..8f724464 100644 --- a/Emulsion/Emulsion.fsproj +++ b/Emulsion/Emulsion.fsproj @@ -38,7 +38,7 @@ - + From a22fc4ea2905c6ac08dc27dea04f1cf0db7540ba Mon Sep 17 00:00:00 2001 From: Friedrich von Never Date: Sat, 2 Jul 2022 18:13:02 +0700 Subject: [PATCH 04/16] (#147) Web: make it work properly in a manual test --- Emulsion.Web/ContentController.fs | 2 +- Emulsion.Web/Emulsion.Web.fsproj | 2 ++ Emulsion.Web/WebServer.fs | 14 +++++++++++--- Emulsion/Emulsion.fsproj | 4 ++-- Emulsion/Program.fs | 8 ++++---- 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/Emulsion.Web/ContentController.fs b/Emulsion.Web/ContentController.fs index 4deacec6..a7b1552a 100644 --- a/Emulsion.Web/ContentController.fs +++ b/Emulsion.Web/ContentController.fs @@ -34,7 +34,7 @@ type ContentController(logger: ILogger, ) } - [] + [] member this.Get(hashId: string): Task = task { match decodeHashId hashId with | None -> return this.BadRequest() diff --git a/Emulsion.Web/Emulsion.Web.fsproj b/Emulsion.Web/Emulsion.Web.fsproj index 0a4a0d3a..eb58afb6 100644 --- a/Emulsion.Web/Emulsion.Web.fsproj +++ b/Emulsion.Web/Emulsion.Web.fsproj @@ -15,6 +15,8 @@ + + diff --git a/Emulsion.Web/WebServer.fs b/Emulsion.Web/WebServer.fs index 9eccb015..3d77e8cf 100644 --- a/Emulsion.Web/WebServer.fs +++ b/Emulsion.Web/WebServer.fs @@ -1,15 +1,23 @@ module Emulsion.Web.WebServer -open System open System.Threading.Tasks +open Emulsion.Database +open Emulsion.Settings open Microsoft.AspNetCore.Builder open Microsoft.Extensions.DependencyInjection +open Serilog -let run(baseUri: Uri): Task = - // TODO: Pass baseUri +let run (logger: ILogger) (hostingSettings: HostingSettings) (databaseSettings: DatabaseSettings): Task = + // TODO: Use baseUri let builder = WebApplication.CreateBuilder() + + builder.Host.UseSerilog(logger) + |> ignore + builder.Services + .AddSingleton(hostingSettings) + .AddTransient(fun _ -> new EmulsionDbContext(databaseSettings.ContextOptions)) .AddControllers() .AddApplicationPart(typeof.Assembly) |> ignore diff --git a/Emulsion/Emulsion.fsproj b/Emulsion/Emulsion.fsproj index 8f724464..fdb0e1ad 100644 --- a/Emulsion/Emulsion.fsproj +++ b/Emulsion/Emulsion.fsproj @@ -38,8 +38,8 @@ - - + + diff --git a/Emulsion/Program.fs b/Emulsion/Program.fs index 652ae643..71cdc01c 100644 --- a/Emulsion/Program.fs +++ b/Emulsion/Program.fs @@ -52,11 +52,11 @@ let private startApp config = | None -> () let webServerTask = - config.Hosting - |> Option.map (fun hosting -> + match config.Hosting, config.Database with + | Some hosting, Some database -> logger.Information "Initializing web server…" - WebServer.run hosting.BaseUri - ) + Some <| WebServer.run logger hosting database + | _ -> None logger.Information "Actor system preparation…" use system = ActorSystem.Create("emulsion") From 7b8bb77544ee62bb46c4c39c5a661379e5802083 Mon Sep 17 00:00:00 2001 From: Friedrich von Never Date: Sat, 2 Jul 2022 18:28:02 +0700 Subject: [PATCH 05/16] (#147) LinkGenerator: deduplicate different sizes of the same photo --- Emulsion.Tests/Telegram/LinkGeneratorTests.fs | 8 ++++++++ Emulsion/Telegram/LinkGenerator.fs | 6 +++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/Emulsion.Tests/Telegram/LinkGeneratorTests.fs b/Emulsion.Tests/Telegram/LinkGeneratorTests.fs index 1a04f3da..228e76dd 100644 --- a/Emulsion.Tests/Telegram/LinkGeneratorTests.fs +++ b/Emulsion.Tests/Telegram/LinkGeneratorTests.fs @@ -87,6 +87,11 @@ let private messageWithMultiplePhotos = }|]) } +let private messageWithMultiplePhotoSizes = + let photoSize1 = messageWithPhoto.Photo |> Option.get |> Seq.head + let photoSize2 = { photoSize1 with Width = 100; Height = 200 } + { messageWithPhoto with Photo = Some [| photoSize1; photoSize2 |] } + let private messageWithSticker = { messageTemplate with Sticker = Some { @@ -197,3 +202,6 @@ let databaseVideoNoteTest(): unit = doDatabaseLinkTest fileId1 messageWithVideoN [] let databaseMultiplePhotosTest(): unit = doDatabaseLinksTest [|fileId1; fileId2|] messageWithMultiplePhotos + +[] +let databaseMultiplePhotoSizesTest(): unit = doDatabaseLinksTest [| fileId1 |] messageWithMultiplePhotoSizes diff --git a/Emulsion/Telegram/LinkGenerator.fs b/Emulsion/Telegram/LinkGenerator.fs index 167efec8..b3f2e9a4 100644 --- a/Emulsion/Telegram/LinkGenerator.fs +++ b/Emulsion/Telegram/LinkGenerator.fs @@ -36,7 +36,11 @@ let private getFileIds(message: FunogramMessage): string seq = Option.iter(fun o -> allFileIds.Add((^a) : (member FileId: string) o)) o let extractPhotoFileIds: PhotoSize seq option -> unit = - Option.iter(Seq.iter(fun photoSize -> allFileIds.Add(photoSize.FileId))) + Option.iter( + Seq.map(fun photoSize -> photoSize.FileId) + >> Seq.distinct + >> Seq.iter(allFileIds.Add) + ) extractFileId message.Document extractFileId message.Audio From b018aeec8799f78b51fd3ef5d137ffdb6dac1aab Mon Sep 17 00:00:00 2001 From: Friedrich von Never Date: Sat, 2 Jul 2022 18:29:51 +0700 Subject: [PATCH 06/16] (#147) .gitignore: ignore SQLite files --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index dab70302..0996b939 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,10 @@ out/ emulsion.json +*.db +*.db-shm +*.db-wal *.user .fake -.ionide \ No newline at end of file +.ionide From 4528cf0f6407ef7019ca70227e5e442115b9594b Mon Sep 17 00:00:00 2001 From: Friedrich von Never Date: Sat, 2 Jul 2022 18:30:04 +0700 Subject: [PATCH 07/16] (#147) ClientControllerTests: open formatting --- Emulsion.Tests/Web/ContentControllerTests.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Emulsion.Tests/Web/ContentControllerTests.fs b/Emulsion.Tests/Web/ContentControllerTests.fs index fa622df7..e0f4e1d2 100644 --- a/Emulsion.Tests/Web/ContentControllerTests.fs +++ b/Emulsion.Tests/Web/ContentControllerTests.fs @@ -1,15 +1,15 @@ namespace Emulsion.Tests.Web open System - open System.Threading.Tasks -open Emulsion.ContentProxy + open Microsoft.AspNetCore.Mvc open Microsoft.Extensions.Logging open Serilog.Extensions.Logging open Xunit open Xunit.Abstractions +open Emulsion.ContentProxy open Emulsion.Database open Emulsion.Database.Entities open Emulsion.Settings From 8cb25346e73657dc8988fc9f9ec1662f1a73f809 Mon Sep 17 00:00:00 2001 From: Friedrich von Never Date: Sat, 2 Jul 2022 19:04:16 +0700 Subject: [PATCH 08/16] (#147) WebServer: use configured startup URI, split bind URI and external URI --- Emulsion.Settings/Settings.fs | 17 ++++++++++------- Emulsion.Tests/SettingsTests.fs | 6 ++++-- Emulsion.Tests/Telegram/LinkGeneratorTests.fs | 7 ++++--- Emulsion.Tests/Web/ContentControllerTests.fs | 3 ++- Emulsion.Web/WebServer.fs | 5 ++--- Emulsion/Telegram/LinkGenerator.fs | 2 +- README.md | 7 +++++-- emulsion.example.json | 3 ++- 8 files changed, 30 insertions(+), 20 deletions(-) diff --git a/Emulsion.Settings/Settings.fs b/Emulsion.Settings/Settings.fs index 9ede62d6..bc89149d 100644 --- a/Emulsion.Settings/Settings.fs +++ b/Emulsion.Settings/Settings.fs @@ -29,7 +29,8 @@ type LogSettings = { } type HostingSettings = { - BaseUri: Uri + ExternalUriBase: Uri + BindUri: Uri HashIdSalt: string } @@ -78,16 +79,18 @@ let read (config : IConfiguration) : EmulsionSettings = |> Option.ofObj |> Option.map(fun dataSource -> { DataSource = dataSource }) let readHosting(section: IConfigurationSection) = - let baseUri = Option.ofObj section["baseUri"] + let externalUriBase = Option.ofObj section["externalUriBase"] + let bindUri = Option.ofObj section["bindUri"] let hashIdSalt = Option.ofObj section["hashIdSalt"] - match baseUri, hashIdSalt with - | Some baseUri, Some hashIdSalt -> + match externalUriBase, bindUri, hashIdSalt with + | Some externalUriBase, Some bindUri, Some hashIdSalt -> Some { - BaseUri = Uri baseUri + ExternalUriBase = Uri externalUriBase + BindUri = Uri bindUri HashIdSalt = hashIdSalt } - | None, None -> None - | other -> failwith $"Pair {other} is not valid for hosting settings" + | None, None, None -> None + | other -> failwith $"Parameter pack {other} represents invalid hosting settings." { Xmpp = readXmpp <| config.GetSection("xmpp") Telegram = readTelegram <| config.GetSection("telegram") diff --git a/Emulsion.Tests/SettingsTests.fs b/Emulsion.Tests/SettingsTests.fs index d1546adc..abfc40bf 100644 --- a/Emulsion.Tests/SettingsTests.fs +++ b/Emulsion.Tests/SettingsTests.fs @@ -81,7 +81,8 @@ let ``Extended settings read properly``(): Task = task { ""dataSource"": "":memory:"" }, ""hosting"": { - ""baseUri"": ""https://example.com"", + ""externalUriBase"": ""https://example.com"", + ""bindUri"": ""http://localhost:5555"", ""hashIdSalt"": ""123123123"" }" let expectedConfiguration = @@ -90,7 +91,8 @@ let ``Extended settings read properly``(): Task = task { DataSource = ":memory:" } Hosting = Some { - BaseUri = Uri("https://example.com") + ExternalUriBase = Uri "https://example.com" + BindUri = Uri "http://localhost:5555" HashIdSalt = "123123123" } } diff --git a/Emulsion.Tests/Telegram/LinkGeneratorTests.fs b/Emulsion.Tests/Telegram/LinkGeneratorTests.fs index 228e76dd..fc8103f8 100644 --- a/Emulsion.Tests/Telegram/LinkGeneratorTests.fs +++ b/Emulsion.Tests/Telegram/LinkGeneratorTests.fs @@ -13,7 +13,8 @@ open Emulsion.Telegram open Emulsion.Tests.TestUtils let private hostingSettings = { - BaseUri = Uri "https://example.com" + ExternalUriBase = Uri "https://example.com" + BindUri = Uri "http://localhost:5556" HashIdSalt = "mySalt" } let private chatName = "test_chat" @@ -153,9 +154,9 @@ let private doDatabaseLinksTest (fileIds: string[]) message = let contentLinks = Seq.toArray links.ContentLinks for fileId, link in Seq.zip fileIds contentLinks do let link = link.ToString() - let baseUri = hostingSettings.BaseUri.ToString() + let baseUri = hostingSettings.ExternalUriBase.ToString() Assert.StartsWith(baseUri, link) - let emptyLinkLength = (Proxy.getLink hostingSettings.BaseUri "").ToString().Length + let emptyLinkLength = (Proxy.getLink hostingSettings.ExternalUriBase "").ToString().Length let id = link.Substring(emptyLinkLength) let! content = DataStorage.transaction databaseSettings (fun context -> ContentStorage.getById context (Proxy.decodeHashId hostingSettings.HashIdSalt id) diff --git a/Emulsion.Tests/Web/ContentControllerTests.fs b/Emulsion.Tests/Web/ContentControllerTests.fs index e0f4e1d2..564b485c 100644 --- a/Emulsion.Tests/Web/ContentControllerTests.fs +++ b/Emulsion.Tests/Web/ContentControllerTests.fs @@ -20,7 +20,8 @@ open Emulsion.Web type ContentControllerTests(output: ITestOutputHelper) = let hostingSettings = { - BaseUri = Uri "https://example.com/emulsion" + ExternalUriBase = Uri "https://example.com/emulsion" + BindUri = Uri "http://localhost:5557" HashIdSalt = "test_salt" } diff --git a/Emulsion.Web/WebServer.fs b/Emulsion.Web/WebServer.fs index 3d77e8cf..24a71a21 100644 --- a/Emulsion.Web/WebServer.fs +++ b/Emulsion.Web/WebServer.fs @@ -9,8 +9,7 @@ open Microsoft.Extensions.DependencyInjection open Serilog let run (logger: ILogger) (hostingSettings: HostingSettings) (databaseSettings: DatabaseSettings): Task = - // TODO: Use baseUri - let builder = WebApplication.CreateBuilder() + let builder = WebApplication.CreateBuilder(WebApplicationOptions()) builder.Host.UseSerilog(logger) |> ignore @@ -24,4 +23,4 @@ let run (logger: ILogger) (hostingSettings: HostingSettings) (databaseSettings: let app = builder.Build() app.MapControllers() |> ignore - app.RunAsync() + app.RunAsync(hostingSettings.BindUri.ToString()) diff --git a/Emulsion/Telegram/LinkGenerator.fs b/Emulsion/Telegram/LinkGenerator.fs index b3f2e9a4..1e064419 100644 --- a/Emulsion/Telegram/LinkGenerator.fs +++ b/Emulsion/Telegram/LinkGenerator.fs @@ -83,7 +83,7 @@ let gatherLinks (logger: ILogger) ) let hashId = Proxy.encodeHashId hostingSettings.HashIdSalt content.Id - return Proxy.getLink hostingSettings.BaseUri hashId + return Proxy.getLink hostingSettings.ExternalUriBase hashId }) |> Async.Parallel return links diff --git a/README.md b/README.md index 9e286f7a..fd5f17be 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,8 @@ There's **unfinished** Telegram content proxy support. To enable it, configure t "dataSource": "sqliteDatabase.db" }, "hosting": { - "baseUri": "https://example.com/api/content", + "externalUriBase": "https://example.com/api/", + "bindUri": "http://localhost:5000/", "hashIdSalt": "test" } } @@ -52,7 +53,9 @@ There's **unfinished** Telegram content proxy support. To enable it, configure t `dataSource` may be a path to the SQLite database file on disk. If set, Emulsion will automatically apply necessary migrations to this database on startup. -If all the parameters are set, then Emulsion will save the incoming messages into the database, and will then insert links to `{baseUri}/content/{contentId}` instead of links to `https://t.me/{messageId}`. +If all the parameters are set, then Emulsion will save the incoming messages into the database, and will then insert links to `{externalUriBase}/content/{contentId}` instead of links to `https://t.me/{messageId}`. + +`bindUri` designates the URI the web server will listen locally (which may or may not be the same as the `externalUriBase`). The content identifiers in question are generated from the database ones using the [hashids.net][hashids.net] library, `hashIdSalt` is used in generation. This should complicate guessing of content ids for any external party not reading the chat directly. diff --git a/emulsion.example.json b/emulsion.example.json index 8263c050..0b7e79b6 100644 --- a/emulsion.example.json +++ b/emulsion.example.json @@ -21,7 +21,8 @@ "dataSource": "sqliteDatabase.db" }, "hosting": { - "baseUri": "https://example.com/api/content", + "externalUriBase": "https://example.com/api/", + "bindUri": "https://localhost:5000", "hashIdSalt": "test" } } From 59db204b5df95d2522cee09f76b7766e310d8c67 Mon Sep 17 00:00:00 2001 From: Friedrich von Never Date: Sat, 2 Jul 2022 20:14:19 +0700 Subject: [PATCH 09/16] (#147) Funogram: update to 6.1.0.1 This update has introduced a FileUniqueId support, that's required to properly handle the photos of different sizes. --- Emulsion.Tests/Telegram/FunogramTests.fs | 134 ++++++++++-------- Emulsion.Tests/Telegram/LinkGeneratorTests.fs | 31 ++-- Emulsion/Emulsion.fsproj | 2 +- Emulsion/Telegram/Client.fs | 2 +- Emulsion/Telegram/Funogram.fs | 4 +- Emulsion/Telegram/LinkGenerator.fs | 4 +- 6 files changed, 107 insertions(+), 70 deletions(-) diff --git a/Emulsion.Tests/Telegram/FunogramTests.fs b/Emulsion.Tests/Telegram/FunogramTests.fs index 337db4e2..7bb1ecff 100644 --- a/Emulsion.Tests/Telegram/FunogramTests.fs +++ b/Emulsion.Tests/Telegram/FunogramTests.fs @@ -18,32 +18,34 @@ let private groupId = 200600L [] let private chatName = "test_room" -let private createUser username firstName lastName = { - Id = 0L - FirstName = firstName - LastName = lastName - Username = username - LanguageCode = None - IsBot = false -} - -let private currentChat = { - defaultChat with - Id = groupId - Username = Some chatName - Type = SuperGroup -} +let private createUser username firstName lastName = User.Create( + id = 0L, + isBot = false, + firstName = firstName, + ?lastName = lastName, + ?username = username +) + +let private currentChat = Chat.Create( + id = groupId, + ``type`` = ChatType.SuperGroup, + username = chatName +) + +let private defaultMessage = Message.Create( + messageId = 1L, + date = DateTime.MinValue, + chat = currentChat +) let private createMessage from text : Funogram.Telegram.Types.Message = { defaultMessage with From = from - Chat = currentChat Text = text } let private createEmptyMessage from : Funogram.Telegram.Types.Message = { defaultMessage with - From = Some from - Chat = currentChat } + From = Some from } let private createReplyMessage from text replyTo : Funogram.Telegram.Types.Message = { createMessage from text with @@ -52,7 +54,6 @@ let private createReplyMessage from text replyTo : Funogram.Telegram.Types.Messa let private createForwardedMessage from (forwarded: Funogram.Telegram.Types.Message) = { defaultMessage with From = Some from - Chat = currentChat ForwardFrom = forwarded.From Text = forwarded.Text } @@ -62,6 +63,7 @@ let private createStickerMessage from emoji = Chat = currentChat Sticker = Some { FileId = "" + FileUniqueId = "" Width = 0 Height = 0 Thumb = None @@ -70,31 +72,36 @@ let private createStickerMessage from emoji = SetName = None MaskPosition = None IsAnimated = false + IsVideo = false + PremiumAnimation = None } } -let private createPhoto() = seq { - { FileId = "" - Width = 0 - Height = 0 - FileSize = None +let private createPhoto() = [| + { + FileId = "" + FileUniqueId = "" + Width = 0 + Height = 0 + FileSize = None } +|] + +let private createAnimation(): Animation = { + FileId = "" + FileUniqueId = "" + Width = 0 + Height = 0 + Duration = 0 + Thumb = None + FileName = None + MimeType = None + FileSize = None } -let private createAnimation() = - { FileId = "" - Width = 0 - Height = 0 - Duration = 0 - Thumb = None - FileName = None - MimeType = None - FileSize = None } - let private createMessageWithCaption from caption = { defaultMessage with From = Some from - Chat = currentChat Caption = Some caption } let private createPoll from (question: string) (options: string[]) = @@ -105,11 +112,16 @@ let private createPoll from (question: string) (options: string[]) = VoterCount = 0 }) - let poll: Poll = - { Id = "" - Question = question - Options = options - IsClosed = false } + let poll= Poll.Create( + id = "", + question = question, + options = options, + totalVoterCount = 0L, + isClosed = false, + isAnonymous = false, + ``type`` = "", + allowsMultipleAnswers = false + ) { defaultMessage with From = Some from @@ -131,11 +143,12 @@ let private createEntity t offset length url = { Length = length Url = Some url User = None + Language = None } -let private createEntities t offset length url = Some <| seq { +let private createEntities t offset length url = Some <| [| createEntity t offset length url -} +|] let private originalUser = createUser (Some "originalUser") "" None let private replyingUser = createUser (Some "replyingUser") "" None @@ -293,7 +306,8 @@ module ReadMessageTests = Url = None Offset = 0L Length = 5L - User = None } |] } + User = None + Language = None } |] } Assert.Equal( authoredTelegramMessage "@originalUser" "Original text", readMessage message @@ -459,7 +473,7 @@ module ReadMessageTests = [] let readUserEntersChat() = let message = { createEmptyMessage originalUser with - NewChatMembers = Some <| seq { originalUser } } + NewChatMembers = Some <| [| originalUser |] } Assert.Equal( eventTelegramMessage "@originalUser has entered the chat", readMessage message @@ -477,7 +491,7 @@ module ReadMessageTests = let readAddedChatMember() = let newUser = createUser None "FirstName1" None let message = { createEmptyMessage originalUser with - NewChatMembers = Some <| seq { newUser } } + NewChatMembers = Some <| [| newUser |] } Assert.Equal( eventTelegramMessage "@originalUser has added FirstName1 the chat", readMessage message @@ -485,10 +499,10 @@ module ReadMessageTests = [] let readNewChatMembers() = - let newUsers = seq { + let newUsers = [| createUser None "FirstName1" None createUser None "FirstName2" None - } + |] let message = { createEmptyMessage originalUser with NewChatMembers = Some newUsers } Assert.Equal( @@ -498,11 +512,11 @@ module ReadMessageTests = [] let readMoreChatMembers() = - let newUsers = seq { + let newUsers = [| createUser None "FirstName1" None createUser None "FirstName2" None createUser None "FirstName3" None - } + |] let message = { createEmptyMessage originalUser with NewChatMembers = Some newUsers } Assert.Equal( @@ -526,12 +540,17 @@ module ReadMessageTests = From = Some { createUser None "" None with Id = selfUserId } Text = Some "Myself\nTests" - Entities = Some <| seq { - yield { Type = "bold" - Offset = 0L - Length = int64 "Myself".Length - Url = None - User = None } } } + Entities = Some <| [| + { + Type = "bold" + Offset = 0L + Length = int64 "Myself".Length + Url = None + User = None + Language = None + } + |] + } let reply = createReplyMessage (Some replyingUser) (Some "reply text") originalMessage Assert.Equal( authoredTelegramReplyMessage "@replyingUser" "reply text" (authoredTelegramMessage "Myself" "Tests").main, @@ -545,7 +564,10 @@ module ProcessMessageTests = [] let messageFromOtherChatShouldBeIgnored(): unit = let message = { createMessage (Some originalUser) (Some "test") with - Chat = defaultChat } + Chat = Chat.Create( + id = 0L, + ``type`` = ChatType.SuperGroup + ) } Assert.Equal(None, processMessage message) module ProcessSendResultTests = diff --git a/Emulsion.Tests/Telegram/LinkGeneratorTests.fs b/Emulsion.Tests/Telegram/LinkGeneratorTests.fs index fc8103f8..d89078c4 100644 --- a/Emulsion.Tests/Telegram/LinkGeneratorTests.fs +++ b/Emulsion.Tests/Telegram/LinkGeneratorTests.fs @@ -22,18 +22,21 @@ let private fileId1 = "123456" let private fileId2 = "654321" let private messageTemplate = - { defaultMessage with - Chat = - { defaultChat with - Type = SuperGroup - Username = Some chatName - } - } + Message.Create( + messageId = 0L, + date = DateTime.MinValue, + chat = Chat.Create( + id = 0L, + ``type`` = ChatType.SuperGroup, + username = chatName + ) + ) let private messageWithDocument = { messageTemplate with Document = Some { FileId = fileId1 + FileUniqueId = fileId1 Thumb = None FileName = None MimeType = None @@ -45,6 +48,8 @@ let private messageWithAudio = { messageTemplate with Audio = Some { FileId = fileId1 + FileUniqueId = fileId1 + FileName = None Duration = 0 Performer = None Title = None @@ -58,6 +63,7 @@ let private messageWithAnimation = { messageTemplate with Animation = Some { FileId = fileId1 + FileUniqueId = fileId1 Width = 0 Height = 0 Duration = 0 @@ -72,6 +78,7 @@ let private messageWithPhoto = { messageTemplate with Photo = Some([|{ FileId = fileId1 + FileUniqueId = fileId1 Width = 0 Height = 0 FileSize = None @@ -80,8 +87,9 @@ let private messageWithPhoto = let private messageWithMultiplePhotos = { messageWithPhoto with - Photo = Some(Seq.append (Option.get messageWithPhoto.Photo) [|{ + Photo = Some(Array.append (Option.get messageWithPhoto.Photo) [|{ FileId = fileId2 + FileUniqueId = fileId2 Width = 0 Height = 0 FileSize = None @@ -97,6 +105,7 @@ let private messageWithSticker = { messageTemplate with Sticker = Some { FileId = fileId1 + FileUniqueId = fileId1 Width = 0 Height = 0 IsAnimated = false @@ -105,6 +114,8 @@ let private messageWithSticker = SetName = None MaskPosition = None FileSize = None + IsVideo = false + PremiumAnimation = None } } @@ -112,6 +123,8 @@ let private messageWithVideo = { messageTemplate with Video = Some { FileId = fileId1 + FileUniqueId = fileId1 + FileName = None Width = 0 Height = 0 Duration = 0 @@ -125,6 +138,7 @@ let private messageWithVoice = { messageTemplate with Voice = Some { FileId = fileId1 + FileUniqueId = fileId1 Duration = 0 MimeType = None FileSize = None @@ -135,6 +149,7 @@ let private messageWithVideoNote = { messageTemplate with VideoNote = Some { FileId = fileId1 + FileUniqueId = fileId1 Length = 0 Duration = 0 Thumb = None diff --git a/Emulsion/Emulsion.fsproj b/Emulsion/Emulsion.fsproj index fdb0e1ad..f968adb0 100644 --- a/Emulsion/Emulsion.fsproj +++ b/Emulsion/Emulsion.fsproj @@ -34,7 +34,7 @@ - + diff --git a/Emulsion/Telegram/Client.fs b/Emulsion/Telegram/Client.fs index 02aa86ef..bf42b1ef 100644 --- a/Emulsion/Telegram/Client.fs +++ b/Emulsion/Telegram/Client.fs @@ -13,7 +13,7 @@ type Client(ctx: ServiceContext, hostingSettings: HostingSettings option) = inherit MessageSystemBase(ctx, cancellationToken) - let botConfig = { Funogram.Telegram.Bot.defaultConfig with Token = telegramSettings.Token } + let botConfig = { Funogram.Telegram.Bot.Config.defaultConfig with Token = telegramSettings.Token } override _.RunUntilError receiver = async { // Run loop of Telegram is in no need of any complicated start, so just return an async that will perform it: diff --git a/Emulsion/Telegram/Funogram.fs b/Emulsion/Telegram/Funogram.fs index e3451f54..fbf7166b 100644 --- a/Emulsion/Telegram/Funogram.fs +++ b/Emulsion/Telegram/Funogram.fs @@ -348,8 +348,8 @@ let internal prepareHtmlMessage: Message -> string = function | Event {text = text} -> Html.escape text let send (settings: TelegramSettings) (botConfig: BotConfig) (OutgoingMessage content): Async = - let sendHtmlMessage groupId text = - Api.sendMessageBase groupId text (Some ParseMode.HTML) None None None None + let sendHtmlMessage (groupId: ChatId) text = + Req.SendMessage.Make(groupId, text, ParseMode.HTML) let groupId = Int(int64 settings.GroupId) let message = prepareHtmlMessage content diff --git a/Emulsion/Telegram/LinkGenerator.fs b/Emulsion/Telegram/LinkGenerator.fs index 1e064419..3931a1df 100644 --- a/Emulsion/Telegram/LinkGenerator.fs +++ b/Emulsion/Telegram/LinkGenerator.fs @@ -35,7 +35,7 @@ let private getFileIds(message: FunogramMessage): string seq = let inline extractFileId(o: ^a option) = Option.iter(fun o -> allFileIds.Add((^a) : (member FileId: string) o)) o - let extractPhotoFileIds: PhotoSize seq option -> unit = + let extractPhotoFileIds: PhotoSize[] option -> unit = Option.iter( Seq.map(fun photoSize -> photoSize.FileId) >> Seq.distinct @@ -53,7 +53,7 @@ let private getFileIds(message: FunogramMessage): string seq = allFileIds -let private getContentIdentities message: ContentStorage.MessageContentIdentity seq = +let private getContentIdentities(message: FunogramMessage): ContentStorage.MessageContentIdentity seq = match message.Chat with | { Type = SuperGroup Username = Some chatName } -> From 238f6edaacf0d9f80b287c0c476814da505a107e Mon Sep 17 00:00:00 2001 From: Friedrich von Never Date: Sat, 2 Jul 2022 20:25:58 +0700 Subject: [PATCH 10/16] (#147) LinkGenerator: only extract the biggest photo from each size group --- Emulsion.Tests/Telegram/LinkGeneratorTests.fs | 17 ++++++++++++++--- Emulsion/Telegram/LinkGenerator.fs | 11 +++++++++-- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/Emulsion.Tests/Telegram/LinkGeneratorTests.fs b/Emulsion.Tests/Telegram/LinkGeneratorTests.fs index d89078c4..dadc312b 100644 --- a/Emulsion.Tests/Telegram/LinkGeneratorTests.fs +++ b/Emulsion.Tests/Telegram/LinkGeneratorTests.fs @@ -20,6 +20,7 @@ let private hostingSettings = { let private chatName = "test_chat" let private fileId1 = "123456" let private fileId2 = "654321" +let private fileId3 = "555555" let private messageTemplate = Message.Create( @@ -97,9 +98,19 @@ let private messageWithMultiplePhotos = } let private messageWithMultiplePhotoSizes = + // Create multiple photos with the same FileUniqueId but different file ids: let photoSize1 = messageWithPhoto.Photo |> Option.get |> Seq.head - let photoSize2 = { photoSize1 with Width = 100; Height = 200 } - { messageWithPhoto with Photo = Some [| photoSize1; photoSize2 |] } + let photoSize2 = { photoSize1 with FileId = fileId2; Width = photoSize1.Width + 1L; Height = photoSize1.Height + 1L } + let photoSize3 = { photoSize1 with FileId = fileId3 } + + Assert.Equal(photoSize1.FileUniqueId, photoSize2.FileUniqueId) + Assert.Equal(photoSize1.FileUniqueId, photoSize3.FileUniqueId) + + Assert.NotEqual(photoSize1.FileId, photoSize2.FileId) + Assert.NotEqual(photoSize1.FileId, photoSize3.FileId) + Assert.NotEqual(photoSize2.FileId, photoSize3.FileId) + + { messageWithPhoto with Photo = Some [| photoSize1; photoSize2; photoSize3 |] } let private messageWithSticker = { messageTemplate with @@ -220,4 +231,4 @@ let databaseVideoNoteTest(): unit = doDatabaseLinkTest fileId1 messageWithVideoN let databaseMultiplePhotosTest(): unit = doDatabaseLinksTest [|fileId1; fileId2|] messageWithMultiplePhotos [] -let databaseMultiplePhotoSizesTest(): unit = doDatabaseLinksTest [| fileId1 |] messageWithMultiplePhotoSizes +let databaseMultiplePhotoSizesTest(): unit = doDatabaseLinksTest [| fileId2 |] messageWithMultiplePhotoSizes diff --git a/Emulsion/Telegram/LinkGenerator.fs b/Emulsion/Telegram/LinkGenerator.fs index 3931a1df..de9f1664 100644 --- a/Emulsion/Telegram/LinkGenerator.fs +++ b/Emulsion/Telegram/LinkGenerator.fs @@ -37,8 +37,15 @@ let private getFileIds(message: FunogramMessage): string seq = let extractPhotoFileIds: PhotoSize[] option -> unit = Option.iter( - Seq.map(fun photoSize -> photoSize.FileId) - >> Seq.distinct + // In one message, several unique photos may have several sizes each. Get the biggest photo for each unique + // file: + Seq.groupBy(fun photoSize -> photoSize.FileUniqueId) + >> Seq.map(fun (_uniqueId, sizes) -> + sizes + |> Seq.sortByDescending(fun size -> size.Height * size.Width) + |> Seq.head + |> fun photoSize -> photoSize.FileId + ) >> Seq.iter(allFileIds.Add) ) From 929573f6ae4b2094e401a7ef4acc65aa244dded3 Mon Sep 17 00:00:00 2001 From: Friedrich von Never Date: Sat, 2 Jul 2022 21:00:01 +0700 Subject: [PATCH 11/16] (#147) Docs: update the notes on the web proxy --- README.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index fd5f17be..8e9e4bec 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,9 @@ Note that `pingInterval` of `null` disables XMPP ping support. ### Telegram Content Proxy -There's **unfinished** Telegram content proxy support. To enable it, configure the `database` and `hosting` configuration file sections: +There's **unfinished** Telegram content proxy support, for XMPP users to access Telegram content without directly opening links on t.me. Right now, it will only generate a redirect to the corresponding t.me URI, so it doesn't help a lot. But in the future, proper content proxy will be supported. + +To enable it, configure the `database` and `hosting` configuration file sections: ```json { @@ -59,6 +61,14 @@ If all the parameters are set, then Emulsion will save the incoming messages int The content identifiers in question are generated from the database ones using the [hashids.net][hashids.net] library, `hashIdSalt` is used in generation. This should complicate guessing of content ids for any external party not reading the chat directly. +### Recommended Network Configuration + +Current configuration system allows the following: + +1. Set up a reverse proxy for, say, `https://example.com/telegram` taking the content from `http://localhost/`. +2. When receiving a piece of Telegram content (a file, a photo, an audio message), the bot will send a link to `https://example.com/telegram/content/` to the XMPP chat. +3. When anyone visits the link, the reverse proxy will send a request to `http://localhost/content/`, which will take a corresponding content from the database. + Test ---- From c14512b9cfd22b146e94c8997c2ecde14ba125d6 Mon Sep 17 00:00:00 2001 From: Friedrich von Never Date: Sat, 2 Jul 2022 21:36:41 +0700 Subject: [PATCH 12/16] (#147) Docs: add a changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 760bd2f9..15d058c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning v2.0.0](https://semver.org/spec/v2.0.0.html). +## [Unreleased] +### Added +- [#147: Telegram content redirector support](https://github.com/codingteam/emulsion/issues/147). Emulsion is now able to generate redirects to t.me from its own embedded web server, knowing the internal content id. + + This feature is not very useful, yet (earlier, the same t.me links were already available to the users directly), but it is the first step for further development of more valuable forms of content proxying. + ## [1.9.0] - 2022-06-01 ### Changed - Runtime: upgrade to .NET 6 From b032537d36e479fc9c55b81bf5bc8e625f406990 Mon Sep 17 00:00:00 2001 From: Friedrich von Never Date: Sun, 3 Jul 2022 16:00:03 +0700 Subject: [PATCH 13/16] (#147) Funogram: update to 2.0.4 --- Emulsion/Emulsion.fsproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Emulsion/Emulsion.fsproj b/Emulsion/Emulsion.fsproj index f968adb0..17fd7ace 100644 --- a/Emulsion/Emulsion.fsproj +++ b/Emulsion/Emulsion.fsproj @@ -33,7 +33,7 @@ - + From ad66bd9f3eb05b9367ceab3cfe0c2704982c7246 Mon Sep 17 00:00:00 2001 From: Friedrich von Never Date: Sun, 3 Jul 2022 16:01:53 +0700 Subject: [PATCH 14/16] (#147) LinkGenerator: fix the issue with photo links --- Emulsion.Tests/Telegram/LinkGeneratorTests.fs | 25 +++---------------- Emulsion/Telegram/LinkGenerator.fs | 19 ++++++-------- 2 files changed, 10 insertions(+), 34 deletions(-) diff --git a/Emulsion.Tests/Telegram/LinkGeneratorTests.fs b/Emulsion.Tests/Telegram/LinkGeneratorTests.fs index dadc312b..474a4df9 100644 --- a/Emulsion.Tests/Telegram/LinkGeneratorTests.fs +++ b/Emulsion.Tests/Telegram/LinkGeneratorTests.fs @@ -20,7 +20,6 @@ let private hostingSettings = { let private chatName = "test_chat" let private fileId1 = "123456" let private fileId2 = "654321" -let private fileId3 = "555555" let private messageTemplate = Message.Create( @@ -91,27 +90,12 @@ let private messageWithMultiplePhotos = Photo = Some(Array.append (Option.get messageWithPhoto.Photo) [|{ FileId = fileId2 FileUniqueId = fileId2 - Width = 0 - Height = 0 + Width = 1000 + Height = 2000 FileSize = None }|]) } -let private messageWithMultiplePhotoSizes = - // Create multiple photos with the same FileUniqueId but different file ids: - let photoSize1 = messageWithPhoto.Photo |> Option.get |> Seq.head - let photoSize2 = { photoSize1 with FileId = fileId2; Width = photoSize1.Width + 1L; Height = photoSize1.Height + 1L } - let photoSize3 = { photoSize1 with FileId = fileId3 } - - Assert.Equal(photoSize1.FileUniqueId, photoSize2.FileUniqueId) - Assert.Equal(photoSize1.FileUniqueId, photoSize3.FileUniqueId) - - Assert.NotEqual(photoSize1.FileId, photoSize2.FileId) - Assert.NotEqual(photoSize1.FileId, photoSize3.FileId) - Assert.NotEqual(photoSize2.FileId, photoSize3.FileId) - - { messageWithPhoto with Photo = Some [| photoSize1; photoSize2; photoSize3 |] } - let private messageWithSticker = { messageTemplate with Sticker = Some { @@ -228,7 +212,4 @@ let databaseVoiceTest(): unit = doDatabaseLinkTest fileId1 messageWithVoice let databaseVideoNoteTest(): unit = doDatabaseLinkTest fileId1 messageWithVideoNote [] -let databaseMultiplePhotosTest(): unit = doDatabaseLinksTest [|fileId1; fileId2|] messageWithMultiplePhotos - -[] -let databaseMultiplePhotoSizesTest(): unit = doDatabaseLinksTest [| fileId2 |] messageWithMultiplePhotoSizes +let databaseMultiplePhotosTest(): unit = doDatabaseLinksTest [|fileId2|] messageWithMultiplePhotos diff --git a/Emulsion/Telegram/LinkGenerator.fs b/Emulsion/Telegram/LinkGenerator.fs index de9f1664..8be513e9 100644 --- a/Emulsion/Telegram/LinkGenerator.fs +++ b/Emulsion/Telegram/LinkGenerator.fs @@ -35,24 +35,19 @@ let private getFileIds(message: FunogramMessage): string seq = let inline extractFileId(o: ^a option) = Option.iter(fun o -> allFileIds.Add((^a) : (member FileId: string) o)) o - let extractPhotoFileIds: PhotoSize[] option -> unit = + let extractPhotoFileId: PhotoSize[] option -> unit = Option.iter( - // In one message, several unique photos may have several sizes each. Get the biggest photo for each unique - // file: - Seq.groupBy(fun photoSize -> photoSize.FileUniqueId) - >> Seq.map(fun (_uniqueId, sizes) -> - sizes - |> Seq.sortByDescending(fun size -> size.Height * size.Width) - |> Seq.head - |> fun photoSize -> photoSize.FileId - ) - >> Seq.iter(allFileIds.Add) + // Telegram may send several differently-sized thumbnails in one message. Pick the biggest one of them. + Seq.sortByDescending(fun size -> size.Height * size.Width) + >> Seq.map(fun photoSize -> photoSize.FileId) + >> Seq.tryHead + >> Option.iter(allFileIds.Add) ) extractFileId message.Document extractFileId message.Audio extractFileId message.Animation - extractPhotoFileIds message.Photo + extractPhotoFileId message.Photo extractFileId message.Sticker extractFileId message.Video extractFileId message.Voice From 3096cd98e4db572882cec3839f41a77c86af0a20 Mon Sep 17 00:00:00 2001 From: Friedrich von Never Date: Sun, 3 Jul 2022 17:30:50 +0700 Subject: [PATCH 15/16] (#147) CI: enable Docker build on PR and periodically --- .github/workflows/docker.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ca0c53d6..6b79f302 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,8 +1,15 @@ name: Docker on: push: + branches: + - master tags: - - '*' + - 'v*' + pull_request: + branches: + - master + schedule: + - cron: '0 0 * * 6' # every Saturday jobs: publish: @@ -18,3 +25,4 @@ jobs: repository: codingteam/emulsion tag_with_ref: true tags: latest + push: ${{ github.event_name == 'push' && contains(github.ref, 'refs/tags/') && 'true' || 'false' }} From eba4b78dbc2c9ae3a18972cc20a19863b15683be Mon Sep 17 00:00:00 2001 From: Friedrich von Never Date: Sun, 3 Jul 2022 17:33:18 +0700 Subject: [PATCH 16/16] (#147) Docker: add new projects --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9d869b3c..13508a58 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,11 @@ FROM mcr.microsoft.com/dotnet/sdk:6.0 AS build-env WORKDIR /app +COPY ./Emulsion/Emulsion.fsproj ./Emulsion/ COPY ./Emulsion.ContentProxy/Emulsion.ContentProxy.fsproj ./Emulsion.ContentProxy/ COPY ./Emulsion.Database/Emulsion.Database.fsproj ./Emulsion.Database/ -COPY ./Emulsion/Emulsion.fsproj ./Emulsion/ +COPY ./Emulsion.Settings/Emulsion.Settings.fsproj ./Emulsion.Settings/ +COPY ./Emulsion.Web/Emulsion.Web.fsproj ./Emulsion.Web/ RUN dotnet restore Emulsion