diff --git a/Emulsion.ContentProxy/ContentStorage.fs b/Emulsion.ContentProxy/ContentStorage.fs index 11604e17..3f05ed68 100644 --- a/Emulsion.ContentProxy/ContentStorage.fs +++ b/Emulsion.ContentProxy/ContentStorage.fs @@ -9,6 +9,8 @@ type MessageContentIdentity = { ChatUserName: string MessageId: int64 FileId: string + FileName: string + MimeType: string } let getOrCreateMessageRecord (context: EmulsionDbContext) (id: MessageContentIdentity): Async = async { @@ -17,7 +19,9 @@ let getOrCreateMessageRecord (context: EmulsionDbContext) (id: MessageContentIde for content in context.TelegramContents do where (content.ChatUserName = id.ChatUserName && content.MessageId = id.MessageId - && content.FileId = id.FileId) + && content.FileId = id.FileId + && content.FileName = id.FileName + && content.MimeType = id.MimeType) } |> tryExactlyOneAsync match existingItem with | None -> @@ -26,6 +30,8 @@ let getOrCreateMessageRecord (context: EmulsionDbContext) (id: MessageContentIde ChatUserName = id.ChatUserName MessageId = id.MessageId FileId = id.FileId + FileName = id.FileName + MimeType = id.MimeType } do! addAsync context.TelegramContents newItem return newItem diff --git a/Emulsion.ContentProxy/Emulsion.ContentProxy.fsproj b/Emulsion.ContentProxy/Emulsion.ContentProxy.fsproj index 3de8d6cb..55bcfafb 100644 --- a/Emulsion.ContentProxy/Emulsion.ContentProxy.fsproj +++ b/Emulsion.ContentProxy/Emulsion.ContentProxy.fsproj @@ -7,14 +7,21 @@ + + + + + + + diff --git a/Emulsion.ContentProxy/FileCache.fs b/Emulsion.ContentProxy/FileCache.fs new file mode 100644 index 00000000..73a303e7 --- /dev/null +++ b/Emulsion.ContentProxy/FileCache.fs @@ -0,0 +1,218 @@ +namespace Emulsion.ContentProxy + +open System +open System.IO +open System.Net.Http +open System.Security.Cryptography +open System.Text +open System.Threading + +open JetBrains.Collections.Viewable +open Serilog +open SimpleBase + +open Emulsion.Settings + +type DownloadRequest = { + Uri: Uri + CacheKey: string + Size: uint64 +} + +module Base58 = + /// Suggested by @ttldtor. + let M4N71KR = Base58(Base58Alphabet "123456789qwertyuiopasdfghjkzxcvbnmQWERTYUPASDFGHJKLZXCVBNM") + +module FileCache = + let EncodeFileName(sha256: SHA256, cacheKey: string): string = + cacheKey + |> Encoding.UTF8.GetBytes + |> sha256.ComputeHash + |> Base58.M4N71KR.Encode + + let TryDecodeFileNameToSha256Hash(fileName: string): byte[] option = + try + Some <| (Base58.M4N71KR.Decode fileName).ToArray() + with + | :? ArgumentException -> None + + let IsMoveAndDeleteModeEnabled = + // NOTE: On older versions of Windows (known to reproduce on windows-2019 GitHub Actions image), the following + // scenario may be defunct: + // + // - open a file with FileShare.Delete (i.e. for download) + // - delete a file (i.e. during the cache cleanup) + // - try to create a file with the same name again + // + // According to this article + // (https://boostgsoc13.github.io/boost.afio/doc/html/afio/FAQ/deleting_open_files.html), it is impossible to do + // since file will occupy its disk name until the last handle is closed. + // + // In practice, this is allowed (checked at least on Windows 10 20H2 and windows-2022 GitHub Actions image), but + // some tests are known to be broken on older versions of Windows (windows-2019). + // + // As a workaround, let's rename the file to a random name before deleting it. + // + // This workaround may be removed after these older versions of Windows goes out of support. + OperatingSystem.IsWindows() + +type FileCache(logger: ILogger, + settings: FileCacheSettings, + httpClientFactory: IHttpClientFactory, + sha256: SHA256) = + + let error = Signal() + + let getFilePath(cacheKey: string) = + Path.Combine(settings.Directory, FileCache.EncodeFileName(sha256, cacheKey)) + + let readFileOptions = + FileStreamOptions(Mode = FileMode.Open, Access = FileAccess.Read, Options = FileOptions.Asynchronous, Share = (FileShare.Read ||| FileShare.Delete)) + + let writeFileOptions = + FileStreamOptions(Mode = FileMode.CreateNew, Access = FileAccess.Write, Options = FileOptions.Asynchronous, Share = FileShare.None) + + let getFromCache(cacheKey: string) = async { + let path = getFilePath cacheKey + return + if File.Exists path then + Some(new FileStream(path, readFileOptions)) + else + None + } + + let enumerateCacheFiles() = + let entries = Directory.EnumerateFileSystemEntries settings.Directory + if FileCache.IsMoveAndDeleteModeEnabled then + entries |> Seq.filter(fun p -> not(p.EndsWith ".deleted")) + else + entries + + let deleteFileSafe (fileInfo: FileInfo) = async { + if FileCache.IsMoveAndDeleteModeEnabled then + fileInfo.MoveTo(Path.Combine(fileInfo.DirectoryName, $"{Guid.NewGuid().ToString()}.deleted")) + fileInfo.Delete() + else + fileInfo.Delete() + } + + let assertCacheDirectoryExists() = async { + Directory.CreateDirectory settings.Directory |> ignore + } + + let assertCacheValid() = async { + enumerateCacheFiles() + |> Seq.iter(fun entry -> + let entryName = Path.GetFileName entry + + if not <| File.Exists entry + then failwith $"Cache directory invalid: contains a subdirectory \"{entryName}\"." + + match FileCache.TryDecodeFileNameToSha256Hash entryName with + | Some hash when hash.Length = sha256.HashSize / 8 -> () + | _ -> + failwith ( + $"Cache directory invalid: contains an entry \"{entryName}\" which doesn't correspond to a " + + "base58-encoded SHA-256 hash." + ) + ) + } + + let ensureFreeCache size = async { + if size > settings.FileSizeLimitBytes || size > settings.TotalCacheSizeLimitBytes then + return false + else + do! assertCacheDirectoryExists() + do! assertCacheValid() + + let allEntries = enumerateCacheFiles() |> Seq.map FileInfo + + // Now, sort the entries from newest to oldest, and start deleting if required at a point when we understand + // that there are too much files: + let entriesByPriority = + allEntries + |> Seq.sortByDescending(fun info -> info.LastWriteTimeUtc) + |> Seq.toArray + + let mutable currentSize = 0UL + for info in entriesByPriority do + currentSize <- currentSize + Checked.uint64 info.Length + if currentSize + size > settings.TotalCacheSizeLimitBytes then + logger.Information("Deleting a cache item \"{FileName}\" ({Size} bytes)", info.Name, info.Length) + do! deleteFileSafe info + + return true + } + + let download(uri: Uri): Async = async { + let! ct = Async.CancellationToken + + use client = httpClientFactory.CreateClient() + let! response = Async.AwaitTask <| client.GetAsync(uri, ct) + return! Async.AwaitTask <| response.EnsureSuccessStatusCode().Content.ReadAsStreamAsync() + } + + let downloadIntoCacheAndGet uri cacheKey: Async = async { + let! ct = Async.CancellationToken + let! stream = download uri + let path = getFilePath cacheKey + logger.Information("Saving {Uri} to path {Path}…", uri, path) + + do! async { // to limit the cachedFile scope + use cachedFile = new FileStream(path, writeFileOptions) + do! Async.AwaitTask(stream.CopyToAsync(cachedFile, ct)) + logger.Information("Download successful: \"{Uri}\" to \"{Path}\".", uri, path) + } + + let! file = getFromCache cacheKey + return upcast Option.get file + } + + let cancellation = new CancellationTokenSource() + let processRequest request: Async = async { + logger.Information("Cache lookup for content {Uri} (cache key {CacheKey})", request.Uri, request.CacheKey) + match! getFromCache request.CacheKey with + | Some content -> + logger.Information("Cache hit for content {Uri} (cache key {CacheKey})", request.Uri, request.CacheKey) + return content + | None -> + logger.Information("No cache hit for content {Uri} (cache key {CacheKey}), will download", request.Uri, request.CacheKey) + let! shouldCache = ensureFreeCache request.Size + if shouldCache then + logger.Information("Resource {Uri} (cache key {CacheKey}, {Size} bytes) will fit into cache, caching", request.Uri, request.CacheKey, request.Size) + let! result = downloadIntoCacheAndGet request.Uri request.CacheKey + logger.Information("Resource {Uri} (cache key {CacheKey}, {Size} bytes) downloaded", request.Uri, request.CacheKey, request.Size) + return result + else + logger.Information("Resource {Uri} (cache key {CacheKey}) won't fit into cache, directly downloading", request.Uri, request.CacheKey) + let! result = download request.Uri + return result + } + + let processLoop(processor: MailboxProcessor<_ * AsyncReplyChannel<_>>) = async { + while true do + let! request, replyChannel = processor.Receive() + try + let! result = processRequest request + replyChannel.Reply(Some result) + with + | ex -> + logger.Error(ex, "Exception while processing the file download queue") + error.Fire ex + replyChannel.Reply None + } + let processor = MailboxProcessor.Start(processLoop, cancellation.Token) + + interface IDisposable with + member _.Dispose() = + cancellation.Dispose() + (processor :> IDisposable).Dispose() + + member _.Download(uri: Uri, cacheKey: string, size: uint64): Async = + processor.PostAndAsyncReply(fun chan -> ({ + Uri = uri + CacheKey = cacheKey + Size = size + }, chan)) + + member _.Error: ISource = error diff --git a/Emulsion.ContentProxy/SimpleHttpClientFactory.fs b/Emulsion.ContentProxy/SimpleHttpClientFactory.fs new file mode 100644 index 00000000..01a91ac3 --- /dev/null +++ b/Emulsion.ContentProxy/SimpleHttpClientFactory.fs @@ -0,0 +1,7 @@ +namespace Emulsion.ContentProxy + +open System.Net.Http + +type SimpleHttpClientFactory() = + interface IHttpClientFactory with + member this.CreateClient _ = new HttpClient() diff --git a/Emulsion.Database/Entities.fs b/Emulsion.Database/Entities.fs index aa6b57c9..db9b169e 100644 --- a/Emulsion.Database/Entities.fs +++ b/Emulsion.Database/Entities.fs @@ -8,4 +8,6 @@ type TelegramContent = { ChatUserName: string MessageId: int64 FileId: string + FileName: string + MimeType: string } diff --git a/Emulsion.Database/Migrations/20220828133844_ContentFileNameAndMimeType.fs b/Emulsion.Database/Migrations/20220828133844_ContentFileNameAndMimeType.fs new file mode 100644 index 00000000..93b73de9 --- /dev/null +++ b/Emulsion.Database/Migrations/20220828133844_ContentFileNameAndMimeType.fs @@ -0,0 +1,91 @@ +// +namespace Emulsion.Database.Migrations + +open System +open Emulsion.Database +open Microsoft.EntityFrameworkCore +open Microsoft.EntityFrameworkCore.Infrastructure +open Microsoft.EntityFrameworkCore.Migrations + +[)>] +[] +type ContentFileNameAndMimeType() = + inherit Migration() + + override this.Up(migrationBuilder:MigrationBuilder) = + migrationBuilder.AddColumn( + name = "FileName" + ,table = "TelegramContents" + ,``type`` = "TEXT" + ,nullable = true + ,defaultValue = "file.bin" + ) |> ignore + + migrationBuilder.AddColumn( + name = "MimeType" + ,table = "TelegramContents" + ,``type`` = "TEXT" + ,nullable = true + ,defaultValue = "application/octet-stream" + ) |> ignore + + migrationBuilder.Sql @" + drop index TelegramContents_Unique; + + create unique index TelegramContents_Unique + on TelegramContents(ChatUserName, MessageId, FileId, FileName, MimeType) + " |> ignore + + + override this.Down(migrationBuilder:MigrationBuilder) = + migrationBuilder.DropColumn( + name = "FileName" + ,table = "TelegramContents" + ) |> ignore + + migrationBuilder.DropColumn( + name = "MimeType" + ,table = "TelegramContents" + ) |> ignore + + migrationBuilder.Sql @" + drop index TelegramContents_Unique; + + create unique index TelegramContents_Unique + on TelegramContents(ChatUserName, MessageId, FileId) + " |> ignore + + + override this.BuildTargetModel(modelBuilder: ModelBuilder) = + modelBuilder + .HasAnnotation("ProductVersion", "5.0.10") + |> ignore + + modelBuilder.Entity("Emulsion.Database.Entities.TelegramContent", (fun b -> + + b.Property("Id") + .IsRequired(true) + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER") |> ignore + b.Property("ChatUserName") + .IsRequired(false) + .HasColumnType("TEXT") |> ignore + b.Property("FileId") + .IsRequired(false) + .HasColumnType("TEXT") |> ignore + b.Property("FileName") + .IsRequired(false) + .HasColumnType("TEXT") |> ignore + b.Property("MessageId") + .IsRequired(true) + .HasColumnType("INTEGER") |> ignore + b.Property("MimeType") + .IsRequired(false) + .HasColumnType("TEXT") |> ignore + + b.HasKey("Id") |> ignore + + b.ToTable("TelegramContents") |> ignore + + )) |> ignore + diff --git a/Emulsion.Database/Migrations/EmulsionDbContextModelSnapshot.fs b/Emulsion.Database/Migrations/EmulsionDbContextModelSnapshot.fs index f2fd07d8..2e930df8 100644 --- a/Emulsion.Database/Migrations/EmulsionDbContextModelSnapshot.fs +++ b/Emulsion.Database/Migrations/EmulsionDbContextModelSnapshot.fs @@ -5,9 +5,6 @@ open System open Emulsion.Database open Microsoft.EntityFrameworkCore open Microsoft.EntityFrameworkCore.Infrastructure -open Microsoft.EntityFrameworkCore.Metadata -open Microsoft.EntityFrameworkCore.Migrations -open Microsoft.EntityFrameworkCore.Storage.ValueConversion [)>] type EmulsionDbContextModelSnapshot() = @@ -30,9 +27,15 @@ type EmulsionDbContextModelSnapshot() = b.Property("FileId") .IsRequired(false) .HasColumnType("TEXT") |> ignore + b.Property("FileName") + .IsRequired(false) + .HasColumnType("TEXT") |> ignore b.Property("MessageId") .IsRequired(true) .HasColumnType("INTEGER") |> ignore + b.Property("MimeType") + .IsRequired(false) + .HasColumnType("TEXT") |> ignore b.HasKey("Id") |> ignore diff --git a/Emulsion.Settings/Settings.fs b/Emulsion.Settings/Settings.fs index 9832e22d..511f23e8 100644 --- a/Emulsion.Settings/Settings.fs +++ b/Emulsion.Settings/Settings.fs @@ -34,12 +34,19 @@ type HostingSettings = { HashIdSalt: string } +type FileCacheSettings = { + Directory: string + FileSizeLimitBytes: uint64 + TotalCacheSizeLimitBytes: uint64 +} + type EmulsionSettings = { Xmpp: XmppSettings Telegram: TelegramSettings Log: LogSettings Database: DatabaseSettings option Hosting: HostingSettings option + FileCache: FileCacheSettings option } let defaultConnectionTimeout = TimeSpan.FromMinutes 5.0 @@ -56,6 +63,12 @@ let private readTimeSpan defaultVal key section = |> Option.defaultValue defaultVal let read (config : IConfiguration) : EmulsionSettings = + let uint64OrDefault value ``default`` = + value + |> Option.ofObj + |> Option.map uint64 + |> Option.defaultValue ``default`` + let readXmpp (section : IConfigurationSection) = { Login = section["login"] Password = section["password"] @@ -91,9 +104,17 @@ let read (config : IConfiguration) : EmulsionSettings = } | None, None, None -> None | other -> failwith $"Parameter pack {other} represents invalid hosting settings." + let readFileCache(section: IConfigurationSection) = + Option.ofObj section["directory"] + |> Option.map(fun directory -> { + Directory = directory + FileSizeLimitBytes = uint64OrDefault section["fileSizeLimitBytes"] (1024UL * 1024UL) + TotalCacheSizeLimitBytes = uint64OrDefault section["totalCacheSizeLimitBytes"] (20UL * 1024UL * 1024UL) + }) { Xmpp = readXmpp <| config.GetSection("xmpp") Telegram = readTelegram <| config.GetSection("telegram") Log = readLog <| config.GetSection "log" Database = readDatabase <| config.GetSection "database" - Hosting = readHosting <| config.GetSection "hosting" } + Hosting = readHosting <| config.GetSection "hosting" + FileCache = readFileCache <| config.GetSection "fileCache" } diff --git a/Emulsion.Telegram/Client.fs b/Emulsion.Telegram/Client.fs index 2f2d708b..459874d8 100644 --- a/Emulsion.Telegram/Client.fs +++ b/Emulsion.Telegram/Client.fs @@ -1,11 +1,20 @@ namespace Emulsion.Telegram +open System open System.Threading open Emulsion.Database open Emulsion.Messaging.MessageSystem open Emulsion.Settings +type FileInfo = { + TemporaryLink: Uri + Size: uint64 +} + +type ITelegramClient = + abstract GetFileInfo: fileId: string -> Async + type Client(ctx: ServiceContext, cancellationToken: CancellationToken, telegramSettings: TelegramSettings, @@ -15,10 +24,27 @@ type Client(ctx: ServiceContext, let botConfig = { Funogram.Telegram.Bot.Config.defaultConfig with Token = telegramSettings.Token } + interface ITelegramClient with + member this.GetFileInfo(fileId) = async { + let logger = ctx.Logger + logger.Information("Querying file information for file {FileId}", fileId) + let! file = Funogram.sendGetFile botConfig fileId + match file.FilePath, file.FileSize with + | None, None -> + logger.Warning("File {FileId} was not found on server", fileId) + return None + | Some fp, Some sz -> + return Some { + TemporaryLink = Uri $"https://api.telegram.org/file/bot{telegramSettings.Token}/{fp}" + Size = Checked.uint64 sz + } + | x, y -> return failwith $"Unknown data received from Telegram server: {x}, {y}" + } + override _.RunUntilError receiver = async { // Run loop of Telegram is in no need of any complicated start, so just return an async that will perform it: return Funogram.run ctx.Logger telegramSettings databaseSettings hostingSettings botConfig receiver } override _.Send message = - Funogram.send telegramSettings botConfig message + Funogram.sendMessage telegramSettings botConfig message diff --git a/Emulsion.Telegram/Funogram.fs b/Emulsion.Telegram/Funogram.fs index a9edad6c..aadf6ae1 100644 --- a/Emulsion.Telegram/Funogram.fs +++ b/Emulsion.Telegram/Funogram.fs @@ -298,9 +298,9 @@ module MessageConverter = else extractMessageContent replyTo links.ReplyToContentLinks { main = mainMessage; replyTo = Some replyToMessage } -let internal processSendResult(result: Result<'a, ApiResponseError>): unit = +let internal processSendResult(result: Result<'a, ApiResponseError>): 'a = match result with - | Ok _ -> () + | Ok x -> x | Error e -> failwith $"Telegram API Call processing error {e.ErrorCode}: {e.Description}" @@ -347,15 +347,23 @@ let internal prepareHtmlMessage: Message -> string = function | Authored {author = author; text = text} -> $"{Html.escape author}\n{Html.escape text}" | Event {text = text} -> Html.escape text -let send (settings: TelegramSettings) (botConfig: BotConfig) (OutgoingMessage content): Async = +let private send (botConfig: BotConfig) request = api botConfig request + +let sendGetFile (botConfig: BotConfig) (fileId: string): Async = async { + let! result = send botConfig (Req.GetFile.Make fileId) + return processSendResult result +} + +let sendMessage (settings: TelegramSettings) (botConfig: BotConfig) (OutgoingMessage content): Async = let sendHtmlMessage (groupId: ChatId) text = Req.SendMessage.Make(groupId, text, ParseMode.HTML) let groupId = Int(int64 settings.GroupId) let message = prepareHtmlMessage content async { - let! result = api botConfig (sendHtmlMessage groupId message) - return processSendResult result + let! result = send botConfig (sendHtmlMessage groupId message) + processSendResult result |> ignore + return () } let run (logger: ILogger) diff --git a/Emulsion.Telegram/LinkGenerator.fs b/Emulsion.Telegram/LinkGenerator.fs index 8be513e9..70c54255 100644 --- a/Emulsion.Telegram/LinkGenerator.fs +++ b/Emulsion.Telegram/LinkGenerator.fs @@ -30,41 +30,74 @@ let private gatherMessageLink(message: FunogramMessage) = | { Text = Some _} | { Poll = Some _ } -> None | _ -> getMessageLink message -let private getFileIds(message: FunogramMessage): string seq = - let allFileIds = ResizeArray() - let inline extractFileId(o: ^a option) = - Option.iter(fun o -> allFileIds.Add((^a) : (member FileId: string) o)) o +type private FileInfo = { + FileId: string + FileName: string option + MimeType: string option +} + +let private getFileInfos(message: FunogramMessage): FileInfo seq = + let allFileInfos = ResizeArray() + let inline extractFileInfo(o: ^a option) = + o |> Option.iter(fun o -> + allFileInfos.Add({ + FileId = ((^a) : (member FileId: string) o) + FileName = ((^a) : (member FileName: string option) o) + MimeType = ((^a) : (member MimeType: string option) o) + })) + + let inline extractFileInfoWithName (fileName: string) (o: ^a option) = + o |> Option.iter(fun o -> + allFileInfos.Add({ + FileId = ((^a) : (member FileId: string) o) + FileName = Some fileName + MimeType = ((^a) : (member MimeType: string option) o) + })) - let extractPhotoFileId: PhotoSize[] option -> unit = + let inline extractFileInfoWithNameAndMimeType (fileName: string) (mimeType: string) (o: ^a option) = + o |> Option.iter(fun o -> + allFileInfos.Add({ + FileId = ((^a) : (member FileId: string) o) + FileName = Some fileName + MimeType = Some mimeType + })) + + let extractPhotoFileInfo: PhotoSize[] option -> unit = Option.iter( // Telegram may send several differently-sized thumbnails in one message. Pick the biggest one of them. Seq.sortByDescending(fun size -> size.Height * size.Width) >> Seq.map(fun photoSize -> photoSize.FileId) >> Seq.tryHead - >> Option.iter(allFileIds.Add) + >> Option.iter(fun fileId -> allFileInfos.Add { + FileId = fileId + FileName = Some "photo.jpg" + MimeType = Some "image/jpeg" + }) ) - extractFileId message.Document - extractFileId message.Audio - extractFileId message.Animation - extractPhotoFileId message.Photo - extractFileId message.Sticker - extractFileId message.Video - extractFileId message.Voice - extractFileId message.VideoNote + extractFileInfo message.Document + extractFileInfo message.Audio + extractFileInfo message.Animation + extractPhotoFileInfo message.Photo + extractFileInfoWithNameAndMimeType "sticker.jpg" "image/jpeg" message.Sticker + extractFileInfo message.Video + extractFileInfoWithName "voice.ogg" message.Voice + extractFileInfoWithNameAndMimeType "video.mp4" "video/mp4" message.VideoNote - allFileIds + allFileInfos let private getContentIdentities(message: FunogramMessage): ContentStorage.MessageContentIdentity seq = match message.Chat with | { Type = SuperGroup Username = Some chatName } -> - getFileIds message - |> Seq.map (fun fileId -> + getFileInfos message + |> Seq.map (fun fileInfo -> { ChatUserName = chatName MessageId = message.MessageId - FileId = fileId + FileId = fileInfo.FileId + FileName = Option.defaultValue "file.bin" fileInfo.FileName + MimeType = Option.defaultValue "application/octet-stream" fileInfo.MimeType } ) | _ -> Seq.empty diff --git a/Emulsion.TestFramework/Emulsion.TestFramework.fsproj b/Emulsion.TestFramework/Emulsion.TestFramework.fsproj new file mode 100644 index 00000000..a0a6baed --- /dev/null +++ b/Emulsion.TestFramework/Emulsion.TestFramework.fsproj @@ -0,0 +1,26 @@ + + + + net6.0 + true + Library + + + + + + + + + + + + + + + + + + + + diff --git a/Emulsion.Tests/TestUtils/Exceptions.fs b/Emulsion.TestFramework/Exceptions.fs similarity index 90% rename from Emulsion.Tests/TestUtils/Exceptions.fs rename to Emulsion.TestFramework/Exceptions.fs index fa5654bd..85c4b55c 100644 --- a/Emulsion.Tests/TestUtils/Exceptions.fs +++ b/Emulsion.TestFramework/Exceptions.fs @@ -1,4 +1,4 @@ -module Emulsion.Tests.TestUtils.Exceptions +module Emulsion.TestFramework.Exceptions open System open Microsoft.EntityFrameworkCore diff --git a/Emulsion.TestFramework/FileCacheUtil.fs b/Emulsion.TestFramework/FileCacheUtil.fs new file mode 100644 index 00000000..e062a4c8 --- /dev/null +++ b/Emulsion.TestFramework/FileCacheUtil.fs @@ -0,0 +1,22 @@ +module Emulsion.TestFramework.FileCacheUtil + +open System.IO + +open Emulsion.ContentProxy +open Emulsion.Settings +open Emulsion.TestFramework.Logging + +let newCacheDirectory() = + let path = Path.GetTempFileName() + File.Delete path + Directory.CreateDirectory path |> ignore + path + +let setUpFileCache outputHelper sha256 cacheDirectory (totalLimitBytes: uint64) = + let settings = { + Directory = cacheDirectory + FileSizeLimitBytes = 10UL * 1024UL * 1024UL + TotalCacheSizeLimitBytes = totalLimitBytes + } + + new FileCache(xunitLogger outputHelper, settings, SimpleHttpClientFactory(), sha256) diff --git a/Emulsion.Tests/TestUtils/LockedBuffer.fs b/Emulsion.TestFramework/LockedBuffer.fs similarity index 90% rename from Emulsion.Tests/TestUtils/LockedBuffer.fs rename to Emulsion.TestFramework/LockedBuffer.fs index 951f7165..dfae3891 100644 --- a/Emulsion.Tests/TestUtils/LockedBuffer.fs +++ b/Emulsion.TestFramework/LockedBuffer.fs @@ -1,4 +1,4 @@ -namespace Emulsion.Tests.TestUtils +namespace Emulsion.TestFramework type LockedBuffer<'T>() = let messages = ResizeArray<'T>() diff --git a/Emulsion.Tests/TestUtils/Logging.fs b/Emulsion.TestFramework/Logging.fs similarity index 82% rename from Emulsion.Tests/TestUtils/Logging.fs rename to Emulsion.TestFramework/Logging.fs index 4d73b083..25018f73 100644 --- a/Emulsion.Tests/TestUtils/Logging.fs +++ b/Emulsion.TestFramework/Logging.fs @@ -1,4 +1,4 @@ -module Emulsion.Tests.TestUtils.Logging +module Emulsion.TestFramework.Logging open Serilog open Xunit.Abstractions diff --git a/Emulsion.TestFramework/StreamUtils.fs b/Emulsion.TestFramework/StreamUtils.fs new file mode 100644 index 00000000..92fccf1c --- /dev/null +++ b/Emulsion.TestFramework/StreamUtils.fs @@ -0,0 +1,10 @@ +module Emulsion.TestFramework.StreamUtils + +open System.IO + +let readAllBytes(stream: Stream) = async { + use buffer = new MemoryStream() + let! ct = Async.CancellationToken + do! Async.AwaitTask(stream.CopyToAsync(buffer, ct)) + return buffer.ToArray() +} diff --git a/Emulsion.TestFramework/TelegramClientMock.fs b/Emulsion.TestFramework/TelegramClientMock.fs new file mode 100644 index 00000000..c66c9181 --- /dev/null +++ b/Emulsion.TestFramework/TelegramClientMock.fs @@ -0,0 +1,14 @@ +namespace Emulsion.TestFramework + +open System.Collections.Generic + +open Emulsion.Telegram + +type TelegramClientMock() = + let responses = Dictionary() + + interface ITelegramClient with + member this.GetFileInfo fileId = async.Return responses[fileId] + + member _.SetResponse(fileId: string, fileInfo: FileInfo option): unit = + responses[fileId] <- fileInfo diff --git a/Emulsion.Tests/TestUtils/TestDataStorage.fs b/Emulsion.TestFramework/TestDataStorage.fs similarity index 88% rename from Emulsion.Tests/TestUtils/TestDataStorage.fs rename to Emulsion.TestFramework/TestDataStorage.fs index 84b1ea06..0787628f 100644 --- a/Emulsion.Tests/TestUtils/TestDataStorage.fs +++ b/Emulsion.TestFramework/TestDataStorage.fs @@ -1,4 +1,4 @@ -module Emulsion.Tests.TestUtils.TestDataStorage +module Emulsion.TestFramework.TestDataStorage open System.IO diff --git a/Emulsion.Tests/TestUtils/Waiter.fs b/Emulsion.TestFramework/Waiter.fs similarity index 91% rename from Emulsion.Tests/TestUtils/Waiter.fs rename to Emulsion.TestFramework/Waiter.fs index f83e3c93..89fe61f2 100644 --- a/Emulsion.Tests/TestUtils/Waiter.fs +++ b/Emulsion.TestFramework/Waiter.fs @@ -1,4 +1,4 @@ -module Emulsion.Tests.TestUtils.Waiter +module Emulsion.TestFramework.Waiter open System open System.Threading diff --git a/Emulsion.TestFramework/WebFileStorage.fs b/Emulsion.TestFramework/WebFileStorage.fs new file mode 100644 index 00000000..ad3d886c --- /dev/null +++ b/Emulsion.TestFramework/WebFileStorage.fs @@ -0,0 +1,40 @@ +namespace Emulsion.TestFramework + +open System +open System.Net +open System.Net.Sockets + +open Microsoft.AspNetCore.Builder +open Microsoft.AspNetCore.Http + +module private NetUtil = + let findFreePort() = + use socket = new Socket(SocketType.Stream, ProtocolType.Tcp) + socket.Bind(IPEndPoint(IPAddress.Loopback, 0)) + (socket.LocalEndPoint :?> IPEndPoint).Port + +type WebFileStorage(data: Map) = + let url = $"http://localhost:{NetUtil.findFreePort()}" + + let startWebApplication() = + let builder = WebApplication.CreateBuilder() + let app = builder.Build() + app.MapGet("/{entry}", Func<_, _>(fun (entry: string) -> task { + match Map.tryFind entry data with + | Some bytes -> return Results.Bytes bytes + | None -> return Results.NotFound() + })) |> ignore + app, app.RunAsync url + + let app, task = startWebApplication() + + member _.Link(entry: string): Uri = + Uri $"{url}/{entry}" + + member _.Content(entry: string): byte[] = + data[entry] + + interface IDisposable with + member this.Dispose(): unit = + app.StopAsync().Wait() + task.Wait() diff --git a/Emulsion.Tests/Actors/Core.fs b/Emulsion.Tests/Actors/Core.fs index 0ea58530..ced5b542 100644 --- a/Emulsion.Tests/Actors/Core.fs +++ b/Emulsion.Tests/Actors/Core.fs @@ -9,7 +9,7 @@ open Xunit.Abstractions open Emulsion open Emulsion.Actors open Emulsion.Messaging -open Emulsion.Tests.TestUtils +open Emulsion.TestFramework type CoreTests(testOutput: ITestOutputHelper) as this = inherit TestKit() diff --git a/Emulsion.Tests/Actors/Telegram.fs b/Emulsion.Tests/Actors/Telegram.fs index 28b496ad..97285a40 100644 --- a/Emulsion.Tests/Actors/Telegram.fs +++ b/Emulsion.Tests/Actors/Telegram.fs @@ -8,7 +8,7 @@ open Emulsion open Emulsion.Actors open Emulsion.Messaging open Emulsion.Messaging.MessageSystem -open Emulsion.Tests.TestUtils +open Emulsion.TestFramework type TelegramTest(testOutput: ITestOutputHelper) = inherit TestKit() diff --git a/Emulsion.Tests/Actors/Xmpp.fs b/Emulsion.Tests/Actors/Xmpp.fs index 24ac6de8..5e256acb 100644 --- a/Emulsion.Tests/Actors/Xmpp.fs +++ b/Emulsion.Tests/Actors/Xmpp.fs @@ -8,7 +8,7 @@ open Emulsion open Emulsion.Actors open Emulsion.Messaging open Emulsion.Messaging.MessageSystem -open Emulsion.Tests.TestUtils +open Emulsion.TestFramework type XmppTest(testOutput: ITestOutputHelper) = inherit TestKit() diff --git a/Emulsion.Tests/ContentProxy/ContentStorageTests.fs b/Emulsion.Tests/ContentProxy/ContentStorageTests.fs index 81bc0ca2..8c598ec6 100644 --- a/Emulsion.Tests/ContentProxy/ContentStorageTests.fs +++ b/Emulsion.Tests/ContentProxy/ContentStorageTests.fs @@ -4,12 +4,14 @@ open Xunit open Emulsion.ContentProxy.ContentStorage open Emulsion.Database -open Emulsion.Tests.TestUtils +open Emulsion.TestFramework let private testIdentity = { ChatUserName = "test" MessageId = 123L FileId = "this_is_file" + FileName = "file.bin" + MimeType = "application/octet-stream" } let private executeQuery settings = diff --git a/Emulsion.Tests/ContentProxy/FileCacheTests.fs b/Emulsion.Tests/ContentProxy/FileCacheTests.fs new file mode 100644 index 00000000..0548d17f --- /dev/null +++ b/Emulsion.Tests/ContentProxy/FileCacheTests.fs @@ -0,0 +1,193 @@ +namespace Emulsion.Tests.ContentProxy + +open System +open System.Collections.Generic +open System.IO +open System.Security.Cryptography +open System.Threading.Tasks + +open JetBrains.Lifetimes +open Xunit +open Xunit.Abstractions + +open Emulsion.ContentProxy +open Emulsion.TestFramework + +type FileCacheTests(outputHelper: ITestOutputHelper) = + + let sha256 = SHA256.Create() + + let cacheDirectory = lazy FileCacheUtil.newCacheDirectory() + + let setUpFileCache(totalLimitBytes: uint64) = + FileCacheUtil.setUpFileCache outputHelper sha256 cacheDirectory.Value totalLimitBytes + + let assertCacheState(entries: (string * byte[]) seq) = + let files = + Directory.EnumerateFileSystemEntries(cacheDirectory.Value) + |> Seq.filter(fun f -> + if FileCache.IsMoveAndDeleteModeEnabled then not(f.EndsWith ".deleted") + else true + ) + |> Seq.map(fun file -> + let name = Path.GetFileName file + let content = File.ReadAllBytes file + name, content + ) + |> Map.ofSeq + + let entries = + entries + |> Seq.map(fun (k, v) -> FileCache.EncodeFileName(sha256, k), v) + |> Map.ofSeq + + Assert.Equal>(entries.Keys, files.Keys) + for key in entries.Keys do + Assert.Equal>(entries[key], files[key]) + + let assertFileDownloaded (fileCache: FileCache) (fileStorage: WebFileStorage) entry size = async { + let! file = fileCache.Download(fileStorage.Link entry, entry, size) + Assert.True file.IsSome + } + + let assertCacheValidationError setUpAction expectedMessage = + use fileCache = setUpFileCache 1UL + use fileStorage = new WebFileStorage(Map.empty) + + setUpAction() + + Lifetime.Using(fun lt -> + let mutable error: Exception option = None + fileCache.Error.Advise(lt, fun e -> error <- Some e) + + let file = Async.RunSynchronously <| fileCache.Download(fileStorage.Link "nonexistent", "nonexistent", 1UL) + Assert.True file.IsNone + + Assert.True error.IsSome + Assert.Equal(expectedMessage, error.Value.Message) + ) + + interface IDisposable with + member _.Dispose() = sha256.Dispose() + + [] + member _.``File cache should throw a validation exception if the cache directory contains directories``(): unit = + assertCacheValidationError + (fun() -> Directory.CreateDirectory(Path.Combine(cacheDirectory.Value, "aaa")) |> ignore) + "Cache directory invalid: contains a subdirectory \"aaa\"." + + [] + member _.``File cache should throw a validation exception if the cache directory contains non-conventionally-named files``(): unit = + assertCacheValidationError + (fun() -> File.Create(Path.Combine(cacheDirectory.Value, "aaa.txt")).Dispose()) + ("Cache directory invalid: contains an entry \"aaa.txt\" which doesn't correspond to a base58-encoded " + + "SHA-256 hash.") + + [] + member _.``File should be cached``(): Task = task { + use fileCache = setUpFileCache 1024UL + use fileStorage = new WebFileStorage(Map.ofArray [| + "a", [| for _ in 1 .. 5 do yield 1uy |] + |]) + + do! assertFileDownloaded fileCache fileStorage "a" 5UL + assertCacheState [| "a", fileStorage.Content("a") |] + } + + [] + member _.``Too big file should be proxied``(): Task = task { + use fileCache = setUpFileCache 1UL + use fileStorage = new WebFileStorage(Map.ofArray [| + "a", [| for _ in 1 .. 2 do yield 1uy |] + |]) + + do! assertFileDownloaded fileCache fileStorage "a" 2UL + assertCacheState Array.empty + } + + [] + member _.``Cleanup should be triggered``(): Task = task { + use fileCache = setUpFileCache 129UL + use fileStorage = new WebFileStorage(Map.ofArray [| + "a", [| for _ in 1 .. 128 do yield 1uy |] + "b", [| for _ in 1 .. 128 do yield 2uy |] + "c", [| 3uy |] + |]) + + do! assertFileDownloaded fileCache fileStorage "a" 128UL + assertCacheState [| "a", fileStorage.Content("a") |] + + do! assertFileDownloaded fileCache fileStorage "b" 128UL + assertCacheState [| "b", fileStorage.Content("b") |] + + do! assertFileDownloaded fileCache fileStorage "c" 1UL + assertCacheState [| + "b", fileStorage.Content("b") + "c", fileStorage.Content("c") + |] + } + + [] + member _.``File cache cleanup works in order by file modification dates``(): Task = task { + use fileCache = setUpFileCache 2UL + use fileStorage = new WebFileStorage(Map.ofArray [| + "a", [| 1uy |] + "b", [| 2uy |] + "c", [| 3uy |] + |]) + + do! assertFileDownloaded fileCache fileStorage "a" 1UL + do! assertFileDownloaded fileCache fileStorage "c" 1UL + do! assertFileDownloaded fileCache fileStorage "b" 1UL // "a" should be deleted + assertCacheState [| "b", [| 2uy |] + "c", [| 3uy |] |] + do! assertFileDownloaded fileCache fileStorage "a" 1UL // "c" should be deleted + assertCacheState [| "a", [| 1uy |] + "b", [| 2uy |] |] + } + + [] + member _.``File should be downloaded even if it was cleaned up during download``(): Task = task { + use fileCache = setUpFileCache (1024UL * 1024UL) + use fileStorage = new WebFileStorage(Map.ofArray [| + "a", [| for _ in 1 .. 1024 * 1024 do 1uy |] + "b", [| for _ in 1 .. 1024 * 1024 do 2uy |] + |]) + + // Start downloading the "a" item: + let! stream = fileCache.Download(fileStorage.Link "a", "a", 1024UL * 1024UL) + let stream = Option.get stream + // Just keep the stream open for now and trigger the cleanup: + do! assertFileDownloaded fileCache fileStorage "b" (1024UL * 1024UL) + // Now there's only "b" item in the cache: + assertCacheState [| "b", fileStorage.Content "b" |] + // We should still be able to read "a" fully: + let! content = StreamUtils.readAllBytes stream + Assert.Equal(fileStorage.Content "a", content) + } + + [] + member _.``File should be re-downloaded after cleanup even if there's a outdated read session in progress``(): Task = task { + let size = 2UL * 1024UL * 1024UL + use fileCache = setUpFileCache size + use fileStorage = new WebFileStorage(Map.ofArray [| + "a", [| for _ in 1UL .. size do 1uy |] + "b", [| for _ in 1UL .. size do 2uy |] + |]) + + // Start downloading the "a" item: + let! stream = fileCache.Download(fileStorage.Link "a", "a", size) + let stream = Option.get stream + // Just keep the stream open for now and trigger the cleanup: + do! assertFileDownloaded fileCache fileStorage "b" size + // Now there's only "b" item in the cache: + assertCacheState [| "b", fileStorage.Content "b" |] + // And now, while still having "a" not downloaded, let's fill the cache with it again (could be broken on + // Windows due to peculiarity of file deletion when opened, see + // https://boostgsoc13.github.io/boost.afio/doc/html/afio/FAQ/deleting_open_files.html): + do! assertFileDownloaded fileCache fileStorage "a" size + assertCacheState [| "a", fileStorage.Content "a" |] + // We should still be able to read "a" fully: + let! content = StreamUtils.readAllBytes stream + Assert.Equal(fileStorage.Content "a", content) + } diff --git a/Emulsion.Tests/Database/DatabaseStructureTests.fs b/Emulsion.Tests/Database/DatabaseStructureTests.fs index e2ca8f79..a2395363 100644 --- a/Emulsion.Tests/Database/DatabaseStructureTests.fs +++ b/Emulsion.Tests/Database/DatabaseStructureTests.fs @@ -5,7 +5,7 @@ open Xunit open Emulsion.Database open Emulsion.Database.Entities -open Emulsion.Tests.TestUtils +open Emulsion.TestFramework [] let ``Unique constraint should hold``(): unit = @@ -16,6 +16,8 @@ let ``Unique constraint should hold``(): unit = ChatUserName = "testChat" MessageId = 666L FileId = "foobar" + FileName = "file.bin" + MimeType = "application/octet-stream" } async { do! DataStorage.addAsync ctx.TelegramContents newContent diff --git a/Emulsion.Tests/Emulsion.Tests.fsproj b/Emulsion.Tests/Emulsion.Tests.fsproj index bcb0b36e..44998653 100644 --- a/Emulsion.Tests/Emulsion.Tests.fsproj +++ b/Emulsion.Tests/Emulsion.Tests.fsproj @@ -4,11 +4,6 @@ false - - - - - @@ -32,6 +27,7 @@ + @@ -39,10 +35,10 @@ - + \ No newline at end of file diff --git a/Emulsion.Tests/MessageSenderTests.fs b/Emulsion.Tests/MessageSenderTests.fs index f0030c27..fa353857 100644 --- a/Emulsion.Tests/MessageSenderTests.fs +++ b/Emulsion.Tests/MessageSenderTests.fs @@ -11,8 +11,8 @@ open Xunit.Abstractions open Emulsion.Messaging open Emulsion.Messaging.MessageSender -open Emulsion.Tests.TestUtils -open Emulsion.Tests.TestUtils.Waiter +open Emulsion.TestFramework +open Emulsion.TestFramework.Waiter type MessageSenderTests(testOutput: ITestOutputHelper) = let testContext = { diff --git a/Emulsion.Tests/MessageSystemTests/MessageSystemBaseTests.fs b/Emulsion.Tests/MessageSystemTests/MessageSystemBaseTests.fs index 6fb5b561..f34747f0 100644 --- a/Emulsion.Tests/MessageSystemTests/MessageSystemBaseTests.fs +++ b/Emulsion.Tests/MessageSystemTests/MessageSystemBaseTests.fs @@ -10,8 +10,8 @@ open Xunit.Abstractions open Emulsion open Emulsion.Messaging open Emulsion.Messaging.MessageSystem -open Emulsion.Tests.TestUtils -open Emulsion.Tests.TestUtils.Waiter +open Emulsion.TestFramework +open Emulsion.TestFramework.Waiter type MessageSystemBaseTests(testLogger: ITestOutputHelper) = let logger = Logging.xunitLogger testLogger diff --git a/Emulsion.Tests/SettingsTests.fs b/Emulsion.Tests/SettingsTests.fs index 4f80a9a2..57b05769 100644 --- a/Emulsion.Tests/SettingsTests.fs +++ b/Emulsion.Tests/SettingsTests.fs @@ -50,6 +50,7 @@ let private testConfiguration = { } Database = None Hosting = None + FileCache = None } let private mockConfiguration groupIdLiteral extendedJson = diff --git a/Emulsion.Tests/Telegram/LinkGeneratorTests.fs b/Emulsion.Tests/Telegram/LinkGeneratorTests.fs index ea579b82..e4960554 100644 --- a/Emulsion.Tests/Telegram/LinkGeneratorTests.fs +++ b/Emulsion.Tests/Telegram/LinkGeneratorTests.fs @@ -10,7 +10,7 @@ open Xunit open Emulsion.Database open Emulsion.Settings open Emulsion.Telegram -open Emulsion.Tests.TestUtils +open Emulsion.TestFramework let private hostingSettings = { ExternalUriBase = Uri "https://example.com" diff --git a/Emulsion.Tests/Web/ContentControllerTests.fs b/Emulsion.Tests/Web/ContentControllerTests.fs index ccad1970..9bec2218 100644 --- a/Emulsion.Tests/Web/ContentControllerTests.fs +++ b/Emulsion.Tests/Web/ContentControllerTests.fs @@ -1,6 +1,7 @@ namespace Emulsion.Tests.Web open System +open System.Security.Cryptography open System.Threading.Tasks open Microsoft.AspNetCore.Mvc @@ -13,8 +14,9 @@ open Emulsion.ContentProxy open Emulsion.Database open Emulsion.Database.Entities open Emulsion.Settings -open Emulsion.Tests.TestUtils -open Emulsion.Tests.TestUtils.Logging +open Emulsion.Telegram +open Emulsion.TestFramework +open Emulsion.TestFramework.Logging open Emulsion.Web type ContentControllerTests(output: ITestOutputHelper) = @@ -26,20 +28,35 @@ type ContentControllerTests(output: ITestOutputHelper) = } let logger = xunitLogger output + let telegramClient = TelegramClientMock() + let sha256 = SHA256.Create() - let performTestWithPreparation prepareAction testAction = Async.StartAsTask(async { + let cacheDirectory = lazy FileCacheUtil.newCacheDirectory() + + let setUpFileCache() = + FileCacheUtil.setUpFileCache output sha256 cacheDirectory.Value 0UL + + let performTestWithPreparation fileCache prepareAction testAction = Async.StartAsTask(async { return! TestDataStorage.doWithDatabase(fun databaseSettings -> async { do! prepareAction databaseSettings use loggerFactory = new SerilogLoggerFactory(logger) let logger = loggerFactory.CreateLogger() use context = new EmulsionDbContext(databaseSettings.ContextOptions) - let controller = ContentController(logger, hostingSettings, context) + let controller = ContentController(logger, hostingSettings, telegramClient, fileCache, context) return! testAction controller }) }) - let performTest = performTestWithPreparation(fun _ -> async.Return()) + let performTest = performTestWithPreparation None (fun _ -> async.Return()) + let performTestWithContent fileCache content = performTestWithPreparation fileCache (fun databaseOptions -> async { + use context = new EmulsionDbContext(databaseOptions.ContextOptions) + do! DataStorage.addAsync context.TelegramContents content + return! Async.Ignore <| Async.AwaitTask(context.SaveChangesAsync()) + }) + + interface IDisposable with + member _.Dispose() = sha256.Dispose() [] member _.``ContentController returns BadRequest on hashId deserialization error``(): Task = @@ -50,7 +67,7 @@ type ContentControllerTests(output: ITestOutputHelper) = }) [] - member _.``ContentController returns NotFound if the content doesn't exist``(): Task = + member _.``ContentController returns NotFound if the content doesn't exist in the database``(): Task = performTest (fun controller -> async { let hashId = Proxy.encodeHashId hostingSettings.HashIdSalt 667L let! result = Async.AwaitTask <| controller.Get hashId @@ -58,23 +75,109 @@ type ContentControllerTests(output: ITestOutputHelper) = }) [] - member _.``ContentController returns a correct result``(): Task = + member _.``ContentController returns a normal redirect if there's no file cache``(): Task = let contentId = 343L let chatUserName = "MySuperExampleChat" let messageId = 777L - performTestWithPreparation (fun databaseOptions -> async { - use context = new EmulsionDbContext(databaseOptions.ContextOptions) - let content = { - Id = contentId - ChatUserName = chatUserName - MessageId = messageId - FileId = "foobar" - } - do! DataStorage.addAsync context.TelegramContents content - return! Async.Ignore <| Async.AwaitTask(context.SaveChangesAsync()) - }) (fun controller -> async { + let content = { + Id = contentId + ChatUserName = chatUserName + MessageId = messageId + FileId = "foobar" + FileName = "file.bin" + MimeType = "application/octet-stream" + } + + performTestWithContent None content (fun controller -> async { let hashId = Proxy.encodeHashId hostingSettings.HashIdSalt contentId let! result = Async.AwaitTask <| controller.Get hashId let redirect = Assert.IsType result - Assert.Equal($"https://t.me/{chatUserName}/{string messageId}", redirect.Url) + Assert.Equal(Uri $"https://t.me/{chatUserName}/{string messageId}", Uri redirect.Url) + }) + + [] + member _.``ContentController returns NotFound if the content doesn't exist on the Telegram server``(): Task = task { + let contentId = 344L + let chatUserName = "MySuperExampleChat" + let messageId = 777L + let fileId = "foobar1" + let content = { + Id = contentId + ChatUserName = chatUserName + MessageId = messageId + FileId = fileId + FileName = "file.bin" + MimeType = "application/octet-stream" + } + + telegramClient.SetResponse(fileId, None) + + use fileCache = setUpFileCache() + do! performTestWithContent (Some fileCache) content (fun controller -> async { + let hashId = Proxy.encodeHashId hostingSettings.HashIdSalt contentId + let! result = Async.AwaitTask <| controller.Get hashId + Assert.IsType result |> ignore }) + } + + [] + member _.``ContentController returns 404 if the cache reports that a file was not found``(): Task = task { + let contentId = 344L + let chatUserName = "MySuperExampleChat" + let messageId = 777L + let fileId = "foobar1" + let content = { + Id = contentId + ChatUserName = chatUserName + MessageId = messageId + FileId = fileId + FileName = "file.bin" + MimeType = "application/octet-stream" + } + + use fileCache = setUpFileCache() + use fileStorage = new WebFileStorage(Map.empty) + telegramClient.SetResponse(fileId, Some { + TemporaryLink = fileStorage.Link fileId + Size = 1UL + }) + + do! performTestWithContent (Some fileCache) content (fun controller -> async { + let hashId = Proxy.encodeHashId hostingSettings.HashIdSalt contentId + let! result = Async.AwaitTask <| controller.Get hashId + Assert.IsType result |> ignore + }) + } + + [] + member _.``ContentController returns a downloaded file from cache``(): Task = task { + let contentId = 343L + let chatUserName = "MySuperExampleChat" + let messageId = 777L + let fileId = "foobar" + let content = { + Id = contentId + ChatUserName = chatUserName + MessageId = messageId + FileId = fileId + FileName = "file.bin" + MimeType = "application/octet-stream" + } + + let onServerFileId = "fileIdOnServer" + use fileCache = setUpFileCache() + use fileStorage = new WebFileStorage(Map.ofArray [| onServerFileId, [| 1uy; 2uy; 3uy |] |]) + let testFileInfo = { + TemporaryLink = fileStorage.Link onServerFileId + Size = 1UL + } + telegramClient.SetResponse(fileId, Some testFileInfo) + + do! performTestWithContent (Some fileCache) content (fun controller -> async { + let hashId = Proxy.encodeHashId hostingSettings.HashIdSalt contentId + let! result = Async.AwaitTask <| controller.Get hashId + let streamResult = Assert.IsType result + let! content = StreamUtils.readAllBytes streamResult.FileStream + Assert.Equal(fileStorage.Content onServerFileId, content) + }) + } diff --git a/Emulsion.Tests/Xmpp/EmulsionXmppTests.fs b/Emulsion.Tests/Xmpp/EmulsionXmppTests.fs index 1e7d894b..5a2bbb80 100644 --- a/Emulsion.Tests/Xmpp/EmulsionXmppTests.fs +++ b/Emulsion.Tests/Xmpp/EmulsionXmppTests.fs @@ -14,7 +14,7 @@ open Xunit.Abstractions open Emulsion open Emulsion.Messaging open Emulsion.Settings -open Emulsion.Tests.TestUtils +open Emulsion.TestFramework open Emulsion.Tests.Xmpp open Emulsion.Xmpp open Emulsion.Xmpp.SharpXmppHelper.Elements diff --git a/Emulsion.Tests/Xmpp/XmppClientRoomTests.fs b/Emulsion.Tests/Xmpp/XmppClientRoomTests.fs index abe1dc97..5d0a678c 100644 --- a/Emulsion.Tests/Xmpp/XmppClientRoomTests.fs +++ b/Emulsion.Tests/Xmpp/XmppClientRoomTests.fs @@ -14,7 +14,7 @@ open Emulsion open Emulsion.Xmpp open Emulsion.Xmpp.SharpXmppHelper.Attributes open Emulsion.Xmpp.SharpXmppHelper.Elements -open Emulsion.Tests.TestUtils.Logging +open Emulsion.TestFramework.Logging type XmppClientRoomTests(output: ITestOutputHelper) = let logger = xunitLogger output diff --git a/Emulsion.Web/ContentController.fs b/Emulsion.Web/ContentController.fs index a7b1552a..9dde6043 100644 --- a/Emulsion.Web/ContentController.fs +++ b/Emulsion.Web/ContentController.fs @@ -7,12 +7,16 @@ open Microsoft.Extensions.Logging open Emulsion.ContentProxy open Emulsion.Database +open Emulsion.Database.Entities open Emulsion.Settings +open Emulsion.Telegram [] [] type ContentController(logger: ILogger, configuration: HostingSettings, + telegram: ITelegramClient, + fileCache: FileCache option, context: EmulsionDbContext) = inherit ControllerBase() @@ -24,22 +28,32 @@ type ContentController(logger: ILogger, logger.LogWarning(ex, "Error during hashId deserializing") None - let produceRedirect contentId: Async = async { - let! content = ContentStorage.getById context contentId - return - content - |> Option.map(fun c -> - let url = $"https://t.me/{c.ChatUserName}/{string c.MessageId}" - RedirectResult url - ) - } - [] member this.Get(hashId: string): Task = task { match decodeHashId hashId with - | None -> return this.BadRequest() + | None -> + logger.LogWarning $"Cannot decode hash id: \"{hashId}\"." + return this.BadRequest() | Some contentId -> - match! produceRedirect contentId with - | None -> return this.NotFound() :> IActionResult - | Some redirect -> return redirect + match! ContentStorage.getById context contentId with + | None -> + logger.LogWarning $"Content \"{contentId}\" not found in content storage." + return this.NotFound() :> IActionResult + | Some content -> + match fileCache with + | None -> + let link = $"https://t.me/{content.ChatUserName}/{string content.MessageId}" + return RedirectResult link + | Some cache -> + match! telegram.GetFileInfo content.FileId with + | None -> + logger.LogWarning $"File \"{content.FileId}\" could not be found on Telegram server." + return this.NotFound() :> IActionResult + | Some fileInfo -> + match! cache.Download(fileInfo.TemporaryLink, content.FileId, fileInfo.Size) with + | None -> + logger.LogWarning $"Link \"{fileInfo}\" could not be downloaded." + return this.NotFound() :> IActionResult + | Some stream -> + return FileStreamResult(stream, content.MimeType, FileDownloadName = content.FileName) } diff --git a/Emulsion.Web/Emulsion.Web.fsproj b/Emulsion.Web/Emulsion.Web.fsproj index eb58afb6..569840ec 100644 --- a/Emulsion.Web/Emulsion.Web.fsproj +++ b/Emulsion.Web/Emulsion.Web.fsproj @@ -12,6 +12,7 @@ + diff --git a/Emulsion.Web/WebServer.fs b/Emulsion.Web/WebServer.fs index a8504922..0953f9ab 100644 --- a/Emulsion.Web/WebServer.fs +++ b/Emulsion.Web/WebServer.fs @@ -2,13 +2,21 @@ module Emulsion.Web.WebServer open System.Threading.Tasks -open Emulsion.Database -open Emulsion.Settings open Microsoft.AspNetCore.Builder open Microsoft.Extensions.DependencyInjection open Serilog -let run (logger: ILogger) (hostingSettings: HostingSettings) (databaseSettings: DatabaseSettings): Task = +open Emulsion.ContentProxy +open Emulsion.Database +open Emulsion.Settings +open Emulsion.Telegram + +let run (logger: ILogger) + (hostingSettings: HostingSettings) + (databaseSettings: DatabaseSettings) + (telegram: ITelegramClient) + (fileCache: FileCache option) + : Task = let builder = WebApplication.CreateBuilder(WebApplicationOptions()) builder.Host.UseSerilog(logger) @@ -16,6 +24,8 @@ let run (logger: ILogger) (hostingSettings: HostingSettings) (databaseSettings: builder.Services .AddSingleton(hostingSettings) + .AddSingleton(telegram) + .AddSingleton(fileCache) .AddTransient(fun _ -> new EmulsionDbContext(databaseSettings.ContextOptions)) .AddControllers() .AddApplicationPart(typeof.Assembly) diff --git a/Emulsion.sln b/Emulsion.sln index ab2ffc42..0f56025d 100644 --- a/Emulsion.sln +++ b/Emulsion.sln @@ -48,6 +48,8 @@ Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Emulsion.Telegram", "Emulsi EndProject Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Emulsion.Messaging", "Emulsion.Messaging\Emulsion.Messaging.fsproj", "{C8DC4049-250B-4E84-BC98-CFC0AF1632AF}" EndProject +Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Emulsion.TestFramework", "Emulsion.TestFramework\Emulsion.TestFramework.fsproj", "{381D687B-6520-48F1-8AA0-3EDB45654AAC}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -89,6 +91,10 @@ Global {C8DC4049-250B-4E84-BC98-CFC0AF1632AF}.Debug|Any CPU.Build.0 = Debug|Any CPU {C8DC4049-250B-4E84-BC98-CFC0AF1632AF}.Release|Any CPU.ActiveCfg = Release|Any CPU {C8DC4049-250B-4E84-BC98-CFC0AF1632AF}.Release|Any CPU.Build.0 = Release|Any CPU + {381D687B-6520-48F1-8AA0-3EDB45654AAC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {381D687B-6520-48F1-8AA0-3EDB45654AAC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {381D687B-6520-48F1-8AA0-3EDB45654AAC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {381D687B-6520-48F1-8AA0-3EDB45654AAC}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(NestedProjects) = preSolution {7D1ADF47-BF1C-4007-BB9B-08C283044467} = {973131E1-E645-4A50-A0D2-1886A1A8F0C6} diff --git a/Emulsion.sln.DotSettings b/Emulsion.sln.DotSettings index 4aa17b0b..b2e4be22 100644 --- a/Emulsion.sln.DotSettings +++ b/Emulsion.sln.DotSettings @@ -1,6 +1,8 @@  True + True True True + True True True \ No newline at end of file diff --git a/Emulsion/Program.fs b/Emulsion/Program.fs index 8b721c08..3ed1e544 100644 --- a/Emulsion/Program.fs +++ b/Emulsion/Program.fs @@ -2,12 +2,14 @@ open System open System.IO +open System.Security.Cryptography open Akka.Actor open Microsoft.Extensions.Configuration open Serilog open Emulsion.Actors +open Emulsion.ContentProxy open Emulsion.Database open Emulsion.Messaging.MessageSystem open Emulsion.Settings @@ -47,21 +49,6 @@ let private startApp config = async { let logger = Logging.createRootLogger config.Log try - match config.Database with - | Some dbSettings -> do! migrateDatabase logger dbSettings - | None -> () - - let webServerTask = - match config.Hosting, config.Database with - | Some hosting, Some database -> - logger.Information "Initializing web server…" - Some <| WebServer.run logger hosting database - | _ -> None - - logger.Information "Actor system preparation…" - use system = ActorSystem.Create("emulsion") - logger.Information "Clients preparation…" - let xmppLogger = Logging.xmppLogger logger let telegramLogger = Logging.telegramLogger logger @@ -72,27 +59,51 @@ let private startApp config = config.Telegram, config.Database, config.Hosting) - let factories = { xmppFactory = Xmpp.spawn xmppLogger xmpp - telegramFactory = Telegram.spawn telegramLogger telegram } - logger.Information "Core preparation…" - let core = Core.spawn logger factories system "core" - logger.Information "Message systems preparation…" - let! telegramSystem = startMessageSystem logger telegram core.Tell - let! xmppSystem = startMessageSystem logger xmpp core.Tell - logger.Information "System ready" - - logger.Information "Waiting for the systems to terminate…" - let! _ = Async.Parallel(seq { - yield Async.AwaitTask system.WhenTerminated - yield telegramSystem - yield xmppSystem - - match webServerTask with - | Some task -> yield Async.AwaitTask task + + use sha256 = SHA256.Create() + let fileCacheOption = config.FileCache |> Option.map(fun settings -> + let httpClientFactory = SimpleHttpClientFactory() + new FileCache(logger, settings, httpClientFactory, sha256) + ) + + try + match config.Database with + | Some dbSettings -> do! migrateDatabase logger dbSettings | None -> () - }) - logger.Information "Terminated successfully." + let webServerTask = + match config.Hosting, config.Database with + | Some hosting, Some database -> + logger.Information "Initializing the web server…" + Some <| WebServer.run logger hosting database telegram fileCacheOption + | _ -> None + + logger.Information "Actor system preparation…" + use system = ActorSystem.Create("emulsion") + logger.Information "Clients preparation…" + + let factories = { xmppFactory = Xmpp.spawn xmppLogger xmpp + telegramFactory = Telegram.spawn telegramLogger telegram } + logger.Information "Core preparation…" + let core = Core.spawn logger factories system "core" + logger.Information "Message systems preparation…" + let! telegramSystem = startMessageSystem logger telegram core.Tell + let! xmppSystem = startMessageSystem logger xmpp core.Tell + logger.Information "System ready" + + logger.Information "Waiting for the systems to terminate…" + do! Async.Ignore <| Async.Parallel(seq { + yield Async.AwaitTask system.WhenTerminated + yield telegramSystem + yield xmppSystem + + match webServerTask with + | Some task -> yield Async.AwaitTask task + | None -> () + }) + finally + fileCacheOption |> Option.iter(fun x -> (x :> IDisposable).Dispose()) + logger.Information "Terminated successfully." with | error -> logger.Fatal(error, "General application failure") diff --git a/README.md b/README.md index 6023e63c..8755dc8d 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,7 @@ $ dotnet build Configure --------- -Copy `emulsion.example.json` to `emulsion.json` and set the settings. For some -settings, there're defaults: +Copy `emulsion.example.json` to `emulsion.json` and set the settings. For some settings, there are defaults: ```json { @@ -26,19 +25,23 @@ settings, there're defaults: "messageTimeout": "00:05:00", "pingInterval": null, "pingTimeout": "00:00:30" + }, + "fileCache": { + "fileSizeLimitBytes": 1048576, + "totalCacheSizeLimitBytes": 20971520 } } ``` -All the other settings are required, except the `database` and `hosting` sections. +All the other settings are required, except the `database`, `hosting` and `fileCache` sections (the corresponding functionality will be turned off if the sections aren't filled). Note that `pingInterval` of `null` disables XMPP ping support. ### Telegram Content Proxy -There's **unfinished** Telegram content proxy support, for XMPP users to access Telegram content without directly opening links on t.me. Right now, it will only generate a redirect to the corresponding t.me URI, so it doesn't help a lot. But in the future, proper content proxy will be supported. +There's Telegram content proxy support, for XMPP users to access Telegram content without directly opening links on t.me. -To enable it, configure the `database` and `hosting` configuration file sections: +To enable it, configure the `database`, `hosting` and `fileCache` configuration file sections: ```json { @@ -49,6 +52,11 @@ To enable it, configure the `database` and `hosting` configuration file sections "externalUriBase": "https://example.com/api/", "bindUri": "http://*:5000/", "hashIdSalt": "test" + }, + "fileCache": { + "directory": "/tmp/emulsion/cache", + "fileSizeLimitBytes": 1048576, + "totalCacheSizeLimitBytes": 20971520 } } ``` @@ -61,6 +69,8 @@ If all the parameters are set, then Emulsion will save the incoming messages int The content identifiers in question are generated from the database ones using the [hashids.net][hashids.net] library, `hashIdSalt` is used in generation. This should complicate guessing of content ids for any external party not reading the chat directly. +If the `fileCache.directory` option is not set, then the content proxy will only generate redirects to corresponding t.me URIs. Otherwise, it will store the downloaded files (that fit the cache) in a cache on disk; the items not fitting into the cache will be proxied to clients. + ### Recommended Network Configuration Current configuration system allows the following: @@ -115,7 +125,7 @@ where - `$EMULSION_VERSION` is the image version you want to deploy, or `latest` for the latest available one - `$CONFIG` is the **absolute** path to the configuration file -- `$DATA` is the absolute path to the data directory +- `$DATA` is the absolute path to the data directory (used by the configuration) - `$WEB_PORT` is the port on the host system which will be used to access the content proxy To build and push the container to Docker Hub, use the following commands: diff --git a/docs/create-migration.md b/docs/create-migration.md index e689820f..a08ed419 100644 --- a/docs/create-migration.md +++ b/docs/create-migration.md @@ -3,7 +3,7 @@ This article explains how to create a database migration using [EFCore.FSharp][efcore.fsharp]. -1. Change the entity type (see `Emulsion.Database/Models.fs`), update the `EmulsionDbContext` if required. +1. Change the entity type (see `Emulsion.Database/Entities.fs`), update the `EmulsionDbContext` if required. 2. Run the following shell commands: ```console diff --git a/emulsion.example.json b/emulsion.example.json index 7ee1ca5b..0d064ede 100644 --- a/emulsion.example.json +++ b/emulsion.example.json @@ -24,5 +24,10 @@ "externalUriBase": "https://example.com/api/", "bindUri": "http://*:5000", "hashIdSalt": "test" + }, + "fileCache": { + "directory": "./cache", + "fileSizeLimitBytes": 1048576, + "totalCacheSizeLimitBytes": 20971520 } }