From 12e33bd11a41b9582aef713a23b0a128f0c14ec7 Mon Sep 17 00:00:00 2001 From: Sergii Date: Fri, 23 Jan 2026 14:10:13 +0100 Subject: [PATCH] refactor: adds proper logger and move templates into module --- apps/api/scripts/buildAkashTemplatesCache.ts | 8 +- apps/api/src/caching/helpers.ts | 16 + .../src/services/external/githubService.ts | 13 - .../templates/template-gallery.service.ts | 124 ---- .../template/template.controller.ts | 16 +- .../template-fetcher.service.spec.ts | 664 ++++++++++++++++++ .../template-fetcher.service.ts | 49 +- .../template-gallery.service.spec.ts | 238 +++++++ .../template-gallery.service.ts | 185 +++++ .../template-processor.service.spec.ts | 475 +++++++++++++ .../template-processor.service.ts | 33 +- apps/api/src/template/types/template.ts | 6 +- .../__snapshots__/template-cache.spec.ts.snap | 218 ------ .../test/functional/template-cache.spec.ts | 77 -- apps/api/test/functional/templates.spec.ts | 109 --- ...kash-network-awesome-akash-cached-sha.json | 91 --- ...ash-network-cosmos-omnibus-cached-sha.json | 57 -- ...ndcoffee-akash-linuxserver-cached-sha.json | 60 -- .../awesome-akash/alpaca-cpp/index.json | 50 -- .../awesome-akash/lunie-lite/index.json | 66 -- .../git/akash-network/awesome-akash/readme.md | 11 - .../awesome-akash/ssh-ubuntu/index.json | 98 --- .../cosmos-omnibus/akash/index.json | 66 -- .../cosmos-omnibus/archway/index.json | 66 -- .../akash-network/cosmos-omnibus/index.json | 82 --- .../adguardhome-sync/index.json | 34 - .../airsonic-advanced/index.json | 34 - .../akash-linuxserver/airsonic/index.json | 34 - .../akash-linuxserver/readme.md | 13 - 29 files changed, 1654 insertions(+), 1339 deletions(-) delete mode 100644 apps/api/src/services/external/githubService.ts delete mode 100644 apps/api/src/services/external/templates/template-gallery.service.ts create mode 100644 apps/api/src/template/services/template-fetcher/template-fetcher.service.spec.ts rename apps/api/src/{services/external/templates => template/services/template-fetcher}/template-fetcher.service.ts (89%) create mode 100644 apps/api/src/template/services/template-gallery/template-gallery.service.spec.ts create mode 100644 apps/api/src/template/services/template-gallery/template-gallery.service.ts create mode 100644 apps/api/src/template/services/template-processor/template-processor.service.spec.ts rename apps/api/src/{services/external/templates => template/services/template-processor}/template-processor.service.ts (75%) delete mode 100644 apps/api/test/functional/__snapshots__/template-cache.spec.ts.snap delete mode 100644 apps/api/test/functional/template-cache.spec.ts delete mode 100644 apps/api/test/functional/templates.spec.ts delete mode 100644 apps/api/test/mocks/templates/cache/akash-network-awesome-akash-cached-sha.json delete mode 100644 apps/api/test/mocks/templates/cache/akash-network-cosmos-omnibus-cached-sha.json delete mode 100644 apps/api/test/mocks/templates/cache/cryptoandcoffee-akash-linuxserver-cached-sha.json delete mode 100644 apps/api/test/mocks/templates/git/akash-network/awesome-akash/alpaca-cpp/index.json delete mode 100644 apps/api/test/mocks/templates/git/akash-network/awesome-akash/lunie-lite/index.json delete mode 100644 apps/api/test/mocks/templates/git/akash-network/awesome-akash/readme.md delete mode 100644 apps/api/test/mocks/templates/git/akash-network/awesome-akash/ssh-ubuntu/index.json delete mode 100644 apps/api/test/mocks/templates/git/akash-network/cosmos-omnibus/akash/index.json delete mode 100644 apps/api/test/mocks/templates/git/akash-network/cosmos-omnibus/archway/index.json delete mode 100644 apps/api/test/mocks/templates/git/akash-network/cosmos-omnibus/index.json delete mode 100644 apps/api/test/mocks/templates/git/cryptoandcoffee/akash-linuxserver/adguardhome-sync/index.json delete mode 100644 apps/api/test/mocks/templates/git/cryptoandcoffee/akash-linuxserver/airsonic-advanced/index.json delete mode 100644 apps/api/test/mocks/templates/git/cryptoandcoffee/akash-linuxserver/airsonic/index.json delete mode 100644 apps/api/test/mocks/templates/git/cryptoandcoffee/akash-linuxserver/readme.md diff --git a/apps/api/scripts/buildAkashTemplatesCache.ts b/apps/api/scripts/buildAkashTemplatesCache.ts index 7265aa5b27..bd1f72f4c1 100644 --- a/apps/api/scripts/buildAkashTemplatesCache.ts +++ b/apps/api/scripts/buildAkashTemplatesCache.ts @@ -1,6 +1,10 @@ +import "reflect-metadata"; import "@akashnetwork/env-loader"; -import { TemplateGalleryService } from "../src/services/external/templates/template-gallery.service"; +import { LoggerService } from "@akashnetwork/logging"; +import { promises as fsp } from "node:fs"; + +import { TemplateGalleryService } from "../src/template/services/template-gallery/template-gallery.service"; import { dataFolderPath } from "../src/utils/constants"; console.log("Warming up Akash templates cache..."); @@ -10,7 +14,7 @@ if (!githubPAT) { process.exit(1); } -const templateGalleryService = new TemplateGalleryService({ +const templateGalleryService = new TemplateGalleryService(LoggerService.forContext("TemplateGalleryService.script"), fsp, { githubPAT, dataFolderPath, categoryProcessingConcurrency: 30, diff --git a/apps/api/src/caching/helpers.ts b/apps/api/src/caching/helpers.ts index 225d7785d6..bbde87bc14 100644 --- a/apps/api/src/caching/helpers.ts +++ b/apps/api/src/caching/helpers.ts @@ -147,3 +147,19 @@ export const cacheKeys = { getGpuUtilization: "getGpuUtilization", getGpuBreakdown: "getGpuBreakdown" }; + +export function reusePendingPromise Promise>(fn: T, options?: { getKey?: (...args: Parameters) => string }): T { + const pendingPromises = new Map>(); + + return ((...args: Parameters) => { + const key = options?.getKey ? options.getKey(...args) : JSON.stringify(args); + + let pendingPromise = pendingPromises.get(key); + if (!pendingPromise) { + pendingPromise = fn(...args).finally(() => pendingPromises.delete(key)) as ReturnType; + pendingPromises.set(key, pendingPromise); + } + + return pendingPromise as ReturnType; + }) as unknown as T; +} diff --git a/apps/api/src/services/external/githubService.ts b/apps/api/src/services/external/githubService.ts deleted file mode 100644 index bcaab5794a..0000000000 --- a/apps/api/src/services/external/githubService.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { Octokit } from "@octokit/rest"; - -export function getOctokit(githubPAT: string) { - if (!githubPAT) { - throw new Error("GITHUB_PAT is missing"); - } - - return new Octokit({ - auth: githubPAT, - userAgent: "Console API", - baseUrl: "https://api.github.com" - }); -} diff --git a/apps/api/src/services/external/templates/template-gallery.service.ts b/apps/api/src/services/external/templates/template-gallery.service.ts deleted file mode 100644 index 6b256fb0ac..0000000000 --- a/apps/api/src/services/external/templates/template-gallery.service.ts +++ /dev/null @@ -1,124 +0,0 @@ -import { minutesToSeconds } from "date-fns"; -import * as fs from "fs"; -import path from "path"; - -import { Memoize } from "@src/caching/helpers"; -import type { Category, FinalCategory, Template } from "@src/template/types/template"; -import { REPOSITORIES, TemplateFetcherService } from "./template-fetcher.service"; -import { TemplateProcessorService } from "./template-processor.service"; - -type Options = { - githubPAT?: string; - dataFolderPath: string; - categoryProcessingConcurrency?: number; - templateSourceProcessingConcurrency?: number; -}; - -export class TemplateGalleryService { - private generatingTasks: Record | null> = {}; - private lastServedData: FinalCategory[] | null = null; - - private readonly templateFetcher: TemplateFetcherService; - private readonly templateProcessor: TemplateProcessorService; - - constructor(private readonly options: Options) { - this.templateProcessor = new TemplateProcessorService(); - this.templateFetcher = new TemplateFetcherService(this.templateProcessor, { - githubPAT: this.options.githubPAT, - categoryProcessingConcurrency: this.options.categoryProcessingConcurrency, - templateSourceProcessingConcurrency: this.options.templateSourceProcessingConcurrency - }); - } - - @Memoize({ ttlInSeconds: minutesToSeconds(5) }) - async getTemplateGallery() { - try { - const [awesomeAkashTemplates, omnibusTemplates, linuxServerTemplates] = await Promise.all([ - this.getTemplatesFromRepo({ - repository: "awesome-akash", - fetchTemplates: this.templateFetcher.fetchAwesomeAkashTemplates.bind(this.templateFetcher) - }), - this.getTemplatesFromRepo({ - repository: "cosmos-omnibus", - fetchTemplates: this.templateFetcher.fetchOmnibusTemplates.bind(this.templateFetcher) - }), - this.getTemplatesFromRepo({ - repository: "akash-linuxserver", - fetchTemplates: this.templateFetcher.fetchLinuxServerTemplates.bind(this.templateFetcher) - }) - ]); - - const templateGallery = this.templateProcessor - .mergeTemplateCategories(omnibusTemplates, awesomeAkashTemplates, linuxServerTemplates) - .map(({ templateSources, ...category }) => category); - - this.lastServedData = templateGallery; - - console.log(`${this.templateFetcher.githubRequestsRemaining} requests remaining`); - - return templateGallery; - } catch (err) { - if (this.lastServedData) { - console.error(err); - console.log("Serving template gallery from last working version"); - return this.lastServedData; - } else { - throw err; - } - } - } - - @Memoize({ ttlInSeconds: minutesToSeconds(5) }) - async getTemplateById(id: Required