diff --git a/apps/api/scripts/buildAkashTemplatesCache.ts b/apps/api/scripts/buildAkashTemplatesCache.ts index 7265aa5b27..bd1f72f4c1 100644 --- a/apps/api/scripts/buildAkashTemplatesCache.ts +++ b/apps/api/scripts/buildAkashTemplatesCache.ts @@ -1,6 +1,10 @@ +import "reflect-metadata"; import "@akashnetwork/env-loader"; -import { TemplateGalleryService } from "../src/services/external/templates/template-gallery.service"; +import { LoggerService } from "@akashnetwork/logging"; +import { promises as fsp } from "node:fs"; + +import { TemplateGalleryService } from "../src/template/services/template-gallery/template-gallery.service"; import { dataFolderPath } from "../src/utils/constants"; console.log("Warming up Akash templates cache..."); @@ -10,7 +14,7 @@ if (!githubPAT) { process.exit(1); } -const templateGalleryService = new TemplateGalleryService({ +const templateGalleryService = new TemplateGalleryService(LoggerService.forContext("TemplateGalleryService.script"), fsp, { githubPAT, dataFolderPath, categoryProcessingConcurrency: 30, diff --git a/apps/api/src/caching/helpers.ts b/apps/api/src/caching/helpers.ts index 225d7785d6..bbde87bc14 100644 --- a/apps/api/src/caching/helpers.ts +++ b/apps/api/src/caching/helpers.ts @@ -147,3 +147,19 @@ export const cacheKeys = { getGpuUtilization: "getGpuUtilization", getGpuBreakdown: "getGpuBreakdown" }; + +export function reusePendingPromise Promise>(fn: T, options?: { getKey?: (...args: Parameters) => string }): T { + const pendingPromises = new Map>(); + + return ((...args: Parameters) => { + const key = options?.getKey ? options.getKey(...args) : JSON.stringify(args); + + let pendingPromise = pendingPromises.get(key); + if (!pendingPromise) { + pendingPromise = fn(...args).finally(() => pendingPromises.delete(key)) as ReturnType; + pendingPromises.set(key, pendingPromise); + } + + return pendingPromise as ReturnType; + }) as unknown as T; +} diff --git a/apps/api/src/services/external/githubService.ts b/apps/api/src/services/external/githubService.ts deleted file mode 100644 index bcaab5794a..0000000000 --- a/apps/api/src/services/external/githubService.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { Octokit } from "@octokit/rest"; - -export function getOctokit(githubPAT: string) { - if (!githubPAT) { - throw new Error("GITHUB_PAT is missing"); - } - - return new Octokit({ - auth: githubPAT, - userAgent: "Console API", - baseUrl: "https://api.github.com" - }); -} diff --git a/apps/api/src/services/external/templates/template-gallery.service.ts b/apps/api/src/services/external/templates/template-gallery.service.ts deleted file mode 100644 index 6b256fb0ac..0000000000 --- a/apps/api/src/services/external/templates/template-gallery.service.ts +++ /dev/null @@ -1,124 +0,0 @@ -import { minutesToSeconds } from "date-fns"; -import * as fs from "fs"; -import path from "path"; - -import { Memoize } from "@src/caching/helpers"; -import type { Category, FinalCategory, Template } from "@src/template/types/template"; -import { REPOSITORIES, TemplateFetcherService } from "./template-fetcher.service"; -import { TemplateProcessorService } from "./template-processor.service"; - -type Options = { - githubPAT?: string; - dataFolderPath: string; - categoryProcessingConcurrency?: number; - templateSourceProcessingConcurrency?: number; -}; - -export class TemplateGalleryService { - private generatingTasks: Record | null> = {}; - private lastServedData: FinalCategory[] | null = null; - - private readonly templateFetcher: TemplateFetcherService; - private readonly templateProcessor: TemplateProcessorService; - - constructor(private readonly options: Options) { - this.templateProcessor = new TemplateProcessorService(); - this.templateFetcher = new TemplateFetcherService(this.templateProcessor, { - githubPAT: this.options.githubPAT, - categoryProcessingConcurrency: this.options.categoryProcessingConcurrency, - templateSourceProcessingConcurrency: this.options.templateSourceProcessingConcurrency - }); - } - - @Memoize({ ttlInSeconds: minutesToSeconds(5) }) - async getTemplateGallery() { - try { - const [awesomeAkashTemplates, omnibusTemplates, linuxServerTemplates] = await Promise.all([ - this.getTemplatesFromRepo({ - repository: "awesome-akash", - fetchTemplates: this.templateFetcher.fetchAwesomeAkashTemplates.bind(this.templateFetcher) - }), - this.getTemplatesFromRepo({ - repository: "cosmos-omnibus", - fetchTemplates: this.templateFetcher.fetchOmnibusTemplates.bind(this.templateFetcher) - }), - this.getTemplatesFromRepo({ - repository: "akash-linuxserver", - fetchTemplates: this.templateFetcher.fetchLinuxServerTemplates.bind(this.templateFetcher) - }) - ]); - - const templateGallery = this.templateProcessor - .mergeTemplateCategories(omnibusTemplates, awesomeAkashTemplates, linuxServerTemplates) - .map(({ templateSources, ...category }) => category); - - this.lastServedData = templateGallery; - - console.log(`${this.templateFetcher.githubRequestsRemaining} requests remaining`); - - return templateGallery; - } catch (err) { - if (this.lastServedData) { - console.error(err); - console.log("Serving template gallery from last working version"); - return this.lastServedData; - } else { - throw err; - } - } - } - - @Memoize({ ttlInSeconds: minutesToSeconds(5) }) - async getTemplateById(id: Required