From 59390d57eea7ecac8cd17d004d7f140a95e93374 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Sat, 7 Jun 2025 21:40:05 +1000 Subject: [PATCH 1/4] feat: move game import to new task system --- pages/admin/library/import.vue | 4 +- server/api/v1/admin/import/game/index.post.ts | 17 +- server/internal/metadata/giantbomb.ts | 30 ++- server/internal/metadata/igdb.ts | 189 ++++++++++-------- server/internal/metadata/index.ts | 175 +++++++++------- server/internal/objects/transactional.ts | 13 ++ server/internal/tasks/index.ts | 16 ++ 7 files changed, 276 insertions(+), 168 deletions(-) diff --git a/pages/admin/library/import.vue b/pages/admin/library/import.vue index 61af7934..5f32f544 100644 --- a/pages/admin/library/import.vue +++ b/pages/admin/library/import.vue @@ -334,7 +334,7 @@ async function importGame(useMetadata: boolean) { : undefined; const option = games.unimportedGames[currentlySelectedGame.value]; - const game = await $dropFetch("/api/v1/admin/import/game", { + const { taskId } = await $dropFetch("/api/v1/admin/import/game", { method: "POST", body: { path: option.game, @@ -343,7 +343,7 @@ async function importGame(useMetadata: boolean) { }, }); - router.push(`/admin/library/${game.id}`); + router.push(`/admin/task/${taskId}`); } function importGame_wrapper(metadata = true) { importLoading.value = true; diff --git a/server/api/v1/admin/import/game/index.post.ts b/server/api/v1/admin/import/game/index.post.ts index 4d4bd5b4..e3e32478 100644 --- a/server/api/v1/admin/import/game/index.post.ts +++ b/server/api/v1/admin/import/game/index.post.ts @@ -37,10 +37,17 @@ export default defineEventHandler<{ body: typeof ImportGameBody.infer }>( statusMessage: "Invalid library or game.", }); - if (!metadata) { - return await metadataHandler.createGameWithoutMetadata(library, path); - } else { - return await metadataHandler.createGame(metadata, library, path); - } + const taskId = metadata + ? await metadataHandler.createGame(metadata, library, path) + : await metadataHandler.createGameWithoutMetadata(library, path); + + if (!taskId) + throw createError({ + statusCode: 400, + statusMessage: + "Duplicate metadata import. Please chose a different game or metadata provider.", + }); + + return { taskId }; }, ); diff --git a/server/internal/metadata/giantbomb.ts b/server/internal/metadata/giantbomb.ts index b75838fd..176d808a 100644 --- a/server/internal/metadata/giantbomb.ts +++ b/server/internal/metadata/giantbomb.ts @@ -12,6 +12,7 @@ import type { import axios, { type AxiosRequestConfig } from "axios"; import TurndownService from "turndown"; import { DateTime } from "luxon"; +import type { TaskRunContext } from "../tasks"; interface GiantBombResponseType { error: "OK" | string; @@ -164,12 +165,12 @@ export class GiantBombProvider implements MetadataProvider { return mapped; } - async fetchGame({ - id, - publisher, - developer, - createObject, - }: _FetchGameMetadataParams): Promise { + async fetchGame( + { id, publisher, developer, createObject }: _FetchGameMetadataParams, + context?: TaskRunContext, + ): Promise { + context?.log("Using GiantBomb provider"); + const result = await this.request("game", id, {}); const gameData = result.data.results; @@ -180,21 +181,29 @@ export class GiantBombProvider implements MetadataProvider { const publishers: Company[] = []; if (gameData.publishers) { for (const pub of gameData.publishers) { + context?.log(`Importing publisher "${pub.name}"`); + const res = await publisher(pub.name); if (res === undefined) continue; publishers.push(res); } } + context?.progress(35); + const developers: Company[] = []; if (gameData.developers) { for (const dev of gameData.developers) { + context?.log(`Importing developer "${dev.name}"`); + const res = await developer(dev.name); if (res === undefined) continue; developers.push(res); } } + context?.progress(70); + const icon = createObject(gameData.image.icon_url); const banner = createObject(gameData.image.screen_large_url); @@ -202,6 +211,8 @@ export class GiantBombProvider implements MetadataProvider { const images = [banner, ...imageURLs.map(createObject)]; + context?.log(`Found all images. Total of ${images.length + 1}.`); + const releaseDate = gameData.original_release_date ? DateTime.fromISO(gameData.original_release_date).toJSDate() : DateTime.fromISO( @@ -210,8 +221,11 @@ export class GiantBombProvider implements MetadataProvider { }-${gameData.expected_release_day ?? 1}`, ).toJSDate(); + context?.progress(85); + const reviews: GameMetadataRating[] = []; if (gameData.reviews) { + context?.log("Found reviews, importing..."); for (const { api_detail_url } of gameData.reviews) { const reviewId = api_detail_url.split("/").at(-2); if (!reviewId) continue; @@ -225,6 +239,7 @@ export class GiantBombProvider implements MetadataProvider { }); } } + const metadata: GameMetadata = { id: gameData.guid, name: gameData.name, @@ -245,6 +260,9 @@ export class GiantBombProvider implements MetadataProvider { images, }; + context?.log("GiantBomb provider finished."); + context?.progress(100); + return metadata; } async fetchCompany({ diff --git a/server/internal/metadata/igdb.ts b/server/internal/metadata/igdb.ts index 505b09b9..877e061c 100644 --- a/server/internal/metadata/igdb.ts +++ b/server/internal/metadata/igdb.ts @@ -13,6 +13,7 @@ import type { AxiosRequestConfig } from "axios"; import axios from "axios"; import { DateTime } from "luxon"; import * as jdenticon from "jdenticon"; +import type { TaskRunContext } from "../tasks"; type IGDBID = number; @@ -345,107 +346,125 @@ export class IGDBProvider implements MetadataProvider { return results; } - async fetchGame({ - id, - publisher, - developer, - createObject, - }: _FetchGameMetadataParams): Promise { + async fetchGame( + { id, publisher, developer, createObject }: _FetchGameMetadataParams, + context?: TaskRunContext, + ): Promise { const body = `where id = ${id}; fields *;`; - const response = await this.request("games", body); + const currentGame = (await this.request("games", body)).at(0); + if (!currentGame) throw new Error("No game found on IGDB with that id"); - for (let i = 0; i < response.length; i++) { - const currentGame = response[i]; - if (!currentGame) continue; + context?.log("Using IDGB provider."); - let iconRaw; - const cover = currentGame.cover; - if (cover !== undefined) { - iconRaw = await this.getCoverURL(cover); - } else { - iconRaw = jdenticon.toPng(id, 512); - } - const icon = createObject(iconRaw); - let banner = ""; - - const images = [icon]; - for (const art of currentGame.artworks ?? []) { - // if banner not set - if (banner.length <= 0) { - banner = createObject(await this.getArtworkURL(art)); - images.push(banner); - } else { - images.push(createObject(await this.getArtworkURL(art))); - } + let iconRaw; + const cover = currentGame.cover; + + if (cover !== undefined) { + context?.log("Found cover URL, using..."); + iconRaw = await this.getCoverURL(cover); + } else { + context?.log("Missing cover URL, using fallback..."); + iconRaw = jdenticon.toPng(id, 512); + } + + const icon = createObject(iconRaw); + let banner; + + const images = [icon]; + for (const art of currentGame.artworks ?? []) { + const objectId = createObject(await this.getArtworkURL(art)); + if (!banner) { + banner = objectId; } + images.push(objectId); + } + + if (!banner) { + banner = createObject(jdenticon.toPng(id, 512)); + } + + context?.progress(20); - const publishers: Company[] = []; - const developers: Company[] = []; - for (const involvedCompany of currentGame.involved_companies ?? []) { - // get details about the involved company - const involved_company_response = - await this.request( - "involved_companies", - `where id = ${involvedCompany}; fields *;`, + const publishers: Company[] = []; + const developers: Company[] = []; + for (const involvedCompany of currentGame.involved_companies ?? []) { + // get details about the involved company + const involved_company_response = await this.request( + "involved_companies", + `where id = ${involvedCompany}; fields *;`, + ); + for (const foundInvolved of involved_company_response) { + // now we need to get the actual company so we can get the name + const findCompanyResponse = await this.request< + { name: string } & IGDBItem + >("companies", `where id = ${foundInvolved.company}; fields name;`); + + for (const company of findCompanyResponse) { + context?.log( + `Found involved company "${company.name}" as: ${foundInvolved.developer ? "developer, " : ""}${foundInvolved.publisher ? "publisher" : ""}`, ); - for (const foundInvolved of involved_company_response) { - // now we need to get the actual company so we can get the name - const findCompanyResponse = await this.request< - { name: string } & IGDBItem - >("companies", `where id = ${foundInvolved.company}; fields name;`); - - for (const company of findCompanyResponse) { - // if company was a dev or publisher - // CANNOT use else since a company can be both - if (foundInvolved.developer) { - const res = await developer(company.name); - if (res === undefined) continue; - developers.push(res); - } - if (foundInvolved.publisher) { - const res = await publisher(company.name); - if (res === undefined) continue; - publishers.push(res); - } + + // if company was a dev or publisher + // CANNOT use else since a company can be both + if (foundInvolved.developer) { + const res = await developer(company.name); + if (res === undefined) continue; + developers.push(res); + } + + if (foundInvolved.publisher) { + const res = await publisher(company.name); + if (res === undefined) continue; + publishers.push(res); } } } + } - const firstReleaseDate = currentGame.first_release_date; + context?.progress(80); - return { - id: "" + response[i].id, - name: response[i].name, - shortDescription: this.trimMessage(currentGame.summary, 280), - description: currentGame.summary, - released: - firstReleaseDate === undefined - ? new Date() - : DateTime.fromSeconds(firstReleaseDate).toJSDate(), + const firstReleaseDate = currentGame.first_release_date; + const released = + firstReleaseDate === undefined + ? new Date() + : DateTime.fromSeconds(firstReleaseDate).toJSDate(); - reviews: [ - { - metadataId: "" + currentGame.id, - metadataSource: MetadataSource.IGDB, - mReviewCount: currentGame.total_rating_count ?? 0, - mReviewRating: (currentGame.total_rating ?? 0) / 100, - mReviewHref: currentGame.url, - }, - ], + const review = { + metadataId: currentGame.id.toString(), + metadataSource: MetadataSource.IGDB, + mReviewCount: currentGame.total_rating_count ?? 0, + mReviewRating: (currentGame.total_rating ?? 0) / 100, + mReviewHref: currentGame.url, + }; - publishers: [], - developers: [], + const tags = await this.getGenres(currentGame.genres); - tags: await this.getGenres(currentGame.genres), + const deck = this.trimMessage(currentGame.summary, 280); - icon, - bannerId: banner, - coverId: icon, - images, - }; - } + const metadata = { + id: currentGame.id.toString(), + name: currentGame.name, + shortDescription: deck, + description: currentGame.summary, + released, + + reviews: [review], + + publishers, + developers, + + tags, + + icon, + bannerId: banner, + coverId: icon, + images, + }; + + context?.log("IGDB provider finished."); + context?.progress(100); - throw new Error("No game found on igdb with that id"); + return metadata; } async fetchCompany({ query, diff --git a/server/internal/metadata/index.ts b/server/internal/metadata/index.ts index 97e59417..8292bd1b 100644 --- a/server/internal/metadata/index.ts +++ b/server/internal/metadata/index.ts @@ -1,4 +1,4 @@ -import { MetadataSource, type GameRating } from "~/prisma/client"; +import { type Prisma, MetadataSource } from "~/prisma/client"; import prisma from "../db/database"; import type { _FetchGameMetadataParams, @@ -12,6 +12,10 @@ import type { import { ObjectTransactionalHandler } from "../objects/transactional"; import { PriorityListIndexed } from "../utils/prioritylist"; import { systemConfig } from "../config/sys-conf"; +import type { TaskRunContext } from "../tasks"; +import taskHandler, { wrapTaskContext } from "../tasks"; +import { randomUUID } from "crypto"; +import { fuzzy } from "fast-fuzzy"; export class MissingMetadataProviderConfig extends Error { private providerName: string; @@ -34,9 +38,13 @@ export abstract class MetadataProvider { abstract source(): MetadataSource; abstract search(query: string): Promise; - abstract fetchGame(params: _FetchGameMetadataParams): Promise; + abstract fetchGame( + params: _FetchGameMetadataParams, + taskRunContext?: TaskRunContext, + ): Promise; abstract fetchCompany( params: _FetchCompanyMetadataParams, + taskRunContext?: TaskRunContext, ): Promise; } @@ -92,7 +100,12 @@ export class MetadataHandler { const successfulResults = results .filter((result) => result.status === "fulfilled") .map((result) => result.value) - .flat(); + .flat() + .map((result) => { + const match = fuzzy(query, result.name); + return { ...result, fuzzy: match }; + }) + .sort((a, b) => b.fuzzy - a.fuzzy); return successfulResults; } @@ -110,14 +123,7 @@ export class MetadataHandler { } private parseTags(tags: string[]) { - const results: { - where: { - name: string; - }; - create: { - name: string; - }; - }[] = []; + const results: Array = []; tags.forEach((t) => results.push({ @@ -134,15 +140,7 @@ export class MetadataHandler { } private parseRatings(ratings: GameMetadataRating[]) { - const results: { - where: { - metadataKey: { - metadataId: string; - metadataSource: MetadataSource; - }; - }; - create: Omit; - }[] = []; + const results: Array = []; ratings.forEach((r) => { results.push({ @@ -178,65 +176,102 @@ export class MetadataHandler { }, }, }); - if (existing) return existing; - - const [createObject, pullObjects, dumpObjects] = this.objectHandler.new( - {}, - ["internal:read"], - ); - - let metadata: GameMetadata | undefined = undefined; - try { - metadata = await provider.fetchGame({ - id: result.id, - name: result.name, - // wrap in anonymous functions to keep references to this - publisher: (name: string) => this.fetchCompany(name), - developer: (name: string) => this.fetchCompany(name), - createObject, - }); - } catch (e) { - dumpObjects(); - throw e; - } + if (existing) return undefined; + + const gameId = randomUUID(); + + const taskId = `import:${gameId}`; + await taskHandler.create({ + name: `Import game "${result.name}" (${libraryPath})`, + id: taskId, + taskGroup: "import:game", + acls: ["system:import:game:read"], + async run(context) { + const { progress, log } = context; + + progress(0); + + const [createObject, pullObjects, dumpObjects] = + metadataHandler.objectHandler.new( + {}, + ["internal:read"], + wrapTaskContext(context, { + min: 63, + max: 100, + prefix: "[object import] ", + }), + ); - const game = await prisma.game.create({ - data: { - metadataSource: provider.source(), - metadataId: metadata.id, + let metadata: GameMetadata | undefined = undefined; + try { + metadata = await provider.fetchGame( + { + id: result.id, + name: result.name, + // wrap in anonymous functions to keep references to this + publisher: (name: string) => metadataHandler.fetchCompany(name), + developer: (name: string) => metadataHandler.fetchCompany(name), + createObject, + }, + wrapTaskContext(context, { + min: 0, + max: 60, + prefix: "[metadata import] ", + }), + ); + } catch (e) { + dumpObjects(); + throw e; + } - mName: metadata.name, - mShortDescription: metadata.shortDescription, - mDescription: metadata.description, - mReleased: metadata.released, + context?.progress(60); - mIconObjectId: metadata.icon, - mBannerObjectId: metadata.bannerId, - mCoverObjectId: metadata.coverId, - mImageLibraryObjectIds: metadata.images, + await prisma.game.create({ + data: { + id: gameId, + metadataSource: provider.source(), + metadataId: metadata.id, + + mName: metadata.name, + mShortDescription: metadata.shortDescription, + mDescription: metadata.description, + mReleased: metadata.released, + + mIconObjectId: metadata.icon, + mBannerObjectId: metadata.bannerId, + mCoverObjectId: metadata.coverId, + mImageLibraryObjectIds: metadata.images, + + publishers: { + connect: metadata.publishers, + }, + developers: { + connect: metadata.developers, + }, + + ratings: { + connectOrCreate: metadataHandler.parseRatings(metadata.reviews), + }, + tags: { + connectOrCreate: metadataHandler.parseTags(metadata.tags), + }, + + libraryId, + libraryPath, + }, + }); - publishers: { - connect: metadata.publishers, - }, - developers: { - connect: metadata.developers, - }, + progress(63); + log(`Successfully fetched all metadata.`); + log(`Importing objects...`); - ratings: { - connectOrCreate: this.parseRatings(metadata.reviews), - }, - tags: { - connectOrCreate: this.parseTags(metadata.tags), - }, + await pullObjects(); - libraryId, - libraryPath, + log(`Finished game import.`); }, }); - await pullObjects(); - - return game; + return taskId; } // Careful with this function, it has no typechecking diff --git a/server/internal/objects/transactional.ts b/server/internal/objects/transactional.ts index 3155aba3..dc9134fe 100644 --- a/server/internal/objects/transactional.ts +++ b/server/internal/objects/transactional.ts @@ -5,6 +5,7 @@ This is used as a utility in metadata handling, so we only fetch the objects if import type { Readable } from "stream"; import { randomUUID } from "node:crypto"; import objectHandler from "."; +import type { TaskRunContext } from "../tasks"; export type TransactionDataType = string | Readable | Buffer; type TransactionTable = Map; // ID to data @@ -20,6 +21,7 @@ export class ObjectTransactionalHandler { new( metadata: { [key: string]: string }, permissions: Array, + context?: TaskRunContext, ): [Register, Pull, Dump] { const transactionId = randomUUID(); @@ -35,7 +37,16 @@ export class ObjectTransactionalHandler { const pull = async () => { const transaction = this.record.get(transactionId); if (!transaction) return; + + let progress = 0; + const increment = (1 / transaction.size) * 100; + for (const [id, data] of transaction) { + if (typeof data === "string") { + context?.log(`Importing object from "${data}"`); + } else { + context?.log(`Importing raw object...`); + } await objectHandler.createFromSource( id, () => { @@ -47,6 +58,8 @@ export class ObjectTransactionalHandler { metadata, permissions, ); + progress += increment; + context?.progress(progress); } }; diff --git a/server/internal/tasks/index.ts b/server/internal/tasks/index.ts index 599f9ebf..e9b9ce8f 100644 --- a/server/internal/tasks/index.ts +++ b/server/internal/tasks/index.ts @@ -332,6 +332,22 @@ export type TaskRunContext = { log: (message: string) => void; }; +export function wrapTaskContext( + context: TaskRunContext, + options: { min: number; max: number; prefix: string }, +): TaskRunContext { + return { + progress(progress) { + const scalar = 100 / (options.max - options.min); + const adjustedProgress = progress * scalar + options.min; + return context.progress(adjustedProgress); + }, + log(message) { + return context.log(options.prefix + message); + }, + }; +} + export interface Task { id: string; taskGroup: TaskGroup; From 4101b386ece8b5cc6493ae02a0f7b05b58bbf962 Mon Sep 17 00:00:00 2001 From: DecDuck Date: Sat, 7 Jun 2025 21:51:43 +1000 Subject: [PATCH 2/4] fix: sizing issue with new task UI --- pages/admin/task/[id]/index.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pages/admin/task/[id]/index.vue b/pages/admin/task/[id]/index.vue index 466ca83e..2bde783b 100644 --- a/pages/admin/task/[id]/index.vue +++ b/pages/admin/task/[id]/index.vue @@ -60,7 +60,7 @@ 
{{ line }}
-
+