From f98af5e70f3c99d6fa212fc8468faa19d8a4b21b Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Fri, 7 Jun 2024 16:30:21 +0200 Subject: [PATCH 01/22] feat(Spotify): initial provider implementation --- providers/Spotify/api_types.ts | 105 +++++++++++ providers/Spotify/mod.ts | 268 +++++++++++++++++++++++++++++ providers/Spotify/regions.ts | 189 ++++++++++++++++++++ providers/mod.ts | 2 + server/components/ProviderIcon.tsx | 1 + server/routes/icon-sprite.svg.tsx | 2 + server/static/harmony.css | 4 + 7 files changed, 571 insertions(+) create mode 100644 providers/Spotify/api_types.ts create mode 100644 providers/Spotify/mod.ts create mode 100644 providers/Spotify/regions.ts diff --git a/providers/Spotify/api_types.ts b/providers/Spotify/api_types.ts new file mode 100644 index 00000000..e6bc92f7 --- /dev/null +++ b/providers/Spotify/api_types.ts @@ -0,0 +1,105 @@ +export type Album = { + id: string; + type: 'album'; + href: string; + name: string; + uri: string; + artists: SimplifiedArtist[]; + tracks: ResultList; + album_type: AlbumType; + total_tracks: number; + release_date: string; + release_date_precision: ReleaseDatePrecision; + external_urls: { spotify: string }; + images: Image[]; + available_markets: string[]; + restrictions: { reason: string }; + copyrights: Copyright[]; + external_ids: ExternalIds; + genres: string[]; + label: string; + /** The popularity of the album. The value will be between 0 and 100, with 100 being the most popular. */ + popularity: number; +}; + +export type SimplifiedArtist = { + id: string; + type: 'artist'; + href: string; + name: string; + uri: string; +}; + +export type Artist = SimplifiedArtist & { + followers: { href: string; total: number }; +}; + +export type LinkedTrack = { + id: string; + type: 'track'; + href: string; + uri: string; + external_urls: { spotify: string }; +}; + +export type SimplifiedTrack = LinkedTrack & { + name: string; + artists: SimplifiedArtist[]; + track_number: number; + disc_number: number; + duration_ms: number; + explicit: boolean; + is_playable: boolean; + is_local: boolean; + preview_url: string; + linked_from: LinkedTrack | undefined; + available_markets: string[]; + restrictions: { reason: string }; +}; + +export type Track = SimplifiedTrack & { + album: Album; + artists: Artist[]; + external_ids: ExternalIds; + popularity: number; +}; + +export type Image = { + url: string; + width: number; + height: number; +}; + +export type Copyright = { + text: string; + type: CopyrightType; +}; + +export type ExternalIds = { + isrc: string; + ean: string; + upc: string; +}; + +export type ReleaseDatePrecision = 'year' | 'month' | 'day'; + +export type AlbumType = 'album' | 'single' | 'compilation'; + +export type CopyrightType = 'C' | 'P'; + +export type ResultList = { + href: string; + limit: number; + offset: number; + total: number; + next: string; + previous: string; + items: T[]; +}; + +export type ApiError = { + error: { + status: number; + message: string; + }; +}; diff --git a/providers/Spotify/mod.ts b/providers/Spotify/mod.ts new file mode 100644 index 00000000..be43a914 --- /dev/null +++ b/providers/Spotify/mod.ts @@ -0,0 +1,268 @@ +import { type CacheEntry, MetadataApiProvider, ReleaseApiLookup } from '@/providers/base.ts'; +import { DurationPrecision, FeatureQuality, FeatureQualityMap } from '@/providers/features.ts'; +import { parseHyphenatedDate, PartialDate } from '@/utils/date.ts'; +import { ResponseError } from '@/utils/errors.ts'; +import { encodeBase64 } from 'std/encoding/base64.ts'; +import { availableRegions } from './regions.ts'; + +import type { Album, ApiError, Image, SimplifiedArtist, SimplifiedTrack } from './api_types.ts'; +import type { + ArtistCreditName, + Artwork, + EntityId, + HarmonyMedium, + HarmonyRelease, + HarmonyTrack, + Label, +} from '@/harmonizer/types.ts'; + +// See https://developer.spotify.com/documentation/web-api + +const spotifyClientId = Deno.env.get('HARMONY_SPOTIFY_CLIENT_ID') || ''; +const spotifyClientSecret = Deno.env.get('HARMONY_SPOTIFY_CLIENT_SECRET') || ''; + +export default class SpotifyProvider extends MetadataApiProvider { + readonly name = 'Spotify'; + + readonly supportedUrls = new URLPattern({ + hostname: 'open.spotify.com', + pathname: '{/intl-:region}?/:type(artist|album)/:id', + }); + + readonly features: FeatureQualityMap = { + 'cover size': 640, + 'duration precision': DurationPrecision.MS, + // 'GTIN lookup': FeatureQuality.GOOD, + 'GTIN lookup': FeatureQuality.MISSING, + 'MBID resolving': FeatureQuality.GOOD, + 'release label': FeatureQuality.PRESENT, + }; + + readonly entityTypeMap = { + artist: 'artist', + release: 'album', + }; + + readonly availableRegions = new Set(availableRegions); + + readonly releaseLookup = SpotifyReleaseLookup; + + readonly launchDate: PartialDate = { + year: 2008, + month: 10, + }; + + readonly apiBaseUrl = 'https://api.spotify.com/v1/'; + + constructUrl(entity: EntityId): URL { + return new URL([entity.type, entity.id].join('/'), 'https://open.spotify.com'); + } + + async query(apiUrl: URL, maxTimestamp?: number): Promise> { + const cacheEntry = await this.fetchJSON(apiUrl, { + policy: { maxTimestamp }, + requestInit: { + headers: { + 'Authorization': `Bearer ${await this.accessToken()}`, + }, + }, + }); + const { error } = cacheEntry.content as { error?: ApiError }; + + if (error) { + throw new SpotifyResponseError(error, apiUrl); + } + return cacheEntry; + } + + private async accessToken(): Promise { + const cacheKey = `${this.name}:accessKey`; + let tokenResult = JSON.parse(localStorage.getItem(cacheKey) || '{}'); + if (!tokenResult?.accessToken || Date.now() > tokenResult?.validUntil) { + tokenResult = await this.requestAccessToken(); + localStorage.setItem(cacheKey, JSON.stringify(tokenResult)); + } + return tokenResult.accessToken; + } + + private async requestAccessToken(): Promise<{ accessToken: string; validUntil: number }> { + // See https://developer.spotify.com/documentation/web-api/tutorials/client-credentials-flow + const url = new URL('https://accounts.spotify.com/api/token'); + const auth = encodeBase64(`${spotifyClientId}:${spotifyClientSecret}`); + const body = new URLSearchParams(); + body.append('grant_type', 'client_credentials'); + + const response = await fetch(url, { + method: 'POST', + headers: { + 'Authorization': `Basic ${auth}`, + 'Content-Type': 'application/x-www-form-urlencoded', + }, + body: body, + }); + + const content = await response.json(); + return { + accessToken: content?.access_token, + validUntil: Date.now() + (content.expires_in * 1000), + }; + } +} + +export class SpotifyReleaseLookup extends ReleaseApiLookup { + constructReleaseApiUrl(): URL { + const { method, value, region } = this.lookup; + let lookupUrl: URL; + const query = new URLSearchParams(); + if (region) { + query.set('market', region); + } + if (method === 'gtin') { + lookupUrl = new URL(`/search`, this.provider.apiBaseUrl); + query.set('type', 'album'); + query.set('q', `upc:${value}`); + } else { // if (method === 'id') + lookupUrl = new URL(`albums/${value}`, this.provider.apiBaseUrl); + } + + lookupUrl.search = query.toString(); + return lookupUrl; + } + + protected async getRawRelease(): Promise { + const apiUrl = this.constructReleaseApiUrl(); + if (this.lookup.method === 'gtin') { + throw new Error('GTIN lookup not implemented.'); + } else { // if (method === 'id') + const cacheEntry = await this.provider.query( + apiUrl, + this.options.snapshotMaxTimestamp, + ); + const release = cacheEntry.content; + this.updateCacheTime(cacheEntry.timestamp); + return release; + } + } + + // deno-lint-ignore require-await + private async getRawTracklist(rawRelease: Album): Promise { + // FIXME: Check whether the track list is complete and perform additional + // queries if it is not. + // Also ISRCs seem to be only available in separate queries. + return rawRelease.tracks.items; + } + + protected async convertRawRelease(rawRelease: Album): Promise { + this.id = rawRelease.id; + const rawTracklist = await this.getRawTracklist(rawRelease); + const media = this.convertRawTracklist(rawTracklist); + return { + title: rawRelease.name, + artists: rawRelease.artists.map(this.convertRawArtist.bind(this)), + gtin: rawRelease.external_ids.ean || rawRelease.external_ids.upc, + externalLinks: [{ + url: new URL(rawRelease.external_urls.spotify), + types: ['free streaming'], + }], + media, + releaseDate: parseHyphenatedDate(rawRelease.release_date), + copyright: rawRelease.copyrights.map((c) => c.text).join('\n'), + status: 'Official', + packaging: 'None', + images: this.getLargestCoverImage(rawRelease.images), + labels: this.getLabels(rawRelease), + availableIn: rawRelease.available_markets, + info: this.generateReleaseInfo(), + }; + } + + private convertRawTracklist(tracklist: SimplifiedTrack[]): HarmonyMedium[] { + const result: HarmonyMedium[] = []; + let medium: HarmonyMedium = { + number: 1, + format: 'Digital Media', + tracklist: [], + }; + + // split flat tracklist into media + tracklist.forEach((item) => { + // store the previous medium and create a new one + if (item.disc_number !== medium.number) { + result.push(medium); + + medium = { + number: item.disc_number, + format: 'Digital Media', + tracklist: [], + }; + } + + medium.tracklist.push(this.convertRawTrack(item)); + }); + + // store the final medium + result.push(medium); + + return result; + } + + private convertRawTrack(track: SimplifiedTrack): HarmonyTrack { + const result: HarmonyTrack = { + number: track.track_number, + title: track.name, + length: track.duration_ms, + // isrc: track., // FIXME + artists: track.artists.map(this.convertRawArtist.bind(this)), + availableIn: track.available_markets, + }; + + return result; + } + + private convertRawArtist(artist: SimplifiedArtist): ArtistCreditName { + return { + name: artist.name, + creditedName: artist.name, + externalIds: this.provider.makeExternalIds({ type: 'artist', id: artist.id }), + }; + } + + private getLargestCoverImage(images: Image[]): Artwork[] { + let largestImage: Image | undefined; + let thumbnail: Image | undefined; + images.forEach((i) => { + if (!largestImage || i.width > largestImage.width) { + largestImage = i; + } + if (i.width >= 200 && (!thumbnail || i.width < thumbnail.width)) { + thumbnail = i; + } + }); + if (!largestImage) return []; + let thumbUrl: URL | undefined; + if (thumbnail) { + thumbUrl = new URL(thumbnail.url); + } + return [{ + url: new URL(largestImage.url), + thumbUrl: thumbUrl, + types: ['front'], + }]; + } + + private getLabels(rawRelease: Album): Label[] { + if (rawRelease.label) { + return [{ + name: rawRelease.label, + }]; + } + + return []; + } +} + +class SpotifyResponseError extends ResponseError { + constructor(readonly details: ApiError, url: URL) { + super('Spotify', details?.error?.message, url); + } +} diff --git a/providers/Spotify/regions.ts b/providers/Spotify/regions.ts new file mode 100644 index 00000000..b7468602 --- /dev/null +++ b/providers/Spotify/regions.ts @@ -0,0 +1,189 @@ +// Fetched from API endpoint https://developer.spotify.com/documentation/web-api/reference/get-available-markets + +export const availableRegions = [ + 'AD', + 'AE', + 'AG', + 'AL', + 'AM', + 'AO', + 'AR', + 'AT', + 'AU', + 'AZ', + 'BA', + 'BB', + 'BD', + 'BE', + 'BF', + 'BG', + 'BH', + 'BI', + 'BJ', + 'BN', + 'BO', + 'BR', + 'BS', + 'BT', + 'BW', + 'BY', + 'BZ', + 'CA', + 'CD', + 'CG', + 'CH', + 'CI', + 'CL', + 'CM', + 'CO', + 'CR', + 'CV', + 'CW', + 'CY', + 'CZ', + 'DE', + 'DJ', + 'DK', + 'DM', + 'DO', + 'DZ', + 'EC', + 'EE', + 'EG', + 'ES', + 'ET', + 'FI', + 'FJ', + 'FM', + 'FR', + 'GA', + 'GB', + 'GD', + 'GE', + 'GH', + 'GM', + 'GN', + 'GQ', + 'GR', + 'GT', + 'GW', + 'GY', + 'HK', + 'HN', + 'HR', + 'HT', + 'HU', + 'ID', + 'IE', + 'IL', + 'IN', + 'IQ', + 'IS', + 'IT', + 'JM', + 'JO', + 'JP', + 'KE', + 'KG', + 'KH', + 'KI', + 'KM', + 'KN', + 'KR', + 'KW', + 'KZ', + 'LA', + 'LB', + 'LC', + 'LI', + 'LK', + 'LR', + 'LS', + 'LT', + 'LU', + 'LV', + 'LY', + 'MA', + 'MC', + 'MD', + 'ME', + 'MG', + 'MH', + 'MK', + 'ML', + 'MN', + 'MO', + 'MR', + 'MT', + 'MU', + 'MV', + 'MW', + 'MX', + 'MY', + 'MZ', + 'NA', + 'NE', + 'NG', + 'NI', + 'NL', + 'NO', + 'NP', + 'NR', + 'NZ', + 'OM', + 'PA', + 'PE', + 'PG', + 'PH', + 'PK', + 'PL', + 'PR', + 'PS', + 'PT', + 'PW', + 'PY', + 'QA', + 'RO', + 'RS', + 'RW', + 'SA', + 'SB', + 'SC', + 'SE', + 'SG', + 'SI', + 'SK', + 'SL', + 'SM', + 'SN', + 'SR', + 'ST', + 'SV', + 'SZ', + 'TD', + 'TG', + 'TH', + 'TJ', + 'TL', + 'TN', + 'TO', + 'TR', + 'TT', + 'TV', + 'TW', + 'TZ', + 'UA', + 'UG', + 'US', + 'UY', + 'UZ', + 'VC', + 'VE', + 'VN', + 'VU', + 'WS', + 'XK', + 'ZA', + 'ZM', + 'ZW', +]; diff --git a/providers/mod.ts b/providers/mod.ts index b97c3556..1e14d8cc 100644 --- a/providers/mod.ts +++ b/providers/mod.ts @@ -5,6 +5,7 @@ import BandcampProvider from './Bandcamp/mod.ts'; import BeatportProvider from './Beatport/mod.ts'; import DeezerProvider from './Deezer/mod.ts'; import iTunesProvider from './iTunes/mod.ts'; +import SpotifyProvider from './Spotify/mod.ts'; import TidalProvider from './Tidal/mod.ts'; /** Registry with all supported providers. */ @@ -14,6 +15,7 @@ export const providers = new ProviderRegistry(); providers.addMultiple( DeezerProvider, iTunesProvider, + SpotifyProvider, TidalProvider, BandcampProvider, BeatportProvider, diff --git a/server/components/ProviderIcon.tsx b/server/components/ProviderIcon.tsx index 6ae44b4b..bf813a9a 100644 --- a/server/components/ProviderIcon.tsx +++ b/server/components/ProviderIcon.tsx @@ -8,6 +8,7 @@ const providerIconMap: Record = { deezer: 'brand-deezer', itunes: 'brand-apple', musicbrainz: 'brand-metabrainz', + spotify: 'brand-spotify', tidal: 'brand-tidal', }; diff --git a/server/routes/icon-sprite.svg.tsx b/server/routes/icon-sprite.svg.tsx index 2efbcd8e..16dbd236 100644 --- a/server/routes/icon-sprite.svg.tsx +++ b/server/routes/icon-sprite.svg.tsx @@ -4,6 +4,7 @@ import IconBrandApple from 'tabler-icons/brand-apple.tsx'; import IconBrandBandcamp from 'tabler-icons/brand-bandcamp.tsx'; import IconBrandDeezer from 'tabler-icons/brand-deezer.tsx'; import IconBrandGit from 'tabler-icons/brand-git.tsx'; +import IconBrandSpotify from 'tabler-icons/brand-spotify.tsx'; import IconBrandTidal from 'tabler-icons/brand-tidal.tsx'; import IconAlertTriangle from 'tabler-icons/alert-triangle.tsx'; import IconBarcode from 'tabler-icons/barcode.tsx'; @@ -48,6 +49,7 @@ const icons: Icon[] = [ IconBrandDeezer, IconBrandGit, IconBrandMetaBrainz, + IconBrandSpotify, IconBrandTidal, IconPuzzle, ]; diff --git a/server/static/harmony.css b/server/static/harmony.css index d56c2e1c..03c7686a 100644 --- a/server/static/harmony.css +++ b/server/static/harmony.css @@ -22,6 +22,7 @@ --bandcamp: #1da0c3; --beatport: #00e586; --deezer: #a238ff; + --spotify: #1db954; --tidal: #000000; } @@ -368,6 +369,9 @@ label.deezer { label.itunes { background-color: var(--apple); } +label.spotify { + background-color: var(--spotify); +} label.tidal { background-color: var(--tidal); } From a5a25b98af5fcf33cf2e5441cfc4de3327e38fac Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Fri, 7 Jun 2024 19:05:57 +0200 Subject: [PATCH 02/22] feat(Spotify): GTIN search --- providers/Spotify/api_types.ts | 18 ++++++++++++++++-- providers/Spotify/mod.ts | 32 ++++++++++++++++++++++---------- 2 files changed, 38 insertions(+), 12 deletions(-) diff --git a/providers/Spotify/api_types.ts b/providers/Spotify/api_types.ts index e6bc92f7..80ecb336 100644 --- a/providers/Spotify/api_types.ts +++ b/providers/Spotify/api_types.ts @@ -1,11 +1,10 @@ -export type Album = { +export type SimplifiedAlbum = { id: string; type: 'album'; href: string; name: string; uri: string; artists: SimplifiedArtist[]; - tracks: ResultList; album_type: AlbumType; total_tracks: number; release_date: string; @@ -14,6 +13,10 @@ export type Album = { images: Image[]; available_markets: string[]; restrictions: { reason: string }; +}; + +export type Album = SimplifiedAlbum & { + tracks: ResultList; copyrights: Copyright[]; external_ids: ExternalIds; genres: string[]; @@ -97,6 +100,17 @@ export type ResultList = { items: T[]; }; +export type SearchResult = { + albums: ResultList; + tracks: ResultList; + artists: ResultList; + // Unsupported / not needed: + // Playlists: ResultList; + // Shows: ResultList; + // Episodes: ResultList; + // Audiobooks: ResultList; +}; + export type ApiError = { error: { status: number; diff --git a/providers/Spotify/mod.ts b/providers/Spotify/mod.ts index be43a914..67a5acc7 100644 --- a/providers/Spotify/mod.ts +++ b/providers/Spotify/mod.ts @@ -5,7 +5,7 @@ import { ResponseError } from '@/utils/errors.ts'; import { encodeBase64 } from 'std/encoding/base64.ts'; import { availableRegions } from './regions.ts'; -import type { Album, ApiError, Image, SimplifiedArtist, SimplifiedTrack } from './api_types.ts'; +import type { Album, ApiError, Image, SearchResult, SimplifiedArtist, SimplifiedTrack } from './api_types.ts'; import type { ArtistCreditName, Artwork, @@ -32,8 +32,7 @@ export default class SpotifyProvider extends MetadataApiProvider { readonly features: FeatureQualityMap = { 'cover size': 640, 'duration precision': DurationPrecision.MS, - // 'GTIN lookup': FeatureQuality.GOOD, - 'GTIN lookup': FeatureQuality.MISSING, + 'GTIN lookup': FeatureQuality.GOOD, 'MBID resolving': FeatureQuality.GOOD, 'release label': FeatureQuality.PRESENT, }; @@ -118,7 +117,7 @@ export class SpotifyReleaseLookup extends ReleaseApiLookup { + this.lookup.region = [...this.options.regions || []][0]; const apiUrl = this.constructReleaseApiUrl(); if (this.lookup.method === 'gtin') { - throw new Error('GTIN lookup not implemented.'); - } else { // if (method === 'id') - const cacheEntry = await this.provider.query( + const cacheEntry = await this.provider.query( apiUrl, this.options.snapshotMaxTimestamp, ); - const release = cacheEntry.content; - this.updateCacheTime(cacheEntry.timestamp); - return release; + if (!cacheEntry.content?.albums?.items?.length) { + throw new ResponseError(this.provider.name, 'API returned no results', apiUrl); + } + + // Result is a SimplifiedAlbum. Perform a regular ID lookup with the found release + // ID to retrieve complete data. + this.lookup.method = 'id'; + this.lookup.value = cacheEntry.content.albums.items[0].id; } + + const cacheEntry = await this.provider.query( + this.constructReleaseApiUrl(), + this.options.snapshotMaxTimestamp, + ); + const release = cacheEntry.content; + + this.updateCacheTime(cacheEntry.timestamp); + return release; } // deno-lint-ignore require-await From 32fc11f7da13c383b6c8d6a966cf600045501fc6 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sat, 8 Jun 2024 00:58:20 +0200 Subject: [PATCH 03/22] fix(Spotify): perform requests without region code The code is not required to obtain the result, and omitting it ensures the available_markets arrays are set in the response. --- providers/Spotify/mod.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/providers/Spotify/mod.ts b/providers/Spotify/mod.ts index 67a5acc7..d1c679a6 100644 --- a/providers/Spotify/mod.ts +++ b/providers/Spotify/mod.ts @@ -110,18 +110,14 @@ export default class SpotifyProvider extends MetadataApiProvider { export class SpotifyReleaseLookup extends ReleaseApiLookup { constructReleaseApiUrl(): URL { - const { method, value, region } = this.lookup; let lookupUrl: URL; const query = new URLSearchParams(); - if (region) { - query.set('market', region); - } - if (method === 'gtin') { + if (this.lookup.method === 'gtin') { lookupUrl = new URL(`search`, this.provider.apiBaseUrl); query.set('type', 'album'); - query.set('q', `upc:${value}`); + query.set('q', `upc:${this.lookup.value}`); } else { // if (method === 'id') - lookupUrl = new URL(`albums/${value}`, this.provider.apiBaseUrl); + lookupUrl = new URL(`albums/${this.lookup.value}`, this.provider.apiBaseUrl); } lookupUrl.search = query.toString(); @@ -129,7 +125,6 @@ export class SpotifyReleaseLookup extends ReleaseApiLookup { - this.lookup.region = [...this.options.regions || []][0]; const apiUrl = this.constructReleaseApiUrl(); if (this.lookup.method === 'gtin') { const cacheEntry = await this.provider.query( From 3be0f60d3b6cb55f65fb01fd745e9f8c815665ab Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sat, 8 Jun 2024 01:20:10 +0200 Subject: [PATCH 04/22] =?UTF-8?q?feat(Spotify):=20extend=20copyright=20inf?= =?UTF-8?q?o=20with=20=E2=84=97=20or=20=C2=A9=20depending=20on=20type?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- providers/Spotify/mod.ts | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/providers/Spotify/mod.ts b/providers/Spotify/mod.ts index d1c679a6..91eca1ad 100644 --- a/providers/Spotify/mod.ts +++ b/providers/Spotify/mod.ts @@ -5,7 +5,15 @@ import { ResponseError } from '@/utils/errors.ts'; import { encodeBase64 } from 'std/encoding/base64.ts'; import { availableRegions } from './regions.ts'; -import type { Album, ApiError, Image, SearchResult, SimplifiedArtist, SimplifiedTrack } from './api_types.ts'; +import type { + Album, + ApiError, + Copyright, + Image, + SearchResult, + SimplifiedArtist, + SimplifiedTrack, +} from './api_types.ts'; import type { ArtistCreditName, Artwork, @@ -173,7 +181,7 @@ export class SpotifyReleaseLookup extends ReleaseApiLookup c.text).join('\n'), + copyright: this.getCopyright(rawRelease.copyrights), status: 'Official', packaging: 'None', images: this.getLargestCoverImage(rawRelease.images), @@ -266,6 +274,23 @@ export class SpotifyReleaseLookup extends ReleaseApiLookup Date: Sat, 8 Jun 2024 01:25:15 +0200 Subject: [PATCH 05/22] feat(Spotify): split release labels on slashes --- providers/Spotify/mod.ts | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/providers/Spotify/mod.ts b/providers/Spotify/mod.ts index 91eca1ad..12859e56 100644 --- a/providers/Spotify/mod.ts +++ b/providers/Spotify/mod.ts @@ -266,13 +266,10 @@ export class SpotifyReleaseLookup extends ReleaseApiLookup ({ + name: label.trim(), + })); } private getCopyright(copyrights: Copyright[]): string { From 6d136577b720167206b2634a0f03683dbbb46452 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sat, 8 Jun 2024 11:56:11 +0200 Subject: [PATCH 06/22] fix(Spotify): try UPC barcode search with prefixed zeros --- providers/Spotify/mod.ts | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/providers/Spotify/mod.ts b/providers/Spotify/mod.ts index 12859e56..09e7fca4 100644 --- a/providers/Spotify/mod.ts +++ b/providers/Spotify/mod.ts @@ -133,20 +133,32 @@ export class SpotifyReleaseLookup extends ReleaseApiLookup { - const apiUrl = this.constructReleaseApiUrl(); if (this.lookup.method === 'gtin') { - const cacheEntry = await this.provider.query( - apiUrl, - this.options.snapshotMaxTimestamp, - ); - if (!cacheEntry.content?.albums?.items?.length) { - throw new ResponseError(this.provider.name, 'API returned no results', apiUrl); + // Spotify does not always find UPC barcodes but expects them prefixed with + // 0 to a max. size of 14 characters. E.g. "810121774182" gives no results, + // but "00810121774182" does. + let albumId: string | undefined; + while (this.lookup.value.length <= 14 && !albumId) { + const cacheEntry = await this.provider.query( + this.constructReleaseApiUrl(), + this.options.snapshotMaxTimestamp, + ); + if (cacheEntry.content?.albums?.items?.length) { + albumId = cacheEntry.content.albums.items[0].id; + } + // Prefix the GTIN with an additional 0 + this.lookup.value = `0${this.lookup.value}`; + } + + // No results found + if (!albumId) { + throw new ResponseError(this.provider.name, 'API returned no results', this.constructReleaseApiUrl()); } // Result is a SimplifiedAlbum. Perform a regular ID lookup with the found release // ID to retrieve complete data. this.lookup.method = 'id'; - this.lookup.value = cacheEntry.content.albums.items[0].id; + this.lookup.value = albumId; } const cacheEntry = await this.provider.query( From faa7ce7ab6a6d5b072839d85a9b4cf7ebca983bc Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sat, 8 Jun 2024 12:21:19 +0200 Subject: [PATCH 07/22] fix(Spotify): load full track length for releases with > 50 tracks --- providers/Spotify/mod.ts | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/providers/Spotify/mod.ts b/providers/Spotify/mod.ts index 09e7fca4..2dabb600 100644 --- a/providers/Spotify/mod.ts +++ b/providers/Spotify/mod.ts @@ -10,6 +10,7 @@ import type { ApiError, Copyright, Image, + ResultList, SearchResult, SimplifiedArtist, SimplifiedTrack, @@ -171,12 +172,27 @@ export class SpotifyReleaseLookup extends ReleaseApiLookup { - // FIXME: Check whether the track list is complete and perform additional - // queries if it is not. - // Also ISRCs seem to be only available in separate queries. - return rawRelease.tracks.items; + let allTracks: SimplifiedTrack[] = []; + allTracks = allTracks.concat(rawRelease.tracks.items); + + // The initial response contains max. 50 tracks. Fetch the remaining + // tracks with separate requests if needed. + let nextUrl = rawRelease.tracks.next; + while (nextUrl && allTracks.length < rawRelease.tracks.total) { + const cacheEntry = await this.provider.query>( + new URL(nextUrl), + this.options.snapshotMaxTimestamp, + ); + this.updateCacheTime(cacheEntry.timestamp); + allTracks = allTracks.concat(cacheEntry.content.items); + nextUrl = cacheEntry.content.next; + } + + // TODO: ISRCs seem to be only available when separately querying tracks + // via the /v1/tracks endpoint. + + return allTracks; } protected async convertRawRelease(rawRelease: Album): Promise { From 7549841c9470b2dbedb1cd69d676fcb50c5887bf Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sat, 8 Jun 2024 14:44:01 +0200 Subject: [PATCH 08/22] feat(Spotify): load extended track information to get ISRCs --- providers/Spotify/api_types.ts | 9 ++++++++- providers/Spotify/mod.ts | 35 ++++++++++++++++++++++++++++------ 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/providers/Spotify/api_types.ts b/providers/Spotify/api_types.ts index 80ecb336..78cbf0f4 100644 --- a/providers/Spotify/api_types.ts +++ b/providers/Spotify/api_types.ts @@ -90,13 +90,20 @@ export type AlbumType = 'album' | 'single' | 'compilation'; export type CopyrightType = 'C' | 'P'; -export type ResultList = { +export type BaseResultList = { href: string; limit: number; offset: number; total: number; next: string; previous: string; +}; + +export type TrackList = BaseResultList & { + tracks: Track[]; +}; + +export type ResultList = BaseResultList & { items: T[]; }; diff --git a/providers/Spotify/mod.ts b/providers/Spotify/mod.ts index 2dabb600..25be6f03 100644 --- a/providers/Spotify/mod.ts +++ b/providers/Spotify/mod.ts @@ -14,6 +14,8 @@ import type { SearchResult, SimplifiedArtist, SimplifiedTrack, + Track, + TrackList, } from './api_types.ts'; import type { ArtistCreditName, @@ -172,7 +174,7 @@ export class SpotifyReleaseLookup extends ReleaseApiLookup { + private async getRawTracklist(rawRelease: Album): Promise { let allTracks: SimplifiedTrack[] = []; allTracks = allTracks.concat(rawRelease.tracks.items); @@ -189,8 +191,29 @@ export class SpotifyReleaseLookup extends ReleaseApiLookup { + let allTracks: Track[] = []; + const trackIds = simplifiedTracks.map((track) => track.id); + + // The SimplifiedTrack entries do not contain ISRCs. + // Perform track queries to obtain the full details of all tracks. + // Each query can return up to 50 tracks. + const maxResults = 50; + const apiUrl = new URL('tracks', this.provider.apiBaseUrl); + for (let index = 0; index < trackIds.length; index += maxResults) { + apiUrl.searchParams.set('ids', trackIds.slice(index, index + maxResults).join(',')); + apiUrl.search = apiUrl.searchParams.toString(); + const cacheEntry = await this.provider.query( + apiUrl, + this.options.snapshotMaxTimestamp, + ); + this.updateCacheTime(cacheEntry.timestamp); + allTracks = allTracks.concat(cacheEntry.content.tracks); + } return allTracks; } @@ -219,7 +242,7 @@ export class SpotifyReleaseLookup extends ReleaseApiLookup Date: Sat, 8 Jun 2024 15:16:56 +0200 Subject: [PATCH 09/22] refactor(provider): moved Tidal / Spotify artwork selection into generic utility Added the generic utility function selectLargestImage to turn a list of same images in different sizes into a single Artwork object with thumbnail. --- providers/Spotify/mod.ts | 29 +++-------------------------- providers/Tidal/mod.ts | 30 ++++-------------------------- utils/image.ts | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 41 insertions(+), 52 deletions(-) create mode 100644 utils/image.ts diff --git a/providers/Spotify/mod.ts b/providers/Spotify/mod.ts index 25be6f03..f1050462 100644 --- a/providers/Spotify/mod.ts +++ b/providers/Spotify/mod.ts @@ -2,6 +2,7 @@ import { type CacheEntry, MetadataApiProvider, ReleaseApiLookup } from '@/provid import { DurationPrecision, FeatureQuality, FeatureQualityMap } from '@/providers/features.ts'; import { parseHyphenatedDate, PartialDate } from '@/utils/date.ts'; import { ResponseError } from '@/utils/errors.ts'; +import { selectLargestImage } from '@/utils/image.ts'; import { encodeBase64 } from 'std/encoding/base64.ts'; import { availableRegions } from './regions.ts'; @@ -9,7 +10,6 @@ import type { Album, ApiError, Copyright, - Image, ResultList, SearchResult, SimplifiedArtist, @@ -19,7 +19,6 @@ import type { } from './api_types.ts'; import type { ArtistCreditName, - Artwork, EntityId, HarmonyMedium, HarmonyRelease, @@ -222,6 +221,7 @@ export class SpotifyReleaseLookup extends ReleaseApiLookup { - if (!largestImage || i.width > largestImage.width) { - largestImage = i; - } - if (i.width >= 200 && (!thumbnail || i.width < thumbnail.width)) { - thumbnail = i; - } - }); - if (!largestImage) return []; - let thumbUrl: URL | undefined; - if (thumbnail) { - thumbUrl = new URL(thumbnail.url); - } - return [{ - url: new URL(largestImage.url), - thumbUrl: thumbUrl, - types: ['front'], - }]; - } - private getLabels(rawRelease: Album): Label[] { // split label string using slashes if the results have at least 3 characters return rawRelease.label?.split(/(?<=[^/]{3,})\/(?=[^/]{3,})/).map((label) => ({ diff --git a/providers/Tidal/mod.ts b/providers/Tidal/mod.ts index b5268bbd..69d1cecc 100644 --- a/providers/Tidal/mod.ts +++ b/providers/Tidal/mod.ts @@ -3,12 +3,12 @@ import { type CacheEntry, MetadataApiProvider, type ProviderOptions, ReleaseApiL import { DurationPrecision, FeatureQuality, FeatureQualityMap } from '@/providers/features.ts'; import { parseHyphenatedDate, PartialDate } from '@/utils/date.ts'; import { ResponseError } from '@/utils/errors.ts'; +import { selectLargestImage } from '@/utils/image.ts'; import { encodeBase64 } from 'std/encoding/base64.ts'; -import type { Album, AlbumItem, ApiError, Image, Resource, Result, SimpleArtist } from './api_types.ts'; +import type { Album, AlbumItem, ApiError, Resource, Result, SimpleArtist } from './api_types.ts'; import type { ArtistCreditName, - Artwork, CountryCode, EntityId, HarmonyMedium, @@ -197,6 +197,7 @@ export class TidalReleaseLookup extends ReleaseApiLookup { this.id = rawRelease.id; const rawTracklist = await this.getRawTracklist(this.id); const media = this.convertRawTracklist(rawTracklist); + const artwork = selectLargestImage(rawRelease.imageCover, ['front']); return { title: rawRelease.title, artists: rawRelease.artists.map(this.convertRawArtist.bind(this)), @@ -210,7 +211,7 @@ export class TidalReleaseLookup extends ReleaseApiLookup { copyright: rawRelease.copyright, status: 'Official', packaging: 'None', - images: this.getLargestCoverImage(rawRelease.imageCover), + images: artwork ? [artwork] : [], labels: this.getLabels(rawRelease), info: this.generateReleaseInfo(), }; @@ -269,29 +270,6 @@ export class TidalReleaseLookup extends ReleaseApiLookup { }; } - private getLargestCoverImage(images: Image[]): Artwork[] { - let largestImage: Image | undefined; - let thumbnail: Image | undefined; - images.forEach((i) => { - if (!largestImage || i.width > largestImage.width) { - largestImage = i; - } - if (i.width >= 200 && (!thumbnail || i.width < thumbnail.width)) { - thumbnail = i; - } - }); - if (!largestImage) return []; - let thumbUrl: URL | undefined; - if (thumbnail) { - thumbUrl = new URL(thumbnail.url); - } - return [{ - url: new URL(largestImage.url), - thumbUrl: thumbUrl, - types: ['front'], - }]; - } - private getLabels(rawRelease: Album): Label[] { // It is unsure whether providerInfo is actually used for some releases, // but it is documented in the API schemas. diff --git a/utils/image.ts b/utils/image.ts new file mode 100644 index 00000000..c0085579 --- /dev/null +++ b/utils/image.ts @@ -0,0 +1,34 @@ +import type { Artwork, ArtworkType } from '@/harmonizer/types.ts'; + +interface ImageObject { + url: string; + width: number; + height: number; +} + +const minThumbnailSize = 200; + +/** + * From a list of images select the largest one (based on image width) and return an `Artwork` object. + * + * The smallest image from the list that is still larger than `minThumbnailSize` will + * be used as the thumbnail image. + */ +export function selectLargestImage(images: ImageObject[], types: ArtworkType[]): Artwork | undefined { + let largestImage: ImageObject | undefined; + let thumbnail: ImageObject | undefined; + images.forEach((image) => { + if (!largestImage || image.width > largestImage.width) { + largestImage = image; + } + if (image.width >= minThumbnailSize && (!thumbnail || image.width < thumbnail.width)) { + thumbnail = image; + } + }); + if (!largestImage) return; + return { + url: new URL(largestImage.url), + thumbUrl: thumbnail ? new URL(thumbnail.url) : undefined, + types, + }; +} From 4f2c7c20cd9ae7b065c21213b50cf9ed53b3abfe Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sat, 8 Jun 2024 16:50:16 +0200 Subject: [PATCH 10/22] refactor(provider): moved caching of API access token to MetadataApiProvider --- providers/Spotify/mod.ts | 19 +++++-------------- providers/Tidal/mod.ts | 25 +++++++++++-------------- providers/base.ts | 24 ++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 28 deletions(-) diff --git a/providers/Spotify/mod.ts b/providers/Spotify/mod.ts index f1050462..75160a34 100644 --- a/providers/Spotify/mod.ts +++ b/providers/Spotify/mod.ts @@ -1,4 +1,4 @@ -import { type CacheEntry, MetadataApiProvider, ReleaseApiLookup } from '@/providers/base.ts'; +import { ApiAccessToken, type CacheEntry, MetadataApiProvider, ReleaseApiLookup } from '@/providers/base.ts'; import { DurationPrecision, FeatureQuality, FeatureQualityMap } from '@/providers/features.ts'; import { parseHyphenatedDate, PartialDate } from '@/utils/date.ts'; import { ResponseError } from '@/utils/errors.ts'; @@ -68,11 +68,12 @@ export default class SpotifyProvider extends MetadataApiProvider { } async query(apiUrl: URL, maxTimestamp?: number): Promise> { + const accessToken = await this.cachedAccessToken(this.requestAccessToken); const cacheEntry = await this.fetchJSON(apiUrl, { policy: { maxTimestamp }, requestInit: { headers: { - 'Authorization': `Bearer ${await this.accessToken()}`, + 'Authorization': `Bearer ${accessToken}`, }, }, }); @@ -84,17 +85,7 @@ export default class SpotifyProvider extends MetadataApiProvider { return cacheEntry; } - private async accessToken(): Promise { - const cacheKey = `${this.name}:accessKey`; - let tokenResult = JSON.parse(localStorage.getItem(cacheKey) || '{}'); - if (!tokenResult?.accessToken || Date.now() > tokenResult?.validUntil) { - tokenResult = await this.requestAccessToken(); - localStorage.setItem(cacheKey, JSON.stringify(tokenResult)); - } - return tokenResult.accessToken; - } - - private async requestAccessToken(): Promise<{ accessToken: string; validUntil: number }> { + private async requestAccessToken(): Promise { // See https://developer.spotify.com/documentation/web-api/tutorials/client-credentials-flow const url = new URL('https://accounts.spotify.com/api/token'); const auth = encodeBase64(`${spotifyClientId}:${spotifyClientSecret}`); @@ -113,7 +104,7 @@ export default class SpotifyProvider extends MetadataApiProvider { const content = await response.json(); return { accessToken: content?.access_token, - validUntil: Date.now() + (content.expires_in * 1000), + validUntilTimestamp: Date.now() + (content.expires_in * 1000), }; } } diff --git a/providers/Tidal/mod.ts b/providers/Tidal/mod.ts index 69d1cecc..f4320715 100644 --- a/providers/Tidal/mod.ts +++ b/providers/Tidal/mod.ts @@ -1,5 +1,11 @@ import { availableRegions } from './regions.ts'; -import { type CacheEntry, MetadataApiProvider, type ProviderOptions, ReleaseApiLookup } from '@/providers/base.ts'; +import { + ApiAccessToken, + type CacheEntry, + MetadataApiProvider, + type ProviderOptions, + ReleaseApiLookup, +} from '@/providers/base.ts'; import { DurationPrecision, FeatureQuality, FeatureQualityMap } from '@/providers/features.ts'; import { parseHyphenatedDate, PartialDate } from '@/utils/date.ts'; import { ResponseError } from '@/utils/errors.ts'; @@ -70,11 +76,12 @@ export default class TidalProvider extends MetadataApiProvider { } async query(apiUrl: URL, maxTimestamp?: number): Promise> { + const accessToken = await this.cachedAccessToken(this.requestAccessToken); const cacheEntry = await this.fetchJSON(apiUrl, { policy: { maxTimestamp }, requestInit: { headers: { - 'Authorization': `Bearer ${await this.accessToken()}`, + 'Authorization': `Bearer ${accessToken}`, 'Content-Type': 'application/vnd.tidal.v1+json', }, }, @@ -87,17 +94,7 @@ export default class TidalProvider extends MetadataApiProvider { return cacheEntry; } - private async accessToken(): Promise { - const cacheKey = `${this.name}:accessKey`; - let tokenResult = JSON.parse(localStorage.getItem(cacheKey) || '{}'); - if (!tokenResult?.accessToken || Date.now() > tokenResult?.validUntil) { - tokenResult = await this.requestAccessToken(); - localStorage.setItem(cacheKey, JSON.stringify(tokenResult)); - } - return tokenResult.accessToken; - } - - private async requestAccessToken(): Promise<{ accessToken: string; validUntil: number }> { + private async requestAccessToken(): Promise { // See https://developer.tidal.com/documentation/api-sdk/api-sdk-quick-start const url = new URL('https://auth.tidal.com/v1/oauth2/token'); const auth = encodeBase64(`${tidalClientId}:${tidalClientSecret}`); @@ -117,7 +114,7 @@ export default class TidalProvider extends MetadataApiProvider { const content = await response.json(); return { accessToken: content?.access_token, - validUntil: Date.now() + (content.expires_in * 1000), + validUntilTimestamp: Date.now() + (content.expires_in * 1000), }; } } diff --git a/providers/base.ts b/providers/base.ts index 492774ff..cc1f8924 100644 --- a/providers/base.ts +++ b/providers/base.ts @@ -351,10 +351,34 @@ export abstract class ReleaseLookup(apiUrl: URL, maxTimestamp?: number): Promise>; + + /** + * Returns a cached API access token. + * + * Must be passed a function `requestAccessToken` which will return a promise resolving to a + * `ApiAccessToken`, containing the access token and an expiration Unix timestamp. The result is + * cached and `requestAccessToken` only gets called if the cached token has expired. + */ + protected async cachedAccessToken(requestAccessToken: () => Promise): Promise { + const cacheKey = `${this.name}:accessToken`; + let tokenResult = JSON.parse(localStorage.getItem(cacheKey) || '{}'); + if ( + !tokenResult?.accessToken || !tokenResult?.validUntilTimestamp || Date.now() > tokenResult.validUntilTimestamp + ) { + tokenResult = await requestAccessToken(); + localStorage.setItem(cacheKey, JSON.stringify(tokenResult)); + } + return tokenResult.accessToken; + } } /** Extends `ReleaseLookup` with functions common to lookups accessing web APIs. */ From eed1b38d4dc81dd95d8f43540bc1b3df7406f517 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sat, 8 Jun 2024 17:23:56 +0200 Subject: [PATCH 11/22] fix(Spotify): SimplifiedTrack.is_playable is optional and often not present --- providers/Spotify/api_types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/Spotify/api_types.ts b/providers/Spotify/api_types.ts index 78cbf0f4..0cfe97e3 100644 --- a/providers/Spotify/api_types.ts +++ b/providers/Spotify/api_types.ts @@ -52,7 +52,7 @@ export type SimplifiedTrack = LinkedTrack & { disc_number: number; duration_ms: number; explicit: boolean; - is_playable: boolean; + is_playable: boolean | undefined; is_local: boolean; preview_url: string; linked_from: LinkedTrack | undefined; From 0e64616e631c23234f0397be5d086fd711f62406 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sun, 9 Jun 2024 17:02:02 +0200 Subject: [PATCH 12/22] refactor(Spotify): only pad GTIN to 14 characters once --- providers/Spotify/mod.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/providers/Spotify/mod.ts b/providers/Spotify/mod.ts index 75160a34..f9966493 100644 --- a/providers/Spotify/mod.ts +++ b/providers/Spotify/mod.ts @@ -128,19 +128,24 @@ export class SpotifyReleaseLookup extends ReleaseApiLookup { if (this.lookup.method === 'gtin') { // Spotify does not always find UPC barcodes but expects them prefixed with - // 0 to a max. size of 14 characters. E.g. "810121774182" gives no results, + // 0 to a length of 14 characters. E.g. "810121774182" gives no results, // but "00810121774182" does. let albumId: string | undefined; - while (this.lookup.value.length <= 14 && !albumId) { + while (true) { const cacheEntry = await this.provider.query( this.constructReleaseApiUrl(), this.options.snapshotMaxTimestamp, ); if (cacheEntry.content?.albums?.items?.length) { albumId = cacheEntry.content.albums.items[0].id; + break; + } else if (this.lookup.value.length < 14) { + // Prefix the GTIN with 0s + this.lookup.value = this.lookup.value.padStart(14, '0'); + } else { + // No results found + break; } - // Prefix the GTIN with an additional 0 - this.lookup.value = `0${this.lookup.value}`; } // No results found From 60ecdf9a7de676114b581453bedf1beac7425f6f Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sun, 9 Jun 2024 17:05:22 +0200 Subject: [PATCH 13/22] refactor(Spotify): use Array.push instead of concat. --- providers/Spotify/mod.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/providers/Spotify/mod.ts b/providers/Spotify/mod.ts index f9966493..aaeeccad 100644 --- a/providers/Spotify/mod.ts +++ b/providers/Spotify/mod.ts @@ -170,8 +170,7 @@ export class SpotifyReleaseLookup extends ReleaseApiLookup { - let allTracks: SimplifiedTrack[] = []; - allTracks = allTracks.concat(rawRelease.tracks.items); + const allTracks: SimplifiedTrack[] = [...rawRelease.tracks.items]; // The initial response contains max. 50 tracks. Fetch the remaining // tracks with separate requests if needed. @@ -182,7 +181,7 @@ export class SpotifyReleaseLookup extends ReleaseApiLookup { - let allTracks: Track[] = []; + const allTracks: Track[] = []; const trackIds = simplifiedTracks.map((track) => track.id); // The SimplifiedTrack entries do not contain ISRCs. @@ -207,7 +206,7 @@ export class SpotifyReleaseLookup extends ReleaseApiLookup Date: Sun, 9 Jun 2024 17:27:16 +0200 Subject: [PATCH 14/22] feat(Spotify): only perform loading of full track info if ISRCs are requested --- providers/Spotify/mod.ts | 14 +++++++++----- server/routes/release.tsx | 1 + server/routes/release/actions.tsx | 1 + 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/providers/Spotify/mod.ts b/providers/Spotify/mod.ts index aaeeccad..6d870221 100644 --- a/providers/Spotify/mod.ts +++ b/providers/Spotify/mod.ts @@ -169,7 +169,7 @@ export class SpotifyReleaseLookup extends ReleaseApiLookup { + private async getRawTracklist(rawRelease: Album): Promise { const allTracks: SimplifiedTrack[] = [...rawRelease.tracks.items]; // The initial response contains max. 50 tracks. Fetch the remaining @@ -186,7 +186,11 @@ export class SpotifyReleaseLookup extends ReleaseApiLookup { @@ -237,7 +241,7 @@ export class SpotifyReleaseLookup extends ReleaseApiLookup { const options: ReleaseOptions = { withSeparateMedia: true, withAllTrackArtists: true, + withISRC: true, regions, providers, snapshotMaxTimestamp, diff --git a/server/routes/release/actions.tsx b/server/routes/release/actions.tsx index 50bc6e1a..ab1fa954 100644 --- a/server/routes/release/actions.tsx +++ b/server/routes/release/actions.tsx @@ -35,6 +35,7 @@ export default defineRoute(async (req, ctx) => { const { gtin, urls, regions, providerIds, providers, snapshotMaxTimestamp } = extractReleaseLookupState(ctx.url); const options: ReleaseOptions = { withSeparateMedia: true, + withISRC: true, regions, providers, snapshotMaxTimestamp, From 7da2a9a29eb732015949289a32fbe2a5ba61b970 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Sun, 9 Jun 2024 18:02:19 +0200 Subject: [PATCH 15/22] refactor(provider): moved label splitting code into utility function splitLabels --- providers/Deezer/mod.ts | 6 ++---- providers/Spotify/mod.ts | 19 +++---------------- utils/label.test.ts | 27 +++++++++++++++++++++++++++ utils/label.ts | 13 +++++++++++++ 4 files changed, 45 insertions(+), 20 deletions(-) create mode 100644 utils/label.test.ts create mode 100644 utils/label.ts diff --git a/providers/Deezer/mod.ts b/providers/Deezer/mod.ts index a60ea5b9..83366a0e 100644 --- a/providers/Deezer/mod.ts +++ b/providers/Deezer/mod.ts @@ -2,6 +2,7 @@ import { availableRegions } from './regions.ts'; import { type CacheEntry, MetadataApiProvider, type ProviderOptions, ReleaseApiLookup } from '@/providers/base.ts'; import { DurationPrecision, FeatureQuality, FeatureQualityMap } from '@/providers/features.ts'; import { parseHyphenatedDate, PartialDate } from '@/utils/date.ts'; +import { splitLabels } from '@/utils/label.ts'; import { ResponseError } from '@/utils/errors.ts'; import { formatGtin } from '@/utils/gtin.ts'; @@ -174,10 +175,7 @@ export class DeezerReleaseLookup extends ReleaseApiLookup ({ - name: label.trim(), - })), + labels: splitLabels(rawRelease.label), status: 'Official', packaging: 'None', images: [{ diff --git a/providers/Spotify/mod.ts b/providers/Spotify/mod.ts index 6d870221..8aec3668 100644 --- a/providers/Spotify/mod.ts +++ b/providers/Spotify/mod.ts @@ -1,6 +1,7 @@ import { ApiAccessToken, type CacheEntry, MetadataApiProvider, ReleaseApiLookup } from '@/providers/base.ts'; import { DurationPrecision, FeatureQuality, FeatureQualityMap } from '@/providers/features.ts'; import { parseHyphenatedDate, PartialDate } from '@/utils/date.ts'; +import { splitLabels } from '@/utils/label.ts'; import { ResponseError } from '@/utils/errors.ts'; import { selectLargestImage } from '@/utils/image.ts'; import { encodeBase64 } from 'std/encoding/base64.ts'; @@ -17,14 +18,7 @@ import type { Track, TrackList, } from './api_types.ts'; -import type { - ArtistCreditName, - EntityId, - HarmonyMedium, - HarmonyRelease, - HarmonyTrack, - Label, -} from '@/harmonizer/types.ts'; +import type { ArtistCreditName, EntityId, HarmonyMedium, HarmonyRelease, HarmonyTrack } from '@/harmonizer/types.ts'; // See https://developer.spotify.com/documentation/web-api @@ -235,7 +229,7 @@ export class SpotifyReleaseLookup extends ReleaseApiLookup ({ - name: label.trim(), - })); - } - private getCopyright(copyrights: Copyright[]): string { return copyrights.map(this.formatCopyright).join('\n'); } diff --git a/utils/label.test.ts b/utils/label.test.ts new file mode 100644 index 00000000..19ae77a5 --- /dev/null +++ b/utils/label.test.ts @@ -0,0 +1,27 @@ +import { splitLabels } from './label.ts'; + +import { assertEquals } from 'std/assert/assert_equals.ts'; +import { describe, it } from 'std/testing/bdd.ts'; + +import type { FunctionSpec } from './test_spec.ts'; + +describe('GTIN validator', () => { + const passingCases: FunctionSpec = [ + ['preserves single label', 'Nuclear Blast', [{ name: 'Nuclear Blast' }]], + ['split multiple labels', 'Roc Nation/RocAFella/IDJ', [{ name: 'Roc Nation' }, { name: 'RocAFella' }, { + name: 'IDJ', + }]], + ['split multiple labels, trim values', 'Roc Nation / RocAFella / IDJ', [{ name: 'Roc Nation' }, { + name: 'RocAFella', + }, { + name: 'IDJ', + }]], + ['preserve label whose suffix contains a slash', 'EMI Belgium SA/NV', [{ name: 'EMI Belgium SA/NV' }]], + ]; + + passingCases.forEach(([description, input, expected]) => { + it(description, () => { + assertEquals(splitLabels(input), expected); + }); + }); +}); diff --git a/utils/label.ts b/utils/label.ts new file mode 100644 index 00000000..5753899a --- /dev/null +++ b/utils/label.ts @@ -0,0 +1,13 @@ +import { Label } from '@/harmonizer/types.ts'; + +/** + * Split label string using slashes if the results have at least 3 characters. + * + * @param labels String containing one or more label name. + * @returns List of `Label` entries + */ +export function splitLabels(labels: string): Label[] { + return labels?.split(/(?<=[^/]{3,})\/(?=[^/]{3,})/).map((label) => ({ + name: label.trim(), + })); +} From 0045aa064f1f81b88276fa95cf80a668b7b32063 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Mon, 10 Jun 2024 08:06:09 +0200 Subject: [PATCH 16/22] feat(Spotify): throw specific SpotifyResponseError This allows the actual error message to be displayed --- providers/Spotify/mod.ts | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/providers/Spotify/mod.ts b/providers/Spotify/mod.ts index 8aec3668..fb907c59 100644 --- a/providers/Spotify/mod.ts +++ b/providers/Spotify/mod.ts @@ -62,21 +62,31 @@ export default class SpotifyProvider extends MetadataApiProvider { } async query(apiUrl: URL, maxTimestamp?: number): Promise> { - const accessToken = await this.cachedAccessToken(this.requestAccessToken); - const cacheEntry = await this.fetchJSON(apiUrl, { - policy: { maxTimestamp }, - requestInit: { - headers: { - 'Authorization': `Bearer ${accessToken}`, + try { + const accessToken = await this.cachedAccessToken(this.requestAccessToken); + const cacheEntry = await this.fetchJSON(apiUrl, { + policy: { maxTimestamp }, + requestInit: { + headers: { + 'Authorization': `Bearer ${accessToken}`, + }, }, - }, - }); - const { error } = cacheEntry.content as { error?: ApiError }; - - if (error) { - throw new SpotifyResponseError(error, apiUrl); + }); + const apiError = cacheEntry.content as ApiError; + if (apiError.error) { + throw new SpotifyResponseError(apiError, apiUrl); + } + return cacheEntry; + } catch (error) { + // Clone the response so the body of the original response can be + // consumed later if the error gets re-thrown. + const apiError = await error.response?.clone().json() as ApiError; + if (apiError?.error) { + throw new SpotifyResponseError(apiError, apiUrl); + } else { + throw error; + } } - return cacheEntry; } private async requestAccessToken(): Promise { From 3532ac4ff97e7bbe9a303de6b855441dd2626749 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Mon, 10 Jun 2024 14:32:48 +0200 Subject: [PATCH 17/22] fix(Spotify): set region on GTIN lookups, do ID lookups without region This ensures GTIN lookups give results for barcodes not available in the default region. ID lookups must be performed without region, only then the API will return available_markets. --- providers/Spotify/mod.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/providers/Spotify/mod.ts b/providers/Spotify/mod.ts index fb907c59..4281a508 100644 --- a/providers/Spotify/mod.ts +++ b/providers/Spotify/mod.ts @@ -117,6 +117,9 @@ export class SpotifyReleaseLookup extends ReleaseApiLookup { if (this.lookup.method === 'gtin') { + // For GTIN lookups use the region + this.lookup.region = this.options?.regions?.values().next().value; // Spotify does not always find UPC barcodes but expects them prefixed with // 0 to a length of 14 characters. E.g. "810121774182" gives no results, // but "00810121774182" does. @@ -161,6 +166,7 @@ export class SpotifyReleaseLookup extends ReleaseApiLookup( From 75fe532daddd364af35cd3a403839b66909e2f5e Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Mon, 10 Jun 2024 15:53:09 +0200 Subject: [PATCH 18/22] refactor(Spotify): simplify code for retrying different length GTIN Also try with GTIN padded to 13 characters again, but only as a last resort. --- providers/Spotify/mod.ts | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/providers/Spotify/mod.ts b/providers/Spotify/mod.ts index 4281a508..7358d513 100644 --- a/providers/Spotify/mod.ts +++ b/providers/Spotify/mod.ts @@ -140,7 +140,8 @@ export class SpotifyReleaseLookup extends ReleaseApiLookup( this.constructReleaseApiUrl(), this.options.snapshotMaxTimestamp, @@ -148,12 +149,6 @@ export class SpotifyReleaseLookup extends ReleaseApiLookup { + if (gtin.length < length) { + candidates.push(gtin.padStart(length, '0')); + } + }); + return candidates; + } + private getCopyright(copyrights: Copyright[]): string { return copyrights.map(this.formatCopyright).join('\n'); } From 3b2e188ca20dd7606cda08bf843bd3a638a5a1bb Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Mon, 10 Jun 2024 16:19:13 +0200 Subject: [PATCH 19/22] fix(Spotify): for GTIN search try all regions --- providers/Spotify/mod.ts | 41 +++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/providers/Spotify/mod.ts b/providers/Spotify/mod.ts index 7358d513..530c0d39 100644 --- a/providers/Spotify/mod.ts +++ b/providers/Spotify/mod.ts @@ -134,23 +134,7 @@ export class SpotifyReleaseLookup extends ReleaseApiLookup { if (this.lookup.method === 'gtin') { - // For GTIN lookups use the region - this.lookup.region = this.options?.regions?.values().next().value; - // Spotify does not always find UPC barcodes but expects them prefixed with - // 0 to a length of 14 characters. E.g. "810121774182" gives no results, - // but "00810121774182" does. - let albumId: string | undefined; - for (const gtin of this.getGtinCandidates(this.lookup.value)) { - this.lookup.value = gtin; - const cacheEntry = await this.provider.query( - this.constructReleaseApiUrl(), - this.options.snapshotMaxTimestamp, - ); - if (cacheEntry.content?.albums?.items?.length) { - albumId = cacheEntry.content.albums.items[0].id; - break; - } - } + const albumId = await this.queryAlbumIdByGtin(this.lookup.value); // No results found if (!albumId) { @@ -174,6 +158,29 @@ export class SpotifyReleaseLookup extends ReleaseApiLookup { + // Spotify does not always find UPC barcodes but expects them prefixed with + // 0 to a length of 14 characters. E.g. "810121774182" gives no results, + // but "00810121774182" does. + const gtins = this.getGtinCandidates(gtin); + // For GTIN lookups use the region + for (const region of this.options?.regions || [undefined]) { + this.lookup.region = region; + for (const gtin of gtins) { + this.lookup.value = gtin; + const cacheEntry = await this.provider.query( + this.constructReleaseApiUrl(), + this.options.snapshotMaxTimestamp, + ); + if (cacheEntry.content?.albums?.items?.length) { + return cacheEntry.content.albums.items[0].id; + } + } + } + + return undefined; + } + private async getRawTracklist(rawRelease: Album): Promise { const allTracks: SimplifiedTrack[] = [...rawRelease.tracks.items]; From 94a0dd1e37d950625bfd5c677ca5dfe529c21410 Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Mon, 10 Jun 2024 16:39:21 +0200 Subject: [PATCH 20/22] fix(Spotify): missing parameter documentation --- providers/Spotify/mod.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/providers/Spotify/mod.ts b/providers/Spotify/mod.ts index 530c0d39..7a7dbf90 100644 --- a/providers/Spotify/mod.ts +++ b/providers/Spotify/mod.ts @@ -310,8 +310,8 @@ export class SpotifyReleaseLookup extends ReleaseApiLookup Date: Mon, 10 Jun 2024 18:26:14 +0200 Subject: [PATCH 21/22] refactor(Spotify): set region only on gtin lookup URLs --- providers/Spotify/mod.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/providers/Spotify/mod.ts b/providers/Spotify/mod.ts index 7a7dbf90..c541cfc1 100644 --- a/providers/Spotify/mod.ts +++ b/providers/Spotify/mod.ts @@ -115,17 +115,19 @@ export default class SpotifyProvider extends MetadataApiProvider { export class SpotifyReleaseLookup extends ReleaseApiLookup { constructReleaseApiUrl(): URL { + const { method, value, region } = this.lookup; let lookupUrl: URL; const query = new URLSearchParams(); - if (this.lookup.region) { - query.set('market', this.lookup.region); - } - if (this.lookup.method === 'gtin') { + + if (method === 'gtin') { lookupUrl = new URL(`search`, this.provider.apiBaseUrl); query.set('type', 'album'); - query.set('q', `upc:${this.lookup.value}`); + query.set('q', `upc:${value}`); + if (region) { + query.set('market', region); + } } else { // if (method === 'id') - lookupUrl = new URL(`albums/${this.lookup.value}`, this.provider.apiBaseUrl); + lookupUrl = new URL(`albums/${value}`, this.provider.apiBaseUrl); } lookupUrl.search = query.toString(); @@ -145,7 +147,6 @@ export class SpotifyReleaseLookup extends ReleaseApiLookup( From 4f583c9417ffab98b3b09ec66d309581f412426f Mon Sep 17 00:00:00 2001 From: Philipp Wolfer Date: Mon, 10 Jun 2024 18:39:47 +0200 Subject: [PATCH 22/22] fix(Spotify): intl-* paths seem to indicate language, not region --- providers/Spotify/mod.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/providers/Spotify/mod.ts b/providers/Spotify/mod.ts index c541cfc1..ba6578e4 100644 --- a/providers/Spotify/mod.ts +++ b/providers/Spotify/mod.ts @@ -30,7 +30,7 @@ export default class SpotifyProvider extends MetadataApiProvider { readonly supportedUrls = new URLPattern({ hostname: 'open.spotify.com', - pathname: '{/intl-:region}?/:type(artist|album)/:id', + pathname: '{/intl-:language}?/:type(artist|album)/:id', }); readonly features: FeatureQualityMap = {