From 5011148bb53d705bc9b5f3f49203d67af4512b71 Mon Sep 17 00:00:00 2001 From: Adam James Date: Tue, 7 May 2024 18:27:29 +0100 Subject: [PATCH 1/5] feat(provider): Add Beatport metadata provider --- .vscode/settings.json | 1 + providers/Beatport/json_types.ts | 159 +++++++++++++++++++++++++++ providers/Beatport/mod.ts | 178 +++++++++++++++++++++++++++++++ providers/mod.ts | 4 +- 4 files changed, 341 insertions(+), 1 deletion(-) create mode 100644 providers/Beatport/json_types.ts create mode 100644 providers/Beatport/mod.ts diff --git a/.vscode/settings.json b/.vscode/settings.json index ea069ac5..29d10160 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,6 +2,7 @@ "cSpell.words": [ "Audiobook", "Bandcamp", + "Beatport", "brainz", "Brainz", "deezer", diff --git a/providers/Beatport/json_types.ts b/providers/Beatport/json_types.ts new file mode 100644 index 00000000..c2188a58 --- /dev/null +++ b/providers/Beatport/json_types.ts @@ -0,0 +1,159 @@ +export interface BeatportNextData { + props: { + pageProps: { + dehydratedState: { + queries: Array | ArrayQueryResult>; + }; + release: Release; + }; + }; +} + +export interface QueryResult { + queryHash: string; + queryKey: Array>; +} + +export interface ScalarQueryResult extends QueryResult { + state: { + data: T; + status: string; + }; +} + +export interface ArrayQueryResult extends QueryResult { + state: { + data: { + next: string | null; + previous: string | null; + count: number; + page: string; + per_page: number; + results: Array; + }; + status: string; + }; +} + +export interface Entity { + id: number; + name: string; +} + +export interface EntityWithUrl extends Entity { + url: string; +} + +export interface Artist extends EntityWithUrl { + image: Image; + slug: string; +} + +export interface Image { + id: number; + uri: string; + dynamic_uri: string; +} + +export interface Label extends Entity { + image: Image; + slug: string; +} + +export interface BpmRange { + min: number; + max: number; +} + +export interface Price { + code: string; + symbol: string; + value: number; + display: string; +} + +export interface MinimalRelease extends Entity { + image: Image; + label: Label; + slug: string; +} + +export interface Release extends MinimalRelease { + artists: Artist[]; + bpm_range: BpmRange; + catalog_number: string; + desc: string | null; + enabled: boolean; + encoded_date: string; + exclusive: boolean; + grid: null; + is_available_for_streaming: boolean; + is_hype: boolean; + new_release_date: string; + override_price: boolean; + pre_order: boolean; + pre_order_date: string | null; + price: Price; + price_override_firm: boolean; + publish_date: string; + remixers: Artist[]; + tracks: string[]; + track_count: number; + type: Entity; + upc: string; + updated: string; +} + +export interface Genre extends EntityWithUrl { + slug: string; +} + +export interface Key extends EntityWithUrl { + camelot_number: number; + camelot_letter: string; + chord_type: EntityWithUrl; + is_sharp: boolean; + is_flat: boolean; + letter: string; +} + +export interface Track extends EntityWithUrl { + artists: Artist[]; + publish_status: string; + available_worldwide: boolean; + bpm: number; + catalog_number: string; + current_status: EntityWithUrl; + encoded_date: string; + exclusive: boolean; + // @todo find tracks where this is not empty + free_downloads: []; + free_download_start_date: null; + free_download_end_date: null; + genre: Genre; + is_available_for_streaming: boolean; + is_hype: boolean; + isrc: string; + key: Key; + label_track_identifier: string; + length: string; + length_ms: number; + mix_name: string; + new_release_date: string; + pre_order: boolean; + price: Price; + publish_date: string; + release: MinimalRelease; + remixers: Artist[]; + sale_type: EntityWithUrl; + sample_url: string; + sample_start_ms: number; + sample_end_ms: number; + slug: string; + // @todo find examples where this is set + sub_genre: null; +} + +export interface BeatportRelease extends Release { + track_objects: Track[]; +} diff --git a/providers/Beatport/mod.ts b/providers/Beatport/mod.ts new file mode 100644 index 00000000..ce39d985 --- /dev/null +++ b/providers/Beatport/mod.ts @@ -0,0 +1,178 @@ +import type { Artist, BeatportNextData, BeatportRelease, Release, Track } from './json_types.ts'; +import type { ArtistCreditName, EntityId, HarmonyRelease, HarmonyTrack, LinkType } from '@/harmonizer/types.ts'; +import { CacheEntry, DurationPrecision, MetadataProvider, ReleaseLookup } from '@/providers/base.ts'; +import { parseHyphenatedDate, PartialDate } from '@/utils/date.ts'; +import { ProviderError, ResponseError } from '@/utils/errors.ts'; +import { extractTextFromHtml } from '@/utils/html.ts'; + +export default class BeatportProvider extends MetadataProvider { + readonly name = 'Beatport'; + + readonly supportedUrls = new URLPattern({ + hostname: 'www.beatport.com', + pathname: '/:type(artist|label|release)/:slug/:id', + }); + + readonly entityTypeMap = { + artist: 'artist', + label: 'label', + release: 'release', + }; + + readonly releaseLookup = BeatportReleaseLookup; + + readonly durationPrecision = DurationPrecision.MS; + + readonly launchDate: PartialDate = { + year: 2005, + month: 1, + day: 7, + }; + + readonly artworkQuality = 1400; + + constructUrl(entity: EntityId): URL { + return new URL([entity.type, entity.slug ?? '-', entity.id].join('/'), 'https://www.beatport.com'); + } + + extractEmbeddedJson(webUrl: URL, maxTimestamp?: number): Promise> { + return this.fetchJSON(webUrl, { + policy: { maxTimestamp }, + responseMutator: async (response) => { + const html = await response.text(); + const nextData = extractTextFromHtml( + html, + /]+?id=["']__NEXT_DATA__["'])(?=[^>]+?type=["']application\/json["'])[^>]+?>(.+?)<\/script>/i, + ); + if (nextData) { + return new Response(nextData, response); + } + + throw new ResponseError(this.name, 'Failed to extract embedded JSON', webUrl); + }, + }); + } +} + +export class BeatportReleaseLookup extends ReleaseLookup { + constructReleaseApiUrl(): URL | undefined { + return undefined; + } + + async getRawRelease(): Promise { + if (this.lookup.method === 'gtin') { + throw new ProviderError(this.provider.name, 'GTIN lookups are not supported (yet)'); + } + + const webUrl = this.provider.constructUrl({ id: this.lookup.value, type: 'release' }); + const { content: data, timestamp } = await this.provider.extractEmbeddedJson( + webUrl, + this.options.snapshotMaxTimestamp, + ); + this.cacheTime = timestamp; + + return this.extractRawRelease(data); + } + + convertRawRelease(rawRelease: BeatportRelease): HarmonyRelease { + this.id = rawRelease.id.toString(); + const releaseUrl = this.provider.constructUrl({ + id: this.id, + type: 'release', + slug: rawRelease.slug, + }); + + const linkTypes: LinkType[] = ['paid download']; + // @todo paid streaming is not currently permitted for Beatport links + // see https://tickets.metabrainz.org/browse/STYLE-2141 + // if (rawRelease.is_available_for_streaming) { + // linkTypes.push('paid streaming'); + //} + + return { + title: rawRelease.name, + artists: rawRelease.artists.map(this.makeArtistCreditName.bind(this)), + labels: [{ + name: rawRelease.label.name, + catalogNumber: rawRelease.catalog_number, + externalIds: this.provider.makeExternalIds({ + type: 'label', + id: rawRelease.label.id.toString(), + slug: rawRelease.label.slug, + }), + }], + gtin: rawRelease.upc, + releaseDate: parseHyphenatedDate(rawRelease.new_release_date), + media: [{ + format: 'Digital Media', + tracklist: rawRelease.track_objects.map(this.convertRawTrack.bind(this)), + }], + externalLinks: [{ + url: releaseUrl, + types: linkTypes, + }], + status: 'Official', + packaging: 'None', + images: [{ + url: new URL(rawRelease.image.uri), + thumbUrl: new URL(rawRelease.image.dynamic_uri.replace('{w}x{h}', '250x250')), + types: ['front'], + }], + info: this.generateReleaseInfo(), + }; + } + + convertRawTrack(rawTrack: Track, index: number): HarmonyTrack { + let title = rawTrack.name; + if (rawTrack.mix_name !== 'Original Mix') { + title += ` (${rawTrack.mix_name})`; + } + + return { + number: index + 1, + title: title, + artists: rawTrack.artists.map(this.makeArtistCreditName.bind(this)), + duration: rawTrack.length_ms, + isrc: rawTrack.isrc, + }; + } + + makeArtistCreditName(artist: Artist): ArtistCreditName { + return { + name: artist.name, + creditedName: artist.name, + externalIds: this.provider.makeExternalIds({ + type: 'artist', + id: artist.id.toString(), + slug: artist.slug, + }), + }; + } + + extractRawRelease(nextData: BeatportNextData): BeatportRelease { + const release = nextData.props.pageProps.release; + if (!release) { + throw new ProviderError(this.provider.name, 'Failed to extract release from embedded JSON'); + } else if (release.track_count > 100) { + throw new ProviderError(this.provider.name, 'Releases with more than 100 tracks are not currently supported'); + } + + const tracksResult = nextData.props.pageProps.dehydratedState.queries[1]; + if (!tracksResult || !('results' in tracksResult.state.data)) { + throw new ProviderError(this.provider.name, 'Failed to extract tracks from embedded JSON'); + } + const tracks = tracksResult.state.data.results; + + // tracks are listed in reverse order + const releaseTracks = release.tracks.slice().reverse().map((url) => { + const track = tracks.find((t) => url === t.url); + if (!track) { + throw new ProviderError(this.provider.name, `Track ${url} not found in embedded JSON`); + } + + return track; + }); + + return { ...release, track_objects: releaseTracks }; + } +} diff --git a/providers/mod.ts b/providers/mod.ts index 4d7d4de5..037de37e 100644 --- a/providers/mod.ts +++ b/providers/mod.ts @@ -1,4 +1,5 @@ import BandcampProvider from './Bandcamp/mod.ts'; +import BeatportProvider from './Beatport/mod.ts'; import DeezerProvider from './Deezer/mod.ts'; import iTunesProvider from './iTunes/mod.ts'; @@ -15,6 +16,7 @@ export const allProviders: MetadataProvider[] = [ DeezerProvider, iTunesProvider, BandcampProvider, + BeatportProvider, ].map((Provider) => new Provider({ snaps })); /** Display names of all supported providers. */ @@ -50,7 +52,7 @@ export const defaultProviderPreferences: ProviderPreferences = { // get cover art from the provider with the highest quality (currently: image resolution) images: sortProvidersByQuality('artworkQuality'), // use region-specific external URLs last (TODO: derive this from provider properties) - externalId: ['Deezer', 'Bandcamp', 'iTunes'], + externalId: ['Deezer', 'Bandcamp', 'Beatport', 'iTunes'], }; /** Returns a list of provider names sorted by the value of the given numeric property (descending). */ From 4df024e971073783b5e5b70988cdca23f96f62f6 Mon Sep 17 00:00:00 2001 From: Adam James Date: Thu, 9 May 2024 17:00:11 +0100 Subject: [PATCH 2/5] style(web): Add Beatport brand icon --- server/components/ProviderIcon.tsx | 1 + server/icons/BrandBeatport.tsx | 25 +++++++++++++++++++++++++ server/routes/icon-sprite.svg.tsx | 2 ++ 3 files changed, 28 insertions(+) create mode 100644 server/icons/BrandBeatport.tsx diff --git a/server/components/ProviderIcon.tsx b/server/components/ProviderIcon.tsx index 401a5834..098051b3 100644 --- a/server/components/ProviderIcon.tsx +++ b/server/components/ProviderIcon.tsx @@ -4,6 +4,7 @@ import { simplifyName } from 'utils/string/simplify.js'; const providerIconMap: Record = { bandcamp: 'brand-bandcamp', + beatport: 'brand-beatport', deezer: 'brand-deezer', itunes: 'brand-apple', musicbrainz: 'brand-metabrainz', diff --git a/server/icons/BrandBeatport.tsx b/server/icons/BrandBeatport.tsx new file mode 100644 index 00000000..734b2fe4 --- /dev/null +++ b/server/icons/BrandBeatport.tsx @@ -0,0 +1,25 @@ +export default function IconBrandBeatport({ + size = 24, + color = 'currentColor', + stroke = 2, + ...props +}) { + return ( + + + + + ); +} diff --git a/server/routes/icon-sprite.svg.tsx b/server/routes/icon-sprite.svg.tsx index 0f1c0876..0f44beec 100644 --- a/server/routes/icon-sprite.svg.tsx +++ b/server/routes/icon-sprite.svg.tsx @@ -1,3 +1,4 @@ +import IconBrandBeatport from '@/server/icons/BrandBeatport.tsx'; import IconBrandMetaBrainz from '@/server/icons/BrandMetaBrainz.tsx'; import IconBrandApple from 'tabler-icons/brand-apple.tsx'; import IconBrandBandcamp from 'tabler-icons/brand-bandcamp.tsx'; @@ -32,6 +33,7 @@ const icons: Icon[] = [ // Brand icons IconBrandApple, IconBrandBandcamp, + IconBrandBeatport, IconBrandDeezer, IconBrandGit, IconBrandMetaBrainz, From fff3756b2040c1c9152c24cc4a84b79b9dfea13f Mon Sep 17 00:00:00 2001 From: Adam James Date: Wed, 8 May 2024 22:10:48 +0100 Subject: [PATCH 3/5] fix(web): Add whitespace between label and cat # --- server/components/Release.tsx | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/components/Release.tsx b/server/components/Release.tsx index b1a4e62a..3c7d396c 100644 --- a/server/components/Release.tsx +++ b/server/components/Release.tsx @@ -59,8 +59,7 @@ export function Release({ release }: { release: HarmonyRelease }) {
    {release.labels?.map((label) => (
  • - - {label.catalogNumber} + {label.catalogNumber}
    • ))}
From af99b6a248e7069d55e98c6a16756f6f99be51c4 Mon Sep 17 00:00:00 2001 From: Adam James Date: Wed, 8 May 2024 22:22:30 +0100 Subject: [PATCH 4/5] chore: Standardise on "length" for property names MusicBrainz recently switched from using a mix of "duration" and "length" to just "length", so follow suit for consistency. --- harmonizer/properties.ts | 2 +- harmonizer/types.ts | 4 ++-- musicbrainz/seeding.ts | 2 +- providers/Bandcamp/mod.ts | 2 +- providers/Beatport/mod.ts | 2 +- providers/Deezer/mod.ts | 2 +- providers/iTunes/mod.ts | 2 +- providers/mod.ts | 4 ++-- server/components/Tracklist.tsx | 4 ++-- 9 files changed, 12 insertions(+), 12 deletions(-) diff --git a/harmonizer/properties.ts b/harmonizer/properties.ts index 45145434..aa9a1bd5 100644 --- a/harmonizer/properties.ts +++ b/harmonizer/properties.ts @@ -31,5 +31,5 @@ export const immutableReleaseProperties = [ /** Track properties which have to be taken from one provider and can not be combined from data of multiple providers. */ export const immutableTrackProperties = [ 'isrc', - 'duration', + 'length', ] as const; diff --git a/harmonizer/types.ts b/harmonizer/types.ts index 7ded7936..04ecec37 100644 --- a/harmonizer/types.ts +++ b/harmonizer/types.ts @@ -68,8 +68,8 @@ export type HarmonyTrack = { title: string; artists?: ArtistCredit; number?: number | string; - /** Track duration in milliseconds. */ - duration?: number; + /** Track length in milliseconds. */ + length?: number; isrc?: string; availableIn?: CountryCode[]; }; diff --git a/musicbrainz/seeding.ts b/musicbrainz/seeding.ts index 0ddaebde..7f9e2bba 100644 --- a/musicbrainz/seeding.ts +++ b/musicbrainz/seeding.ts @@ -41,7 +41,7 @@ export function createReleaseSeed(release: HarmonyRelease, options: ReleaseSeedO name: track.title, artist_credit: convertArtistCredit(track.artists), number: track.number?.toString(), - length: track.duration, + length: track.length, })), })), language: release.language?.code, diff --git a/providers/Bandcamp/mod.ts b/providers/Bandcamp/mod.ts index bc7ea49d..0d491e64 100644 --- a/providers/Bandcamp/mod.ts +++ b/providers/Bandcamp/mod.ts @@ -220,7 +220,7 @@ export class BandcampReleaseLookup extends ReleaseLookup const result: HarmonyTrack = { number: index + 1, title: track.title, - duration: track.duration * 1000, + length: track.duration * 1000, }; if ('isrc' in track) { diff --git a/providers/iTunes/mod.ts b/providers/iTunes/mod.ts index 1bb98d4a..01141d5f 100644 --- a/providers/iTunes/mod.ts +++ b/providers/iTunes/mod.ts @@ -223,7 +223,7 @@ export class iTunesReleaseLookup extends ReleaseLookupTrack Title Artists - Duration + Length ISRC {medium.tracklist.some((track) => track.availableIn) && Availability} @@ -37,7 +37,7 @@ export function Tracklist({ medium, showTitle = false }: Props) { {track.number} {track.title} {track.artists && } - {formatDuration(track.duration, { showMs: true })} + {formatDuration(track.length, { showMs: true })} {track.isrc && } From 50d42460241b9d5ec6c6bbf3687b08fe0a7ebd0b Mon Sep 17 00:00:00 2001 From: Adam James Date: Thu, 9 May 2024 17:19:06 +0100 Subject: [PATCH 5/5] feat(provider): Add support for Beatport GTIN lookups --- providers/Beatport/json_types.ts | 17 ++++++++++++++-- providers/Beatport/mod.ts | 34 +++++++++++++++++++++++++++++--- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/providers/Beatport/json_types.ts b/providers/Beatport/json_types.ts index c2188a58..cf4f0325 100644 --- a/providers/Beatport/json_types.ts +++ b/providers/Beatport/json_types.ts @@ -2,9 +2,11 @@ export interface BeatportNextData { props: { pageProps: { dehydratedState: { - queries: Array | ArrayQueryResult>; + queries: + | Array | ArrayQueryResult> + | Array>; }; - release: Release; + release?: Release; }; }; } @@ -35,6 +37,17 @@ export interface ArrayQueryResult extends QueryResult { }; } +export interface ReleaseSearchResult { + release_id: number; + upc: string; +} + +export interface SearchResult { + releases: { + data: ReleaseSearchResult[]; + }; +} + export interface Entity { id: number; name: string; diff --git a/providers/Beatport/mod.ts b/providers/Beatport/mod.ts index 176dfaca..ce884a7b 100644 --- a/providers/Beatport/mod.ts +++ b/providers/Beatport/mod.ts @@ -31,8 +31,10 @@ export default class BeatportProvider extends MetadataProvider { readonly artworkQuality = 1400; + readonly baseUrl = 'https://www.beatport.com'; + constructUrl(entity: EntityId): URL { - return new URL([entity.type, entity.slug ?? '-', entity.id].join('/'), 'https://www.beatport.com'); + return new URL([entity.type, entity.slug ?? '-', entity.id].join('/'), this.baseUrl); } extractEmbeddedJson(webUrl: URL, maxTimestamp?: number): Promise> { @@ -60,11 +62,18 @@ export class BeatportReleaseLookup extends ReleaseLookup { + let releaseId = this.lookup.value; + if (this.lookup.method === 'gtin') { - throw new ProviderError(this.provider.name, 'GTIN lookups are not supported (yet)'); + const id = await this.searchReleaseByGtin(this.lookup.value); + if (!id) { + throw new ProviderError(this.provider.name, 'Search returned no matching results'); + } + + releaseId = id; } - const webUrl = this.provider.constructUrl({ id: this.lookup.value, type: 'release' }); + const webUrl = this.provider.constructUrl({ id: releaseId, type: 'release' }); const { content: data, timestamp } = await this.provider.extractEmbeddedJson( webUrl, this.options.snapshotMaxTimestamp, @@ -74,6 +83,25 @@ export class BeatportReleaseLookup extends ReleaseLookup { + const webUrl = new URL('search', this.provider.baseUrl); + webUrl.searchParams.set('q', gtin); + + const { content: data } = await this.provider.extractEmbeddedJson( + webUrl, + this.options.snapshotMaxTimestamp, + ); + + const result = data.props.pageProps.dehydratedState.queries[0]; + if (!('releases' in result.state.data)) { + throw new ProviderError(this.provider.name, 'Failed to extract results from embedded JSON'); + } + + const release = result.state.data.releases.data.find((r) => r.upc === gtin); + + return release?.release_id.toString(); + } + convertRawRelease(rawRelease: BeatportRelease): HarmonyRelease { this.id = rawRelease.id.toString(); const releaseUrl = this.provider.constructUrl({