From 68d39d6752f4ec3eefab4938786ff2444d55d8ba Mon Sep 17 00:00:00 2001 From: Brandon Bothell Date: Mon, 10 Jul 2023 00:41:02 -0400 Subject: [PATCH 01/10] feat(cache): better paginated and individual item caching --- src/index.ts | 44 ++++-- src/oauth/captions.ts | 10 +- src/oauth/index.ts | 2 +- src/services/generic-service.ts | 210 ++++++++++++++++++++--------- src/services/resolution-service.ts | 13 +- src/types/GetItem.ts | 4 +- src/util/arrays.ts | 10 ++ src/util/cache.ts | 154 ++++++++++++++++++++- src/util/index.ts | 1 + test/playlist.spec.ts | 2 + 10 files changed, 360 insertions(+), 90 deletions(-) create mode 100644 src/util/arrays.ts diff --git a/src/index.ts b/src/index.ts index a8436193a..3d4da9d70 100644 --- a/src/index.ts +++ b/src/index.ts @@ -133,7 +133,35 @@ export class YouTube { return } - Cache.set(id, value, this._cacheTTL > 0 ? this._cacheTTL * 1000 + new Date().getTime() : 0) + Cache.set(id, value, + this._cacheTTL > 0 ? this._cacheTTL * 1000 + new Date().getTime() : 0) + } + + /** + * @ignore + */ + public _cacheItem (type: T.ItemTypes, id: string, parts: string[] | undefined, + value: InstanceType) { + if (!this._shouldCache) { + return + } + + Cache.setItem(type, id, parts, value, + this._cacheTTL > 0 ? this._cacheTTL * 1000 + new Date().getTime() : 0) + } + + /** + * @ignore + */ + public _cachePage (endpoint: string, page: number, options: T.PaginatedRequestParams, + auth: T.AuthorizationOptions | undefined, + value: T.PaginatedResponse) { + if (!this._shouldCache) { + return + } + + Cache.setPage(endpoint, page, options, auth, value, + this._cacheTTL > 0 ? this._cacheTTL * 1000 + new Date().getTime() : 0) } /** @@ -188,7 +216,7 @@ export class YouTube { */ public async getVideo (videoResolvable: T, parts?: Part.VideoParts) { const video = await this._resolutionService.resolve(videoResolvable, Entity.Video) - return this._genericService.getItem(Entity.Video, false, video, parts) + return this._genericService.getItem(Entity.Video, { resolvableEntity: video }, parts) } /** @@ -200,7 +228,7 @@ export class YouTube { */ public async getChannel (channelResolvable: T, parts?: Part.ChannelParts) { const channel = await this._resolutionService.resolve(channelResolvable, Entity.Channel) - return this._genericService.getItem(Entity.Channel, false, channel, parts) + return this._genericService.getItem(Entity.Channel, { resolvableEntity: channel }, parts) } /** @@ -211,7 +239,7 @@ export class YouTube { */ public async getPlaylist (playlistResolvable: T, parts?: Part.PlaylistParts) { const playlist = await this._resolutionService.resolve(playlistResolvable, Entity.Playlist) - return this._genericService.getItem(Entity.Playlist, false, playlist, parts) + return this._genericService.getItem(Entity.Playlist, { resolvableEntity: playlist }, parts) } /** @@ -220,7 +248,7 @@ export class YouTube { * @param parts The parts of the comment to fetch (saves quota if you aren't using certain properties!) */ public async getComment (commentId: T, parts?: Part.CommentParts) { - return this._genericService.getItem(Entity.Comment, false, commentId, parts) + return this._genericService.getItem(Entity.Comment, { resolvableEntity: commentId }, parts) } /** @@ -231,7 +259,7 @@ export class YouTube { * @param parts The parts of the subscription to fetch (saves quota if you aren't using certain properties!) */ public async getSubscription (subscriptionId: T, parts?: Part.SubscriptionParts) { - return this._genericService.getItem(Entity.Subscription, false, subscriptionId, parts) + return this._genericService.getItem(Entity.Subscription, { resolvableEntity: subscriptionId }, parts) } /** @@ -239,7 +267,7 @@ export class YouTube { * @param categoryId The ID of the category. */ public async getCategory (categoryId: T) { - return this._genericService.getItem(Entity.VideoCategory, false, categoryId) + return this._genericService.getItem(Entity.VideoCategory, { resolvableEntity: categoryId }) } /** @@ -248,7 +276,7 @@ export class YouTube { * @param parts The parts of the channel section to fetch (saves quota if you aren't using certain properties!) */ public async getChannelSection (sectionId: T, parts?: Part.ChannelSectionParts) { - return this._genericService.getItem(Entity.ChannelSection, false, sectionId, parts) + return this._genericService.getItem(Entity.ChannelSection, { resolvableEntity: sectionId }, parts) } /** diff --git a/src/oauth/captions.ts b/src/oauth/captions.ts index 82062c767..97d43435f 100644 --- a/src/oauth/captions.ts +++ b/src/oauth/captions.ts @@ -10,7 +10,8 @@ export class OAuthCaptions { constructor (public oauth: OAuth) {} /** - * Get a [Caption](./Library_Exports.Caption#) object from the ID of the caption. + * Get a [Caption](./Library_Exports.Caption#) object + * from the URL, ID, or search query of its video and the ID of the caption. * Last tested 06/11/2020 04:50. PASSING * @param videoResolvable The Title, URL, or ID of the video to get the caption from. * @param captionId The ID of the caption. @@ -19,8 +20,13 @@ export class OAuthCaptions { this.oauth.checkTokenAndThrow() const video = await this.oauth.youtube._resolutionService.resolve(videoResolvable, YT.Video) + const params: { videoId: string; id?: string } = + { videoId: typeof video === 'string' ? video : video.id } + + if (captionId) params.id = captionId + const data = await this.oauth.youtube._request.get('captions', { - params: { part: 'snippet', videoId: typeof video === 'string' ? video : video.id, id: captionId }, + params: { part: 'snippet', id: captionId }, authorizationOptions: { accessToken: true } }) diff --git a/src/oauth/index.ts b/src/oauth/index.ts index 5e6174350..7489e3aca 100644 --- a/src/oauth/index.ts +++ b/src/oauth/index.ts @@ -72,7 +72,7 @@ export class OAuth { */ public getMe (parts?: Part.ChannelParts): Promise { this.checkTokenAndThrow() - return this.youtube._genericService.getItem(YT.Channel, true, null, parts) as Promise + return this.youtube._genericService.getItem(YT.Channel, { mine: true }, parts) } /** diff --git a/src/services/generic-service.ts b/src/services/generic-service.ts index 75cbb9eab..abd168760 100644 --- a/src/services/generic-service.ts +++ b/src/services/generic-service.ts @@ -15,8 +15,11 @@ export class GenericService { } - public async getItem | Resolvable[], K extends ItemTypes> (type: K, mine: boolean, id?: T, parts?: string[]): - Promise> { + public async getItem | Resolvable[], K extends ItemTypes, M extends boolean = false> + (type: K, { mine, resolvableEntity }: + { mine?: M; resolvableEntity?: T }, parts?: string[]): + Promise> { + if (!this._getItemAllowedTypes.has(type.name)) { return Promise.reject(new Error(`${type.name}s cannot be directly fetched. The item may be paginated or not directly accessible.`)) } @@ -25,47 +28,65 @@ export class GenericService { return Promise.reject(new Error(`${type.name}s cannot be filtered by the 'mine' parameter.`)) } - if (!mine && !id) { + if (!mine && !resolvableEntity) { return Promise.reject(new Error('Items must either specify an ID or the \'mine\' parameter.')) } + // Convert to array full of IDs and/or entity class instances + const resolvables: Resolvable[] = mine ? [] : Array.isArray(resolvableEntity) ? + resolvableEntity : + [ resolvableEntity as Resolvable ] + + if (!mine && resolvables.length === 0) { + return Promise.reject( + new Error('The resolvableEntity parameter must include at least one item.') + ) + } + + // IDs + let resolvableStrings: string[] + + // Instances of entity classes + let resolvedEntities: InstanceType[] let alreadyResolvedCount = 0 - const alreadyResolved: InstanceType[] = [] - const idsToGet: string[] = [] - if (typeof id === 'string') idsToGet.push(id) - else if (Array.isArray(id)) { - for (let i = 0; i < id.length; i++) { - const entry = id[i] + for (let i = 0; i < resolvables.length; i++) { + const entry = resolvables[i] - if (typeof entry === 'string') { - idsToGet.push(entry) + // An ID + if (typeof entry === 'string') { + const cachedEntity = this.youtube._shouldCache ? Cache.getItem(type, entry, parts) : undefined + + if (!cachedEntity) { + if (!resolvableStrings) resolvableStrings = [] + resolvableStrings.push(entry) + // An instance we have cached and can just return } else { - if (alreadyResolved.length < i + 1) alreadyResolved.length = id.length - alreadyResolved[i] = entry + if (!resolvedEntities) resolvedEntities = new Array(resolvables.length).fill(undefined) + resolvedEntities[i] = cachedEntity as InstanceType alreadyResolvedCount++ } + // An instance that we can just return + } else if (entry instanceof type) { + if (!resolvedEntities) resolvedEntities = new Array(resolvables.length).fill(undefined) + resolvedEntities[i] = entry + alreadyResolvedCount++ + } else { + return Promise.reject(new TypeError( + 'The resolvableEntity parameter must be a string, array, or proper entity class.' + )) } - - // If all of the items are entities, what are we even doing here? - if (alreadyResolvedCount === id.length) { - return alreadyResolved as ItemReturns - } - } else if (id) { - return id as ItemReturns } - const idString = idsToGet.join(',') - let preresolvedCacheString = alreadyResolved.length ? alreadyResolved.map(r => r.id).join(',') : '' - - let cacheKey: string - - if (this.youtube._shouldCache) { - cacheKey = `get://${type.endpoint}/${id ? idString : 'mine'}/${preresolvedCacheString}/${parts?.join(',')}` - const cached = Cache.get(cacheKey) - if (cached) return cached + // If all of the items are instances, what are we even doing here? + if (alreadyResolvedCount === resolvables.length) { + return (resolvedEntities.length === 1 ? + resolvedEntities[0] : + resolvedEntities) as ItemReturns } + const idString = resolvableStrings?.join(',') + const options: { id?: string mine?: boolean @@ -93,21 +114,28 @@ export class GenericService { return Promise.reject(new Error('Item not found')) } - let endResult: ItemReturns - - if (!alreadyResolved.length) endResult = await Promise.all(result.items.map(async item => new (type)(this.youtube, await item, true))) - else { - for (const item of result.items) { - alreadyResolved[alreadyResolved.findIndex(value => value === undefined)] = new (type)(this.youtube, item, true) as InstanceType + // All resolvable entries are strings (IDs, URLs, and search queries) + if (!resolvedEntities) { + resolvedEntities = result.items.map((item, index) => + this.transformItemToCachedEntity(type, item, resolvables[index] as string, parts)) + // Otherwise there are some entity objects to include + } else { + let resultIndex = 0 + // All undefined indices should coorelate with a string value in resolvables[] + // and a result from the API + resolvedEntities = resolvedEntities.map((entity, index) => + entity ?? this.transformItemToCachedEntity(type, result.items[resultIndex++], + resolvables[index] as string, parts)) + + if (resultIndex < result.items.length - 1) { + return Promise.reject(new Error(`There are ${ + result.items.length - resultIndex - 1} extra items in the response`)) } - - endResult = alreadyResolved } - if (endResult.length === 1) endResult = endResult[0] - if (this.youtube._shouldCache) this.youtube._cache(cacheKey, endResult) - - return endResult as ItemReturns + return (resolvedEntities.length === 1 ? + resolvedEntities[0] : + resolvedEntities) as ItemReturns } /** @@ -138,13 +166,13 @@ export class GenericService { return Promise.reject(new Error(`${name} cannot be filtered by the 'mine' parameter.`)) } - let cacheKey: string + /* let cacheKey: string if (this.youtube._shouldCache) { cacheKey = `getpage://${type}/${id ? id : 'mine'}/${subId}/${parts?.join(',')}/${maxPerPage >= 1 ? maxPerPage : 0}/${pages}/${pageToken}` const cached = Cache.get(cacheKey) if (cached) return cached - } + } */ const options: PaginatedRequestParams = { part: parts?.join(',') @@ -244,8 +272,10 @@ export class GenericService { if (maxForEndpoint !== undefined) { if (pages < 1 || maxPerPage < 1) options.maxResults = maxForEndpoint - else if (maxPerPage > maxForEndpoint) return Promise.reject(new Error(`Max per page must be ${maxForEndpoint} or below for ${endpoint}`)) - else options.maxResults = maxPerPage + else if (maxPerPage > maxForEndpoint) { + return Promise.reject( + new Error(`Max per page must be ${maxForEndpoint} or below for ${endpoint}`)) + } else options.maxResults = maxPerPage } if (pages < 1) { @@ -254,70 +284,118 @@ export class GenericService { if (pageToken) options.pageToken = pageToken + // Caching handled here const toReturn = await this.fetchPages(pages, endpoint, options, clazz, auth) as PaginatedResponse - if (this.youtube._shouldCache) this.youtube._cache(cacheKey, toReturn) + // if (this.youtube._shouldCache) this.youtube._cache(cacheKey, toReturn) return toReturn } /** + * Caching handled here * @param clazz Most endpoints only return one type of entity, so set this to that entity. * If endpoint is `search`, then leave undefined. */ - public async fetchPages (pages: number, endpoint: string, options: - { part: string; maxResults?: number; hl?: string; [key: string]: any }, clazz?: T, - auth?: AuthorizationOptions): Promise>> { + public async fetchPages ( + pages: number, endpoint: string, options: PaginatedRequestParams, clazz?: T, + auth?: AuthorizationOptions): + Promise>> { if (!clazz && endpoint !== 'search') { return Promise.reject(new Error('Endpoints other than search must specify an entity class')) } - const toReturn: PaginatedResponse> = { + let toReturn: PaginatedResponse> = { items: [] } + const cachedPages = this.youtube._shouldCache ? + Cache.getPages>(endpoint, options, auth) : + undefined + let pagesFetched = 0 + while (true) { - const apiResponse = await this.youtube._request.get(endpoint, { + let page: PaginatedResponse + + if (this.youtube._shouldCache) { + page = cachedPages?.[pagesFetched] + + // We have a cached page, use that + if (page) { + toReturn.items = toReturn.items.concat(page.items) + if (page.prevPageToken) toReturn.prevPageToken = page.prevPageToken + if (page.nextPageToken) toReturn.nextPageToken = page.nextPageToken + if (++pagesFetched >= pages || !page.nextPageToken) break + options.pageToken = toReturn.nextPageToken + continue + } + } + + // No cached page, request one from the API + page = await this.youtube._request.get(endpoint, { params: options, - authorizationOptions: auth ?? { [options.mine ? 'accessToken' : 'apiKey']: true } - }) + authorizationOptions: auth ?? { [options.mine ? 'accessToken' : 'apiKey']: true } }) - if (!apiResponse.items?.length) { - if (apiResponse.prevPageToken) toReturn.prevPageToken = apiResponse.prevPageToken + // No more items found, return + if (!page.items?.length) { + if (page.prevPageToken) toReturn.prevPageToken = page.prevPageToken break } - pagesFetched++ + const instances: InstanceType[] = [] - for (const data of apiResponse.items) { + for (const data of page.items) { if (endpoint === 'search') { if (data.id.videoId) { - toReturn.items.push(new Video(this.youtube, data) as InstanceType) + instances.push(new Video(this.youtube, data) as InstanceType) } else if (data.id.channelId) { - toReturn.items.push(new Channel(this.youtube, data) as InstanceType) + instances.push(new Channel(this.youtube, data) as InstanceType) } else if (data.id.playlistId) { - toReturn.items.push(new Playlist(this.youtube, data) as InstanceType) + instances.push(new Playlist(this.youtube, data) as InstanceType) } } else if (data.kind === 'youtube#commentThread') { - toReturn.items.push(new Comment(this.youtube, data, true) as InstanceType) + instances.push(new Comment(this.youtube, data, true) as InstanceType) } else { - toReturn.items.push(new clazz(this.youtube, data, false) as InstanceType) + instances.push(new clazz(this.youtube, data, false) as InstanceType) } } - if (pagesFetched >= pages || !apiResponse.nextPageToken) { - if (apiResponse.prevPageToken) toReturn.prevPageToken = apiResponse.prevPageToken - if (apiResponse.nextPageToken) toReturn.nextPageToken = apiResponse.nextPageToken + toReturn.items = toReturn.items.concat(instances) + + const toCache: PaginatedResponse> = { + items: instances + } + + if (page.prevPageToken) toCache.prevPageToken = page.prevPageToken + if (page.nextPageToken) toCache.nextPageToken = page.nextPageToken + + this.youtube._cachePage(endpoint, pagesFetched, options, auth, toCache) + + if (++pagesFetched >= pages || !page.nextPageToken) { + if (page.prevPageToken) toReturn.prevPageToken = page.prevPageToken + if (page.nextPageToken) toReturn.nextPageToken = page.nextPageToken break } - options.pageToken = apiResponse.nextPageToken + options.pageToken = page.nextPageToken } return toReturn } + + private transformItemToCachedEntity + (type: K, item: any, cacheString: string, parts?: string[]): + InstanceType { + const entity = new (type)(this.youtube, item, true) as InstanceType + + if (this.youtube._shouldCache) { + this.youtube._cacheItem(type, cacheString, parts, entity) + } + + return entity + } } diff --git a/src/services/resolution-service.ts b/src/services/resolution-service.ts index feb4a30ba..14baf21c1 100644 --- a/src/services/resolution-service.ts +++ b/src/services/resolution-service.ts @@ -16,7 +16,8 @@ export class ResolutionService { else return this.resolveStringToIdOrEntity(input, type) as Promise> } - const resolutions: (Resolvable | Promise>)[] = new Array(input.length) // either the ID or the input if resolution failed + // either the ID or the input if resolution failed + const resolutions: (Resolvable | Promise>)[] = new Array(input.length).fill(undefined) const resolvableStrings: { [resolutionIndex: number]: string } = {} // could be ID, URL, or search query let preresolvedCount = 0 @@ -50,16 +51,16 @@ export class ResolutionService { * **Anything that isn't an ID, Entity, or Username URL returns the first result of a search query of `type`.** */ public async resolveStringToIdOrEntity (input: string, type: T): Promise> { - if (this.youtube._shouldCache) { - const cached = Cache.get(`get_id://${type.endpoint}/${input}`) - if (cached) return cached - } - // types that resolve only by class or ID if (type === VideoCategory || type === Comment || type === Subscription) { return input } + if (this.youtube._shouldCache) { + const cached = Cache.get(`get_id://${type.endpoint}/${input}`) + if (cached) return cached + } + // The search query or ID parameter for later (used for setting legacy custom channel URL search query) let idOrSearchQuery = input let resolution: Resolvable diff --git a/src/types/GetItem.ts b/src/types/GetItem.ts index 394dce9c5..1bf27788c 100644 --- a/src/types/GetItem.ts +++ b/src/types/GetItem.ts @@ -4,4 +4,6 @@ export const GETTABLE_CLASSES = [ Video, Channel, Playlist, Comment, Subscriptio export type ItemTypes = typeof GETTABLE_CLASSES[number] -export type ItemReturns = T extends any[] ? InstanceType[] : InstanceType +export type ItemReturns = + M extends true ? InstanceType : + T extends any[] ? InstanceType[] : InstanceType diff --git a/src/util/arrays.ts b/src/util/arrays.ts new file mode 100644 index 000000000..68f8a7409 --- /dev/null +++ b/src/util/arrays.ts @@ -0,0 +1,10 @@ +export function findArrayIndexFrom +(predicate: (element: T) => boolean, array: T[], startFrom: number): number { + for (let i = startFrom; i < array.length; i++) { + if (predicate(array[i])) { + return i + } + } + + return -1 +} diff --git a/src/util/cache.ts b/src/util/cache.ts index 5ce829ac6..bca9e7ddd 100644 --- a/src/util/cache.ts +++ b/src/util/cache.ts @@ -1,8 +1,12 @@ +import { AuthorizationOptions, ItemTypes, PaginatedInstance, PaginatedRequestParams, PaginatedResponse } from '..' + /** * @ignore */ export class Cache { private static map: Map = new Map() + private static itemsMap: Map>> = new Map() + private static pagesMap: Map>[]> = new Map() public static set (key: string, value: any, ttl: number) { Cache.map.set(key, { v: value, t: ttl }) @@ -19,27 +23,165 @@ export class Cache { return item.v } + public static setItem (type: ItemTypes, name: string, parts: string[] | undefined, + value: InstanceType, ttl: number) { + Cache.itemsMap.set(`${type.name.toLowerCase()}/${name}/${parts?.join(',')}`, + { v: value, t: ttl }) + } + + public static getItem (type: ItemTypes, name: string, parts?: string[]): + InstanceType { + const key = `${type.name.toLowerCase()}/${name}/${parts?.join(',')}` + const item = Cache.itemsMap.get(key) + + if (!item || (item.t > 0 && new Date().getTime() >= item.t)) { + Cache._deleteItem(key) + return undefined + } + + return item.v + } + + public static setPage (endpoint: string, page: number, + options: PaginatedRequestParams, auth: AuthorizationOptions | undefined, + value: PaginatedResponse, ttl: number) { + + const key = `${endpoint}/${JSON.stringify({ ...options, pageToken: undefined })}/${ + auth?.accessToken ?? false}/${auth?.apiKey ?? false}` + const toCache = Cache.pagesMap.get(key) ?? [] + + if (page > toCache.length - 1) toCache.length = page + 1 + toCache[page] = { v: value, t: ttl } + console.log(`New page length: ${value.items.length}`) + + Cache.pagesMap.set(key, toCache) + } + + public static getPage ( + endpoint: string, page: number, options: PaginatedRequestParams, + auth: AuthorizationOptions | undefined): + PaginatedResponse { + const key = `${endpoint}/${JSON.stringify({ ...options, pageToken: undefined })}/${ + auth?.accessToken ?? false}/${auth?.apiKey ?? false}` + const pages = Cache.pagesMap.get(key) + const item = pages ? page > pages.length - 1 ? undefined : pages[page] : undefined + + if (!item) return undefined + + if (item.t > 0 && new Date().getTime() >= item.t) { + Cache._deletePage(endpoint, page, options, auth) + return undefined + } + + return item.v as PaginatedResponse + } + + public static getPages ( + endpoint: string, options: PaginatedRequestParams, + auth: AuthorizationOptions | undefined): + PaginatedResponse[] { + const key = `${endpoint}/${JSON.stringify({ ...options, pageToken: undefined })}/${ + auth?.accessToken ?? false}/${auth?.apiKey ?? false}` + const pages = Cache.pagesMap.get(key) + + if (!pages) { + return undefined + } + + const toReturn: PaginatedResponse[] = new Array(pages.length) + + if (pages) console.log(`Number of pages from fetched: ${pages.length}`) + + for (let page = 0; page < pages.length; page++) { + const item = pages[page] + + if (!item) continue + if (item.t > 0 && new Date().getTime() >= item.t) { + Cache._deletePage(endpoint, page, options, auth) + return undefined + } + + toReturn[page] = item.v as PaginatedResponse + } + + return toReturn + } + public static checkTTLs () { const time = new Date().getTime() - for (const [ name, value ] of Cache.map.entries()) { + for (const [ key, value ] of Cache.map.entries()) { const timeToDelete = value.t if (timeToDelete > 0 && time >= timeToDelete) { - Cache.map.delete(name) + Cache.map.delete(key) } } + + for (const [ key, value ] of Cache.itemsMap.entries()) { + const timeToDelete = value.t + + if (timeToDelete > 0 && time >= timeToDelete) { + Cache.itemsMap.delete(key) + } + } + + for (let [ key, pages ] of Cache.pagesMap.entries()) { + for (let page = 0; page < pages.length; page++) { + if (!pages[page]) continue + + const timeToDelete = pages[page].t + + if (timeToDelete > 0 && time >= timeToDelete) { + pages = Cache._deletePageByKey(key, page) + page-- + } + } + } + } + + public static _delete (key: string) { + Cache.map.delete(key) + } + + public static _deleteItem (key: string) { + Cache.itemsMap.delete(key) + } + + /** + * @returns Pages left in the key + */ + public static _deletePageByKey (key: string, page: number) { + const pages = Cache.pagesMap.get(key) + + if (page > pages.length - 1) return pages + + pages.splice(page, 1) + + if (pages.length === 0) { + Cache.pagesMap.delete(key) + } else { + Cache.pagesMap.set(key, pages) + } + + return pages } - public static _delete (name: string) { - Cache.map.delete(name) + /** + * @returns Pages left in the key + */ + public static _deletePage (endpoint: string, page: number, + options: PaginatedRequestParams, auth: AuthorizationOptions | undefined) { + const key = `${endpoint}/${JSON.stringify(options)}/${ + auth?.accessToken ?? false}/${auth?.apiKey ?? false}` + return Cache._deletePageByKey(key, page) } } /** * @ignore */ -type CacheItem = { - v: any +type CacheItem = { + v: T t: number } diff --git a/src/util/index.ts b/src/util/index.ts index 27c5f8ec8..aab068847 100644 --- a/src/util/index.ts +++ b/src/util/index.ts @@ -1,3 +1,4 @@ export * from './parser' export * from './request' export * from './cache' +export * from './arrays' diff --git a/test/playlist.spec.ts b/test/playlist.spec.ts index a14e68c7b..2be17118c 100644 --- a/test/playlist.spec.ts +++ b/test/playlist.spec.ts @@ -42,8 +42,10 @@ describe('Playlists', () => { }) it('should work with fetching a page of videos', async () => { + console.log('----- before page -----') const playlist = await youtube.getPlaylist('PLMC9KNkIncKvYin_USF1qoJQnIyMAfRxl', [ 'id' ]) const videos = (await playlist.fetchVideos(undefined, [ 'id' ])).items + console.log('----- after page -----') expect(videos.length).to.equal(50) expect(playlist.videos.items.length).to.equal(50) From 0b10064bda6718807b6a0d285a826fef0a2c77b6 Mon Sep 17 00:00:00 2001 From: Brandon Bothell Date: Mon, 10 Jul 2023 11:53:45 -0400 Subject: [PATCH 02/10] feat(cache): handle items with extra data properly and clean up --- src/entities/channel-section.ts | 2 +- src/entities/channel.ts | 3 +- src/entities/playlist.ts | 2 +- src/entities/subscription.ts | 2 +- src/entities/video.ts | 3 +- src/services/generic-service.ts | 49 +++++++++++---------------- src/services/search-service.ts | 9 ++--- src/util/cache.ts | 60 ++++++++++++++++++++++++--------- test/cache.spec.ts | 2 +- test/playlist.spec.ts | 2 -- test/search.spec.ts | 11 ++++++ 11 files changed, 84 insertions(+), 61 deletions(-) diff --git a/src/entities/channel-section.ts b/src/entities/channel-section.ts index efe8418a0..20a8749bf 100644 --- a/src/entities/channel-section.ts +++ b/src/entities/channel-section.ts @@ -13,7 +13,7 @@ export class ChannelSection { /** * The parts to request for this entity. */ - public static part = 'snippet,contentDetails' + public static part = 'contentDetails,snippet' /** * The fields to request for this entity. diff --git a/src/entities/channel.ts b/src/entities/channel.ts index ba833f241..580bfd12f 100644 --- a/src/entities/channel.ts +++ b/src/entities/channel.ts @@ -13,7 +13,8 @@ export class Channel { /** * The parts to request for this entity. */ - public static part = 'snippet,contentDetails,statistics,status,brandingSettings,localizations' + public static part = + 'brandingSettings,contentDetails,localizations,snippet,statistics,status' /** * The fields to request for this entity. diff --git a/src/entities/playlist.ts b/src/entities/playlist.ts index c781cd9cb..82610bc9f 100644 --- a/src/entities/playlist.ts +++ b/src/entities/playlist.ts @@ -13,7 +13,7 @@ export class Playlist { /** * The parts to request for this entity. */ - public static part = 'snippet,contentDetails,player,status' + public static part = 'contentDetails,player,snippet,status' /** * The fields to request for this entity. diff --git a/src/entities/subscription.ts b/src/entities/subscription.ts index 79cc1bf02..8f708d4c3 100644 --- a/src/entities/subscription.ts +++ b/src/entities/subscription.ts @@ -14,7 +14,7 @@ export class Subscription { /** * The parts to request for this entity. */ - public static part = 'snippet,contentDetails,subscriberSnippet' + public static part = 'contentDetails,snippet,subscriberSnippet' /** * The fields to request for this entity. diff --git a/src/entities/video.ts b/src/entities/video.ts index 4b410e3dd..43c0f67ea 100644 --- a/src/entities/video.ts +++ b/src/entities/video.ts @@ -16,7 +16,8 @@ export class Video { /** * The parts to request for this entity. */ - public static part = 'snippet,contentDetails,statistics,status,recordingDetails,localizations' + public static part = + 'contentDetails,localizations,recordingDetails,snippet,statistics,status' /** * The fields to request for this entity. diff --git a/src/services/generic-service.ts b/src/services/generic-service.ts index abd168760..e76083bd2 100644 --- a/src/services/generic-service.ts +++ b/src/services/generic-service.ts @@ -202,9 +202,10 @@ export class GenericService { maxForEndpoint = 100 clazz = Comment options[`${commentType}Id`] = id + options.textFormat = 'plainText' + if (!options.part) options.part = 'snippet,replies' if (otherFilters.order) options.order = otherFilters.order - options.textFormat = 'plainText' break case PaginatedItemType.CommentReplies: @@ -215,23 +216,28 @@ export class GenericService { options.parentId = id break + case PaginatedItemType.Captions: + endpoint = 'captions' + clazz = Caption + options.videoId = id + break + case PaginatedItemType.Playlists: endpoint = 'playlists' clazz = Playlist // falls through - case PaginatedItemType.Subscriptions: if (!endpoint) endpoint = 'subscriptions' if (!clazz) clazz = Subscription maxForEndpoint = 50 // falls through - case PaginatedItemType.ChannelSections: if (!endpoint) endpoint = 'channelSections' if (!clazz) clazz = ChannelSection - if (mine) options.mine = mine; else options.channelId = id + if (mine) options.mine = mine + else options.channelId = id break case PaginatedItemType.VideoCategories: @@ -239,57 +245,40 @@ export class GenericService { clazz = VideoCategory options.regionCode = this.youtube.region // falls through - case PaginatedItemType.VideoAbuseReportReasons: if (!endpoint) endpoint = 'videoAbuseReportReasons' if (!clazz) clazz = VideoAbuseReportReason // falls through - case PaginatedItemType.Languages: if (!endpoint) endpoint = 'i18nLanguages' if (!clazz) clazz = Language // falls through - case PaginatedItemType.Regions: if (!endpoint) endpoint = 'i18nRegions' if (!clazz) clazz = Region - options.hl = this.youtube.language - break - case PaginatedItemType.Captions: - endpoint = 'captions' - clazz = Caption - options.videoId = id + options.hl = this.youtube.language break default: return Promise.reject(new TypeError('Unknown item type: ' + type)) } - if (!options.part) { - options.part = clazz.part - } + if (!options.part) options.part = clazz.part + if (pages < 1) pages = Infinity + if (pageToken) options.pageToken = pageToken if (maxForEndpoint !== undefined) { - if (pages < 1 || maxPerPage < 1) options.maxResults = maxForEndpoint - else if (maxPerPage > maxForEndpoint) { + if (pages === Infinity || maxPerPage < 1) options.maxResults = maxForEndpoint + else if (maxPerPage <= maxForEndpoint) options.maxResults = maxPerPage + else { return Promise.reject( new Error(`Max per page must be ${maxForEndpoint} or below for ${endpoint}`)) - } else options.maxResults = maxPerPage - } - - if (pages < 1) { - pages = Infinity + } } - if (pageToken) options.pageToken = pageToken - // Caching handled here - const toReturn = await this.fetchPages(pages, endpoint, options, clazz, auth) as PaginatedResponse - - // if (this.youtube._shouldCache) this.youtube._cache(cacheKey, toReturn) - - return toReturn + return this.fetchPages(pages, endpoint, options, clazz, auth) as Promise> } /** diff --git a/src/services/search-service.ts b/src/services/search-service.ts index 7760a7771..f7f194291 100644 --- a/src/services/search-service.ts +++ b/src/services/search-service.ts @@ -75,12 +75,7 @@ export class SearchService { if ('order' in searchFilters) options.order = searchFilters.order if ('videoEmbeddable' in searchFilters) options.videoEmbeddable = 'true' - const toReturn = await this.youtube._genericService.fetchPages(pages, 'search', options) - - if (this.youtube._shouldCache && this.youtube._cacheSearches) { - this.youtube._cache(`search://${type}/"${searchTerm}"/${pages}/${maxPerPage}/${pageToken}/${JSON.stringify(otherFilters)}`, toReturn) - } - - return toReturn + // Caching handled here + return this.youtube._genericService.fetchPages(pages, 'search', options) } } diff --git a/src/util/cache.ts b/src/util/cache.ts index bca9e7ddd..888a8a666 100644 --- a/src/util/cache.ts +++ b/src/util/cache.ts @@ -5,7 +5,7 @@ import { AuthorizationOptions, ItemTypes, PaginatedInstance, PaginatedRequestPar */ export class Cache { private static map: Map = new Map() - private static itemsMap: Map>> = new Map() + private static itemsMap: Map>[]> = new Map() private static pagesMap: Map>[]> = new Map() public static set (key: string, value: any, ttl: number) { @@ -25,17 +25,29 @@ export class Cache { public static setItem (type: ItemTypes, name: string, parts: string[] | undefined, value: InstanceType, ttl: number) { - Cache.itemsMap.set(`${type.name.toLowerCase()}/${name}/${parts?.join(',')}`, - { v: value, t: ttl }) + const key = `${type.name.toLowerCase()}/${name}` + const cachedItems = Cache.itemsMap.get(key) ?? [] + + cachedItems.push({ + v: value, + t: ttl, + p: parts ? parts?.sort()?.join(',') : type.part + }) + Cache.itemsMap.set(key, cachedItems) } public static getItem (type: ItemTypes, name: string, parts?: string[]): InstanceType { - const key = `${type.name.toLowerCase()}/${name}/${parts?.join(',')}` - const item = Cache.itemsMap.get(key) + const key = `${type.name.toLowerCase()}/${name}` + const cachedItems = Cache.itemsMap.get(key) + const item = cachedItems?.find(parts ? + item => item.p?.startsWith(parts.sort().join(',')) : + item => item.p === type.part + ) - if (!item || (item.t > 0 && new Date().getTime() >= item.t)) { - Cache._deleteItem(key) + if (!item) return undefined + if (item.t > 0 && new Date().getTime() >= item.t) { + Cache._deleteItem(key, item.p) return undefined } @@ -52,7 +64,6 @@ export class Cache { if (page > toCache.length - 1) toCache.length = page + 1 toCache[page] = { v: value, t: ttl } - console.log(`New page length: ${value.items.length}`) Cache.pagesMap.set(key, toCache) } @@ -90,8 +101,6 @@ export class Cache { const toReturn: PaginatedResponse[] = new Array(pages.length) - if (pages) console.log(`Number of pages from fetched: ${pages.length}`) - for (let page = 0; page < pages.length; page++) { const item = pages[page] @@ -118,11 +127,13 @@ export class Cache { } } - for (const [ key, value ] of Cache.itemsMap.entries()) { - const timeToDelete = value.t + for (const [ key, items ] of Cache.itemsMap.entries()) { + for (const item of items) { + const timeToDelete = item.t - if (timeToDelete > 0 && time >= timeToDelete) { - Cache.itemsMap.delete(key) + if (timeToDelete > 0 && time >= timeToDelete) { + Cache._deleteItem(key, item.p) + } } } @@ -144,8 +155,21 @@ export class Cache { Cache.map.delete(key) } - public static _deleteItem (key: string) { - Cache.itemsMap.delete(key) + public static _deleteItem (key: string, parts?: string) { + const cachedItems = Cache.itemsMap.get(key) + const index = cachedItems?.findIndex(item => item.p === parts) + + if (index === undefined || index < 0) return + + cachedItems.splice(index, 1) + + if (cachedItems.length === 0) { + Cache.itemsMap.delete(key) + } else { + Cache.itemsMap.set(key, cachedItems) + } + + return cachedItems } /** @@ -180,8 +204,12 @@ export class Cache { /** * @ignore + * v = Value + * t = Time to live + * p = Parts string (sorted!) */ type CacheItem = { v: T t: number + p?: string } diff --git a/test/cache.spec.ts b/test/cache.spec.ts index 0b3b12d16..9223d24f2 100644 --- a/test/cache.spec.ts +++ b/test/cache.spec.ts @@ -17,7 +17,7 @@ describe('Caching', () => { await youtube.getVideo('dQw4w9WgXcQ') - const video = youtube.getVideo('dQw4w9WgXcQ') + const video = youtube.getVideo('dQw4w9WgXcQ', [ 'contentDetails' ]) const time = new Date().getTime() await video diff --git a/test/playlist.spec.ts b/test/playlist.spec.ts index 2be17118c..a14e68c7b 100644 --- a/test/playlist.spec.ts +++ b/test/playlist.spec.ts @@ -42,10 +42,8 @@ describe('Playlists', () => { }) it('should work with fetching a page of videos', async () => { - console.log('----- before page -----') const playlist = await youtube.getPlaylist('PLMC9KNkIncKvYin_USF1qoJQnIyMAfRxl', [ 'id' ]) const videos = (await playlist.fetchVideos(undefined, [ 'id' ])).items - console.log('----- after page -----') expect(videos.length).to.equal(50) expect(playlist.videos.items.length).to.equal(50) diff --git a/test/search.spec.ts b/test/search.spec.ts index 42a019f3a..eeba772e1 100644 --- a/test/search.spec.ts +++ b/test/search.spec.ts @@ -84,6 +84,17 @@ describe('Searching', () => { expect(video.channel.name).to.be.a('string') }) + it('should work with caching', async () => { + let video: Video | Promise> = youtube.searchVideos('bukkit', { searchFilters: { channel: 'UC6mi9rp7vRYninucP61qOjg' }, pageOptions: { maxPerPage: 1 } }) + const time = new Date().getTime() + + video = (await video).items[0] + + expect(new Date().getTime() - time).to.be.lessThan(50) + expect(video.channel.id).to.equal('UC6mi9rp7vRYninucP61qOjg') + expect(video.channel.name).to.be.a('string') + }) + it('should return an array with a size of pages * maxPerPage', async () => { expect((await youtube.search('gaming moments', { pageOptions: { pages: 3, maxPerPage: 6 } })).items.length).to.equal(18) }) From 3c7d1dff820cbbb1ddcdbef8bbb61801fcff9deb Mon Sep 17 00:00:00 2001 From: Brandon Bothell Date: Mon, 10 Jul 2023 14:01:57 -0400 Subject: [PATCH 03/10] feat(cache): use map for parts of pages --- src/index.ts | 6 +- src/services/generic-service.ts | 10 +-- src/util/cache.ts | 107 ++++++++++++++++++-------------- 3 files changed, 69 insertions(+), 54 deletions(-) diff --git a/src/index.ts b/src/index.ts index 3d4da9d70..8ecee7e4c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -154,13 +154,13 @@ export class YouTube { * @ignore */ public _cachePage (endpoint: string, page: number, options: T.PaginatedRequestParams, - auth: T.AuthorizationOptions | undefined, - value: T.PaginatedResponse) { + auth: T.AuthorizationOptions | undefined, parts: string[] | undefined, + type: T.PaginatedType | undefined, value: T.PaginatedResponse) { if (!this._shouldCache) { return } - Cache.setPage(endpoint, page, options, auth, value, + Cache.setPage(endpoint, page, options, auth, parts, type, value, this._cacheTTL > 0 ? this._cacheTTL * 1000 + new Date().getTime() : 0) } diff --git a/src/services/generic-service.ts b/src/services/generic-service.ts index e76083bd2..fc3742bb0 100644 --- a/src/services/generic-service.ts +++ b/src/services/generic-service.ts @@ -278,7 +278,8 @@ export class GenericService { } // Caching handled here - return this.fetchPages(pages, endpoint, options, clazz, auth) as Promise> + return this.fetchPages( + pages, endpoint, options, clazz, auth, parts) as Promise> } /** @@ -288,7 +289,7 @@ export class GenericService { */ public async fetchPages ( pages: number, endpoint: string, options: PaginatedRequestParams, clazz?: T, - auth?: AuthorizationOptions): + auth?: AuthorizationOptions, parts?: string[]): Promise>> { if (!clazz && endpoint !== 'search') { @@ -300,7 +301,7 @@ export class GenericService { } const cachedPages = this.youtube._shouldCache ? - Cache.getPages>(endpoint, options, auth) : + Cache.getPages>(endpoint, options, auth, parts, clazz) : undefined let pagesFetched = 0 @@ -361,7 +362,8 @@ export class GenericService { if (page.prevPageToken) toCache.prevPageToken = page.prevPageToken if (page.nextPageToken) toCache.nextPageToken = page.nextPageToken - this.youtube._cachePage(endpoint, pagesFetched, options, auth, toCache) + this.youtube._cachePage( + endpoint, pagesFetched, options, auth, parts, clazz, toCache) if (++pagesFetched >= pages || !page.nextPageToken) { if (page.prevPageToken) toReturn.prevPageToken = page.prevPageToken diff --git a/src/util/cache.ts b/src/util/cache.ts index 888a8a666..b3534ca9c 100644 --- a/src/util/cache.ts +++ b/src/util/cache.ts @@ -1,12 +1,25 @@ -import { AuthorizationOptions, ItemTypes, PaginatedInstance, PaginatedRequestParams, PaginatedResponse } from '..' +import { AuthorizationOptions, ItemTypes, PaginatedInstance, PaginatedRequestParams, PaginatedResponse, PaginatedType } from '..' /** * @ignore */ export class Cache { + /** + * The plan is to deprecate this global map as soon as the below maps are + * fully-featured. + */ private static map: Map = new Map() + + /** + * List of items that are separated by which parts were requested from the API. + */ private static itemsMap: Map>[]> = new Map() - private static pagesMap: Map>[]> = new Map() + + /** + * Map of pages of items that are separated by which parts were requested from the API. + */ + private static pagesMap: + Map>[]>> = new Map() public static set (key: string, value: any, ttl: number) { Cache.map.set(key, { v: value, t: ttl }) @@ -31,8 +44,9 @@ export class Cache { cachedItems.push({ v: value, t: ttl, - p: parts ? parts?.sort()?.join(',') : type.part + p: parts ? parts.sort().join(',') : type.part }) + Cache.itemsMap.set(key, cachedItems) } @@ -56,48 +70,38 @@ export class Cache { public static setPage (endpoint: string, page: number, options: PaginatedRequestParams, auth: AuthorizationOptions | undefined, + parts: string[] | undefined, type: PaginatedType | undefined, value: PaginatedResponse, ttl: number) { - const key = `${endpoint}/${JSON.stringify({ ...options, pageToken: undefined })}/${ + const part = parts ? parts.sort().join(',') : type?.part ?? 'default' + const cachedWithSameParts = Cache.pagesMap.get(part) ?? + Cache.pagesMap.set(part, new Map()).get(part) + + const key = `${endpoint}/${ + JSON.stringify({ ...options, pageToken: undefined, part: undefined })}/${ auth?.accessToken ?? false}/${auth?.apiKey ?? false}` - const toCache = Cache.pagesMap.get(key) ?? [] + const toCache = cachedWithSameParts.get(key) ?? [] if (page > toCache.length - 1) toCache.length = page + 1 toCache[page] = { v: value, t: ttl } - Cache.pagesMap.set(key, toCache) - } - - public static getPage ( - endpoint: string, page: number, options: PaginatedRequestParams, - auth: AuthorizationOptions | undefined): - PaginatedResponse { - const key = `${endpoint}/${JSON.stringify({ ...options, pageToken: undefined })}/${ - auth?.accessToken ?? false}/${auth?.apiKey ?? false}` - const pages = Cache.pagesMap.get(key) - const item = pages ? page > pages.length - 1 ? undefined : pages[page] : undefined - - if (!item) return undefined - - if (item.t > 0 && new Date().getTime() >= item.t) { - Cache._deletePage(endpoint, page, options, auth) - return undefined - } - - return item.v as PaginatedResponse + cachedWithSameParts.set(key, toCache) } public static getPages ( endpoint: string, options: PaginatedRequestParams, - auth: AuthorizationOptions | undefined): + auth?: AuthorizationOptions, parts?: string[], type?: PaginatedType): PaginatedResponse[] { - const key = `${endpoint}/${JSON.stringify({ ...options, pageToken: undefined })}/${ + const part = parts ? parts.sort().join(',') : type?.part ?? 'default' + const cachedWithSameParts = Cache.pagesMap.get(part) ?? + Cache.pagesMap.set(part, new Map()).get(part) + + const key = `${endpoint}/${ + JSON.stringify({ ...options, pageToken: undefined, part: undefined })}/${ auth?.accessToken ?? false}/${auth?.apiKey ?? false}` - const pages = Cache.pagesMap.get(key) + const pages = cachedWithSameParts.get(key) - if (!pages) { - return undefined - } + if (!pages) return undefined const toReturn: PaginatedResponse[] = new Array(pages.length) @@ -106,8 +110,8 @@ export class Cache { if (!item) continue if (item.t > 0 && new Date().getTime() >= item.t) { - Cache._deletePage(endpoint, page, options, auth) - return undefined + Cache._deletePage(endpoint, page, options, auth, parts, type) + continue } toReturn[page] = item.v as PaginatedResponse @@ -137,15 +141,17 @@ export class Cache { } } - for (let [ key, pages ] of Cache.pagesMap.entries()) { - for (let page = 0; page < pages.length; page++) { - if (!pages[page]) continue + for (const [ part, cache ] of Cache.pagesMap.entries()) { + for (let [ key, pages ] of cache.entries()) { + for (let page = 0; page < pages.length; page++) { + if (!pages[page]) continue - const timeToDelete = pages[page].t + const timeToDelete = pages[page].t - if (timeToDelete > 0 && time >= timeToDelete) { - pages = Cache._deletePageByKey(key, page) - page-- + if (timeToDelete > 0 && time >= timeToDelete) { + pages = Cache._deletePageByKey(part, key, page) + page-- + } } } } @@ -175,17 +181,19 @@ export class Cache { /** * @returns Pages left in the key */ - public static _deletePageByKey (key: string, page: number) { - const pages = Cache.pagesMap.get(key) + public static _deletePageByKey (parts: string, key: string, page: number) { + const cachedItems = Cache.pagesMap.get(parts) + const pages = cachedItems?.get(key) + if (!pages) return if (page > pages.length - 1) return pages pages.splice(page, 1) if (pages.length === 0) { - Cache.pagesMap.delete(key) + cachedItems.delete(key) } else { - Cache.pagesMap.set(key, pages) + cachedItems.set(key, pages) } return pages @@ -195,10 +203,15 @@ export class Cache { * @returns Pages left in the key */ public static _deletePage (endpoint: string, page: number, - options: PaginatedRequestParams, auth: AuthorizationOptions | undefined) { - const key = `${endpoint}/${JSON.stringify(options)}/${ + options: PaginatedRequestParams, auth?: AuthorizationOptions, parts?: string[], + type?: PaginatedType) { + + const part = parts ? parts.sort().join(',') : type?.part ?? 'default' + const key = `${endpoint}/${ + JSON.stringify({ ...options, pageToken: undefined, part: undefined })}/${ auth?.accessToken ?? false}/${auth?.apiKey ?? false}` - return Cache._deletePageByKey(key, page) + + return Cache._deletePageByKey(part, key, page) } } From 06a3763f647db2ce5650f7932d918abb8b8a2713 Mon Sep 17 00:00:00 2001 From: Brandon Bothell Date: Mon, 10 Jul 2023 14:53:46 -0400 Subject: [PATCH 04/10] fix(cache): set cacheCheckInterval correctly and fix tests --- src/index.ts | 16 +++++++++++----- src/util/cache.ts | 25 +++++++++++++++++++++++-- test/cache.spec.ts | 22 ++++++++++++++++++---- 3 files changed, 52 insertions(+), 11 deletions(-) diff --git a/src/index.ts b/src/index.ts index 8ecee7e4c..45c894216 100644 --- a/src/index.ts +++ b/src/index.ts @@ -31,6 +31,11 @@ export class YouTube { */ public _cacheTTL: number + /** + * @ignore + */ + public _cacheCheckInterval: number + /** * @ignore */ @@ -113,15 +118,16 @@ export class YouTube { this._upload = new Request('https://www.googleapis.com/upload/youtube/v3/', this.#auth) this.oauth = new OAuth(this) - this._shouldCache = options.cache - this._cacheSearches = options.cacheSearches - this._cacheTTL = options.cacheTTL + this._shouldCache = options.cache ?? true + this._cacheSearches = options.cacheSearches ?? true + this._cacheTTL = options.cacheTTL ?? 600 + this._cacheCheckInterval = options.cacheCheckInterval ?? 600 this.language = language this.region = region - if (options.cacheCheckInterval > 0) { - setInterval(Cache.checkTTLs, options.cacheCheckInterval * 1000) + if (this._cacheCheckInterval > 0) { + setInterval(Cache.checkTTLs, this._cacheCheckInterval * 1000) } } diff --git a/src/util/cache.ts b/src/util/cache.ts index b3534ca9c..78d58d453 100644 --- a/src/util/cache.ts +++ b/src/util/cache.ts @@ -11,7 +11,8 @@ export class Cache { private static map: Map = new Map() /** - * List of items that are separated by which parts were requested from the API. + * Items mapped to IDs/URLs/search queries that are separated + * by which parts were requested from the API. */ private static itemsMap: Map>[]> = new Map() @@ -147,7 +148,6 @@ export class Cache { if (!pages[page]) continue const timeToDelete = pages[page].t - if (timeToDelete > 0 && time >= timeToDelete) { pages = Cache._deletePageByKey(part, key, page) page-- @@ -157,6 +157,27 @@ export class Cache { } } + public static setTTLs (ttl: number) { + for (const [ key, item ] of Cache.map.entries()) { + Cache.map.set(key, { ...item, t: ttl }) + } + + for (const [ key, items ] of Cache.itemsMap.entries()) { + Cache.itemsMap.set(key, items.map(i => ({ ...i, t: ttl }))) + } + + for (const cache of Cache.pagesMap.values()) { + for (let [ key, pages ] of cache.entries()) { + for (let page = 0; page < pages.length; page++) { + if (!pages[page]) continue + + pages[page].t = ttl + cache.set(key, pages) + } + } + } + } + public static _delete (key: string) { Cache.map.delete(key) } diff --git a/test/cache.spec.ts b/test/cache.spec.ts index 9223d24f2..a58d8866a 100644 --- a/test/cache.spec.ts +++ b/test/cache.spec.ts @@ -1,6 +1,7 @@ import 'mocha' import './setup-instance' +import { setTimeout } from 'timers/promises' import { Cache } from '../src/util/cache' import { YouTube } from '../src' import { expect } from 'chai' @@ -36,13 +37,26 @@ describe('Caching', () => { }) it('should not use expired items', async () => { + Cache.setTTLs(1) // for clearing previous test cache + const youtube = new YouTube(apiKey, undefined, { cacheTTL: 0.001, cacheCheckInterval: 0.009 }) + await setTimeout(20) // clearing previous test cache + await youtube.getVideo('dQw4w9WgXcQ') + await setTimeout(20) // give enough time to clear cache - const time = new Date().getTime() - const video = youtube.getVideo('dQw4w9WgXcQ') + let time = new Date().getTime() + const video = await youtube.getVideo('dQw4w9WgXcQ') - await video + expect(new Date().getTime() - time).to.be.greaterThan(30) + + await video.fetchComments() + await setTimeout(20) + + time = new Date().getTime() + const comments = video.fetchComments() + + await comments expect(new Date().getTime() - time).to.be.greaterThan(30) }) @@ -70,7 +84,7 @@ describe('Caching', () => { expect(Cache.get('test')).to.equal(undefined) }) - it('should not cache if _shouldCalche is false', () => { + it('should not cache if _shouldCache is false', () => { const youtube = new YouTube(apiKey, undefined, { cache: false }) youtube._cache('test', 'value') From 8a740df0c71d5d71ef4d822e8283889d59ca54e3 Mon Sep 17 00:00:00 2001 From: Brandon Bothell Date: Mon, 10 Jul 2023 23:35:16 -0400 Subject: [PATCH 05/10] feat(cache): add partial part functionality to items --- src/util/arrays.ts | 11 +++++ src/util/cache.ts | 110 ++++++++++++++++++++++++------------------- test/channel.spec.ts | 24 ++++++++-- 3 files changed, 93 insertions(+), 52 deletions(-) diff --git a/src/util/arrays.ts b/src/util/arrays.ts index 68f8a7409..f1bde3c47 100644 --- a/src/util/arrays.ts +++ b/src/util/arrays.ts @@ -8,3 +8,14 @@ export function findArrayIndexFrom return -1 } + +export function findArrayFrom +(predicate: (element: T) => boolean, array: T[], startFrom: number): [number, T] { + for (let i = startFrom; i < array.length; i++) { + if (predicate(array[i])) { + return [i, array[i]] + } + } + + return undefined +} diff --git a/src/util/cache.ts b/src/util/cache.ts index 78d58d453..5f98c9f44 100644 --- a/src/util/cache.ts +++ b/src/util/cache.ts @@ -1,4 +1,6 @@ -import { AuthorizationOptions, ItemTypes, PaginatedInstance, PaginatedRequestParams, PaginatedResponse, PaginatedType } from '..' +import { AuthorizationOptions, ItemTypes, PaginatedInstance, + PaginatedRequestParams, PaginatedResponse, PaginatedType } from '..' +import { findArrayFrom } from '.' /** * @ignore @@ -11,13 +13,15 @@ export class Cache { private static map: Map = new Map() /** - * Items mapped to IDs/URLs/search queries that are separated - * by which parts were requested from the API. + * Map of items that are separated by which parts were requested from the API and then + * their ID/URL/search query. */ - private static itemsMap: Map>[]> = new Map() + private static itemsMap: + Map>>> = new Map() /** - * Map of pages of items that are separated by which parts were requested from the API. + * Map of pages of items that are separated by which parts were requested from the API + * and then a cache key generated from the request options. */ private static pagesMap: Map>[]>> = new Map() @@ -39,30 +43,25 @@ export class Cache { public static setItem (type: ItemTypes, name: string, parts: string[] | undefined, value: InstanceType, ttl: number) { - const key = `${type.name.toLowerCase()}/${name}` - const cachedItems = Cache.itemsMap.get(key) ?? [] - - cachedItems.push({ - v: value, - t: ttl, - p: parts ? parts.sort().join(',') : type.part - }) + const key = `${type.name.toLowerCase()}/${name}` // The item key string + const part = parts ? parts.sort().join(',') : type.part // The parts key string + const cachedWithSameParts = Cache.itemsMap.get(part) ?? // The parts map + Cache.itemsMap.set(part, new Map()).get(part) - Cache.itemsMap.set(key, cachedItems) + cachedWithSameParts.set(key, { v: value, t: ttl }) } public static getItem (type: ItemTypes, name: string, parts?: string[]): InstanceType { + const part = parts ? parts.sort().join(',') : type.part const key = `${type.name.toLowerCase()}/${name}` - const cachedItems = Cache.itemsMap.get(key) - const item = cachedItems?.find(parts ? - item => item.p?.startsWith(parts.sort().join(',')) : - item => item.p === type.part - ) + + // Search for an item that has the same or more parts than we need + const item = Cache.findCachedItemWithParts(Cache.itemsMap, part, key) if (!item) return undefined if (item.t > 0 && new Date().getTime() >= item.t) { - Cache._deleteItem(key, item.p) + Cache._deleteItem(key, part) return undefined } @@ -75,12 +74,12 @@ export class Cache { value: PaginatedResponse, ttl: number) { const part = parts ? parts.sort().join(',') : type?.part ?? 'default' - const cachedWithSameParts = Cache.pagesMap.get(part) ?? - Cache.pagesMap.set(part, new Map()).get(part) - const key = `${endpoint}/${ JSON.stringify({ ...options, pageToken: undefined, part: undefined })}/${ auth?.accessToken ?? false}/${auth?.apiKey ?? false}` + + const cachedWithSameParts = Cache.pagesMap.get(part) ?? + Cache.pagesMap.set(part, new Map()).get(part) const toCache = cachedWithSameParts.get(key) ?? [] if (page > toCache.length - 1) toCache.length = page + 1 @@ -93,14 +92,13 @@ export class Cache { endpoint: string, options: PaginatedRequestParams, auth?: AuthorizationOptions, parts?: string[], type?: PaginatedType): PaginatedResponse[] { - const part = parts ? parts.sort().join(',') : type?.part ?? 'default' - const cachedWithSameParts = Cache.pagesMap.get(part) ?? - Cache.pagesMap.set(part, new Map()).get(part) - const key = `${endpoint}/${ JSON.stringify({ ...options, pageToken: undefined, part: undefined })}/${ auth?.accessToken ?? false}/${auth?.apiKey ?? false}` - const pages = cachedWithSameParts.get(key) + const part = parts ? parts.sort().join(',') : type?.part ?? 'default' + + // Search for a page that has the same or more parts than we need + const pages = Cache.findCachedItemWithParts(Cache.pagesMap, part, key) if (!pages) return undefined @@ -121,6 +119,30 @@ export class Cache { return toReturn } + private static findCachedItemWithParts (cache: Map>, + part: string, key: string): T { + let cachedWithSameParts = cache.get(part) // A map with matching parts + let item = cachedWithSameParts?.get(key) // A matching page with our parts + + if (item) return item + + let currentIndex = 0 + const partKeys = Array.from(cache.keys()) + + while (!item) { + // Find a page that contains our data + const matchingPart = findArrayFrom(p => p.includes(part), + partKeys, currentIndex) + if (!matchingPart) break + + currentIndex = matchingPart[0] + 1 // Increment index + cachedWithSameParts = cache.get(matchingPart[1]) + item = cachedWithSameParts?.get(key) + } + + return item + } + public static checkTTLs () { const time = new Date().getTime() @@ -132,12 +154,12 @@ export class Cache { } } - for (const [ key, items ] of Cache.itemsMap.entries()) { - for (const item of items) { + for (const [ part, cache ] of Cache.itemsMap.entries()) { + for (const [ key, item ] of cache.entries()) { const timeToDelete = item.t if (timeToDelete > 0 && time >= timeToDelete) { - Cache._deleteItem(key, item.p) + Cache._deleteItem(key, part) } } } @@ -162,8 +184,10 @@ export class Cache { Cache.map.set(key, { ...item, t: ttl }) } - for (const [ key, items ] of Cache.itemsMap.entries()) { - Cache.itemsMap.set(key, items.map(i => ({ ...i, t: ttl }))) + for (const cache of Cache.itemsMap.values()) { + for (let [ key, item ] of cache.entries()) { + cache.set(key, { ...item, t: ttl }) + } } for (const cache of Cache.pagesMap.values()) { @@ -182,21 +206,12 @@ export class Cache { Cache.map.delete(key) } - public static _deleteItem (key: string, parts?: string) { - const cachedItems = Cache.itemsMap.get(key) - const index = cachedItems?.findIndex(item => item.p === parts) - - if (index === undefined || index < 0) return - - cachedItems.splice(index, 1) - - if (cachedItems.length === 0) { - Cache.itemsMap.delete(key) - } else { - Cache.itemsMap.set(key, cachedItems) - } + public static _deleteItem (key: string, parts: string) { + const cachedWithSameParts = Cache.itemsMap.get(parts) + const cachedItem = cachedWithSameParts?.get(key) - return cachedItems + cachedWithSameParts?.delete(key) + return cachedItem } /** @@ -245,5 +260,4 @@ export class Cache { type CacheItem = { v: T t: number - p?: string } diff --git a/test/channel.spec.ts b/test/channel.spec.ts index eee65908c..ec5d3a8f9 100644 --- a/test/channel.spec.ts +++ b/test/channel.spec.ts @@ -1,5 +1,5 @@ import 'mocha' -import { Channel, Playlist, Subscription, ChannelSection } from '../src' +import { Channel, Playlist, Subscription, ChannelSection, PaginatedResponse } from '../src' import { youtube } from './setup-instance' import { expect } from 'chai' @@ -49,17 +49,33 @@ describe('Channels', () => { it('should work with fetching pages of playlists', async () => { const channel = await youtube.getChannel('UCBR8-60-B28hp2BmDPdntcQ', [ 'id' ]) - expect((await channel.fetchPlaylists({ pages: 2 }, [ 'id' ])).items.length).to.equal(100) + expect((await channel.fetchPlaylists({ pages: 2 }, [ 'id' ])) + .items.length).to.equal(100) }) it('should work with fetching playlists', async () => { const channel = await youtube.getChannel('UC6mi9rp7vRYninucP61qOjg', [ 'id' ]) - expect((await channel.fetchPlaylists(undefined, [ 'id' ])).items[0]).to.be.an.instanceOf(Playlist) + expect((await channel.fetchPlaylists(undefined, [ 'id', 'contentDetails' ])) + .items[0]).to.be.an.instanceOf(Playlist) + }) + + it('should work with caching playlists', async () => { + const channel = await youtube.getChannel('UC6mi9rp7vRYninucP61qOjg', [ 'id' ]) + let playlist: Playlist | Promise> = + channel.fetchPlaylists(undefined, [ 'contentDetails' ]) + const time = new Date().getTime() + + playlist = (await playlist).items[0] + + expect(new Date().getTime() - time).to.be.lessThan(50) + expect(playlist).to.be.an.instanceOf(Playlist) + expect(playlist.length).to.be.a('number') }) it('should work with fetching pages of subscriptions', async () => { const channel = await youtube.getChannel('UCg4XK-l40KZD7fLi12pJ1YA', [ 'id' ]) - expect((await channel.fetchSubscriptions({ pages: 1 }, [ 'id' ])).items[0]).to.be.an.instanceOf(Subscription) + expect((await channel.fetchSubscriptions({ pages: 1 }, [ 'id' ])) + .items[0]).to.be.an.instanceOf(Subscription) }) it('should work with fetching sections', async () => { From 905907b0b628c193eab3f49b537901f5b692f1cb Mon Sep 17 00:00:00 2001 From: Brandon Bothell Date: Tue, 25 Jul 2023 14:46:01 -0400 Subject: [PATCH 06/10] feat: cache video ratings, seperate them into own class, and start standardizing oauth method requests --- src/entities/index.ts | 1 + src/entities/playlist.ts | 13 ++++-- src/entities/video-rating.ts | 75 ++++++++++++++++++++++++++++++ src/entities/video.ts | 4 +- src/index.ts | 20 ++++++-- src/oauth/index.ts | 9 ++-- src/oauth/videos.ts | 50 +++++++++++++++----- src/services/generic-service.ts | 30 ++++++++---- src/services/resolution-service.ts | 10 ++-- src/types/GetItem.ts | 6 ++- src/types/GetPaginatedItems.ts | 7 ++- src/util/cache.ts | 31 +++++++++--- test/oauth/videos.spec.ts | 9 ++++ 13 files changed, 219 insertions(+), 46 deletions(-) create mode 100644 src/entities/video-rating.ts diff --git a/src/entities/index.ts b/src/entities/index.ts index b1a8636ae..a52f6b82f 100644 --- a/src/entities/index.ts +++ b/src/entities/index.ts @@ -4,6 +4,7 @@ */ export * from './video' +export * from './video-rating' export * from './channel' export * from './playlist' export * from './comment' diff --git a/src/entities/playlist.ts b/src/entities/playlist.ts index 82610bc9f..71eae1493 100644 --- a/src/entities/playlist.ts +++ b/src/entities/playlist.ts @@ -239,15 +239,22 @@ export class Playlist { */ public async removeVideo (videoResolvable: VideoResolvable) { const video = await this.youtube._resolutionService.resolve(videoResolvable, Video) - const playlistItemId = (this.youtube._genericService.getPaginatedItems({ + const matchingItems = (await this.youtube._genericService.getPaginatedItems