feat(fetch): store cookies between requests

src/server/cookieStore.ts

@@ -0,0 +1,133 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as types from './types';
+
+class Cookie {
+  private _raw: types.NetworkCookie;
+  constructor(data: types.NetworkCookie) {
+    this._raw = data;
+  }
+
+  name(): string {
+    return this._raw.name;
+  }
+
+  // https://datatracker.ietf.org/doc/html/rfc6265#section-5.4
+  matches(url: URL): boolean {
+    if (this._raw.secure && url.protocol !== 'https:')
+      return false;
+    if (!domainMatches(url.hostname, this._raw.domain))
+      return false;
+    if (!pathMatches(url.pathname, this._raw.path))
+      return false;
+    return true;
+  }
+
+  equals(other: Cookie) {
+    return this._raw.name === other._raw.name &&
+      this._raw.domain === other._raw.domain &&
+      this._raw.path === other._raw.path;
+  }
+
+  networkCookie(): types.NetworkCookie {
+    return this._raw;
+  }
+
+  updateExpiresFrom(other: Cookie) {
+    this._raw.expires = other._raw.expires;
+  }
+
+  expired() {
+    if (this._raw.expires === -1)
+      return false;
+    return this._raw.expires * 1000 < Date.now();
+  }
+}
+
+export class CookieStore {
+  private readonly _nameToCookies: Map<string, Set<Cookie>> = new Map();
+
+  addCookies(cookies: types.NetworkCookie[]) {
+    for (const cookie of cookies)
+      this._addCookie(new Cookie(cookie));
+  }
+
+  cookies(url: URL): types.NetworkCookie[] {
+    const result = [];
+    for (const cookie of this._allCookies()) {
+      if (cookie.matches(url))
+        result.push(cookie.networkCookie());
+    }
+    return result;
+  }
+
+  private _addCookie(cookie: Cookie) {
+    if (cookie.expired())
+      return;
+    let set = this._nameToCookies.get(cookie.name());
+    if (!set) {
+      set = new Set();
+      this._nameToCookies.set(cookie.name(), set);
+    }
+    CookieStore.pruneExpired(set);
+    // https://datatracker.ietf.org/doc/html/rfc6265#section-5.3
+    for (const other of set) {
+      if (other.equals(cookie)) {
+        cookie.updateExpiresFrom(other);
+        set.delete(other);
+      }
+    }
+    set.add(cookie);
+  }
+
+  private *_allCookies(): IterableIterator<Cookie> {
+    for (const [name, cookies] of this._nameToCookies) {
+      CookieStore.pruneExpired(cookies);
+      for (const cookie of cookies)
+        yield cookie;
+      if (cookies.size === 0)
+        this._nameToCookies.delete(name);
+    }
+  }
+
+  private static pruneExpired(cookies: Set<Cookie>) {
+    for (const cookie of cookies) {
+      if (cookie.expired())
+        cookies.delete(cookie);
+    }
+  }
+}
+
+export function domainMatches(value: string, domain: string): boolean {
+  if (value === domain)
+    return true;
+  // Only strict match is allowed if domain doesn't start with '.' (host-only-flag is true in the spec)
+  if (!domain.startsWith('.'))
+    return false;
+  value = '.' + value;
+  return value.endsWith(domain);
+}
+
+function pathMatches(value: string, path: string): boolean {
+  if (value === path)
+    return true;
+  if (!value.endsWith('/'))
+    value = value + '/';
+  if (!path.endsWith('/'))
+    path = path + '/';
+  return value.startsWith(path);
+}

src/server/fetch.ts

@@ -23,8 +23,9 @@ import zlib from 'zlib';
 import { HTTPCredentials } from '../../types/types';
 import { NameValue, NewRequestOptions } from '../common/types';
 import { TimeoutSettings } from '../utils/timeoutSettings';
-import { createGuid, getPlaywrightVersion, isFilePayload, monotonicTime } from '../utils/utils';
+import { assert, createGuid, getPlaywrightVersion, isFilePayload, monotonicTime } from '../utils/utils';
 import { BrowserContext } from './browserContext';
+import { CookieStore, domainMatches } from './cookieStore';
 import { MultipartFormData } from './formData';
 import { SdkObject } from './instrumentation';
 import { Playwright } from './playwright';
@@ -73,8 +74,8 @@ export abstract class FetchRequest extends SdkObject {
   abstract dispose(): void;
 
   abstract _defaultOptions(): FetchRequestOptions;
-  abstract _addCookies(cookies: types.SetNetworkCookieParam[]): Promise<void>;
-  abstract _cookies(url: string): Promise<types.NetworkCookie[]>;
+  abstract _addCookies(cookies: types.NetworkCookie[]): Promise<void>;
+  abstract _cookies(url: URL): Promise<types.NetworkCookie[]>;
 
   private _storeResponseBody(body: Buffer): string {
     const uid = createGuid();
@@ -155,15 +156,18 @@ export abstract class FetchRequest extends SdkObject {
     const url = new URL(responseUrl);
     // https://datatracker.ietf.org/doc/html/rfc6265#section-5.1.4
     const defaultPath = '/' + url.pathname.substr(1).split('/').slice(0, -1).join('/');
-    const cookies: types.SetNetworkCookieParam[] = [];
+    const cookies: types.NetworkCookie[] = [];
     for (const header of setCookie) {
       // Decode cookie value?
-      const cookie: types.SetNetworkCookieParam | null = parseCookie(header);
+      const cookie: types.NetworkCookie | null = parseCookie(header);
       if (!cookie)
         continue;
+      // https://datatracker.ietf.org/doc/html/rfc6265#section-5.2.3
       if (!cookie.domain)
         cookie.domain = url.hostname;
-      if (!canSetCookie(cookie.domain!, url.hostname))
+      else
+        assert(cookie.domain.startsWith('.'));
+      if (!domainMatches(url.hostname, cookie.domain!))
         continue;
       // https://datatracker.ietf.org/doc/html/rfc6265#section-5.2.4
       if (!cookie.path || !cookie.path.startsWith('/'))
@@ -177,7 +181,7 @@ export abstract class FetchRequest extends SdkObject {
   private async _updateRequestCookieHeader(url: URL, options: http.RequestOptions) {
     if (options.headers!['cookie'] !== undefined)
       return;
-    const cookies = await this._cookies(url.toString());
+    const cookies = await this._cookies(url);
     if (cookies.length) {
       const valueArray = cookies.map(c => `${c.name}=${c.value}`);
       options.headers!['cookie'] = valueArray.join('; ');
@@ -326,17 +330,18 @@ export class BrowserContextFetchRequest extends FetchRequest {
     };
   }
 
-  async _addCookies(cookies: types.SetNetworkCookieParam[]): Promise<void> {
+  async _addCookies(cookies: types.NetworkCookie[]): Promise<void> {
     await this._context.addCookies(cookies);
   }
 
-  async _cookies(url: string): Promise<types.NetworkCookie[]> {
-    return await this._context.cookies(url);
+  async _cookies(url: URL): Promise<types.NetworkCookie[]> {
+    return await this._context.cookies(url.toString());
   }
 }
 
 
 export class GlobalFetchRequest extends FetchRequest {
+  private readonly _cookieStore: CookieStore = new CookieStore();
   private readonly _options: FetchRequestOptions;
   constructor(playwright: Playwright, options: Omit<NewRequestOptions, 'extraHTTPHeaders'> & { extraHTTPHeaders?: NameValue[] }) {
     super(playwright);
@@ -370,11 +375,12 @@ export class GlobalFetchRequest extends FetchRequest {
     return this._options;
   }
 
-  async _addCookies(cookies: types.SetNetworkCookieParam[]): Promise<void> {
+  async _addCookies(cookies: types.NetworkCookie[]): Promise<void> {
+    this._cookieStore.addCookies(cookies);
   }
 
-  async _cookies(url: string): Promise<types.NetworkCookie[]> {
-    return [];
+  async _cookies(url: URL): Promise<types.NetworkCookie[]> {
+    return this._cookieStore.cookies(url);
   }
 }
 
@@ -387,15 +393,7 @@ function toHeadersArray(rawHeaders: string[]): types.HeadersArray {
 
 const redirectStatus = [301, 302, 303, 307, 308];
 
-function canSetCookie(cookieDomain: string, hostname: string) {
-  // TODO: check public suffix list?
-  hostname = '.' + hostname;
-  if (!cookieDomain.startsWith('.'))
-    cookieDomain = '.' + cookieDomain;
-  return hostname.endsWith(cookieDomain);
-}
-
-function parseCookie(header: string) {
+function parseCookie(header: string): types.NetworkCookie | null {
   const pairs = header.split(';').filter(s => s.trim().length > 0).map(p => p.split('=').map(s => s.trim()));
   if (!pairs.length)
     return null;
@@ -424,7 +422,9 @@ function parseCookie(header: string) {
           cookie.expires = Date.now() / 1000 + maxAgeSec;
         break;
       case 'domain':
-        cookie.domain = value || '';
+        cookie.domain = value.toLocaleLowerCase() || '';
+        if (cookie.domain && !cookie.domain.startsWith('.'))
+          cookie.domain = '.' + cookie.domain;
         break;
       case 'path':
         cookie.path = value || '';