From 09f999adaf3d7501dfd483acb48b15645cdb2cc8 Mon Sep 17 00:00:00 2001 From: tusharmctrl Date: Wed, 30 Apr 2025 14:22:36 +0530 Subject: [PATCH 1/8] Add support for running automation for mobile devices to check UI/UX --- src/agent/index.ts | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/agent/index.ts b/src/agent/index.ts index edf93d3..a00f2eb 100644 --- a/src/agent/index.ts +++ b/src/agent/index.ts @@ -1,6 +1,6 @@ import { BaseChatModel } from "@langchain/core/language_models/chat_models"; import { ChatOpenAI } from "@langchain/openai"; -import { Browser, BrowserContext, Page } from "playwright"; +import { Browser, BrowserContext, devices, Page } from "playwright"; import { v4 as uuidv4 } from "uuid"; import { @@ -40,6 +40,7 @@ export class HyperAgent { private tokenLimit = 128000; private debug = false; private mcpClient: MCPClient | undefined; + private useMobile: boolean = false; private browserProvider: T extends "Hyperbrowser" ? HyperbrowserProvider : LocalBrowserProvider; @@ -75,6 +76,7 @@ export class HyperAgent { } else { this.llm = params.llm; } + this.useMobile = params?.useMobile ?? false; this.browserProviderType = (params.browserProvider ?? "Local") as T; this.browserProvider = ( @@ -99,10 +101,22 @@ export class HyperAgent { */ public async initBrowser(): Promise { if (!this.browser) { + const iphone12 = devices["iPhone 12"]; + this.browser = await this.browserProvider.start(); - this.context = await this.browser.newContext({ - viewport: null, - }); + this.context = await this.browser.newContext( + this.useMobile + ? { + userAgent: iphone12.userAgent, + viewport: { + width: iphone12.viewport.width, + height: iphone12.viewport.height + 100, + }, + } + : { + viewport: null, + } + ); // Inject script to track event listeners await this.context.addInitScript(() => { From b64214ea96eeb961da12285d0ba1f7b0dab89e3b Mon Sep 17 00:00:00 2001 From: tusharmctrl Date: Wed, 30 Apr 2025 14:25:19 +0530 Subject: [PATCH 2/8] Added missing property in a type --- src/types/config.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/types/config.ts b/src/types/config.ts index 8ea80bf..0120d0a 100644 --- a/src/types/config.ts +++ b/src/types/config.ts @@ -68,5 +68,6 @@ export interface HyperAgentConfig { NonNullable[0]>, "debug" >; + useMobile?: boolean; localConfig?: ConstructorParameters[0]; } From 4ffaa462d271c479251bd78fc79144a2ad751f6b Mon Sep 17 00:00:00 2001 From: tusharmctrl Date: Thu, 1 May 2025 17:26:51 +0530 Subject: [PATCH 3/8] feat: add device support in localConfig --- src/browser-providers/local.ts | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/src/browser-providers/local.ts b/src/browser-providers/local.ts index 976c772..938bc5b 100644 --- a/src/browser-providers/local.ts +++ b/src/browser-providers/local.ts @@ -1,13 +1,21 @@ -import { chromium, Browser, LaunchOptions } from "playwright"; +import { chromium, Browser, LaunchOptions, devices } from "playwright"; import BrowserProvider from "@/types/browser-providers/types"; +type LocalBrowserProviderOptions = + | (Omit, "channel"> & { + device?: keyof typeof devices; + }) + | undefined; + export class LocalBrowserProvider extends BrowserProvider { - options: Omit, "channel"> | undefined; + options: LocalBrowserProviderOptions; session: Browser | undefined; - constructor(options?: Omit, "channel">) { + + constructor(options?: LocalBrowserProviderOptions) { super(); this.options = options; } + async start(): Promise { const launchArgs = this.options?.args ?? []; const browser = await chromium.launch({ @@ -16,7 +24,21 @@ export class LocalBrowserProvider extends BrowserProvider { headless: false, args: ["--disable-blink-features=AutomationControlled", ...launchArgs], }); + this.session = browser; + if (this?.options?.device) { + const defaultContexts = browser.contexts(); + for (const context of defaultContexts) { + await context.close(); + } + + const newContext = await browser.newContext({ + ...devices[this.options.device], + }); + + await newContext.newPage(); + } + return this.session; } async close(): Promise { From fb498f841e1afcc5cb98f18e75dfc596d95f058a Mon Sep 17 00:00:00 2001 From: tusharmctrl Date: Thu, 1 May 2025 17:29:40 +0530 Subject: [PATCH 4/8] fix: remove unwanted type --- src/types/config.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/types/config.ts b/src/types/config.ts index 0120d0a..8ea80bf 100644 --- a/src/types/config.ts +++ b/src/types/config.ts @@ -68,6 +68,5 @@ export interface HyperAgentConfig { NonNullable[0]>, "debug" >; - useMobile?: boolean; localConfig?: ConstructorParameters[0]; } From 0f6ec535d12813cfd2def957cce45fd9d2df8a89 Mon Sep 17 00:00:00 2001 From: tusharmctrl Date: Thu, 1 May 2025 17:31:11 +0530 Subject: [PATCH 5/8] remove unwanted code from HyperbrowserProvider --- src/agent/index.ts | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/src/agent/index.ts b/src/agent/index.ts index a00f2eb..7c8434e 100644 --- a/src/agent/index.ts +++ b/src/agent/index.ts @@ -1,6 +1,6 @@ import { BaseChatModel } from "@langchain/core/language_models/chat_models"; import { ChatOpenAI } from "@langchain/openai"; -import { Browser, BrowserContext, devices, Page } from "playwright"; +import { Browser, BrowserContext, Page } from "playwright"; import { v4 as uuidv4 } from "uuid"; import { @@ -76,7 +76,6 @@ export class HyperAgent { } else { this.llm = params.llm; } - this.useMobile = params?.useMobile ?? false; this.browserProviderType = (params.browserProvider ?? "Local") as T; this.browserProvider = ( @@ -101,22 +100,10 @@ export class HyperAgent { */ public async initBrowser(): Promise { if (!this.browser) { - const iphone12 = devices["iPhone 12"]; - this.browser = await this.browserProvider.start(); - this.context = await this.browser.newContext( - this.useMobile - ? { - userAgent: iphone12.userAgent, - viewport: { - width: iphone12.viewport.width, - height: iphone12.viewport.height + 100, - }, - } - : { - viewport: null, - } - ); + this.context = await this.browser.newContext({ + viewport: null, + }); // Inject script to track event listeners await this.context.addInitScript(() => { From e066278dffc53391f4864505f8f1033eabd7f5a3 Mon Sep 17 00:00:00 2001 From: tusharmctrl Date: Thu, 1 May 2025 17:31:55 +0530 Subject: [PATCH 6/8] final cleanup of useMobile --- src/agent/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/agent/index.ts b/src/agent/index.ts index 7c8434e..edf93d3 100644 --- a/src/agent/index.ts +++ b/src/agent/index.ts @@ -40,7 +40,6 @@ export class HyperAgent { private tokenLimit = 128000; private debug = false; private mcpClient: MCPClient | undefined; - private useMobile: boolean = false; private browserProvider: T extends "Hyperbrowser" ? HyperbrowserProvider : LocalBrowserProvider; From 30f4cb24bbaf9e8157e87d2b9486026870728fbd Mon Sep 17 00:00:00 2001 From: tusharmctrl Date: Wed, 7 May 2025 18:59:38 +0530 Subject: [PATCH 7/8] feat: added new param --- src/agent/index.ts | 12 +++++-- src/browser-providers/hyperbrowser.ts | 23 ++++++++++++- src/browser-providers/local.ts | 49 ++++++++++++++------------- src/types/browser-providers/types.ts | 5 +-- src/types/config.ts | 2 +- 5 files changed, 61 insertions(+), 30 deletions(-) diff --git a/src/agent/index.ts b/src/agent/index.ts index edf93d3..8e52181 100644 --- a/src/agent/index.ts +++ b/src/agent/index.ts @@ -45,6 +45,7 @@ export class HyperAgent { : LocalBrowserProvider; private browserProviderType: T; private actions: Array = [...DEFAULT_ACTIONS]; + private clientType: "desktop" | "mobile"; public browser: Browser | null = null; public context: BrowserContext | null = null; @@ -76,6 +77,7 @@ export class HyperAgent { this.llm = params.llm; } this.browserProviderType = (params.browserProvider ?? "Local") as T; + this.clientType = params.clientType ?? "desktop"; this.browserProvider = ( this.browserProviderType === "Hyperbrowser" @@ -100,9 +102,13 @@ export class HyperAgent { public async initBrowser(): Promise { if (!this.browser) { this.browser = await this.browserProvider.start(); - this.context = await this.browser.newContext({ - viewport: null, - }); + this.context = await this.browserProvider.getContext(this.clientType); + + if (!this.context) { + this.context = await this.browser.newContext({ + viewport: null, + }); + } // Inject script to track event listeners await this.context.addInitScript(() => { diff --git a/src/browser-providers/hyperbrowser.ts b/src/browser-providers/hyperbrowser.ts index 78084d7..a84a3b3 100644 --- a/src/browser-providers/hyperbrowser.ts +++ b/src/browser-providers/hyperbrowser.ts @@ -1,4 +1,10 @@ -import { chromium, Browser, ConnectOverCDPOptions } from "playwright"; +import { + chromium, + Browser, + ConnectOverCDPOptions, + BrowserContext, + devices, +} from "playwright"; import { Hyperbrowser } from "@hyperbrowser/sdk"; import { CreateSessionParams, @@ -62,6 +68,21 @@ export class HyperbrowserProvider extends BrowserProvider { } } + public async getContext( + device: string = "desktop" + ): Promise { + if (!this.browser) return null; + + if (device === "mobile") { + const iPhone = devices["iPhone 12"]; + return await this.browser.newContext({ + ...iPhone, + }); + } + + return await this.browser.newContext(); + } + public getSession() { if (!this.session) { return null; diff --git a/src/browser-providers/local.ts b/src/browser-providers/local.ts index 938bc5b..7452fb1 100644 --- a/src/browser-providers/local.ts +++ b/src/browser-providers/local.ts @@ -1,17 +1,17 @@ -import { chromium, Browser, LaunchOptions, devices } from "playwright"; +import { + chromium, + Browser, + LaunchOptions, + devices, + BrowserContext, +} from "playwright"; import BrowserProvider from "@/types/browser-providers/types"; -type LocalBrowserProviderOptions = - | (Omit, "channel"> & { - device?: keyof typeof devices; - }) - | undefined; - export class LocalBrowserProvider extends BrowserProvider { - options: LocalBrowserProviderOptions; + options: Omit, "channel"> | undefined; session: Browser | undefined; - constructor(options?: LocalBrowserProviderOptions) { + constructor(options?: Omit, "channel">) { super(); this.options = options; } @@ -24,30 +24,33 @@ export class LocalBrowserProvider extends BrowserProvider { headless: false, args: ["--disable-blink-features=AutomationControlled", ...launchArgs], }); - this.session = browser; - if (this?.options?.device) { - const defaultContexts = browser.contexts(); - for (const context of defaultContexts) { - await context.close(); - } - - const newContext = await browser.newContext({ - ...devices[this.options.device], - }); - - await newContext.newPage(); - } - return this.session; } + async close(): Promise { return await this.session?.close(); } + public getSession() { if (!this.session) { return null; } return this.session; } + + public async getContext( + device: string = "desktop" + ): Promise { + if (!this.session) return null; + + if (device === "mobile") { + const iPhone = devices["iPhone 12"]; + return await this.session.newContext({ + ...iPhone, + }); + } + + return await this.session.newContext(); + } } diff --git a/src/types/browser-providers/types.ts b/src/types/browser-providers/types.ts index f01e62a..3e33e4e 100644 --- a/src/types/browser-providers/types.ts +++ b/src/types/browser-providers/types.ts @@ -1,10 +1,11 @@ -import { Browser } from "playwright"; +import { Browser, BrowserContext } from "playwright"; abstract class BrowserProvider { abstract session: unknown; abstract start(): Promise; abstract close(): Promise; - abstract getSession(): T|null; + abstract getSession(): T | null; + abstract getContext(device?: string): Promise; } export default BrowserProvider; diff --git a/src/types/config.ts b/src/types/config.ts index 8ea80bf..f690130 100644 --- a/src/types/config.ts +++ b/src/types/config.ts @@ -63,7 +63,7 @@ export interface HyperAgentConfig { debug?: boolean; llm?: BaseChatModel; - + clientType?: "mobile" | "desktop"; hyperbrowserConfig?: Omit< NonNullable[0]>, "debug" From 436f3627afb686f2466609474b5fcfffbf085538 Mon Sep 17 00:00:00 2001 From: tusharmctrl Date: Wed, 7 May 2025 19:14:54 +0530 Subject: [PATCH 8/8] fix: resolved sharp error about deviceFactor --- src/browser-providers/hyperbrowser.ts | 6 +++++- src/browser-providers/local.ts | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/browser-providers/hyperbrowser.ts b/src/browser-providers/hyperbrowser.ts index a84a3b3..63bcd14 100644 --- a/src/browser-providers/hyperbrowser.ts +++ b/src/browser-providers/hyperbrowser.ts @@ -76,7 +76,11 @@ export class HyperbrowserProvider extends BrowserProvider { if (device === "mobile") { const iPhone = devices["iPhone 12"]; return await this.browser.newContext({ - ...iPhone, + userAgent: iPhone.userAgent, + viewport: { + width: iPhone.viewport.width + 50, + height: iPhone.viewport.height + 50, + }, }); } diff --git a/src/browser-providers/local.ts b/src/browser-providers/local.ts index 7452fb1..3b50db5 100644 --- a/src/browser-providers/local.ts +++ b/src/browser-providers/local.ts @@ -47,7 +47,11 @@ export class LocalBrowserProvider extends BrowserProvider { if (device === "mobile") { const iPhone = devices["iPhone 12"]; return await this.session.newContext({ - ...iPhone, + userAgent: iPhone.userAgent, + viewport: { + width: iPhone.viewport.width + 50, + height: iPhone.viewport.height + 50, + }, }); }