diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..111cf31 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,33 @@ +name: build-and-test + +on: + pull_request: + branches: [ main ] + push: + branches: [ main ] + +env: + DOTNET_VERSION: '10.0.x' + +jobs: + build-and-test: + name: build-and-test + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v5 + + - name: Setup .NET + uses: actions/setup-dotnet@v4 + with: + dotnet-version: ${{ env.DOTNET_VERSION }} + + - name: Restore dependencies + run: dotnet restore ManagedCode.Playwright.Stealth.sln + + - name: Build + run: dotnet build ManagedCode.Playwright.Stealth.sln --configuration Release --no-restore + + - name: Test + run: dotnet test --solution ManagedCode.Playwright.Stealth.sln --configuration Release --no-build --verbosity normal diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bc78471 --- /dev/null +++ b/.gitignore @@ -0,0 +1,484 @@ +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. +## +## Get latest from `dotnet new gitignore` + +# dotenv files +.env + +# User-specific files +*.rsuser +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Mono auto generated files +mono_crash.* + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +[Ww][Ii][Nn]32/ +[Aa][Rr][Mm]/ +[Aa][Rr][Mm]64/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ +[Ll]ogs/ + +# Visual Studio 2015/2017 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# Visual Studio 2017 auto generated files +Generated\ Files/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUnit +*.VisualState.xml +TestResult.xml +nunit-*.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# Benchmark Results +BenchmarkDotNet.Artifacts/ + +# .NET +project.lock.json +project.fragment.lock.json +artifacts/ + +# Tye +.tye/ + +# ASP.NET Scaffolding +ScaffoldingReadMe.txt + +# StyleCop +StyleCopReport.xml + +# Files built by Visual Studio +*_i.c +*_p.c +*_h.h +*.ilk +*.meta +*.obj +*.iobj +*.pch +*.pdb +*.ipdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*_wpftmp.csproj +*.log +*.tlog +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# Visual Studio Trace Files +*.e2e + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# AxoCover is a Code Coverage Tool +.axoCover/* +!.axoCover/settings.json + +# Coverlet is a free, cross platform Code Coverage Tool +coverage*.json +coverage*.xml +coverage*.info + +# Visual Studio code coverage results +*.coverage +*.coveragexml + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# Note: Comment the next line if you want to checkin your web deploy settings, +# but database connection strings (with potential passwords) will be unencrypted +*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# NuGet Symbol Packages +*.snupkg +# The packages folder can be ignored because of Package Restore +**/[Pp]ackages/* +# except build/, which is used as an MSBuild target. +!**/[Pp]ackages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/[Pp]ackages/repositories.config +# NuGet v3's project.json files produces more ignorable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt +*.appx +*.appxbundle +*.appxupload + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!?*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +orleans.codegen.cs + +# Including strong name files can present a security risk +# (https://github.com/github/gitignore/pull/2483#issue-259490424) +#*.snk + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm +ServiceFabricBackup/ +*.rptproj.bak + +# SQL Server files +*.mdf +*.ldf +*.ndf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings +*.rptproj.rsuser +*- [Bb]ackup.rdl +*- [Bb]ackup ([0-9]).rdl +*- [Bb]ackup ([0-9][0-9]).rdl + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat +node_modules/ + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) +*.vbw + +# Visual Studio 6 auto-generated project file (contains which files were open etc.) +*.vbp + +# Visual Studio 6 workspace and project file (working project files containing files to include in project) +*.dsw +*.dsp + +# Visual Studio 6 technical files +*.ncb +*.aps + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# CodeRush personal settings +.cr/personal + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc + +# Cake - Uncomment if you are using it +# tools/** +# !tools/packages.config + +# Tabs Studio +*.tss + +# Telerik's JustMock configuration file +*.jmconfig + +# BizTalk build output +*.btp.cs +*.btm.cs +*.odx.cs +*.xsd.cs + +# OpenCover UI analysis results +OpenCover/ + +# Azure Stream Analytics local run output +ASALocalRun/ + +# MSBuild Binary and Structured Log +*.binlog + +# NVidia Nsight GPU debugger configuration file +*.nvuser + +# MFractors (Xamarin productivity tool) working folder +.mfractor/ + +# Local History for Visual Studio +.localhistory/ + +# Visual Studio History (VSHistory) files +.vshistory/ + +# BeatPulse healthcheck temp database +healthchecksdb + +# Backup folder for Package Reference Convert tool in Visual Studio 2017 +MigrationBackup/ + +# Ionide (cross platform F# VS Code tools) working folder +.ionide/ + +# Fody - auto-generated XML schema +FodyWeavers.xsd + +# VS Code files for those working on multiple tools +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace + +# Local History for Visual Studio Code +.history/ + +# Windows Installer files from build outputs +*.cab +*.msi +*.msix +*.msm +*.msp + +# JetBrains Rider +*.sln.iml +.idea/ + +## +## Visual studio for Mac +## + + +# globs +Makefile.in +*.userprefs +*.usertasks +config.make +config.status +aclocal.m4 +install-sh +autom4te.cache/ +*.tar.gz +tarballs/ +test-results/ + +# Mac bundle stuff +*.dmg +*.app + +# content below from: https://github.com/github/gitignore/blob/main/Global/macOS.gitignore +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +# content below from: https://github.com/github/gitignore/blob/main/Global/Windows.gitignore +# Windows thumbnail cache files +Thumbs.db +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +# Vim temporary swap files +*.swp diff --git a/Directory.Build.props b/Directory.Build.props new file mode 100644 index 0000000..252af69 --- /dev/null +++ b/Directory.Build.props @@ -0,0 +1,34 @@ + + + net10.0 + 14 + true + embedded + enable + enable + true + $(NoWarn);1591 + true + + + + ManagedCode Community + true + Github + MIT + README.md + https://github.com/AtuboDad/playwright_stealth + https://github.com/AtuboDad/playwright_stealth + ManagedCode Playwright Stealth + 1.0.0 + 1.0.0 + + + + true + + + + + + diff --git a/ManagedCode.Playwright.Stealth.sln b/ManagedCode.Playwright.Stealth.sln new file mode 100644 index 0000000..2fab6e3 --- /dev/null +++ b/ManagedCode.Playwright.Stealth.sln @@ -0,0 +1,56 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31903.59 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{827E0CD3-B72D-47B6-A68D-7590B98EB39B}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ManagedCode.Playwright.Stealth", "src\Playwright.Stealth\ManagedCode.Playwright.Stealth.csproj", "{4FEFBA12-C714-433F-BA7E-8BAC95CCB72A}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{0AB3BF05-4346-4AA6-1389-037BE0695223}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ManagedCode.Playwright.Stealth.Tests", "tests\Playwright.Stealth.Tests\ManagedCode.Playwright.Stealth.Tests.csproj", "{E168CA70-F764-4D1F-A424-23C692241414}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {4FEFBA12-C714-433F-BA7E-8BAC95CCB72A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4FEFBA12-C714-433F-BA7E-8BAC95CCB72A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4FEFBA12-C714-433F-BA7E-8BAC95CCB72A}.Debug|x64.ActiveCfg = Debug|Any CPU + {4FEFBA12-C714-433F-BA7E-8BAC95CCB72A}.Debug|x64.Build.0 = Debug|Any CPU + {4FEFBA12-C714-433F-BA7E-8BAC95CCB72A}.Debug|x86.ActiveCfg = Debug|Any CPU + {4FEFBA12-C714-433F-BA7E-8BAC95CCB72A}.Debug|x86.Build.0 = Debug|Any CPU + {4FEFBA12-C714-433F-BA7E-8BAC95CCB72A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4FEFBA12-C714-433F-BA7E-8BAC95CCB72A}.Release|Any CPU.Build.0 = Release|Any CPU + {4FEFBA12-C714-433F-BA7E-8BAC95CCB72A}.Release|x64.ActiveCfg = Release|Any CPU + {4FEFBA12-C714-433F-BA7E-8BAC95CCB72A}.Release|x64.Build.0 = Release|Any CPU + {4FEFBA12-C714-433F-BA7E-8BAC95CCB72A}.Release|x86.ActiveCfg = Release|Any CPU + {4FEFBA12-C714-433F-BA7E-8BAC95CCB72A}.Release|x86.Build.0 = Release|Any CPU + {E168CA70-F764-4D1F-A424-23C692241414}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {E168CA70-F764-4D1F-A424-23C692241414}.Debug|Any CPU.Build.0 = Debug|Any CPU + {E168CA70-F764-4D1F-A424-23C692241414}.Debug|x64.ActiveCfg = Debug|Any CPU + {E168CA70-F764-4D1F-A424-23C692241414}.Debug|x64.Build.0 = Debug|Any CPU + {E168CA70-F764-4D1F-A424-23C692241414}.Debug|x86.ActiveCfg = Debug|Any CPU + {E168CA70-F764-4D1F-A424-23C692241414}.Debug|x86.Build.0 = Debug|Any CPU + {E168CA70-F764-4D1F-A424-23C692241414}.Release|Any CPU.ActiveCfg = Release|Any CPU + {E168CA70-F764-4D1F-A424-23C692241414}.Release|Any CPU.Build.0 = Release|Any CPU + {E168CA70-F764-4D1F-A424-23C692241414}.Release|x64.ActiveCfg = Release|Any CPU + {E168CA70-F764-4D1F-A424-23C692241414}.Release|x64.Build.0 = Release|Any CPU + {E168CA70-F764-4D1F-A424-23C692241414}.Release|x86.ActiveCfg = Release|Any CPU + {E168CA70-F764-4D1F-A424-23C692241414}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {4FEFBA12-C714-433F-BA7E-8BAC95CCB72A} = {827E0CD3-B72D-47B6-A68D-7590B98EB39B} + {E168CA70-F764-4D1F-A424-23C692241414} = {0AB3BF05-4346-4AA6-1389-037BE0695223} + EndGlobalSection +EndGlobal diff --git a/README.md b/README.md new file mode 100644 index 0000000..f7fe59d --- /dev/null +++ b/README.md @@ -0,0 +1,70 @@ +# ManagedCode.Playwright.Stealth (.NET) + +A .NET port of the Playwright stealth scripts from [AtuboDad/playwright_stealth](https://github.com/AtuboDad/playwright_stealth). +This package adds a single call to apply a collection of stealth init scripts to a Playwright page or browser context. + +## Install + +```bash + dotnet add package ManagedCode.Playwright.Stealth +``` + +## Usage + +```csharp +using Microsoft.Playwright; +using ManagedCode.Playwright.Stealth; + +using var playwright = await Playwright.CreateAsync(); +await using var browser = await playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions +{ + Headless = true +}); + +var context = await browser.NewContextAsync(); +await context.ApplyStealthAsync(); + +var page = await context.NewPageAsync(); +await page.GotoAsync("https://www.browserscan.net/bot-detection"); +``` + +## Configuration + +```csharp +var config = new StealthConfig +{ + NavigatorHardwareConcurrency = 8, + NavigatorLanguages = true, + NavigatorUserAgentValue = "Mozilla/5.0 ...", + WebglVendor = true +}; + +await page.ApplyStealthAsync(config); +``` + +## Development + +1. Install .NET SDK 10.0.102 or later. +2. Restore dependencies: + +```bash + dotnet restore ManagedCode.Playwright.Stealth.sln +``` + +3. Run tests (uses Microsoft.Testing.Platform via `global.json`): + +```bash + dotnet test --solution ManagedCode.Playwright.Stealth.sln --configuration Release +``` + +## Test Sites + +The integration tests exercise stealth checks on these sites: + +- https://www.browserscan.net/bot-detection +- https://bot.sannysoft.com/ +- https://www.intoli.com/blog/not-possible-to-block-chrome-headless/chrome-headless-test.html +- https://fingerprint.com/demo +- https://nowsecure.nl + +> These sites can change at any time; adjust assertions as needed. diff --git a/global.json b/global.json new file mode 100644 index 0000000..2cf953f --- /dev/null +++ b/global.json @@ -0,0 +1,8 @@ +{ + "sdk": { + "version": "10.0.102" + }, + "test": { + "runner": "Microsoft.Testing.Platform" + } +} diff --git a/src/Playwright.Stealth/ManagedCode.Playwright.Stealth.csproj b/src/Playwright.Stealth/ManagedCode.Playwright.Stealth.csproj new file mode 100644 index 0000000..9569f86 --- /dev/null +++ b/src/Playwright.Stealth/ManagedCode.Playwright.Stealth.csproj @@ -0,0 +1,20 @@ + + + + ManagedCode.Playwright.Stealth + ManagedCode.Playwright.Stealth + Stealth tweaks for Microsoft.Playwright. + playwright;stealth;bot-detection + ManagedCode.Playwright.Stealth + ManagedCode.Playwright.Stealth + + + + + + + + + + + diff --git a/src/Playwright.Stealth/PlaywrightStealthExtensions.cs b/src/Playwright.Stealth/PlaywrightStealthExtensions.cs new file mode 100644 index 0000000..fe2685b --- /dev/null +++ b/src/Playwright.Stealth/PlaywrightStealthExtensions.cs @@ -0,0 +1,24 @@ +using Microsoft.Playwright; + +namespace ManagedCode.Playwright.Stealth; + +public static class PlaywrightStealthExtensions +{ + public static async Task ApplyStealthAsync(this IPage page, StealthConfig? config = null) + { + var stealthConfig = config ?? new StealthConfig(); + foreach (var script in StealthScriptProvider.BuildScripts(stealthConfig)) + { + await page.AddInitScriptAsync(script).ConfigureAwait(false); + } + } + + public static async Task ApplyStealthAsync(this IBrowserContext context, StealthConfig? config = null) + { + var stealthConfig = config ?? new StealthConfig(); + foreach (var script in StealthScriptProvider.BuildScripts(stealthConfig)) + { + await context.AddInitScriptAsync(script).ConfigureAwait(false); + } + } +} diff --git a/src/Playwright.Stealth/Resources/js/chrome.app.js b/src/Playwright.Stealth/Resources/js/chrome.app.js new file mode 100644 index 0000000..7a73929 --- /dev/null +++ b/src/Playwright.Stealth/Resources/js/chrome.app.js @@ -0,0 +1,71 @@ +if (!window.chrome) { + // Use the exact property descriptor found in headful Chrome + // fetch it via `Object.getOwnPropertyDescriptor(window, 'chrome')` + Object.defineProperty(window, 'chrome', { + writable: true, + enumerable: true, + configurable: false, // note! + value: {} // We'll extend that later + }) +} + +// app in window.chrome means we're running headful and don't need to mock anything +if (!('app' in window.chrome)) { + const makeError = { + ErrorInInvocation: fn => { + const err = new TypeError(`Error in invocation of app.${fn}()`) + return utils.stripErrorWithAnchor( + err, + `at ${fn} (eval at ` + ) + } + } + +// There's a some static data in that property which doesn't seem to change, +// we should periodically check for updates: `JSON.stringify(window.app, null, 2)` + const APP_STATIC_DATA = JSON.parse( + ` +{ + "isInstalled": false, + "InstallState": { + "DISABLED": "disabled", + "INSTALLED": "installed", + "NOT_INSTALLED": "not_installed" + }, + "RunningState": { + "CANNOT_RUN": "cannot_run", + "READY_TO_RUN": "ready_to_run", + "RUNNING": "running" + } +} + `.trim() + ) + + window.chrome.app = { + ...APP_STATIC_DATA, + + get isInstalled() { + return false + }, + + getDetails: function getDetails() { + if (arguments.length) { + throw makeError.ErrorInInvocation(`getDetails`) + } + return null + }, + getIsInstalled: function getDetails() { + if (arguments.length) { + throw makeError.ErrorInInvocation(`getIsInstalled`) + } + return false + }, + runningState: function getDetails() { + if (arguments.length) { + throw makeError.ErrorInInvocation(`runningState`) + } + return 'cannot_run' + } + } + utils.patchToStringNested(window.chrome.app) +} \ No newline at end of file diff --git a/src/Playwright.Stealth/Resources/js/chrome.csi.js b/src/Playwright.Stealth/Resources/js/chrome.csi.js new file mode 100644 index 0000000..388e39f --- /dev/null +++ b/src/Playwright.Stealth/Resources/js/chrome.csi.js @@ -0,0 +1,27 @@ +if (!window.chrome) { + // Use the exact property descriptor found in headful Chrome + // fetch it via `Object.getOwnPropertyDescriptor(window, 'chrome')` + Object.defineProperty(window, 'chrome', { + writable: true, + enumerable: true, + configurable: false, // note! + value: {} // We'll extend that later + }) +} + +// Check if we're running headful and don't need to mock anything +// Check that the Navigation Timing API v1 is available, we need that +if (!('csi' in window.chrome) && (window.performance || window.performance.timing)) { + const {csi_timing} = window.performance + + log.info('loading chrome.csi.js') + window.chrome.csi = function () { + return { + onloadT: csi_timing.domContentLoadedEventEnd, + startE: csi_timing.navigationStart, + pageT: Date.now() - csi_timing.navigationStart, + tran: 15 // Transition type or something + } + } + utils.patchToString(window.chrome.csi) +} \ No newline at end of file diff --git a/src/Playwright.Stealth/Resources/js/chrome.hairline.js b/src/Playwright.Stealth/Resources/js/chrome.hairline.js new file mode 100644 index 0000000..4a4a633 --- /dev/null +++ b/src/Playwright.Stealth/Resources/js/chrome.hairline.js @@ -0,0 +1,14 @@ +// https://intoli.com/blog/making-chrome-headless-undetectable/ +// store the existing descriptor +const elementDescriptor = Object.getOwnPropertyDescriptor(HTMLElement.prototype, 'offsetHeight'); + +// redefine the property with a patched descriptor +Object.defineProperty(HTMLDivElement.prototype, 'offsetHeight', { + ...elementDescriptor, + get: function() { + if (this.id === 'modernizr') { + return 1; + } + return elementDescriptor.get.apply(this); + }, +}); \ No newline at end of file diff --git a/src/Playwright.Stealth/Resources/js/chrome.load.times.js b/src/Playwright.Stealth/Resources/js/chrome.load.times.js new file mode 100644 index 0000000..7025665 --- /dev/null +++ b/src/Playwright.Stealth/Resources/js/chrome.load.times.js @@ -0,0 +1,122 @@ +if (!window.chrome) { + // Use the exact property descriptor found in headful Chrome + // fetch it via `Object.getOwnPropertyDescriptor(window, 'chrome')` + Object.defineProperty(window, 'chrome', { + writable: true, + enumerable: true, + configurable: false, // note! + value: {} // We'll extend that later + }) +} + +// That means we're running headful and don't need to mock anything +if ('loadTimes' in window.chrome) { + throw new Error('skipping chrome loadtimes update, running in headfull mode') +} + +// Check that the Navigation Timing API v1 + v2 is available, we need that +if ( + window.performance || + window.performance.timing || + window.PerformancePaintTiming +) { + + const {performance} = window + + // Some stuff is not available on about:blank as it requires a navigation to occur, + // let's harden the code to not fail then: + const ntEntryFallback = { + nextHopProtocol: 'h2', + type: 'other' + } + + // The API exposes some funky info regarding the connection + const protocolInfo = { + get connectionInfo() { + const ntEntry = + performance.getEntriesByType('navigation')[0] || ntEntryFallback + return ntEntry.nextHopProtocol + }, + get npnNegotiatedProtocol() { + // NPN is deprecated in favor of ALPN, but this implementation returns the + // HTTP/2 or HTTP2+QUIC/39 requests negotiated via ALPN. + const ntEntry = + performance.getEntriesByType('navigation')[0] || ntEntryFallback + return ['h2', 'hq'].includes(ntEntry.nextHopProtocol) + ? ntEntry.nextHopProtocol + : 'unknown' + }, + get navigationType() { + const ntEntry = + performance.getEntriesByType('navigation')[0] || ntEntryFallback + return ntEntry.type + }, + get wasAlternateProtocolAvailable() { + // The Alternate-Protocol header is deprecated in favor of Alt-Svc + // (https://www.mnot.net/blog/2016/03/09/alt-svc), so technically this + // should always return false. + return false + }, + get wasFetchedViaSpdy() { + // SPDY is deprecated in favor of HTTP/2, but this implementation returns + // true for HTTP/2 or HTTP2+QUIC/39 as well. + const ntEntry = + performance.getEntriesByType('navigation')[0] || ntEntryFallback + return ['h2', 'hq'].includes(ntEntry.nextHopProtocol) + }, + get wasNpnNegotiated() { + // NPN is deprecated in favor of ALPN, but this implementation returns true + // for HTTP/2 or HTTP2+QUIC/39 requests negotiated via ALPN. + const ntEntry = + performance.getEntriesByType('navigation')[0] || ntEntryFallback + return ['h2', 'hq'].includes(ntEntry.nextHopProtocol) + } + } + + const {timing} = window.performance + +// Truncate number to specific number of decimals, most of the `loadTimes` stuff has 3 + function toFixed(num, fixed) { + var re = new RegExp('^-?\\d+(?:.\\d{0,' + (fixed || -1) + '})?') + return num.toString().match(re)[0] + } + + const timingInfo = { + get firstPaintAfterLoadTime() { + // This was never actually implemented and always returns 0. + return 0 + }, + get requestTime() { + return timing.navigationStart / 1000 + }, + get startLoadTime() { + return timing.navigationStart / 1000 + }, + get commitLoadTime() { + return timing.responseStart / 1000 + }, + get finishDocumentLoadTime() { + return timing.domContentLoadedEventEnd / 1000 + }, + get finishLoadTime() { + return timing.loadEventEnd / 1000 + }, + get firstPaintTime() { + const fpEntry = performance.getEntriesByType('paint')[0] || { + startTime: timing.loadEventEnd / 1000 // Fallback if no navigation occured (`about:blank`) + } + return toFixed( + (fpEntry.startTime + performance.timeOrigin) / 1000, + 3 + ) + } + } + + window.chrome.loadTimes = function () { + return { + ...protocolInfo, + ...timingInfo + } + } + utils.patchToString(window.chrome.loadTimes) +} \ No newline at end of file diff --git a/src/Playwright.Stealth/Resources/js/chrome.runtime.js b/src/Playwright.Stealth/Resources/js/chrome.runtime.js new file mode 100644 index 0000000..b47b37e --- /dev/null +++ b/src/Playwright.Stealth/Resources/js/chrome.runtime.js @@ -0,0 +1,262 @@ +const STATIC_DATA = { + "OnInstalledReason": { + "CHROME_UPDATE": "chrome_update", + "INSTALL": "install", + "SHARED_MODULE_UPDATE": "shared_module_update", + "UPDATE": "update" + }, + "OnRestartRequiredReason": { + "APP_UPDATE": "app_update", + "OS_UPDATE": "os_update", + "PERIODIC": "periodic" + }, + "PlatformArch": { + "ARM": "arm", + "ARM64": "arm64", + "MIPS": "mips", + "MIPS64": "mips64", + "X86_32": "x86-32", + "X86_64": "x86-64" + }, + "PlatformNaclArch": { + "ARM": "arm", + "MIPS": "mips", + "MIPS64": "mips64", + "X86_32": "x86-32", + "X86_64": "x86-64" + }, + "PlatformOs": { + "ANDROID": "android", + "CROS": "cros", + "LINUX": "linux", + "MAC": "mac", + "OPENBSD": "openbsd", + "WIN": "win" + }, + "RequestUpdateCheckStatus": { + "NO_UPDATE": "no_update", + "THROTTLED": "throttled", + "UPDATE_AVAILABLE": "update_available" + } +} + +if (!window.chrome) { + // Use the exact property descriptor found in headful Chrome + // fetch it via `Object.getOwnPropertyDescriptor(window, 'chrome')` + Object.defineProperty(window, 'chrome', { + writable: true, + enumerable: true, + configurable: false, // note! + value: {} // We'll extend that later + }) +} + +// That means we're running headfull and don't need to mock anything +const existsAlready = 'runtime' in window.chrome +// `chrome.runtime` is only exposed on secure origins +const isNotSecure = !window.location.protocol.startsWith('https') +if (!(existsAlready || (isNotSecure && !opts.runOnInsecureOrigins))) { + window.chrome.runtime = { + // There's a bunch of static data in that property which doesn't seem to change, + // we should periodically check for updates: `JSON.stringify(window.chrome.runtime, null, 2)` + ...STATIC_DATA, + // `chrome.runtime.id` is extension related and returns undefined in Chrome + get id() { + return undefined + }, + // These two require more sophisticated mocks + connect: null, + sendMessage: null + } + + const makeCustomRuntimeErrors = (preamble, method, extensionId) => ({ + NoMatchingSignature: new TypeError( + preamble + `No matching signature.` + ), + MustSpecifyExtensionID: new TypeError( + preamble + + `${method} called from a webpage must specify an Extension ID (string) for its first argument.` + ), + InvalidExtensionID: new TypeError( + preamble + `Invalid extension id: '${extensionId}'` + ) + }) + + // Valid Extension IDs are 32 characters in length and use the letter `a` to `p`: + // https://source.chromium.org/chromium/chromium/src/+/master:components/crx_file/id_util.cc;drc=14a055ccb17e8c8d5d437fe080faba4c6f07beac;l=90 + const isValidExtensionID = str => + str.length === 32 && str.toLowerCase().match(/^[a-p]+$/) + + /** Mock `chrome.runtime.sendMessage` */ + const sendMessageHandler = { + apply: function (target, ctx, args) { + const [extensionId, options, responseCallback] = args || [] + + // Define custom errors + const errorPreamble = `Error in invocation of runtime.sendMessage(optional string extensionId, any message, optional object options, optional function responseCallback): ` + const Errors = makeCustomRuntimeErrors( + errorPreamble, + `chrome.runtime.sendMessage()`, + extensionId + ) + + // Check if the call signature looks ok + const noArguments = args.length === 0 + const tooManyArguments = args.length > 4 + const incorrectOptions = options && typeof options !== 'object' + const incorrectResponseCallback = + responseCallback && typeof responseCallback !== 'function' + if ( + noArguments || + tooManyArguments || + incorrectOptions || + incorrectResponseCallback + ) { + throw Errors.NoMatchingSignature + } + + // At least 2 arguments are required before we even validate the extension ID + if (args.length < 2) { + throw Errors.MustSpecifyExtensionID + } + + // Now let's make sure we got a string as extension ID + if (typeof extensionId !== 'string') { + throw Errors.NoMatchingSignature + } + + if (!isValidExtensionID(extensionId)) { + throw Errors.InvalidExtensionID + } + + return undefined // Normal behavior + } + } + utils.mockWithProxy( + window.chrome.runtime, + 'sendMessage', + function sendMessage() { + }, + sendMessageHandler + ) + + /** + * Mock `chrome.runtime.connect` + * + * @see https://developer.chrome.com/apps/runtime#method-connect + */ + const connectHandler = { + apply: function (target, ctx, args) { + const [extensionId, connectInfo] = args || [] + + // Define custom errors + const errorPreamble = `Error in invocation of runtime.connect(optional string extensionId, optional object connectInfo): ` + const Errors = makeCustomRuntimeErrors( + errorPreamble, + `chrome.runtime.connect()`, + extensionId + ) + + // Behavior differs a bit from sendMessage: + const noArguments = args.length === 0 + const emptyStringArgument = args.length === 1 && extensionId === '' + if (noArguments || emptyStringArgument) { + throw Errors.MustSpecifyExtensionID + } + + const tooManyArguments = args.length > 2 + const incorrectConnectInfoType = + connectInfo && typeof connectInfo !== 'object' + + if (tooManyArguments || incorrectConnectInfoType) { + throw Errors.NoMatchingSignature + } + + const extensionIdIsString = typeof extensionId === 'string' + if (extensionIdIsString && extensionId === '') { + throw Errors.MustSpecifyExtensionID + } + if (extensionIdIsString && !isValidExtensionID(extensionId)) { + throw Errors.InvalidExtensionID + } + + // There's another edge-case here: extensionId is optional so we might find a connectInfo object as first param, which we need to validate + const validateConnectInfo = ci => { + // More than a first param connectInfo as been provided + if (args.length > 1) { + throw Errors.NoMatchingSignature + } + // An empty connectInfo has been provided + if (Object.keys(ci).length === 0) { + throw Errors.MustSpecifyExtensionID + } + // Loop over all connectInfo props an check them + Object.entries(ci).forEach(([k, v]) => { + const isExpected = ['name', 'includeTlsChannelId'].includes(k) + if (!isExpected) { + throw new TypeError( + errorPreamble + `Unexpected property: '${k}'.` + ) + } + const MismatchError = (propName, expected, found) => + TypeError( + errorPreamble + + `Error at property '${propName}': Invalid type: expected ${expected}, found ${found}.` + ) + if (k === 'name' && typeof v !== 'string') { + throw MismatchError(k, 'string', typeof v) + } + if (k === 'includeTlsChannelId' && typeof v !== 'boolean') { + throw MismatchError(k, 'boolean', typeof v) + } + }) + } + if (typeof extensionId === 'object') { + validateConnectInfo(extensionId) + throw Errors.MustSpecifyExtensionID + } + + // Unfortunately even when the connect fails Chrome will return an object with methods we need to mock as well + return utils.patchToStringNested(makeConnectResponse()) + } + } + utils.mockWithProxy( + window.chrome.runtime, + 'connect', + function connect() { + }, + connectHandler + ) + + function makeConnectResponse() { + const onSomething = () => ({ + addListener: function addListener() { + }, + dispatch: function dispatch() { + }, + hasListener: function hasListener() { + }, + hasListeners: function hasListeners() { + return false + }, + removeListener: function removeListener() { + } + }) + + const response = { + name: '', + sender: undefined, + disconnect: function disconnect() { + }, + onDisconnect: onSomething(), + onMessage: onSomething(), + postMessage: function postMessage() { + if (!arguments.length) { + throw new TypeError(`Insufficient number of arguments.`) + } + throw new Error(`Attempting to use a disconnected port object`) + } + } + return response + } +} diff --git a/src/Playwright.Stealth/Resources/js/generate.magic.arrays.js b/src/Playwright.Stealth/Resources/js/generate.magic.arrays.js new file mode 100644 index 0000000..b9c2811 --- /dev/null +++ b/src/Playwright.Stealth/Resources/js/generate.magic.arrays.js @@ -0,0 +1,142 @@ +generateFunctionMocks = ( + proto, + itemMainProp, + dataArray +) => ({ + item: utils.createProxy(proto.item, { + apply(target, ctx, args) { + if (!args.length) { + throw new TypeError( + `Failed to execute 'item' on '${ + proto[Symbol.toStringTag] + }': 1 argument required, but only 0 present.` + ) + } + // Special behavior alert: + // - Vanilla tries to cast strings to Numbers (only integers!) and use them as property index lookup + // - If anything else than an integer (including as string) is provided it will return the first entry + const isInteger = args[0] && Number.isInteger(Number(args[0])) // Cast potential string to number first, then check for integer + // Note: Vanilla never returns `undefined` + return (isInteger ? dataArray[Number(args[0])] : dataArray[0]) || null + } + }), + /** Returns the MimeType object with the specified name. */ + namedItem: utils.createProxy(proto.namedItem, { + apply(target, ctx, args) { + if (!args.length) { + throw new TypeError( + `Failed to execute 'namedItem' on '${ + proto[Symbol.toStringTag] + }': 1 argument required, but only 0 present.` + ) + } + return dataArray.find(mt => mt[itemMainProp] === args[0]) || null // Not `undefined`! + } + }), + /** Does nothing and shall return nothing */ + refresh: proto.refresh + ? utils.createProxy(proto.refresh, { + apply(target, ctx, args) { + return undefined + } + }) + : undefined +}) + +function generateMagicArray( + dataArray = [], + proto = MimeTypeArray.prototype, + itemProto = MimeType.prototype, + itemMainProp = 'type' +) { + // Quick helper to set props with the same descriptors vanilla is using + const defineProp = (obj, prop, value) => + Object.defineProperty(obj, prop, { + value, + writable: false, + enumerable: false, // Important for mimeTypes & plugins: `JSON.stringify(navigator.mimeTypes)` + configurable: false + }) + + // Loop over our fake data and construct items + const makeItem = data => { + const item = {} + for (const prop of Object.keys(data)) { + if (prop.startsWith('__')) { + continue + } + defineProp(item, prop, data[prop]) + } + // navigator.plugins[i].length should always be 1 + if (itemProto === Plugin.prototype) { + defineProp(item, 'length', 1) + } + // We need to spoof a specific `MimeType` or `Plugin` object + return Object.create(itemProto, Object.getOwnPropertyDescriptors(item)) + } + + const magicArray = [] + + // Loop through our fake data and use that to create convincing entities + dataArray.forEach(data => { + magicArray.push(makeItem(data)) + }) + + // Add direct property access based on types (e.g. `obj['application/pdf']`) afterwards + magicArray.forEach(entry => { + defineProp(magicArray, entry[itemMainProp], entry) + }) + + // This is the best way to fake the type to make sure this is false: `Array.isArray(navigator.mimeTypes)` + const magicArrayObj = Object.create(proto, { + ...Object.getOwnPropertyDescriptors(magicArray), + + // There's one ugly quirk we unfortunately need to take care of: + // The `MimeTypeArray` prototype has an enumerable `length` property, + // but headful Chrome will still skip it when running `Object.getOwnPropertyNames(navigator.mimeTypes)`. + // To strip it we need to make it first `configurable` and can then overlay a Proxy with an `ownKeys` trap. + length: { + value: magicArray.length, + writable: false, + enumerable: false, + configurable: true // Important to be able to use the ownKeys trap in a Proxy to strip `length` + } + }) + + // Generate our functional function mocks :-) + const functionMocks = generateFunctionMocks( + proto, + itemMainProp, + magicArray + ) + + // Override custom object with proxy + return new Proxy(magicArrayObj, { + get(target, key = '') { + // Redirect function calls to our custom proxied versions mocking the vanilla behavior + if (key === 'item') { + return functionMocks.item + } + if (key === 'namedItem') { + return functionMocks.namedItem + } + if (proto === PluginArray.prototype && key === 'refresh') { + return functionMocks.refresh + } + // Everything else can pass through as normal + return utils.cache.Reflect.get(...arguments) + }, + ownKeys(target) { + // There are a couple of quirks where the original property demonstrates "magical" behavior that makes no sense + // This can be witnessed when calling `Object.getOwnPropertyNames(navigator.mimeTypes)` and the absense of `length` + // My guess is that it has to do with the recent change of not allowing data enumeration and this being implemented weirdly + // For that reason we just completely fake the available property names based on our data to match what regular Chrome is doing + // Specific issues when not patching this: `length` property is available, direct `types` props (e.g. `obj['application/pdf']`) are missing + const keys = [] + const typeProps = magicArray.map(mt => mt[itemMainProp]) + typeProps.forEach((_, i) => keys.push(`${i}`)) + typeProps.forEach(propName => keys.push(propName)) + return keys + } + }) +} \ No newline at end of file diff --git a/src/Playwright.Stealth/Resources/js/iframe.contentWindow.js b/src/Playwright.Stealth/Resources/js/iframe.contentWindow.js new file mode 100644 index 0000000..9bdae19 --- /dev/null +++ b/src/Playwright.Stealth/Resources/js/iframe.contentWindow.js @@ -0,0 +1,97 @@ +try { + // Adds a contentWindow proxy to the provided iframe element + const addContentWindowProxy = iframe => { + const contentWindowProxy = { + get(target, key) { + // Now to the interesting part: + // We actually make this thing behave like a regular iframe window, + // by intercepting calls to e.g. `.self` and redirect it to the correct thing. :) + // That makes it possible for these assertions to be correct: + // iframe.contentWindow.self === window.top // must be false + if (key === 'self') { + return this + } + // iframe.contentWindow.frameElement === iframe // must be true + if (key === 'frameElement') { + return iframe + } + return Reflect.get(target, key) + } + } + + if (!iframe.contentWindow) { + const proxy = new Proxy(window, contentWindowProxy) + Object.defineProperty(iframe, 'contentWindow', { + get() { + return proxy + }, + set(newValue) { + return newValue // contentWindow is immutable + }, + enumerable: true, + configurable: false + }) + } + } + + // Handles iframe element creation, augments `srcdoc` property so we can intercept further + const handleIframeCreation = (target, thisArg, args) => { + const iframe = target.apply(thisArg, args) + + // We need to keep the originals around + const _iframe = iframe + const _srcdoc = _iframe.srcdoc + + // Add hook for the srcdoc property + // We need to be very surgical here to not break other iframes by accident + Object.defineProperty(iframe, 'srcdoc', { + configurable: true, // Important, so we can reset this later + get: function () { + return _iframe.srcdoc + }, + set: function (newValue) { + addContentWindowProxy(this) + // Reset property, the hook is only needed once + Object.defineProperty(iframe, 'srcdoc', { + configurable: false, + writable: false, + value: _srcdoc + }) + _iframe.srcdoc = newValue + } + }) + return iframe + } + + // Adds a hook to intercept iframe creation events + const addIframeCreationSniffer = () => { + /* global document */ + const createElementHandler = { + // Make toString() native + get(target, key) { + return Reflect.get(target, key) + }, + apply: function (target, thisArg, args) { + const isIframe = + args && args.length && `${args[0]}`.toLowerCase() === 'iframe' + if (!isIframe) { + // Everything as usual + return target.apply(thisArg, args) + } else { + return handleIframeCreation(target, thisArg, args) + } + } + } + // All this just due to iframes with srcdoc bug + utils.replaceWithProxy( + document, + 'createElement', + createElementHandler + ) + } + + // Let's go + addIframeCreationSniffer() +} catch (err) { + // console.warn(err) +} \ No newline at end of file diff --git a/src/Playwright.Stealth/Resources/js/media.codecs.js b/src/Playwright.Stealth/Resources/js/media.codecs.js new file mode 100644 index 0000000..7761032 --- /dev/null +++ b/src/Playwright.Stealth/Resources/js/media.codecs.js @@ -0,0 +1,63 @@ +/** + * Input might look funky, we need to normalize it so e.g. whitespace isn't an issue for our spoofing. + * + * @example + * video/webm; codecs="vp8, vorbis" + * video/mp4; codecs="avc1.42E01E" + * audio/x-m4a; + * audio/ogg; codecs="vorbis" + * @param {String} arg + */ +const parseInput = arg => { + const [mime, codecStr] = arg.trim().split(';') + let codecs = [] + if (codecStr && codecStr.includes('codecs="')) { + codecs = codecStr + .trim() + .replace(`codecs="`, '') + .replace(`"`, '') + .trim() + .split(',') + .filter(x => !!x) + .map(x => x.trim()) + } + return { + mime, + codecStr, + codecs + } +} + +const canPlayType = { + // Intercept certain requests + apply: function (target, ctx, args) { + if (!args || !args.length) { + return target.apply(ctx, args) + } + const {mime, codecs} = parseInput(args[0]) + // This specific mp4 codec is missing in Chromium + if (mime === 'video/mp4') { + if (codecs.includes('avc1.42E01E')) { + return 'probably' + } + } + // This mimetype is only supported if no codecs are specified + if (mime === 'audio/x-m4a' && !codecs.length) { + return 'maybe' + } + + // This mimetype is only supported if no codecs are specified + if (mime === 'audio/aac' && !codecs.length) { + return 'probably' + } + // Everything else as usual + return target.apply(ctx, args) + } +} + +/* global HTMLMediaElement */ +utils.replaceWithProxy( + HTMLMediaElement.prototype, + 'canPlayType', + canPlayType +) \ No newline at end of file diff --git a/src/Playwright.Stealth/Resources/js/navigator.hardwareConcurrency.js b/src/Playwright.Stealth/Resources/js/navigator.hardwareConcurrency.js new file mode 100644 index 0000000..739c458 --- /dev/null +++ b/src/Playwright.Stealth/Resources/js/navigator.hardwareConcurrency.js @@ -0,0 +1,8 @@ +const patchNavigator = (name, value) => + utils.replaceProperty(Object.getPrototypeOf(navigator), name, { + get() { + return value + } + }) + +patchNavigator('hardwareConcurrency', opts.navigator_hardware_concurrency || 4); \ No newline at end of file diff --git a/src/Playwright.Stealth/Resources/js/navigator.languages.js b/src/Playwright.Stealth/Resources/js/navigator.languages.js new file mode 100644 index 0000000..75d6b01 --- /dev/null +++ b/src/Playwright.Stealth/Resources/js/navigator.languages.js @@ -0,0 +1,3 @@ +Object.defineProperty(Object.getPrototypeOf(navigator), 'languages', { + get: () => opts.languages || ['en-US', 'en'] +}) diff --git a/src/Playwright.Stealth/Resources/js/navigator.permissions.js b/src/Playwright.Stealth/Resources/js/navigator.permissions.js new file mode 100644 index 0000000..1d90531 --- /dev/null +++ b/src/Playwright.Stealth/Resources/js/navigator.permissions.js @@ -0,0 +1,19 @@ +const handler = { + apply: function (target, ctx, args) { + const param = (args || [])[0] + + if (param && param.name && param.name === 'notifications') { + const result = {state: Notification.permission} + Object.setPrototypeOf(result, PermissionStatus.prototype) + return Promise.resolve(result) + } + + return utils.cache.Reflect.apply(...arguments) + } +} + +utils.replaceWithProxy( + window.navigator.permissions.__proto__, // eslint-disable-line no-proto + 'query', + handler +) diff --git a/src/Playwright.Stealth/Resources/js/navigator.platform.js b/src/Playwright.Stealth/Resources/js/navigator.platform.js new file mode 100644 index 0000000..3655de0 --- /dev/null +++ b/src/Playwright.Stealth/Resources/js/navigator.platform.js @@ -0,0 +1,5 @@ +if (opts.navigator_platform) { + Object.defineProperty(Object.getPrototypeOf(navigator), 'platform', { + get: () => opts.navigator_platform, + }) +} diff --git a/src/Playwright.Stealth/Resources/js/navigator.plugins.js b/src/Playwright.Stealth/Resources/js/navigator.plugins.js new file mode 100644 index 0000000..117aeb6 --- /dev/null +++ b/src/Playwright.Stealth/Resources/js/navigator.plugins.js @@ -0,0 +1,92 @@ +data = { + "mimeTypes": [ + { + "type": "application/pdf", + "suffixes": "pdf", + "description": "", + "__pluginName": "Chrome PDF Viewer" + }, + { + "type": "application/x-google-chrome-pdf", + "suffixes": "pdf", + "description": "Portable Document Format", + "__pluginName": "Chrome PDF Plugin" + }, + { + "type": "application/x-nacl", + "suffixes": "", + "description": "Native Client Executable", + "__pluginName": "Native Client" + }, + { + "type": "application/x-pnacl", + "suffixes": "", + "description": "Portable Native Client Executable", + "__pluginName": "Native Client" + } + ], + "plugins": [ + { + "name": "Chrome PDF Plugin", + "filename": "internal-pdf-viewer", + "description": "Portable Document Format", + "__mimeTypes": ["application/x-google-chrome-pdf"] + }, + { + "name": "Chrome PDF Viewer", + "filename": "mhjfbmdgcfjbbpaeojofohoefgiehjai", + "description": "", + "__mimeTypes": ["application/pdf"] + }, + { + "name": "Native Client", + "filename": "internal-nacl-plugin", + "description": "", + "__mimeTypes": ["application/x-nacl", "application/x-pnacl"] + } + ] +} + + +// That means we're running headful +const hasPlugins = 'plugins' in navigator && navigator.plugins.length +if (!(hasPlugins)) { + + const mimeTypes = generateMagicArray( + data.mimeTypes, + MimeTypeArray.prototype, + MimeType.prototype, + 'type' + ) + const plugins = generateMagicArray( + data.plugins, + PluginArray.prototype, + Plugin.prototype, + 'name' + ) + + // Plugin and MimeType cross-reference each other, let's do that now + // Note: We're looping through `data.plugins` here, not the generated `plugins` + for (const pluginData of data.plugins) { + pluginData.__mimeTypes.forEach((type, index) => { + plugins[pluginData.name][index] = mimeTypes[type] + plugins[type] = mimeTypes[type] + Object.defineProperty(mimeTypes[type], 'enabledPlugin', { + value: JSON.parse(JSON.stringify(plugins[pluginData.name])), + writable: false, + enumerable: false, // Important: `JSON.stringify(navigator.plugins)` + configurable: false + }) + }) + } + + const patchNavigator = (name, value) => + utils.replaceProperty(Object.getPrototypeOf(navigator), name, { + get() { + return value + } + }) + + patchNavigator('mimeTypes', mimeTypes) + patchNavigator('plugins', plugins) +} \ No newline at end of file diff --git a/src/Playwright.Stealth/Resources/js/navigator.userAgent.js b/src/Playwright.Stealth/Resources/js/navigator.userAgent.js new file mode 100644 index 0000000..f35f07a --- /dev/null +++ b/src/Playwright.Stealth/Resources/js/navigator.userAgent.js @@ -0,0 +1,5 @@ +// replace Headless references in default useragent +const current_ua = navigator.userAgent +Object.defineProperty(Object.getPrototypeOf(navigator), 'userAgent', { + get: () => opts.navigator_user_agent || current_ua.replace('HeadlessChrome/', 'Chrome/') +}) diff --git a/src/Playwright.Stealth/Resources/js/navigator.vendor.js b/src/Playwright.Stealth/Resources/js/navigator.vendor.js new file mode 100644 index 0000000..1349b67 --- /dev/null +++ b/src/Playwright.Stealth/Resources/js/navigator.vendor.js @@ -0,0 +1,3 @@ +Object.defineProperty(Object.getPrototypeOf(navigator), 'vendor', { + get: () => opts.navigator_vendor || 'Google Inc.', +}) diff --git a/src/Playwright.Stealth/Resources/js/navigator.webdriver.js b/src/Playwright.Stealth/Resources/js/navigator.webdriver.js new file mode 100644 index 0000000..6f40b85 --- /dev/null +++ b/src/Playwright.Stealth/Resources/js/navigator.webdriver.js @@ -0,0 +1,14 @@ +// this is close to the most accurate way to emulate this: https://stackoverflow.com/a/69533548 +Object.defineProperty(Object.getPrototypeOf(navigator), 'webdriver', { + set: undefined, + enumerable: true, + configurable: true, + get: new Proxy( + Object.getOwnPropertyDescriptor(Object.getPrototypeOf(navigator), 'webdriver').get, + { apply: (target, thisArg, args) => { + // emulate getter call validation + Reflect.apply(target, thisArg, args); + return false; + }} + ) +}); \ No newline at end of file diff --git a/src/Playwright.Stealth/Resources/js/utils.js b/src/Playwright.Stealth/Resources/js/utils.js new file mode 100644 index 0000000..5e26ae6 --- /dev/null +++ b/src/Playwright.Stealth/Resources/js/utils.js @@ -0,0 +1,456 @@ +/** + * A set of shared utility functions specifically for the purpose of modifying native browser APIs without leaving traces. + * + * Meant to be passed down in puppeteer and used in the context of the page (everything in here runs in NodeJS as well as a browser). + * + * Note: If for whatever reason you need to use this outside of `puppeteer-extra`: + * Just remove the `module.exports` statement at the very bottom, the rest can be copy pasted into any browser context. + * + * Alternatively take a look at the `extract-stealth-evasions` package to create a finished bundle which includes these utilities. + * + */ +const utils = {} + +/** + * Wraps a JS Proxy Handler and strips it's presence from error stacks, in case the traps throw. + * + * The presence of a JS Proxy can be revealed as it shows up in error stack traces. + * + * @param {object} handler - The JS Proxy handler to wrap + */ +utils.stripProxyFromErrors = (handler = {}) => { + const newHandler = {} + // We wrap each trap in the handler in a try/catch and modify the error stack if they throw + const traps = Object.getOwnPropertyNames(handler) + traps.forEach(trap => { + newHandler[trap] = function() { + try { + // Forward the call to the defined proxy handler + return handler[trap].apply(this, arguments || []) + } catch (err) { + // Stack traces differ per browser, we only support chromium based ones currently + if (!err || !err.stack || !err.stack.includes(`at `)) { + throw err + } + + // When something throws within one of our traps the Proxy will show up in error stacks + // An earlier implementation of this code would simply strip lines with a blacklist, + // but it makes sense to be more surgical here and only remove lines related to our Proxy. + // We try to use a known "anchor" line for that and strip it with everything above it. + // If the anchor line cannot be found for some reason we fall back to our blacklist approach. + + const stripWithBlacklist = stack => { + const blacklist = [ + `at Reflect.${trap} `, // e.g. Reflect.get or Reflect.apply + `at Object.${trap} `, // e.g. Object.get or Object.apply + `at Object.newHandler. [as ${trap}] ` // caused by this very wrapper :-) + ] + return ( + err.stack + .split('\n') + // Always remove the first (file) line in the stack (guaranteed to be our proxy) + .filter((line, index) => index !== 1) + // Check if the line starts with one of our blacklisted strings + .filter(line => !blacklist.some(bl => line.trim().startsWith(bl))) + .join('\n') + ) + } + + const stripWithAnchor = stack => { + const stackArr = stack.split('\n') + const anchor = `at Object.newHandler. [as ${trap}] ` // Known first Proxy line in chromium + const anchorIndex = stackArr.findIndex(line => + line.trim().startsWith(anchor) + ) + if (anchorIndex === -1) { + return false // 404, anchor not found + } + // Strip everything from the top until we reach the anchor line + // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`) + stackArr.splice(1, anchorIndex) + return stackArr.join('\n') + } + + // Try using the anchor method, fallback to blacklist if necessary + err.stack = stripWithAnchor(err.stack) || stripWithBlacklist(err.stack) + + throw err // Re-throw our now sanitized error + } + } + }) + return newHandler +} + +/** + * Strip error lines from stack traces until (and including) a known line the stack. + * + * @param {object} err - The error to sanitize + * @param {string} anchor - The string the anchor line starts with + */ +utils.stripErrorWithAnchor = (err, anchor) => { + const stackArr = err.stack.split('\n') + const anchorIndex = stackArr.findIndex(line => line.trim().startsWith(anchor)) + if (anchorIndex === -1) { + return err // 404, anchor not found + } + // Strip everything from the top until we reach the anchor line (remove anchor line as well) + // Note: We're keeping the 1st line (zero index) as it's unrelated (e.g. `TypeError`) + stackArr.splice(1, anchorIndex) + err.stack = stackArr.join('\n') + return err +} + +/** + * Replace the property of an object in a stealthy way. + * + * Note: You also want to work on the prototype of an object most often, + * as you'd otherwise leave traces (e.g. showing up in Object.getOwnPropertyNames(obj)). + * + * @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/defineProperty + * + * @example + * replaceProperty(WebGLRenderingContext.prototype, 'getParameter', { value: "alice" }) + * // or + * replaceProperty(Object.getPrototypeOf(navigator), 'languages', { get: () => ['en-US', 'en'] }) + * + * @param {object} obj - The object which has the property to replace + * @param {string} propName - The property name to replace + * @param {object} descriptorOverrides - e.g. { value: "alice" } + */ +utils.replaceProperty = (obj, propName, descriptorOverrides = {}) => { + return Object.defineProperty(obj, propName, { + // Copy over the existing descriptors (writable, enumerable, configurable, etc) + ...(Object.getOwnPropertyDescriptor(obj, propName) || {}), + // Add our overrides (e.g. value, get()) + ...descriptorOverrides + }) +} + +/** + * Preload a cache of function copies and data. + * + * For a determined enough observer it would be possible to overwrite and sniff usage of functions + * we use in our internal Proxies, to combat that we use a cached copy of those functions. + * + * This is evaluated once per execution context (e.g. window) + */ +utils.preloadCache = () => { + if (utils.cache) { + return + } + utils.cache = { + // Used in our proxies + Reflect: { + get: Reflect.get.bind(Reflect), + apply: Reflect.apply.bind(Reflect) + }, + // Used in `makeNativeString` + nativeToStringStr: Function.toString + '' // => `function toString() { [native code] }` + } +} + +/** + * Utility function to generate a cross-browser `toString` result representing native code. + * + * There's small differences: Chromium uses a single line, whereas FF & Webkit uses multiline strings. + * To future-proof this we use an existing native toString result as the basis. + * + * The only advantage we have over the other team is that our JS runs first, hence we cache the result + * of the native toString result once, so they cannot spoof it afterwards and reveal that we're using it. + * + * Note: Whenever we add a `Function.prototype.toString` proxy we should preload the cache before, + * by executing `utils.preloadCache()` before the proxy is applied (so we don't cause recursive lookups). + * + * @example + * makeNativeString('foobar') // => `function foobar() { [native code] }` + * + * @param {string} [name] - Optional function name + */ +utils.makeNativeString = (name = '') => { + // Cache (per-window) the original native toString or use that if available + utils.preloadCache() + return utils.cache.nativeToStringStr.replace('toString', name || '') +} + +/** + * Helper function to modify the `toString()` result of the provided object. + * + * Note: Use `utils.redirectToString` instead when possible. + * + * There's a quirk in JS Proxies that will cause the `toString()` result to differ from the vanilla Object. + * If no string is provided we will generate a `[native code]` thing based on the name of the property object. + * + * @example + * patchToString(WebGLRenderingContext.prototype.getParameter, 'function getParameter() { [native code] }') + * + * @param {object} obj - The object for which to modify the `toString()` representation + * @param {string} str - Optional string used as a return value + */ +utils.patchToString = (obj, str = '') => { + utils.preloadCache() + + const toStringProxy = new Proxy(Function.prototype.toString, { + apply: function(target, ctx) { + // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + ""` + if (ctx === Function.prototype.toString) { + return utils.makeNativeString('toString') + } + // `toString` targeted at our proxied Object detected + if (ctx === obj) { + // We either return the optional string verbatim or derive the most desired result automatically + return str || utils.makeNativeString(obj.name) + } + // Check if the toString protype of the context is the same as the global prototype, + // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case + const hasSameProto = Object.getPrototypeOf( + Function.prototype.toString + ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins + if (!hasSameProto) { + // Pass the call on to the local Function.prototype.toString instead + return ctx.toString() + } + return target.call(ctx) + } + }) + utils.replaceProperty(Function.prototype, 'toString', { + value: toStringProxy + }) +} + +/** + * Make all nested functions of an object native. + * + * @param {object} obj + */ +utils.patchToStringNested = (obj = {}) => { + return utils.execRecursively(obj, ['function'], utils.patchToString) +} + +/** + * Redirect toString requests from one object to another. + * + * @param {object} proxyObj - The object that toString will be called on + * @param {object} originalObj - The object which toString result we wan to return + */ +utils.redirectToString = (proxyObj, originalObj) => { + utils.preloadCache() + + const toStringProxy = new Proxy(Function.prototype.toString, { + apply: function(target, ctx) { + // This fixes e.g. `HTMLMediaElement.prototype.canPlayType.toString + ""` + if (ctx === Function.prototype.toString) { + return utils.makeNativeString('toString') + } + + // `toString` targeted at our proxied Object detected + if (ctx === proxyObj) { + const fallback = () => + originalObj && originalObj.name + ? utils.makeNativeString(originalObj.name) + : utils.makeNativeString(proxyObj.name) + + // Return the toString representation of our original object if possible + return originalObj + '' || fallback() + } + + // Check if the toString protype of the context is the same as the global prototype, + // if not indicates that we are doing a check across different windows., e.g. the iframeWithdirect` test case + const hasSameProto = Object.getPrototypeOf( + Function.prototype.toString + ).isPrototypeOf(ctx.toString) // eslint-disable-line no-prototype-builtins + if (!hasSameProto) { + // Pass the call on to the local Function.prototype.toString instead + return ctx.toString() + } + + return target.call(ctx) + } + }) + utils.replaceProperty(Function.prototype, 'toString', { + value: toStringProxy + }) +} + +/** + * All-in-one method to replace a property with a JS Proxy using the provided Proxy handler with traps. + * + * Will stealthify these aspects (strip error stack traces, redirect toString, etc). + * Note: This is meant to modify native Browser APIs and works best with prototype objects. + * + * @example + * replaceWithProxy(WebGLRenderingContext.prototype, 'getParameter', proxyHandler) + * + * @param {object} obj - The object which has the property to replace + * @param {string} propName - The name of the property to replace + * @param {object} handler - The JS Proxy handler to use + */ +utils.replaceWithProxy = (obj, propName, handler) => { + utils.preloadCache() + const originalObj = obj[propName] + const proxyObj = new Proxy(obj[propName], utils.stripProxyFromErrors(handler)) + + utils.replaceProperty(obj, propName, { value: proxyObj }) + utils.redirectToString(proxyObj, originalObj) + + return true +} + +/** + * All-in-one method to mock a non-existing property with a JS Proxy using the provided Proxy handler with traps. + * + * Will stealthify these aspects (strip error stack traces, redirect toString, etc). + * + * @example + * mockWithProxy(chrome.runtime, 'sendMessage', function sendMessage() {}, proxyHandler) + * + * @param {object} obj - The object which has the property to replace + * @param {string} propName - The name of the property to replace or create + * @param {object} pseudoTarget - The JS Proxy target to use as a basis + * @param {object} handler - The JS Proxy handler to use + */ +utils.mockWithProxy = (obj, propName, pseudoTarget, handler) => { + utils.preloadCache() + const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler)) + + utils.replaceProperty(obj, propName, { value: proxyObj }) + utils.patchToString(proxyObj) + + return true +} + +/** + * All-in-one method to create a new JS Proxy with stealth tweaks. + * + * This is meant to be used whenever we need a JS Proxy but don't want to replace or mock an existing known property. + * + * Will stealthify certain aspects of the Proxy (strip error stack traces, redirect toString, etc). + * + * @example + * createProxy(navigator.mimeTypes.__proto__.namedItem, proxyHandler) // => Proxy + * + * @param {object} pseudoTarget - The JS Proxy target to use as a basis + * @param {object} handler - The JS Proxy handler to use + */ +utils.createProxy = (pseudoTarget, handler) => { + utils.preloadCache() + const proxyObj = new Proxy(pseudoTarget, utils.stripProxyFromErrors(handler)) + utils.patchToString(proxyObj) + + return proxyObj +} + +/** + * Helper function to split a full path to an Object into the first part and property. + * + * @example + * splitObjPath(`HTMLMediaElement.prototype.canPlayType`) + * // => {objName: "HTMLMediaElement.prototype", propName: "canPlayType"} + * + * @param {string} objPath - The full path to an object as dot notation string + */ +utils.splitObjPath = objPath => ({ + // Remove last dot entry (property) ==> `HTMLMediaElement.prototype` + objName: objPath + .split('.') + .slice(0, -1) + .join('.'), + // Extract last dot entry ==> `canPlayType` + propName: objPath.split('.').slice(-1)[0] +}) + +/** + * Convenience method to replace a property with a JS Proxy using the provided objPath. + * + * Supports a full path (dot notation) to the object as string here, in case that makes it easier. + * + * @example + * replaceObjPathWithProxy('WebGLRenderingContext.prototype.getParameter', proxyHandler) + * + * @param {string} objPath - The full path to an object (dot notation string) to replace + * @param {object} handler - The JS Proxy handler to use + */ +utils.replaceObjPathWithProxy = (objPath, handler) => { + const { objName, propName } = utils.splitObjPath(objPath) + const obj = eval(objName) // eslint-disable-line no-eval + return utils.replaceWithProxy(obj, propName, handler) +} + +/** + * Traverse nested properties of an object recursively and apply the given function on a whitelist of value types. + * + * @param {object} obj + * @param {array} typeFilter - e.g. `['function']` + * @param {Function} fn - e.g. `utils.patchToString` + */ +utils.execRecursively = (obj = {}, typeFilter = [], fn) => { + function recurse(obj) { + for (const key in obj) { + if (obj[key] === undefined) { + continue + } + if (obj[key] && typeof obj[key] === 'object') { + recurse(obj[key]) + } else { + if (obj[key] && typeFilter.includes(typeof obj[key])) { + fn.call(this, obj[key]) + } + } + } + } + recurse(obj) + return obj +} + +/** + * Everything we run through e.g. `page.evaluate` runs in the browser context, not the NodeJS one. + * That means we cannot just use reference variables and functions from outside code, we need to pass everything as a parameter. + * + * Unfortunately the data we can pass is only allowed to be of primitive types, regular functions don't survive the built-in serialization process. + * This utility function will take an object with functions and stringify them, so we can pass them down unharmed as strings. + * + * We use this to pass down our utility functions as well as any other functions (to be able to split up code better). + * + * @see utils.materializeFns + * + * @param {object} fnObj - An object containing functions as properties + */ +utils.stringifyFns = (fnObj = { hello: () => 'world' }) => { + // Object.fromEntries() ponyfill (in 6 lines) - supported only in Node v12+, modern browsers are fine + // https://github.com/feross/fromentries + function fromEntries(iterable) { + return [...iterable].reduce((obj, [key, val]) => { + obj[key] = val + return obj + }, {}) + } + return (Object.fromEntries || fromEntries)( + Object.entries(fnObj) + .filter(([key, value]) => typeof value === 'function') + .map(([key, value]) => [key, value.toString()]) // eslint-disable-line no-eval + ) +} + +/** + * Utility function to reverse the process of `utils.stringifyFns`. + * Will materialize an object with stringified functions (supports classic and fat arrow functions). + * + * @param {object} fnStrObj - An object containing stringified functions as properties + */ +utils.materializeFns = (fnStrObj = { hello: "() => 'world'" }) => { + return Object.fromEntries( + Object.entries(fnStrObj).map(([key, value]) => { + if (value.startsWith('function')) { + // some trickery is needed to make oldschool functions work :-) + return [key, eval(`() => ${value}`)()] // eslint-disable-line no-eval + } else { + // arrow functions just work + return [key, eval(value)] // eslint-disable-line no-eval + } + }) + ) +} + +// -- +// Stuff starting below this line is NodeJS specific. +// -- +// module.exports = utils \ No newline at end of file diff --git a/src/Playwright.Stealth/Resources/js/webgl.vendor.js b/src/Playwright.Stealth/Resources/js/webgl.vendor.js new file mode 100644 index 0000000..6b585a9 --- /dev/null +++ b/src/Playwright.Stealth/Resources/js/webgl.vendor.js @@ -0,0 +1,25 @@ +console.log(opts) +const getParameterProxyHandler = { + apply: function (target, ctx, args) { + const param = (args || [])[0] + // UNMASKED_VENDOR_WEBGL + if (param === 37445) { + return opts.webgl_vendor || 'Intel Inc.' // default in headless: Google Inc. + } + // UNMASKED_RENDERER_WEBGL + if (param === 37446) { + return opts.webgl_renderer || 'Intel Iris OpenGL Engine' // default in headless: Google SwiftShader + } + return utils.cache.Reflect.apply(target, ctx, args) + } +} + +// There's more than one WebGL rendering context +// https://developer.mozilla.org/en-US/docs/Web/API/WebGL2RenderingContext#Browser_compatibility +// To find out the original values here: Object.getOwnPropertyDescriptors(WebGLRenderingContext.prototype.getParameter) +const addProxy = (obj, propName) => { + utils.replaceWithProxy(obj, propName, getParameterProxyHandler) +} +// For whatever weird reason loops don't play nice with Object.defineProperty, here's the next best thing: +addProxy(WebGLRenderingContext.prototype, 'getParameter') +addProxy(WebGL2RenderingContext.prototype, 'getParameter') \ No newline at end of file diff --git a/src/Playwright.Stealth/Resources/js/window.outerdimensions.js b/src/Playwright.Stealth/Resources/js/window.outerdimensions.js new file mode 100644 index 0000000..e9ef868 --- /dev/null +++ b/src/Playwright.Stealth/Resources/js/window.outerdimensions.js @@ -0,0 +1,12 @@ +'use strict' + +try { + if (!!window.outerWidth && !!window.outerHeight) { + const windowFrame = 85 // probably OS and WM dependent + window.outerWidth = window.innerWidth + console.log(`current window outer height ${window.outerHeight}`) + window.outerHeight = window.innerHeight + windowFrame + console.log(`new window outer height ${window.outerHeight}`) + } +} catch (err) { +} diff --git a/src/Playwright.Stealth/StealthConfig.cs b/src/Playwright.Stealth/StealthConfig.cs new file mode 100644 index 0000000..38a5997 --- /dev/null +++ b/src/Playwright.Stealth/StealthConfig.cs @@ -0,0 +1,50 @@ +using System.Collections.Generic; +using System.Text.Json; + +namespace ManagedCode.Playwright.Stealth; + +public sealed class StealthConfig +{ + public bool WebDriver { get; init; } = true; + public bool WebglVendor { get; init; } = true; + public bool ChromeApp { get; init; } = true; + public bool ChromeCsi { get; init; } = true; + public bool ChromeLoadTimes { get; init; } = true; + public bool ChromeRuntime { get; init; } = true; + public bool IframeContentWindow { get; init; } = true; + public bool MediaCodecs { get; init; } = true; + public int NavigatorHardwareConcurrency { get; init; } = 4; + public bool NavigatorLanguages { get; init; } = true; + public bool NavigatorPermissions { get; init; } = true; + public bool NavigatorPlatform { get; init; } = true; + public bool NavigatorPlugins { get; init; } = true; + public bool NavigatorUserAgent { get; init; } = true; + public bool NavigatorVendor { get; init; } = true; + public bool OuterDimensions { get; init; } = true; + public bool Hairline { get; init; } = true; + + public string Vendor { get; init; } = "Intel Inc."; + public string Renderer { get; init; } = "Intel Iris OpenGL Engine"; + public string NavigatorVendorValue { get; init; } = "Google Inc."; + public string? NavigatorUserAgentValue { get; init; } + public string? NavigatorPlatformValue { get; init; } + public IReadOnlyList Languages { get; init; } = new[] { "en-US", "en" }; + public bool? RunOnInsecureOrigins { get; init; } + + internal string BuildOptionsScript() + { + var payload = new Dictionary + { + ["webgl_vendor"] = Vendor, + ["webgl_renderer"] = Renderer, + ["navigator_vendor"] = NavigatorVendorValue, + ["navigator_platform"] = NavigatorPlatformValue, + ["navigator_user_agent"] = NavigatorUserAgentValue, + ["languages"] = Languages, + ["runOnInsecureOrigins"] = RunOnInsecureOrigins, + ["navigator_hardware_concurrency"] = NavigatorHardwareConcurrency + }; + + return $"const opts = {JsonSerializer.Serialize(payload)}"; + } +} diff --git a/src/Playwright.Stealth/StealthScriptProvider.cs b/src/Playwright.Stealth/StealthScriptProvider.cs new file mode 100644 index 0000000..a158b46 --- /dev/null +++ b/src/Playwright.Stealth/StealthScriptProvider.cs @@ -0,0 +1,137 @@ +using System.Collections.Generic; +using System.IO; +using System.Reflection; + +namespace ManagedCode.Playwright.Stealth; + +internal static class StealthScriptProvider +{ + private static readonly IReadOnlyDictionary Scripts = new Dictionary + { + ["chrome_app"] = LoadScript("chrome.app.js"), + ["chrome_csi"] = LoadScript("chrome.csi.js"), + ["chrome_hairline"] = LoadScript("chrome.hairline.js"), + ["chrome_load_times"] = LoadScript("chrome.load.times.js"), + ["chrome_runtime"] = LoadScript("chrome.runtime.js"), + ["generate_magic_arrays"] = LoadScript("generate.magic.arrays.js"), + ["iframe_content_window"] = LoadScript("iframe.contentWindow.js"), + ["media_codecs"] = LoadScript("media.codecs.js"), + ["navigator_hardware_concurrency"] = LoadScript("navigator.hardwareConcurrency.js"), + ["navigator_languages"] = LoadScript("navigator.languages.js"), + ["navigator_permissions"] = LoadScript("navigator.permissions.js"), + ["navigator_platform"] = LoadScript("navigator.platform.js"), + ["navigator_plugins"] = LoadScript("navigator.plugins.js"), + ["navigator_user_agent"] = LoadScript("navigator.userAgent.js"), + ["navigator_vendor"] = LoadScript("navigator.vendor.js"), + ["outerdimensions"] = LoadScript("window.outerdimensions.js"), + ["utils"] = LoadScript("utils.js"), + ["webdriver"] = LoadScript("navigator.webdriver.js"), + ["webgl_vendor"] = LoadScript("webgl.vendor.js") + }; + + public static IEnumerable BuildScripts(StealthConfig config) + { + yield return config.BuildOptionsScript(); + yield return Scripts["utils"]; + yield return Scripts["generate_magic_arrays"]; + + if (config.ChromeApp) + { + yield return Scripts["chrome_app"]; + } + + if (config.ChromeCsi) + { + yield return Scripts["chrome_csi"]; + } + + if (config.Hairline) + { + yield return Scripts["chrome_hairline"]; + } + + if (config.ChromeLoadTimes) + { + yield return Scripts["chrome_load_times"]; + } + + if (config.ChromeRuntime) + { + yield return Scripts["chrome_runtime"]; + } + + if (config.IframeContentWindow) + { + yield return Scripts["iframe_content_window"]; + } + + if (config.MediaCodecs) + { + yield return Scripts["media_codecs"]; + } + + if (config.NavigatorLanguages) + { + yield return Scripts["navigator_languages"]; + } + + if (config.NavigatorPermissions) + { + yield return Scripts["navigator_permissions"]; + } + + if (config.NavigatorPlatform) + { + yield return Scripts["navigator_platform"]; + } + + if (config.NavigatorPlugins) + { + yield return Scripts["navigator_plugins"]; + } + + if (config.NavigatorUserAgent) + { + yield return Scripts["navigator_user_agent"]; + } + + if (config.NavigatorVendor) + { + yield return Scripts["navigator_vendor"]; + } + + if (config.WebDriver) + { + yield return Scripts["webdriver"]; + } + + if (config.OuterDimensions) + { + yield return Scripts["outerdimensions"]; + } + + if (config.WebglVendor) + { + yield return Scripts["webgl_vendor"]; + } + + if (config.NavigatorHardwareConcurrency > 0) + { + yield return Scripts["navigator_hardware_concurrency"]; + } + } + + private static string LoadScript(string fileName) + { + var assembly = Assembly.GetExecutingAssembly(); + var resourceName = $"{assembly.GetName().Name}.Resources.js.{fileName}"; + using var stream = assembly.GetManifestResourceStream(resourceName); + if (stream is null) + { + throw new InvalidOperationException($"Missing embedded script resource: {resourceName}"); + } + + using var reader = new StreamReader(stream); + return reader.ReadToEnd(); + } +} diff --git a/tests/Playwright.Stealth.Tests/ManagedCode.Playwright.Stealth.Tests.csproj b/tests/Playwright.Stealth.Tests/ManagedCode.Playwright.Stealth.Tests.csproj new file mode 100644 index 0000000..fafb97e --- /dev/null +++ b/tests/Playwright.Stealth.Tests/ManagedCode.Playwright.Stealth.Tests.csproj @@ -0,0 +1,18 @@ + + + + true + ManagedCode.Playwright.Stealth.Tests + + + + + + + + + + + + + diff --git a/tests/Playwright.Stealth.Tests/StealthConfigTests.cs b/tests/Playwright.Stealth.Tests/StealthConfigTests.cs new file mode 100644 index 0000000..3a7f6de --- /dev/null +++ b/tests/Playwright.Stealth.Tests/StealthConfigTests.cs @@ -0,0 +1,19 @@ +using TUnit.Assertions; +using TUnit.Assertions.Extensions; +using TUnit.Core; + +namespace ManagedCode.Playwright.Stealth.Tests; + +public sealed class StealthConfigTests +{ + [Test] + public async Task Defaults_Should_Enable_Core_Stealth_Scripts() + { + var config = new StealthConfig(); + + await Assert.That(config.WebDriver).IsTrue(); + await Assert.That(config.WebglVendor).IsTrue(); + await Assert.That(config.NavigatorLanguages).IsTrue(); + await Assert.That(config.NavigatorHardwareConcurrency).IsGreaterThan(0); + } +} diff --git a/tests/Playwright.Stealth.Tests/StealthIntegrationTests.cs b/tests/Playwright.Stealth.Tests/StealthIntegrationTests.cs new file mode 100644 index 0000000..a9857cf --- /dev/null +++ b/tests/Playwright.Stealth.Tests/StealthIntegrationTests.cs @@ -0,0 +1,74 @@ +using System.Globalization; +using Microsoft.Playwright; +using ManagedCode.Playwright.Stealth; +using TUnit.Assertions; +using TUnit.Assertions.Extensions; +using TUnit.Core; + +namespace ManagedCode.Playwright.Stealth.Tests; + +public sealed class StealthIntegrationTests +{ + private static readonly IReadOnlyList Sites = + [ + new("https://www.browserscan.net/bot-detection", "BrowserScan"), + new("https://bot.sannysoft.com/", "SannySoft"), + new("https://www.intoli.com/blog/not-possible-to-block-chrome-headless/chrome-headless-test.html", "Intoli"), + new("https://fingerprint.com/demo", "Fingerprint"), + new("https://nowsecure.nl", "NowSecure") + ]; + + [Test] + [Explicit("Integration test hitting external bot-detection sites. Set RUN_STEALTH_INTEGRATION_TESTS=1 to run intentionally.")] + public async Task Pages_Should_Not_Report_As_Bot() + { + if (!string.Equals(Environment.GetEnvironmentVariable("RUN_STEALTH_INTEGRATION_TESTS"), "1", StringComparison.Ordinal)) + { + return; + } + + using var playwright = await Microsoft.Playwright.Playwright.CreateAsync(); + await using var browser = await playwright.Chromium.LaunchAsync(new BrowserTypeLaunchOptions + { + Headless = true + }); + + var context = await browser.NewContextAsync(new BrowserNewContextOptions + { + Locale = CultureInfo.CurrentCulture.Name + }); + + await context.ApplyStealthAsync(); + var page = await context.NewPageAsync(); + + foreach (var site in Sites) + { + await page.GotoAsync(site.Url, new PageGotoOptions + { + WaitUntil = WaitUntilState.NetworkIdle, + Timeout = 120_000 + }); + + await EnsureNoBotSignalsAsync(page, site); + } + } + + private static async Task EnsureNoBotSignalsAsync(IPage page, StealthSite site) + { + var hasWebDriver = await page.EvaluateAsync("() => navigator.webdriver === true"); + var pluginCount = await page.EvaluateAsync("() => navigator.plugins ? navigator.plugins.length : 0"); + var languageCount = await page.EvaluateAsync("() => navigator.languages ? navigator.languages.length : 0"); + var hasChrome = await page.EvaluateAsync("() => typeof window.chrome !== 'undefined'"); + var bodyText = await page.InnerTextAsync("body"); + + await Assert.That(hasWebDriver).IsFalse(); + await Assert.That(pluginCount).IsGreaterThan(0); + await Assert.That(languageCount).IsGreaterThan(0); + await Assert.That(hasChrome).IsTrue(); + await Assert.That(bodyText.Contains("bot", StringComparison.OrdinalIgnoreCase)).IsFalse(); + await Assert.That(bodyText.Contains("headless", StringComparison.OrdinalIgnoreCase)).IsFalse(); + await Assert.That(bodyText.Contains("automation", StringComparison.OrdinalIgnoreCase)).IsFalse(); + } + + private sealed record StealthSite(string Url, string Name); +}