From 582d5eaf326a74742b7dbaf9a646d4ec03a215bd Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Mon, 20 Sep 2021 22:03:11 +0530 Subject: [PATCH 1/9] feat(cli): Support trace file URLs --- src/server/trace/viewer/traceViewer.ts | 19 +++++++++++--- src/utils/browserFetcher.ts | 30 +++------------------- src/utils/utils.ts | 35 ++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 30 deletions(-) diff --git a/src/server/trace/viewer/traceViewer.ts b/src/server/trace/viewer/traceViewer.ts index 11fba5d72c62e..a82c0757602ed 100644 --- a/src/server/trace/viewer/traceViewer.ts +++ b/src/server/trace/viewer/traceViewer.ts @@ -25,7 +25,7 @@ import { PersistentSnapshotStorage, TraceModel } from './traceModel'; import { ServerRouteHandler, HttpServer } from '../../../utils/httpServer'; import { SnapshotServer } from '../../snapshot/snapshotServer'; import * as consoleApiSource from '../../../generated/consoleApiSource'; -import { isUnderTest } from '../../../utils/utils'; +import { downloadFile, getDownloadProgress, getErrorMessage, isUnderTest } from '../../../utils/utils'; import { internalCallMetadata } from '../../instrumentation'; import { ProgressController } from '../../progress'; import { BrowserContext } from '../../browserContext'; @@ -196,6 +196,21 @@ async function appendTraceEvents(model: TraceModel, file: string) { } export async function showTraceViewer(tracePath: string, browserName: string, headless = false): Promise { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), `playwright-trace`)); + process.on('exit', () => rimraf.sync(dir)); + + if (tracePath.startsWith('http')){ + const downloadZipPath = path.join(dir, 'trace.zip'); + const {error} = await downloadFile(tracePath, downloadZipPath, { + progressCallback: getDownloadProgress('Trace File'), + }); + if (error) { + console.log(`Download failed. ${getErrorMessage(error)}`); // eslint-disable-line no-console + return; + } + tracePath = downloadZipPath; + } + let stat; try { stat = fs.statSync(tracePath); @@ -210,8 +225,6 @@ export async function showTraceViewer(tracePath: string, browserName: string, he } const zipFile = tracePath; - const dir = fs.mkdtempSync(path.join(os.tmpdir(), `playwright-trace`)); - process.on('exit', () => rimraf.sync(dir)); try { await extract(zipFile, { dir }); } catch (e) { diff --git a/src/utils/browserFetcher.ts b/src/utils/browserFetcher.ts index 254f5d486f901..7a24af21ad78c 100644 --- a/src/utils/browserFetcher.ts +++ b/src/utils/browserFetcher.ts @@ -19,8 +19,7 @@ import extract from 'extract-zip'; import fs from 'fs'; import os from 'os'; import path from 'path'; -import ProgressBar from 'progress'; -import { downloadFile, existsAsync } from './utils'; +import { downloadFile, existsAsync, getDownloadProgress, getErrorMessage } from './utils'; import { debugLogger } from './debugLogger'; export async function downloadBrowserWithProgressBar(title: string, browserDirectory: string, executablePath: string, downloadURL: string, downloadFileName: string): Promise { @@ -31,36 +30,17 @@ export async function downloadBrowserWithProgressBar(title: string, browserDirec return false; } - let progressBar: ProgressBar; - let lastDownloadedBytes = 0; - - function progress(downloadedBytes: number, totalBytes: number) { - if (!process.stderr.isTTY) - return; - if (!progressBar) { - progressBar = new ProgressBar(`Downloading ${progressBarName} - ${toMegabytes(totalBytes)} [:bar] :percent :etas `, { - complete: '=', - incomplete: ' ', - width: 20, - total: totalBytes, - }); - } - const delta = downloadedBytes - lastDownloadedBytes; - lastDownloadedBytes = downloadedBytes; - progressBar.tick(delta); - } - const url = downloadURL; const zipPath = path.join(os.tmpdir(), downloadFileName); try { for (let attempt = 1, N = 3; attempt <= N; ++attempt) { debugLogger.log('install', `downloading ${progressBarName} - attempt #${attempt}`); - const {error} = await downloadFile(url, zipPath, {progressCallback: progress, log: debugLogger.log.bind(debugLogger, 'install')}); + const {error} = await downloadFile(url, zipPath, {progressCallback: getDownloadProgress(progressBarName), log: debugLogger.log.bind(debugLogger, 'install')}); if (!error) { debugLogger.log('install', `SUCCESS downloading ${progressBarName}`); break; } - const errorMessage = typeof error === 'object' && typeof error.message === 'string' ? error.message : ''; + const errorMessage = getErrorMessage(error); debugLogger.log('install', `attempt #${attempt} - ERROR: ${errorMessage}`); if (attempt < N && (errorMessage.includes('ECONNRESET') || errorMessage.includes('ETIMEDOUT'))) { // Maximum delay is 3rd retry: 1337.5ms @@ -89,10 +69,6 @@ export async function downloadBrowserWithProgressBar(title: string, browserDirec return true; } -function toMegabytes(bytes: number) { - const mb = bytes / 1024 / 1024; - return `${Math.round(mb * 10) / 10} Mb`; -} export function logPolitely(toBeLogged: string) { const logLevel = process.env.npm_config_loglevel; diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 4ba1abb6eab3a..198b73dbdf72c 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -25,6 +25,7 @@ import { getProxyForUrl } from 'proxy-from-env'; import * as URL from 'url'; import { getUbuntuVersionSync } from './ubuntuVersion'; import { NameValue } from '../protocol/channels'; +import ProgressBar from 'progress'; // `https-proxy-agent` v5 is written in TypeScript and exposes generated types. // However, as of June 2020, its types are generated with tsconfig that enables @@ -415,3 +416,37 @@ export function wrapInASCIIBox(text: string, padding = 0): string { export function isFilePayload(value: any): boolean { return typeof value === 'object' && value['name'] && value['mimeType'] && value['buffer']; } + + +function toMegabytes(bytes: number) { + const mb = bytes / 1024 / 1024; + return `${Math.round(mb * 10) / 10} Mb`; +} + +export function getDownloadProgress(progressBarName: string) { + + let progressBar: ProgressBar; + let lastDownloadedBytes = 0; + + return (downloadedBytes: number, totalBytes: number) => { + if (!process.stderr.isTTY) return; + if (!progressBar) { + progressBar = new ProgressBar( + `Downloading ${progressBarName} - ${toMegabytes( + totalBytes + )} [:bar] :percent :etas `, + { + complete: '=', + incomplete: ' ', + width: 20, + total: totalBytes, + } + ); + } + const delta = downloadedBytes - lastDownloadedBytes; + lastDownloadedBytes = downloadedBytes; + progressBar.tick(delta); + }; +} + +export const getErrorMessage = (error: Error) => typeof error === 'object' && typeof error.message === 'string' ? error.message : ''; \ No newline at end of file From d6fba6c211f9a59e8489fab09c942686d5200753 Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Mon, 20 Sep 2021 22:14:09 +0530 Subject: [PATCH 2/9] feat(cli): Check Protocol of URL for trace zip. --- src/server/trace/viewer/traceViewer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/trace/viewer/traceViewer.ts b/src/server/trace/viewer/traceViewer.ts index a82c0757602ed..d92f1cb056ba2 100644 --- a/src/server/trace/viewer/traceViewer.ts +++ b/src/server/trace/viewer/traceViewer.ts @@ -199,7 +199,7 @@ export async function showTraceViewer(tracePath: string, browserName: string, he const dir = fs.mkdtempSync(path.join(os.tmpdir(), `playwright-trace`)); process.on('exit', () => rimraf.sync(dir)); - if (tracePath.startsWith('http')){ + if (/^https?:\/\//i.test(tracePath)){ const downloadZipPath = path.join(dir, 'trace.zip'); const {error} = await downloadFile(tracePath, downloadZipPath, { progressCallback: getDownloadProgress('Trace File'), From 3856ddbf3c604d205d9eda3f447ac6bbf82128cd Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Mon, 20 Sep 2021 23:07:39 +0530 Subject: [PATCH 3/9] docs(cli): Add URL example to show-trace --- docs/src/trace-viewer.md | 7 ++++++- src/cli/cli.ts | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/src/trace-viewer.md b/docs/src/trace-viewer.md index 0fe4dafcd766b..3e1d81859a3bb 100644 --- a/docs/src/trace-viewer.md +++ b/docs/src/trace-viewer.md @@ -136,22 +136,27 @@ This will record the trace and place it into the file named `trace.zip`. ## Viewing the trace -You can open the saved trace using Playwright CLI: +You can open the saved trace using Playwright CLI. +You can also open traces generated by CI runs using it's URL. ```bash js npx playwright show-trace trace.zip +npx playwright show-trace https://example.com/trace.zip ``` ```bash java mvn exec:java -e -Dexec.mainClass=com.microsoft.playwright.CLI -Dexec.args="show-trace trace.zip" +mvn exec:java -e -Dexec.mainClass=com.microsoft.playwright.CLI -Dexec.args="show-trace https://example.com/trace.zip" ``` ```bash python playwright show-trace trace.zip +playwright show-trace https://example.com/trace.zip ``` ```bash csharp playwright show-trace trace.zip +playwright show-trace https://example.com/trace.zip ``` ## Actions diff --git a/src/cli/cli.ts b/src/cli/cli.ts index 87c5c45d7228b..50b52332940f0 100755 --- a/src/cli/cli.ts +++ b/src/cli/cli.ts @@ -239,6 +239,8 @@ program console.log('Examples:'); console.log(''); console.log(' $ show-trace trace/directory'); + console.log(''); + console.log(' $ show-trace https://example.com/trace.zip'); }); if (!process.env.PW_CLI_TARGET_LANG) { From 22a04c6559426ceb7b745e9f9dfe5ca5319afb3d Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Fri, 24 Sep 2021 13:43:47 +0530 Subject: [PATCH 4/9] docs(trace-viewer): Separate Remote trace docs. --- docs/src/trace-viewer.md | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/docs/src/trace-viewer.md b/docs/src/trace-viewer.md index 3e1d81859a3bb..cbf07bbaa8b3b 100644 --- a/docs/src/trace-viewer.md +++ b/docs/src/trace-viewer.md @@ -136,27 +136,22 @@ This will record the trace and place it into the file named `trace.zip`. ## Viewing the trace -You can open the saved trace using Playwright CLI. -You can also open traces generated by CI runs using it's URL. +You can open the saved trace using Playwright CLI: ```bash js npx playwright show-trace trace.zip -npx playwright show-trace https://example.com/trace.zip ``` ```bash java mvn exec:java -e -Dexec.mainClass=com.microsoft.playwright.CLI -Dexec.args="show-trace trace.zip" -mvn exec:java -e -Dexec.mainClass=com.microsoft.playwright.CLI -Dexec.args="show-trace https://example.com/trace.zip" ``` ```bash python playwright show-trace trace.zip -playwright show-trace https://example.com/trace.zip ``` ```bash csharp playwright show-trace trace.zip -playwright show-trace https://example.com/trace.zip ``` ## Actions @@ -203,3 +198,25 @@ Here is what the typical Action snapshot looks like: Notice how it highlights both, the DOM Node as well as the exact click position. + + +## Viewing remote Traces + +You can open remote traces using it's URL. +They could be generated in a CI run and makes it easy to view the remote trace without having to manually download the file. + +```bash js +npx playwright show-trace https://example.com/trace.zip +``` + +```bash java +mvn exec:java -e -Dexec.mainClass=com.microsoft.playwright.CLI -Dexec.args="show-trace https://example.com/trace.zip" +``` + +```bash python +playwright show-trace https://example.com/trace.zip +``` + +```bash csharp +playwright show-trace https://example.com/trace.zip +``` From 0f96a6709c8e063aa8b7aae71bfa83ce2f00aa00 Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Sat, 25 Sep 2021 10:16:22 +0530 Subject: [PATCH 5/9] feat(trace-viewer): Retry file download --- src/server/trace/viewer/traceViewer.ts | 12 ++- src/utils/browserFetcher.ts | 24 +---- src/utils/debugLogger.ts | 1 + src/utils/download.ts | 144 +++++++++++++++++++++++++ src/utils/utils.ts | 80 +------------- 5 files changed, 159 insertions(+), 102 deletions(-) create mode 100644 src/utils/download.ts diff --git a/src/server/trace/viewer/traceViewer.ts b/src/server/trace/viewer/traceViewer.ts index d92f1cb056ba2..16f91a71aad63 100644 --- a/src/server/trace/viewer/traceViewer.ts +++ b/src/server/trace/viewer/traceViewer.ts @@ -25,7 +25,8 @@ import { PersistentSnapshotStorage, TraceModel } from './traceModel'; import { ServerRouteHandler, HttpServer } from '../../../utils/httpServer'; import { SnapshotServer } from '../../snapshot/snapshotServer'; import * as consoleApiSource from '../../../generated/consoleApiSource'; -import { downloadFile, getDownloadProgress, getErrorMessage, isUnderTest } from '../../../utils/utils'; +import { getErrorMessage, isUnderTest } from '../../../utils/utils'; +import { download } from '../../../utils/download'; import { internalCallMetadata } from '../../instrumentation'; import { ProgressController } from '../../progress'; import { BrowserContext } from '../../browserContext'; @@ -201,10 +202,11 @@ export async function showTraceViewer(tracePath: string, browserName: string, he if (/^https?:\/\//i.test(tracePath)){ const downloadZipPath = path.join(dir, 'trace.zip'); - const {error} = await downloadFile(tracePath, downloadZipPath, { - progressCallback: getDownloadProgress('Trace File'), - }); - if (error) { + try { + await download(tracePath, downloadZipPath, { + progressBarName: 'Trace File', + }); + } catch (error) { console.log(`Download failed. ${getErrorMessage(error)}`); // eslint-disable-line no-console return; } diff --git a/src/utils/browserFetcher.ts b/src/utils/browserFetcher.ts index 7a24af21ad78c..e7c06e5934d3b 100644 --- a/src/utils/browserFetcher.ts +++ b/src/utils/browserFetcher.ts @@ -19,7 +19,8 @@ import extract from 'extract-zip'; import fs from 'fs'; import os from 'os'; import path from 'path'; -import { downloadFile, existsAsync, getDownloadProgress, getErrorMessage } from './utils'; +import { existsAsync } from './utils'; +import { download } from './download'; import { debugLogger } from './debugLogger'; export async function downloadBrowserWithProgressBar(title: string, browserDirectory: string, executablePath: string, downloadURL: string, downloadFileName: string): Promise { @@ -33,24 +34,9 @@ export async function downloadBrowserWithProgressBar(title: string, browserDirec const url = downloadURL; const zipPath = path.join(os.tmpdir(), downloadFileName); try { - for (let attempt = 1, N = 3; attempt <= N; ++attempt) { - debugLogger.log('install', `downloading ${progressBarName} - attempt #${attempt}`); - const {error} = await downloadFile(url, zipPath, {progressCallback: getDownloadProgress(progressBarName), log: debugLogger.log.bind(debugLogger, 'install')}); - if (!error) { - debugLogger.log('install', `SUCCESS downloading ${progressBarName}`); - break; - } - const errorMessage = getErrorMessage(error); - debugLogger.log('install', `attempt #${attempt} - ERROR: ${errorMessage}`); - if (attempt < N && (errorMessage.includes('ECONNRESET') || errorMessage.includes('ETIMEDOUT'))) { - // Maximum delay is 3rd retry: 1337.5ms - const millis = (Math.random() * 200) + (250 * Math.pow(1.5, attempt)); - debugLogger.log('install', `sleeping ${millis}ms before retry...`); - await new Promise(c => setTimeout(c, millis)); - } else { - throw error; - } - } + await download(url, zipPath, { + progressBarName, + }); debugLogger.log('install', `extracting archive`); debugLogger.log('install', `-- zip: ${zipPath}`); debugLogger.log('install', `-- location: ${browserDirectory}`); diff --git a/src/utils/debugLogger.ts b/src/utils/debugLogger.ts index 754509fd8491f..f6cab23c7c36d 100644 --- a/src/utils/debugLogger.ts +++ b/src/utils/debugLogger.ts @@ -21,6 +21,7 @@ const debugLoggerColorMap = { 'api': 45, // cyan 'protocol': 34, // green 'install': 34, // green + 'download': 34, // green 'browser': 0, // reset 'proxy': 92, // purple 'error': 160, // red, diff --git a/src/utils/download.ts b/src/utils/download.ts new file mode 100644 index 0000000000000..6c37db4eb7711 --- /dev/null +++ b/src/utils/download.ts @@ -0,0 +1,144 @@ +/** + * Copyright (c) Microsoft Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import ProgressBar from 'progress'; +import { debugLogger } from './debugLogger'; +import { getErrorMessage, httpRequest } from './utils'; +import * as fs from 'fs'; + + +type OnProgressCallback = (downloadedBytes: number, totalBytes: number) => void; +type DownloadFileLogger = (message: string) => void; + +function toMegabytes(bytes: number): string { + const mb = bytes / 1024 / 1024; + return `${Math.round(mb * 10) / 10} Mb`; +} + +function getDownloadProgress(progressBarName: string): OnProgressCallback { + let progressBar: ProgressBar; + let lastDownloadedBytes = 0; + + return (downloadedBytes: number, totalBytes: number) => { + if (!process.stderr.isTTY) return; + if (!progressBar) { + progressBar = new ProgressBar( + `Downloading ${progressBarName} - ${toMegabytes( + totalBytes + )} [:bar] :percent :etas `, + { + complete: '=', + incomplete: ' ', + width: 20, + total: totalBytes, + } + ); + } + const delta = downloadedBytes - lastDownloadedBytes; + lastDownloadedBytes = downloadedBytes; + progressBar.tick(delta); + }; +} + +function downloadFile( + url: string, + destinationPath: string, + options: { + progressCallback?: OnProgressCallback; + log?: DownloadFileLogger; + } = {} +): Promise<{ error: any }> { + const { progressCallback, log = () => {} } = options; + log(`running download:`); + log(`-- from url: ${url}`); + log(`-- to location: ${destinationPath}`); + let fulfill: ({ error }: { error: any }) => void = ({ error }) => {}; + let downloadedBytes = 0; + let totalBytes = 0; + + const promise: Promise<{ error: any }> = new Promise(x => { + fulfill = x; + }); + + httpRequest( + { url }, + response => { + log(`-- response status code: ${response.statusCode}`); + if (response.statusCode !== 200) { + const error = new Error( + `Download failed: server returned code ${response.statusCode}. URL: ${url}` + ); + // consume response data to free up memory + response.resume(); + fulfill({ error }); + return; + } + const file = fs.createWriteStream(destinationPath); + file.on('finish', () => fulfill({ error: null })); + file.on('error', error => fulfill({ error })); + response.pipe(file); + totalBytes = parseInt(response.headers['content-length'] || '0', 10); + log(`-- total bytes: ${totalBytes}`); + if (progressCallback) response.on('data', onData); + }, + (error: any) => fulfill({ error }) + ); + return promise; + + function onData(chunk: string) { + downloadedBytes += chunk.length; + progressCallback!(downloadedBytes, totalBytes); + } +} + +export async function download( + url: string, + destination: string, + options: { + progressBarName?: string, + retryCount?: number + } = {} +) { + const { progressBarName = 'file', retryCount = 3 } = options; + for (let attempt = 1; attempt <= retryCount; ++attempt) { + debugLogger.log( + 'download', + `downloading ${progressBarName} - attempt #${attempt}` + ); + const { error } = await downloadFile(url, destination, { + progressCallback: getDownloadProgress(progressBarName), + log: debugLogger.log.bind(debugLogger, 'download'), + }); + if (!error) { + debugLogger.log('download', `SUCCESS downloading ${progressBarName}`); + break; + } + const errorMessage = getErrorMessage(error); + debugLogger.log('download', `attempt #${attempt} - ERROR: ${errorMessage}`); + if ( + attempt < retryCount && + (errorMessage.includes('ECONNRESET') || + errorMessage.includes('ETIMEDOUT')) + ) { + // Maximum default delay is 3rd retry: 1337.5ms + const millis = Math.random() * 200 + 250 * Math.pow(1.5, attempt); + debugLogger.log('download', `sleeping ${millis}ms before retry...`); + await new Promise(c => setTimeout(c, millis)); + } else { + throw error; + } + } +} diff --git a/src/utils/utils.ts b/src/utils/utils.ts index db5bfd7611743..bac47511c9ae1 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -27,7 +27,6 @@ import { getProxyForUrl } from 'proxy-from-env'; import * as URL from 'url'; import { getUbuntuVersionSync } from './ubuntuVersion'; import { NameValue } from '../protocol/channels'; -import ProgressBar from 'progress'; // `https-proxy-agent` v5 is written in TypeScript and exposes generated types. // However, as of June 2020, its types are generated with tsconfig that enables @@ -48,7 +47,7 @@ type HTTPRequestParams = { timeout?: number, }; -function httpRequest(params: HTTPRequestParams, onResponse: (r: http.IncomingMessage) => void, onError: (error: Error) => void) { +export function httpRequest(params: HTTPRequestParams, onResponse: (r: http.IncomingMessage) => void, onError: (error: Error) => void) { const parsedUrl = URL.parse(params.url); let options: https.RequestOptions = { ...parsedUrl }; options.method = params.method || 'GET'; @@ -113,49 +112,6 @@ export function fetchData(params: HTTPRequestParams, onError?: (response: http.I }); } -type OnProgressCallback = (downloadedBytes: number, totalBytes: number) => void; -type DownloadFileLogger = (message: string) => void; - -export function downloadFile(url: string, destinationPath: string, options: {progressCallback?: OnProgressCallback, log?: DownloadFileLogger} = {}): Promise<{error: any}> { - const { - progressCallback, - log = () => {}, - } = options; - log(`running download:`); - log(`-- from url: ${url}`); - log(`-- to location: ${destinationPath}`); - let fulfill: ({error}: {error: any}) => void = ({error}) => {}; - let downloadedBytes = 0; - let totalBytes = 0; - - const promise: Promise<{error: any}> = new Promise(x => { fulfill = x; }); - - httpRequest({ url }, response => { - log(`-- response status code: ${response.statusCode}`); - if (response.statusCode !== 200) { - const error = new Error(`Download failed: server returned code ${response.statusCode}. URL: ${url}`); - // consume response data to free up memory - response.resume(); - fulfill({error}); - return; - } - const file = fs.createWriteStream(destinationPath); - file.on('finish', () => fulfill({error: null})); - file.on('error', error => fulfill({error})); - response.pipe(file); - totalBytes = parseInt(response.headers['content-length'] || '0', 10); - log(`-- total bytes: ${totalBytes}`); - if (progressCallback) - response.on('data', onData); - }, (error: any) => fulfill({error})); - return promise; - - function onData(chunk: string) { - downloadedBytes += chunk.length; - progressCallback!(downloadedBytes, totalBytes); - } -} - export function spawnAsync(cmd: string, args: string[], options?: SpawnOptions): Promise<{stdout: string, stderr: string, code: number, error?: Error}> { const process = spawn(cmd, args, options); @@ -441,39 +397,7 @@ export function isFilePayload(value: any): boolean { return typeof value === 'object' && value['name'] && value['mimeType'] && value['buffer']; } - -function toMegabytes(bytes: number) { - const mb = bytes / 1024 / 1024; - return `${Math.round(mb * 10) / 10} Mb`; -} - -export function getDownloadProgress(progressBarName: string) { - - let progressBar: ProgressBar; - let lastDownloadedBytes = 0; - - return (downloadedBytes: number, totalBytes: number) => { - if (!process.stderr.isTTY) return; - if (!progressBar) { - progressBar = new ProgressBar( - `Downloading ${progressBarName} - ${toMegabytes( - totalBytes - )} [:bar] :percent :etas `, - { - complete: '=', - incomplete: ' ', - width: 20, - total: totalBytes, - } - ); - } - const delta = downloadedBytes - lastDownloadedBytes; - lastDownloadedBytes = downloadedBytes; - progressBar.tick(delta); - }; -} - -export const getErrorMessage = (error: Error) => typeof error === 'object' && typeof error.message === 'string' ? error.message : ''; +export const getErrorMessage = (error: Error): string => typeof error === 'object' && typeof error.message === 'string' ? error.message : ''; export function streamToString(stream: stream.Readable): Promise { return new Promise((resolve, reject) => { From e362243aa1ab213f7bde35f45afa6ea1ebcce38c Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Tue, 28 Sep 2021 09:33:35 +0530 Subject: [PATCH 6/9] chore(trace-viewer): Reorganise files --- src/server/trace/viewer/traceViewer.ts | 7 +- src/utils/browserFetcher.ts | 4 +- src/utils/download.ts | 144 ------------------------- src/utils/utils.ts | 118 +++++++++++++++++++- 4 files changed, 121 insertions(+), 152 deletions(-) delete mode 100644 src/utils/download.ts diff --git a/src/server/trace/viewer/traceViewer.ts b/src/server/trace/viewer/traceViewer.ts index 16f91a71aad63..5fb885b25b025 100644 --- a/src/server/trace/viewer/traceViewer.ts +++ b/src/server/trace/viewer/traceViewer.ts @@ -25,13 +25,13 @@ import { PersistentSnapshotStorage, TraceModel } from './traceModel'; import { ServerRouteHandler, HttpServer } from '../../../utils/httpServer'; import { SnapshotServer } from '../../snapshot/snapshotServer'; import * as consoleApiSource from '../../../generated/consoleApiSource'; -import { getErrorMessage, isUnderTest } from '../../../utils/utils'; -import { download } from '../../../utils/download'; +import { isUnderTest, download } from '../../../utils/utils'; import { internalCallMetadata } from '../../instrumentation'; import { ProgressController } from '../../progress'; import { BrowserContext } from '../../browserContext'; import { registry } from '../../../utils/registry'; import { installAppIcon } from '../../chromium/crApp'; +import { debugLogger } from '../../../utils/debugLogger'; export class TraceViewer { private _server: HttpServer; @@ -205,9 +205,10 @@ export async function showTraceViewer(tracePath: string, browserName: string, he try { await download(tracePath, downloadZipPath, { progressBarName: 'Trace File', + log: console.log // eslint-disable-line no-console }); } catch (error) { - console.log(`Download failed. ${getErrorMessage(error)}`); // eslint-disable-line no-console + console.log(`Download failed. ${error?.message || ''}`); // eslint-disable-line no-console return; } tracePath = downloadZipPath; diff --git a/src/utils/browserFetcher.ts b/src/utils/browserFetcher.ts index e7c06e5934d3b..8a54e500d2ea4 100644 --- a/src/utils/browserFetcher.ts +++ b/src/utils/browserFetcher.ts @@ -19,8 +19,7 @@ import extract from 'extract-zip'; import fs from 'fs'; import os from 'os'; import path from 'path'; -import { existsAsync } from './utils'; -import { download } from './download'; +import { existsAsync, download } from './utils'; import { debugLogger } from './debugLogger'; export async function downloadBrowserWithProgressBar(title: string, browserDirectory: string, executablePath: string, downloadURL: string, downloadFileName: string): Promise { @@ -36,6 +35,7 @@ export async function downloadBrowserWithProgressBar(title: string, browserDirec try { await download(url, zipPath, { progressBarName, + log: debugLogger.log.bind(debugLogger, 'install') }); debugLogger.log('install', `extracting archive`); debugLogger.log('install', `-- zip: ${zipPath}`); diff --git a/src/utils/download.ts b/src/utils/download.ts deleted file mode 100644 index 6c37db4eb7711..0000000000000 --- a/src/utils/download.ts +++ /dev/null @@ -1,144 +0,0 @@ -/** - * Copyright (c) Microsoft Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import ProgressBar from 'progress'; -import { debugLogger } from './debugLogger'; -import { getErrorMessage, httpRequest } from './utils'; -import * as fs from 'fs'; - - -type OnProgressCallback = (downloadedBytes: number, totalBytes: number) => void; -type DownloadFileLogger = (message: string) => void; - -function toMegabytes(bytes: number): string { - const mb = bytes / 1024 / 1024; - return `${Math.round(mb * 10) / 10} Mb`; -} - -function getDownloadProgress(progressBarName: string): OnProgressCallback { - let progressBar: ProgressBar; - let lastDownloadedBytes = 0; - - return (downloadedBytes: number, totalBytes: number) => { - if (!process.stderr.isTTY) return; - if (!progressBar) { - progressBar = new ProgressBar( - `Downloading ${progressBarName} - ${toMegabytes( - totalBytes - )} [:bar] :percent :etas `, - { - complete: '=', - incomplete: ' ', - width: 20, - total: totalBytes, - } - ); - } - const delta = downloadedBytes - lastDownloadedBytes; - lastDownloadedBytes = downloadedBytes; - progressBar.tick(delta); - }; -} - -function downloadFile( - url: string, - destinationPath: string, - options: { - progressCallback?: OnProgressCallback; - log?: DownloadFileLogger; - } = {} -): Promise<{ error: any }> { - const { progressCallback, log = () => {} } = options; - log(`running download:`); - log(`-- from url: ${url}`); - log(`-- to location: ${destinationPath}`); - let fulfill: ({ error }: { error: any }) => void = ({ error }) => {}; - let downloadedBytes = 0; - let totalBytes = 0; - - const promise: Promise<{ error: any }> = new Promise(x => { - fulfill = x; - }); - - httpRequest( - { url }, - response => { - log(`-- response status code: ${response.statusCode}`); - if (response.statusCode !== 200) { - const error = new Error( - `Download failed: server returned code ${response.statusCode}. URL: ${url}` - ); - // consume response data to free up memory - response.resume(); - fulfill({ error }); - return; - } - const file = fs.createWriteStream(destinationPath); - file.on('finish', () => fulfill({ error: null })); - file.on('error', error => fulfill({ error })); - response.pipe(file); - totalBytes = parseInt(response.headers['content-length'] || '0', 10); - log(`-- total bytes: ${totalBytes}`); - if (progressCallback) response.on('data', onData); - }, - (error: any) => fulfill({ error }) - ); - return promise; - - function onData(chunk: string) { - downloadedBytes += chunk.length; - progressCallback!(downloadedBytes, totalBytes); - } -} - -export async function download( - url: string, - destination: string, - options: { - progressBarName?: string, - retryCount?: number - } = {} -) { - const { progressBarName = 'file', retryCount = 3 } = options; - for (let attempt = 1; attempt <= retryCount; ++attempt) { - debugLogger.log( - 'download', - `downloading ${progressBarName} - attempt #${attempt}` - ); - const { error } = await downloadFile(url, destination, { - progressCallback: getDownloadProgress(progressBarName), - log: debugLogger.log.bind(debugLogger, 'download'), - }); - if (!error) { - debugLogger.log('download', `SUCCESS downloading ${progressBarName}`); - break; - } - const errorMessage = getErrorMessage(error); - debugLogger.log('download', `attempt #${attempt} - ERROR: ${errorMessage}`); - if ( - attempt < retryCount && - (errorMessage.includes('ECONNRESET') || - errorMessage.includes('ETIMEDOUT')) - ) { - // Maximum default delay is 3rd retry: 1337.5ms - const millis = Math.random() * 200 + 250 * Math.pow(1.5, attempt); - debugLogger.log('download', `sleeping ${millis}ms before retry...`); - await new Promise(c => setTimeout(c, millis)); - } else { - throw error; - } - } -} diff --git a/src/utils/utils.ts b/src/utils/utils.ts index bac47511c9ae1..cd344de3704b5 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -27,6 +27,7 @@ import { getProxyForUrl } from 'proxy-from-env'; import * as URL from 'url'; import { getUbuntuVersionSync } from './ubuntuVersion'; import { NameValue } from '../protocol/channels'; +import ProgressBar from 'progress'; // `https-proxy-agent` v5 is written in TypeScript and exposes generated types. // However, as of June 2020, its types are generated with tsconfig that enables @@ -47,7 +48,7 @@ type HTTPRequestParams = { timeout?: number, }; -export function httpRequest(params: HTTPRequestParams, onResponse: (r: http.IncomingMessage) => void, onError: (error: Error) => void) { +function httpRequest(params: HTTPRequestParams, onResponse: (r: http.IncomingMessage) => void, onError: (error: Error) => void) { const parsedUrl = URL.parse(params.url); let options: https.RequestOptions = { ...parsedUrl }; options.method = params.method || 'GET'; @@ -112,6 +113,119 @@ export function fetchData(params: HTTPRequestParams, onError?: (response: http.I }); } +type OnProgressCallback = (downloadedBytes: number, totalBytes: number) => void; +type DownloadFileLogger = (message: string) => void; + +function downloadFile(url: string, destinationPath: string, options: {progressCallback?: OnProgressCallback, log?: DownloadFileLogger} = {}): Promise<{error: any}> { + const { + progressCallback, + log = () => {}, + } = options; + log(`running download:`); + log(`-- from url: ${url}`); + log(`-- to location: ${destinationPath}`); + let fulfill: ({error}: {error: any}) => void = ({error}) => {}; + let downloadedBytes = 0; + let totalBytes = 0; + + const promise: Promise<{error: any}> = new Promise(x => { fulfill = x; }); + + httpRequest({ url }, response => { + log(`-- response status code: ${response.statusCode}`); + if (response.statusCode !== 200) { + const error = new Error(`Download failed: server returned code ${response.statusCode}. URL: ${url}`); + // consume response data to free up memory + response.resume(); + fulfill({error}); + return; + } + const file = fs.createWriteStream(destinationPath); + file.on('finish', () => fulfill({error: null})); + file.on('error', error => fulfill({error})); + response.pipe(file); + totalBytes = parseInt(response.headers['content-length'] || '0', 10); + log(`-- total bytes: ${totalBytes}`); + if (progressCallback) + response.on('data', onData); + }, (error: any) => fulfill({error})); + return promise; + + function onData(chunk: string) { + downloadedBytes += chunk.length; + progressCallback!(downloadedBytes, totalBytes); + } +} + +export async function download( + url: string, + destination: string, + options: { + progressBarName?: string, + retryCount?: number + log?: DownloadFileLogger + } = {} +) { + const { progressBarName = 'file', retryCount = 3, log = () => {} } = options; + for (let attempt = 1; attempt <= retryCount; ++attempt) { + log( + `downloading ${progressBarName} - attempt #${attempt}` + ); + const { error } = await downloadFile(url, destination, { + progressCallback: getDownloadProgress(progressBarName), + log, + }); + if (!error) { + log(`SUCCESS downloading ${progressBarName}`); + break; + } + const errorMessage = error?.message || ''; + log(`attempt #${attempt} - ERROR: ${errorMessage}`); + if ( + attempt < retryCount && + (errorMessage.includes('ECONNRESET') || + errorMessage.includes('ETIMEDOUT')) + ) { + // Maximum default delay is 3rd retry: 1337.5ms + const millis = Math.random() * 200 + 250 * Math.pow(1.5, attempt); + log(`sleeping ${millis}ms before retry...`); + await new Promise(c => setTimeout(c, millis)); + } else { + throw error; + } + } +} + +function getDownloadProgress(progressBarName: string): OnProgressCallback { + let progressBar: ProgressBar; + let lastDownloadedBytes = 0; + + return (downloadedBytes: number, totalBytes: number) => { + if (!process.stderr.isTTY) + return; + if (!progressBar) { + progressBar = new ProgressBar( + `Downloading ${progressBarName} - ${toMegabytes( + totalBytes + )} [:bar] :percent :etas `, + { + complete: '=', + incomplete: ' ', + width: 20, + total: totalBytes, + } + ); + } + const delta = downloadedBytes - lastDownloadedBytes; + lastDownloadedBytes = downloadedBytes; + progressBar.tick(delta); + }; +} + +function toMegabytes(bytes: number) { + const mb = bytes / 1024 / 1024; + return `${Math.round(mb * 10) / 10} Mb`; +} + export function spawnAsync(cmd: string, args: string[], options?: SpawnOptions): Promise<{stdout: string, stderr: string, code: number, error?: Error}> { const process = spawn(cmd, args, options); @@ -397,8 +511,6 @@ export function isFilePayload(value: any): boolean { return typeof value === 'object' && value['name'] && value['mimeType'] && value['buffer']; } -export const getErrorMessage = (error: Error): string => typeof error === 'object' && typeof error.message === 'string' ? error.message : ''; - export function streamToString(stream: stream.Readable): Promise { return new Promise((resolve, reject) => { const chunks: Buffer[] = []; From a0a974162d133b8de73a1c8f8bdf8361f1415cad Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Tue, 28 Sep 2021 09:38:32 +0530 Subject: [PATCH 7/9] chore(trace-viewer): Reorganise files --- src/server/trace/viewer/traceViewer.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/server/trace/viewer/traceViewer.ts b/src/server/trace/viewer/traceViewer.ts index 5fb885b25b025..a70b509188646 100644 --- a/src/server/trace/viewer/traceViewer.ts +++ b/src/server/trace/viewer/traceViewer.ts @@ -31,7 +31,6 @@ import { ProgressController } from '../../progress'; import { BrowserContext } from '../../browserContext'; import { registry } from '../../../utils/registry'; import { installAppIcon } from '../../chromium/crApp'; -import { debugLogger } from '../../../utils/debugLogger'; export class TraceViewer { private _server: HttpServer; From 0ba5f5d1e755a8357bc4062df6c2cd0b44a0cfc0 Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Tue, 28 Sep 2021 09:39:28 +0530 Subject: [PATCH 8/9] chore(trace-viewer): Remove download in logger --- src/utils/debugLogger.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/utils/debugLogger.ts b/src/utils/debugLogger.ts index 6302fb3b30362..3f6fad4000b42 100644 --- a/src/utils/debugLogger.ts +++ b/src/utils/debugLogger.ts @@ -21,7 +21,6 @@ const debugLoggerColorMap = { 'api': 45, // cyan 'protocol': 34, // green 'install': 34, // green - 'download': 34, // green 'browser': 0, // reset 'proxy': 92, // purple 'error': 160, // red, From bdfc6031ab2ac1db7d144e75fc4937e57c6c81d6 Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Thu, 30 Sep 2021 08:15:14 +0530 Subject: [PATCH 9/9] chore(traceViewer): Cleanup download messages --- src/server/trace/viewer/traceViewer.ts | 7 ++++--- src/utils/debugLogger.ts | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/server/trace/viewer/traceViewer.ts b/src/server/trace/viewer/traceViewer.ts index a70b509188646..be44bd8694229 100644 --- a/src/server/trace/viewer/traceViewer.ts +++ b/src/server/trace/viewer/traceViewer.ts @@ -31,6 +31,7 @@ import { ProgressController } from '../../progress'; import { BrowserContext } from '../../browserContext'; import { registry } from '../../../utils/registry'; import { installAppIcon } from '../../chromium/crApp'; +import { debugLogger } from '../../../utils/debugLogger'; export class TraceViewer { private _server: HttpServer; @@ -203,11 +204,11 @@ export async function showTraceViewer(tracePath: string, browserName: string, he const downloadZipPath = path.join(dir, 'trace.zip'); try { await download(tracePath, downloadZipPath, { - progressBarName: 'Trace File', - log: console.log // eslint-disable-line no-console + progressBarName: tracePath, + log: debugLogger.log.bind(debugLogger, 'download') }); } catch (error) { - console.log(`Download failed. ${error?.message || ''}`); // eslint-disable-line no-console + console.log(`${error?.message || ''}`); // eslint-disable-line no-console return; } tracePath = downloadZipPath; diff --git a/src/utils/debugLogger.ts b/src/utils/debugLogger.ts index 3f6fad4000b42..6302fb3b30362 100644 --- a/src/utils/debugLogger.ts +++ b/src/utils/debugLogger.ts @@ -21,6 +21,7 @@ const debugLoggerColorMap = { 'api': 45, // cyan 'protocol': 34, // green 'install': 34, // green + 'download': 34, // green 'browser': 0, // reset 'proxy': 92, // purple 'error': 160, // red,