From 3412ec174bb1d7c8953d5e7b776446c6c213c18c Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Wed, 18 Apr 2018 13:07:12 -0700 Subject: [PATCH 01/12] Send normalized lines to terminal --- .../codeExecution/codeExecutionManager.ts | 2 +- .../codeExecutionManager.test.ts | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/client/terminals/codeExecution/codeExecutionManager.ts b/src/client/terminals/codeExecution/codeExecutionManager.ts index 5a6159a50aae..f3db05292310 100644 --- a/src/client/terminals/codeExecution/codeExecutionManager.ts +++ b/src/client/terminals/codeExecution/codeExecutionManager.ts @@ -64,6 +64,6 @@ export class CodeExecutionManager implements ICodeExecutionManager { return; } - await executionService.execute(codeToExecute!, activeEditor!.document.uri); + await executionService.execute(normalizedCode, activeEditor!.document.uri); } } diff --git a/src/test/terminals/codeExecution/codeExecutionManager.test.ts b/src/test/terminals/codeExecution/codeExecutionManager.test.ts index 8dec4d7f5025..17696de0e189 100644 --- a/src/test/terminals/codeExecution/codeExecutionManager.test.ts +++ b/src/test/terminals/codeExecution/codeExecutionManager.test.ts @@ -69,7 +69,7 @@ suite('Terminal - Code Execution Manager', () => { serviceContainer.setup(s => s.get(TypeMoq.It.isValue(ICodeExecutionHelper))).returns(() => helper.object); await commandHandler!(); - helper.verify(async h => await h.getFileToExecute(), TypeMoq.Times.once()); + helper.verify(async h => h.getFileToExecute(), TypeMoq.Times.once()); }); test('Ensure executeFileInterTerminal will use provided file', async () => { @@ -96,8 +96,8 @@ suite('Terminal - Code Execution Manager', () => { const fileToExecute = Uri.file('x'); await commandHandler!(fileToExecute); - helper.verify(async h => await h.getFileToExecute(), TypeMoq.Times.never()); - executionService.verify(async e => await e.executeFile(TypeMoq.It.isValue(fileToExecute)), TypeMoq.Times.once()); + helper.verify(async h => h.getFileToExecute(), TypeMoq.Times.never()); + executionService.verify(async e => e.executeFile(TypeMoq.It.isValue(fileToExecute)), TypeMoq.Times.once()); }); test('Ensure executeFileInterTerminal will use active file', async () => { @@ -119,12 +119,12 @@ suite('Terminal - Code Execution Manager', () => { const fileToExecute = Uri.file('x'); const helper = TypeMoq.Mock.ofType(); serviceContainer.setup(s => s.get(TypeMoq.It.isValue(ICodeExecutionHelper))).returns(() => helper.object); - helper.setup(async h => await h.getFileToExecute()).returns(() => Promise.resolve(fileToExecute)); + helper.setup(async h => h.getFileToExecute()).returns(() => Promise.resolve(fileToExecute)); const executionService = TypeMoq.Mock.ofType(); serviceContainer.setup(s => s.get(TypeMoq.It.isValue(ICodeExecutionService), TypeMoq.It.isValue('standard'))).returns(() => executionService.object); await commandHandler!(fileToExecute); - executionService.verify(async e => await e.executeFile(TypeMoq.It.isValue(fileToExecute)), TypeMoq.Times.once()); + executionService.verify(async e => e.executeFile(TypeMoq.It.isValue(fileToExecute)), TypeMoq.Times.once()); }); async function testExecutionOfSelectionWithoutAnyActiveDocument(commandId: string, executionSericeId: string) { @@ -150,7 +150,7 @@ suite('Terminal - Code Execution Manager', () => { documentManager.setup(d => d.activeTextEditor).returns(() => undefined); await commandHandler!(); - executionService.verify(async e => await e.execute(TypeMoq.It.isAny()), TypeMoq.Times.never()); + executionService.verify(async e => e.execute(TypeMoq.It.isAny()), TypeMoq.Times.never()); } test('Ensure executeSelectionInTerminal will do nothing if theres no active document', async () => { @@ -186,7 +186,7 @@ suite('Terminal - Code Execution Manager', () => { documentManager.setup(d => d.activeTextEditor).returns(() => { return {} as any; }); await commandHandler!(); - executionService.verify(async e => await e.execute(TypeMoq.It.isAny()), TypeMoq.Times.never()); + executionService.verify(async e => e.execute(TypeMoq.It.isAny()), TypeMoq.Times.never()); } test('Ensure executeSelectionInTerminal will do nothing if no text is selected', async () => { @@ -218,7 +218,7 @@ suite('Terminal - Code Execution Manager', () => { const helper = TypeMoq.Mock.ofType(); serviceContainer.setup(s => s.get(TypeMoq.It.isValue(ICodeExecutionHelper))).returns(() => helper.object); helper.setup(h => h.getSelectedTextToExecute).returns(() => () => Promise.resolve(textSelected)); - helper.setup(h => h.normalizeLines).returns(() => () => textSelected); + helper.setup(h => h.normalizeLines).returns(() => () => textSelected).verifiable(TypeMoq.Times.once()); const executionService = TypeMoq.Mock.ofType(); serviceContainer.setup(s => s.get(TypeMoq.It.isValue(ICodeExecutionService), TypeMoq.It.isValue(executionServiceId))).returns(() => executionService.object); const document = TypeMoq.Mock.ofType(); @@ -228,7 +228,8 @@ suite('Terminal - Code Execution Manager', () => { documentManager.setup(d => d.activeTextEditor).returns(() => activeEditor.object); await commandHandler!(); - executionService.verify(async e => await e.execute(TypeMoq.It.isValue(textSelected), TypeMoq.It.isValue(activeDocumentUri)), TypeMoq.Times.once()); + executionService.verify(async e => e.execute(TypeMoq.It.isValue(textSelected), TypeMoq.It.isValue(activeDocumentUri)), TypeMoq.Times.once()); + helper.verifyAll(); } test('Ensure executeSelectionInTerminal will normalize selected text and send it to the terminal', async () => { await testExecutionOfSelectionIsSentToTerminal(Commands.Exec_Selection_In_Terminal, 'standard'); From fff01c946d797f11bb1a8f99c21f49c263738759 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Wed, 18 Apr 2018 13:08:58 -0700 Subject: [PATCH 02/12] :memo: news entry --- news/1 Enhancements/1207.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/1 Enhancements/1207.md diff --git a/news/1 Enhancements/1207.md b/news/1 Enhancements/1207.md new file mode 100644 index 000000000000..f668cdf00aae --- /dev/null +++ b/news/1 Enhancements/1207.md @@ -0,0 +1 @@ +Remove empty spaces from the selected text of the active editor when executing in a terminal. From fbdb935143c978989055c18f87a254af538611bc Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Wed, 18 Apr 2018 13:18:08 -0700 Subject: [PATCH 03/12] Add key board shortcut --- news/1 Enhancements/1349.md | 1 + package.json | 6 ++++++ 2 files changed, 7 insertions(+) create mode 100644 news/1 Enhancements/1349.md diff --git a/news/1 Enhancements/1349.md b/news/1 Enhancements/1349.md new file mode 100644 index 000000000000..7bcbcd8f1a88 --- /dev/null +++ b/news/1 Enhancements/1349.md @@ -0,0 +1 @@ +Add `Ctrl+Enter` keyboard shortcut for `Run Selection/Line in Python Terminal`. diff --git a/package.json b/package.json index 906ad9e94b14..a6a9a8f4a1e5 100644 --- a/package.json +++ b/package.json @@ -94,6 +94,12 @@ "path": "./snippets/python.json" } ], + "keybindings":[ + { + "command": "python.execSelectionInTerminal", + "key": "ctrl+enter" + } + ], "commands": [ { "command": "python.sortImports", From c321fb1b39df5e90fc2082e91ee0f7c36de49347 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Wed, 18 Apr 2018 14:07:05 -0700 Subject: [PATCH 04/12] Save file before executing --- src/client/common/configSettings.ts | 2 +- src/client/common/types.ts | 2 +- .../codeExecution/djangoShellCodeExecution.ts | 2 +- src/client/terminals/codeExecution/helper.ts | 5 ++- .../codeExecution/terminalCodeExecution.ts | 11 +++--- .../terminals/codeExecution/helper.test.ts | 39 +++++++++++++++++++ 6 files changed, 51 insertions(+), 10 deletions(-) diff --git a/src/client/common/configSettings.ts b/src/client/common/configSettings.ts index 62d75c0ec0ba..131228a28677 100644 --- a/src/client/common/configSettings.ts +++ b/src/client/common/configSettings.ts @@ -37,7 +37,7 @@ export class PythonSettings extends EventEmitter implements IPythonSettings { public formatting?: IFormattingSettings; public autoComplete?: IAutoCompleteSettings; public unitTest?: IUnitTestSettings; - public terminal?: ITerminalSettings; + public terminal!: ITerminalSettings; public sortImports?: ISortImportSettings; public workspaceSymbols?: IWorkspaceSymbolSettings; public disableInstallationChecks = false; diff --git a/src/client/common/types.ts b/src/client/common/types.ts index 5e16a4557786..240b6f5809e3 100644 --- a/src/client/common/types.ts +++ b/src/client/common/types.ts @@ -106,7 +106,7 @@ export interface IPythonSettings { readonly formatting?: IFormattingSettings; readonly unitTest?: IUnitTestSettings; readonly autoComplete?: IAutoCompleteSettings; - readonly terminal?: ITerminalSettings; + readonly terminal: ITerminalSettings; readonly sortImports?: ISortImportSettings; readonly workspaceSymbols?: IWorkspaceSymbolSettings; readonly envFile: string; diff --git a/src/client/terminals/codeExecution/djangoShellCodeExecution.ts b/src/client/terminals/codeExecution/djangoShellCodeExecution.ts index 4fc230b6e21a..3066ec27fb71 100644 --- a/src/client/terminals/codeExecution/djangoShellCodeExecution.ts +++ b/src/client/terminals/codeExecution/djangoShellCodeExecution.ts @@ -32,7 +32,7 @@ export class DjangoShellCodeExecutionProvider extends TerminalCodeExecutionProvi public getReplCommandArgs(resource?: Uri): { command: string; args: string[] } { const pythonSettings = this.configurationService.getSettings(resource); const command = this.platformService.isWindows ? pythonSettings.pythonPath.replace(/\\/g, '/') : pythonSettings.pythonPath; - const args = pythonSettings.terminal!.launchArgs.slice(); + const args = pythonSettings.terminal.launchArgs.slice(); const workspaceUri = resource ? this.workspace.getWorkspaceFolder(resource) : undefined; const defaultWorkspace = Array.isArray(this.workspace.workspaceFolders) && this.workspace.workspaceFolders.length > 0 ? this.workspace.workspaceFolders[0].uri.fsPath : ''; diff --git a/src/client/terminals/codeExecution/helper.ts b/src/client/terminals/codeExecution/helper.ts index efa468dcbc95..de335c7f7610 100644 --- a/src/client/terminals/codeExecution/helper.ts +++ b/src/client/terminals/codeExecution/helper.ts @@ -11,7 +11,7 @@ import { ICodeExecutionHelper } from '../types'; @injectable() export class CodeExecutionHelper implements ICodeExecutionHelper { - constructor( @inject(IDocumentManager) private documentManager: IDocumentManager, + constructor(@inject(IDocumentManager) private documentManager: IDocumentManager, @inject(IApplicationShell) private applicationShell: IApplicationShell) { } @@ -35,6 +35,9 @@ export class CodeExecutionHelper implements ICodeExecutionHelper { this.applicationShell.showErrorMessage('The active file is not a Python source file'); return; } + if (activeEditor.document.isDirty) { + await activeEditor.document.save(); + } return activeEditor.document.uri; } diff --git a/src/client/terminals/codeExecution/terminalCodeExecution.ts b/src/client/terminals/codeExecution/terminalCodeExecution.ts index d6ab0601c885..7e541ee12126 100644 --- a/src/client/terminals/codeExecution/terminalCodeExecution.ts +++ b/src/client/terminals/codeExecution/terminalCodeExecution.ts @@ -10,16 +10,15 @@ import { IWorkspaceService } from '../../common/application/types'; import '../../common/extensions'; import { IPlatformService } from '../../common/platform/types'; import { ITerminalService, ITerminalServiceFactory } from '../../common/terminal/types'; -import { IConfigurationService } from '../../common/types'; -import { IDisposableRegistry } from '../../common/types'; +import { IConfigurationService, IDisposableRegistry } from '../../common/types'; import { ICodeExecutionService } from '../../terminals/types'; @injectable() export class TerminalCodeExecutionProvider implements ICodeExecutionService { - protected terminalTitle: string; - private _terminalService: ITerminalService; + protected terminalTitle!: string; + private _terminalService!: ITerminalService; private replActive?: Promise; - constructor( @inject(ITerminalServiceFactory) protected readonly terminalServiceFactory: ITerminalServiceFactory, + constructor(@inject(ITerminalServiceFactory) protected readonly terminalServiceFactory: ITerminalServiceFactory, @inject(IConfigurationService) protected readonly configurationService: IConfigurationService, @inject(IWorkspaceService) protected readonly workspace: IWorkspaceService, @inject(IDisposableRegistry) protected readonly disposables: Disposable[], @@ -60,7 +59,7 @@ export class TerminalCodeExecutionProvider implements ICodeExecutionService { await this.replActive; } - public getReplCommandArgs(resource?: Uri): { command: string, args: string[] } { + public getReplCommandArgs(resource?: Uri): { command: string; args: string[] } { const pythonSettings = this.configurationService.getSettings(resource); const command = this.platformService.isWindows ? pythonSettings.pythonPath.replace(/\\/g, '/') : pythonSettings.pythonPath; const args = pythonSettings.terminal.launchArgs.slice(); diff --git a/src/test/terminals/codeExecution/helper.test.ts b/src/test/terminals/codeExecution/helper.test.ts index 064ca0d55e72..2ec1e85d3e01 100644 --- a/src/test/terminals/codeExecution/helper.test.ts +++ b/src/test/terminals/codeExecution/helper.test.ts @@ -73,6 +73,45 @@ suite('Terminal - Code Execution Helper', () => { expect(uri).to.be.deep.equal(expectedUri); }); + test('Returns file uri even if saving fails', async () => { + document.setup(doc => doc.isUntitled).returns(() => false); + document.setup(doc => doc.isDirty).returns(() => true); + document.setup(doc => doc.languageId).returns(() => PythonLanguage.language); + document.setup(doc => doc.save()).returns(() => Promise.resolve(false)); + const expectedUri = Uri.file('one.py'); + document.setup(doc => doc.uri).returns(() => expectedUri); + documentManager.setup(doc => doc.activeTextEditor).returns(() => editor.object); + + const uri = await helper.getFileToExecute(); + expect(uri).to.be.deep.equal(expectedUri); + }); + + test('Dirty files are saved', async () => { + document.setup(doc => doc.isUntitled).returns(() => false); + document.setup(doc => doc.isDirty).returns(() => true); + document.setup(doc => doc.languageId).returns(() => PythonLanguage.language); + const expectedUri = Uri.file('one.py'); + document.setup(doc => doc.uri).returns(() => expectedUri); + documentManager.setup(doc => doc.activeTextEditor).returns(() => editor.object); + + const uri = await helper.getFileToExecute(); + expect(uri).to.be.deep.equal(expectedUri); + document.verify(doc => doc.save(), TypeMoq.Times.once()); + }); + + test('Non-Dirty files are not-saved', async () => { + document.setup(doc => doc.isUntitled).returns(() => false); + document.setup(doc => doc.isDirty).returns(() => false); + document.setup(doc => doc.languageId).returns(() => PythonLanguage.language); + const expectedUri = Uri.file('one.py'); + document.setup(doc => doc.uri).returns(() => expectedUri); + documentManager.setup(doc => doc.activeTextEditor).returns(() => editor.object); + + const uri = await helper.getFileToExecute(); + expect(uri).to.be.deep.equal(expectedUri); + document.verify(doc => doc.save(), TypeMoq.Times.never()); + }); + test('Returns current line if nothing is selected', async () => { const lineContents = 'Line Contents'; editor.setup(e => e.selection).returns(() => new Selection(3, 0, 3, 0)); From 840b6ef1b6938ad298537f948c38de734c1bb957 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Wed, 18 Apr 2018 14:24:50 -0700 Subject: [PATCH 05/12] Save before executing code --- .../codeExecution/codeExecutionManager.ts | 1 + src/client/terminals/codeExecution/helper.ts | 6 ++++ src/client/terminals/types.ts | 1 + .../terminals/codeExecution/helper.test.ts | 33 +++++++++++++++++++ 4 files changed, 41 insertions(+) diff --git a/src/client/terminals/codeExecution/codeExecutionManager.ts b/src/client/terminals/codeExecution/codeExecutionManager.ts index f3db05292310..43257a094525 100644 --- a/src/client/terminals/codeExecution/codeExecutionManager.ts +++ b/src/client/terminals/codeExecution/codeExecutionManager.ts @@ -35,6 +35,7 @@ export class CodeExecutionManager implements ICodeExecutionManager { if (!fileToExecute) { return; } + await codeExecutionHelper.saveFileIfDirty(fileToExecute); const executionService = this.serviceContainer.get(ICodeExecutionService, 'standard'); await executionService.executeFile(fileToExecute); } diff --git a/src/client/terminals/codeExecution/helper.ts b/src/client/terminals/codeExecution/helper.ts index de335c7f7610..26a1246199d9 100644 --- a/src/client/terminals/codeExecution/helper.ts +++ b/src/client/terminals/codeExecution/helper.ts @@ -56,4 +56,10 @@ export class CodeExecutionHelper implements ICodeExecutionHelper { } return code; } + public async saveFileIfDirty(file: Uri): Promise { + const docs = this.documentManager.textDocuments.filter(d => d.uri.path === file.path); + if (docs.length === 1 && docs[0].isDirty) { + await docs[0].save(); + } + } } diff --git a/src/client/terminals/types.ts b/src/client/terminals/types.ts index 67c50d4b887b..c10e0bde7d1e 100644 --- a/src/client/terminals/types.ts +++ b/src/client/terminals/types.ts @@ -16,6 +16,7 @@ export const ICodeExecutionHelper = Symbol('ICodeExecutionHelper'); export interface ICodeExecutionHelper { normalizeLines(code: string): string; getFileToExecute(): Promise; + saveFileIfDirty(file: Uri): Promise; getSelectedTextToExecute(textEditor: TextEditor): Promise; } diff --git a/src/test/terminals/codeExecution/helper.test.ts b/src/test/terminals/codeExecution/helper.test.ts index 2ec1e85d3e01..fe00a2451957 100644 --- a/src/test/terminals/codeExecution/helper.test.ts +++ b/src/test/terminals/codeExecution/helper.test.ts @@ -133,4 +133,37 @@ suite('Terminal - Code Execution Helper', () => { const content = await helper.getSelectedTextToExecute(editor.object); expect(content).to.be.equal('3.0.10.5'); }); + + test('saveFileIfDirty will not fail if file is not opened', async () => { + documentManager.setup(d => d.textDocuments).returns(() => []).verifiable(TypeMoq.Times.once()); + + await helper.saveFileIfDirty(Uri.file(`${__filename}.py`)); + documentManager.verifyAll(); + }); + + test('File will be saved if file is dirty', async () => { + documentManager.setup(d => d.textDocuments).returns(() => [document.object]).verifiable(TypeMoq.Times.once()); + document.setup(doc => doc.isUntitled).returns(() => false); + document.setup(doc => doc.isDirty).returns(() => true); + document.setup(doc => doc.languageId).returns(() => PythonLanguage.language); + const expectedUri = Uri.file('one.py'); + document.setup(doc => doc.uri).returns(() => expectedUri); + + await helper.saveFileIfDirty(expectedUri); + documentManager.verifyAll(); + document.verify(doc => doc.save(), TypeMoq.Times.once()); + }); + + test('File will be not saved if file is not dirty', async () => { + documentManager.setup(d => d.textDocuments).returns(() => [document.object]).verifiable(TypeMoq.Times.once()); + document.setup(doc => doc.isUntitled).returns(() => false); + document.setup(doc => doc.isDirty).returns(() => false); + document.setup(doc => doc.languageId).returns(() => PythonLanguage.language); + const expectedUri = Uri.file('one.py'); + document.setup(doc => doc.uri).returns(() => expectedUri); + + await helper.saveFileIfDirty(expectedUri); + documentManager.verifyAll(); + document.verify(doc => doc.save(), TypeMoq.Times.never()); + }); }); From 03322b7d164033545598e06ede6bf988cbc68db2 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Wed, 18 Apr 2018 14:26:58 -0700 Subject: [PATCH 06/12] :memo: news entry --- news/1 Enhancements/1316.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/1 Enhancements/1316.md diff --git a/news/1 Enhancements/1316.md b/news/1 Enhancements/1316.md new file mode 100644 index 000000000000..ca6c6a80eb77 --- /dev/null +++ b/news/1 Enhancements/1316.md @@ -0,0 +1 @@ +Save the python file before running it in the terminal using the command/menu `Run Python File in Terminal`. From 2c4a166646fd58718ff9b2b31f6a2efac4927d3c Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Fri, 20 Apr 2018 14:10:42 -0700 Subject: [PATCH 07/12] :tada: --- news/2 Fixes/259.md | 1 + pythonFiles/experimental/ptvsd | 1 + pythonFiles/listNewLinesAndDedents.py | 23 ++++ .../codeExecution/codeExecutionManager.ts | 2 +- src/client/terminals/codeExecution/helper.ts | 112 ++++++++++++++++-- src/client/terminals/types.ts | 2 +- src/test/index.ts | 2 +- .../terminalExec/sample1_normalized.py | 19 +++ .../pythonFiles/terminalExec/sample1_raw.py | 18 +++ .../terminalExec/sample2_normalized.py | 7 ++ .../pythonFiles/terminalExec/sample2_raw.py | 8 ++ .../terminalExec/sample3_normalized.py | 5 + .../pythonFiles/terminalExec/sample3_raw.py | 5 + .../terminalExec/sample4_normalized.py | 7 ++ .../pythonFiles/terminalExec/sample4_raw.py | 7 ++ .../terminalExec/sample5_normalized.py | 9 ++ .../pythonFiles/terminalExec/sample5_raw.py | 11 ++ .../codeExecutionManager.test.ts | 2 +- .../terminals/codeExecution/helper.test.ts | 61 +++++++++- 19 files changed, 285 insertions(+), 17 deletions(-) create mode 100644 news/2 Fixes/259.md create mode 160000 pythonFiles/experimental/ptvsd create mode 100644 pythonFiles/listNewLinesAndDedents.py create mode 100644 src/test/pythonFiles/terminalExec/sample1_normalized.py create mode 100644 src/test/pythonFiles/terminalExec/sample1_raw.py create mode 100644 src/test/pythonFiles/terminalExec/sample2_normalized.py create mode 100644 src/test/pythonFiles/terminalExec/sample2_raw.py create mode 100644 src/test/pythonFiles/terminalExec/sample3_normalized.py create mode 100644 src/test/pythonFiles/terminalExec/sample3_raw.py create mode 100644 src/test/pythonFiles/terminalExec/sample4_normalized.py create mode 100644 src/test/pythonFiles/terminalExec/sample4_raw.py create mode 100644 src/test/pythonFiles/terminalExec/sample5_normalized.py create mode 100644 src/test/pythonFiles/terminalExec/sample5_raw.py diff --git a/news/2 Fixes/259.md b/news/2 Fixes/259.md new file mode 100644 index 000000000000..8579bc07f4cc --- /dev/null +++ b/news/2 Fixes/259.md @@ -0,0 +1 @@ +Add blank lines to seprate blocks of indented code (function defs, classes, and the like) to ensure the code can be run within a Python interactive prompt. diff --git a/pythonFiles/experimental/ptvsd b/pythonFiles/experimental/ptvsd new file mode 160000 index 000000000000..8355787a1024 --- /dev/null +++ b/pythonFiles/experimental/ptvsd @@ -0,0 +1 @@ +Subproject commit 8355787a1024c56ee87cbb76a41654e6d0c63725 diff --git a/pythonFiles/listNewLinesAndDedents.py b/pythonFiles/listNewLinesAndDedents.py new file mode 100644 index 000000000000..b8e180e4e56c --- /dev/null +++ b/pythonFiles/listNewLinesAndDedents.py @@ -0,0 +1,23 @@ +import io +import sys +import token +import tokenize + +try: + unicode +except: + unicode = str + +def printTokens(content): + tokens = tokenize.generate_tokens(io.StringIO(content).readline) + newline_detected = False + for toknum, _, spos, epos, line in tokens: + if len(line.strip()) > 0: + continue + elif toknum == token.DEDENT and spos[0] == epos[0] and newline_detected: + print('DEDENT, {}'.format(spos[0], epos[0])) + elif token.tok_name[toknum] == 'NL' and spos[0] == epos[0]: + newline_detected = True + print('NL, {}'.format(spos[0], epos[0])) + +printTokens(unicode(sys.argv[1])) diff --git a/src/client/terminals/codeExecution/codeExecutionManager.ts b/src/client/terminals/codeExecution/codeExecutionManager.ts index 43257a094525..04ccf5e61c19 100644 --- a/src/client/terminals/codeExecution/codeExecutionManager.ts +++ b/src/client/terminals/codeExecution/codeExecutionManager.ts @@ -60,7 +60,7 @@ export class CodeExecutionManager implements ICodeExecutionManager { } const codeExecutionHelper = this.serviceContainer.get(ICodeExecutionHelper); const codeToExecute = await codeExecutionHelper.getSelectedTextToExecute(activeEditor!); - const normalizedCode = codeExecutionHelper.normalizeLines(codeToExecute!); + const normalizedCode = await codeExecutionHelper.normalizeLines(codeToExecute!); if (!normalizedCode || normalizedCode.trim().length === 0) { return; } diff --git a/src/client/terminals/codeExecution/helper.ts b/src/client/terminals/codeExecution/helper.ts index 26a1246199d9..7f8aa5e9d164 100644 --- a/src/client/terminals/codeExecution/helper.ts +++ b/src/client/terminals/codeExecution/helper.ts @@ -2,23 +2,104 @@ // Licensed under the MIT License. import { inject, injectable } from 'inversify'; +import * as _ from 'lodash'; import { EOL } from 'os'; +import * as path from 'path'; import { Range, TextEditor, Uri } from 'vscode'; +import { RegistrationRequest } from 'vscode-languageclient/lib/main'; import { IApplicationShell, IDocumentManager } from '../../common/application/types'; -import { PythonLanguage } from '../../common/constants'; +import { EXTENSION_ROOT_DIR, PythonLanguage } from '../../common/constants'; import '../../common/extensions'; +import { IProcessService } from '../../common/process/types'; +import { IConfigurationService } from '../../common/types'; +import { IEnvironmentVariablesProvider } from '../../common/variables/types'; +import { IServiceContainer } from '../../ioc/types'; import { ICodeExecutionHelper } from '../types'; +// Tokens returned by Python Tokenizer. +enum Tokens { + Dedent = 'DEDENT', + NewLine = 'NL' +} +type LineAndDedent = { + type: Tokens.Dedent | Tokens.NewLine; + lineIndex: number; +}; + @injectable() export class CodeExecutionHelper implements ICodeExecutionHelper { - constructor(@inject(IDocumentManager) private documentManager: IDocumentManager, - @inject(IApplicationShell) private applicationShell: IApplicationShell) { - + private readonly documentManager: IDocumentManager; + private readonly applicationShell: IApplicationShell; + private readonly envVariablesProvider: IEnvironmentVariablesProvider; + private readonly processService: IProcessService; + private readonly configurationService: IConfigurationService; + constructor(@inject(IServiceContainer) private serviceContainer: IServiceContainer) { + this.documentManager = serviceContainer.get(IDocumentManager); + this.applicationShell = serviceContainer.get(IApplicationShell); + this.envVariablesProvider = serviceContainer.get(IEnvironmentVariablesProvider); + this.processService = serviceContainer.get(IProcessService); + this.configurationService = serviceContainer.get(IConfigurationService); } - public normalizeLines(code: string): string { - const codeLines = code.splitLines({ trim: false, removeEmptyEntries: false }); - const codeLinesWithoutEmptyLines = codeLines.filter(line => line.trim().length > 0); - return codeLinesWithoutEmptyLines.join(EOL); + public async normalizeLines(code: string, resource?: Uri): Promise { + try { + if (code.trim().length === 0) { + return ''; + } + const lines = code.splitLines({ trim: false, removeEmptyEntries: false }); + // const emptyFirstLine = lines.length > 0 && lines[0].length > 0 + const linesAndDedents = await this.getLinesAndDedents(code, resource); + + const hasTrailingLine = lines[lines.length - 1].trim().length === 0; + + // Remove empty lines + let nonEmptyLineFound = false; + const fixedLines = lines + .reverse() + .filter((line, i, items) => { + const isEmptyLine = line.trim().length === 0; + let skipThisLine = false; + if (!nonEmptyLineFound) { + if (isEmptyLine) { + skipThisLine = true; + } else { + nonEmptyLineFound = true; + } + } + const index = Math.abs(items.length - i - 1); + if (skipThisLine || linesAndDedents.findIndex(entry => entry.type === Tokens.NewLine && entry.lineIndex === index) >= 0) { + // Adjust line numbers for lines with DEDENTS + linesAndDedents + .filter(entry => entry.type === Tokens.Dedent && entry.lineIndex >= index) + .forEach(entry => entry.lineIndex -= 1); + return false; + } + return true; + }) + .reverse(); + + // Find dendented lines and add blank lines above it + // We're only interested in lines that are not already indented + const re = new RegExp('^\\\s+\\S+'); + linesAndDedents + .filter(entry => entry.type === Tokens.Dedent && entry.lineIndex < fixedLines.length) + .reverse() + .forEach(entry => { + if (re.test(fixedLines[entry.lineIndex])) { + fixedLines.splice(entry.lineIndex, 0, ''); + } else { + // Add a blank line, and ensure that link is indented at this same level + const line = fixedLines[entry.lineIndex]; + const indentation = line.substr(0, line.indexOf(line.trim())); + fixedLines.splice(entry.lineIndex, 0, indentation); + } + }); + + // If we had a trailing line, then add it + return fixedLines.join(EOL) + (hasTrailingLine ? EOL : ''); + } catch (ex) { + console.error(ex, 'Python: Failed to normalize code for execution in terminal'); + return code; + } } public async getFileToExecute(): Promise { @@ -62,4 +143,19 @@ export class CodeExecutionHelper implements ICodeExecutionHelper { await docs[0].save(); } } + + private async getLinesAndDedents(source: string, resource?: Uri): Promise { + const env = await this.envVariablesProvider.getEnvironmentVariables(resource); + const pythonPath = this.configurationService.getSettings(resource).pythonPath; + const args = [path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'listNewLinesAndDedents.py'), source]; + const proc = await this.processService.exec(pythonPath, args, { env, throwOnStdErr: true }); + const entries = proc.stdout.splitLines({ removeEmptyEntries: true, trim: true }); + return entries.map(line => { + const parts = line.split(','); + return { + type: parts[0] as Tokens, + lineIndex: parseInt(parts[1], 10) - 1 + }; + }); + } } diff --git a/src/client/terminals/types.ts b/src/client/terminals/types.ts index c10e0bde7d1e..cd9e1bc96a9b 100644 --- a/src/client/terminals/types.ts +++ b/src/client/terminals/types.ts @@ -14,7 +14,7 @@ export interface ICodeExecutionService { export const ICodeExecutionHelper = Symbol('ICodeExecutionHelper'); export interface ICodeExecutionHelper { - normalizeLines(code: string): string; + normalizeLines(code: string): Promise; getFileToExecute(): Promise; saveFileIfDirty(file: Uri): Promise; getSelectedTextToExecute(textEditor: TextEditor): Promise; diff --git a/src/test/index.ts b/src/test/index.ts index 135ca5d8b6c1..dbd835ed92ae 100644 --- a/src/test/index.ts +++ b/src/test/index.ts @@ -13,7 +13,7 @@ process.env.IS_MULTI_ROOT_TEST = IS_MULTI_ROOT_TEST.toString(); // If running on CI server and we're running the debugger tests, then ensure we only run debug tests. // We do this to ensure we only run debugger test, as debugger tests are very flaky on CI. // So the solution is to run them separately and first on CI. -const grep = IS_CI_SERVER && IS_CI_SERVER_TEST_DEBUGGER ? 'Debug' : undefined; +const grep = IS_CI_SERVER && IS_CI_SERVER_TEST_DEBUGGER ? 'Debug' : 'Terminal - Code Execution Helper'; // You can directly control Mocha options by uncommenting the following lines. // See https://github.com/mochajs/mocha/wiki/Using-mocha-programmatically#set-options for more info. diff --git a/src/test/pythonFiles/terminalExec/sample1_normalized.py b/src/test/pythonFiles/terminalExec/sample1_normalized.py new file mode 100644 index 000000000000..1dfae64f6f17 --- /dev/null +++ b/src/test/pythonFiles/terminalExec/sample1_normalized.py @@ -0,0 +1,19 @@ +# Sample block 1 +def square(x): + return x**2 + +print('hello') +# Sample block 2 +a = 2 +if a < 2: + print('less than 2') + +else: + print('more than 2') + +print('hello') +# Sample block 3 +for i in range(5): + print(i) + +print('complete') diff --git a/src/test/pythonFiles/terminalExec/sample1_raw.py b/src/test/pythonFiles/terminalExec/sample1_raw.py new file mode 100644 index 000000000000..aea382c6b2cd --- /dev/null +++ b/src/test/pythonFiles/terminalExec/sample1_raw.py @@ -0,0 +1,18 @@ +# Sample block 1 +def square(x): + return x**2 + +print('hello') +# Sample block 2 +a = 2 +if a < 2: + print('less than 2') +else: + print('more than 2') + +print('hello') +# Sample block 3 +for i in range(5): + print(i) + +print('complete') diff --git a/src/test/pythonFiles/terminalExec/sample2_normalized.py b/src/test/pythonFiles/terminalExec/sample2_normalized.py new file mode 100644 index 000000000000..a333d4e0daae --- /dev/null +++ b/src/test/pythonFiles/terminalExec/sample2_normalized.py @@ -0,0 +1,7 @@ +def add(x, y): + """Adds x to y""" + # Some comment + return x + y + +v = add(1, 7) +print(v) diff --git a/src/test/pythonFiles/terminalExec/sample2_raw.py b/src/test/pythonFiles/terminalExec/sample2_raw.py new file mode 100644 index 000000000000..6ab7e67637f4 --- /dev/null +++ b/src/test/pythonFiles/terminalExec/sample2_raw.py @@ -0,0 +1,8 @@ +def add(x, y): + """Adds x to y""" + # Some comment + + return x + y + +v = add(1, 7) +print(v) diff --git a/src/test/pythonFiles/terminalExec/sample3_normalized.py b/src/test/pythonFiles/terminalExec/sample3_normalized.py new file mode 100644 index 000000000000..4fa62091c66d --- /dev/null +++ b/src/test/pythonFiles/terminalExec/sample3_normalized.py @@ -0,0 +1,5 @@ +if True: + print(1) + print(2) + +print(3) diff --git a/src/test/pythonFiles/terminalExec/sample3_raw.py b/src/test/pythonFiles/terminalExec/sample3_raw.py new file mode 100644 index 000000000000..5865e6d2cbde --- /dev/null +++ b/src/test/pythonFiles/terminalExec/sample3_raw.py @@ -0,0 +1,5 @@ +if True: + print(1) + + print(2) +print(3) diff --git a/src/test/pythonFiles/terminalExec/sample4_normalized.py b/src/test/pythonFiles/terminalExec/sample4_normalized.py new file mode 100644 index 000000000000..b3468dafe8d8 --- /dev/null +++ b/src/test/pythonFiles/terminalExec/sample4_normalized.py @@ -0,0 +1,7 @@ +class pc(object): + def __init__(self, pcname, model): + self.pcname = pcname + self.model = model + + def print_name(self): + print('Workstation name is', self.pcname, 'model is', self.model) diff --git a/src/test/pythonFiles/terminalExec/sample4_raw.py b/src/test/pythonFiles/terminalExec/sample4_raw.py new file mode 100644 index 000000000000..fbf0d68fe5f8 --- /dev/null +++ b/src/test/pythonFiles/terminalExec/sample4_raw.py @@ -0,0 +1,7 @@ +class pc(object): + def __init__(self, pcname, model): + self.pcname = pcname + self.model = model + + def print_name(self): + print('Workstation name is', self.pcname, 'model is', self.model) diff --git a/src/test/pythonFiles/terminalExec/sample5_normalized.py b/src/test/pythonFiles/terminalExec/sample5_normalized.py new file mode 100644 index 000000000000..954edb313a73 --- /dev/null +++ b/src/test/pythonFiles/terminalExec/sample5_normalized.py @@ -0,0 +1,9 @@ +for i in range(10): + print('a') + for j in range(5): + print('b') + print('b2') + for k in range(2): + print('c') + + print('done with first loop') diff --git a/src/test/pythonFiles/terminalExec/sample5_raw.py b/src/test/pythonFiles/terminalExec/sample5_raw.py new file mode 100644 index 000000000000..d1415d4bf5de --- /dev/null +++ b/src/test/pythonFiles/terminalExec/sample5_raw.py @@ -0,0 +1,11 @@ +for i in range(10): + print('a') + for j in range(5): + print('b') + + print('b2') + + for k in range(2): + print('c') + + print('done with first loop') diff --git a/src/test/terminals/codeExecution/codeExecutionManager.test.ts b/src/test/terminals/codeExecution/codeExecutionManager.test.ts index 17696de0e189..05c395a1e571 100644 --- a/src/test/terminals/codeExecution/codeExecutionManager.test.ts +++ b/src/test/terminals/codeExecution/codeExecutionManager.test.ts @@ -218,7 +218,7 @@ suite('Terminal - Code Execution Manager', () => { const helper = TypeMoq.Mock.ofType(); serviceContainer.setup(s => s.get(TypeMoq.It.isValue(ICodeExecutionHelper))).returns(() => helper.object); helper.setup(h => h.getSelectedTextToExecute).returns(() => () => Promise.resolve(textSelected)); - helper.setup(h => h.normalizeLines).returns(() => () => textSelected).verifiable(TypeMoq.Times.once()); + helper.setup(h => h.normalizeLines).returns(() => () => Promise.resolve(textSelected)).verifiable(TypeMoq.Times.once()); const executionService = TypeMoq.Mock.ofType(); serviceContainer.setup(s => s.get(TypeMoq.It.isValue(ICodeExecutionService), TypeMoq.It.isValue(executionServiceId))).returns(() => executionService.object); const document = TypeMoq.Mock.ofType(); diff --git a/src/test/terminals/codeExecution/helper.test.ts b/src/test/terminals/codeExecution/helper.test.ts index fe00a2451957..36df594316bf 100644 --- a/src/test/terminals/codeExecution/helper.test.ts +++ b/src/test/terminals/codeExecution/helper.test.ts @@ -4,13 +4,24 @@ // tslint:disable:no-multiline-string no-trailing-whitespace import { expect } from 'chai'; +import * as fs from 'fs-extra'; import { EOL } from 'os'; +import * as path from 'path'; import * as TypeMoq from 'typemoq'; import { Range, Selection, TextDocument, TextEditor, TextLine, Uri } from 'vscode'; import { IApplicationShell, IDocumentManager } from '../../../client/common/application/types'; -import { PythonLanguage } from '../../../client/common/constants'; +import { EXTENSION_ROOT_DIR, PythonLanguage } from '../../../client/common/constants'; +import { BufferDecoder } from '../../../client/common/process/decoder'; +import { ProcessService } from '../../../client/common/process/proc'; +import { IProcessService } from '../../../client/common/process/types'; +import { IConfigurationService, IPythonSettings } from '../../../client/common/types'; +import { IEnvironmentVariablesProvider } from '../../../client/common/variables/types'; +import { IServiceContainer } from '../../../client/ioc/types'; import { CodeExecutionHelper } from '../../../client/terminals/codeExecution/helper'; import { ICodeExecutionHelper } from '../../../client/terminals/types'; +import { PYTHON_PATH } from '../../common'; + +const TEST_FILES_PATH = path.join(EXTENSION_ROOT_DIR, 'src', 'test', 'pythonFiles', 'terminalExec'); // tslint:disable-next-line:max-func-body-length suite('Terminal - Code Execution Helper', () => { @@ -19,21 +30,61 @@ suite('Terminal - Code Execution Helper', () => { let helper: ICodeExecutionHelper; let document: TypeMoq.IMock; let editor: TypeMoq.IMock; + let processService: TypeMoq.IMock; + let configService: TypeMoq.IMock; setup(() => { + const serviceContainer = TypeMoq.Mock.ofType(); documentManager = TypeMoq.Mock.ofType(); applicationShell = TypeMoq.Mock.ofType(); - helper = new CodeExecutionHelper(documentManager.object, applicationShell.object); + const envVariablesProvider = TypeMoq.Mock.ofType(); + processService = TypeMoq.Mock.ofType(); + configService = TypeMoq.Mock.ofType(); + const pythonSettings = TypeMoq.Mock.ofType(); + pythonSettings.setup(p => p.pythonPath).returns(() => PYTHON_PATH); + configService.setup(c => c.getSettings(TypeMoq.It.isAny())).returns(() => pythonSettings.object); + envVariablesProvider.setup(e => e.getEnvironmentVariables(TypeMoq.It.isAny())).returns(() => Promise.resolve({})); + serviceContainer.setup(c => c.get(TypeMoq.It.isValue(IDocumentManager), TypeMoq.It.isAny())).returns(() => documentManager.object); + serviceContainer.setup(c => c.get(TypeMoq.It.isValue(IApplicationShell), TypeMoq.It.isAny())).returns(() => applicationShell.object); + serviceContainer.setup(c => c.get(TypeMoq.It.isValue(IEnvironmentVariablesProvider), TypeMoq.It.isAny())).returns(() => envVariablesProvider.object); + serviceContainer.setup(c => c.get(TypeMoq.It.isValue(IProcessService), TypeMoq.It.isAny())).returns(() => processService.object); + serviceContainer.setup(c => c.get(TypeMoq.It.isValue(IConfigurationService), TypeMoq.It.isAny())).returns(() => configService.object); + helper = new CodeExecutionHelper(serviceContainer.object); document = TypeMoq.Mock.ofType(); editor = TypeMoq.Mock.ofType(); editor.setup(e => e.document).returns(() => document.object); }); - test('Ensure blank lines are removed', async () => { + async function ensureBlankLinesAreRemoved(source: string, expectedSource: string) { + const actualProcessService = new ProcessService(new BufferDecoder()); + processService.setup(p => p.exec(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny())) + .returns((file, args, options) => { + return actualProcessService.exec.apply(actualProcessService, [file, args, options]); + }); + const normalizedZCode = await helper.normalizeLines(source); + expect(normalizedZCode).to.be.equal(expectedSource); + } + test('Ensure blank lines are removed (simple)', async () => { const code = ['import sys', '', '', '', 'print(sys.executable)', '', 'print("1234")', '', '', 'print(1)', 'print(2)']; const expectedCode = code.filter(line => line.trim().length > 0).join(EOL); - const normalizedZCode = helper.normalizeLines(code.join(EOL)); - expect(normalizedZCode).to.be.equal(expectedCode); + await ensureBlankLinesAreRemoved(code.join(EOL), expectedCode); + }); + ['sample1', 'sample2', 'sample3', 'sample4', 'sample5'].forEach(fileName => { + test(`Ensure blank lines are removed (${fileName})`, async () => { + const code = await fs.readFile(path.join(TEST_FILES_PATH, `${fileName}_raw.py`), 'utf8'); + const expectedCode = await fs.readFile(path.join(TEST_FILES_PATH, `${fileName}_normalized.py`), 'utf8'); + await ensureBlankLinesAreRemoved(code, expectedCode); + }); + test(`Ensure blank lines are removed, including leading empty lines (${fileName})`, async () => { + const code = await fs.readFile(path.join(TEST_FILES_PATH, `${fileName}_raw.py`), 'utf8'); + const expectedCode = await fs.readFile(path.join(TEST_FILES_PATH, `${fileName}_normalized.py`), 'utf8'); + await ensureBlankLinesAreRemoved(['', '', ''].join(EOL) + EOL + code, expectedCode); + }); + }); + test('Ensure blank lines are removed (sample2)', async () => { + const code = await fs.readFile(path.join(TEST_FILES_PATH, 'sample2_raw.py'), 'utf8'); + const expectedCode = await fs.readFile(path.join(TEST_FILES_PATH, 'sample2_normalized.py'), 'utf8'); + await ensureBlankLinesAreRemoved(code, expectedCode); }); test('Display message if there\s no active file', async () => { documentManager.setup(doc => doc.activeTextEditor).returns(() => undefined); From 748fdb65a4589536f8c1f9352fd94be53b0d9c4a Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Fri, 20 Apr 2018 15:06:05 -0700 Subject: [PATCH 08/12] more changes --- pythonFiles/listNewLinesAndDedents.py | 58 +++++++++++--- src/client/terminals/codeExecution/helper.ts | 81 ++------------------ 2 files changed, 52 insertions(+), 87 deletions(-) diff --git a/pythonFiles/listNewLinesAndDedents.py b/pythonFiles/listNewLinesAndDedents.py index b8e180e4e56c..3710659423fa 100644 --- a/pythonFiles/listNewLinesAndDedents.py +++ b/pythonFiles/listNewLinesAndDedents.py @@ -1,4 +1,8 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + import io +import os import sys import token import tokenize @@ -8,16 +12,48 @@ except: unicode = str -def printTokens(content): + +def normalizeLines(content): + """Removes empty lines and adds empty lines only to sepaparate + indented code. So that the code can be used for execution in + the Python interactive prompt """ + + lines = content.splitlines(False) + + # Find out if we have any trailing blank lines + has_blank_lines = len(lines[-1].strip()) == 0 or content.endswith(os.linesep) + + # Remove empty lines tokens = tokenize.generate_tokens(io.StringIO(content).readline) - newline_detected = False + + new_lines_to_remove = [] for toknum, _, spos, epos, line in tokens: - if len(line.strip()) > 0: - continue - elif toknum == token.DEDENT and spos[0] == epos[0] and newline_detected: - print('DEDENT, {}'.format(spos[0], epos[0])) - elif token.tok_name[toknum] == 'NL' and spos[0] == epos[0]: - newline_detected = True - print('NL, {}'.format(spos[0], epos[0])) - -printTokens(unicode(sys.argv[1])) + if token.tok_name[toknum] == 'NL' and len(line.strip()) == 0 and spos[0] == epos[0]: + new_lines_to_remove.append(spos[0] - 1) + + for line_index in reversed(new_lines_to_remove): + lines.pop(line_index) + + # Add new lines just before every dedent + content = os.linesep.join(lines) + tokens = tokenize.generate_tokens(io.StringIO(content).readline) + dedented_lines = [] + for toknum, _, spos, epos, line in tokens: + if toknum == token.DEDENT and spos[0] == epos[0] and spos[0] <= len(lines): + index = spos[0] - 1 + if not index in dedented_lines: + dedented_lines.append(index) + + for line_index in reversed(dedented_lines): + line = lines[line_index] + indent_size = line.index(line.strip()) + indentation = line[0:indent_size] + lines.insert(line_index, indentation) + + sys.stdout.write(os.linesep.join(lines) + (os.linesep if has_blank_lines else '')) + sys.stdout.flush() + + +if __name__ == '__main__': + contents = unicode(sys.argv[1]) + normalizeLines(contents) diff --git a/src/client/terminals/codeExecution/helper.ts b/src/client/terminals/codeExecution/helper.ts index 7f8aa5e9d164..ad90f0fbd760 100644 --- a/src/client/terminals/codeExecution/helper.ts +++ b/src/client/terminals/codeExecution/helper.ts @@ -16,16 +16,6 @@ import { IEnvironmentVariablesProvider } from '../../common/variables/types'; import { IServiceContainer } from '../../ioc/types'; import { ICodeExecutionHelper } from '../types'; -// Tokens returned by Python Tokenizer. -enum Tokens { - Dedent = 'DEDENT', - NewLine = 'NL' -} -type LineAndDedent = { - type: Tokens.Dedent | Tokens.NewLine; - lineIndex: number; -}; - @injectable() export class CodeExecutionHelper implements ICodeExecutionHelper { private readonly documentManager: IDocumentManager; @@ -45,57 +35,11 @@ export class CodeExecutionHelper implements ICodeExecutionHelper { if (code.trim().length === 0) { return ''; } - const lines = code.splitLines({ trim: false, removeEmptyEntries: false }); - // const emptyFirstLine = lines.length > 0 && lines[0].length > 0 - const linesAndDedents = await this.getLinesAndDedents(code, resource); - - const hasTrailingLine = lines[lines.length - 1].trim().length === 0; - - // Remove empty lines - let nonEmptyLineFound = false; - const fixedLines = lines - .reverse() - .filter((line, i, items) => { - const isEmptyLine = line.trim().length === 0; - let skipThisLine = false; - if (!nonEmptyLineFound) { - if (isEmptyLine) { - skipThisLine = true; - } else { - nonEmptyLineFound = true; - } - } - const index = Math.abs(items.length - i - 1); - if (skipThisLine || linesAndDedents.findIndex(entry => entry.type === Tokens.NewLine && entry.lineIndex === index) >= 0) { - // Adjust line numbers for lines with DEDENTS - linesAndDedents - .filter(entry => entry.type === Tokens.Dedent && entry.lineIndex >= index) - .forEach(entry => entry.lineIndex -= 1); - return false; - } - return true; - }) - .reverse(); - - // Find dendented lines and add blank lines above it - // We're only interested in lines that are not already indented - const re = new RegExp('^\\\s+\\S+'); - linesAndDedents - .filter(entry => entry.type === Tokens.Dedent && entry.lineIndex < fixedLines.length) - .reverse() - .forEach(entry => { - if (re.test(fixedLines[entry.lineIndex])) { - fixedLines.splice(entry.lineIndex, 0, ''); - } else { - // Add a blank line, and ensure that link is indented at this same level - const line = fixedLines[entry.lineIndex]; - const indentation = line.substr(0, line.indexOf(line.trim())); - fixedLines.splice(entry.lineIndex, 0, indentation); - } - }); - - // If we had a trailing line, then add it - return fixedLines.join(EOL) + (hasTrailingLine ? EOL : ''); + const env = await this.envVariablesProvider.getEnvironmentVariables(resource); + const pythonPath = this.configurationService.getSettings(resource).pythonPath; + const args = [path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'listNewLinesAndDedents.py'), code]; + const proc = await this.processService.exec(pythonPath, args, { env, throwOnStdErr: true }); + return proc.stdout; } catch (ex) { console.error(ex, 'Python: Failed to normalize code for execution in terminal'); return code; @@ -143,19 +87,4 @@ export class CodeExecutionHelper implements ICodeExecutionHelper { await docs[0].save(); } } - - private async getLinesAndDedents(source: string, resource?: Uri): Promise { - const env = await this.envVariablesProvider.getEnvironmentVariables(resource); - const pythonPath = this.configurationService.getSettings(resource).pythonPath; - const args = [path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'listNewLinesAndDedents.py'), source]; - const proc = await this.processService.exec(pythonPath, args, { env, throwOnStdErr: true }); - const entries = proc.stdout.splitLines({ removeEmptyEntries: true, trim: true }); - return entries.map(line => { - const parts = line.split(','); - return { - type: parts[0] as Tokens, - lineIndex: parseInt(parts[1], 10) - 1 - }; - }); - } } From a1ee823ec7ba142c97963707aaa47b94959ba123 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Fri, 20 Apr 2018 15:13:02 -0700 Subject: [PATCH 09/12] :hammer: rename file --- ...istNewLinesAndDedents.py => normalizeForInterpreter.py} | 0 src/client/terminals/codeExecution/helper.ts | 7 ++----- 2 files changed, 2 insertions(+), 5 deletions(-) rename pythonFiles/{listNewLinesAndDedents.py => normalizeForInterpreter.py} (100%) diff --git a/pythonFiles/listNewLinesAndDedents.py b/pythonFiles/normalizeForInterpreter.py similarity index 100% rename from pythonFiles/listNewLinesAndDedents.py rename to pythonFiles/normalizeForInterpreter.py diff --git a/src/client/terminals/codeExecution/helper.ts b/src/client/terminals/codeExecution/helper.ts index ad90f0fbd760..0382b3ebd6eb 100644 --- a/src/client/terminals/codeExecution/helper.ts +++ b/src/client/terminals/codeExecution/helper.ts @@ -2,11 +2,8 @@ // Licensed under the MIT License. import { inject, injectable } from 'inversify'; -import * as _ from 'lodash'; -import { EOL } from 'os'; import * as path from 'path'; import { Range, TextEditor, Uri } from 'vscode'; -import { RegistrationRequest } from 'vscode-languageclient/lib/main'; import { IApplicationShell, IDocumentManager } from '../../common/application/types'; import { EXTENSION_ROOT_DIR, PythonLanguage } from '../../common/constants'; import '../../common/extensions'; @@ -23,7 +20,7 @@ export class CodeExecutionHelper implements ICodeExecutionHelper { private readonly envVariablesProvider: IEnvironmentVariablesProvider; private readonly processService: IProcessService; private readonly configurationService: IConfigurationService; - constructor(@inject(IServiceContainer) private serviceContainer: IServiceContainer) { + constructor(@inject(IServiceContainer) serviceContainer: IServiceContainer) { this.documentManager = serviceContainer.get(IDocumentManager); this.applicationShell = serviceContainer.get(IApplicationShell); this.envVariablesProvider = serviceContainer.get(IEnvironmentVariablesProvider); @@ -37,7 +34,7 @@ export class CodeExecutionHelper implements ICodeExecutionHelper { } const env = await this.envVariablesProvider.getEnvironmentVariables(resource); const pythonPath = this.configurationService.getSettings(resource).pythonPath; - const args = [path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'listNewLinesAndDedents.py'), code]; + const args = [path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'normalizeForInterpreter.py'), code]; const proc = await this.processService.exec(pythonPath, args, { env, throwOnStdErr: true }); return proc.stdout; } catch (ex) { From 057ff8f991d493ddbf65af03ee4163836c6d2520 Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Mon, 23 Apr 2018 13:33:52 -0700 Subject: [PATCH 10/12] Fix tests --- pythonFiles/normalizeForInterpreter.py | 59 ------------------- src/client/terminals/codeExecution/helper.ts | 24 +++----- src/test/index.ts | 2 +- .../terminalExec/sample1_normalized.py | 5 +- .../pythonFiles/terminalExec/sample1_raw.py | 6 ++ .../terminalExec/sample3_normalized.py | 1 - .../terminalExec/sample4_normalized.py | 1 - .../terminalExec/sample5_normalized.py | 1 - .../pythonFiles/terminalExec/sample5_raw.py | 2 +- .../terminals/codeExecution/helper.test.ts | 39 +++--------- 10 files changed, 28 insertions(+), 112 deletions(-) delete mode 100644 pythonFiles/normalizeForInterpreter.py diff --git a/pythonFiles/normalizeForInterpreter.py b/pythonFiles/normalizeForInterpreter.py deleted file mode 100644 index 3710659423fa..000000000000 --- a/pythonFiles/normalizeForInterpreter.py +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -import io -import os -import sys -import token -import tokenize - -try: - unicode -except: - unicode = str - - -def normalizeLines(content): - """Removes empty lines and adds empty lines only to sepaparate - indented code. So that the code can be used for execution in - the Python interactive prompt """ - - lines = content.splitlines(False) - - # Find out if we have any trailing blank lines - has_blank_lines = len(lines[-1].strip()) == 0 or content.endswith(os.linesep) - - # Remove empty lines - tokens = tokenize.generate_tokens(io.StringIO(content).readline) - - new_lines_to_remove = [] - for toknum, _, spos, epos, line in tokens: - if token.tok_name[toknum] == 'NL' and len(line.strip()) == 0 and spos[0] == epos[0]: - new_lines_to_remove.append(spos[0] - 1) - - for line_index in reversed(new_lines_to_remove): - lines.pop(line_index) - - # Add new lines just before every dedent - content = os.linesep.join(lines) - tokens = tokenize.generate_tokens(io.StringIO(content).readline) - dedented_lines = [] - for toknum, _, spos, epos, line in tokens: - if toknum == token.DEDENT and spos[0] == epos[0] and spos[0] <= len(lines): - index = spos[0] - 1 - if not index in dedented_lines: - dedented_lines.append(index) - - for line_index in reversed(dedented_lines): - line = lines[line_index] - indent_size = line.index(line.strip()) - indentation = line[0:indent_size] - lines.insert(line_index, indentation) - - sys.stdout.write(os.linesep.join(lines) + (os.linesep if has_blank_lines else '')) - sys.stdout.flush() - - -if __name__ == '__main__': - contents = unicode(sys.argv[1]) - normalizeLines(contents) diff --git a/src/client/terminals/codeExecution/helper.ts b/src/client/terminals/codeExecution/helper.ts index 0382b3ebd6eb..555a64b11a3f 100644 --- a/src/client/terminals/codeExecution/helper.ts +++ b/src/client/terminals/codeExecution/helper.ts @@ -2,14 +2,10 @@ // Licensed under the MIT License. import { inject, injectable } from 'inversify'; -import * as path from 'path'; import { Range, TextEditor, Uri } from 'vscode'; import { IApplicationShell, IDocumentManager } from '../../common/application/types'; -import { EXTENSION_ROOT_DIR, PythonLanguage } from '../../common/constants'; +import { PythonLanguage } from '../../common/constants'; import '../../common/extensions'; -import { IProcessService } from '../../common/process/types'; -import { IConfigurationService } from '../../common/types'; -import { IEnvironmentVariablesProvider } from '../../common/variables/types'; import { IServiceContainer } from '../../ioc/types'; import { ICodeExecutionHelper } from '../types'; @@ -17,26 +13,22 @@ import { ICodeExecutionHelper } from '../types'; export class CodeExecutionHelper implements ICodeExecutionHelper { private readonly documentManager: IDocumentManager; private readonly applicationShell: IApplicationShell; - private readonly envVariablesProvider: IEnvironmentVariablesProvider; - private readonly processService: IProcessService; - private readonly configurationService: IConfigurationService; constructor(@inject(IServiceContainer) serviceContainer: IServiceContainer) { this.documentManager = serviceContainer.get(IDocumentManager); this.applicationShell = serviceContainer.get(IApplicationShell); - this.envVariablesProvider = serviceContainer.get(IEnvironmentVariablesProvider); - this.processService = serviceContainer.get(IProcessService); - this.configurationService = serviceContainer.get(IConfigurationService); } public async normalizeLines(code: string, resource?: Uri): Promise { try { if (code.trim().length === 0) { return ''; } - const env = await this.envVariablesProvider.getEnvironmentVariables(resource); - const pythonPath = this.configurationService.getSettings(resource).pythonPath; - const args = [path.join(EXTENSION_ROOT_DIR, 'pythonFiles', 'normalizeForInterpreter.py'), code]; - const proc = await this.processService.exec(pythonPath, args, { env, throwOnStdErr: true }); - return proc.stdout; + const regex = /(\n)([ \t]*\n)([ \t]+\S+)/gm; + return code.replace(regex, (_, a, b, c) => { + // console.log('found'); + // console.log(match); + // console.log(`${a}${c}`); + return `${a}${c}`; + }); } catch (ex) { console.error(ex, 'Python: Failed to normalize code for execution in terminal'); return code; diff --git a/src/test/index.ts b/src/test/index.ts index dbd835ed92ae..135ca5d8b6c1 100644 --- a/src/test/index.ts +++ b/src/test/index.ts @@ -13,7 +13,7 @@ process.env.IS_MULTI_ROOT_TEST = IS_MULTI_ROOT_TEST.toString(); // If running on CI server and we're running the debugger tests, then ensure we only run debug tests. // We do this to ensure we only run debugger test, as debugger tests are very flaky on CI. // So the solution is to run them separately and first on CI. -const grep = IS_CI_SERVER && IS_CI_SERVER_TEST_DEBUGGER ? 'Debug' : 'Terminal - Code Execution Helper'; +const grep = IS_CI_SERVER && IS_CI_SERVER_TEST_DEBUGGER ? 'Debug' : undefined; // You can directly control Mocha options by uncommenting the following lines. // See https://github.com/mochajs/mocha/wiki/Using-mocha-programmatically#set-options for more info. diff --git a/src/test/pythonFiles/terminalExec/sample1_normalized.py b/src/test/pythonFiles/terminalExec/sample1_normalized.py index 1dfae64f6f17..0896de65d22f 100644 --- a/src/test/pythonFiles/terminalExec/sample1_normalized.py +++ b/src/test/pythonFiles/terminalExec/sample1_normalized.py @@ -7,13 +7,16 @@ def square(x): a = 2 if a < 2: print('less than 2') - else: print('more than 2') print('hello') + # Sample block 3 for i in range(5): print(i) + print(i) + print(i) + print(i) print('complete') diff --git a/src/test/pythonFiles/terminalExec/sample1_raw.py b/src/test/pythonFiles/terminalExec/sample1_raw.py index aea382c6b2cd..fe050c7af289 100644 --- a/src/test/pythonFiles/terminalExec/sample1_raw.py +++ b/src/test/pythonFiles/terminalExec/sample1_raw.py @@ -11,8 +11,14 @@ def square(x): print('more than 2') print('hello') + # Sample block 3 for i in range(5): print(i) + print(i) + print(i) + + print(i) + print('complete') diff --git a/src/test/pythonFiles/terminalExec/sample3_normalized.py b/src/test/pythonFiles/terminalExec/sample3_normalized.py index 4fa62091c66d..e4f028b0b778 100644 --- a/src/test/pythonFiles/terminalExec/sample3_normalized.py +++ b/src/test/pythonFiles/terminalExec/sample3_normalized.py @@ -1,5 +1,4 @@ if True: print(1) print(2) - print(3) diff --git a/src/test/pythonFiles/terminalExec/sample4_normalized.py b/src/test/pythonFiles/terminalExec/sample4_normalized.py index b3468dafe8d8..2c49d10253ff 100644 --- a/src/test/pythonFiles/terminalExec/sample4_normalized.py +++ b/src/test/pythonFiles/terminalExec/sample4_normalized.py @@ -2,6 +2,5 @@ class pc(object): def __init__(self, pcname, model): self.pcname = pcname self.model = model - def print_name(self): print('Workstation name is', self.pcname, 'model is', self.model) diff --git a/src/test/pythonFiles/terminalExec/sample5_normalized.py b/src/test/pythonFiles/terminalExec/sample5_normalized.py index 954edb313a73..822d51bd15d9 100644 --- a/src/test/pythonFiles/terminalExec/sample5_normalized.py +++ b/src/test/pythonFiles/terminalExec/sample5_normalized.py @@ -5,5 +5,4 @@ print('b2') for k in range(2): print('c') - print('done with first loop') diff --git a/src/test/pythonFiles/terminalExec/sample5_raw.py b/src/test/pythonFiles/terminalExec/sample5_raw.py index d1415d4bf5de..19caa9cf26a6 100644 --- a/src/test/pythonFiles/terminalExec/sample5_raw.py +++ b/src/test/pythonFiles/terminalExec/sample5_raw.py @@ -7,5 +7,5 @@ for k in range(2): print('c') - + print('done with first loop') diff --git a/src/test/terminals/codeExecution/helper.test.ts b/src/test/terminals/codeExecution/helper.test.ts index 36df594316bf..a9344344d548 100644 --- a/src/test/terminals/codeExecution/helper.test.ts +++ b/src/test/terminals/codeExecution/helper.test.ts @@ -11,15 +11,9 @@ import * as TypeMoq from 'typemoq'; import { Range, Selection, TextDocument, TextEditor, TextLine, Uri } from 'vscode'; import { IApplicationShell, IDocumentManager } from '../../../client/common/application/types'; import { EXTENSION_ROOT_DIR, PythonLanguage } from '../../../client/common/constants'; -import { BufferDecoder } from '../../../client/common/process/decoder'; -import { ProcessService } from '../../../client/common/process/proc'; -import { IProcessService } from '../../../client/common/process/types'; -import { IConfigurationService, IPythonSettings } from '../../../client/common/types'; -import { IEnvironmentVariablesProvider } from '../../../client/common/variables/types'; import { IServiceContainer } from '../../../client/ioc/types'; import { CodeExecutionHelper } from '../../../client/terminals/codeExecution/helper'; import { ICodeExecutionHelper } from '../../../client/terminals/types'; -import { PYTHON_PATH } from '../../common'; const TEST_FILES_PATH = path.join(EXTENSION_ROOT_DIR, 'src', 'test', 'pythonFiles', 'terminalExec'); @@ -30,24 +24,12 @@ suite('Terminal - Code Execution Helper', () => { let helper: ICodeExecutionHelper; let document: TypeMoq.IMock; let editor: TypeMoq.IMock; - let processService: TypeMoq.IMock; - let configService: TypeMoq.IMock; setup(() => { const serviceContainer = TypeMoq.Mock.ofType(); documentManager = TypeMoq.Mock.ofType(); applicationShell = TypeMoq.Mock.ofType(); - const envVariablesProvider = TypeMoq.Mock.ofType(); - processService = TypeMoq.Mock.ofType(); - configService = TypeMoq.Mock.ofType(); - const pythonSettings = TypeMoq.Mock.ofType(); - pythonSettings.setup(p => p.pythonPath).returns(() => PYTHON_PATH); - configService.setup(c => c.getSettings(TypeMoq.It.isAny())).returns(() => pythonSettings.object); - envVariablesProvider.setup(e => e.getEnvironmentVariables(TypeMoq.It.isAny())).returns(() => Promise.resolve({})); serviceContainer.setup(c => c.get(TypeMoq.It.isValue(IDocumentManager), TypeMoq.It.isAny())).returns(() => documentManager.object); serviceContainer.setup(c => c.get(TypeMoq.It.isValue(IApplicationShell), TypeMoq.It.isAny())).returns(() => applicationShell.object); - serviceContainer.setup(c => c.get(TypeMoq.It.isValue(IEnvironmentVariablesProvider), TypeMoq.It.isAny())).returns(() => envVariablesProvider.object); - serviceContainer.setup(c => c.get(TypeMoq.It.isValue(IProcessService), TypeMoq.It.isAny())).returns(() => processService.object); - serviceContainer.setup(c => c.get(TypeMoq.It.isValue(IConfigurationService), TypeMoq.It.isAny())).returns(() => configService.object); helper = new CodeExecutionHelper(serviceContainer.object); document = TypeMoq.Mock.ofType(); @@ -56,17 +38,12 @@ suite('Terminal - Code Execution Helper', () => { }); async function ensureBlankLinesAreRemoved(source: string, expectedSource: string) { - const actualProcessService = new ProcessService(new BufferDecoder()); - processService.setup(p => p.exec(TypeMoq.It.isAny(), TypeMoq.It.isAny(), TypeMoq.It.isAny())) - .returns((file, args, options) => { - return actualProcessService.exec.apply(actualProcessService, [file, args, options]); - }); const normalizedZCode = await helper.normalizeLines(source); expect(normalizedZCode).to.be.equal(expectedSource); } - test('Ensure blank lines are removed (simple)', async () => { - const code = ['import sys', '', '', '', 'print(sys.executable)', '', 'print("1234")', '', '', 'print(1)', 'print(2)']; - const expectedCode = code.filter(line => line.trim().length > 0).join(EOL); + test('Ensure blank lines are NOT removed when code is not indented (simple)', async () => { + const code = ['import sys', '', 'print(sys.executable)', '', 'print("1234")', '', 'print(1)', 'print(2)']; + const expectedCode = code.join(EOL); await ensureBlankLinesAreRemoved(code.join(EOL), expectedCode); }); ['sample1', 'sample2', 'sample3', 'sample4', 'sample5'].forEach(fileName => { @@ -75,11 +52,11 @@ suite('Terminal - Code Execution Helper', () => { const expectedCode = await fs.readFile(path.join(TEST_FILES_PATH, `${fileName}_normalized.py`), 'utf8'); await ensureBlankLinesAreRemoved(code, expectedCode); }); - test(`Ensure blank lines are removed, including leading empty lines (${fileName})`, async () => { - const code = await fs.readFile(path.join(TEST_FILES_PATH, `${fileName}_raw.py`), 'utf8'); - const expectedCode = await fs.readFile(path.join(TEST_FILES_PATH, `${fileName}_normalized.py`), 'utf8'); - await ensureBlankLinesAreRemoved(['', '', ''].join(EOL) + EOL + code, expectedCode); - }); + // test(`Ensure blank lines are removed, including leading empty lines (${fileName})`, async () => { + // const code = await fs.readFile(path.join(TEST_FILES_PATH, `${fileName}_raw.py`), 'utf8'); + // const expectedCode = await fs.readFile(path.join(TEST_FILES_PATH, `${fileName}_normalized.py`), 'utf8'); + // await ensureBlankLinesAreRemoved(['', '', ''].join(EOL) + EOL + code, expectedCode); + // }); }); test('Ensure blank lines are removed (sample2)', async () => { const code = await fs.readFile(path.join(TEST_FILES_PATH, 'sample2_raw.py'), 'utf8'); From 86d17eae6a2d8d5246e45e6d703fa4dab6e9ce9b Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Mon, 23 Apr 2018 13:35:06 -0700 Subject: [PATCH 11/12] Remove PTVSD --- pythonFiles/experimental/ptvsd | 1 - 1 file changed, 1 deletion(-) delete mode 160000 pythonFiles/experimental/ptvsd diff --git a/pythonFiles/experimental/ptvsd b/pythonFiles/experimental/ptvsd deleted file mode 160000 index 8355787a1024..000000000000 --- a/pythonFiles/experimental/ptvsd +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8355787a1024c56ee87cbb76a41654e6d0c63725 From 578c35933625d4cd64a9797ae95bded87a77a69a Mon Sep 17 00:00:00 2001 From: Don Jayamanne Date: Mon, 23 Apr 2018 13:38:40 -0700 Subject: [PATCH 12/12] Account for carriage returns --- src/client/terminals/codeExecution/helper.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/client/terminals/codeExecution/helper.ts b/src/client/terminals/codeExecution/helper.ts index 555a64b11a3f..f5c2e0baeb27 100644 --- a/src/client/terminals/codeExecution/helper.ts +++ b/src/client/terminals/codeExecution/helper.ts @@ -22,11 +22,8 @@ export class CodeExecutionHelper implements ICodeExecutionHelper { if (code.trim().length === 0) { return ''; } - const regex = /(\n)([ \t]*\n)([ \t]+\S+)/gm; + const regex = /(\n)([ \t]*\r?\n)([ \t]+\S+)/gm; return code.replace(regex, (_, a, b, c) => { - // console.log('found'); - // console.log(match); - // console.log(`${a}${c}`); return `${a}${c}`; }); } catch (ex) {