Handle OOM in Java language server

USAGE_DATA.md

@@ -18,6 +18,8 @@ vscode-xml has opt-in telemetry collection, provided by [vscode-redhat-telemetry
     * If the download fails, the associated error is attached to the telemetry event
  * A telemetry event is sent every time you click the "Open Proxy Configuration Documentation" link that is provided when the language server binary download fails due to a proxy related issue.
  * A telemetry event is sent every time you click the "Download Java" link that appears when you have [LemMinX extensions](./docs/Extensions.md) installed but don't have Java installed.
+ * A telemetry event is sent every time the Java XML language server crashes due to an Out Of Memory Error, which also contains the maximum heap space for the JVM (-Xmx) that you've set.
+ * A telemetry event is sent every time you click on the link to the documentation that appears after the Java XML language server crashes due to an Out Of Memory Error.
 
 ## What's included in the general telemetry data
 

docs/Troubleshooting.md

@@ -21,3 +21,22 @@ You can kill the process by:
 
   * on Windows OS: run `taskkill /F /PID ...` all instances
   * on other OS: run `kill -9 ...` all instances
+
+### The Language Server Crashes Due to an Out Of Memory Error
+
+If you are working with large XML files or referencing large schema files,
+this may lead to the language server running out of memory.
+The Java language server is more likely to run out memory than the binary language server.
+Switching to the binary language server
+or increasing the memory available to the Java language server could resolve this issue.
+
+If you get an Out of Memory Error, but aren't working with large XML files,
+then there may be a memory leak in the language server.
+Please [file a issue](https://github.com/redhat-developer/vscode-xml/issues) with a description of what you were doing if this is the case.
+
+#### How to increase the amount of memory available to the Java Language Server
+
+1. Go to settings
+2. Navigate to the setting `xml.server.vmargs`
+3. Add `-Xmx512m` to the setting string. This allows the the language server to use at most 512 megabytes of memory.
+4. If the problem persists, you can increase the `512m` to `1G` or higher

package.json

@@ -67,7 +67,8 @@
     "Snippets"
   ],
   "devDependencies": {
-    "@types/fs-extra": "^8.0.0",
+    "@types/fs-extra": "^8.1.2",
+    "@types/glob": "^7.1.4",
     "@types/node": "^10.14.16",
     "@types/vscode": "^1.37.0",
     "@types/yauzl": "^2.9.1",

src/client/clientErrorHandler.ts

@@ -1,20 +1,30 @@
-import { window } from "vscode";
+import * as fs from "fs-extra";
+import { commands, ExtensionContext, window, workspace } from "vscode";
 import { CloseAction, ErrorAction, ErrorHandler, Message } from "vscode-languageclient";
+import { ClientCommandConstants } from "../commands/commandConstants";
+import { HEAP_DUMP_LOCATION } from "../server/java/jvmArguments";
+import { Telemetry } from "../telemetry";
+import glob = require("glob");
 
 /**
  * An error handler that restarts the language server,
- * unless it has been restarted 5 times in the last 3 minutes
+ * unless it has been restarted 5 times in the last 10 minutes,
+ * or if it crashed due to an Out Of Memory Error
  *
  * Adapted from [vscode-java](https://github.com/redhat-developer/vscode-java)
  */
 export class ClientErrorHandler implements ErrorHandler {
 
   private restarts: number[];
   private name: string;
+  private context: ExtensionContext;
+  private heapDumpFolder: string;
 
-  constructor(name: string) {
+  constructor(name: string, context: ExtensionContext) {
     this.name = name;
     this.restarts = [];
+    this.context = context;
+    this.heapDumpFolder = getHeapDumpFolderFromSettings() || context.globalStorageUri.fsPath;
   }
 
   error(_error: Error, _message: Message, _count: number): ErrorAction {
@@ -23,12 +33,24 @@ export class ClientErrorHandler implements ErrorHandler {
 
   closed(): CloseAction {
     this.restarts.push(Date.now());
+    const heapProfileGlob = new glob.GlobSync(`${this.heapDumpFolder}/java_*.hprof`);
+    if (heapProfileGlob.found.length) {
+      // Only clean heap dumps that are generated in the default location.
+      // The default location is the extension global storage
+      // This means that if users change the folder where the heap dumps are placed,
+      // then they will be able to read the heap dumps,
+      // since they aren't immediately deleted.
+      cleanUpHeapDumps(this.context);
+      Telemetry.sendTelemetry(Telemetry.JAVA_OOM_EVT, { 'jvm.xmx': getXmxFromSettings() });
+      showOOMMessage();
+      return CloseAction.DoNotRestart;
+    }
     if (this.restarts.length < 5) {
       return CloseAction.Restart;
     } else {
       const diff = this.restarts[this.restarts.length - 1] - this.restarts[0];
-      if (diff <= 3 * 60 * 1000) {
-        window.showErrorMessage(`The ${this.name} language server crashed 5 times in the last 3 minutes. The server will not be restarted.`);
+      if (diff <= 10 * 60 * 1000) {
+        window.showErrorMessage(`The ${this.name} language server crashed 5 times in the last 10 minutes. The server will not be restarted.`);
         return CloseAction.DoNotRestart;
       }
       this.restarts.shift();
@@ -37,3 +59,67 @@ export class ClientErrorHandler implements ErrorHandler {
   }
 
 }
+
+/**
+ * Deletes all the heap dumps generated by Out Of Memory errors
+ *
+ * @returns when the heap dumps have been deleted
+ */
+export async function cleanUpHeapDumps(context: ExtensionContext): Promise<void> {
+  const heapProfileGlob = new glob.GlobSync(`${context.globalStorageUri.fsPath}/java_*.hprof`);
+  for (let heapProfile of heapProfileGlob.found) {
+    await fs.remove(heapProfile);
+  }
+}
+
+/**
+ * Shows a message about the server crashing due to an out of memory issue
+ */
+async function showOOMMessage(): Promise<void> {
+  const DOCS = 'More info...';
+  const result = await window.showErrorMessage('The XML Language Server crashed due to an Out Of Memory Error, and will not be restarted. ', //
+    DOCS);
+  if (result === DOCS) {
+    Telemetry.sendTelemetry(Telemetry.OPEN_OOM_DOCS_EVT);
+    await commands.executeCommand(ClientCommandConstants.OPEN_DOCS,
+      {
+        page: 'Troubleshooting',
+        section: 'the-language-server-crashes-due-to-an-out-of-memory-error'
+      }
+    );
+  }
+}
+
+const HEAP_DUMP_FOLDER_EXTRACTOR = new RegExp(`${HEAP_DUMP_LOCATION}(?:'([^']+)'|"([^"]+)"|([^\\s]+))`);
+
+/**
+ * Returns the heap dump folder defined in the user's preferences, or undefined if the user does not set the heap dump folder
+ *
+ * @returns the heap dump folder defined in the user's preferences, or undefined if the user does not set the heap dump folder
+ */
+function getHeapDumpFolderFromSettings(): string {
+  const jvmArgs: string = workspace.getConfiguration('xml.server').get('vmargs');
+  const results = HEAP_DUMP_FOLDER_EXTRACTOR.exec(jvmArgs);
+  if (!results || !results[0]) {
+    return undefined;
+  }
+  return results[1] || results[2] || results[3];
+}
+
+const XMX_EXTRACTOR = /-Xmx([^\s]+)/;
+
+/**
+ * Returns the value that the user set for Xmx, or DEFAULT if the user didn't set Xmx
+ *
+ * @returns the value that the user set for Xmx, or DEFAULT if the user didn't set Xmx
+ */
+function getXmxFromSettings(): string {
+  const vmargs: string = workspace.getConfiguration('xml.server').get('vmargs', null);
+  if (vmargs != null) {
+    const extractOfVmargs: RegExpExecArray = XMX_EXTRACTOR.exec(vmargs);
+    if (extractOfVmargs.length && extractOfVmargs[1]) {
+      return extractOfVmargs[1];
+    }
+  }
+  return 'DEFAULT';
+}

src/client/xmlClient.ts

@@ -3,7 +3,7 @@ import { commands, ExtensionContext, extensions, Position, TextDocument, TextEdi
 import { Command, ConfigurationParams, ConfigurationRequest, DidChangeConfigurationNotification, ExecuteCommandParams, LanguageClientOptions, MessageType, NotificationType, RequestType, RevealOutputChannelOn, TextDocumentPositionParams } from "vscode-languageclient";
 import { Executable, LanguageClient } from 'vscode-languageclient/node';
 import { XMLFileAssociation } from '../api/xmlExtensionApi';
-import { ClientCommandConstants, ServerCommandConstants } from '../commands/commandConstants';
+import { ServerCommandConstants } from '../commands/commandConstants';
 import { registerClientServerCommands } from '../commands/registerCommands';
 import { onExtensionChange } from '../plugin';
 import { RequirementsData } from "../server/requirements";
@@ -37,7 +37,7 @@ let languageClient: LanguageClient;
 
 export async function startLanguageClient(context: ExtensionContext, executable: Executable, logfile: string, externalXmlSettings: ExternalXmlSettings, requirementsData: RequirementsData): Promise<LanguageClient> {
 
-  const languageClientOptions: LanguageClientOptions = getLanguageClientOptions(logfile, externalXmlSettings, requirementsData);
+  const languageClientOptions: LanguageClientOptions = getLanguageClientOptions(logfile, externalXmlSettings, requirementsData, context);
   languageClient = new LanguageClient('xml', 'XML Support', executable, languageClientOptions);
 
   languageClient.onTelemetry(async (e: TelemetryEvent) => {
@@ -106,7 +106,11 @@ export async function startLanguageClient(context: ExtensionContext, executable:
   return languageClient;
 }
 
-function getLanguageClientOptions(logfile: string, externalXmlSettings: ExternalXmlSettings, requirementsData: RequirementsData): LanguageClientOptions {
+function getLanguageClientOptions(
+    logfile: string,
+    externalXmlSettings: ExternalXmlSettings,
+    requirementsData: RequirementsData,
+    context: ExtensionContext): LanguageClientOptions {
   return {
     // Register the server for xml and xsl
     documentSelector: [
@@ -134,7 +138,7 @@ function getLanguageClientOptions(logfile: string, externalXmlSettings: External
         shouldLanguageServerExitOnShutdown: true
       }
     },
-    errorHandler: new ClientErrorHandler('XML'),
+    errorHandler: new ClientErrorHandler('XML', context),
     synchronize: {
       //preferences starting with these will trigger didChangeConfiguration
       configurationSection: ['xml', '[xml]', 'files.trimFinalNewlines', 'files.trimTrailingWhitespace', 'files.insertFinalNewline']

src/extension.ts

@@ -10,21 +10,23 @@
  *  Microsoft Corporation - Auto Closing Tags
  */
 
+import * as fs from 'fs-extra';
 import * as os from 'os';
 import * as path from 'path';
 import { ExtensionContext, extensions, languages } from "vscode";
 import { Executable, LanguageClient } from 'vscode-languageclient/node';
 import { XMLExtensionApi } from './api/xmlExtensionApi';
 import { getXmlExtensionApiImplementation } from './api/xmlExtensionApiImplementation';
+import { cleanUpHeapDumps } from './client/clientErrorHandler';
 import { getIndentationRules } from './client/indentation';
 import { startLanguageClient } from './client/xmlClient';
+import { registerClientOnlyCommands } from './commands/registerCommands';
 import { collectXmlJavaExtensions } from './plugin';
 import * as requirements from './server/requirements';
 import { prepareExecutable } from './server/serverStarter';
 import { ExternalXmlSettings } from "./settings/externalXmlSettings";
 import { getXMLConfiguration } from './settings/settings';
 import { Telemetry } from './telemetry';
-import { registerClientOnlyCommands } from './commands/registerCommands';
 
 let languageClient: LanguageClient;
 
@@ -52,6 +54,8 @@ export async function activate(context: ExtensionContext): Promise<XMLExtensionA
     storagePath = os.homedir() + "/.lemminx";
   }
   const logfile = path.resolve(storagePath + '/lemminx.log');
+  await fs.ensureDir(context.globalStorageUri.fsPath);
+  await cleanUpHeapDumps(context);
 
   const externalXmlSettings: ExternalXmlSettings = new ExternalXmlSettings();
 

src/server/binary/binaryServerStarter.ts

@@ -13,7 +13,7 @@ import { getProxySettings, getProxySettingsAsEnvironmentVariables, ProxySettings
 import { getXMLConfiguration } from "../../settings/settings";
 import { Telemetry } from '../../telemetry';
 import { addTrustedHash, getTrustedHashes } from './binaryHashManager';
-const glob = require('glob');
+import glob = require('glob');
 
 const HTTPS_PATTERN = /^https:\/\//;
 const JAR_ZIP_AND_HASH_REJECTOR = /(?:\.jar)|(?:\.zip)|(?:\.sha256)$/;
@@ -430,4 +430,4 @@ async function acceptBinaryDownloadResponse(response: http.IncomingMessage): Pro
 
 async function openProxyDocumentation(): Promise<void> {
   await commands.executeCommand(ClientCommandConstants.OPEN_DOCS, { page: "Proxy" });
-}
+}