diff --git a/csharp/autobuilder/Semmle.Autobuild.Tests/BuildScripts.cs b/csharp/autobuilder/Semmle.Autobuild.Tests/BuildScripts.cs index 4cd50926dba1..2b655310e09a 100644 --- a/csharp/autobuilder/Semmle.Autobuild.Tests/BuildScripts.cs +++ b/csharp/autobuilder/Semmle.Autobuild.Tests/BuildScripts.cs @@ -130,7 +130,7 @@ IEnumerable IBuildActions.EnumerateDirectories(string dir) string IBuildActions.PathCombine(params string[] parts) { - return string.Join('\\', parts.Where(p => !string.IsNullOrWhiteSpace(p))); + return string.Join(IsWindows ? '\\' : '/', parts.Where(p => !string.IsNullOrWhiteSpace(p))); } string IBuildActions.GetFullPath(string path) => path; @@ -146,6 +146,13 @@ XmlDocument IBuildActions.LoadXml(string filename) return xml; throw new ArgumentException("Missing LoadXml " + filename); } + + public string EnvironmentExpandEnvironmentVariables(string s) + { + foreach (var kvp in GetEnvironmentVariable) + s = s.Replace($"%{kvp.Key}%", kvp.Value); + return s; + } } /// @@ -394,9 +401,9 @@ public void TestLinuxCSharpAutoBuilder() Actions.RunProcess["dotnet --info"] = 0; Actions.RunProcess["dotnet clean test.csproj"] = 0; Actions.RunProcess["dotnet restore test.csproj"] = 0; - Actions.RunProcess[@"C:\odasa\tools\odasa index --auto dotnet build --no-incremental /p:UseSharedCompilation=false test.csproj"] = 0; - Actions.RunProcess[@"C:\odasa\tools\java\bin\java -jar C:\odasa\tools\extractor-asp.jar ."] = 0; - Actions.RunProcess[@"C:\odasa\tools\odasa index --xml --extensions config csproj props xml"] = 0; + Actions.RunProcess[@"C:\odasa/tools/odasa index --auto dotnet build --no-incremental /p:UseSharedCompilation=false test.csproj"] = 0; + Actions.RunProcess[@"C:\odasa\tools\java/bin/java -jar C:\odasa/tools/extractor-asp.jar ."] = 0; + Actions.RunProcess[@"C:\odasa/tools/odasa index --xml --extensions config csproj props xml"] = 0; Actions.FileExists["csharp.log"] = true; Actions.FileExists["test.csproj"] = true; Actions.GetEnvironmentVariable["TRAP_FOLDER"] = null; @@ -470,6 +477,7 @@ public void TestCppAutobuilderSuccess() [Fact] public void TestVsWhereSucceeded() { + Actions.IsWindows = true; Actions.GetEnvironmentVariable["ProgramFiles(x86)"] = @"C:\Program Files (x86)"; Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = true; Actions.RunProcess[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe -prerelease -legacy -property installationPath"] = 0; @@ -487,6 +495,7 @@ public void TestVsWhereSucceeded() [Fact] public void TestVsWhereNotExist() { + Actions.IsWindows = true; Actions.GetEnvironmentVariable["ProgramFiles(x86)"] = @"C:\Program Files (x86)"; Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false; @@ -497,6 +506,7 @@ public void TestVsWhereNotExist() [Fact] public void TestVcVarsAllBatFiles() { + Actions.IsWindows = true; Actions.GetEnvironmentVariable["ProgramFiles(x86)"] = @"C:\Program Files (x86)"; Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio\Installer\vswhere.exe"] = false; Actions.FileExists[@"C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat"] = true; @@ -511,9 +521,9 @@ public void TestVcVarsAllBatFiles() [Fact] public void TestLinuxBuildlessExtractionSuccess() { - Actions.RunProcess[@"C:\odasa\tools\csharp\Semmle.Extraction.CSharp.Standalone --references:."] = 0; - Actions.RunProcess[@"C:\odasa\tools\java\bin\java -jar C:\odasa\tools\extractor-asp.jar ."] = 0; - Actions.RunProcess[@"C:\odasa\tools\odasa index --xml --extensions config csproj props xml"] = 0; + Actions.RunProcess[@"C:\odasa\tools/csharp/Semmle.Extraction.CSharp.Standalone --references:."] = 0; + Actions.RunProcess[@"C:\odasa\tools\java/bin/java -jar C:\odasa/tools/extractor-asp.jar ."] = 0; + Actions.RunProcess[@"C:\odasa/tools/odasa index --xml --extensions config csproj props xml"] = 0; Actions.FileExists["csharp.log"] = true; Actions.GetEnvironmentVariable["TRAP_FOLDER"] = null; Actions.GetEnvironmentVariable["SOURCE_ARCHIVE"] = null; @@ -527,9 +537,7 @@ public void TestLinuxBuildlessExtractionSuccess() [Fact] public void TestLinuxBuildlessExtractionFailed() { - Actions.RunProcess[@"C:\odasa\tools\csharp\Semmle.Extraction.CSharp.Standalone --references:."] = 10; - Actions.RunProcess[@"C:\odasa\tools\java\bin\java -jar C:\odasa\tools\extractor-asp.jar ."] = 0; - Actions.RunProcess[@"C:\odasa\tools\odasa index --xml --extensions config"] = 0; + Actions.RunProcess[@"C:\odasa\tools/csharp/Semmle.Extraction.CSharp.Standalone --references:."] = 10; Actions.FileExists["csharp.log"] = true; Actions.GetEnvironmentVariable["TRAP_FOLDER"] = null; Actions.GetEnvironmentVariable["SOURCE_ARCHIVE"] = null; @@ -543,9 +551,9 @@ public void TestLinuxBuildlessExtractionFailed() [Fact] public void TestLinuxBuildlessExtractionSolution() { - Actions.RunProcess[@"C:\odasa\tools\csharp\Semmle.Extraction.CSharp.Standalone foo.sln --references:."] = 0; - Actions.RunProcess[@"C:\odasa\tools\java\bin\java -jar C:\odasa\tools\extractor-asp.jar ."] = 0; - Actions.RunProcess[@"C:\odasa\tools\odasa index --xml --extensions config csproj props xml"] = 0; + Actions.RunProcess[@"C:\odasa\tools/csharp/Semmle.Extraction.CSharp.Standalone foo.sln --references:."] = 0; + Actions.RunProcess[@"C:\odasa\tools\java/bin/java -jar C:\odasa/tools/extractor-asp.jar ."] = 0; + Actions.RunProcess[@"C:\odasa/tools/odasa index --xml --extensions config csproj props xml"] = 0; Actions.FileExists["csharp.log"] = true; Actions.GetEnvironmentVariable["TRAP_FOLDER"] = null; Actions.GetEnvironmentVariable["SOURCE_ARCHIVE"] = null; @@ -587,9 +595,9 @@ void TestAutobuilderScript(Autobuilder autobuilder, int expectedOutput, int comm [Fact] public void TestLinuxBuildCommand() { - Actions.RunProcess["C:\\odasa\\tools\\odasa index --auto \"./build.sh --skip-tests\""] = 0; - Actions.RunProcess[@"C:\odasa\tools\java\bin\java -jar C:\odasa\tools\extractor-asp.jar ."] = 0; - Actions.RunProcess[@"C:\odasa\tools\odasa index --xml --extensions config csproj props xml"] = 0; + Actions.RunProcess[@"C:\odasa/tools/odasa index --auto ""./build.sh --skip-tests"""] = 0; + Actions.RunProcess[@"C:\odasa\tools\java/bin/java -jar C:\odasa/tools/extractor-asp.jar ."] = 0; + Actions.RunProcess[@"C:\odasa/tools/odasa index --xml --extensions config csproj props xml"] = 0; Actions.FileExists["csharp.log"] = true; Actions.GetEnvironmentVariable["TRAP_FOLDER"] = null; Actions.GetEnvironmentVariable["SOURCE_ARCHIVE"] = null; @@ -610,10 +618,10 @@ public void TestLinuxBuildSh() Actions.GetEnvironmentVariable["TRAP_FOLDER"] = null; Actions.GetEnvironmentVariable["SOURCE_ARCHIVE"] = null; Actions.RunProcess["/bin/chmod u+x build/build.sh"] = 0; - Actions.RunProcess[@"C:\odasa\tools\odasa index --auto build/build.sh"] = 0; - Actions.RunProcessWorkingDirectory[@"C:\odasa\tools\odasa index --auto build/build.sh"] = "build"; - Actions.RunProcess[@"C:\odasa\tools\java\bin\java -jar C:\odasa\tools\extractor-asp.jar ."] = 0; - Actions.RunProcess[@"C:\odasa\tools\odasa index --xml --extensions config csproj props xml"] = 0; + Actions.RunProcess[@"C:\odasa/tools/odasa index --auto build/build.sh"] = 0; + Actions.RunProcessWorkingDirectory[@"C:\odasa/tools/odasa index --auto build/build.sh"] = "build"; + Actions.RunProcess[@"C:\odasa\tools\java/bin/java -jar C:\odasa/tools/extractor-asp.jar ."] = 0; + Actions.RunProcess[@"C:\odasa/tools/odasa index --xml --extensions config csproj props xml"] = 0; Actions.FileExists["csharp.log"] = true; var autobuilder = CreateAutoBuilder("csharp", false); @@ -629,8 +637,8 @@ public void TestLinuxBuildShCSharpLogMissing() Actions.GetEnvironmentVariable["SOURCE_ARCHIVE"] = null; Actions.RunProcess["/bin/chmod u+x build.sh"] = 0; - Actions.RunProcess[@"C:\odasa\tools\odasa index --auto build.sh"] = 0; - Actions.RunProcessWorkingDirectory[@"C:\odasa\tools\odasa index --auto build.sh"] = ""; + Actions.RunProcess[@"C:\odasa/tools/odasa index --auto build.sh"] = 0; + Actions.RunProcessWorkingDirectory[@"C:\odasa/tools/odasa index --auto build.sh"] = ""; Actions.FileExists["csharp.log"] = false; var autobuilder = CreateAutoBuilder("csharp", false); @@ -646,8 +654,8 @@ public void TestLinuxBuildShFailed() Actions.GetEnvironmentVariable["SOURCE_ARCHIVE"] = null; Actions.RunProcess["/bin/chmod u+x build.sh"] = 0; - Actions.RunProcess[@"C:\odasa\tools\odasa index --auto build.sh"] = 5; - Actions.RunProcessWorkingDirectory[@"C:\odasa\tools\odasa index --auto build.sh"] = ""; + Actions.RunProcess[@"C:\odasa/tools/odasa index --auto build.sh"] = 5; + Actions.RunProcessWorkingDirectory[@"C:\odasa/tools/odasa index --auto build.sh"] = ""; Actions.FileExists["csharp.log"] = true; var autobuilder = CreateAutoBuilder("csharp", false); @@ -796,9 +804,9 @@ public void TestSkipNugetMsBuild() [Fact] public void TestSkipNugetBuildless() { - Actions.RunProcess[@"C:\odasa\tools\csharp\Semmle.Extraction.CSharp.Standalone foo.sln --references:. --skip-nuget"] = 0; - Actions.RunProcess[@"C:\odasa\tools\java\bin\java -jar C:\odasa\tools\extractor-asp.jar ."] = 0; - Actions.RunProcess[@"C:\odasa\tools\odasa index --xml --extensions config csproj props xml"] = 0; + Actions.RunProcess[@"C:\odasa\tools/csharp/Semmle.Extraction.CSharp.Standalone foo.sln --references:. --skip-nuget"] = 0; + Actions.RunProcess[@"C:\odasa\tools\java/bin/java -jar C:\odasa/tools/extractor-asp.jar ."] = 0; + Actions.RunProcess[@"C:\odasa/tools/odasa index --xml --extensions config csproj props xml"] = 0; Actions.FileExists["csharp.log"] = true; Actions.GetEnvironmentVariable["TRAP_FOLDER"] = null; Actions.GetEnvironmentVariable["SOURCE_ARCHIVE"] = null; @@ -816,9 +824,9 @@ public void TestSkipNugetDotnet() Actions.RunProcess["dotnet --info"] = 0; Actions.RunProcess["dotnet clean test.csproj"] = 0; Actions.RunProcess["dotnet restore test.csproj"] = 0; - Actions.RunProcess[@"C:\odasa\tools\odasa index --auto dotnet build --no-incremental /p:UseSharedCompilation=false --no-restore test.csproj"] = 0; - Actions.RunProcess[@"C:\odasa\tools\java\bin\java -jar C:\odasa\tools\extractor-asp.jar ."] = 0; - Actions.RunProcess[@"C:\odasa\tools\odasa index --xml --extensions config csproj props xml"] = 0; + Actions.RunProcess[@"C:\odasa/tools/odasa index --auto dotnet build --no-incremental /p:UseSharedCompilation=false --no-restore test.csproj"] = 0; + Actions.RunProcess[@"C:\odasa\tools\java/bin/java -jar C:\odasa/tools/extractor-asp.jar ."] = 0; + Actions.RunProcess[@"C:\odasa/tools/odasa index --xml --extensions config csproj props xml"] = 0; Actions.FileExists["csharp.log"] = true; Actions.FileExists["test.csproj"] = true; Actions.GetEnvironmentVariable["TRAP_FOLDER"] = null; @@ -846,13 +854,13 @@ public void TestDotnetVersionNotInstalled() Actions.RunProcessOut["dotnet --list-sdks"] = "2.1.2 [C:\\Program Files\\dotnet\\sdks]\n2.1.4 [C:\\Program Files\\dotnet\\sdks]"; Actions.RunProcess[@"curl -sO https://dot.net/v1/dotnet-install.sh"] = 0; Actions.RunProcess[@"chmod u+x dotnet-install.sh"] = 0; - Actions.RunProcess[@"./dotnet-install.sh --channel release --version 2.1.3 --install-dir C:\Project\.dotnet"] = 0; - Actions.RunProcess[@"C:\Project\.dotnet\dotnet --info"] = 0; - Actions.RunProcess[@"C:\Project\.dotnet\dotnet clean test.csproj"] = 0; - Actions.RunProcess[@"C:\Project\.dotnet\dotnet restore test.csproj"] = 0; - Actions.RunProcess[@"C:\odasa\tools\odasa index --auto C:\Project\.dotnet\dotnet build --no-incremental /p:UseSharedCompilation=false test.csproj"] = 0; - Actions.RunProcess[@"C:\odasa\tools\java\bin\java -jar C:\odasa\tools\extractor-asp.jar ."] = 0; - Actions.RunProcess[@"C:\odasa\tools\odasa index --xml --extensions config csproj props xml"] = 0; + Actions.RunProcess[@"./dotnet-install.sh --channel release --version 2.1.3 --install-dir C:\Project/.dotnet"] = 0; + Actions.RunProcess[@"C:\Project/.dotnet/dotnet --info"] = 0; + Actions.RunProcess[@"C:\Project/.dotnet/dotnet clean test.csproj"] = 0; + Actions.RunProcess[@"C:\Project/.dotnet/dotnet restore test.csproj"] = 0; + Actions.RunProcess[@"C:\odasa/tools/odasa index --auto C:\Project/.dotnet/dotnet build --no-incremental /p:UseSharedCompilation=false test.csproj"] = 0; + Actions.RunProcess[@"C:\odasa\tools\java/bin/java -jar C:\odasa/tools/extractor-asp.jar ."] = 0; + Actions.RunProcess[@"C:\odasa/tools/odasa index --xml --extensions config csproj props xml"] = 0; Actions.FileExists["csharp.log"] = true; Actions.FileExists["test.csproj"] = true; Actions.GetEnvironmentVariable["TRAP_FOLDER"] = null; @@ -881,13 +889,13 @@ public void TestDotnetVersionAlreadyInstalled() Actions.RunProcessOut["dotnet --list-sdks"] = "2.1.3 [C:\\Program Files\\dotnet\\sdks]\n2.1.4 [C:\\Program Files\\dotnet\\sdks]"; Actions.RunProcess[@"curl -sO https://dot.net/v1/dotnet-install.sh"] = 0; Actions.RunProcess[@"chmod u+x dotnet-install.sh"] = 0; - Actions.RunProcess[@"./dotnet-install.sh --channel release --version 2.1.3 --install-dir C:\Project\.dotnet"] = 0; - Actions.RunProcess[@"C:\Project\.dotnet\dotnet --info"] = 0; - Actions.RunProcess[@"C:\Project\.dotnet\dotnet clean test.csproj"] = 0; - Actions.RunProcess[@"C:\Project\.dotnet\dotnet restore test.csproj"] = 0; - Actions.RunProcess[@"C:\odasa\tools\odasa index --auto C:\Project\.dotnet\dotnet build --no-incremental /p:UseSharedCompilation=false test.csproj"] = 0; - Actions.RunProcess[@"C:\odasa\tools\java\bin\java -jar C:\odasa\tools\extractor-asp.jar ."] = 0; - Actions.RunProcess[@"C:\odasa\tools\odasa index --xml --extensions config csproj props xml"] = 0; + Actions.RunProcess[@"./dotnet-install.sh --channel release --version 2.1.3 --install-dir C:\Project/.dotnet"] = 0; + Actions.RunProcess[@"C:\Project/.dotnet/dotnet --info"] = 0; + Actions.RunProcess[@"C:\Project/.dotnet/dotnet clean test.csproj"] = 0; + Actions.RunProcess[@"C:\Project/.dotnet/dotnet restore test.csproj"] = 0; + Actions.RunProcess[@"C:\odasa/tools/odasa index --auto C:\Project/.dotnet/dotnet build --no-incremental /p:UseSharedCompilation=false test.csproj"] = 0; + Actions.RunProcess[@"C:\odasa\tools\java/bin/java -jar C:\odasa/tools/extractor-asp.jar ."] = 0; + Actions.RunProcess[@"C:\odasa/tools/odasa index --xml --extensions config csproj props xml"] = 0; Actions.FileExists["csharp.log"] = true; Actions.FileExists["test.csproj"] = true; Actions.GetEnvironmentVariable["TRAP_FOLDER"] = null; @@ -988,10 +996,10 @@ public void TestDirsProjWindows() [Fact] public void TestDirsProjLinux() { - Actions.RunProcess[@"mono C:\odasa\tools\csharp\nuget\nuget.exe restore dirs.proj"] = 1; - Actions.RunProcess[@"C:\odasa\tools\odasa index --auto msbuild dirs.proj /p:UseSharedCompilation=false /t:rebuild /p:MvcBuildViews=true"] = 0; - Actions.RunProcess[@"C:\odasa\tools\java\bin\java -jar C:\odasa\tools\extractor-asp.jar ."] = 0; - Actions.RunProcess[@"C:\odasa\tools\odasa index --xml --extensions config csproj props xml"] = 0; + Actions.RunProcess[@"mono C:\odasa\tools/csharp/nuget/nuget.exe restore dirs.proj"] = 1; + Actions.RunProcess[@"C:\odasa/tools/odasa index --auto msbuild dirs.proj /p:UseSharedCompilation=false /t:rebuild /p:MvcBuildViews=true"] = 0; + Actions.RunProcess[@"C:\odasa\tools\java/bin/java -jar C:\odasa/tools/extractor-asp.jar ."] = 0; + Actions.RunProcess[@"C:\odasa/tools/odasa index --xml --extensions config csproj props xml"] = 0; Actions.FileExists["csharp.log"] = true; Actions.FileExists["a/test.csproj"] = true; Actions.FileExists["dirs.proj"] = true; @@ -1042,5 +1050,21 @@ public void TestCyclicDirsProj() var autobuilder = CreateAutoBuilder("csharp", false); TestAutobuilderScript(autobuilder, 1, 0); } + + [Fact] + public void TestAsStringWithExpandedEnvVarsWindows() + { + Actions.IsWindows = true; + Actions.GetEnvironmentVariable["LGTM_SRC"] = @"C:\repo"; + Assert.Equal(@"C:\repo\test", @"%LGTM_SRC%\test".AsStringWithExpandedEnvVars(Actions)); + } + + [Fact] + public void TestAsStringWithExpandedEnvVarsLinux() + { + Actions.IsWindows = false; + Actions.GetEnvironmentVariable["LGTM_SRC"] = "/tmp/repo"; + Assert.Equal("/tmp/repo/test", "$LGTM_SRC/test".AsStringWithExpandedEnvVars(Actions)); + } } } diff --git a/csharp/autobuilder/Semmle.Autobuild/AspBuildRule.cs b/csharp/autobuilder/Semmle.Autobuild/AspBuildRule.cs index 7765e94f5e16..a39481545d88 100644 --- a/csharp/autobuilder/Semmle.Autobuild/AspBuildRule.cs +++ b/csharp/autobuilder/Semmle.Autobuild/AspBuildRule.cs @@ -7,7 +7,7 @@ namespace Semmle.Autobuild /// class AspBuildRule : IBuildRule { - public BuildScript Analyse(Autobuilder builder) + public BuildScript Analyse(Autobuilder builder, bool auto) { var command = new CommandBuilder(builder.Actions). RunCommand(builder.Actions.PathCombine(builder.SemmleJavaHome, "bin", "java")). diff --git a/csharp/autobuilder/Semmle.Autobuild/AutobuildOptions.cs b/csharp/autobuilder/Semmle.Autobuild/AutobuildOptions.cs index 53ecf697721e..4a2e906fc81a 100644 --- a/csharp/autobuilder/Semmle.Autobuild/AutobuildOptions.cs +++ b/csharp/autobuilder/Semmle.Autobuild/AutobuildOptions.cs @@ -1,5 +1,6 @@ using System; using System.Linq; +using System.Text.RegularExpressions; namespace Semmle.Autobuild { @@ -37,14 +38,14 @@ public void ReadEnvironment(IBuildActions actions) { RootDirectory = actions.GetCurrentDirectory(); VsToolsVersion = actions.GetEnvironmentVariable(prefix + "VSTOOLS_VERSION"); - MsBuildArguments = actions.GetEnvironmentVariable(prefix + "MSBUILD_ARGUMENTS"); + MsBuildArguments = actions.GetEnvironmentVariable(prefix + "MSBUILD_ARGUMENTS").AsStringWithExpandedEnvVars(actions); MsBuildPlatform = actions.GetEnvironmentVariable(prefix + "MSBUILD_PLATFORM"); MsBuildConfiguration = actions.GetEnvironmentVariable(prefix + "MSBUILD_CONFIGURATION"); MsBuildTarget = actions.GetEnvironmentVariable(prefix + "MSBUILD_TARGET"); - DotNetArguments = actions.GetEnvironmentVariable(prefix + "DOTNET_ARGUMENTS"); + DotNetArguments = actions.GetEnvironmentVariable(prefix + "DOTNET_ARGUMENTS").AsStringWithExpandedEnvVars(actions); DotNetVersion = actions.GetEnvironmentVariable(prefix + "DOTNET_VERSION"); BuildCommand = actions.GetEnvironmentVariable(prefix + "BUILD_COMMAND"); - Solution = actions.GetEnvironmentVariable(prefix + "SOLUTION").AsList(new string[0]); + Solution = actions.GetEnvironmentVariable(prefix + "SOLUTION").AsListWithExpandedEnvVars(actions, new string[0]); IgnoreErrors = actions.GetEnvironmentVariable(prefix + "IGNORE_ERRORS").AsBool("ignore_errors", false); Buildless = actions.GetEnvironmentVariable(prefix + "BUILDLESS").AsBool("buildless", false); @@ -92,12 +93,29 @@ public static Language AsLanguage(this string key) } } - public static string[] AsList(this string value, string[] defaultValue) + public static string[] AsListWithExpandedEnvVars(this string value, IBuildActions actions, string[] defaultValue) { if (value == null) return defaultValue; - return value.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries).ToArray(); + return value. + Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries). + Select(s => AsStringWithExpandedEnvVars(s, actions)).ToArray(); + } + + static readonly Regex linuxEnvRegEx = new Regex(@"\$([a-zA-Z_][a-zA-Z_0-9]*)", RegexOptions.Compiled); + + public static string AsStringWithExpandedEnvVars(this string value, IBuildActions actions) + { + if (string.IsNullOrEmpty(value)) + return value; + + // `Environment.ExpandEnvironmentVariables` only works with Windows-style + // environment variables + var windowsStyle = actions.IsWindows() + ? value + : linuxEnvRegEx.Replace(value, m => $"%{m.Groups[1].Value}%"); + return actions.EnvironmentExpandEnvironmentVariables(windowsStyle); } } } diff --git a/csharp/autobuilder/Semmle.Autobuild/Autobuilder.cs b/csharp/autobuilder/Semmle.Autobuild/Autobuilder.cs index 02e3b672511f..1734afc8b856 100644 --- a/csharp/autobuilder/Semmle.Autobuild/Autobuilder.cs +++ b/csharp/autobuilder/Semmle.Autobuild/Autobuilder.cs @@ -16,7 +16,8 @@ interface IBuildRule /// Analyse the files and produce a build script. /// /// The files and options relating to the build. - BuildScript Analyse(Autobuilder builder); + /// Whether this build rule is being automatically applied. + BuildScript Analyse(Autobuilder builder, bool auto); } /// @@ -135,9 +136,9 @@ public Autobuilder(IBuildActions actions, AutobuildOptions options) foreach (var solution in options.Solution) { if (actions.FileExists(solution)) - ret.Add(new Solution(this, solution)); + ret.Add(new Solution(this, solution, true)); else - Log(Severity.Error, $"The specified solution file {solution} was not found"); + Log(Severity.Error, $"The specified project or solution file {solution} was not found"); } return ret; } @@ -172,7 +173,7 @@ IEnumerable FindFiles(string extension, Func new Solution(this, f))?.ToList(); + ret = FindFiles(".sln", f => new Solution(this, f, false))?.ToList(); if (ret != null) return ret; @@ -250,17 +251,17 @@ BuildScript CheckExtractorRun(bool warnOnFailure) => switch (GetCSharpBuildStrategy()) { case CSharpBuildStrategy.CustomBuildCommand: - attempt = new BuildCommandRule().Analyse(this) & CheckExtractorRun(true); + attempt = new BuildCommandRule().Analyse(this, false) & CheckExtractorRun(true); break; case CSharpBuildStrategy.Buildless: // No need to check that the extractor has been executed in buildless mode - attempt = new StandaloneBuildRule().Analyse(this); + attempt = new StandaloneBuildRule().Analyse(this, false); break; case CSharpBuildStrategy.MSBuild: - attempt = new MsBuildRule().Analyse(this) & CheckExtractorRun(true); + attempt = new MsBuildRule().Analyse(this, false) & CheckExtractorRun(true); break; case CSharpBuildStrategy.DotNet: - attempt = new DotNetRule().Analyse(this) & CheckExtractorRun(true); + attempt = new DotNetRule().Analyse(this, false) & CheckExtractorRun(true); break; case CSharpBuildStrategy.Auto: var cleanTrapFolder = @@ -285,11 +286,11 @@ BuildScript IntermediateAttempt(BuildScript s) => attempt = // First try .NET Core - IntermediateAttempt(new DotNetRule().Analyse(this)) | + IntermediateAttempt(new DotNetRule().Analyse(this, true)) | // Then MSBuild - (() => IntermediateAttempt(new MsBuildRule().Analyse(this))) | + (() => IntermediateAttempt(new MsBuildRule().Analyse(this, true))) | // And finally look for a script that might be a build script - (() => new BuildCommandAutoRule().Analyse(this) & CheckExtractorRun(true)) | + (() => new BuildCommandAutoRule().Analyse(this, true) & CheckExtractorRun(true)) | // All attempts failed: print message AutobuildFailure(); break; @@ -297,8 +298,8 @@ BuildScript IntermediateAttempt(BuildScript s) => return attempt & - (() => new AspBuildRule().Analyse(this)) & - (() => new XmlBuildRule().Analyse(this)); + (() => new AspBuildRule().Analyse(this, false)) & + (() => new XmlBuildRule().Analyse(this, false)); } /// @@ -337,13 +338,13 @@ enum CSharpBuildStrategy BuildScript GetCppBuildScript() { if (Options.BuildCommand != null) - return new BuildCommandRule().Analyse(this); + return new BuildCommandRule().Analyse(this, false); return // First try MSBuild - new MsBuildRule().Analyse(this) | + new MsBuildRule().Analyse(this, true) | // Then look for a script that might be a build script - (() => new BuildCommandAutoRule().Analyse(this)) | + (() => new BuildCommandAutoRule().Analyse(this, true)) | // All attempts failed: print message AutobuildFailure(); } diff --git a/csharp/autobuilder/Semmle.Autobuild/BuildActions.cs b/csharp/autobuilder/Semmle.Autobuild/BuildActions.cs index 433598c5c16c..edf4fc752c1c 100644 --- a/csharp/autobuilder/Semmle.Autobuild/BuildActions.cs +++ b/csharp/autobuilder/Semmle.Autobuild/BuildActions.cs @@ -113,6 +113,12 @@ public interface IBuildActions /// Loads the XML document from . /// XmlDocument LoadXml(string filename); + + /// + /// Expand all Windows-style environment variables in , + /// Environment.ExpandEnvironmentVariables() + /// + string EnvironmentExpandEnvironmentVariables(string s); } /// @@ -187,6 +193,8 @@ XmlDocument IBuildActions.LoadXml(string filename) string IBuildActions.GetFullPath(string path) => Path.GetFullPath(path); + public string EnvironmentExpandEnvironmentVariables(string s) => Environment.ExpandEnvironmentVariables(s); + public static readonly IBuildActions Instance = new SystemBuildActions(); } } diff --git a/csharp/autobuilder/Semmle.Autobuild/BuildCommandAutoRule.cs b/csharp/autobuilder/Semmle.Autobuild/BuildCommandAutoRule.cs index 9ec8a3e16eb0..8ae3ee609667 100644 --- a/csharp/autobuilder/Semmle.Autobuild/BuildCommandAutoRule.cs +++ b/csharp/autobuilder/Semmle.Autobuild/BuildCommandAutoRule.cs @@ -25,7 +25,7 @@ class BuildCommandAutoRule : IBuildRule "build" }; - public BuildScript Analyse(Autobuilder builder) + public BuildScript Analyse(Autobuilder builder, bool auto) { builder.Log(Severity.Info, "Attempting to locate build script"); diff --git a/csharp/autobuilder/Semmle.Autobuild/BuildCommandRule.cs b/csharp/autobuilder/Semmle.Autobuild/BuildCommandRule.cs index 2dd3a1442223..8019b5798bc5 100644 --- a/csharp/autobuilder/Semmle.Autobuild/BuildCommandRule.cs +++ b/csharp/autobuilder/Semmle.Autobuild/BuildCommandRule.cs @@ -5,7 +5,7 @@ /// class BuildCommandRule : IBuildRule { - public BuildScript Analyse(Autobuilder builder) + public BuildScript Analyse(Autobuilder builder, bool auto) { if (builder.Options.BuildCommand == null) return BuildScript.Failure; diff --git a/csharp/autobuilder/Semmle.Autobuild/DotNetRule.cs b/csharp/autobuilder/Semmle.Autobuild/DotNetRule.cs index c9622d2cf26f..9cce54ab50ef 100644 --- a/csharp/autobuilder/Semmle.Autobuild/DotNetRule.cs +++ b/csharp/autobuilder/Semmle.Autobuild/DotNetRule.cs @@ -14,22 +14,25 @@ namespace Semmle.Autobuild /// class DotNetRule : IBuildRule { - public BuildScript Analyse(Autobuilder builder) + public BuildScript Analyse(Autobuilder builder, bool auto) { if (!builder.ProjectsOrSolutionsToBuild.Any()) return BuildScript.Failure; - var notDotNetProject = builder.ProjectsOrSolutionsToBuild. - SelectMany(p => Enumerators.Singleton(p).Concat(p.IncludedProjects)). - OfType(). - FirstOrDefault(p => !p.DotNetProject); - if (notDotNetProject != null) + if (auto) { - builder.Log(Severity.Info, "Not using .NET Core because of incompatible project {0}", notDotNetProject); - return BuildScript.Failure; - } + var notDotNetProject = builder.ProjectsOrSolutionsToBuild. + SelectMany(p => Enumerators.Singleton(p).Concat(p.IncludedProjects)). + OfType(). + FirstOrDefault(p => !p.DotNetProject); + if (notDotNetProject != null) + { + builder.Log(Severity.Info, "Not using .NET Core because of incompatible project {0}", notDotNetProject); + return BuildScript.Failure; + } - builder.Log(Severity.Info, "Attempting to build using .NET Core"); + builder.Log(Severity.Info, "Attempting to build using .NET Core"); + } return WithDotNet(builder, dotNet => { diff --git a/csharp/autobuilder/Semmle.Autobuild/MsBuildRule.cs b/csharp/autobuilder/Semmle.Autobuild/MsBuildRule.cs index 2b46d8d526e8..593039919ed9 100644 --- a/csharp/autobuilder/Semmle.Autobuild/MsBuildRule.cs +++ b/csharp/autobuilder/Semmle.Autobuild/MsBuildRule.cs @@ -13,12 +13,13 @@ class MsBuildRule : IBuildRule /// const string MsBuild = "msbuild"; - public BuildScript Analyse(Autobuilder builder) + public BuildScript Analyse(Autobuilder builder, bool auto) { if (!builder.ProjectsOrSolutionsToBuild.Any()) return BuildScript.Failure; - builder.Log(Severity.Info, "Attempting to build using MSBuild"); + if (auto) + builder.Log(Severity.Info, "Attempting to build using MSBuild"); var vsTools = GetVcVarsBatFile(builder); diff --git a/csharp/autobuilder/Semmle.Autobuild/Project.cs b/csharp/autobuilder/Semmle.Autobuild/Project.cs index 87088d1edfb5..47752f691a49 100644 --- a/csharp/autobuilder/Semmle.Autobuild/Project.cs +++ b/csharp/autobuilder/Semmle.Autobuild/Project.cs @@ -39,9 +39,9 @@ public Project(Autobuilder builder, string path) : base(builder, path) { projFile = builder.Actions.LoadXml(FullPath); } - catch (XmlException) + catch (Exception e) when (e is XmlException || e is FileNotFoundException) { - builder.Log(Severity.Info, $"Skipping project file {path} as it is not a valid XML document."); + builder.Log(Severity.Info, $"Unable to read project file {path}."); return; } @@ -80,7 +80,7 @@ public Project(Autobuilder builder, string path) : base(builder, path) var projectFilesIncludes = root.SelectNodes("//msbuild:Project/msbuild:ItemGroup/msbuild:ProjectFiles/@Include", mgr).OfType(); foreach (var include in projectFileIncludes.Concat(projectFilesIncludes)) { - var includePath = builder.Actions.IsWindows() ? include.Value : include.Value.Replace("\\", "/"); + var includePath = builder.Actions.PathCombine(include.Value.Split('\\', StringSplitOptions.RemoveEmptyEntries)); ret.Add(new Project(builder, builder.Actions.PathCombine(Path.GetDirectoryName(this.FullPath), includePath))); } return ret; diff --git a/csharp/autobuilder/Semmle.Autobuild/Solution.cs b/csharp/autobuilder/Semmle.Autobuild/Solution.cs index 312e7deeb54b..a5512a84ea66 100644 --- a/csharp/autobuilder/Semmle.Autobuild/Solution.cs +++ b/csharp/autobuilder/Semmle.Autobuild/Solution.cs @@ -4,6 +4,8 @@ using System.Collections.Generic; using System.Linq; using Semmle.Util; +using System.IO; +using Semmle.Util.Logging; namespace Semmle.Autobuild { @@ -55,25 +57,33 @@ class Solution : ProjectOrSolution, ISolution public string DefaultPlatformName => solution == null ? "" : solution.GetDefaultPlatformName(); - public Solution(Autobuilder builder, string path) : base(builder, path) + public Solution(Autobuilder builder, string path, bool allowProject) : base(builder, path) { try { solution = SolutionFile.Parse(FullPath); - - includedProjects = - solution.ProjectsInOrder. - Where(p => p.ProjectType == SolutionProjectType.KnownToBeMSBuildFormat). - Select(p => builder.Actions.GetFullPath(FileUtils.ConvertToNative(p.AbsolutePath))). - Select(p => new Project(builder, p)). - ToArray(); } - catch (InvalidProjectFileException) + catch (Exception e) when (e is InvalidProjectFileException || e is FileNotFoundException) { // We allow specifying projects as solutions in lgtm.yml, so model // that scenario as a solution with just that one project - includedProjects = new[] { new Project(builder, path) }; + if (allowProject) + { + includedProjects = new[] { new Project(builder, path) }; + return; + } + + builder.Log(Severity.Info, $"Unable to read solution file {path}."); + includedProjects = new Project[0]; + return; } + + includedProjects = + solution.ProjectsInOrder. + Where(p => p.ProjectType == SolutionProjectType.KnownToBeMSBuildFormat). + Select(p => builder.Actions.PathCombine(Path.GetDirectoryName(path), builder.Actions.PathCombine(p.RelativePath.Split('\\', StringSplitOptions.RemoveEmptyEntries)))). + Select(p => new Project(builder, p)). + ToArray(); } IEnumerable ToolsVersions => includedProjects.Where(p => p.ValidToolsVersion).Select(p => p.ToolsVersion); diff --git a/csharp/autobuilder/Semmle.Autobuild/StandaloneBuildRule.cs b/csharp/autobuilder/Semmle.Autobuild/StandaloneBuildRule.cs index 3fafa84e454b..366ce1f08fc6 100644 --- a/csharp/autobuilder/Semmle.Autobuild/StandaloneBuildRule.cs +++ b/csharp/autobuilder/Semmle.Autobuild/StandaloneBuildRule.cs @@ -7,7 +7,7 @@ namespace Semmle.Autobuild /// class StandaloneBuildRule : IBuildRule { - public BuildScript Analyse(Autobuilder builder) + public BuildScript Analyse(Autobuilder builder, bool auto) { BuildScript GetCommand(string solution) { diff --git a/csharp/autobuilder/Semmle.Autobuild/XmlBuildRule.cs b/csharp/autobuilder/Semmle.Autobuild/XmlBuildRule.cs index 420fc508bfb7..8d34fde6413a 100644 --- a/csharp/autobuilder/Semmle.Autobuild/XmlBuildRule.cs +++ b/csharp/autobuilder/Semmle.Autobuild/XmlBuildRule.cs @@ -7,7 +7,7 @@ namespace Semmle.Autobuild /// class XmlBuildRule : IBuildRule { - public BuildScript Analyse(Autobuilder builder) + public BuildScript Analyse(Autobuilder builder, bool auto) { var command = new CommandBuilder(builder.Actions). RunCommand(builder.Odasa).