diff --git a/PackageIndexer/CsvUtils.cs b/PackageIndexer/CsvUtils.cs new file mode 100644 index 00000000..622250f6 --- /dev/null +++ b/PackageIndexer/CsvUtils.cs @@ -0,0 +1,128 @@ +using System.Globalization; +using CsvHelper; +using CsvHelper.Configuration; + +namespace PackageIndexer; + +internal class CsvUtils +{ + static readonly Dictionary s_targetFrameworks = new() + { + { "8.", "net8.0" }, // Latest stable. + { "9.", "net9.0" } // Latest preview. + }; + + static readonly Dictionary s_opsMonikers = new() + { + { "8.", "dotnet-plat-ext-8.0" }, // Latest stable. + { "9.", "dotnet-plat-ext-9.0" } // Latest preview. + }; + + internal static void GenerateCSVFiles(string indexPackagesPath, string csvPath) + { + Console.WriteLine("Generating CSV files from package index."); + + // For each package XML file + // For each framework + // Map it to a known framework name + // Create a dictionary or add to an existing dictionary *for that version* that will become the CSV file - + // pac,[tfm=;includeXml=false], + // Example: pac01,[tfm=net9.0;includeXml=false]Microsoft.Extensions.Caching.Abstractions,9.0.0-preview.2.24128.5 + // Generate a CSV file from each dictionary + + Dictionary> csvDictionary = []; + Dictionary packageCounter = []; // Used for "pac" number in CSV file. + foreach (string moniker in s_opsMonikers.Values) + { + csvDictionary.Add(moniker, []); + packageCounter.Add(moniker, 1); + } + + // Get all XML files (ignores disabled indexes). + IEnumerable packageIndexFiles = Directory.EnumerateFiles(indexPackagesPath, "*.xml"); + foreach (string packageIndexFile in packageIndexFiles) + { + PackageEntry packageEntry = XmlEntryFormat.ReadPackageEntry(packageIndexFile); + + Console.WriteLine($"Creating CSV entries for package {packageEntry.Name}."); + + // Get OPS moniker and TFM from the package version. + string shortVersion = packageEntry.Version.Substring(0, 2); + if (!s_targetFrameworks.TryGetValue(shortVersion, out string tfm)) + { + Console.WriteLine($"Ignoring package {packageEntry.Name} {packageEntry.Version}."); + continue; + } + string opsMoniker = s_opsMonikers[shortVersion]; + + AddCsvEntryToDict(opsMoniker, csvDictionary, packageCounter, packageEntry, tfm); + } + + // Create the directory. + Directory.CreateDirectory(csvPath); + + foreach (KeyValuePair> tfm in csvDictionary) + { + // CSV file names must match the folder name in the "binaries" repo: + // e.g. netframework-4.6.2, netstandard-2.0, net-8.0. + string filePath = Path.Combine(csvPath, string.Concat(tfm.Key, ".csv")); + + // Delete the file if it already exists. + if (File.Exists(filePath)) + { + File.Delete(filePath); + } + + using var writer = new StreamWriter(filePath); + + var config = new CsvConfiguration(CultureInfo.InvariantCulture) + { + // Don't write the header. + HasHeaderRecord = false, + }; + using var csv = new CsvWriter(writer, config); + csv.WriteRecords(tfm.Value); + } + + static void AddCsvEntryToDict( + string opsMoniker, + Dictionary> csvDictionary, + Dictionary packageCounter, + PackageEntry packageEntry, + string frameworkName + ) + { + // Special case for packages from dotnet/extensions repo - include XML files. + bool includeXml = string.Equals( + packageEntry.Repository, + "https://github.com/dotnet/extensions", + StringComparison.InvariantCultureIgnoreCase + ); + + // Except don't include XML file for Microsoft.Extensions.Diagnostics.ResourceMonitoring + // See https://github.com/dotnet/dotnet-api-docs/pull/10395#discussion_r1758128787. + if (string.Equals( + packageEntry.Name, + "Microsoft.Extensions.Diagnostics.ResourceMonitoring", + StringComparison.InvariantCultureIgnoreCase)) + includeXml = false; + + // Special case for newer assemblies - include XML documentation files. + if (PlatformPackageDefinition.PackagesWithTruthDocs.Contains(packageEntry.Name)) + includeXml = true; + + string squareBrackets = $"[tfm={frameworkName};includeXml={includeXml}]"; + + // Special case for System.ServiceModel.Primitives - use reference assemblies. + if (string.Equals(packageEntry.Name, "System.ServiceModel.Primitives", StringComparison.InvariantCultureIgnoreCase)) + squareBrackets = $"[tfm={frameworkName};includeXml={includeXml};libpath=ref]"; + + CsvEntry entry = CsvEntry.Create( + string.Concat("pac", packageCounter[opsMoniker]++), + string.Concat(squareBrackets, packageEntry.Name), + packageEntry.Version + ); + csvDictionary[opsMoniker].Add(entry); + } + } +} diff --git a/PackageIndexer/DotnetPackageIndex.cs b/PackageIndexer/DotnetPackageIndex.cs index 8af4919c..0fcd9df9 100644 --- a/PackageIndexer/DotnetPackageIndex.cs +++ b/PackageIndexer/DotnetPackageIndex.cs @@ -1,5 +1,7 @@ using System.Collections.Concurrent; using System.Xml.Linq; +using Kusto.Data.Net.Client; +using Kusto.Data; using NuGet.Packaging.Core; using NuGet.Versioning; @@ -7,11 +9,6 @@ namespace PackageIndexer; public static class DotnetPackageIndex { - private static readonly string[] s_dotnetPlatformOwners = [ - "aspnet", - "dotnetframework" - ]; - public static async Task CreateAsync(string packageListPath, bool usePreviewVersions) { // TODO - use nightly feed?? @@ -64,313 +61,40 @@ params string[] feedUrls Console.WriteLine($"Getting packages from {feed.FeedUrl}..."); if (feed.FeedUrl == NuGetFeeds.NuGetOrg) - return await GetPackagesFromNuGetOrgAsync(feed); + return await GetPackagesFromKustoAsync(feed); else //return await GetPackagesFromOtherGalleryAsync(feed); throw new ApplicationException("NuGetOrg should be the only feed."); } - private static async Task> GetPackagesFromNuGetOrgAsync(NuGetFeed feed) + private static async Task> GetPackagesFromKustoAsync(NuGetFeed feed) { - Console.WriteLine("Fetching owner information..."); - - // TODO - // 9/11/24 - Commented out until I can get owner info from Kusto. - //Dictionary ownerInformation = await feed.GetOwnerMappingAsync(); - - //string[] packageIds = ownerInformation.Keys - // .ToHashSet(StringComparer.OrdinalIgnoreCase) - // .Where(id => IsOwnedByDotNet(ownerInformation, id) && - // PackageFilter.Default.IsMatch(id)) - // .ToArray(); - - // Temporarily hard-code the .NET package IDs. - string[] packageIds = [ - "Microsoft.Bcl.AsyncInterfaces", - "Microsoft.Bcl.Build", - "Microsoft.Bcl.Cryptography", - "Microsoft.Bcl.HashCode", - "Microsoft.Bcl.Numerics", - "Microsoft.Bcl.TimeProvider", - "Microsoft.Extensions.AmbientMetadata.Application", - "Microsoft.Extensions.ApiDescription.Client", - "Microsoft.Extensions.ApiDescription.Server", - "Microsoft.Extensions.AsyncState", - "Microsoft.Extensions.AuditReports", - "Microsoft.Extensions.Caching.Abstractions", - "Microsoft.Extensions.Caching.Hybrid", - "Microsoft.Extensions.Caching.Memory", - "Microsoft.Extensions.Caching.SqlServer", - "Microsoft.Extensions.Caching.StackExchangeRedis", - "Microsoft.Extensions.Compliance.Abstractions", - "Microsoft.Extensions.Compliance.Redaction", - "Microsoft.Extensions.Compliance.Testing", - "Microsoft.Extensions.Configuration", - "Microsoft.Extensions.Configuration.Abstractions", - "Microsoft.Extensions.Configuration.Binder", - "Microsoft.Extensions.Configuration.CommandLine", - "Microsoft.Extensions.Configuration.EnvironmentVariables", - "Microsoft.Extensions.Configuration.FileExtensions", - "Microsoft.Extensions.Configuration.Ini", - "Microsoft.Extensions.Configuration.Json", - "Microsoft.Extensions.Configuration.KeyPerFile", - "Microsoft.Extensions.Configuration.UserSecrets", - "Microsoft.Extensions.Configuration.Xml", - "Microsoft.Extensions.DependencyInjection", - "Microsoft.Extensions.DependencyInjection.Abstractions", - "Microsoft.Extensions.DependencyInjection.AutoActivation", - "Microsoft.Extensions.DependencyInjection.Specification.Tests", - "Microsoft.Extensions.DependencyModel", - "Microsoft.Extensions.DiagnosticAdapter", - "Microsoft.Extensions.Diagnostics", - "Microsoft.Extensions.Diagnostics.Abstractions", - "Microsoft.Extensions.Diagnostics.ExceptionSummarization", - "Microsoft.Extensions.Diagnostics.HealthChecks", - "Microsoft.Extensions.Diagnostics.HealthChecks.Abstractions", - "Microsoft.Extensions.Diagnostics.HealthChecks.Common", - "Microsoft.Extensions.Diagnostics.HealthChecks.EntityFrameworkCore", - "Microsoft.Extensions.Diagnostics.HealthChecks.ResourceUtilization", - "Microsoft.Extensions.Diagnostics.Probes", - "Microsoft.Extensions.Diagnostics.ResourceMonitoring", - "Microsoft.Extensions.Diagnostics.Testing", - "Microsoft.Extensions.FileProviders.Abstractions", - "Microsoft.Extensions.FileProviders.Composite", - "Microsoft.Extensions.FileProviders.Embedded", - "Microsoft.Extensions.FileProviders.Physical", - "Microsoft.Extensions.FileSystemGlobbing", - "Microsoft.Extensions.Hosting", - "Microsoft.Extensions.Hosting.Abstractions", - "Microsoft.Extensions.Hosting.Systemd", - "Microsoft.Extensions.Hosting.Testing", - "Microsoft.Extensions.Hosting.WindowsServices", - "Microsoft.Extensions.Http", - "Microsoft.Extensions.Http.Diagnostics", - "Microsoft.Extensions.Http.Polly", - "Microsoft.Extensions.Http.Resilience", - "Microsoft.Extensions.Localization", - "Microsoft.Extensions.Localization.Abstractions", - "Microsoft.Extensions.Logging", - "Microsoft.Extensions.Logging.Abstractions", - "Microsoft.Extensions.Logging.AzureAppServices", - "Microsoft.Extensions.Logging.Configuration", - "Microsoft.Extensions.Logging.Console", - "Microsoft.Extensions.Logging.Debug", - "Microsoft.Extensions.Logging.EventLog", - "Microsoft.Extensions.Logging.EventSource", - "Microsoft.Extensions.Logging.TraceSource", - "Microsoft.Extensions.ObjectPool", - "Microsoft.Extensions.ObjectPool.DependencyInjection", - "Microsoft.Extensions.Options", - "Microsoft.Extensions.Options.ConfigurationExtensions", - "Microsoft.Extensions.Options.Contextual", - "Microsoft.Extensions.Options.DataAnnotations", - "Microsoft.Extensions.Primitives", - "Microsoft.Extensions.Resilience", - "Microsoft.Extensions.StaticAnalysis", - "Microsoft.Extensions.Telemetry", - "Microsoft.Extensions.Telemetry.Abstractions", - "Microsoft.Extensions.TimeProvider.Testing", - "Microsoft.Extensions.WebEncoders", - "Microsoft.Win32.Primitives", - "Microsoft.Win32.Registry", - "Microsoft.Win32.Registry.AccessControl", - "Microsoft.Win32.SystemEvents", - "System.AppContext", - "System.Buffers", - "System.CodeDom", - "System.Collections", - "System.Collections.Concurrent", - "System.Collections.Immutable", - "System.Collections.NonGeneric", - "System.Collections.Specialized", - "System.ComponentModel", - "System.ComponentModel.Annotations", - "System.ComponentModel.Composition", - "System.ComponentModel.Composition.Registration", - "System.ComponentModel.EventBasedAsync", - "System.ComponentModel.Primitives", - "System.ComponentModel.TypeConverter", - "System.Composition", - "System.Composition.AttributedModel", - "System.Composition.Convention", - "System.Composition.Hosting", - "System.Composition.Runtime", - "System.Composition.TypedParts", - "System.Configuration.ConfigurationManager", - "System.Console", - "System.Data.Common", - "System.Data.DataSetExtensions", - "System.Data.Odbc", - "System.Data.OleDb", - "System.Data.SqlClient", - "System.Diagnostics.Contracts", - "System.Diagnostics.Debug", - "System.Diagnostics.DiagnosticSource", - "System.Diagnostics.EventLog", - "System.Diagnostics.FileVersionInfo", - "System.Diagnostics.PerformanceCounter", - "System.Diagnostics.Process", - "System.Diagnostics.StackTrace", - "System.Diagnostics.TextWriterTraceListener", - "System.Diagnostics.Tools", - "System.Diagnostics.TraceSource", - "System.Diagnostics.Tracing", - "System.DirectoryServices", - "System.DirectoryServices.AccountManagement", - "System.DirectoryServices.Protocols", - "System.Drawing.Common", - "System.Drawing.Primitives", - "System.Dynamic.Runtime", - "System.Formats.Asn1", - "System.Formats.Cbor", - "System.Formats.Nrbf", - "System.Globalization", - "System.Globalization.Calendars", - "System.Globalization.Extensions", - "System.IO", - "System.IO.Compression", - "System.IO.Compression.clrcompression-arm", - "System.IO.Compression.clrcompression-x64", - "System.IO.Compression.clrcompression-x86", - "System.IO.Compression.ZipFile", - "System.IO.FileSystem", - "System.IO.FileSystem.AccessControl", - "System.IO.FileSystem.DriveInfo", - "System.IO.FileSystem.Primitives", - "System.IO.FileSystem.Watcher", - "System.IO.Hashing", - "System.IO.IsolatedStorage", - "System.IO.MemoryMappedFiles", - "System.IO.Packaging", - "System.IO.Pipelines", - "System.IO.Pipes", - "System.IO.Pipes.AccessControl", - "System.IO.Ports", - "System.IO.UnmanagedMemoryStream", - "System.Json", - "System.Linq", - "System.Linq.Expressions", - "System.Linq.Parallel", - "System.Linq.Queryable", - "System.Management", - "System.Memory", - "System.Memory.Data", - "System.Net.Http", - "System.Net.Http.Json", - "System.Net.Http.Rtc", - "System.Net.Http.WinHttpHandler", - "System.Net.NameResolution", - "System.Net.NetworkInformation", - "System.Net.Ping", - "System.Net.Primitives", - "System.Net.Requests", - "System.Net.Security", - "System.Net.ServerSentEvents", - "System.Net.Sockets", - "System.Net.WebHeaderCollection", - "System.Net.WebSockets", - "System.Net.WebSockets.Client", - "System.Net.WebSockets.WebSocketProtocol", - "System.Numerics.Tensors", - "System.Numerics.Vectors", - "System.Numerics.Vectors.WindowsRuntime", - "System.ObjectModel", - "System.Private.DataContractSerialization", - "System.Private.Networking", - "System.Private.Uri", - "System.Reflection", - "System.Reflection.Context", - "System.Reflection.DispatchProxy", - "System.Reflection.Emit", - "System.Reflection.Emit.ILGeneration", - "System.Reflection.Emit.Lightweight", - "System.Reflection.Extensions", - "System.Reflection.Metadata", - "System.Reflection.MetadataLoadContext", - "System.Reflection.Primitives", - "System.Reflection.TypeExtensions", - "System.Resources.Extensions", - "System.Resources.Reader", - "System.Resources.ResourceManager", - "System.Resources.Writer", - "System.Runtime", - "System.Runtime.Caching", - "System.Runtime.CompilerServices.Unsafe", - "System.Runtime.CompilerServices.VisualC", - "System.Runtime.Experimental", - "System.Runtime.Extensions", - "System.Runtime.Handles", - "System.Runtime.InteropServices", - "System.Runtime.InteropServices.NFloat.Internal", - "System.Runtime.InteropServices.RuntimeInformation", - "System.Runtime.InteropServices.WindowsRuntime", - "System.Runtime.Loader", - "System.Runtime.Numerics", - "System.Runtime.Serialization.Formatters", - "System.Runtime.Serialization.Json", - "System.Runtime.Serialization.Primitives", - "System.Runtime.Serialization.Schema", - "System.Runtime.Serialization.Xml", - "System.Security.AccessControl", - "System.Security.Claims", - "System.Security.Cryptography.Algorithms", - "System.Security.Cryptography.Cng", - "System.Security.Cryptography.Cose", - "System.Security.Cryptography.Csp", - "System.Security.Cryptography.Encoding", - "System.Security.Cryptography.OpenSsl", - "System.Security.Cryptography.Pkcs", - "System.Security.Cryptography.Primitives", - "System.Security.Cryptography.ProtectedData", - "System.Security.Cryptography.X509Certificates", - "System.Security.Cryptography.Xml", - "System.Security.Permissions", - "System.Security.Principal", - "System.Security.Principal.Windows", - "System.Security.SecureString", - "System.ServiceModel.Duplex", - "System.ServiceModel.Federation", - "System.ServiceModel.Http", - "System.ServiceModel.NetFramingBase", - "System.ServiceModel.NetNamedPipe", - "System.ServiceModel.NetTcp", - "System.ServiceModel.Primitives", - "System.ServiceModel.Security", - "System.ServiceModel.Syndication", - "System.ServiceModel.UnixDomainSocket", - "System.ServiceProcess.ServiceController", - "System.Speech", - "System.Text.Encoding", - "System.Text.Encoding.CodePages", - "System.Text.Encoding.Extensions", - "System.Text.Encodings.Web", - "System.Text.Json", - "System.Text.RegularExpressions", - "System.Threading", - "System.Threading.AccessControl", - "System.Threading.Channels", - "System.Threading.Overlapped", - "System.Threading.Tasks", - "System.Threading.Tasks.Dataflow", - "System.Threading.Tasks.Extensions", - "System.Threading.Tasks.Parallel", - "System.Threading.Thread", - "System.Threading.ThreadPool", - "System.Threading.Timer", - "System.ValueTuple", - "System.Web.Http.Common", - "System.Web.Services.Description", - "System.Windows.Extensions", - "System.Xml.ReaderWriter", - "System.Xml.XDocument", - "System.Xml.XmlDocument", - "System.Xml.XmlSerializer", - "System.Xml.XPath", - "System.Xml.XPath.XDocument", - "System.Xml.XPath.XmlDocument" - ]; - - Console.WriteLine($"Found {packageIds.Length:N0} package IDs owned by .NET."); + Console.WriteLine("Fetching package IDs..."); + + var cluster = "ddteldata.kusto.windows.net"; + var databaseName = "ClientToolsInsights"; + var predicate = string.Join(" or ", PlatformPackageDefinition.Owners.Select(n => $"set_has_element(Owners, \"{n}\")")); + var query = $""" + NiPackageOwners + | where {predicate} + | project Id + | order by Id asc + """; + + var connectionString = new KustoConnectionStringBuilder(cluster).WithAadUserPromptAuthentication(); + using var queryProvider = KustoClientFactory.CreateCslQueryProvider(connectionString); + using var reader = queryProvider.ExecuteQuery(databaseName, query, null); + + List packageIds = []; + + while (reader.Read()) + { + string packageId = reader.GetString(0); + if (PlatformPackageDefinition.Filter.IsMatch(packageId)) + packageIds.Add(packageId); + } + + Console.WriteLine($"Found {packageIds.Count:N0} package IDs owned by .NET."); Console.WriteLine("Getting versions..."); @@ -447,21 +171,4 @@ bool usePreviewVersions return result; } - - private static bool IsOwnedByDotNet(Dictionary ownerInformation, string id) - { - if (ownerInformation.TryGetValue(id, out string[] owners)) - { - foreach (string owner in owners) - { - foreach (string platformOwner in s_dotnetPlatformOwners) - { - if (string.Equals(owner, platformOwner, StringComparison.OrdinalIgnoreCase)) - return true; - } - } - } - - return false; - } } diff --git a/PackageIndexer/PackageIndexer.csproj b/PackageIndexer/PackageIndexer.csproj index 7b9d5341..e39f52bb 100644 --- a/PackageIndexer/PackageIndexer.csproj +++ b/PackageIndexer/PackageIndexer.csproj @@ -8,10 +8,10 @@ - + - + diff --git a/PackageIndexer/PlatformPackageDefinition.cs b/PackageIndexer/PlatformPackageDefinition.cs new file mode 100644 index 00000000..00fc11cb --- /dev/null +++ b/PackageIndexer/PlatformPackageDefinition.cs @@ -0,0 +1,111 @@ +using System.Collections.Frozen; + +namespace PackageIndexer; + +internal static class PlatformPackageDefinition +{ + private static FrozenSet s_packageIds; + + public static readonly List PackagesWithTruthDocs = + [ + "Microsoft.Bcl.AsyncInterfaces", + "Microsoft.Bcl.Cryptography", + "Microsoft.Bcl.Memory", + "Microsoft.Bcl.Numerics", + "Microsoft.Bcl.TimeProvider", + "Microsoft.Extensions.Caching.Abstractions", + "Microsoft.Extensions.Caching.Memory", + "Microsoft.Extensions.Configuration", + "Microsoft.Extensions.Configuration.Abstractions", + "Microsoft.Extensions.Configuration.Binder", + "Microsoft.Extensions.Configuration.CommandLine", + "Microsoft.Extensions.Configuration.EnvironmentVariables", + "Microsoft.Extensions.Configuration.FileExtensions", + "Microsoft.Extensions.Configuration.Ini", + "Microsoft.Extensions.Configuration.Json", + "Microsoft.Extensions.Configuration.UserSecrets", + "Microsoft.Extensions.Configuration.Xml", + "Microsoft.Extensions.DependencyInjection", + "Microsoft.Extensions.DependencyInjection.Abstractions", + "Microsoft.Extensions.DependencyInjection.Specification.Tests", + "Microsoft.Extensions.Diagnostics", + "Microsoft.Extensions.Diagnostics.Abstractions", + "Microsoft.Extensions.FileProviders.Abstractions", + "Microsoft.Extensions.FileProviders.Composite", + "Microsoft.Extensions.FileProviders.Physical", + "Microsoft.Extensions.HostFactoryResolver.Sources", + "Microsoft.Extensions.Hosting", + "Microsoft.Extensions.Hosting.Abstractions", + "Microsoft.Extensions.Hosting.Systemd", + "Microsoft.Extensions.Hosting.WindowsServices", + "Microsoft.Extensions.Http", + "Microsoft.Extensions.Logging", + "Microsoft.Extensions.Logging.Abstractions", + "Microsoft.Extensions.Logging.Configuration", + "Microsoft.Extensions.Logging.Console", + "Microsoft.Extensions.Logging.Debug", + "Microsoft.Extensions.Logging.EventLog", + "Microsoft.Extensions.Logging.EventSource", + "Microsoft.Extensions.Logging.TraceSource", + "Microsoft.Extensions.Options", + "Microsoft.Extensions.Options.ConfigurationExtensions", + "Microsoft.Extensions.Options.DataAnnotations", + "Microsoft.Extensions.Primitives", + "System.Composition", + "System.Diagnostics.EventLog.Messages", + "System.Formats.Cbor", + "System.Formats.Nrbf", + "System.Net.ServerSentEvents", + "System.Numerics.Tensors", + "System.Runtime.Serialization.Schema", + "System.Speech" + ]; + + public static FrozenSet Owners = FrozenSet.ToFrozenSet( + [ + "aspnet", + "dotnetframework" + ], StringComparer.OrdinalIgnoreCase); + + public static PackageFilter Filter { get; } = new( + includes: + [ + PackageFilterExpression.Parse("Microsoft.Bcl.*"), + PackageFilterExpression.Parse("Microsoft.Extensions.*"), + PackageFilterExpression.Parse("Microsoft.IO.Redist"), + PackageFilterExpression.Parse("Microsoft.Win32.*"), + PackageFilterExpression.Parse("System.*") + ], + excludes: + [ + PackageFilterExpression.Parse("System.Private.ServiceModel"), + PackageFilterExpression.Parse("System.Runtime.WindowsRuntime"), + PackageFilterExpression.Parse("System.Runtime.WindowsRuntime.UI.Xaml"), + // Documented under Azure SDK for .NET moniker. + PackageFilterExpression.Parse("System.ClientModel"), + // Documented under ASP.NET Core moniker. + PackageFilterExpression.Parse("System.Threading.RateLimiting"), + PackageFilterExpression.Parse("Microsoft.Extensions.Features"), + PackageFilterExpression.Parse("Microsoft.Extensions.Identity.Core"), + PackageFilterExpression.Parse("Microsoft.Extensions.Identity.Stores"), + // Documented under ML.NET moniker. + PackageFilterExpression.Parse("Microsoft.Extensions.ML"), + // Documented under .NET Aspire moniker. + PackageFilterExpression.Parse("Microsoft.Extensions.ServiceDiscovery*"), + // Suffixes. + PackageFilterExpression.Parse("*.cs"), + PackageFilterExpression.Parse("*.de"), + PackageFilterExpression.Parse("*.es"), + PackageFilterExpression.Parse("*.fr"), + PackageFilterExpression.Parse("*.it"), + PackageFilterExpression.Parse("*.ja"), + PackageFilterExpression.Parse("*.ko"), + PackageFilterExpression.Parse("*.pl"), + PackageFilterExpression.Parse("*.pt-br"), + PackageFilterExpression.Parse("*.ru"), + PackageFilterExpression.Parse("*.tr"), + PackageFilterExpression.Parse("*.zh-Hans"), + PackageFilterExpression.Parse("*.zh-Hant"), + ] + ); +} diff --git a/PackageIndexer/Program.cs b/PackageIndexer/Program.cs index 599f4216..f150ebc1 100644 --- a/PackageIndexer/Program.cs +++ b/PackageIndexer/Program.cs @@ -1,4 +1,5 @@ -using System.Diagnostics; +using System.Collections.ObjectModel; +using System.Diagnostics; using System.Globalization; using System.Xml.Linq; using CsvHelper; @@ -8,81 +9,10 @@ namespace PackageIndexer; internal static class Program { - static readonly Dictionary s_tfmToOpsMoniker = new() - { - { "net462", "netframework-4.6.2" }, - { "net47", "netframework-4.7" }, - { "net471", "netframework-4.7.1" }, - { "net472", "netframework-4.7.2" }, - { "net48", "netframework-4.8" }, - { "net481", "netframework-4.8.1" }, - { "net6.0", "net-6.0" }, - { "net7.0", "net-7.0" }, - { "net8.0", "net-8.0" }, - { "net9.0", "net-9.0" }, - { "netstandard2.0", "netstandard-2.0" }, - { "netstandard2.1", "netstandard-2.1" } - }; - - static readonly List s_packagesWithTruthDocs = - [ - "Microsoft.Bcl.AsyncInterfaces", - "Microsoft.Bcl.Cryptography", - "Microsoft.Bcl.Memory", - "Microsoft.Bcl.Numerics", - "Microsoft.Bcl.TimeProvider", - "Microsoft.Extensions.Caching.Abstractions", - "Microsoft.Extensions.Caching.Memory", - "Microsoft.Extensions.Configuration", - "Microsoft.Extensions.Configuration.Abstractions", - "Microsoft.Extensions.Configuration.Binder", - "Microsoft.Extensions.Configuration.CommandLine", - "Microsoft.Extensions.Configuration.EnvironmentVariables", - "Microsoft.Extensions.Configuration.FileExtensions", - "Microsoft.Extensions.Configuration.Ini", - "Microsoft.Extensions.Configuration.Json", - "Microsoft.Extensions.Configuration.UserSecrets", - "Microsoft.Extensions.Configuration.Xml", - "Microsoft.Extensions.DependencyInjection", - "Microsoft.Extensions.DependencyInjection.Abstractions", - "Microsoft.Extensions.DependencyInjection.Specification.Tests", - "Microsoft.Extensions.Diagnostics", - "Microsoft.Extensions.Diagnostics.Abstractions", - "Microsoft.Extensions.FileProviders.Abstractions", - "Microsoft.Extensions.FileProviders.Composite", - "Microsoft.Extensions.FileProviders.Physical", - "Microsoft.Extensions.HostFactoryResolver.Sources", - "Microsoft.Extensions.Hosting", - "Microsoft.Extensions.Hosting.Abstractions", - "Microsoft.Extensions.Hosting.Systemd", - "Microsoft.Extensions.Hosting.WindowsServices", - "Microsoft.Extensions.Http", - "Microsoft.Extensions.Logging", - "Microsoft.Extensions.Logging.Abstractions", - "Microsoft.Extensions.Logging.Configuration", - "Microsoft.Extensions.Logging.Console", - "Microsoft.Extensions.Logging.Debug", - "Microsoft.Extensions.Logging.EventLog", - "Microsoft.Extensions.Logging.EventSource", - "Microsoft.Extensions.Logging.TraceSource", - "Microsoft.Extensions.Options", - "Microsoft.Extensions.Options.ConfigurationExtensions", - "Microsoft.Extensions.Options.DataAnnotations", - "Microsoft.Extensions.Primitives", - "System.Composition", - "System.Diagnostics.EventLog.Messages", - "System.Formats.Cbor", - "System.Formats.Nrbf", - "System.Net.ServerSentEvents", - "System.Numerics.Tensors", - "System.Runtime.Serialization.Schema", - "System.Speech" - ]; - private static async Task Main(string[] args) { #if DEBUG - args = [@"c:\users\gewarren\desktop\Package Index 0911b", "preview"]; + args = [@"c:\users\gewarren\desktop\Package Index 1030", "preview"]; #endif if ((args.Length == 0) || (args.Length > 2)) @@ -130,168 +60,12 @@ private static async Task RunAsync(string rootPath, bool usePreviewVersions) await DownloadDotnetPackageListAsync(packageListPath, usePreviewVersions); await GeneratePackageIndexAsync(packageListPath, packagesPath, indexPackagesPath); //, frameworksPath); - GenerateCSVFiles(indexPackagesPath, csvPath); + CsvUtils.GenerateCSVFiles(indexPackagesPath, csvPath); Console.WriteLine($"Completed in {stopwatch.Elapsed}"); Console.WriteLine($"Peak working set: {Process.GetCurrentProcess().PeakWorkingSet64 / (1024 * 1024):N2} MB"); } - private static void GenerateCSVFiles(string indexPackagesPath, string csvPath) - { - Console.WriteLine("Generating CSV files from package index."); - - // For each package XML file - // For each framework - // Map it to a known framework name - // Generate a collection of that version + later versions of that framework - // (e.g. add 4.7, 4.7.1, 4.7.2, 4.8, 4.8.1 for net462; add 2.1 for netstandard2.0; add 7.0, 8.0, 9.0 for net6.0) - // Create a dictionary or add to an existing dictionary *for that version* that will become the CSV file - - // pac,[tfm=;includeXml=false], - // Example: pac01,[tfm=net9.0;includeXml=false]Microsoft.Extensions.Caching.Abstractions,9.0.0-preview.2.24128.5 - // Generate a CSV file from each dictionary - - Dictionary> csvDictionary = []; - Dictionary packageCounter = []; // Used for "pac" number in CSV file. - foreach (string moniker in s_tfmToOpsMoniker.Values) - { - csvDictionary.Add(moniker, []); - packageCounter.Add(moniker, 1); - } - - // Get all XML files (ignores disabled indexes). - IEnumerable packageIndexFiles = Directory.EnumerateFiles(indexPackagesPath, "*.xml"); - foreach (string packageIndexFile in packageIndexFiles) - { - // Read XML file to get each listed framework. - PackageEntry packageEntry = XmlEntryFormat.ReadPackageEntry(packageIndexFile); - - Console.WriteLine($"Creating CSV entries for package {packageEntry.Name}."); - - // Add to each applicable CSV file. - foreach (FrameworkEntry frameworkEntry in packageEntry.FrameworkEntries) - { - string framework = frameworkEntry.FrameworkName; - string opsMoniker; - switch (framework) - { - case "net462": - opsMoniker = s_tfmToOpsMoniker[framework]; - AddCsvEntryToDict(opsMoniker, csvDictionary, packageCounter, packageEntry, frameworkEntry); - framework = "net47"; - goto case "net47"; - case "net47": - opsMoniker = s_tfmToOpsMoniker[framework]; - AddCsvEntryToDict(opsMoniker, csvDictionary, packageCounter, packageEntry, frameworkEntry); - framework = "net471"; - goto case "net471"; - case "net471": - opsMoniker = s_tfmToOpsMoniker[framework]; - AddCsvEntryToDict(opsMoniker, csvDictionary, packageCounter, packageEntry, frameworkEntry); - framework = "net472"; - goto case "net472"; - case "net472": - opsMoniker = s_tfmToOpsMoniker[framework]; - AddCsvEntryToDict(opsMoniker, csvDictionary, packageCounter, packageEntry, frameworkEntry); - framework = "net48"; - goto case "net48"; - case "net48": - opsMoniker = s_tfmToOpsMoniker[framework]; - AddCsvEntryToDict(opsMoniker, csvDictionary, packageCounter, packageEntry, frameworkEntry); - framework = "net481"; - goto case "net481"; - case "net481": - case "net6.0": - case "net7.0": - case "net8.0": - case "net9.0": - case "netstandard2.0": - case "netstandard2.1": - opsMoniker = s_tfmToOpsMoniker[framework]; - AddCsvEntryToDict(opsMoniker, csvDictionary, packageCounter, packageEntry, frameworkEntry); - break; - default: - Console.WriteLine($"Ignoring target framework {framework}."); - break; - } - } - } - - // Update 08/14: Removed since it caused the pipeline to fail on dependencies. - //// Special case for System.ServiceModel.Primitives - add version 4.10.3. - //// (See https://github.com/dotnet/dotnet-api-docs/pull/10164#discussion_r1696016010.) - //AddCsvEntryToDict("netstandard-2.0", csvDictionary, packageCounter, - // PackageEntry.Create("System.ServiceModel.Primitives", "4.10.3", "https://github.com/dotnet/wcf", []), - // FrameworkEntry.Create("netstandard2.0") - // ); - - // Create the directory. - Directory.CreateDirectory(csvPath); - - foreach (KeyValuePair> tfm in csvDictionary) - { - // CSV file names must match the folder name in the "binaries" repo: - // e.g. netframework-4.6.2, netstandard-2.0, net-8.0. - string filePath = Path.Combine(csvPath, string.Concat(tfm.Key, ".csv")); - - // Delete the file if it already exists. - if (File.Exists(filePath)) - { - File.Delete(filePath); - } - - using var writer = new StreamWriter(filePath); - - var config = new CsvConfiguration(CultureInfo.InvariantCulture) - { - // Don't write the header. - HasHeaderRecord = false, - }; - using var csv = new CsvWriter(writer, config); - csv.WriteRecords(tfm.Value); - } - - static void AddCsvEntryToDict( - string opsMoniker, - Dictionary> csvDictionary, - Dictionary packageCounter, - PackageEntry packageEntry, - FrameworkEntry tfm - ) - { - // Special case for packages from dotnet/extensions repo - include XML files. - bool includeXml = string.Equals( - packageEntry.Repository, - "https://github.com/dotnet/extensions", - StringComparison.InvariantCultureIgnoreCase - ); - - // Except don't include XML file for Microsoft.Extensions.Diagnostics.ResourceMonitoring - // See https://github.com/dotnet/dotnet-api-docs/pull/10395#discussion_r1758128787. - if (string.Equals( - packageEntry.Name, - "Microsoft.Extensions.Diagnostics.ResourceMonitoring", - StringComparison.InvariantCultureIgnoreCase)) - includeXml = false; - - // Special case for newer assemblies - include XML documentation files. - if (s_packagesWithTruthDocs.Contains(packageEntry.Name)) - includeXml = true; - - string squareBrackets = $"[tfm={tfm.FrameworkName};includeXml={includeXml}]"; - - // Special case for System.ServiceModel.Primitives - use reference assemblies. - if (string.Equals(packageEntry.Name, "System.ServiceModel.Primitives", StringComparison.InvariantCultureIgnoreCase)) - squareBrackets = $"[tfm={tfm.FrameworkName};includeXml={includeXml};libpath=ref]"; - - CsvEntry entry = CsvEntry.Create( - string.Concat("pac", packageCounter[opsMoniker]++), - string.Concat(squareBrackets, packageEntry.Name), - packageEntry.Version - ); - csvDictionary[opsMoniker].Add(entry); - } - } - private static async Task DownloadDotnetPackageListAsync(string packageListPath, bool usePreviewVersions) { if (!File.Exists(packageListPath))