From de4712294f542a34084d0f5b444298fa82459734 Mon Sep 17 00:00:00 2001 From: Yang Cao Date: Mon, 11 Dec 2023 12:49:16 -0500 Subject: [PATCH 01/19] Use Azure SDK for uploading blobs --- src/Sdk/Sdk.csproj | 1 + src/Sdk/WebApi/WebApi/ResultsHttpClient.cs | 99 ++++++++++------------ 2 files changed, 47 insertions(+), 53 deletions(-) diff --git a/src/Sdk/Sdk.csproj b/src/Sdk/Sdk.csproj index dbd96f33616..ff1cb85a4fe 100644 --- a/src/Sdk/Sdk.csproj +++ b/src/Sdk/Sdk.csproj @@ -13,6 +13,7 @@ + diff --git a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs index 3bdc8cc0289..3d68fa4e512 100644 --- a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs +++ b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.IO; using System.Linq; using System.Net.Http; @@ -8,8 +7,11 @@ using System.Threading; using System.Threading.Tasks; using System.Net.Http.Formatting; +using Azure; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using Azure.Storage.Blobs.Specialized; using GitHub.DistributedTask.WebApi; -using GitHub.Services.Common; using GitHub.Services.Results.Contracts; using Sdk.WebApi.WebApi; @@ -91,7 +93,6 @@ private async Task GetJobLogUploadUrlAsync(string p } // Create metadata calls - private async Task SendRequest(Uri uri, CancellationToken cancellationToken, R request, string timestamp) { using (HttpRequestMessage requestMessage = new HttpRequestMessage(HttpMethod.Post, uri)) @@ -161,73 +162,68 @@ private async Task JobLogUploadCompleteAsync(string planId, string jobId, long l await SendRequest(createJobLogsMetadataEndpoint, cancellationToken, request, timestamp); } - private async Task UploadBlockFileAsync(string url, string blobStorageType, FileStream file, CancellationToken cancellationToken) + private (Uri path, string sas) ParseSasToken(string url) { - // Upload the file to the url - var request = new HttpRequestMessage(HttpMethod.Put, url) + if (String.IsNullOrEmpty(url)) { - Content = new StreamContent(file) - }; + throw new Exception($"SAS url is empty"); + } + var blobUri = new Uri(url); + var blobPath = $"{blobUri.Scheme}://{blobUri.Authority}/{blobUri.AbsolutePath}"; + var sasUrl = blobUri.Query.Substring(1); //remove starting "?" + return (new Uri(blobPath), sasUrl); + } + + private async Task UploadBlockFileAsync(string url, string blobStorageType, FileStream file, CancellationToken cancellationToken) + { if (blobStorageType == BlobStorageTypes.AzureBlobStorage) { - request.Content.Headers.Add(Constants.AzureBlobTypeHeader, Constants.AzureBlockBlob); - } + var blobUri = ParseSasToken(url); + var blobClient = new BlobClient(blobUri.path, new AzureSasCredential(blobUri.sas)); - using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken)) - { - if (!response.IsSuccessStatusCode) + try + { + await blobClient.UploadAsync(file, cancellationToken); + } + catch (RequestFailedException e) { - throw new Exception($"Failed to upload file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}"); + throw new Exception($"Failed to upload block to Azure blob: {e.Message}"); } - return response; } } - private async Task CreateAppendFileAsync(string url, string blobStorageType, CancellationToken cancellationToken) + private async Task CreateAppendFileAsync(string url, string blobStorageType, CancellationToken cancellationToken) { - var request = new HttpRequestMessage(HttpMethod.Put, url) - { - Content = new StringContent("") - }; if (blobStorageType == BlobStorageTypes.AzureBlobStorage) { - request.Content.Headers.Add(Constants.AzureBlobTypeHeader, Constants.AzureAppendBlob); - request.Content.Headers.Add("Content-Length", "0"); - } - - using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken)) - { - if (!response.IsSuccessStatusCode) + var blobUri = ParseSasToken(url); + var appendBlobClient = new AppendBlobClient(blobUri.path, new AzureSasCredential(blobUri.sas)); + try + { + await appendBlobClient.CreateAsync(cancellationToken: cancellationToken); + } + catch (RequestFailedException e) { - throw new Exception($"Failed to create append file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}"); + throw new Exception($"Failed to create append blob in Azure blob: {e.Message}"); } - return response; } } - private async Task UploadAppendFileAsync(string url, string blobStorageType, FileStream file, bool finalize, long fileSize, CancellationToken cancellationToken) + private async Task UploadAppendFileAsync(string url, string blobStorageType, FileStream file, bool finalize, long fileSize, CancellationToken cancellationToken) { - var comp = finalize ? "&comp=appendblock&seal=true" : "&comp=appendblock"; - // Upload the file to the url - var request = new HttpRequestMessage(HttpMethod.Put, url + comp) - { - Content = new StreamContent(file) - }; - if (blobStorageType == BlobStorageTypes.AzureBlobStorage) { - request.Content.Headers.Add("Content-Length", fileSize.ToString()); - request.Content.Headers.Add(Constants.AzureBlobSealedHeader, finalize.ToString()); - } - - using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken)) - { - if (!response.IsSuccessStatusCode) + var blobUri = ParseSasToken(url); + var appendBlobClient = new AppendBlobClient(blobUri.path, new AzureSasCredential(blobUri.sas)); + try { - throw new Exception($"Failed to upload append file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}, object: {response}, fileSize: {fileSize}"); + await appendBlobClient.AppendBlockAsync(file, cancellationToken: cancellationToken); + } + catch (RequestFailedException e) + { + throw new Exception($"Failed to upload append block in Azure blob: {e.Message}"); } - return response; } } @@ -251,23 +247,22 @@ public async Task UploadStepSummaryAsync(string planId, string jobId, Guid stepI // Upload the file using (var fileStream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true)) { - var response = await UploadBlockFileAsync(uploadUrlResponse.SummaryUrl, uploadUrlResponse.BlobStorageType, fileStream, cancellationToken); + await UploadBlockFileAsync(uploadUrlResponse.SummaryUrl, uploadUrlResponse.BlobStorageType, fileStream, cancellationToken); } // Send step summary upload complete message await StepSummaryUploadCompleteAsync(planId, jobId, stepId, fileSize, cancellationToken); } - private async Task UploadLogFile(string file, bool finalize, bool firstBlock, string sasUrl, string blobStorageType, + private async Task UploadLogFile(string file, bool finalize, bool firstBlock, string sasUrl, string blobStorageType, CancellationToken cancellationToken) { - HttpResponseMessage response; if (firstBlock && finalize) { // This is the one and only block, just use a block blob using (var fileStream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true)) { - response = await UploadBlockFileAsync(sasUrl, blobStorageType, fileStream, cancellationToken); + await UploadBlockFileAsync(sasUrl, blobStorageType, fileStream, cancellationToken); } } else @@ -283,11 +278,9 @@ private async Task UploadLogFile(string file, bool finalize var fileSize = new FileInfo(file).Length; using (var fileStream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true)) { - response = await UploadAppendFileAsync(sasUrl, blobStorageType, fileStream, finalize, fileSize, cancellationToken); + await UploadAppendFileAsync(sasUrl, blobStorageType, fileStream, finalize, fileSize, cancellationToken); } } - - return response; } // Handle file upload for step log From 745ecd1c4e7081624d256bdbcfea81381110b60d Mon Sep 17 00:00:00 2001 From: Yang Cao Date: Mon, 11 Dec 2023 12:51:47 -0500 Subject: [PATCH 02/19] Remove unused consts --- src/Sdk/WebApi/WebApi/ResultsHttpClient.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs index 3d68fa4e512..89f1735fc1e 100644 --- a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs +++ b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs @@ -414,11 +414,6 @@ public static class Constants public static readonly string CreateJobLogsMetadata = ResultsReceiverTwirpEndpoint + "CreateJobLogsMetadata"; public static readonly string ResultsProtoApiV1Endpoint = "twirp/github.actions.results.api.v1.WorkflowStepUpdateService/"; public static readonly string WorkflowStepsUpdate = ResultsProtoApiV1Endpoint + "WorkflowStepsUpdate"; - - public static readonly string AzureBlobSealedHeader = "x-ms-blob-sealed"; - public static readonly string AzureBlobTypeHeader = "x-ms-blob-type"; - public static readonly string AzureBlockBlob = "BlockBlob"; - public static readonly string AzureAppendBlob = "AppendBlob"; } } From af780c56b6c1a0d4af9fbb3ef1fb46b7a0bcb2ad Mon Sep 17 00:00:00 2001 From: Yang Cao Date: Mon, 11 Dec 2023 17:33:40 -0500 Subject: [PATCH 03/19] Setup retries and timeout --- src/Sdk/WebApi/WebApi/ResultsHttpClient.cs | 53 ++++++++++++++++++---- 1 file changed, 44 insertions(+), 9 deletions(-) diff --git a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs index 89f1735fc1e..06c49af6fb7 100644 --- a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs +++ b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs @@ -175,13 +175,43 @@ private async Task JobLogUploadCompleteAsync(string planId, string jobId, long l return (new Uri(blobPath), sasUrl); } + private BlobClient GetBlobClient(string url) + { + var blobUri = ParseSasToken(url); + + var opts = new BlobClientOptions + { + Retry = + { + MaxRetries = Constants.DefaultBlobUploadRetries, + NetworkTimeout = TimeSpan.FromSeconds(Constants.DefaultNetworkTimeoutInSeconds) + } + }; + + return new BlobClient(blobUri.path, new AzureSasCredential(blobUri.sas), opts); + } + + private AppendBlobClient GetAppendBlobClient(string url) + { + var blobUri = ParseSasToken(url); + + var opts = new BlobClientOptions + { + Retry = + { + MaxRetries = Constants.DefaultBlobUploadRetries, + NetworkTimeout = TimeSpan.FromSeconds(Constants.DefaultNetworkTimeoutInSeconds) + } + }; + + return new AppendBlobClient(blobUri.path, new AzureSasCredential(blobUri.sas), opts); + } + private async Task UploadBlockFileAsync(string url, string blobStorageType, FileStream file, CancellationToken cancellationToken) { if (blobStorageType == BlobStorageTypes.AzureBlobStorage) { - var blobUri = ParseSasToken(url); - var blobClient = new BlobClient(blobUri.path, new AzureSasCredential(blobUri.sas)); - + var blobClient = GetBlobClient(url); try { await blobClient.UploadAsync(file, cancellationToken); @@ -197,8 +227,7 @@ private async Task CreateAppendFileAsync(string url, string blobStorageType, Can { if (blobStorageType == BlobStorageTypes.AzureBlobStorage) { - var blobUri = ParseSasToken(url); - var appendBlobClient = new AppendBlobClient(blobUri.path, new AzureSasCredential(blobUri.sas)); + var appendBlobClient = GetAppendBlobClient(url); try { await appendBlobClient.CreateAsync(cancellationToken: cancellationToken); @@ -210,15 +239,18 @@ private async Task CreateAppendFileAsync(string url, string blobStorageType, Can } } - private async Task UploadAppendFileAsync(string url, string blobStorageType, FileStream file, bool finalize, long fileSize, CancellationToken cancellationToken) + private async Task UploadAppendFileAsync(string url, string blobStorageType, FileStream file, bool finalize, CancellationToken cancellationToken) { if (blobStorageType == BlobStorageTypes.AzureBlobStorage) { - var blobUri = ParseSasToken(url); - var appendBlobClient = new AppendBlobClient(blobUri.path, new AzureSasCredential(blobUri.sas)); + var appendBlobClient = GetAppendBlobClient(url); try { await appendBlobClient.AppendBlockAsync(file, cancellationToken: cancellationToken); + if (finalize) + { + await appendBlobClient.SealAsync(cancellationToken: cancellationToken); + } } catch (RequestFailedException e) { @@ -278,7 +310,7 @@ private async Task UploadLogFile(string file, bool finalize, bool firstBlock, st var fileSize = new FileInfo(file).Length; using (var fileStream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true)) { - await UploadAppendFileAsync(sasUrl, blobStorageType, fileStream, finalize, fileSize, cancellationToken); + await UploadAppendFileAsync(sasUrl, blobStorageType, fileStream, finalize, cancellationToken); } } } @@ -414,6 +446,9 @@ public static class Constants public static readonly string CreateJobLogsMetadata = ResultsReceiverTwirpEndpoint + "CreateJobLogsMetadata"; public static readonly string ResultsProtoApiV1Endpoint = "twirp/github.actions.results.api.v1.WorkflowStepUpdateService/"; public static readonly string WorkflowStepsUpdate = ResultsProtoApiV1Endpoint + "WorkflowStepsUpdate"; + + public static readonly int DefaultNetworkTimeoutInSeconds = 30; + public static readonly int DefaultBlobUploadRetries = 3; } } From 04408a31ccc6bac23ad22418c090c9e4b8f069c9 Mon Sep 17 00:00:00 2001 From: Yang Cao Date: Mon, 11 Dec 2023 17:49:16 -0500 Subject: [PATCH 04/19] whitespace format --- src/Sdk/WebApi/WebApi/ResultsHttpClient.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs index 06c49af6fb7..9180be8b52c 100644 --- a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs +++ b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs @@ -178,7 +178,7 @@ private async Task JobLogUploadCompleteAsync(string planId, string jobId, long l private BlobClient GetBlobClient(string url) { var blobUri = ParseSasToken(url); - + var opts = new BlobClientOptions { Retry = @@ -190,11 +190,11 @@ private BlobClient GetBlobClient(string url) return new BlobClient(blobUri.path, new AzureSasCredential(blobUri.sas), opts); } - + private AppendBlobClient GetAppendBlobClient(string url) { var blobUri = ParseSasToken(url); - + var opts = new BlobClientOptions { Retry = From ced9b5e1efbe2ba109f9ca4e5b531f40fbab784d Mon Sep 17 00:00:00 2001 From: Yang Cao Date: Tue, 12 Dec 2023 09:17:04 -0500 Subject: [PATCH 05/19] update dotnet runtime hash --- src/Misc/contentHash/dotnetRuntime/linux-arm64 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Misc/contentHash/dotnetRuntime/linux-arm64 b/src/Misc/contentHash/dotnetRuntime/linux-arm64 index 3380d9dcbfa..b72052c44bd 100644 --- a/src/Misc/contentHash/dotnetRuntime/linux-arm64 +++ b/src/Misc/contentHash/dotnetRuntime/linux-arm64 @@ -1 +1 @@ -722dd5fa5ecc207fcccf67f6e502d689f2119d8117beff2041618fba17dc66a4 \ No newline at end of file +0cbdca75fa9afb0f7c726eff95cc6d663348e8cc76536affe07df92251a61483 From c2cf1df4d5fcbeefbc2e58c800fe3cbeab9de171 Mon Sep 17 00:00:00 2001 From: Yang Cao Date: Tue, 12 Dec 2023 09:33:01 -0500 Subject: [PATCH 06/19] update dotnet runtime hash for linux-arm --- src/Misc/contentHash/dotnetRuntime/linux-arm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Misc/contentHash/dotnetRuntime/linux-arm b/src/Misc/contentHash/dotnetRuntime/linux-arm index c750b23ed50..fb961f28f0b 100644 --- a/src/Misc/contentHash/dotnetRuntime/linux-arm +++ b/src/Misc/contentHash/dotnetRuntime/linux-arm @@ -1 +1 @@ -531b31914e525ecb12cc5526415bc70a112ebc818f877347af1a231011f539c5 \ No newline at end of file +a1761f8cbd803571e37b5ce4b0511c33e2ffd0dbe8519f0cbc80503a6d9e356c From 64f52a9f5b74a3f25951bb06db931dbb82ecb6c9 Mon Sep 17 00:00:00 2001 From: Yang Cao Date: Tue, 2 Jan 2024 11:47:38 -0500 Subject: [PATCH 07/19] use a flag on server to determine if we switch to SDK --- src/Runner.Common/JobServerQueue.cs | 4 +- src/Runner.Common/ResultsServer.cs | 10 +-- src/Sdk/WebApi/WebApi/ResultsHttpClient.cs | 84 ++++++++++++++++++++-- 3 files changed, 85 insertions(+), 13 deletions(-) diff --git a/src/Runner.Common/JobServerQueue.cs b/src/Runner.Common/JobServerQueue.cs index e6a00f1c823..6e25f40bf78 100644 --- a/src/Runner.Common/JobServerQueue.cs +++ b/src/Runner.Common/JobServerQueue.cs @@ -134,8 +134,8 @@ public void Start(Pipelines.AgentJobRequestMessage jobRequest, bool resultsServi { liveConsoleFeedUrl = feedStreamUrl; } - - _resultsServer.InitializeResultsClient(new Uri(resultsReceiverEndpoint), liveConsoleFeedUrl, accessToken); + jobRequest.Variables.TryGetValue("system.github.results_upload_with_sdk", out VariableValue resultsUseSdkVariable); + _resultsServer.InitializeResultsClient(new Uri(resultsReceiverEndpoint), liveConsoleFeedUrl, accessToken, Convert.ToBoolean(resultsEndpointVariable?.Value)); _resultsClientInitiated = true; } diff --git a/src/Runner.Common/ResultsServer.cs b/src/Runner.Common/ResultsServer.cs index ef97ebcfc65..f3bf9910cfe 100644 --- a/src/Runner.Common/ResultsServer.cs +++ b/src/Runner.Common/ResultsServer.cs @@ -19,7 +19,7 @@ namespace GitHub.Runner.Common [ServiceLocator(Default = typeof(ResultServer))] public interface IResultsServer : IRunnerService, IAsyncDisposable { - void InitializeResultsClient(Uri uri, string liveConsoleFeedUrl, string token); + void InitializeResultsClient(Uri uri, string liveConsoleFeedUrl, string token, bool useSdk); Task AppendLiveConsoleFeedAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, Guid stepId, IList lines, long? startLine, CancellationToken cancellationToken); @@ -51,9 +51,9 @@ public sealed class ResultServer : RunnerService, IResultsServer private String _liveConsoleFeedUrl; private string _token; - public void InitializeResultsClient(Uri uri, string liveConsoleFeedUrl, string token) + public void InitializeResultsClient(Uri uri, string liveConsoleFeedUrl, string token, bool useSdk) { - this._resultsClient = CreateHttpClient(uri, token); + this._resultsClient = CreateHttpClient(uri, token, useSdk); _token = token; if (!string.IsNullOrEmpty(liveConsoleFeedUrl)) @@ -63,7 +63,7 @@ public void InitializeResultsClient(Uri uri, string liveConsoleFeedUrl, string t } } - public ResultsHttpClient CreateHttpClient(Uri uri, string token) + public ResultsHttpClient CreateHttpClient(Uri uri, string token, bool useSdk) { // Using default 100 timeout RawClientHttpRequestSettings settings = VssUtil.GetHttpRequestSettings(null); @@ -80,7 +80,7 @@ public ResultsHttpClient CreateHttpClient(Uri uri, string token) var pipeline = HttpClientFactory.CreatePipeline(httpMessageHandler, delegatingHandlers); - return new ResultsHttpClient(uri, pipeline, token, disposeHandler: true); + return new ResultsHttpClient(uri, pipeline, token, disposeHandler: true, useSdk: useSdk); } public Task CreateResultsStepSummaryAsync(string planId, string jobId, Guid stepId, string file, diff --git a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs index 9180be8b52c..73282d26258 100644 --- a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs +++ b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs @@ -23,13 +23,15 @@ public ResultsHttpClient( Uri baseUrl, HttpMessageHandler pipeline, string token, - bool disposeHandler) + bool disposeHandler, + bool useSdk) : base(baseUrl, pipeline, disposeHandler) { m_token = token; m_resultsServiceUrl = baseUrl; m_formatter = new JsonMediaTypeFormatter(); m_changeIdCounter = 1; + m_useSdk = useSdk; } // Get Sas URL calls @@ -209,7 +211,7 @@ private AppendBlobClient GetAppendBlobClient(string url) private async Task UploadBlockFileAsync(string url, string blobStorageType, FileStream file, CancellationToken cancellationToken) { - if (blobStorageType == BlobStorageTypes.AzureBlobStorage) + if (m_useSdk && blobStorageType == BlobStorageTypes.AzureBlobStorage) { var blobClient = GetBlobClient(url); try @@ -221,11 +223,32 @@ private async Task UploadBlockFileAsync(string url, string blobStorageType, File throw new Exception($"Failed to upload block to Azure blob: {e.Message}"); } } + else + { + // Upload the file to the url + var request = new HttpRequestMessage(HttpMethod.Put, url) + { + Content = new StreamContent(file) + }; + + if (blobStorageType == BlobStorageTypes.AzureBlobStorage) + { + request.Content.Headers.Add(Constants.AzureBlobTypeHeader, Constants.AzureBlockBlob); + } + + using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken)) + { + if (!response.IsSuccessStatusCode) + { + throw new Exception($"Failed to upload file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}"); + } + } + } } private async Task CreateAppendFileAsync(string url, string blobStorageType, CancellationToken cancellationToken) { - if (blobStorageType == BlobStorageTypes.AzureBlobStorage) + if (m_useSdk && blobStorageType == BlobStorageTypes.AzureBlobStorage) { var appendBlobClient = GetAppendBlobClient(url); try @@ -237,11 +260,31 @@ private async Task CreateAppendFileAsync(string url, string blobStorageType, Can throw new Exception($"Failed to create append blob in Azure blob: {e.Message}"); } } + else + { + var request = new HttpRequestMessage(HttpMethod.Put, url) + { + Content = new StringContent("") + }; + if (blobStorageType == BlobStorageTypes.AzureBlobStorage) + { + request.Content.Headers.Add(Constants.AzureBlobTypeHeader, Constants.AzureAppendBlob); + request.Content.Headers.Add("Content-Length", "0"); + } + + using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken)) + { + if (!response.IsSuccessStatusCode) + { + throw new Exception($"Failed to create append file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}"); + } + } + } } - private async Task UploadAppendFileAsync(string url, string blobStorageType, FileStream file, bool finalize, CancellationToken cancellationToken) + private async Task UploadAppendFileAsync(string url, string blobStorageType, FileStream file, bool finalize, long fileSize, CancellationToken cancellationToken) { - if (blobStorageType == BlobStorageTypes.AzureBlobStorage) + if (m_useSdk && blobStorageType == BlobStorageTypes.AzureBlobStorage) { var appendBlobClient = GetAppendBlobClient(url); try @@ -257,6 +300,29 @@ private async Task UploadAppendFileAsync(string url, string blobStorageType, Fil throw new Exception($"Failed to upload append block in Azure blob: {e.Message}"); } } + else + { + var comp = finalize ? "&comp=appendblock&seal=true" : "&comp=appendblock"; + // Upload the file to the url + var request = new HttpRequestMessage(HttpMethod.Put, url + comp) + { + Content = new StreamContent(file) + }; + + if (blobStorageType == BlobStorageTypes.AzureBlobStorage) + { + request.Content.Headers.Add("Content-Length", fileSize.ToString()); + request.Content.Headers.Add(Constants.AzureBlobSealedHeader, finalize.ToString()); + } + + using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken)) + { + if (!response.IsSuccessStatusCode) + { + throw new Exception($"Failed to upload append file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}, object: {response}, fileSize: {fileSize}"); + } + } + } } // Handle file upload for step summary @@ -310,7 +376,7 @@ private async Task UploadLogFile(string file, bool finalize, bool firstBlock, st var fileSize = new FileInfo(file).Length; using (var fileStream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true)) { - await UploadAppendFileAsync(sasUrl, blobStorageType, fileStream, finalize, cancellationToken); + await UploadAppendFileAsync(sasUrl, blobStorageType, fileStream, finalize, fileSize, cancellationToken); } } } @@ -430,6 +496,7 @@ public async Task UpdateWorkflowStepsAsync(Guid planId, IEnumerable Date: Tue, 2 Jan 2024 12:56:27 -0500 Subject: [PATCH 08/19] fix typo --- src/Runner.Common/JobServerQueue.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Runner.Common/JobServerQueue.cs b/src/Runner.Common/JobServerQueue.cs index 6e25f40bf78..7ef54f418ca 100644 --- a/src/Runner.Common/JobServerQueue.cs +++ b/src/Runner.Common/JobServerQueue.cs @@ -135,7 +135,7 @@ public void Start(Pipelines.AgentJobRequestMessage jobRequest, bool resultsServi liveConsoleFeedUrl = feedStreamUrl; } jobRequest.Variables.TryGetValue("system.github.results_upload_with_sdk", out VariableValue resultsUseSdkVariable); - _resultsServer.InitializeResultsClient(new Uri(resultsReceiverEndpoint), liveConsoleFeedUrl, accessToken, Convert.ToBoolean(resultsEndpointVariable?.Value)); + _resultsServer.InitializeResultsClient(new Uri(resultsReceiverEndpoint), liveConsoleFeedUrl, accessToken, Convert.ToBoolean(resultsUseSdkVariable?.Value)); _resultsClientInitiated = true; } From 09503c559b81fd46721d3eaf8b1b26d3d1666709 Mon Sep 17 00:00:00 2001 From: adjn <104127038+adjn@users.noreply.github.com> Date: Tue, 2 Jan 2024 08:09:10 -0800 Subject: [PATCH 09/19] Update envlinux.md (#3040) Matching supported distros to https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/about-self-hosted-runners --- docs/start/envlinux.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/start/envlinux.md b/docs/start/envlinux.md index 5fddae96f17..11eff187693 100644 --- a/docs/start/envlinux.md +++ b/docs/start/envlinux.md @@ -5,9 +5,9 @@ ## Supported Distributions and Versions x64 - - Red Hat Enterprise Linux 7 - - CentOS 7 - - Oracle Linux 7 + - Red Hat Enterprise Linux 7+ + - CentOS 7+ + - Oracle Linux 7+ - Fedora 29+ - Debian 9+ - Ubuntu 16.04+ From 037cf994d5f8337677f0f0b9196d6a0d8d86cc53 Mon Sep 17 00:00:00 2001 From: Yang Cao Date: Tue, 2 Jan 2024 12:52:14 -0500 Subject: [PATCH 10/19] Adding new dlls and Fixing hashes --- src/Misc/contentHash/dotnetRuntime/linux-arm | 2 +- src/Misc/contentHash/dotnetRuntime/linux-arm64 | 2 +- src/Misc/contentHash/dotnetRuntime/linux-x64 | 2 +- src/Misc/contentHash/dotnetRuntime/osx-arm64 | 2 +- src/Misc/contentHash/dotnetRuntime/osx-x64 | 2 +- src/Misc/contentHash/dotnetRuntime/win-arm64 | 2 +- src/Misc/contentHash/dotnetRuntime/win-x64 | 2 +- src/Misc/runnerdotnetruntimeassets | 6 ++++++ 8 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/Misc/contentHash/dotnetRuntime/linux-arm b/src/Misc/contentHash/dotnetRuntime/linux-arm index fb961f28f0b..53b5c955283 100644 --- a/src/Misc/contentHash/dotnetRuntime/linux-arm +++ b/src/Misc/contentHash/dotnetRuntime/linux-arm @@ -1 +1 @@ -a1761f8cbd803571e37b5ce4b0511c33e2ffd0dbe8519f0cbc80503a6d9e356c +a1761f8cbd803571e37b5ce4b0511c33e2ffd0dbe8519f0cbc80503a6d9e356c \ No newline at end of file diff --git a/src/Misc/contentHash/dotnetRuntime/linux-arm64 b/src/Misc/contentHash/dotnetRuntime/linux-arm64 index b72052c44bd..d97719ae03c 100644 --- a/src/Misc/contentHash/dotnetRuntime/linux-arm64 +++ b/src/Misc/contentHash/dotnetRuntime/linux-arm64 @@ -1 +1 @@ -0cbdca75fa9afb0f7c726eff95cc6d663348e8cc76536affe07df92251a61483 +0cbdca75fa9afb0f7c726eff95cc6d663348e8cc76536affe07df92251a61483 \ No newline at end of file diff --git a/src/Misc/contentHash/dotnetRuntime/linux-x64 b/src/Misc/contentHash/dotnetRuntime/linux-x64 index b2f1fc1a743..c804397f529 100644 --- a/src/Misc/contentHash/dotnetRuntime/linux-x64 +++ b/src/Misc/contentHash/dotnetRuntime/linux-x64 @@ -1 +1 @@ -8ca75c76e15ab9dc7fe49a66c5c74e171e7fabd5d26546fda8931bd11bff30f9 \ No newline at end of file +2b646b8c212b5a13655c587fe7967a85e899b8be334b67047c68f68ba5721458 \ No newline at end of file diff --git a/src/Misc/contentHash/dotnetRuntime/osx-arm64 b/src/Misc/contentHash/dotnetRuntime/osx-arm64 index 783fa8b5599..14fa6b8c307 100644 --- a/src/Misc/contentHash/dotnetRuntime/osx-arm64 +++ b/src/Misc/contentHash/dotnetRuntime/osx-arm64 @@ -1 +1 @@ -70496eb1c99b39b3373b5088c95a35ebbaac1098e6c47c8aab94771f3ffbf501 \ No newline at end of file +b4e13f78dedd7cdb9f42517a6290e2012f3fcc1f802bb4e95552e62d5ad12bda \ No newline at end of file diff --git a/src/Misc/contentHash/dotnetRuntime/osx-x64 b/src/Misc/contentHash/dotnetRuntime/osx-x64 index f593273294e..686ff731539 100644 --- a/src/Misc/contentHash/dotnetRuntime/osx-x64 +++ b/src/Misc/contentHash/dotnetRuntime/osx-x64 @@ -1 +1 @@ -4f8d48727d535daabcaec814e0dafb271c10625366c78e7e022ca7477a73023f \ No newline at end of file +52a5e9e6f9806b456d60c813454363024ff3c9f67b460fac44e46f196e59136e \ No newline at end of file diff --git a/src/Misc/contentHash/dotnetRuntime/win-arm64 b/src/Misc/contentHash/dotnetRuntime/win-arm64 index d050cb89ef2..75c029e1bcd 100644 --- a/src/Misc/contentHash/dotnetRuntime/win-arm64 +++ b/src/Misc/contentHash/dotnetRuntime/win-arm64 @@ -1 +1 @@ -d54d7428f2b9200a0030365a6a4e174e30a1b29b922f8254dffb2924bd09549d \ No newline at end of file +72f4d94edbd7cc0023a41b1a643ac21d5b6f3ae746338a6de4b80e4fb5d22f36 \ No newline at end of file diff --git a/src/Misc/contentHash/dotnetRuntime/win-x64 b/src/Misc/contentHash/dotnetRuntime/win-x64 index 881293ccbd4..2674f4c3a2e 100644 --- a/src/Misc/contentHash/dotnetRuntime/win-x64 +++ b/src/Misc/contentHash/dotnetRuntime/win-x64 @@ -1 +1 @@ -eaa939c45307f46b7003902255b3a2a09287215d710984107667e03ac493eb26 \ No newline at end of file +595ffe8f503d44daaa726e03b7f85cee62f7ea66637c125b577727af49e83bb4 \ No newline at end of file diff --git a/src/Misc/runnerdotnetruntimeassets b/src/Misc/runnerdotnetruntimeassets index 3d9d1ea0a57..016cbcae41a 100644 --- a/src/Misc/runnerdotnetruntimeassets +++ b/src/Misc/runnerdotnetruntimeassets @@ -64,6 +64,10 @@ libmscordaccore.dylib libmscordaccore.so libmscordbi.dylib libmscordbi.so +Azure.Core.dll +Azure.Storage.Blobs.dll +Azure.Storage.Common.dll +Microsoft.Bcl.AsyncInterfaces.dll Microsoft.CSharp.dll Microsoft.DiaSymReader.Native.amd64.dll Microsoft.DiaSymReader.Native.arm64.dll @@ -138,6 +142,7 @@ System.IO.FileSystem.dll System.IO.FileSystem.DriveInfo.dll System.IO.FileSystem.Primitives.dll System.IO.FileSystem.Watcher.dll +System.IO.Hashing.dll System.IO.IsolatedStorage.dll System.IO.MemoryMappedFiles.dll System.IO.Pipes.AccessControl.dll @@ -148,6 +153,7 @@ System.Linq.Expressions.dll System.Linq.Parallel.dll System.Linq.Queryable.dll System.Memory.dll +System.Memory.Data.dll System.Native.a System.Native.dylib System.Native.so From 2b3967d8aa6ca4646b8443499815cb1a78ae63bf Mon Sep 17 00:00:00 2001 From: Yang Cao Date: Tue, 2 Jan 2024 14:23:29 -0500 Subject: [PATCH 11/19] Add extra dlls to runnercoresassets --- src/Misc/runnercoreassets | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Misc/runnercoreassets b/src/Misc/runnercoreassets index 34f873d5b63..46354a5d987 100644 --- a/src/Misc/runnercoreassets +++ b/src/Misc/runnercoreassets @@ -6,6 +6,10 @@ darwin.svc.sh.template hashFiles/index.js installdependencies.sh macos-run-invoker.js +Azure.Core.dll +Azure.Storage.Blobs.dll +Azure.Storage.Common.dll +Microsoft.Bcl.AsyncInterfaces.dll Microsoft.IdentityModel.Logging.dll Microsoft.IdentityModel.Tokens.dll Minimatch.dll @@ -46,7 +50,10 @@ runsvc.sh Sdk.deps.json Sdk.dll Sdk.pdb +System.Diagnostics.DiagnosticSource.dll System.IdentityModel.Tokens.Jwt.dll +System.IO.Hashing.dll +System.Memory.Data.dll System.Net.Http.Formatting.dll System.Security.Cryptography.Pkcs.dll System.Security.Cryptography.ProtectedData.dll @@ -54,4 +61,4 @@ System.ServiceProcess.ServiceController.dll systemd.svc.sh.template update.cmd.template update.sh.template -YamlDotNet.dll \ No newline at end of file +YamlDotNet.dll From 8609d3c93d40201a6aef04c76f4902bbbdd4e034 Mon Sep 17 00:00:00 2001 From: Yang Cao Date: Tue, 2 Jan 2024 14:32:53 -0500 Subject: [PATCH 12/19] fix hashes on linux-arm and liinux-arm64 after update core assets file --- src/Misc/contentHash/dotnetRuntime/linux-arm | 2 +- src/Misc/contentHash/dotnetRuntime/linux-arm64 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Misc/contentHash/dotnetRuntime/linux-arm b/src/Misc/contentHash/dotnetRuntime/linux-arm index 53b5c955283..9f55d62ef2a 100644 --- a/src/Misc/contentHash/dotnetRuntime/linux-arm +++ b/src/Misc/contentHash/dotnetRuntime/linux-arm @@ -1 +1 @@ -a1761f8cbd803571e37b5ce4b0511c33e2ffd0dbe8519f0cbc80503a6d9e356c \ No newline at end of file +54d95a44d118dba852395991224a6b9c1abe916858c87138656f80c619e85331 \ No newline at end of file diff --git a/src/Misc/contentHash/dotnetRuntime/linux-arm64 b/src/Misc/contentHash/dotnetRuntime/linux-arm64 index d97719ae03c..c03c98ade6c 100644 --- a/src/Misc/contentHash/dotnetRuntime/linux-arm64 +++ b/src/Misc/contentHash/dotnetRuntime/linux-arm64 @@ -1 +1 @@ -0cbdca75fa9afb0f7c726eff95cc6d663348e8cc76536affe07df92251a61483 \ No newline at end of file +68015af17f06a824fa478e62ae7393766ce627fd5599ab916432a14656a19a52 \ No newline at end of file From 177dc8858f7f779697d942f4a1149cfaea6d8e25 Mon Sep 17 00:00:00 2001 From: Yang Cao Date: Tue, 2 Jan 2024 14:41:39 -0500 Subject: [PATCH 13/19] fix hashes for linux-x64 --- src/Misc/contentHash/dotnetRuntime/linux-x64 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Misc/contentHash/dotnetRuntime/linux-x64 b/src/Misc/contentHash/dotnetRuntime/linux-x64 index c804397f529..95a7155f74d 100644 --- a/src/Misc/contentHash/dotnetRuntime/linux-x64 +++ b/src/Misc/contentHash/dotnetRuntime/linux-x64 @@ -1 +1 @@ -2b646b8c212b5a13655c587fe7967a85e899b8be334b67047c68f68ba5721458 \ No newline at end of file +a2628119ca419cb54e279103ffae7986cdbd0814d57c73ff0dc74c38be08b9ae \ No newline at end of file From 358880a4b89196b12f7be93d1af810eed76d025d Mon Sep 17 00:00:00 2001 From: Yang Cao Date: Tue, 2 Jan 2024 14:47:56 -0500 Subject: [PATCH 14/19] fix hashes for win-x64 and osx-x64 --- src/Misc/contentHash/dotnetRuntime/osx-x64 | 2 +- src/Misc/contentHash/dotnetRuntime/win-x64 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Misc/contentHash/dotnetRuntime/osx-x64 b/src/Misc/contentHash/dotnetRuntime/osx-x64 index 686ff731539..085b329b2a0 100644 --- a/src/Misc/contentHash/dotnetRuntime/osx-x64 +++ b/src/Misc/contentHash/dotnetRuntime/osx-x64 @@ -1 +1 @@ -52a5e9e6f9806b456d60c813454363024ff3c9f67b460fac44e46f196e59136e \ No newline at end of file +d009e05e6b26d614d65be736a15d1bd151932121c16a9ff1b986deadecc982b9 \ No newline at end of file diff --git a/src/Misc/contentHash/dotnetRuntime/win-x64 b/src/Misc/contentHash/dotnetRuntime/win-x64 index 2674f4c3a2e..6be8253b146 100644 --- a/src/Misc/contentHash/dotnetRuntime/win-x64 +++ b/src/Misc/contentHash/dotnetRuntime/win-x64 @@ -1 +1 @@ -595ffe8f503d44daaa726e03b7f85cee62f7ea66637c125b577727af49e83bb4 \ No newline at end of file +a35b5722375490e9473cdcccb5e18b41eba3dbf4344fe31abc9821e21f18ea5a \ No newline at end of file From 0928715ed5ab10d8a9a8ae65f18b0f28d25ee769 Mon Sep 17 00:00:00 2001 From: Yang Cao Date: Tue, 2 Jan 2024 14:54:08 -0500 Subject: [PATCH 15/19] fix hashes for win-arm64 and osx-arm64 --- src/Misc/contentHash/dotnetRuntime/osx-arm64 | 2 +- src/Misc/contentHash/dotnetRuntime/win-arm64 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Misc/contentHash/dotnetRuntime/osx-arm64 b/src/Misc/contentHash/dotnetRuntime/osx-arm64 index 14fa6b8c307..d99ff5942f0 100644 --- a/src/Misc/contentHash/dotnetRuntime/osx-arm64 +++ b/src/Misc/contentHash/dotnetRuntime/osx-arm64 @@ -1 +1 @@ -b4e13f78dedd7cdb9f42517a6290e2012f3fcc1f802bb4e95552e62d5ad12bda \ No newline at end of file +de71ca09ead807e1a2ce9df0a5b23eb7690cb71fff51169a77e4c3992be53dda \ No newline at end of file diff --git a/src/Misc/contentHash/dotnetRuntime/win-arm64 b/src/Misc/contentHash/dotnetRuntime/win-arm64 index 75c029e1bcd..5c84f556e8d 100644 --- a/src/Misc/contentHash/dotnetRuntime/win-arm64 +++ b/src/Misc/contentHash/dotnetRuntime/win-arm64 @@ -1 +1 @@ -72f4d94edbd7cc0023a41b1a643ac21d5b6f3ae746338a6de4b80e4fb5d22f36 \ No newline at end of file +f730db39c2305800b4653795360ba9c10c68f384a46b85d808f1f9f0ed3c42e4 \ No newline at end of file From 8816eb810c1e0435a1a21af382844090ab6b850b Mon Sep 17 00:00:00 2001 From: Yang Cao Date: Tue, 2 Jan 2024 15:02:56 -0500 Subject: [PATCH 16/19] core assets should not overlap with dotnet runtime assets --- src/Misc/runnerdotnetruntimeassets | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/Misc/runnerdotnetruntimeassets b/src/Misc/runnerdotnetruntimeassets index 016cbcae41a..3d9d1ea0a57 100644 --- a/src/Misc/runnerdotnetruntimeassets +++ b/src/Misc/runnerdotnetruntimeassets @@ -64,10 +64,6 @@ libmscordaccore.dylib libmscordaccore.so libmscordbi.dylib libmscordbi.so -Azure.Core.dll -Azure.Storage.Blobs.dll -Azure.Storage.Common.dll -Microsoft.Bcl.AsyncInterfaces.dll Microsoft.CSharp.dll Microsoft.DiaSymReader.Native.amd64.dll Microsoft.DiaSymReader.Native.arm64.dll @@ -142,7 +138,6 @@ System.IO.FileSystem.dll System.IO.FileSystem.DriveInfo.dll System.IO.FileSystem.Primitives.dll System.IO.FileSystem.Watcher.dll -System.IO.Hashing.dll System.IO.IsolatedStorage.dll System.IO.MemoryMappedFiles.dll System.IO.Pipes.AccessControl.dll @@ -153,7 +148,6 @@ System.Linq.Expressions.dll System.Linq.Parallel.dll System.Linq.Queryable.dll System.Memory.dll -System.Memory.Data.dll System.Native.a System.Native.dylib System.Native.so From f1514c3f8a79fd0f407649b485dcf325a31fc914 Mon Sep 17 00:00:00 2001 From: Yang Cao Date: Wed, 3 Jan 2024 13:10:50 -0500 Subject: [PATCH 17/19] Remove duplicate System.Diagnostics.DiagnosticSource.dll from dotnet runtime assets file --- src/Misc/runnerdotnetruntimeassets | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Misc/runnerdotnetruntimeassets b/src/Misc/runnerdotnetruntimeassets index 3d9d1ea0a57..32d947339ce 100644 --- a/src/Misc/runnerdotnetruntimeassets +++ b/src/Misc/runnerdotnetruntimeassets @@ -106,7 +106,6 @@ System.Data.DataSetExtensions.dll System.Data.dll System.Diagnostics.Contracts.dll System.Diagnostics.Debug.dll -System.Diagnostics.DiagnosticSource.dll System.Diagnostics.FileVersionInfo.dll System.Diagnostics.Process.dll System.Diagnostics.StackTrace.dll From 05f16f15efc24e5f52fa93d626478a07116fa956 Mon Sep 17 00:00:00 2001 From: Yang Cao Date: Mon, 8 Jan 2024 14:00:55 -0500 Subject: [PATCH 18/19] Update src/Runner.Common/JobServerQueue.cs Co-authored-by: Tingluo Huang --- src/Runner.Common/JobServerQueue.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Runner.Common/JobServerQueue.cs b/src/Runner.Common/JobServerQueue.cs index 7ef54f418ca..04450e01161 100644 --- a/src/Runner.Common/JobServerQueue.cs +++ b/src/Runner.Common/JobServerQueue.cs @@ -135,7 +135,7 @@ public void Start(Pipelines.AgentJobRequestMessage jobRequest, bool resultsServi liveConsoleFeedUrl = feedStreamUrl; } jobRequest.Variables.TryGetValue("system.github.results_upload_with_sdk", out VariableValue resultsUseSdkVariable); - _resultsServer.InitializeResultsClient(new Uri(resultsReceiverEndpoint), liveConsoleFeedUrl, accessToken, Convert.ToBoolean(resultsUseSdkVariable?.Value)); + _resultsServer.InitializeResultsClient(new Uri(resultsReceiverEndpoint), liveConsoleFeedUrl, accessToken, StringUtil.ConvertToBoolean(resultsUseSdkVariable?.Value)); _resultsClientInitiated = true; } From 7a680e24deb6494257b60d5b49fd90530b7e594c Mon Sep 17 00:00:00 2001 From: Yang Cao Date: Mon, 8 Jan 2024 15:55:10 -0500 Subject: [PATCH 19/19] Use uriBuilder to avoid manually constructing the url --- src/Sdk/WebApi/WebApi/ResultsHttpClient.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs index 73282d26258..9a7eb990b4e 100644 --- a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs +++ b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs @@ -171,10 +171,10 @@ private async Task JobLogUploadCompleteAsync(string planId, string jobId, long l throw new Exception($"SAS url is empty"); } - var blobUri = new Uri(url); - var blobPath = $"{blobUri.Scheme}://{blobUri.Authority}/{blobUri.AbsolutePath}"; + var blobUri = new UriBuilder(url); var sasUrl = blobUri.Query.Substring(1); //remove starting "?" - return (new Uri(blobPath), sasUrl); + blobUri.Query = null; // remove query params + return (blobUri.Uri, sasUrl); } private BlobClient GetBlobClient(string url)