diff --git a/PenguinLogisticsOnlineNetworkDistributionSystem/src/Plonds.Core/Publishing/PlondsS3Client.cs b/PenguinLogisticsOnlineNetworkDistributionSystem/src/Plonds.Core/Publishing/PlondsS3Client.cs index 2eec6ee..e65f039 100644 --- a/PenguinLogisticsOnlineNetworkDistributionSystem/src/Plonds.Core/Publishing/PlondsS3Client.cs +++ b/PenguinLogisticsOnlineNetworkDistributionSystem/src/Plonds.Core/Publishing/PlondsS3Client.cs @@ -27,11 +27,16 @@ public sealed class PlondsS3Client : IDisposable AccessKey = Require(options.AccessKey, nameof(options.AccessKey)), SecretKey = Require(options.SecretKey, nameof(options.SecretKey)), PublicBaseUrl = Require(options.PublicBaseUrl, nameof(options.PublicBaseUrl)).TrimEnd('/'), - PublicBaseKeyPrefix = NormalizeOptionalKeyPrefix(options.PublicBaseKeyPrefix) + PublicBaseKeyPrefix = NormalizeOptionalKeyPrefix(options.PublicBaseKeyPrefix), + RequestTimeout = options.RequestTimeout <= TimeSpan.Zero ? TimeSpan.FromMinutes(30) : options.RequestTimeout, + MaxUploadAttempts = Math.Max(1, options.MaxUploadAttempts) }; - this.httpClient = httpClient ?? new HttpClient(); ownsHttpClient = httpClient is null; + this.httpClient = httpClient ?? new HttpClient + { + Timeout = this.options.RequestTimeout + }; } public async Task UploadFileAsync(PlondsS3ObjectUpload upload, CancellationToken cancellationToken = default) @@ -47,20 +52,47 @@ public sealed class PlondsS3Client : IDisposable var key = NormalizeKey(upload.Key); var payloadHash = PayloadUtilities.ComputeSha256(sourcePath); var contentLength = new FileInfo(sourcePath).Length; + + for (var attempt = 1; attempt <= options.MaxUploadAttempts; attempt++) + { + try + { + await UploadFileOnceAsync(sourcePath, key, upload.ContentType, payloadHash, contentLength, attempt, cancellationToken).ConfigureAwait(false); + return; + } + catch (Exception ex) when (attempt < options.MaxUploadAttempts && IsRetriable(ex)) + { + var delay = TimeSpan.FromSeconds(Math.Min(30, Math.Pow(2, attempt))); + Console.Error.WriteLine($"S3 upload retry {attempt + 1}/{options.MaxUploadAttempts} for {key} after {delay.TotalSeconds:0}s: {ex.Message}"); + await Task.Delay(delay, cancellationToken).ConfigureAwait(false); + } + } + } + + private async Task UploadFileOnceAsync( + string sourcePath, + string key, + string? contentType, + string payloadHash, + long contentLength, + int attempt, + CancellationToken cancellationToken = default) + { var now = DateTimeOffset.UtcNow; var requestUri = BuildObjectUri(key); + Console.WriteLine($"Uploading S3 object {key} ({FormatBytes(contentLength)}), attempt {attempt}/{options.MaxUploadAttempts}."); - using var content = new StreamContent(File.OpenRead(sourcePath)); - content.Headers.ContentType = new MediaTypeHeaderValue(string.IsNullOrWhiteSpace(upload.ContentType) + await using var fileStream = File.OpenRead(sourcePath); + using var content = new StreamContent(fileStream); + content.Headers.ContentType = new MediaTypeHeaderValue(string.IsNullOrWhiteSpace(contentType) ? "application/octet-stream" - : upload.ContentType); + : contentType); content.Headers.ContentLength = contentLength; using var request = new HttpRequestMessage(HttpMethod.Put, requestUri) { Content = content }; - SignRequest(request, key, payloadHash, now); using var response = await httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false); @@ -69,6 +101,8 @@ public sealed class PlondsS3Client : IDisposable var body = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); throw new InvalidOperationException($"S3 upload failed for {key}: HTTP {(int)response.StatusCode} {response.ReasonPhrase}. {Truncate(body, 512)}"); } + + Console.WriteLine($"Uploaded S3 object {key}."); } public async Task EnsureObjectExistsAsync(string key, CancellationToken cancellationToken = default) @@ -257,4 +291,28 @@ public sealed class PlondsS3Client : IDisposable return value[..maxLength]; } + + private static bool IsRetriable(Exception exception) + { + if (exception is TaskCanceledException or TimeoutException or HttpRequestException) + { + return true; + } + + return exception.InnerException is not null && IsRetriable(exception.InnerException); + } + + private static string FormatBytes(long bytes) + { + string[] units = ["B", "KB", "MB", "GB"]; + double value = bytes; + var unit = 0; + while (value >= 1024 && unit < units.Length - 1) + { + value /= 1024; + unit++; + } + + return $"{value:0.##} {units[unit]}"; + } } diff --git a/PenguinLogisticsOnlineNetworkDistributionSystem/src/Plonds.Core/Publishing/PlondsS3ClientOptions.cs b/PenguinLogisticsOnlineNetworkDistributionSystem/src/Plonds.Core/Publishing/PlondsS3ClientOptions.cs index 7d1ee4a..be2ed14 100644 --- a/PenguinLogisticsOnlineNetworkDistributionSystem/src/Plonds.Core/Publishing/PlondsS3ClientOptions.cs +++ b/PenguinLogisticsOnlineNetworkDistributionSystem/src/Plonds.Core/Publishing/PlondsS3ClientOptions.cs @@ -7,4 +7,9 @@ public sealed record PlondsS3ClientOptions( string AccessKey, string SecretKey, string PublicBaseUrl, - string PublicBaseKeyPrefix = ""); + string PublicBaseKeyPrefix = "") +{ + public TimeSpan RequestTimeout { get; init; } = TimeSpan.FromMinutes(30); + + public int MaxUploadAttempts { get; init; } = 3; +}