From dc735730646c350cecb198ceb386775c0d5fced5 Mon Sep 17 00:00:00 2001 From: ahmedraza118 Date: Mon, 16 Mar 2026 12:27:38 +0200 Subject: [PATCH 1/4] fix: s3 route issue --- src/@types/commands.ts | 5 +++-- src/@types/fileObject.ts | 1 + src/components/httpRoutes/fileInfo.ts | 22 ++++++++++++++++++++-- 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/@types/commands.ts b/src/@types/commands.ts index cf2069824..290cc29f5 100644 --- a/src/@types/commands.ts +++ b/src/@types/commands.ts @@ -14,7 +14,8 @@ import { FtpFileObject, IpfsFileObject, UrlFileObject, - BaseFileObject + BaseFileObject, + S3FileObject } from './fileObject' export interface Command { @@ -69,7 +70,7 @@ export interface FileInfoCommand extends Command { did?: string serviceId?: string fileIndex?: number - file?: UrlFileObject | ArweaveFileObject | IpfsFileObject | FtpFileObject + file?: UrlFileObject | ArweaveFileObject | IpfsFileObject | FtpFileObject | S3FileObject checksum?: boolean } // group these 2 diff --git a/src/@types/fileObject.ts b/src/@types/fileObject.ts index c9526fd74..7a6f3a909 100644 --- a/src/@types/fileObject.ts +++ b/src/@types/fileObject.ts @@ -94,4 +94,5 @@ export interface FileInfoHttpRequest { url?: string transactionId?: string serviceId?: string + s3Access?: S3Object } diff --git a/src/components/httpRoutes/fileInfo.ts b/src/components/httpRoutes/fileInfo.ts index ee6df98d7..14faf611e 100644 --- a/src/components/httpRoutes/fileInfo.ts +++ b/src/components/httpRoutes/fileInfo.ts @@ -5,7 +5,8 @@ import { FileObjectType, FtpFileObject, IpfsFileObject, - UrlFileObject + UrlFileObject, + S3FileObject } from '../../@types/fileObject.js' import { PROTOCOL_COMMANDS, SERVICES_API_BASE_PATH } from '../../utils/constants.js' import { FileInfoHandler } from '../core/handler/fileInfoHandler.js' @@ -29,6 +30,7 @@ const validateFileInfoRequest = (req: FileInfoHttpRequest): boolean => { if (req.type === 'url' && !req.url) return false // 'url' is required if 'type' is 'url' if (req.type === 'ftp' && !req.url) return false // 'url' is required if 'type' is 'ftp' if (req.type === 'arweave' && !req.transactionId) return false // 'transactionId' is required if 'type' is 'arweave' + if (req.type === 's3' && !req.s3Access) return false // 's3Access' is required if 'type' is 's3' if (!req.type && !req.serviceId) return false // 'serviceId' is required if 'type' is not provided return true @@ -48,7 +50,12 @@ fileInfoRoute.post( try { // Retrieve the file info - let fileObject: UrlFileObject | IpfsFileObject | ArweaveFileObject | FtpFileObject + let fileObject: + | UrlFileObject + | IpfsFileObject + | ArweaveFileObject + | FtpFileObject + | S3FileObject let fileInfoTask: FileInfoCommand if (fileInfoReq.did && fileInfoReq.serviceId) { @@ -105,6 +112,17 @@ fileInfoRoute.post( type: FileObjectType.FTP, caller: req.caller } + } else if (fileInfoReq.type === 's3' && fileInfoReq.s3Access) { + fileObject = { + type: 's3', + s3Access: fileInfoReq.s3Access + } as S3FileObject + fileInfoTask = { + command: PROTOCOL_COMMANDS.FILE_INFO, + file: fileObject, + type: FileObjectType.S3, + caller: req.caller + } } const response = await new FileInfoHandler(req.oceanNode).handle(fileInfoTask) if (response.stream) { From a979e42c1099e269f940d45433ce674f123b8c68 Mon Sep 17 00:00:00 2001 From: ahmedraza118 Date: Mon, 16 Mar 2026 16:05:40 +0200 Subject: [PATCH 2/4] fix: for s3 compatibility --- src/components/storage/S3Storage.ts | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/components/storage/S3Storage.ts b/src/components/storage/S3Storage.ts index 52e8e4e0d..45b9e6ad0 100644 --- a/src/components/storage/S3Storage.ts +++ b/src/components/storage/S3Storage.ts @@ -12,19 +12,16 @@ import { CORE_LOGGER } from '../../utils/logging/common.js' import { Storage } from './Storage.js' function createS3Client(s3Access: S3FileObject['s3Access']): S3Client { - const endpoint = s3Access.endpoint.startsWith('http') - ? s3Access.endpoint - : `https://${s3Access.endpoint}` return new S3Client({ - endpoint, - // Region is optional; default to us-east-1 if not provided + endpoint: s3Access.endpoint, region: s3Access.region ?? 'us-east-1', - // Path-style (e.g. endpoint/bucket/key) required for some S3-compatible services (e.g. MinIO); default false for AWS virtual-host style forcePathStyle: s3Access.forcePathStyle ?? false, credentials: { accessKeyId: s3Access.accessKeyId, secretAccessKey: s3Access.secretAccessKey - } + }, + requestChecksumCalculation: 'WHEN_REQUIRED', + responseChecksumValidation: 'WHEN_REQUIRED' }) } From 72f7ad88740c7dd6596912ecd138794ae833f030 Mon Sep 17 00:00:00 2001 From: ahmedraza118 Date: Mon, 16 Mar 2026 16:27:14 +0200 Subject: [PATCH 3/4] push debug to test --- .../core/handler/fileInfoHandler.ts | 108 +++++++-- src/components/storage/S3Storage.ts | 217 +++++++++++++++--- 2 files changed, 274 insertions(+), 51 deletions(-) diff --git a/src/components/core/handler/fileInfoHandler.ts b/src/components/core/handler/fileInfoHandler.ts index c9579fea5..00f3bcffe 100644 --- a/src/components/core/handler/fileInfoHandler.ts +++ b/src/components/core/handler/fileInfoHandler.ts @@ -26,15 +26,37 @@ async function formatMetadata( name: string type: string }> { - const storage = Storage.getStorageClass(file, config) - const fileInfo = await storage.fetchSpecificFileMetadata(file, false) CORE_LOGGER.logMessage( - `Metadata for file: ${fileInfo.contentLength} ${fileInfo.contentType}` + `[formatMetadata] Formatting metadata for file type: ${file.type}`, + true ) - return fileInfo + + try { + const storage = Storage.getStorageClass(file, config) + CORE_LOGGER.logMessage( + `[formatMetadata] Got storage class for type: ${file.type}`, + true + ) + + const fileInfo = await storage.fetchSpecificFileMetadata(file, false) + CORE_LOGGER.logMessage( + `[formatMetadata] Metadata for file: ${fileInfo.contentLength} ${fileInfo.contentType}`, + true + ) + return fileInfo + } catch (error) { + CORE_LOGGER.error(`[formatMetadata] Error: ${error.message}`) + if (error instanceof Error) { + CORE_LOGGER.error(`[formatMetadata] Stack: ${error.stack}`) + } + throw error + } } + export class FileInfoHandler extends CommandHandler { validate(command: FileInfoCommand): ValidateParams { + CORE_LOGGER.logMessage(`[FileInfoHandler] Validating command`, true) + let validation = validateCommandParameters(command, []) // all optional? weird if (validation.valid) { if (command.did) { @@ -58,37 +80,88 @@ export class FileInfoHandler extends CommandHandler { } async handle(task: FileInfoCommand): Promise { + CORE_LOGGER.logMessage( + `[FileInfoHandler] Handling task: ${JSON.stringify(task, null, 2)}`, + true + ) + const validationResponse = await this.verifyParamsAndRateLimits(task) if (this.shouldDenyTaskHandling(validationResponse)) { return validationResponse } + try { const oceanNode = this.getOceanNode() const config = await getConfiguration() let fileInfo = [] if (task.file && task.type) { - const storage = Storage.getStorageClass(task.file, config) + CORE_LOGGER.logMessage( + `[FileInfoHandler] Processing file of type: ${task.type}`, + true + ) + CORE_LOGGER.logMessage( + `[FileInfoHandler] File object: ${JSON.stringify(task.file, null, 2)}`, + true + ) - fileInfo = await storage.getFileInfo({ - type: task.type, - fileIndex: task.fileIndex - }) + try { + const storage = Storage.getStorageClass(task.file, config) + CORE_LOGGER.logMessage( + `[FileInfoHandler] Storage class created successfully`, + true + ) + + fileInfo = await storage.getFileInfo({ + type: task.type, + fileIndex: task.fileIndex + }) + CORE_LOGGER.logMessage( + `[FileInfoHandler] getFileInfo returned: ${JSON.stringify(fileInfo, null, 2)}`, + true + ) + } catch (storageError) { + CORE_LOGGER.error(`[FileInfoHandler] Storage error: ${storageError.message}`) + if (storageError instanceof Error) { + CORE_LOGGER.error( + `[FileInfoHandler] Storage error stack: ${storageError.stack}` + ) + } + throw storageError + } } else if (task.did && task.serviceId) { + CORE_LOGGER.logMessage( + `[FileInfoHandler] Processing DID: ${task.did}, serviceId: ${task.serviceId}`, + true + ) + const fileArray = await getFile(task.did, task.serviceId, oceanNode) + CORE_LOGGER.logMessage( + `[FileInfoHandler] Got fileArray with ${fileArray.length} files`, + true + ) + if (task.fileIndex) { + CORE_LOGGER.logMessage( + `[FileInfoHandler] Getting metadata for file index: ${task.fileIndex}`, + true + ) const fileMetadata = await formatMetadata(fileArray[task.fileIndex], config) fileInfo.push(fileMetadata) } else { - for (const file of fileArray) { - const fileMetadata = await formatMetadata(file, config) + for (let i = 0; i < fileArray.length; i++) { + CORE_LOGGER.logMessage( + `[FileInfoHandler] Getting metadata for file index: ${i}`, + true + ) + const fileMetadata = await formatMetadata(fileArray[i], config) fileInfo.push(fileMetadata) } } } else { const errorMessage = 'Invalid arguments. Please provide either file && Type OR did && serviceId' - CORE_LOGGER.error(errorMessage) + CORE_LOGGER.error(`[FileInfoHandler] ${errorMessage}`) return { stream: null, status: { @@ -97,8 +170,9 @@ export class FileInfoHandler extends CommandHandler { } } } + CORE_LOGGER.logMessage( - 'File Info Response: ' + JSON.stringify(fileInfo, null, 2), + '[FileInfoHandler] File Info Response: ' + JSON.stringify(fileInfo, null, 2), true ) @@ -109,12 +183,16 @@ export class FileInfoHandler extends CommandHandler { } } } catch (error) { - CORE_LOGGER.error(error.message) + CORE_LOGGER.error(`[FileInfoHandler] Error: ${error.message}`) + if (error instanceof Error) { + CORE_LOGGER.error(`[FileInfoHandler] Error stack: ${error.stack}`) + CORE_LOGGER.error(`[FileInfoHandler] Error name: ${error.name}`) + } return { stream: null, status: { httpStatus: 500, - error: error.message + error: error.message || 'UnknownError' } } } diff --git a/src/components/storage/S3Storage.ts b/src/components/storage/S3Storage.ts index 45b9e6ad0..61916be38 100644 --- a/src/components/storage/S3Storage.ts +++ b/src/components/storage/S3Storage.ts @@ -12,6 +12,11 @@ import { CORE_LOGGER } from '../../utils/logging/common.js' import { Storage } from './Storage.js' function createS3Client(s3Access: S3FileObject['s3Access']): S3Client { + CORE_LOGGER.logMessage( + `[S3Storage] Creating S3 client with endpoint: ${s3Access.endpoint}, region: ${s3Access.region || 'us-east-1'}, forcePathStyle: ${s3Access.forcePathStyle}`, + true + ) + return new S3Client({ endpoint: s3Access.endpoint, region: s3Access.region ?? 'us-east-1', @@ -28,41 +33,78 @@ function createS3Client(s3Access: S3FileObject['s3Access']): S3Client { export class S3Storage extends Storage { public constructor(file: S3FileObject, config: OceanNodeConfig) { super(file, config, true) + CORE_LOGGER.logMessage( + `[S3Storage] Constructor called with file type: ${file.type}`, + true + ) const [isValid, message] = this.validate() if (isValid === false) { + CORE_LOGGER.error(`[S3Storage] Validation failed: ${message}`) throw new Error(`Error validating the S3 file: ${message}`) } + CORE_LOGGER.logMessage(`[S3Storage] Validation successful`, true) } validate(): [boolean, string] { const file: S3FileObject = this.getFile() as S3FileObject + CORE_LOGGER.logMessage(`[S3Storage] Validating S3 file...`, true) + if (!file.s3Access) { + CORE_LOGGER.error(`[S3Storage] Missing s3Access object`) return [false, 'Missing s3Access'] } - const { bucket, objectKey, endpoint, accessKeyId, secretAccessKey } = file.s3Access + + const { + bucket, + objectKey, + endpoint, + accessKeyId, + secretAccessKey, + region, + forcePathStyle + } = file.s3Access + + CORE_LOGGER.logMessage( + `[S3Storage] Validating fields - bucket: ${bucket}, objectKey: ${objectKey}, endpoint: ${endpoint}, region: ${region}, forcePathStyle: ${forcePathStyle}`, + true + ) + if (!bucket?.trim()) { + CORE_LOGGER.error(`[S3Storage] Missing bucket`) return [false, 'Missing bucket'] } if (!objectKey?.trim()) { + CORE_LOGGER.error(`[S3Storage] Missing objectKey`) return [false, 'Missing objectKey'] } if (!endpoint?.trim()) { + CORE_LOGGER.error(`[S3Storage] Missing endpoint`) return [false, 'Missing endpoint'] } if (!accessKeyId?.trim()) { + CORE_LOGGER.error(`[S3Storage] Missing accessKeyId`) return [false, 'Missing accessKeyId'] } if (!secretAccessKey?.trim()) { + CORE_LOGGER.error(`[S3Storage] Missing secretAccessKey`) return [false, 'Missing secretAccessKey'] } + + CORE_LOGGER.logMessage(`[S3Storage] All fields validated successfully`, true) return [true, ''] } override async getReadableStream(): Promise { const { s3Access } = this.getFile() as S3FileObject + CORE_LOGGER.logMessage( + `[S3Storage] Getting readable stream for bucket: ${s3Access.bucket}, key: ${s3Access.objectKey}`, + true + ) + const s3Client = createS3Client(s3Access) try { + CORE_LOGGER.logMessage(`[S3Storage] Sending GetObjectCommand...`, true) const response = await s3Client.send( new GetObjectCommand({ Bucket: s3Access.bucket, @@ -70,7 +112,13 @@ export class S3Storage extends Storage { }) ) + CORE_LOGGER.logMessage( + `[S3Storage] GetObjectCommand successful, statusCode: ${response.$metadata.httpStatusCode}`, + true + ) + if (!response.Body) { + CORE_LOGGER.error(`[S3Storage] GetObject returned no body`) throw new Error('S3 GetObject returned no body') } @@ -82,7 +130,23 @@ export class S3Storage extends Storage { : undefined } } catch (err) { - CORE_LOGGER.error(`Error fetching object from S3: ${err}`) + CORE_LOGGER.error(`[S3Storage] Error fetching object from S3: ${err}`) + if (err instanceof Error) { + CORE_LOGGER.error( + `[S3Storage] Error name: ${err.name}, message: ${err.message}, stack: ${err.stack}` + ) + + // Log specific AWS error details + if ('$metadata' in err) { + const awsError = err as any + CORE_LOGGER.error( + `[S3Storage] AWS Error metadata: ${JSON.stringify(awsError.$metadata)}` + ) + CORE_LOGGER.error( + `[S3Storage] AWS Error code: ${awsError.Code}, requestId: ${awsError.$metadata?.requestId}` + ) + } + } throw err } } @@ -96,26 +160,50 @@ export class S3Storage extends Storage { stream: Readable ): Promise<{ httpStatus: number; headers?: Record }> { const { s3Access } = this.getFile() as S3FileObject + CORE_LOGGER.logMessage( + `[S3Storage] Uploading file: ${filename} to bucket: ${s3Access.bucket}`, + true + ) + const s3Client = createS3Client(s3Access) let key = s3Access.objectKey if (key.endsWith('/')) { key = `${key.replace(/\/+$/, '')}/${filename}` + CORE_LOGGER.logMessage( + `[S3Storage] Key ends with '/', updated key to: ${key}`, + true + ) + } + + try { + const upload = new Upload({ + client: s3Client, + params: { + Bucket: s3Access.bucket, + Key: key, + Body: stream, + ContentType: 'application/octet-stream', + ContentDisposition: `attachment; filename="${filename.replace(/"/g, '\\"')}"` + }, + queueSize: 4, + partSize: 5 * 1024 * 1024, // 5MB minimum for S3 + leavePartsOnError: false + }) + + CORE_LOGGER.logMessage(`[S3Storage] Starting upload...`, true) + await upload.done() + CORE_LOGGER.logMessage(`[S3Storage] Upload completed successfully`, true) + + return { httpStatus: 200, headers: {} } + } catch (err) { + CORE_LOGGER.error(`[S3Storage] Upload error: ${err}`) + if (err instanceof Error) { + CORE_LOGGER.error( + `[S3Storage] Upload error name: ${err.name}, message: ${err.message}` + ) + } + throw err } - const upload = new Upload({ - client: s3Client, - params: { - Bucket: s3Access.bucket, - Key: key, - Body: stream, - ContentType: 'application/octet-stream', - ContentDisposition: `attachment; filename="${filename.replace(/"/g, '\\"')}"` - }, - queueSize: 4, - partSize: 5 * 1024 * 1024, // 5MB minimum for S3 - leavePartsOnError: false - }) - await upload.done() - return { httpStatus: 200, headers: {} } } async fetchSpecificFileMetadata( @@ -123,28 +211,85 @@ export class S3Storage extends Storage { _forceChecksum: boolean ): Promise { const { s3Access } = fileObject + CORE_LOGGER.logMessage( + `[S3Storage] Fetching metadata for bucket: ${s3Access.bucket}, key: ${s3Access.objectKey}`, + true + ) + const s3Client = createS3Client(s3Access) - const data = await s3Client.send( - new HeadObjectCommand({ - Bucket: s3Access.bucket, - Key: s3Access.objectKey - }) - ) + try { + CORE_LOGGER.logMessage(`[S3Storage] Sending HeadObjectCommand...`, true) + const data = await s3Client.send( + new HeadObjectCommand({ + Bucket: s3Access.bucket, + Key: s3Access.objectKey + }) + ) + + CORE_LOGGER.logMessage( + `[S3Storage] HeadObjectCommand successful, statusCode: ${data.$metadata.httpStatusCode}`, + true + ) + CORE_LOGGER.logMessage( + `[S3Storage] Metadata received - ContentLength: ${data.ContentLength}, ContentType: ${data.ContentType}, ETag: ${data.ETag}`, + true + ) + + const contentLength = data.ContentLength != null ? String(data.ContentLength) : '0' + const contentType = data.ContentType ?? 'application/octet-stream' + const name = s3Access.objectKey.split('/').pop() ?? s3Access.objectKey + + CORE_LOGGER.logMessage( + `[S3Storage] Returning file info - name: ${name}, type: ${contentType}, size: ${contentLength}`, + true + ) + + return { + valid: true, + contentLength, + contentType, + checksum: data.ETag?.replace(/"/g, ''), + name, + type: 's3', + encryptedBy: fileObject.encryptedBy, + encryptMethod: fileObject.encryptMethod + } + } catch (err) { + CORE_LOGGER.error(`[S3Storage] Error fetching metadata from S3: ${err}`) + if (err instanceof Error) { + CORE_LOGGER.error( + `[S3Storage] Error name: ${err.name}, message: ${err.message}, stack: ${err.stack}` + ) - const contentLength = data.ContentLength != null ? String(data.ContentLength) : '0' - const contentType = data.ContentType ?? 'application/octet-stream' - const name = s3Access.objectKey.split('/').pop() ?? s3Access.objectKey - - return { - valid: true, - contentLength, - contentType, - checksum: data.ETag?.replace(/"/g, ''), - name, - type: 's3', - encryptedBy: fileObject.encryptedBy, - encryptMethod: fileObject.encryptMethod + // Log specific AWS error details + if ('$metadata' in err) { + const awsError = err as any + CORE_LOGGER.error( + `[S3Storage] AWS Error metadata: ${JSON.stringify(awsError.$metadata)}` + ) + CORE_LOGGER.error(`[S3Storage] AWS Error code: ${awsError.Code}`) + + if (awsError.Code === 'NoSuchKey') { + CORE_LOGGER.error( + `[S3Storage] File not found: ${s3Access.bucket}/${s3Access.objectKey}` + ) + } else if (awsError.Code === 'AccessDenied') { + CORE_LOGGER.error( + `[S3Storage] Access denied - check credentials and bucket permissions` + ) + } else if (awsError.Code === 'NoSuchBucket') { + CORE_LOGGER.error(`[S3Storage] Bucket does not exist: ${s3Access.bucket}`) + } else if (awsError.Code === 'SignatureDoesNotMatch') { + CORE_LOGGER.error( + `[S3Storage] Signature does not match - check credentials and endpoint` + ) + } else if (awsError.Code === 'InvalidAccessKeyId') { + CORE_LOGGER.error(`[S3Storage] Invalid Access Key ID`) + } + } + } + throw err } } } From 48d77f8ffc35c365e758f60bc51a38beadd53fe2 Mon Sep 17 00:00:00 2001 From: ahmedraza118 Date: Mon, 16 Mar 2026 16:56:04 +0200 Subject: [PATCH 4/4] debug revert --- .../core/handler/fileInfoHandler.ts | 108 ++------- src/components/storage/S3Storage.ts | 228 ++++-------------- 2 files changed, 58 insertions(+), 278 deletions(-) diff --git a/src/components/core/handler/fileInfoHandler.ts b/src/components/core/handler/fileInfoHandler.ts index 00f3bcffe..c9579fea5 100644 --- a/src/components/core/handler/fileInfoHandler.ts +++ b/src/components/core/handler/fileInfoHandler.ts @@ -26,37 +26,15 @@ async function formatMetadata( name: string type: string }> { + const storage = Storage.getStorageClass(file, config) + const fileInfo = await storage.fetchSpecificFileMetadata(file, false) CORE_LOGGER.logMessage( - `[formatMetadata] Formatting metadata for file type: ${file.type}`, - true + `Metadata for file: ${fileInfo.contentLength} ${fileInfo.contentType}` ) - - try { - const storage = Storage.getStorageClass(file, config) - CORE_LOGGER.logMessage( - `[formatMetadata] Got storage class for type: ${file.type}`, - true - ) - - const fileInfo = await storage.fetchSpecificFileMetadata(file, false) - CORE_LOGGER.logMessage( - `[formatMetadata] Metadata for file: ${fileInfo.contentLength} ${fileInfo.contentType}`, - true - ) - return fileInfo - } catch (error) { - CORE_LOGGER.error(`[formatMetadata] Error: ${error.message}`) - if (error instanceof Error) { - CORE_LOGGER.error(`[formatMetadata] Stack: ${error.stack}`) - } - throw error - } + return fileInfo } - export class FileInfoHandler extends CommandHandler { validate(command: FileInfoCommand): ValidateParams { - CORE_LOGGER.logMessage(`[FileInfoHandler] Validating command`, true) - let validation = validateCommandParameters(command, []) // all optional? weird if (validation.valid) { if (command.did) { @@ -80,88 +58,37 @@ export class FileInfoHandler extends CommandHandler { } async handle(task: FileInfoCommand): Promise { - CORE_LOGGER.logMessage( - `[FileInfoHandler] Handling task: ${JSON.stringify(task, null, 2)}`, - true - ) - const validationResponse = await this.verifyParamsAndRateLimits(task) if (this.shouldDenyTaskHandling(validationResponse)) { return validationResponse } - try { const oceanNode = this.getOceanNode() const config = await getConfiguration() let fileInfo = [] if (task.file && task.type) { - CORE_LOGGER.logMessage( - `[FileInfoHandler] Processing file of type: ${task.type}`, - true - ) - CORE_LOGGER.logMessage( - `[FileInfoHandler] File object: ${JSON.stringify(task.file, null, 2)}`, - true - ) + const storage = Storage.getStorageClass(task.file, config) - try { - const storage = Storage.getStorageClass(task.file, config) - CORE_LOGGER.logMessage( - `[FileInfoHandler] Storage class created successfully`, - true - ) - - fileInfo = await storage.getFileInfo({ - type: task.type, - fileIndex: task.fileIndex - }) - CORE_LOGGER.logMessage( - `[FileInfoHandler] getFileInfo returned: ${JSON.stringify(fileInfo, null, 2)}`, - true - ) - } catch (storageError) { - CORE_LOGGER.error(`[FileInfoHandler] Storage error: ${storageError.message}`) - if (storageError instanceof Error) { - CORE_LOGGER.error( - `[FileInfoHandler] Storage error stack: ${storageError.stack}` - ) - } - throw storageError - } + fileInfo = await storage.getFileInfo({ + type: task.type, + fileIndex: task.fileIndex + }) } else if (task.did && task.serviceId) { - CORE_LOGGER.logMessage( - `[FileInfoHandler] Processing DID: ${task.did}, serviceId: ${task.serviceId}`, - true - ) - const fileArray = await getFile(task.did, task.serviceId, oceanNode) - CORE_LOGGER.logMessage( - `[FileInfoHandler] Got fileArray with ${fileArray.length} files`, - true - ) - if (task.fileIndex) { - CORE_LOGGER.logMessage( - `[FileInfoHandler] Getting metadata for file index: ${task.fileIndex}`, - true - ) const fileMetadata = await formatMetadata(fileArray[task.fileIndex], config) fileInfo.push(fileMetadata) } else { - for (let i = 0; i < fileArray.length; i++) { - CORE_LOGGER.logMessage( - `[FileInfoHandler] Getting metadata for file index: ${i}`, - true - ) - const fileMetadata = await formatMetadata(fileArray[i], config) + for (const file of fileArray) { + const fileMetadata = await formatMetadata(file, config) fileInfo.push(fileMetadata) } } } else { const errorMessage = 'Invalid arguments. Please provide either file && Type OR did && serviceId' - CORE_LOGGER.error(`[FileInfoHandler] ${errorMessage}`) + CORE_LOGGER.error(errorMessage) return { stream: null, status: { @@ -170,9 +97,8 @@ export class FileInfoHandler extends CommandHandler { } } } - CORE_LOGGER.logMessage( - '[FileInfoHandler] File Info Response: ' + JSON.stringify(fileInfo, null, 2), + 'File Info Response: ' + JSON.stringify(fileInfo, null, 2), true ) @@ -183,16 +109,12 @@ export class FileInfoHandler extends CommandHandler { } } } catch (error) { - CORE_LOGGER.error(`[FileInfoHandler] Error: ${error.message}`) - if (error instanceof Error) { - CORE_LOGGER.error(`[FileInfoHandler] Error stack: ${error.stack}`) - CORE_LOGGER.error(`[FileInfoHandler] Error name: ${error.name}`) - } + CORE_LOGGER.error(error.message) return { stream: null, status: { httpStatus: 500, - error: error.message || 'UnknownError' + error: error.message } } } diff --git a/src/components/storage/S3Storage.ts b/src/components/storage/S3Storage.ts index 61916be38..52e8e4e0d 100644 --- a/src/components/storage/S3Storage.ts +++ b/src/components/storage/S3Storage.ts @@ -12,99 +12,60 @@ import { CORE_LOGGER } from '../../utils/logging/common.js' import { Storage } from './Storage.js' function createS3Client(s3Access: S3FileObject['s3Access']): S3Client { - CORE_LOGGER.logMessage( - `[S3Storage] Creating S3 client with endpoint: ${s3Access.endpoint}, region: ${s3Access.region || 'us-east-1'}, forcePathStyle: ${s3Access.forcePathStyle}`, - true - ) - + const endpoint = s3Access.endpoint.startsWith('http') + ? s3Access.endpoint + : `https://${s3Access.endpoint}` return new S3Client({ - endpoint: s3Access.endpoint, + endpoint, + // Region is optional; default to us-east-1 if not provided region: s3Access.region ?? 'us-east-1', + // Path-style (e.g. endpoint/bucket/key) required for some S3-compatible services (e.g. MinIO); default false for AWS virtual-host style forcePathStyle: s3Access.forcePathStyle ?? false, credentials: { accessKeyId: s3Access.accessKeyId, secretAccessKey: s3Access.secretAccessKey - }, - requestChecksumCalculation: 'WHEN_REQUIRED', - responseChecksumValidation: 'WHEN_REQUIRED' + } }) } export class S3Storage extends Storage { public constructor(file: S3FileObject, config: OceanNodeConfig) { super(file, config, true) - CORE_LOGGER.logMessage( - `[S3Storage] Constructor called with file type: ${file.type}`, - true - ) const [isValid, message] = this.validate() if (isValid === false) { - CORE_LOGGER.error(`[S3Storage] Validation failed: ${message}`) throw new Error(`Error validating the S3 file: ${message}`) } - CORE_LOGGER.logMessage(`[S3Storage] Validation successful`, true) } validate(): [boolean, string] { const file: S3FileObject = this.getFile() as S3FileObject - CORE_LOGGER.logMessage(`[S3Storage] Validating S3 file...`, true) - if (!file.s3Access) { - CORE_LOGGER.error(`[S3Storage] Missing s3Access object`) return [false, 'Missing s3Access'] } - - const { - bucket, - objectKey, - endpoint, - accessKeyId, - secretAccessKey, - region, - forcePathStyle - } = file.s3Access - - CORE_LOGGER.logMessage( - `[S3Storage] Validating fields - bucket: ${bucket}, objectKey: ${objectKey}, endpoint: ${endpoint}, region: ${region}, forcePathStyle: ${forcePathStyle}`, - true - ) - + const { bucket, objectKey, endpoint, accessKeyId, secretAccessKey } = file.s3Access if (!bucket?.trim()) { - CORE_LOGGER.error(`[S3Storage] Missing bucket`) return [false, 'Missing bucket'] } if (!objectKey?.trim()) { - CORE_LOGGER.error(`[S3Storage] Missing objectKey`) return [false, 'Missing objectKey'] } if (!endpoint?.trim()) { - CORE_LOGGER.error(`[S3Storage] Missing endpoint`) return [false, 'Missing endpoint'] } if (!accessKeyId?.trim()) { - CORE_LOGGER.error(`[S3Storage] Missing accessKeyId`) return [false, 'Missing accessKeyId'] } if (!secretAccessKey?.trim()) { - CORE_LOGGER.error(`[S3Storage] Missing secretAccessKey`) return [false, 'Missing secretAccessKey'] } - - CORE_LOGGER.logMessage(`[S3Storage] All fields validated successfully`, true) return [true, ''] } override async getReadableStream(): Promise { const { s3Access } = this.getFile() as S3FileObject - CORE_LOGGER.logMessage( - `[S3Storage] Getting readable stream for bucket: ${s3Access.bucket}, key: ${s3Access.objectKey}`, - true - ) - const s3Client = createS3Client(s3Access) try { - CORE_LOGGER.logMessage(`[S3Storage] Sending GetObjectCommand...`, true) const response = await s3Client.send( new GetObjectCommand({ Bucket: s3Access.bucket, @@ -112,13 +73,7 @@ export class S3Storage extends Storage { }) ) - CORE_LOGGER.logMessage( - `[S3Storage] GetObjectCommand successful, statusCode: ${response.$metadata.httpStatusCode}`, - true - ) - if (!response.Body) { - CORE_LOGGER.error(`[S3Storage] GetObject returned no body`) throw new Error('S3 GetObject returned no body') } @@ -130,23 +85,7 @@ export class S3Storage extends Storage { : undefined } } catch (err) { - CORE_LOGGER.error(`[S3Storage] Error fetching object from S3: ${err}`) - if (err instanceof Error) { - CORE_LOGGER.error( - `[S3Storage] Error name: ${err.name}, message: ${err.message}, stack: ${err.stack}` - ) - - // Log specific AWS error details - if ('$metadata' in err) { - const awsError = err as any - CORE_LOGGER.error( - `[S3Storage] AWS Error metadata: ${JSON.stringify(awsError.$metadata)}` - ) - CORE_LOGGER.error( - `[S3Storage] AWS Error code: ${awsError.Code}, requestId: ${awsError.$metadata?.requestId}` - ) - } - } + CORE_LOGGER.error(`Error fetching object from S3: ${err}`) throw err } } @@ -160,50 +99,26 @@ export class S3Storage extends Storage { stream: Readable ): Promise<{ httpStatus: number; headers?: Record }> { const { s3Access } = this.getFile() as S3FileObject - CORE_LOGGER.logMessage( - `[S3Storage] Uploading file: ${filename} to bucket: ${s3Access.bucket}`, - true - ) - const s3Client = createS3Client(s3Access) let key = s3Access.objectKey if (key.endsWith('/')) { key = `${key.replace(/\/+$/, '')}/${filename}` - CORE_LOGGER.logMessage( - `[S3Storage] Key ends with '/', updated key to: ${key}`, - true - ) - } - - try { - const upload = new Upload({ - client: s3Client, - params: { - Bucket: s3Access.bucket, - Key: key, - Body: stream, - ContentType: 'application/octet-stream', - ContentDisposition: `attachment; filename="${filename.replace(/"/g, '\\"')}"` - }, - queueSize: 4, - partSize: 5 * 1024 * 1024, // 5MB minimum for S3 - leavePartsOnError: false - }) - - CORE_LOGGER.logMessage(`[S3Storage] Starting upload...`, true) - await upload.done() - CORE_LOGGER.logMessage(`[S3Storage] Upload completed successfully`, true) - - return { httpStatus: 200, headers: {} } - } catch (err) { - CORE_LOGGER.error(`[S3Storage] Upload error: ${err}`) - if (err instanceof Error) { - CORE_LOGGER.error( - `[S3Storage] Upload error name: ${err.name}, message: ${err.message}` - ) - } - throw err } + const upload = new Upload({ + client: s3Client, + params: { + Bucket: s3Access.bucket, + Key: key, + Body: stream, + ContentType: 'application/octet-stream', + ContentDisposition: `attachment; filename="${filename.replace(/"/g, '\\"')}"` + }, + queueSize: 4, + partSize: 5 * 1024 * 1024, // 5MB minimum for S3 + leavePartsOnError: false + }) + await upload.done() + return { httpStatus: 200, headers: {} } } async fetchSpecificFileMetadata( @@ -211,85 +126,28 @@ export class S3Storage extends Storage { _forceChecksum: boolean ): Promise { const { s3Access } = fileObject - CORE_LOGGER.logMessage( - `[S3Storage] Fetching metadata for bucket: ${s3Access.bucket}, key: ${s3Access.objectKey}`, - true - ) - const s3Client = createS3Client(s3Access) - try { - CORE_LOGGER.logMessage(`[S3Storage] Sending HeadObjectCommand...`, true) - const data = await s3Client.send( - new HeadObjectCommand({ - Bucket: s3Access.bucket, - Key: s3Access.objectKey - }) - ) - - CORE_LOGGER.logMessage( - `[S3Storage] HeadObjectCommand successful, statusCode: ${data.$metadata.httpStatusCode}`, - true - ) - CORE_LOGGER.logMessage( - `[S3Storage] Metadata received - ContentLength: ${data.ContentLength}, ContentType: ${data.ContentType}, ETag: ${data.ETag}`, - true - ) - - const contentLength = data.ContentLength != null ? String(data.ContentLength) : '0' - const contentType = data.ContentType ?? 'application/octet-stream' - const name = s3Access.objectKey.split('/').pop() ?? s3Access.objectKey - - CORE_LOGGER.logMessage( - `[S3Storage] Returning file info - name: ${name}, type: ${contentType}, size: ${contentLength}`, - true - ) - - return { - valid: true, - contentLength, - contentType, - checksum: data.ETag?.replace(/"/g, ''), - name, - type: 's3', - encryptedBy: fileObject.encryptedBy, - encryptMethod: fileObject.encryptMethod - } - } catch (err) { - CORE_LOGGER.error(`[S3Storage] Error fetching metadata from S3: ${err}`) - if (err instanceof Error) { - CORE_LOGGER.error( - `[S3Storage] Error name: ${err.name}, message: ${err.message}, stack: ${err.stack}` - ) - - // Log specific AWS error details - if ('$metadata' in err) { - const awsError = err as any - CORE_LOGGER.error( - `[S3Storage] AWS Error metadata: ${JSON.stringify(awsError.$metadata)}` - ) - CORE_LOGGER.error(`[S3Storage] AWS Error code: ${awsError.Code}`) + const data = await s3Client.send( + new HeadObjectCommand({ + Bucket: s3Access.bucket, + Key: s3Access.objectKey + }) + ) - if (awsError.Code === 'NoSuchKey') { - CORE_LOGGER.error( - `[S3Storage] File not found: ${s3Access.bucket}/${s3Access.objectKey}` - ) - } else if (awsError.Code === 'AccessDenied') { - CORE_LOGGER.error( - `[S3Storage] Access denied - check credentials and bucket permissions` - ) - } else if (awsError.Code === 'NoSuchBucket') { - CORE_LOGGER.error(`[S3Storage] Bucket does not exist: ${s3Access.bucket}`) - } else if (awsError.Code === 'SignatureDoesNotMatch') { - CORE_LOGGER.error( - `[S3Storage] Signature does not match - check credentials and endpoint` - ) - } else if (awsError.Code === 'InvalidAccessKeyId') { - CORE_LOGGER.error(`[S3Storage] Invalid Access Key ID`) - } - } - } - throw err + const contentLength = data.ContentLength != null ? String(data.ContentLength) : '0' + const contentType = data.ContentType ?? 'application/octet-stream' + const name = s3Access.objectKey.split('/').pop() ?? s3Access.objectKey + + return { + valid: true, + contentLength, + contentType, + checksum: data.ETag?.replace(/"/g, ''), + name, + type: 's3', + encryptedBy: fileObject.encryptedBy, + encryptMethod: fileObject.encryptMethod } } }