From c7d7046e914e3b5f38fc8af8af085b52c44cd520 Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Thu, 4 Nov 2021 10:34:50 -0700 Subject: [PATCH 1/3] PYTHON-2459 Implement unified gridfs tests --- .evergreen/resync-specs.sh | 3 + test/gridfs/delete.json | 1019 +++++++++++++++++++++------- test/gridfs/download.json | 861 ++++++++++++----------- test/gridfs/downloadByName.json | 330 +++++++++ test/gridfs/download_by_name.json | 240 ------- test/gridfs/upload-disableMD5.json | 172 +++++ test/gridfs/upload.json | 891 +++++++++++++++--------- test/test_gridfs_spec.py | 214 +----- test/unified_format.py | 54 +- 9 files changed, 2349 insertions(+), 1435 deletions(-) create mode 100644 test/gridfs/downloadByName.json delete mode 100644 test/gridfs/download_by_name.json create mode 100644 test/gridfs/upload-disableMD5.json diff --git a/.evergreen/resync-specs.sh b/.evergreen/resync-specs.sh index 4f5366098b..b64868c5a9 100755 --- a/.evergreen/resync-specs.sh +++ b/.evergreen/resync-specs.sh @@ -108,6 +108,9 @@ do csot|CSOT|client-side-operations-timeout) cpjson client-side-operations-timeout/tests csot ;; + gridfs) + cpjson gridfs/tests gridfs + ;; load-balancers|load_balancer) cpjson load-balancers/tests load_balancer ;; diff --git a/test/gridfs/delete.json b/test/gridfs/delete.json index fb5de861f1..7a4ec27f88 100644 --- a/test/gridfs/delete.json +++ b/test/gridfs/delete.json @@ -1,304 +1,799 @@ { - "data": { - "files": [ - { - "_id": { - "$oid": "000000000000000000000001" - }, - "length": 0, - "chunkSize": 4, - "uploadDate": { - "$date": "1970-01-01T00:00:00.000Z" - }, - "md5": "d41d8cd98f00b204e9800998ecf8427e", - "filename": "length-0", - "contentType": "application/octet-stream", - "aliases": [], - "metadata": {} - }, - { - "_id": { - "$oid": "000000000000000000000002" - }, - "length": 0, - "chunkSize": 4, - "uploadDate": { - "$date": "1970-01-01T00:00:00.000Z" - }, - "md5": "d41d8cd98f00b204e9800998ecf8427e", - "filename": "length-0-with-empty-chunk", - "contentType": "application/octet-stream", - "aliases": [], - "metadata": {} - }, - { - "_id": { - "$oid": "000000000000000000000003" - }, - "length": 2, - "chunkSize": 4, - "uploadDate": { - "$date": "1970-01-01T00:00:00.000Z" - }, - "md5": "c700ed4fdb1d27055aa3faa2c2432283", - "filename": "length-2", - "contentType": "application/octet-stream", - "aliases": [], - "metadata": {} - }, - { - "_id": { - "$oid": "000000000000000000000004" - }, - "length": 8, - "chunkSize": 4, - "uploadDate": { - "$date": "1970-01-01T00:00:00.000Z" - }, - "md5": "dd254cdc958e53abaa67da9f797125f5", - "filename": "length-8", - "contentType": "application/octet-stream", - "aliases": [], - "metadata": {} + "description": "gridfs-delete", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" } - ], - "chunks": [ - { - "_id": { - "$oid": "000000000000000000000001" - }, - "files_id": { - "$oid": "000000000000000000000002" + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "gridfs-tests" + } + }, + { + "bucket": { + "id": "bucket0", + "database": "database0" + } + }, + { + "collection": { + "id": "bucket0_files_collection", + "database": "database0", + "collectionName": "fs.files" + } + }, + { + "collection": { + "id": "bucket0_chunks_collection", + "database": "database0", + "collectionName": "fs.chunks" + } + } + ], + "initialData": [ + { + "collectionName": "fs.files", + "databaseName": "gridfs-tests", + "documents": [ + { + "_id": { + "$oid": "000000000000000000000001" + }, + "length": 0, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "d41d8cd98f00b204e9800998ecf8427e", + "filename": "length-0", + "contentType": "application/octet-stream", + "aliases": [], + "metadata": {} }, - "n": 0, - "data": { - "$hex": "" - } - }, - { - "_id": { - "$oid": "000000000000000000000002" + { + "_id": { + "$oid": "000000000000000000000002" + }, + "length": 0, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "d41d8cd98f00b204e9800998ecf8427e", + "filename": "length-0-with-empty-chunk", + "contentType": "application/octet-stream", + "aliases": [], + "metadata": {} }, - "files_id": { - "$oid": "000000000000000000000003" + { + "_id": { + "$oid": "000000000000000000000003" + }, + "length": 2, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "c700ed4fdb1d27055aa3faa2c2432283", + "filename": "length-2", + "contentType": "application/octet-stream", + "aliases": [], + "metadata": {} }, - "n": 0, - "data": { - "$hex": "1122" + { + "_id": { + "$oid": "000000000000000000000004" + }, + "length": 8, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "dd254cdc958e53abaa67da9f797125f5", + "filename": "length-8", + "contentType": "application/octet-stream", + "aliases": [], + "metadata": {} } - }, - { - "_id": { - "$oid": "000000000000000000000003" - }, - "files_id": { - "$oid": "000000000000000000000004" + ] + }, + { + "collectionName": "fs.chunks", + "databaseName": "gridfs-tests", + "documents": [ + { + "_id": { + "$oid": "000000000000000000000001" + }, + "files_id": { + "$oid": "000000000000000000000002" + }, + "n": 0, + "data": { + "$binary": { + "base64": "", + "subType": "00" + } + } }, - "n": 0, - "data": { - "$hex": "11223344" - } - }, - { - "_id": { - "$oid": "000000000000000000000004" + { + "_id": { + "$oid": "000000000000000000000002" + }, + "files_id": { + "$oid": "000000000000000000000003" + }, + "n": 0, + "data": { + "$binary": { + "base64": "ESI=", + "subType": "00" + } + } }, - "files_id": { - "$oid": "000000000000000000000004" + { + "_id": { + "$oid": "000000000000000000000003" + }, + "files_id": { + "$oid": "000000000000000000000004" + }, + "n": 0, + "data": { + "$binary": { + "base64": "ESIzRA==", + "subType": "00" + } + } }, - "n": 1, - "data": { - "$hex": "55667788" + { + "_id": { + "$oid": "000000000000000000000004" + }, + "files_id": { + "$oid": "000000000000000000000004" + }, + "n": 1, + "data": { + "$binary": { + "base64": "VWZ3iA==", + "subType": "00" + } + } } - } - ] - }, + ] + } + ], "tests": [ { - "description": "Delete when length is 0", - "act": { - "operation": "delete", - "arguments": { - "id": { - "$oid": "000000000000000000000001" + "description": "delete when length is 0", + "operations": [ + { + "name": "delete", + "object": "bucket0", + "arguments": { + "id": { + "$oid": "000000000000000000000001" + } } } - }, - "assert": { - "result": "void", - "data": [ - { - "delete": "expected.files", - "deletes": [ - { - "q": { - "_id": { - "$oid": "000000000000000000000001" - } - }, - "limit": 1 + ], + "outcome": [ + { + "collectionName": "fs.files", + "databaseName": "gridfs-tests", + "documents": [ + { + "_id": { + "$oid": "000000000000000000000002" + }, + "length": 0, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "d41d8cd98f00b204e9800998ecf8427e", + "filename": "length-0-with-empty-chunk", + "contentType": "application/octet-stream", + "aliases": [], + "metadata": {} + }, + { + "_id": { + "$oid": "000000000000000000000003" + }, + "length": 2, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "c700ed4fdb1d27055aa3faa2c2432283", + "filename": "length-2", + "contentType": "application/octet-stream", + "aliases": [], + "metadata": {} + }, + { + "_id": { + "$oid": "000000000000000000000004" + }, + "length": 8, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "dd254cdc958e53abaa67da9f797125f5", + "filename": "length-8", + "contentType": "application/octet-stream", + "aliases": [], + "metadata": {} + } + ] + }, + { + "collectionName": "fs.chunks", + "databaseName": "gridfs-tests", + "documents": [ + { + "_id": { + "$oid": "000000000000000000000001" + }, + "files_id": { + "$oid": "000000000000000000000002" + }, + "n": 0, + "data": { + "$binary": { + "base64": "", + "subType": "00" + } } - ] - } - ] - } + }, + { + "_id": { + "$oid": "000000000000000000000002" + }, + "files_id": { + "$oid": "000000000000000000000003" + }, + "n": 0, + "data": { + "$binary": { + "base64": "ESI=", + "subType": "00" + } + } + }, + { + "_id": { + "$oid": "000000000000000000000003" + }, + "files_id": { + "$oid": "000000000000000000000004" + }, + "n": 0, + "data": { + "$binary": { + "base64": "ESIzRA==", + "subType": "00" + } + } + }, + { + "_id": { + "$oid": "000000000000000000000004" + }, + "files_id": { + "$oid": "000000000000000000000004" + }, + "n": 1, + "data": { + "$binary": { + "base64": "VWZ3iA==", + "subType": "00" + } + } + } + ] + } + ] }, { - "description": "Delete when length is 0 and there is one extra empty chunk", - "act": { - "operation": "delete", - "arguments": { - "id": { - "$oid": "000000000000000000000002" + "description": "delete when length is 0 and there is one extra empty chunk", + "operations": [ + { + "name": "delete", + "object": "bucket0", + "arguments": { + "id": { + "$oid": "000000000000000000000002" + } } } - }, - "assert": { - "result": "void", - "data": [ - { - "delete": "expected.files", - "deletes": [ - { - "q": { - "_id": { - "$oid": "000000000000000000000002" - } - }, - "limit": 1 + ], + "outcome": [ + { + "collectionName": "fs.files", + "databaseName": "gridfs-tests", + "documents": [ + { + "_id": { + "$oid": "000000000000000000000001" + }, + "length": 0, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "d41d8cd98f00b204e9800998ecf8427e", + "filename": "length-0", + "contentType": "application/octet-stream", + "aliases": [], + "metadata": {} + }, + { + "_id": { + "$oid": "000000000000000000000003" + }, + "length": 2, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "c700ed4fdb1d27055aa3faa2c2432283", + "filename": "length-2", + "contentType": "application/octet-stream", + "aliases": [], + "metadata": {} + }, + { + "_id": { + "$oid": "000000000000000000000004" + }, + "length": 8, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "dd254cdc958e53abaa67da9f797125f5", + "filename": "length-8", + "contentType": "application/octet-stream", + "aliases": [], + "metadata": {} + } + ] + }, + { + "collectionName": "fs.chunks", + "databaseName": "gridfs-tests", + "documents": [ + { + "_id": { + "$oid": "000000000000000000000002" + }, + "files_id": { + "$oid": "000000000000000000000003" + }, + "n": 0, + "data": { + "$binary": { + "base64": "ESI=", + "subType": "00" + } } - ] - }, - { - "delete": "expected.chunks", - "deletes": [ - { - "q": { - "files_id": { - "$oid": "000000000000000000000002" - } - }, - "limit": 0 + }, + { + "_id": { + "$oid": "000000000000000000000003" + }, + "files_id": { + "$oid": "000000000000000000000004" + }, + "n": 0, + "data": { + "$binary": { + "base64": "ESIzRA==", + "subType": "00" + } } - ] - } - ] - } + }, + { + "_id": { + "$oid": "000000000000000000000004" + }, + "files_id": { + "$oid": "000000000000000000000004" + }, + "n": 1, + "data": { + "$binary": { + "base64": "VWZ3iA==", + "subType": "00" + } + } + } + ] + } + ] }, { - "description": "Delete when length is 8", - "act": { - "operation": "delete", - "arguments": { - "id": { - "$oid": "000000000000000000000004" + "description": "delete when length is 8", + "operations": [ + { + "name": "delete", + "object": "bucket0", + "arguments": { + "id": { + "$oid": "000000000000000000000004" + } } } - }, - "assert": { - "result": "void", - "data": [ - { - "delete": "expected.files", - "deletes": [ - { - "q": { - "_id": { - "$oid": "000000000000000000000004" - } - }, - "limit": 1 + ], + "outcome": [ + { + "collectionName": "fs.files", + "databaseName": "gridfs-tests", + "documents": [ + { + "_id": { + "$oid": "000000000000000000000001" + }, + "length": 0, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "d41d8cd98f00b204e9800998ecf8427e", + "filename": "length-0", + "contentType": "application/octet-stream", + "aliases": [], + "metadata": {} + }, + { + "_id": { + "$oid": "000000000000000000000002" + }, + "length": 0, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "d41d8cd98f00b204e9800998ecf8427e", + "filename": "length-0-with-empty-chunk", + "contentType": "application/octet-stream", + "aliases": [], + "metadata": {} + }, + { + "_id": { + "$oid": "000000000000000000000003" + }, + "length": 2, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "c700ed4fdb1d27055aa3faa2c2432283", + "filename": "length-2", + "contentType": "application/octet-stream", + "aliases": [], + "metadata": {} + } + ] + }, + { + "collectionName": "fs.chunks", + "databaseName": "gridfs-tests", + "documents": [ + { + "_id": { + "$oid": "000000000000000000000001" + }, + "files_id": { + "$oid": "000000000000000000000002" + }, + "n": 0, + "data": { + "$binary": { + "base64": "", + "subType": "00" + } } - ] - }, - { - "delete": "expected.chunks", - "deletes": [ - { - "q": { - "files_id": { - "$oid": "000000000000000000000004" - } - }, - "limit": 0 + }, + { + "_id": { + "$oid": "000000000000000000000002" + }, + "files_id": { + "$oid": "000000000000000000000003" + }, + "n": 0, + "data": { + "$binary": { + "base64": "ESI=", + "subType": "00" + } } - ] - } - ] - } + } + ] + } + ] }, { - "description": "Delete when files entry does not exist", - "act": { - "operation": "delete", - "arguments": { - "id": { - "$oid": "000000000000000000000000" + "description": "delete when files entry does not exist", + "operations": [ + { + "name": "delete", + "object": "bucket0", + "arguments": { + "id": { + "$oid": "000000000000000000000000" + } + }, + "expectError": { + "isError": true } } - }, - "assert": { - "error": "FileNotFound" - } + ], + "outcome": [ + { + "collectionName": "fs.files", + "databaseName": "gridfs-tests", + "documents": [ + { + "_id": { + "$oid": "000000000000000000000001" + }, + "length": 0, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "d41d8cd98f00b204e9800998ecf8427e", + "filename": "length-0", + "contentType": "application/octet-stream", + "aliases": [], + "metadata": {} + }, + { + "_id": { + "$oid": "000000000000000000000002" + }, + "length": 0, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "d41d8cd98f00b204e9800998ecf8427e", + "filename": "length-0-with-empty-chunk", + "contentType": "application/octet-stream", + "aliases": [], + "metadata": {} + }, + { + "_id": { + "$oid": "000000000000000000000003" + }, + "length": 2, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "c700ed4fdb1d27055aa3faa2c2432283", + "filename": "length-2", + "contentType": "application/octet-stream", + "aliases": [], + "metadata": {} + }, + { + "_id": { + "$oid": "000000000000000000000004" + }, + "length": 8, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "dd254cdc958e53abaa67da9f797125f5", + "filename": "length-8", + "contentType": "application/octet-stream", + "aliases": [], + "metadata": {} + } + ] + }, + { + "collectionName": "fs.chunks", + "databaseName": "gridfs-tests", + "documents": [ + { + "_id": { + "$oid": "000000000000000000000001" + }, + "files_id": { + "$oid": "000000000000000000000002" + }, + "n": 0, + "data": { + "$binary": { + "base64": "", + "subType": "00" + } + } + }, + { + "_id": { + "$oid": "000000000000000000000002" + }, + "files_id": { + "$oid": "000000000000000000000003" + }, + "n": 0, + "data": { + "$binary": { + "base64": "ESI=", + "subType": "00" + } + } + }, + { + "_id": { + "$oid": "000000000000000000000003" + }, + "files_id": { + "$oid": "000000000000000000000004" + }, + "n": 0, + "data": { + "$binary": { + "base64": "ESIzRA==", + "subType": "00" + } + } + }, + { + "_id": { + "$oid": "000000000000000000000004" + }, + "files_id": { + "$oid": "000000000000000000000004" + }, + "n": 1, + "data": { + "$binary": { + "base64": "VWZ3iA==", + "subType": "00" + } + } + } + ] + } + ] }, { - "description": "Delete when files entry does not exist and there are orphaned chunks", - "arrange": { - "data": [ - { - "delete": "fs.files", - "deletes": [ - { - "q": { - "_id": { - "$oid": "000000000000000000000004" - } - }, - "limit": 1 + "description": "delete when files entry does not exist and there are orphaned chunks", + "operations": [ + { + "name": "deleteOne", + "object": "bucket0_files_collection", + "arguments": { + "filter": { + "_id": { + "$oid": "000000000000000000000004" } - ] + } + }, + "expectResult": { + "deletedCount": 1 } - ] - }, - "act": { - "operation": "delete", - "arguments": { - "id": { - "$oid": "000000000000000000000004" + }, + { + "name": "delete", + "object": "bucket0", + "arguments": { + "id": { + "$oid": "000000000000000000000004" + } + }, + "expectError": { + "isError": true } } - }, - "assert": { - "error": "FileNotFound", - "data": [ - { - "delete": "expected.files", - "deletes": [ - { - "q": { - "_id": { - "$oid": "000000000000000000000004" - } - }, - "limit": 1 + ], + "outcome": [ + { + "collectionName": "fs.files", + "databaseName": "gridfs-tests", + "documents": [ + { + "_id": { + "$oid": "000000000000000000000001" + }, + "length": 0, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "d41d8cd98f00b204e9800998ecf8427e", + "filename": "length-0", + "contentType": "application/octet-stream", + "aliases": [], + "metadata": {} + }, + { + "_id": { + "$oid": "000000000000000000000002" + }, + "length": 0, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "d41d8cd98f00b204e9800998ecf8427e", + "filename": "length-0-with-empty-chunk", + "contentType": "application/octet-stream", + "aliases": [], + "metadata": {} + }, + { + "_id": { + "$oid": "000000000000000000000003" + }, + "length": 2, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "c700ed4fdb1d27055aa3faa2c2432283", + "filename": "length-2", + "contentType": "application/octet-stream", + "aliases": [], + "metadata": {} + } + ] + }, + { + "collectionName": "fs.chunks", + "databaseName": "gridfs-tests", + "documents": [ + { + "_id": { + "$oid": "000000000000000000000001" + }, + "files_id": { + "$oid": "000000000000000000000002" + }, + "n": 0, + "data": { + "$binary": { + "base64": "", + "subType": "00" + } } - ] - }, - { - "delete": "expected.chunks", - "deletes": [ - { - "q": { - "files_id": { - "$oid": "000000000000000000000004" - } - }, - "limit": 0 + }, + { + "_id": { + "$oid": "000000000000000000000002" + }, + "files_id": { + "$oid": "000000000000000000000003" + }, + "n": 0, + "data": { + "$binary": { + "base64": "ESI=", + "subType": "00" + } } - ] - } - ] - } + } + ] + } + ] } ] } diff --git a/test/gridfs/download.json b/test/gridfs/download.json index 5092fba981..48d3246218 100644 --- a/test/gridfs/download.json +++ b/test/gridfs/download.json @@ -1,467 +1,558 @@ { - "data": { - "files": [ - { - "_id": { - "$oid": "000000000000000000000001" - }, - "length": 0, - "chunkSize": 4, - "uploadDate": { - "$date": "1970-01-01T00:00:00.000Z" - }, - "md5": "d41d8cd98f00b204e9800998ecf8427e", - "filename": "length-0", - "contentType": "application/octet-stream", - "aliases": [], - "metadata": {} - }, - { - "_id": { - "$oid": "000000000000000000000002" - }, - "length": 0, - "chunkSize": 4, - "uploadDate": { - "$date": "1970-01-01T00:00:00.000Z" - }, - "md5": "d41d8cd98f00b204e9800998ecf8427e", - "filename": "length-0-with-empty-chunk", - "contentType": "application/octet-stream", - "aliases": [], - "metadata": {} - }, - { - "_id": { - "$oid": "000000000000000000000003" - }, - "length": 2, - "chunkSize": 4, - "uploadDate": { - "$date": "1970-01-01T00:00:00.000Z" - }, - "md5": "c700ed4fdb1d27055aa3faa2c2432283", - "filename": "length-2", - "contentType": "application/octet-stream", - "aliases": [], - "metadata": {} - }, - { - "_id": { - "$oid": "000000000000000000000004" - }, - "length": 8, - "chunkSize": 4, - "uploadDate": { - "$date": "1970-01-01T00:00:00.000Z" - }, - "md5": "dd254cdc958e53abaa67da9f797125f5", - "filename": "length-8", - "contentType": "application/octet-stream", - "aliases": [], - "metadata": {} - }, - { - "_id": { - "$oid": "000000000000000000000005" - }, - "length": 10, - "chunkSize": 4, - "uploadDate": { - "$date": "1970-01-01T00:00:00.000Z" - }, - "md5": "57d83cd477bfb1ccd975ab33d827a92b", - "filename": "length-10", - "contentType": "application/octet-stream", - "aliases": [], - "metadata": {} - }, - { - "_id": { - "$oid": "000000000000000000000006" - }, - "length": 2, - "chunkSize": 4, - "uploadDate": { - "$date": "1970-01-01T00:00:00.000Z" - }, - "md5": "c700ed4fdb1d27055aa3faa2c2432283", - "contentType": "application/octet-stream", - "aliases": [], - "metadata": {} + "description": "gridfs-download", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" } - ], - "chunks": [ - { - "_id": { - "$oid": "000000000000000000000001" - }, - "files_id": { - "$oid": "000000000000000000000002" - }, - "n": 0, - "data": { - "$hex": "" - } - }, - { - "_id": { - "$oid": "000000000000000000000002" - }, - "files_id": { - "$oid": "000000000000000000000003" + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "gridfs-tests" + } + }, + { + "bucket": { + "id": "bucket0", + "database": "database0" + } + }, + { + "collection": { + "id": "bucket0_files_collection", + "database": "database0", + "collectionName": "fs.files" + } + }, + { + "collection": { + "id": "bucket0_chunks_collection", + "database": "database0", + "collectionName": "fs.chunks" + } + } + ], + "initialData": [ + { + "collectionName": "fs.files", + "databaseName": "gridfs-tests", + "documents": [ + { + "_id": { + "$oid": "000000000000000000000001" + }, + "length": 0, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "d41d8cd98f00b204e9800998ecf8427e", + "filename": "length-0", + "contentType": "application/octet-stream", + "aliases": [], + "metadata": {} }, - "n": 0, - "data": { - "$hex": "1122" - } - }, - { - "_id": { - "$oid": "000000000000000000000003" + { + "_id": { + "$oid": "000000000000000000000002" + }, + "length": 0, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "d41d8cd98f00b204e9800998ecf8427e", + "filename": "length-0-with-empty-chunk", + "contentType": "application/octet-stream", + "aliases": [], + "metadata": {} }, - "files_id": { - "$oid": "000000000000000000000004" + { + "_id": { + "$oid": "000000000000000000000003" + }, + "length": 2, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "c700ed4fdb1d27055aa3faa2c2432283", + "filename": "length-2", + "contentType": "application/octet-stream", + "aliases": [], + "metadata": {} }, - "n": 0, - "data": { - "$hex": "11223344" - } - }, - { - "_id": { - "$oid": "000000000000000000000004" + { + "_id": { + "$oid": "000000000000000000000004" + }, + "length": 8, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "dd254cdc958e53abaa67da9f797125f5", + "filename": "length-8", + "contentType": "application/octet-stream", + "aliases": [], + "metadata": {} }, - "files_id": { - "$oid": "000000000000000000000004" + { + "_id": { + "$oid": "000000000000000000000005" + }, + "length": 10, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "57d83cd477bfb1ccd975ab33d827a92b", + "filename": "length-10", + "contentType": "application/octet-stream", + "aliases": [], + "metadata": {} }, - "n": 1, - "data": { - "$hex": "55667788" + { + "_id": { + "$oid": "000000000000000000000006" + }, + "length": 2, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "c700ed4fdb1d27055aa3faa2c2432283", + "contentType": "application/octet-stream", + "aliases": [], + "metadata": {} } - }, - { - "_id": { - "$oid": "000000000000000000000005" - }, - "files_id": { - "$oid": "000000000000000000000005" + ] + }, + { + "collectionName": "fs.chunks", + "databaseName": "gridfs-tests", + "documents": [ + { + "_id": { + "$oid": "000000000000000000000001" + }, + "files_id": { + "$oid": "000000000000000000000002" + }, + "n": 0, + "data": { + "$binary": { + "base64": "", + "subType": "00" + } + } }, - "n": 0, - "data": { - "$hex": "11223344" - } - }, - { - "_id": { - "$oid": "000000000000000000000006" + { + "_id": { + "$oid": "000000000000000000000002" + }, + "files_id": { + "$oid": "000000000000000000000003" + }, + "n": 0, + "data": { + "$binary": { + "base64": "ESI=", + "subType": "00" + } + } }, - "files_id": { - "$oid": "000000000000000000000005" + { + "_id": { + "$oid": "000000000000000000000003" + }, + "files_id": { + "$oid": "000000000000000000000004" + }, + "n": 0, + "data": { + "$binary": { + "base64": "ESIzRA==", + "subType": "00" + } + } }, - "n": 1, - "data": { - "$hex": "55667788" - } - }, - { - "_id": { - "$oid": "000000000000000000000007" + { + "_id": { + "$oid": "000000000000000000000004" + }, + "files_id": { + "$oid": "000000000000000000000004" + }, + "n": 1, + "data": { + "$binary": { + "base64": "VWZ3iA==", + "subType": "00" + } + } }, - "files_id": { - "$oid": "000000000000000000000005" + { + "_id": { + "$oid": "000000000000000000000005" + }, + "files_id": { + "$oid": "000000000000000000000005" + }, + "n": 0, + "data": { + "$binary": { + "base64": "ESIzRA==", + "subType": "00" + } + } }, - "n": 2, - "data": { - "$hex": "99aa" - } - }, - { - "_id": { - "$oid": "000000000000000000000008" + { + "_id": { + "$oid": "000000000000000000000006" + }, + "files_id": { + "$oid": "000000000000000000000005" + }, + "n": 1, + "data": { + "$binary": { + "base64": "VWZ3iA==", + "subType": "00" + } + } }, - "files_id": { - "$oid": "000000000000000000000006" + { + "_id": { + "$oid": "000000000000000000000007" + }, + "files_id": { + "$oid": "000000000000000000000005" + }, + "n": 2, + "data": { + "$binary": { + "base64": "mao=", + "subType": "00" + } + } }, - "n": 0, - "data": { - "$hex": "1122" + { + "_id": { + "$oid": "000000000000000000000008" + }, + "files_id": { + "$oid": "000000000000000000000006" + }, + "n": 0, + "data": { + "$binary": { + "base64": "ESI=", + "subType": "00" + } + } } - } - ] - }, + ] + } + ], "tests": [ { - "description": "Download when length is zero", - "act": { - "operation": "download", - "arguments": { - "id": { - "$oid": "000000000000000000000001" + "description": "download when length is zero", + "operations": [ + { + "name": "download", + "object": "bucket0", + "arguments": { + "id": { + "$oid": "000000000000000000000001" + } }, - "options": {} - } - }, - "assert": { - "result": { - "$hex": "" + "expectResult": { + "$$matchesHexBytes": "" + } } - } + ] }, { - "description": "Download when length is zero and there is one empty chunk", - "act": { - "operation": "download", - "arguments": { - "id": { - "$oid": "000000000000000000000002" + "description": "download when length is zero and there is one empty chunk", + "operations": [ + { + "name": "download", + "object": "bucket0", + "arguments": { + "id": { + "$oid": "000000000000000000000002" + } }, - "options": {} - } - }, - "assert": { - "result": { - "$hex": "" + "expectResult": { + "$$matchesHexBytes": "" + } } - } + ] }, { - "description": "Download when there is one chunk", - "act": { - "operation": "download", - "arguments": { - "id": { - "$oid": "000000000000000000000003" + "description": "download when there is one chunk", + "operations": [ + { + "name": "download", + "object": "bucket0", + "arguments": { + "id": { + "$oid": "000000000000000000000003" + } }, - "options": {} - } - }, - "assert": { - "result": { - "$hex": "1122" + "expectResult": { + "$$matchesHexBytes": "1122" + } } - } + ] }, { - "description": "Download when there are two chunks", - "act": { - "operation": "download", - "arguments": { - "id": { - "$oid": "000000000000000000000004" + "description": "download when there are two chunks", + "operations": [ + { + "name": "download", + "object": "bucket0", + "arguments": { + "id": { + "$oid": "000000000000000000000004" + } }, - "options": {} - } - }, - "assert": { - "result": { - "$hex": "1122334455667788" + "expectResult": { + "$$matchesHexBytes": "1122334455667788" + } } - } + ] }, { - "description": "Download when there are three chunks", - "act": { - "operation": "download", - "arguments": { - "id": { - "$oid": "000000000000000000000005" + "description": "download when there are three chunks", + "operations": [ + { + "name": "download", + "object": "bucket0", + "arguments": { + "id": { + "$oid": "000000000000000000000005" + } }, - "options": {} - } - }, - "assert": { - "result": { - "$hex": "112233445566778899aa" + "expectResult": { + "$$matchesHexBytes": "112233445566778899aa" + } } - } + ] }, { - "description": "Download when files entry does not exist", - "act": { - "operation": "download", - "arguments": { - "id": { - "$oid": "000000000000000000000000" - }, - "options": {} + "description": "download when files entry does not exist", + "operations": [ + { + "name": "download", + "object": "bucket0", + "arguments": { + "id": { + "$oid": "000000000000000000000000" + } + }, + "expectError": { + "isError": true + } } - }, - "assert": { - "error": "FileNotFound" - } + ] }, { - "description": "Download when an intermediate chunk is missing", - "arrange": { - "data": [ - { - "delete": "fs.chunks", - "deletes": [ - { - "q": { - "files_id": { - "$oid": "000000000000000000000005" - }, - "n": 1 - }, - "limit": 1 - } - ] + "description": "download when an intermediate chunk is missing", + "operations": [ + { + "name": "deleteOne", + "object": "bucket0_chunks_collection", + "arguments": { + "filter": { + "files_id": { + "$oid": "000000000000000000000005" + }, + "n": 1 + } + }, + "expectResult": { + "deletedCount": 1 } - ] - }, - "act": { - "operation": "download", - "arguments": { - "id": { - "$oid": "000000000000000000000005" + }, + { + "name": "download", + "object": "bucket0", + "arguments": { + "id": { + "$oid": "000000000000000000000005" + } + }, + "expectError": { + "isError": true } } - }, - "assert": { - "error": "ChunkIsMissing" - } + ] }, { - "description": "Download when final chunk is missing", - "arrange": { - "data": [ - { - "delete": "fs.chunks", - "deletes": [ - { - "q": { - "files_id": { - "$oid": "000000000000000000000005" - }, - "n": 1 - }, - "limit": 1 - } - ] + "description": "download when final chunk is missing", + "operations": [ + { + "name": "deleteOne", + "object": "bucket0_chunks_collection", + "arguments": { + "filter": { + "files_id": { + "$oid": "000000000000000000000005" + }, + "n": 2 + } + }, + "expectResult": { + "deletedCount": 1 } - ] - }, - "act": { - "operation": "download", - "arguments": { - "id": { - "$oid": "000000000000000000000005" + }, + { + "name": "download", + "object": "bucket0", + "arguments": { + "id": { + "$oid": "000000000000000000000005" + } + }, + "expectError": { + "isError": true } } - }, - "assert": { - "error": "ChunkIsMissing" - } + ] }, { - "description": "Download when an intermediate chunk is the wrong size", - "arrange": { - "data": [ - { - "update": "fs.chunks", - "updates": [ + "description": "download when an intermediate chunk is the wrong size", + "operations": [ + { + "name": "bulkWrite", + "object": "bucket0_chunks_collection", + "arguments": { + "requests": [ { - "q": { - "files_id": { - "$oid": "000000000000000000000005" + "updateOne": { + "filter": { + "files_id": { + "$oid": "000000000000000000000005" + }, + "n": 1 }, - "n": 1 - }, - "u": { - "$set": { - "data": { - "$hex": "556677" + "update": { + "$set": { + "data": { + "$binary": { + "base64": "VWZ3", + "subType": "00" + } + } } } } }, { - "q": { - "files_id": { - "$oid": "000000000000000000000005" + "updateOne": { + "filter": { + "files_id": { + "$oid": "000000000000000000000005" + }, + "n": 2 }, - "n": 2 - }, - "u": { - "$set": { - "data": { - "$hex": "8899aa" + "update": { + "$set": { + "data": { + "$binary": { + "base64": "iJmq", + "subType": "00" + } + } } } } } ] + }, + "expectResult": { + "matchedCount": 2, + "modifiedCount": 2 } - ] - }, - "act": { - "operation": "download", - "arguments": { - "id": { - "$oid": "000000000000000000000005" + }, + { + "name": "download", + "object": "bucket0", + "arguments": { + "id": { + "$oid": "000000000000000000000005" + } + }, + "expectError": { + "isError": true } } - }, - "assert": { - "error": "ChunkIsWrongSize" - } + ] }, { - "description": "Download when final chunk is the wrong size", - "arrange": { - "data": [ - { - "update": "fs.chunks", - "updates": [ - { - "q": { - "files_id": { - "$oid": "000000000000000000000005" - }, - "n": 2 - }, - "u": { - "$set": { - "data": { - "$hex": "99" - } + "description": "download when final chunk is the wrong size", + "operations": [ + { + "name": "updateOne", + "object": "bucket0_chunks_collection", + "arguments": { + "filter": { + "files_id": { + "$oid": "000000000000000000000005" + }, + "n": 2 + }, + "update": { + "$set": { + "data": { + "$binary": { + "base64": "mQ==", + "subType": "00" } } } - ] + } + }, + "expectResult": { + "matchedCount": 1, + "modifiedCount": 1 } - ] - }, - "act": { - "operation": "download", - "arguments": { - "id": { - "$oid": "000000000000000000000005" + }, + { + "name": "download", + "object": "bucket0", + "arguments": { + "id": { + "$oid": "000000000000000000000005" + } + }, + "expectError": { + "isError": true } } - }, - "assert": { - "error": "ChunkIsWrongSize" - } + ] }, { - "description": "Download legacy file with no name", - "act": { - "operation": "download", - "arguments": { - "id": { - "$oid": "000000000000000000000006" + "description": "download legacy file with no name", + "operations": [ + { + "name": "download", + "object": "bucket0", + "arguments": { + "id": { + "$oid": "000000000000000000000006" + } }, - "options": {} - } - }, - "assert": { - "result": { - "$hex": "1122" + "expectResult": { + "$$matchesHexBytes": "1122" + } } - } + ] } ] } diff --git a/test/gridfs/downloadByName.json b/test/gridfs/downloadByName.json new file mode 100644 index 0000000000..cd44663957 --- /dev/null +++ b/test/gridfs/downloadByName.json @@ -0,0 +1,330 @@ +{ + "description": "gridfs-downloadByName", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "gridfs-tests" + } + }, + { + "bucket": { + "id": "bucket0", + "database": "database0" + } + }, + { + "collection": { + "id": "bucket0_files_collection", + "database": "database0", + "collectionName": "fs.files" + } + }, + { + "collection": { + "id": "bucket0_chunks_collection", + "database": "database0", + "collectionName": "fs.chunks" + } + } + ], + "initialData": [ + { + "collectionName": "fs.files", + "databaseName": "gridfs-tests", + "documents": [ + { + "_id": { + "$oid": "000000000000000000000001" + }, + "length": 1, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-01T00:00:00.000Z" + }, + "md5": "47ed733b8d10be225eceba344d533586", + "filename": "abc", + "contentType": "application/octet-stream", + "aliases": [], + "metadata": {} + }, + { + "_id": { + "$oid": "000000000000000000000002" + }, + "length": 1, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-02T00:00:00.000Z" + }, + "md5": "b15835f133ff2e27c7cb28117bfae8f4", + "filename": "abc", + "contentType": "application/octet-stream", + "aliases": [], + "metadata": {} + }, + { + "_id": { + "$oid": "000000000000000000000003" + }, + "length": 1, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-03T00:00:00.000Z" + }, + "md5": "eccbc87e4b5ce2fe28308fd9f2a7baf3", + "filename": "abc", + "contentType": "application/octet-stream", + "aliases": [], + "metadata": {} + }, + { + "_id": { + "$oid": "000000000000000000000004" + }, + "length": 1, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-04T00:00:00.000Z" + }, + "md5": "f623e75af30e62bbd73d6df5b50bb7b5", + "filename": "abc", + "contentType": "application/octet-stream", + "aliases": [], + "metadata": {} + }, + { + "_id": { + "$oid": "000000000000000000000005" + }, + "length": 1, + "chunkSize": 4, + "uploadDate": { + "$date": "1970-01-05T00:00:00.000Z" + }, + "md5": "4c614360da93c0a041b22e537de151eb", + "filename": "abc", + "contentType": "application/octet-stream", + "aliases": [], + "metadata": {} + } + ] + }, + { + "collectionName": "fs.chunks", + "databaseName": "gridfs-tests", + "documents": [ + { + "_id": { + "$oid": "000000000000000000000001" + }, + "files_id": { + "$oid": "000000000000000000000001" + }, + "n": 0, + "data": { + "$binary": { + "base64": "EQ==", + "subType": "00" + } + } + }, + { + "_id": { + "$oid": "000000000000000000000002" + }, + "files_id": { + "$oid": "000000000000000000000002" + }, + "n": 0, + "data": { + "$binary": { + "base64": "Ig==", + "subType": "00" + } + } + }, + { + "_id": { + "$oid": "000000000000000000000003" + }, + "files_id": { + "$oid": "000000000000000000000003" + }, + "n": 0, + "data": { + "$binary": { + "base64": "Mw==", + "subType": "00" + } + } + }, + { + "_id": { + "$oid": "000000000000000000000004" + }, + "files_id": { + "$oid": "000000000000000000000004" + }, + "n": 0, + "data": { + "$binary": { + "base64": "RA==", + "subType": "00" + } + } + }, + { + "_id": { + "$oid": "000000000000000000000005" + }, + "files_id": { + "$oid": "000000000000000000000005" + }, + "n": 0, + "data": { + "$binary": { + "base64": "VQ==", + "subType": "00" + } + } + } + ] + } + ], + "tests": [ + { + "description": "downloadByName defaults to latest revision (-1)", + "operations": [ + { + "name": "downloadByName", + "object": "bucket0", + "arguments": { + "filename": "abc" + }, + "expectResult": { + "$$matchesHexBytes": "55" + } + } + ] + }, + { + "description": "downloadByName when revision is 0", + "operations": [ + { + "name": "downloadByName", + "object": "bucket0", + "arguments": { + "filename": "abc", + "revision": 0 + }, + "expectResult": { + "$$matchesHexBytes": "11" + } + } + ] + }, + { + "description": "downloadByName when revision is 1", + "operations": [ + { + "name": "downloadByName", + "object": "bucket0", + "arguments": { + "filename": "abc", + "revision": 1 + }, + "expectResult": { + "$$matchesHexBytes": "22" + } + } + ] + }, + { + "description": "downloadByName when revision is 2", + "operations": [ + { + "name": "downloadByName", + "object": "bucket0", + "arguments": { + "filename": "abc", + "revision": 2 + }, + "expectResult": { + "$$matchesHexBytes": "33" + } + } + ] + }, + { + "description": "downloadByName when revision is -2", + "operations": [ + { + "name": "downloadByName", + "object": "bucket0", + "arguments": { + "filename": "abc", + "revision": -2 + }, + "expectResult": { + "$$matchesHexBytes": "44" + } + } + ] + }, + { + "description": "downloadByName when revision is -1", + "operations": [ + { + "name": "downloadByName", + "object": "bucket0", + "arguments": { + "filename": "abc", + "revision": -1 + }, + "expectResult": { + "$$matchesHexBytes": "55" + } + } + ] + }, + { + "description": "downloadByName when files entry does not exist", + "operations": [ + { + "name": "downloadByName", + "object": "bucket0", + "arguments": { + "filename": "xyz" + }, + "expectError": { + "isError": true + } + } + ] + }, + { + "description": "downloadByName when revision does not exist", + "operations": [ + { + "name": "downloadByName", + "object": "bucket0", + "arguments": { + "filename": "abc", + "revision": 999 + }, + "expectError": { + "isError": true + } + } + ] + } + ] +} diff --git a/test/gridfs/download_by_name.json b/test/gridfs/download_by_name.json deleted file mode 100644 index ecc8c9e2cc..0000000000 --- a/test/gridfs/download_by_name.json +++ /dev/null @@ -1,240 +0,0 @@ -{ - "data": { - "files": [ - { - "_id": { - "$oid": "000000000000000000000001" - }, - "length": 1, - "chunkSize": 4, - "uploadDate": { - "$date": "1970-01-01T00:00:00.000Z" - }, - "md5": "47ed733b8d10be225eceba344d533586", - "filename": "abc", - "contentType": "application/octet-stream", - "aliases": [], - "metadata": {} - }, - { - "_id": { - "$oid": "000000000000000000000002" - }, - "length": 1, - "chunkSize": 4, - "uploadDate": { - "$date": "1970-01-02T00:00:00.000Z" - }, - "md5": "b15835f133ff2e27c7cb28117bfae8f4", - "filename": "abc", - "contentType": "application/octet-stream", - "aliases": [], - "metadata": {} - }, - { - "_id": { - "$oid": "000000000000000000000003" - }, - "length": 1, - "chunkSize": 4, - "uploadDate": { - "$date": "1970-01-03T00:00:00.000Z" - }, - "md5": "eccbc87e4b5ce2fe28308fd9f2a7baf3", - "filename": "abc", - "contentType": "application/octet-stream", - "aliases": [], - "metadata": {} - }, - { - "_id": { - "$oid": "000000000000000000000004" - }, - "length": 1, - "chunkSize": 4, - "uploadDate": { - "$date": "1970-01-04T00:00:00.000Z" - }, - "md5": "f623e75af30e62bbd73d6df5b50bb7b5", - "filename": "abc", - "contentType": "application/octet-stream", - "aliases": [], - "metadata": {} - }, - { - "_id": { - "$oid": "000000000000000000000005" - }, - "length": 1, - "chunkSize": 4, - "uploadDate": { - "$date": "1970-01-05T00:00:00.000Z" - }, - "md5": "4c614360da93c0a041b22e537de151eb", - "filename": "abc", - "contentType": "application/octet-stream", - "aliases": [], - "metadata": {} - } - ], - "chunks": [ - { - "_id": { - "$oid": "000000000000000000000001" - }, - "files_id": { - "$oid": "000000000000000000000001" - }, - "n": 0, - "data": { - "$hex": "11" - } - }, - { - "_id": { - "$oid": "000000000000000000000002" - }, - "files_id": { - "$oid": "000000000000000000000002" - }, - "n": 0, - "data": { - "$hex": "22" - } - }, - { - "_id": { - "$oid": "000000000000000000000003" - }, - "files_id": { - "$oid": "000000000000000000000003" - }, - "n": 0, - "data": { - "$hex": "33" - } - }, - { - "_id": { - "$oid": "000000000000000000000004" - }, - "files_id": { - "$oid": "000000000000000000000004" - }, - "n": 0, - "data": { - "$hex": "44" - } - }, - { - "_id": { - "$oid": "000000000000000000000005" - }, - "files_id": { - "$oid": "000000000000000000000005" - }, - "n": 0, - "data": { - "$hex": "55" - } - } - ] - }, - "tests": [ - { - "description": "Download_by_name when revision is 0", - "act": { - "operation": "download_by_name", - "arguments": { - "filename": "abc", - "options": { - "revision": 0 - } - } - }, - "assert": { - "result": { - "$hex": "11" - } - } - }, - { - "description": "Download_by_name when revision is 1", - "act": { - "operation": "download_by_name", - "arguments": { - "filename": "abc", - "options": { - "revision": 1 - } - } - }, - "assert": { - "result": { - "$hex": "22" - } - } - }, - { - "description": "Download_by_name when revision is -2", - "act": { - "operation": "download_by_name", - "arguments": { - "filename": "abc", - "options": { - "revision": -2 - } - } - }, - "assert": { - "result": { - "$hex": "44" - } - } - }, - { - "description": "Download_by_name when revision is -1", - "act": { - "operation": "download_by_name", - "arguments": { - "filename": "abc", - "options": { - "revision": -1 - } - } - }, - "assert": { - "result": { - "$hex": "55" - } - } - }, - { - "description": "Download_by_name when files entry does not exist", - "act": { - "operation": "download_by_name", - "arguments": { - "filename": "xyz" - } - }, - "assert": { - "error": "FileNotFound" - } - }, - { - "description": "Download_by_name when revision does not exist", - "act": { - "operation": "download_by_name", - "arguments": { - "filename": "abc", - "options": { - "revision": 999 - } - } - }, - "assert": { - "error": "RevisionNotFound" - } - } - ] -} diff --git a/test/gridfs/upload-disableMD5.json b/test/gridfs/upload-disableMD5.json new file mode 100644 index 0000000000..d5a9d6f4ab --- /dev/null +++ b/test/gridfs/upload-disableMD5.json @@ -0,0 +1,172 @@ +{ + "description": "gridfs-upload-disableMD5", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "gridfs-tests" + } + }, + { + "bucket": { + "id": "bucket0", + "database": "database0" + } + }, + { + "collection": { + "id": "bucket0_files_collection", + "database": "database0", + "collectionName": "fs.files" + } + }, + { + "collection": { + "id": "bucket0_chunks_collection", + "database": "database0", + "collectionName": "fs.chunks" + } + } + ], + "initialData": [ + { + "collectionName": "fs.files", + "databaseName": "gridfs-tests", + "documents": [] + }, + { + "collectionName": "fs.chunks", + "databaseName": "gridfs-tests", + "documents": [] + } + ], + "tests": [ + { + "description": "upload when length is 0 sans MD5", + "operations": [ + { + "name": "upload", + "object": "bucket0", + "arguments": { + "filename": "filename", + "source": { + "$$hexBytes": "" + }, + "chunkSizeBytes": 4, + "disableMD5": true + }, + "expectResult": { + "$$type": "objectId" + }, + "saveResultAsEntity": "uploadedObjectId" + }, + { + "name": "find", + "object": "bucket0_files_collection", + "arguments": { + "filter": {} + }, + "expectResult": [ + { + "_id": { + "$$matchesEntity": "uploadedObjectId" + }, + "length": 0, + "chunkSize": 4, + "uploadDate": { + "$$type": "date" + }, + "md5": { + "$$exists": false + }, + "filename": "filename" + } + ] + }, + { + "name": "find", + "object": "bucket0_chunks_collection", + "arguments": { + "filter": {} + }, + "expectResult": [] + } + ] + }, + { + "description": "upload when length is 1 sans MD5", + "operations": [ + { + "name": "upload", + "object": "bucket0", + "arguments": { + "filename": "filename", + "source": { + "$$hexBytes": "11" + }, + "chunkSizeBytes": 4, + "disableMD5": true + }, + "expectResult": { + "$$type": "objectId" + }, + "saveResultAsEntity": "uploadedObjectId" + }, + { + "name": "find", + "object": "bucket0_files_collection", + "arguments": { + "filter": {} + }, + "expectResult": [ + { + "_id": { + "$$matchesEntity": "uploadedObjectId" + }, + "length": 1, + "chunkSize": 4, + "uploadDate": { + "$$type": "date" + }, + "md5": { + "$$exists": false + }, + "filename": "filename" + } + ] + }, + { + "name": "find", + "object": "bucket0_chunks_collection", + "arguments": { + "filter": {} + }, + "expectResult": [ + { + "_id": { + "$$type": "objectId" + }, + "files_id": { + "$$matchesEntity": "uploadedObjectId" + }, + "n": 0, + "data": { + "$binary": { + "base64": "EQ==", + "subType": "00" + } + } + } + ] + } + ] + } + ] +} diff --git a/test/gridfs/upload.json b/test/gridfs/upload.json index 7d4adec1d8..97e18d2bc2 100644 --- a/test/gridfs/upload.json +++ b/test/gridfs/upload.json @@ -1,379 +1,616 @@ { - "data": { - "files": [], - "chunks": [] - }, + "description": "gridfs-upload", + "schemaVersion": "1.0", + "createEntities": [ + { + "client": { + "id": "client0" + } + }, + { + "database": { + "id": "database0", + "client": "client0", + "databaseName": "gridfs-tests" + } + }, + { + "bucket": { + "id": "bucket0", + "database": "database0" + } + }, + { + "collection": { + "id": "bucket0_files_collection", + "database": "database0", + "collectionName": "fs.files" + } + }, + { + "collection": { + "id": "bucket0_chunks_collection", + "database": "database0", + "collectionName": "fs.chunks" + } + } + ], + "initialData": [ + { + "collectionName": "fs.files", + "databaseName": "gridfs-tests", + "documents": [] + }, + { + "collectionName": "fs.chunks", + "databaseName": "gridfs-tests", + "documents": [] + } + ], "tests": [ { - "description": "Upload when length is 0", - "act": { - "operation": "upload", - "arguments": { - "filename": "filename", - "source": { - "$hex": "" - }, - "options": { + "description": "upload when length is 0", + "operations": [ + { + "name": "upload", + "object": "bucket0", + "arguments": { + "filename": "filename", + "source": { + "$$hexBytes": "" + }, "chunkSizeBytes": 4 - } + }, + "expectResult": { + "$$type": "objectId" + }, + "saveResultAsEntity": "uploadedObjectId" + }, + { + "name": "find", + "object": "bucket0_files_collection", + "arguments": { + "filter": {} + }, + "expectResult": [ + { + "_id": { + "$$matchesEntity": "uploadedObjectId" + }, + "length": 0, + "chunkSize": 4, + "uploadDate": { + "$$type": "date" + }, + "md5": { + "$$unsetOrMatches": "d41d8cd98f00b204e9800998ecf8427e" + }, + "filename": "filename" + } + ] + }, + { + "name": "find", + "object": "bucket0_chunks_collection", + "arguments": { + "filter": {} + }, + "expectResult": [] } - }, - "assert": { - "result": "&result", - "data": [ - { - "insert": "expected.files", - "documents": [ - { - "_id": "*result", - "length": 0, - "chunkSize": 4, - "uploadDate": "*actual", - "filename": "filename" - } - ] - } - ] - } + ] }, { - "description": "Upload when length is 1", - "act": { - "operation": "upload", - "arguments": { - "filename": "filename", - "source": { - "$hex": "11" - }, - "options": { + "description": "upload when length is 1", + "operations": [ + { + "name": "upload", + "object": "bucket0", + "arguments": { + "filename": "filename", + "source": { + "$$hexBytes": "11" + }, "chunkSizeBytes": 4 - } - } - }, - "assert": { - "result": "&result", - "data": [ - { - "insert": "expected.files", - "documents": [ - { - "_id": "*result", - "length": 1, - "chunkSize": 4, - "uploadDate": "*actual", - "filename": "filename" - } - ] - }, - { - "insert": "expected.chunks", - "documents": [ - { - "_id": "*actual", - "files_id": "*result", - "n": 0, - "data": { - "$hex": "11" + }, + "expectResult": { + "$$type": "objectId" + }, + "saveResultAsEntity": "uploadedObjectId" + }, + { + "name": "find", + "object": "bucket0_files_collection", + "arguments": { + "filter": {} + }, + "expectResult": [ + { + "_id": { + "$$matchesEntity": "uploadedObjectId" + }, + "length": 1, + "chunkSize": 4, + "uploadDate": { + "$$type": "date" + }, + "md5": { + "$$unsetOrMatches": "47ed733b8d10be225eceba344d533586" + }, + "filename": "filename" + } + ] + }, + { + "name": "find", + "object": "bucket0_chunks_collection", + "arguments": { + "filter": {} + }, + "expectResult": [ + { + "_id": { + "$$type": "objectId" + }, + "files_id": { + "$$matchesEntity": "uploadedObjectId" + }, + "n": 0, + "data": { + "$binary": { + "base64": "EQ==", + "subType": "00" } } - ] - } - ] - } + } + ] + } + ] }, { - "description": "Upload when length is 3", - "act": { - "operation": "upload", - "arguments": { - "filename": "filename", - "source": { - "$hex": "112233" - }, - "options": { + "description": "upload when length is 3", + "operations": [ + { + "name": "upload", + "object": "bucket0", + "arguments": { + "filename": "filename", + "source": { + "$$hexBytes": "112233" + }, "chunkSizeBytes": 4 - } - } - }, - "assert": { - "result": "&result", - "data": [ - { - "insert": "expected.files", - "documents": [ - { - "_id": "*result", - "length": 3, - "chunkSize": 4, - "uploadDate": "*actual", - "filename": "filename" - } - ] - }, - { - "insert": "expected.chunks", - "documents": [ - { - "_id": "*actual", - "files_id": "*result", - "n": 0, - "data": { - "$hex": "112233" + }, + "expectResult": { + "$$type": "objectId" + }, + "saveResultAsEntity": "uploadedObjectId" + }, + { + "name": "find", + "object": "bucket0_files_collection", + "arguments": { + "filter": {} + }, + "expectResult": [ + { + "_id": { + "$$matchesEntity": "uploadedObjectId" + }, + "length": 3, + "chunkSize": 4, + "uploadDate": { + "$$type": "date" + }, + "md5": { + "$$unsetOrMatches": "bafae3a174ab91fc70db7a6aa50f4f52" + }, + "filename": "filename" + } + ] + }, + { + "name": "find", + "object": "bucket0_chunks_collection", + "arguments": { + "filter": {} + }, + "expectResult": [ + { + "_id": { + "$$type": "objectId" + }, + "files_id": { + "$$matchesEntity": "uploadedObjectId" + }, + "n": 0, + "data": { + "$binary": { + "base64": "ESIz", + "subType": "00" } } - ] - } - ] - } + } + ] + } + ] }, { - "description": "Upload when length is 4", - "act": { - "operation": "upload", - "arguments": { - "filename": "filename", - "source": { - "$hex": "11223344" - }, - "options": { + "description": "upload when length is 4", + "operations": [ + { + "name": "upload", + "object": "bucket0", + "arguments": { + "filename": "filename", + "source": { + "$$hexBytes": "11223344" + }, "chunkSizeBytes": 4 - } - } - }, - "assert": { - "result": "&result", - "data": [ - { - "insert": "expected.files", - "documents": [ - { - "_id": "*result", - "length": 4, - "chunkSize": 4, - "uploadDate": "*actual", - "filename": "filename" - } - ] - }, - { - "insert": "expected.chunks", - "documents": [ - { - "_id": "*actual", - "files_id": "*result", - "n": 0, - "data": { - "$hex": "11223344" + }, + "expectResult": { + "$$type": "objectId" + }, + "saveResultAsEntity": "uploadedObjectId" + }, + { + "name": "find", + "object": "bucket0_files_collection", + "arguments": { + "filter": {} + }, + "expectResult": [ + { + "_id": { + "$$matchesEntity": "uploadedObjectId" + }, + "length": 4, + "chunkSize": 4, + "uploadDate": { + "$$type": "date" + }, + "md5": { + "$$unsetOrMatches": "7e7c77cff5705d1f7574a25ef6662117" + }, + "filename": "filename" + } + ] + }, + { + "name": "find", + "object": "bucket0_chunks_collection", + "arguments": { + "filter": {} + }, + "expectResult": [ + { + "_id": { + "$$type": "objectId" + }, + "files_id": { + "$$matchesEntity": "uploadedObjectId" + }, + "n": 0, + "data": { + "$binary": { + "base64": "ESIzRA==", + "subType": "00" } } - ] - } - ] - } + } + ] + } + ] }, { - "description": "Upload when length is 5", - "act": { - "operation": "upload", - "arguments": { - "filename": "filename", - "source": { - "$hex": "1122334455" - }, - "options": { + "description": "upload when length is 5", + "operations": [ + { + "name": "upload", + "object": "bucket0", + "arguments": { + "filename": "filename", + "source": { + "$$hexBytes": "1122334455" + }, "chunkSizeBytes": 4 - } - } - }, - "assert": { - "result": "&result", - "data": [ - { - "insert": "expected.files", - "documents": [ - { - "_id": "*result", - "length": 5, - "chunkSize": 4, - "uploadDate": "*actual", - "filename": "filename" - } - ] - }, - { - "insert": "expected.chunks", - "documents": [ - { - "_id": "*actual", - "files_id": "*result", - "n": 0, - "data": { - "$hex": "11223344" + }, + "expectResult": { + "$$type": "objectId" + }, + "saveResultAsEntity": "uploadedObjectId" + }, + { + "name": "find", + "object": "bucket0_files_collection", + "arguments": { + "filter": {} + }, + "expectResult": [ + { + "_id": { + "$$matchesEntity": "uploadedObjectId" + }, + "length": 5, + "chunkSize": 4, + "uploadDate": { + "$$type": "date" + }, + "md5": { + "$$unsetOrMatches": "283d4fea5dded59cf837d3047328f5af" + }, + "filename": "filename" + } + ] + }, + { + "name": "find", + "object": "bucket0_chunks_collection", + "arguments": { + "filter": {}, + "sort": { + "n": 1 + } + }, + "expectResult": [ + { + "_id": { + "$$type": "objectId" + }, + "files_id": { + "$$matchesEntity": "uploadedObjectId" + }, + "n": 0, + "data": { + "$binary": { + "base64": "ESIzRA==", + "subType": "00" } + } + }, + { + "_id": { + "$$type": "objectId" }, - { - "_id": "*actual", - "files_id": "*result", - "n": 1, - "data": { - "$hex": "55" + "files_id": { + "$$matchesEntity": "uploadedObjectId" + }, + "n": 1, + "data": { + "$binary": { + "base64": "VQ==", + "subType": "00" } } - ] - } - ] - } + } + ] + } + ] }, { - "description": "Upload when length is 8", - "act": { - "operation": "upload", - "arguments": { - "filename": "filename", - "source": { - "$hex": "1122334455667788" - }, - "options": { + "description": "upload when length is 8", + "operations": [ + { + "name": "upload", + "object": "bucket0", + "arguments": { + "filename": "filename", + "source": { + "$$hexBytes": "1122334455667788" + }, "chunkSizeBytes": 4 - } - } - }, - "assert": { - "result": "&result", - "data": [ - { - "insert": "expected.files", - "documents": [ - { - "_id": "*result", - "length": 8, - "chunkSize": 4, - "uploadDate": "*actual", - "filename": "filename" - } - ] - }, - { - "insert": "expected.chunks", - "documents": [ - { - "_id": "*actual", - "files_id": "*result", - "n": 0, - "data": { - "$hex": "11223344" + }, + "expectResult": { + "$$type": "objectId" + }, + "saveResultAsEntity": "uploadedObjectId" + }, + { + "name": "find", + "object": "bucket0_files_collection", + "arguments": { + "filter": {} + }, + "expectResult": [ + { + "_id": { + "$$matchesEntity": "uploadedObjectId" + }, + "length": 8, + "chunkSize": 4, + "uploadDate": { + "$$type": "date" + }, + "md5": { + "$$unsetOrMatches": "dd254cdc958e53abaa67da9f797125f5" + }, + "filename": "filename" + } + ] + }, + { + "name": "find", + "object": "bucket0_chunks_collection", + "arguments": { + "filter": {}, + "sort": { + "n": 1 + } + }, + "expectResult": [ + { + "_id": { + "$$type": "objectId" + }, + "files_id": { + "$$matchesEntity": "uploadedObjectId" + }, + "n": 0, + "data": { + "$binary": { + "base64": "ESIzRA==", + "subType": "00" } + } + }, + { + "_id": { + "$$type": "objectId" + }, + "files_id": { + "$$matchesEntity": "uploadedObjectId" }, - { - "_id": "*actual", - "files_id": "*result", - "n": 1, - "data": { - "$hex": "55667788" + "n": 1, + "data": { + "$binary": { + "base64": "VWZ3iA==", + "subType": "00" } } - ] - } - ] - } + } + ] + } + ] }, { - "description": "Upload when contentType is provided", - "act": { - "operation": "upload", - "arguments": { - "filename": "filename", - "source": { - "$hex": "11" - }, - "options": { + "description": "upload when contentType is provided", + "operations": [ + { + "name": "upload", + "object": "bucket0", + "arguments": { + "filename": "filename", + "source": { + "$$hexBytes": "11" + }, "chunkSizeBytes": 4, "contentType": "image/jpeg" - } - } - }, - "assert": { - "result": "&result", - "data": [ - { - "insert": "expected.files", - "documents": [ - { - "_id": "*result", - "length": 1, - "chunkSize": 4, - "uploadDate": "*actual", - "filename": "filename", - "contentType": "image/jpeg" - } - ] - }, - { - "insert": "expected.chunks", - "documents": [ - { - "_id": "*actual", - "files_id": "*result", - "n": 0, - "data": { - "$hex": "11" + }, + "expectResult": { + "$$type": "objectId" + }, + "saveResultAsEntity": "uploadedObjectId" + }, + { + "name": "find", + "object": "bucket0_files_collection", + "arguments": { + "filter": {} + }, + "expectResult": [ + { + "_id": { + "$$matchesEntity": "uploadedObjectId" + }, + "length": 1, + "chunkSize": 4, + "uploadDate": { + "$$type": "date" + }, + "md5": { + "$$unsetOrMatches": "47ed733b8d10be225eceba344d533586" + }, + "filename": "filename", + "contentType": "image/jpeg" + } + ] + }, + { + "name": "find", + "object": "bucket0_chunks_collection", + "arguments": { + "filter": {} + }, + "expectResult": [ + { + "_id": { + "$$type": "objectId" + }, + "files_id": { + "$$matchesEntity": "uploadedObjectId" + }, + "n": 0, + "data": { + "$binary": { + "base64": "EQ==", + "subType": "00" } } - ] - } - ] - } + } + ] + } + ] }, { - "description": "Upload when metadata is provided", - "act": { - "operation": "upload", - "arguments": { - "filename": "filename", - "source": { - "$hex": "11" - }, - "options": { + "description": "upload when metadata is provided", + "operations": [ + { + "name": "upload", + "object": "bucket0", + "arguments": { + "filename": "filename", + "source": { + "$$hexBytes": "11" + }, "chunkSizeBytes": 4, "metadata": { "x": 1 } - } - } - }, - "assert": { - "result": "&result", - "data": [ - { - "insert": "expected.files", - "documents": [ - { - "_id": "*result", - "length": 1, - "chunkSize": 4, - "uploadDate": "*actual", - "filename": "filename", - "metadata": { - "x": 1 - } + }, + "expectResult": { + "$$type": "objectId" + }, + "saveResultAsEntity": "uploadedObjectId" + }, + { + "name": "find", + "object": "bucket0_files_collection", + "arguments": { + "filter": {} + }, + "expectResult": [ + { + "_id": { + "$$matchesEntity": "uploadedObjectId" + }, + "length": 1, + "chunkSize": 4, + "uploadDate": { + "$$type": "date" + }, + "md5": { + "$$unsetOrMatches": "47ed733b8d10be225eceba344d533586" + }, + "filename": "filename", + "metadata": { + "x": 1 } - ] - }, - { - "insert": "expected.chunks", - "documents": [ - { - "_id": "*actual", - "files_id": "*result", - "n": 0, - "data": { - "$hex": "11" + } + ] + }, + { + "name": "find", + "object": "bucket0_chunks_collection", + "arguments": { + "filter": {} + }, + "expectResult": [ + { + "_id": { + "$$type": "objectId" + }, + "files_id": { + "$$matchesEntity": "uploadedObjectId" + }, + "n": 0, + "data": { + "$binary": { + "base64": "EQ==", + "subType": "00" } } - ] - } - ] - } + } + ] + } + ] } ] } diff --git a/test/test_gridfs_spec.py b/test/test_gridfs_spec.py index 3c6f6b76c4..d080c05c4d 100644 --- a/test/test_gridfs_spec.py +++ b/test/test_gridfs_spec.py @@ -1,4 +1,4 @@ -# Copyright 2015 MongoDB, Inc. +# Copyright 2015-present MongoDB, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,221 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Test GridFSBucket class.""" +"""Test the GridFS unified spec tests.""" -import copy -import datetime import os -import re import sys -from json import loads sys.path[0:0] = [""] -from test import IntegrationTest, unittest - -import gridfs -from bson import Binary -from bson.int64 import Int64 -from bson.json_util import object_hook -from gridfs.errors import CorruptGridFile, NoFile - -# Commands. -_COMMANDS = { - "delete": lambda coll, doc: [coll.delete_many(d["q"]) for d in doc["deletes"]], - "insert": lambda coll, doc: coll.insert_many(doc["documents"]), - "update": lambda coll, doc: [coll.update_many(u["q"], u["u"]) for u in doc["updates"]], -} +from test import unittest +from test.unified_format import generate_test_classes # Location of JSON test specifications. -_TEST_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "gridfs") - - -def camel_to_snake(camel): - # Regex to convert CamelCase to snake_case. Special case for _id. - if camel == "id": - return "file_id" - snake = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", camel) - return re.sub("([a-z0-9])([A-Z])", r"\1_\2", snake).lower() - - -class TestAllScenarios(IntegrationTest): - fs: gridfs.GridFSBucket - str_to_cmd: dict - - @classmethod - def setUpClass(cls): - super(TestAllScenarios, cls).setUpClass() - cls.fs = gridfs.GridFSBucket(cls.db) - cls.str_to_cmd = { - "upload": cls.fs.upload_from_stream, - "download": cls.fs.open_download_stream, - "delete": cls.fs.delete, - "download_by_name": cls.fs.open_download_stream_by_name, - } - - def init_db(self, data, test): - self.cleanup_colls( - self.db.fs.files, self.db.fs.chunks, self.db.expected.files, self.db.expected.chunks - ) - - # Read in data. - if data["files"]: - self.db.fs.files.insert_many(data["files"]) - self.db.expected.files.insert_many(data["files"]) - if data["chunks"]: - self.db.fs.chunks.insert_many(data["chunks"]) - self.db.expected.chunks.insert_many(data["chunks"]) - - # Make initial modifications. - if "arrange" in test: - for cmd in test["arrange"].get("data", []): - for key in cmd.keys(): - if key in _COMMANDS: - coll = self.db.get_collection(cmd[key]) - _COMMANDS[key](coll, cmd) - - def init_expected_db(self, test, result): - # Modify outcome DB. - for cmd in test["assert"].get("data", []): - for key in cmd.keys(): - if key in _COMMANDS: - # Replace wildcards in inserts. - for doc in cmd.get("documents", []): - keylist = doc.keys() - for dockey in copy.deepcopy(list(keylist)): - if "result" in str(doc[dockey]): - doc[dockey] = result - if "actual" in str(doc[dockey]): # Avoid duplicate - doc.pop(dockey) - # Move contentType to metadata. - if dockey == "contentType": - doc["metadata"] = {dockey: doc.pop(dockey)} - coll = self.db.get_collection(cmd[key]) - _COMMANDS[key](coll, cmd) - - if test["assert"].get("result") == "&result": - test["assert"]["result"] = result - - def sorted_list(self, coll, ignore_id): - to_sort = [] - for doc in coll.find(): - docstr = "{" - if ignore_id: # Cannot compare _id in chunks collection. - doc.pop("_id") - for k in sorted(doc.keys()): - if k == "uploadDate": # Can't compare datetime. - self.assertTrue(isinstance(doc[k], datetime.datetime)) - else: - docstr += "%s:%s " % (k, repr(doc[k])) - to_sort.append(docstr + "}") - return to_sort - - -def create_test(scenario_def): - def run_scenario(self): - - # Run tests. - self.assertTrue(scenario_def["tests"], "tests cannot be empty") - for test in scenario_def["tests"]: - self.init_db(scenario_def["data"], test) - - # Run GridFs Operation. - operation = self.str_to_cmd[test["act"]["operation"]] - args = test["act"]["arguments"] - extra_opts = args.pop("options", {}) - if "contentType" in extra_opts: - extra_opts["metadata"] = {"contentType": extra_opts.pop("contentType")} - - args.update(extra_opts) - - converted_args = dict((camel_to_snake(c), v) for c, v in args.items()) - - expect_error = test["assert"].get("error", False) - result = None - error = None - try: - result = operation(**converted_args) - - if "download" in test["act"]["operation"]: - result = Binary(result.read()) - except Exception as exc: - if not expect_error: - raise - error = exc - - self.init_expected_db(test, result) - - # Asserts. - errors = { - "FileNotFound": NoFile, - "ChunkIsMissing": CorruptGridFile, - "ExtraChunk": CorruptGridFile, - "ChunkIsWrongSize": CorruptGridFile, - "RevisionNotFound": NoFile, - } - - if expect_error: - self.assertIsNotNone(error) - self.assertIsInstance(error, errors[test["assert"]["error"]], test["description"]) - else: - self.assertIsNone(error) - - if "result" in test["assert"]: - if test["assert"]["result"] == "void": - test["assert"]["result"] = None - self.assertEqual(result, test["assert"].get("result")) - - if "data" in test["assert"]: - # Create alphabetized list - self.assertEqual( - set(self.sorted_list(self.db.fs.chunks, True)), - set(self.sorted_list(self.db.expected.chunks, True)), - ) - - self.assertEqual( - set(self.sorted_list(self.db.fs.files, False)), - set(self.sorted_list(self.db.expected.files, False)), - ) - - return run_scenario - - -def _object_hook(dct): - if "length" in dct: - dct["length"] = Int64(dct["length"]) - return object_hook(dct) - - -def create_tests(): - for dirpath, _, filenames in os.walk(_TEST_PATH): - for filename in filenames: - with open(os.path.join(dirpath, filename)) as scenario_stream: - scenario_def = loads(scenario_stream.read(), object_hook=_object_hook) - - # Because object_hook is already defined by bson.json_util, - # and everything is named 'data' - def str2hex(jsn): - for key, val in jsn.items(): - if key in ("data", "source", "result"): - if "$hex" in val: - jsn[key] = Binary(bytes.fromhex(val["$hex"])) - if isinstance(jsn[key], dict): - str2hex(jsn[key]) - if isinstance(jsn[key], list): - for k in jsn[key]: - str2hex(k) - - str2hex(scenario_def) - - # Construct test from scenario. - new_test = create_test(scenario_def) - test_name = "test_%s" % (os.path.splitext(filename)[0]) - new_test.__name__ = test_name - setattr(TestAllScenarios, new_test.__name__, new_test) - +TEST_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "gridfs") -create_tests() +# Generate unified tests. +globals().update(generate_test_classes(TEST_PATH, module=__name__)) if __name__ == "__main__": unittest.main() diff --git a/test/unified_format.py b/test/unified_format.py index e37bc1bb6d..002a71322f 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -16,6 +16,7 @@ https://github.com/mongodb/specifications/blob/master/source/unified-test-format/unified-test-format.rst """ +import binascii import collections import copy import datetime @@ -457,8 +458,10 @@ def _create_entity(self, entity_spec, uri=None): self.test.addCleanup(session.end_session) return elif entity_type == "bucket": - # TODO: implement the 'bucket' entity type - self.test.skipTest("GridFS is not currently supported (PYTHON-2459)") + db = self[spec["database"]] + kwargs = parse_spec_options(spec.get("bucketOptions", {}).copy()) + self[spec["id"]] = GridFSBucket(db, **kwargs) + return elif entity_type == "clientEncryption": opts = camel_to_snake_args(spec["clientEncryptionOpts"].copy()) if isinstance(opts["key_vault_client"], str): @@ -575,11 +578,12 @@ def _operation_type(self, spec, actual, key_to_compare): def _operation_matchesEntity(self, spec, actual, key_to_compare): expected_entity = self.test.entity_map[spec] - self.test.assertIsInstance(expected_entity, abc.Mapping) self.test.assertEqual(expected_entity, actual[key_to_compare]) def _operation_matchesHexBytes(self, spec, actual, key_to_compare): - raise NotImplementedError + expected = binascii.unhexlify(spec) + value = actual[key_to_compare] if key_to_compare else actual + self.test.assertEqual(value, expected) def _operation_unsetOrMatches(self, spec, actual, key_to_compare): if key_to_compare is None and not actual: @@ -908,17 +912,9 @@ def maybe_skip_test(self, spec): self.skipTest("Test commands must be enabled to use fail points") if "timeoutMode" in op.get("arguments", {}): self.skipTest("PyMongo does not support timeoutMode") - if name == "createEntities": - self.maybe_skip_entity(op.get("arguments", {}).get("entities", [])) if name == "modifyCollection": self.skipTest("PyMongo does not support modifyCollection") - def maybe_skip_entity(self, entities): - for entity in entities: - entity_type = next(iter(entity)) - if entity_type == "bucket": - self.skipTest("GridFS is not currently supported (PYTHON-2459)") - def process_error(self, exception, spec): is_error = spec.get("isError") is_client_error = spec.get("isClientError") @@ -1116,9 +1112,30 @@ def _clientEncryptionOperation_rewrapManyDataKey(self, target, *args, **kwargs): return dict(bulkWriteResult=parse_bulk_write_result(data.bulk_write_result)) return dict() + def _bucketOperation_download(self, target: GridFSBucket, *args: Any, **kwargs: Any) -> bytes: + with target.open_download_stream(*args, **kwargs) as gout: + return gout.read() + + def _bucketOperation_downloadByName( + self, target: GridFSBucket, *args: Any, **kwargs: Any + ) -> bytes: + with target.open_download_stream_by_name(*args, **kwargs) as gout: + return gout.read() + + def _bucketOperation_upload(self, target: GridFSBucket, *args: Any, **kwargs: Any) -> ObjectId: + kwargs["source"] = binascii.unhexlify(kwargs.pop("source")["$$hexBytes"]) + if "content_type" in kwargs: + kwargs.setdefault("metadata", {})["contentType"] = kwargs.pop("content_type") + return target.upload_from_stream(*args, **kwargs) + + def _bucketOperation_uploadWithId(self, target: GridFSBucket, *args: Any, **kwargs: Any) -> Any: + kwargs["source"] = binascii.unhexlify(kwargs.pop("source")["$$hexBytes"]) + if "content_type" in kwargs: + kwargs.setdefault("metadata", {})["contentType"] = kwargs.pop("content_type") + return target.upload_from_stream_with_id(*args, **kwargs) + def run_entity_operation(self, spec): target = self.entity_map[spec["object"]] - client = target opname = spec["name"] opargs = spec.get("arguments") expect_error = spec.get("expectError") @@ -1144,6 +1161,11 @@ def run_entity_operation(self, spec): method_name = "_databaseOperation_%s" % (opname,) elif isinstance(target, Collection): method_name = "_collectionOperation_%s" % (opname,) + # contentType is always stored in metadata in pymongo. + if target.name.endswith(".files") and opname == "find": + for doc in spec.get("expectResult", []): + if "contentType" in doc: + doc.setdefault("metadata", {})["contentType"] = doc.pop("contentType") elif isinstance(target, ChangeStream): method_name = "_changeStreamOperation_%s" % (opname,) elif isinstance(target, NonLazyCursor): @@ -1151,7 +1173,11 @@ def run_entity_operation(self, spec): elif isinstance(target, ClientSession): method_name = "_sessionOperation_%s" % (opname,) elif isinstance(target, GridFSBucket): - raise NotImplementedError + method_name = "_bucketOperation_%s" % (opname,) + if "id" in arguments: + arguments["file_id"] = arguments.pop("id") + # MD5 is always disabled in pymongo. + arguments.pop("disable_md5", None) elif isinstance(target, ClientEncryption): method_name = "_clientEncryptionOperation_%s" % (opname,) else: From e97d9f34f43905c8816caee479501878ecb68a7c Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Tue, 19 Jul 2022 12:12:45 -0700 Subject: [PATCH 2/3] PYTHON-2459 Fix test runner findOne maxTimeMS --- test/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/utils.py b/test/utils.py index 5421d584d1..29ee1ca477 100644 --- a/test/utils.py +++ b/test/utils.py @@ -1087,7 +1087,7 @@ def prepare_spec_arguments(spec, arguments, opname, entity_map, with_txn_callbac arguments["session"] = entity_map[arguments["session"]] elif opname == "open_download_stream" and arg_name == "id": arguments["file_id"] = arguments.pop(arg_name) - elif opname != "find" and c2s == "max_time_ms": + elif opname not in ("find", "find_one") and c2s == "max_time_ms": # find is the only method that accepts snake_case max_time_ms. # All other methods take kwargs which must use the server's # camelCase maxTimeMS. See PYTHON-1855. From 30c5686e648c7ff66a7c1a2f8340b4ad4620497b Mon Sep 17 00:00:00 2001 From: Shane Harvey Date: Tue, 19 Jul 2022 13:02:29 -0700 Subject: [PATCH 3/3] PYTHON-2459 Keep skipping CSOT gridfs tests for now --- test/unified_format.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/test/unified_format.py b/test/unified_format.py index 002a71322f..d36b5d0a48 100644 --- a/test/unified_format.py +++ b/test/unified_format.py @@ -910,10 +910,21 @@ def maybe_skip_test(self, spec): if not client_context.test_commands_enabled: if name == "failPoint" or name == "targetedFailPoint": self.skipTest("Test commands must be enabled to use fail points") - if "timeoutMode" in op.get("arguments", {}): - self.skipTest("PyMongo does not support timeoutMode") if name == "modifyCollection": self.skipTest("PyMongo does not support modifyCollection") + if "timeoutMode" in op.get("arguments", {}): + self.skipTest("PyMongo does not support timeoutMode") + if "csot" in class_name: + if "bucket" in op["object"]: + self.skipTest("CSOT not implemented for GridFS") + if name == "createEntities": + self.maybe_skip_entity(op.get("arguments", {}).get("entities", [])) + + def maybe_skip_entity(self, entities): + for entity in entities: + entity_type = next(iter(entity)) + if entity_type == "bucket": + self.skipTest("GridFS is not currently supported (PYTHON-2459)") def process_error(self, exception, spec): is_error = spec.get("isError") @@ -1134,6 +1145,11 @@ def _bucketOperation_uploadWithId(self, target: GridFSBucket, *args: Any, **kwar kwargs.setdefault("metadata", {})["contentType"] = kwargs.pop("content_type") return target.upload_from_stream_with_id(*args, **kwargs) + def _bucketOperation_drop(self, target: GridFSBucket, *args: Any, **kwargs: Any) -> None: + # PyMongo does not support GridFSBucket.drop(), emulate it. + target._files.drop(*args, **kwargs) + target._chunks.drop(*args, **kwargs) + def run_entity_operation(self, spec): target = self.entity_map[spec["object"]] opname = spec["name"]