diff --git a/sample-configs/config-diskann-fv-paramsweep.json b/sample-configs/config-diskann-fv-paramsweep.json new file mode 100644 index 000000000..d9610da99 --- /dev/null +++ b/sample-configs/config-diskann-fv-paramsweep.json @@ -0,0 +1,113 @@ +{ + "database": { + "host": "${HORIZONDB_HOST}", + "username": "${HORIZONDB_USERNAME}", + "password": "${HORIZONDB_PASSWORD}", + "db-name": "postgres" + }, + "benchmark-info": { + "name": "ann-benchmark-param-sweep", + "instance-size": "HorizonDB-10Core-64GB", + "instance-service": "horizondb", + "provider": "azure-horizondb", + "description": "Running param sweep for DiskANN full vector with pg_diskann v0.5.0" + }, + "cases": [ + { + "db-label": "diskann-fv-mn32-ib64-param-sweep", + "vdb-command": "pgdiskann", + "vector-ext": "pg_diskann", + "index-type": "diskann-fv", + "case-type": "Performance768D10M", + "drop-old": true, + "load": true, + "search-serial": false, + "search-concurrent": false, + "index-params": { + "max-neighbors": 32, + "l-value-ib": 64, + "maintenance-work-mem": "75GB", + "max-parallel-workers": 9 + }, + "search-params": { + "l-value-is": [32, 64, 128, 256, 512] + }, + "num-concurrency": "1,10,20,30,40,50,60,70,80,90,100", + "concurrency-duration": 30, + "k": 10, + "run-count": 1 + }, + { + "db-label": "diskann-fv-mn32-ib128-param-sweep", + "vdb-command": "pgdiskann", + "vector-ext": "pg_diskann", + "index-type": "diskann-fv", + "case-type": "Performance768D10M", + "drop-old": true, + "load": true, + "search-serial": false, + "search-concurrent": false, + "index-params": { + "max-neighbors": 32, + "l-value-ib": 128, + "maintenance-work-mem": "75GB", + "max-parallel-workers": 9 + }, + "search-params": { + "l-value-is": [32, 64, 128, 256, 512] + }, + "num-concurrency": "1,10,20,30,40,50,60,70,80,90,100", + "concurrency-duration": 30, + "k": 10, + "run-count": 1 + }, + { + "db-label": "diskann-fv-mn64-ib64-param-sweep", + "vdb-command": "pgdiskann", + "vector-ext": "pg_diskann", + "index-type": "diskann-fv", + "case-type": "Performance768D10M", + "drop-old": true, + "load": true, + "search-serial": false, + "search-concurrent": false, + "index-params": { + "max-neighbors": 64, + "l-value-ib": 64, + "maintenance-work-mem": "75GB", + "max-parallel-workers": 9 + }, + "search-params": { + "l-value-is": [32, 64, 128, 256, 512] + }, + "num-concurrency": "1,10,20,30,40,50,60,70,80,90,100", + "concurrency-duration": 30, + "k": 10, + "run-count": 1 + }, + { + "db-label": "diskann-fv-mn64-ib128-param-sweep", + "vdb-command": "pgdiskann", + "vector-ext": "pg_diskann", + "index-type": "diskann-fv", + "case-type": "Performance768D10M", + "drop-old": true, + "load": true, + "search-serial": false, + "search-concurrent": false, + "index-params": { + "max-neighbors": 64, + "l-value-ib": 128, + "maintenance-work-mem": "75GB", + "max-parallel-workers": 9 + }, + "search-params": { + "l-value-is": [32, 64, 128, 256, 512] + }, + "num-concurrency": "1,10,20,30,40,50,60,70,80,90,100", + "concurrency-duration": 30, + "k": 10, + "run-count": 1 + } + ] +} \ No newline at end of file diff --git a/sample-configs/config-hnsw-bq-paramsweep.json b/sample-configs/config-hnsw-bq-paramsweep.json new file mode 100644 index 000000000..2683faf6a --- /dev/null +++ b/sample-configs/config-hnsw-bq-paramsweep.json @@ -0,0 +1,494 @@ +{ + "database": { + "host": "${HORIZONDB_HOST}", + "username": "${HORIZONDB_USERNAME}", + "password": "${HORIZONDB_PASSWORD}", + "db-name": "postgres" + }, + "benchmark-info": { + "name": "ann-benchmark-param-sweep", + "instance-size": "HorizonDB-10Core-64GB", + "instance-service": "horizondb", + "provider": "azure-horizondb", + "description": "Running param sweep for HNSW binary quantization with reranking using pgvector v0.8.0" + }, + "cases": [ + { + "db-label": "hnsw-bq-m8-efc32-param-sweep", + "vdb-command": "pgvectorhnsw", + "vector-ext": "vector", + "index-type": "hnsw-bq", + "case-type": "Performance768D10M", + "drop-old": true, + "load": true, + "search-serial": false, + "search-concurrent": false, + "index-params": { + "m": 8, + "ef-construction": 32, + "maintenance-work-mem": "75GB", + "max-parallel-workers": 9, + "quantization-type": "bit" + }, + "search-params": { + "ef-search": [40, 80, 120, 200, 400, 800] + }, + "reranking": true, + "num-concurrency": "1,10,20,30,40,50,60,70,80,90,100", + "concurrency-duration": 30, + "k": 10, + "run-count": 1 + }, + { + "db-label": "hnsw-bq-m8-efc64-param-sweep", + "vdb-command": "pgvectorhnsw", + "vector-ext": "vector", + "index-type": "hnsw-bq", + "case-type": "Performance768D10M", + "drop-old": true, + "load": true, + "search-serial": false, + "search-concurrent": false, + "index-params": { + "m": 8, + "ef-construction": 64, + "maintenance-work-mem": "75GB", + "max-parallel-workers": 9, + "quantization-type": "bit" + }, + "search-params": { + "ef-search": [40, 80, 120, 200, 400, 800] + }, + "reranking": true, + "num-concurrency": "1,10,20,30,40,50,60,70,80,90,100", + "concurrency-duration": 30, + "k": 10, + "run-count": 1 + }, + { + "db-label": "hnsw-bq-m8-efc128-param-sweep", + "vdb-command": "pgvectorhnsw", + "vector-ext": "vector", + "index-type": "hnsw-bq", + "case-type": "Performance768D10M", + "drop-old": true, + "load": true, + "search-serial": false, + "search-concurrent": false, + "index-params": { + "m": 8, + "ef-construction": 128, + "maintenance-work-mem": "75GB", + "max-parallel-workers": 9, + "quantization-type": "bit" + }, + "search-params": { + "ef-search": [40, 80, 120, 200, 400, 800] + }, + "reranking": true, + "num-concurrency": "1,10,20,30,40,50,60,70,80,90,100", + "concurrency-duration": 30, + "k": 10, + "run-count": 1 + }, + { + "db-label": "hnsw-bq-m16-efc32-param-sweep", + "vdb-command": "pgvectorhnsw", + "vector-ext": "vector", + "index-type": "hnsw-bq", + "case-type": "Performance768D10M", + "drop-old": true, + "load": true, + "search-serial": false, + "search-concurrent": false, + "index-params": { + "m": 16, + "ef-construction": 32, + "maintenance-work-mem": "75GB", + "max-parallel-workers": 9, + "quantization-type": "bit" + }, + "search-params": { + "ef-search": [40, 80, 120, 200, 400, 800] + }, + "reranking": true, + "num-concurrency": "1,10,20,30,40,50,60,70,80,90,100", + "concurrency-duration": 30, + "k": 10, + "run-count": 1 + }, + { + "db-label": "hnsw-bq-m16-efc64-param-sweep", + "vdb-command": "pgvectorhnsw", + "vector-ext": "vector", + "index-type": "hnsw-bq", + "case-type": "Performance768D10M", + "drop-old": true, + "load": true, + "search-serial": false, + "search-concurrent": false, + "index-params": { + "m": 16, + "ef-construction": 64, + "maintenance-work-mem": "75GB", + "max-parallel-workers": 9, + "quantization-type": "bit" + }, + "search-params": { + "ef-search": [40, 80, 120, 200, 400, 800] + }, + "reranking": true, + "num-concurrency": "1,10,20,30,40,50,60,70,80,90,100", + "concurrency-duration": 30, + "k": 10, + "run-count": 1 + }, + { + "db-label": "hnsw-bq-m16-efc128-param-sweep", + "vdb-command": "pgvectorhnsw", + "vector-ext": "vector", + "index-type": "hnsw-bq", + "case-type": "Performance768D10M", + "drop-old": true, + "load": true, + "search-serial": false, + "search-concurrent": false, + "index-params": { + "m": 16, + "ef-construction": 128, + "maintenance-work-mem": "75GB", + "max-parallel-workers": 9, + "quantization-type": "bit" + }, + "search-params": { + "ef-search": [40, 80, 120, 200, 400, 800] + }, + "reranking": true, + "num-concurrency": "1,10,20,30,40,50,60,70,80,90,100", + "concurrency-duration": 30, + "k": 10, + "run-count": 1 + }, + { + "db-label": "hnsw-bq-m32-efc64-param-sweep", + "vdb-command": "pgvectorhnsw", + "vector-ext": "vector", + "index-type": "hnsw-bq", + "case-type": "Performance768D10M", + "drop-old": true, + "load": true, + "search-serial": false, + "search-concurrent": false, + "index-params": { + "m": 32, + "ef-construction": 64, + "maintenance-work-mem": "75GB", + "max-parallel-workers": 9, + "quantization-type": "bit" + }, + "search-params": { + "ef-search": [40, 80, 120, 200, 400, 800] + }, + "reranking": true, + "num-concurrency": "1,10,20,30,40,50,60,70,80,90,100", + "concurrency-duration": 30, + "k": 10, + "run-count": 1 + }, + { + "db-label": "hnsw-bq-m32-efc128-param-sweep", + "vdb-command": "pgvectorhnsw", + "vector-ext": "vector", + "index-type": "hnsw-bq", + "case-type": "Performance768D10M", + "drop-old": true, + "load": true, + "search-serial": false, + "search-concurrent": false, + "index-params": { + "m": 32, + "ef-construction": 128, + "maintenance-work-mem": "75GB", + "max-parallel-workers": 9, + "quantization-type": "bit" + }, + "search-params": { + "ef-search": [40, 80, 120, 200, 400, 800] + }, + "reranking": true, + "num-concurrency": "1,10,20,30,40,50,60,70,80,90,100", + "concurrency-duration": 30, + "k": 10, + "run-count": 1 + }, + { + "db-label": "hnsw-bq-m64-efc128-param-sweep", + "vdb-command": "pgvectorhnsw", + "vector-ext": "vector", + "index-type": "hnsw-bq", + "case-type": "Performance768D10M", + "drop-old": true, + "load": true, + "search-serial": false, + "search-concurrent": false, + "index-params": { + "m": 64, + "ef-construction": 128, + "maintenance-work-mem": "75GB", + "max-parallel-workers": 9, + "quantization-type": "bit" + }, + "search-params": { + "ef-search": [40, 80, 120, 200, 400, 800] + }, + "reranking": true, + "num-concurrency": "1,10,20,30,40,50,60,70,80,90,100", + "concurrency-duration": 30, + "k": 10, + "run-count": 1 + }, + { + "db-label": "hnsw-bq-m8-efc32-hqfl-param-sweep", + "vdb-command": "pgvectorhnsw", + "vector-ext": "vector", + "index-type": "hnsw-bq", + "case-type": "Performance768D10M", + "drop-old": true, + "load": true, + "search-serial": false, + "search-concurrent": false, + "index-params": { + "m": 8, + "ef-construction": 32, + "maintenance-work-mem": "75GB", + "max-parallel-workers": 9, + "quantization-type": "bit" + }, + "search-params": { + "ef-search": [40, 80, 120, 200, 400, 800] + }, + "reranking": true, + "half-quantized-fetch-limit": true, + "num-concurrency": "1,10,20,30,40,50,60,70,80,90,100", + "concurrency-duration": 30, + "k": 10, + "run-count": 1 + }, + { + "db-label": "hnsw-bq-m8-efc64-hqfl-param-sweep", + "vdb-command": "pgvectorhnsw", + "vector-ext": "vector", + "index-type": "hnsw-bq", + "case-type": "Performance768D10M", + "drop-old": true, + "load": true, + "search-serial": false, + "search-concurrent": false, + "index-params": { + "m": 8, + "ef-construction": 64, + "maintenance-work-mem": "75GB", + "max-parallel-workers": 9, + "quantization-type": "bit" + }, + "search-params": { + "ef-search": [40, 80, 120, 200, 400, 800] + }, + "reranking": true, + "half-quantized-fetch-limit": true, + "num-concurrency": "1,10,20,30,40,50,60,70,80,90,100", + "concurrency-duration": 30, + "k": 10, + "run-count": 1 + }, + { + "db-label": "hnsw-bq-m8-efc128-hqfl-param-sweep", + "vdb-command": "pgvectorhnsw", + "vector-ext": "vector", + "index-type": "hnsw-bq", + "case-type": "Performance768D10M", + "drop-old": true, + "load": true, + "search-serial": false, + "search-concurrent": false, + "index-params": { + "m": 8, + "ef-construction": 128, + "maintenance-work-mem": "75GB", + "max-parallel-workers": 9, + "quantization-type": "bit" + }, + "search-params": { + "ef-search": [40, 80, 120, 200, 400, 800] + }, + "reranking": true, + "half-quantized-fetch-limit": true, + "num-concurrency": "1,10,20,30,40,50,60,70,80,90,100", + "concurrency-duration": 30, + "k": 10, + "run-count": 1 + }, + { + "db-label": "hnsw-bq-m16-efc32-hqfl-param-sweep", + "vdb-command": "pgvectorhnsw", + "vector-ext": "vector", + "index-type": "hnsw-bq", + "case-type": "Performance768D10M", + "drop-old": true, + "load": true, + "search-serial": false, + "search-concurrent": false, + "index-params": { + "m": 16, + "ef-construction": 32, + "maintenance-work-mem": "75GB", + "max-parallel-workers": 9, + "quantization-type": "bit" + }, + "search-params": { + "ef-search": [40, 80, 120, 200, 400, 800] + }, + "reranking": true, + "half-quantized-fetch-limit": true, + "num-concurrency": "1,10,20,30,40,50,60,70,80,90,100", + "concurrency-duration": 30, + "k": 10, + "run-count": 1 + }, + { + "db-label": "hnsw-bq-m16-efc64-hqfl-param-sweep", + "vdb-command": "pgvectorhnsw", + "vector-ext": "vector", + "index-type": "hnsw-bq", + "case-type": "Performance768D10M", + "drop-old": true, + "load": true, + "search-serial": false, + "search-concurrent": false, + "index-params": { + "m": 16, + "ef-construction": 64, + "maintenance-work-mem": "75GB", + "max-parallel-workers": 9, + "quantization-type": "bit" + }, + "search-params": { + "ef-search": [40, 80, 120, 200, 400, 800] + }, + "reranking": true, + "half-quantized-fetch-limit": true, + "num-concurrency": "1,10,20,30,40,50,60,70,80,90,100", + "concurrency-duration": 30, + "k": 10, + "run-count": 1 + }, + { + "db-label": "hnsw-bq-m16-efc128-hqfl-param-sweep", + "vdb-command": "pgvectorhnsw", + "vector-ext": "vector", + "index-type": "hnsw-bq", + "case-type": "Performance768D10M", + "drop-old": true, + "load": true, + "search-serial": false, + "search-concurrent": false, + "index-params": { + "m": 16, + "ef-construction": 128, + "maintenance-work-mem": "75GB", + "max-parallel-workers": 9, + "quantization-type": "bit" + }, + "search-params": { + "ef-search": [40, 80, 120, 200, 400, 800] + }, + "reranking": true, + "half-quantized-fetch-limit": true, + "num-concurrency": "1,10,20,30,40,50,60,70,80,90,100", + "concurrency-duration": 30, + "k": 10, + "run-count": 1 + }, + { + "db-label": "hnsw-bq-m32-efc64-hqfl-param-sweep", + "vdb-command": "pgvectorhnsw", + "vector-ext": "vector", + "index-type": "hnsw-bq", + "case-type": "Performance768D10M", + "drop-old": true, + "load": true, + "search-serial": false, + "search-concurrent": false, + "index-params": { + "m": 32, + "ef-construction": 64, + "maintenance-work-mem": "75GB", + "max-parallel-workers": 9, + "quantization-type": "bit" + }, + "search-params": { + "ef-search": [40, 80, 120, 200, 400, 800] + }, + "reranking": true, + "half-quantized-fetch-limit": true, + "num-concurrency": "1,10,20,30,40,50,60,70,80,90,100", + "concurrency-duration": 30, + "k": 10, + "run-count": 1 + }, + { + "db-label": "hnsw-bq-m32-efc128-hqfl-param-sweep", + "vdb-command": "pgvectorhnsw", + "vector-ext": "vector", + "index-type": "hnsw-bq", + "case-type": "Performance768D10M", + "drop-old": true, + "load": true, + "search-serial": false, + "search-concurrent": false, + "index-params": { + "m": 32, + "ef-construction": 128, + "maintenance-work-mem": "75GB", + "max-parallel-workers": 9, + "quantization-type": "bit" + }, + "search-params": { + "ef-search": [40, 80, 120, 200, 400, 800] + }, + "reranking": true, + "half-quantized-fetch-limit": true, + "num-concurrency": "1,10,20,30,40,50,60,70,80,90,100", + "concurrency-duration": 30, + "k": 10, + "run-count": 1 + }, + { + "db-label": "hnsw-bq-m64-efc128-hqfl-param-sweep", + "vdb-command": "pgvectorhnsw", + "vector-ext": "vector", + "index-type": "hnsw-bq", + "case-type": "Performance768D10M", + "drop-old": true, + "load": true, + "search-serial": false, + "search-concurrent": false, + "index-params": { + "m": 64, + "ef-construction": 128, + "maintenance-work-mem": "75GB", + "max-parallel-workers": 9, + "quantization-type": "bit" + }, + "search-params": { + "ef-search": [40, 80, 120, 200, 400, 800] + }, + "reranking": true, + "half-quantized-fetch-limit": true, + "num-concurrency": "1,10,20,30,40,50,60,70,80,90,100", + "concurrency-duration": 30, + "k": 10, + "run-count": 1 + } + ] +} \ No newline at end of file diff --git a/sample-configs/config-hnsw-fv-paramsweep.json b/sample-configs/config-hnsw-fv-paramsweep.json new file mode 100644 index 000000000..e53074abc --- /dev/null +++ b/sample-configs/config-hnsw-fv-paramsweep.json @@ -0,0 +1,161 @@ +{ + "database": { + "host": "${HORIZONDB_HOST}", + "username": "${HORIZONDB_USERNAME}", + "password": "${HORIZONDB_PASSWORD}", + "db-name": "postgres" + }, + "benchmark-info": { + "name": "ann-benchmark-param-sweep", + "instance-size": "HorizonDB-10Core-64GB", + "instance-service": "horizondb", + "provider": "azure-horizondb", + "description": "Running param sweep for HNSW full vector with pgvector v0.8.0" + }, + "cases": [ + { + "db-label": "hnsw-fv-m16-efc32-param-sweep", + "vdb-command": "pgvectorhnsw", + "vector-ext": "vector", + "index-type": "hnsw-fv", + "case-type": "Performance768D10M", + "drop-old": true, + "load": true, + "search-serial": false, + "search-concurrent": false, + "index-params": { + "m": 16, + "ef-construction": 32, + "maintenance-work-mem": "75GB", + "max-parallel-workers": 9 + }, + "search-params": { + "ef-search": [10, 20, 40, 80, 120, 200, 400] + }, + "num-concurrency": "1,10,20,30,40,50,60,70,80,90,100", + "concurrency-duration": 30, + "k": 10, + "run-count": 1 + }, + { + "db-label": "hnsw-fv-m16-efc64-param-sweep", + "vdb-command": "pgvectorhnsw", + "vector-ext": "vector", + "index-type": "hnsw-fv", + "case-type": "Performance768D10M", + "drop-old": true, + "load": true, + "search-serial": false, + "search-concurrent": false, + "index-params": { + "m": 16, + "ef-construction": 64, + "maintenance-work-mem": "75GB", + "max-parallel-workers": 9 + }, + "search-params": { + "ef-search": [10, 20, 40, 80, 120, 200, 400] + }, + "num-concurrency": "1,10,20,30,40,50,60,70,80,90,100", + "concurrency-duration": 30, + "k": 10, + "run-count": 1 + }, + { + "db-label": "hnsw-fv-m16-efc128-param-sweep", + "vdb-command": "pgvectorhnsw", + "vector-ext": "vector", + "index-type": "hnsw-fv", + "case-type": "Performance768D10M", + "drop-old": true, + "load": true, + "search-serial": false, + "search-concurrent": false, + "index-params": { + "m": 16, + "ef-construction": 128, + "maintenance-work-mem": "75GB", + "max-parallel-workers": 9 + }, + "search-params": { + "ef-search": [10, 20, 40, 80, 120, 200, 400] + }, + "num-concurrency": "1,10,20,30,40,50,60,70,80,90,100", + "concurrency-duration": 30, + "k": 10, + "run-count": 1 + }, + { + "db-label": "hnsw-fv-m32-efc64-param-sweep", + "vdb-command": "pgvectorhnsw", + "vector-ext": "vector", + "index-type": "hnsw-fv", + "case-type": "Performance768D10M", + "drop-old": true, + "load": true, + "search-serial": false, + "search-concurrent": false, + "index-params": { + "m": 32, + "ef-construction": 64, + "maintenance-work-mem": "75GB", + "max-parallel-workers": 9 + }, + "search-params": { + "ef-search": [10, 20, 40, 80, 120, 200, 400] + }, + "num-concurrency": "1,10,20,30,40,50,60,70,80,90,100", + "concurrency-duration": 30, + "k": 10, + "run-count": 1 + }, + { + "db-label": "hnsw-fv-m32-efc128-param-sweep", + "vdb-command": "pgvectorhnsw", + "vector-ext": "vector", + "index-type": "hnsw-fv", + "case-type": "Performance768D10M", + "drop-old": true, + "load": true, + "search-serial": false, + "search-concurrent": false, + "index-params": { + "m": 32, + "ef-construction": 128, + "maintenance-work-mem": "75GB", + "max-parallel-workers": 9 + }, + "search-params": { + "ef-search": [10, 20, 40, 80, 120, 200, 400] + }, + "num-concurrency": "1,10,20,30,40,50,60,70,80,90,100", + "concurrency-duration": 30, + "k": 10, + "run-count": 1 + }, + { + "db-label": "hnsw-fv-m64-efc128-param-sweep", + "vdb-command": "pgvectorhnsw", + "vector-ext": "vector", + "index-type": "hnsw-fv", + "case-type": "Performance768D10M", + "drop-old": true, + "load": true, + "search-serial": false, + "search-concurrent": false, + "index-params": { + "m": 64, + "ef-construction": 128, + "maintenance-work-mem": "75GB", + "max-parallel-workers": 9 + }, + "search-params": { + "ef-search": [10, 20, 40, 80, 120, 200, 400] + }, + "num-concurrency": "1,10,20,30,40,50,60,70,80,90,100", + "concurrency-duration": 30, + "k": 10, + "run-count": 1 + } + ] +} \ No newline at end of file