diff --git a/.env.example b/.env.example index 618cac556..e495ea999 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,5 @@ -# LOG_LEVEL= -# LOG_PATH= -# LOG_NAME= +LOG_LEVEL=INFO +LOG_FILE="logs/vectordb_bench.log" # TIMEZONE= # NUM_PER_BATCH= diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index ea346dcd0..e9de2c3b9 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -4,6 +4,7 @@ on: pull_request: branches: - main + - vdbbench_* jobs: build: diff --git a/README.md b/README.md index 6d83671da..bd7568da1 100644 --- a/README.md +++ b/README.md @@ -1,19 +1,19 @@ -# VectorDBBench: A Benchmark Tool for VectorDB +# VectorDBBench(VDBBench): A Benchmark Tool for VectorDB [![version](https://img.shields.io/pypi/v/vectordb-bench.svg?color=blue)](https://pypi.org/project/vectordb-bench/) [![Downloads](https://pepy.tech/badge/vectordb-bench)](https://pepy.tech/project/vectordb-bench) -## What is VectorDBBench -VectorDBBench is not just an offering of benchmark results for mainstream vector databases and cloud services, it's your go-to tool for the ultimate performance and cost-effectiveness comparison. Designed with ease-of-use in mind, VectorDBBench is devised to help users, even non-professionals, reproduce results or test new systems, making the hunt for the optimal choice amongst a plethora of cloud services and open-source vector databases a breeze. +## What is VDBBench +VDBBench is not just an offering of benchmark results for mainstream vector databases and cloud services, it's your go-to tool for the ultimate performance and cost-effectiveness comparison. Designed with ease-of-use in mind, VDBBench is devised to help users, even non-professionals, reproduce results or test new systems, making the hunt for the optimal choice amongst a plethora of cloud services and open-source vector databases a breeze. Understanding the importance of user experience, we provide an intuitive visual interface. This not only empowers users to initiate benchmarks at ease, but also to view comparative result reports, thereby reproducing benchmark results effortlessly. To add more relevance and practicality, we provide cost-effectiveness reports particularly for cloud services. This allows for a more realistic and applicable benchmarking process. Closely mimicking real-world production environments, we've set up diverse testing scenarios including insertion, searching, and filtered searching. To provide you with credible and reliable data, we've included public datasets from actual production scenarios, such as [SIFT](http://corpus-texmex.irisa.fr/), [GIST](http://corpus-texmex.irisa.fr/), [Cohere](https://huggingface.co/datasets/Cohere/wikipedia-22-12/tree/main/en), and a dataset generated by OpenAI from an opensource [raw dataset](https://huggingface.co/datasets/allenai/c4). It's fascinating to discover how a relatively unknown open-source database might excel in certain circumstances! -Prepare to delve into the world of VectorDBBench, and let it guide you in uncovering your perfect vector database match. +Prepare to delve into the world of VDBBench, and let it guide you in uncovering your perfect vector database match. -VectorDBBench is sponsered by Zilliz，the leading opensource vectorDB company behind Milvus. Choose smarter with VectorDBBench- start your free test on [zilliz cloud](https://zilliz.com/) today! +VDBBench is sponsored by Zilliz，the leading opensource vectorDB company behind Milvus. Choose smarter with VDBBench - start your free test on [zilliz cloud](https://zilliz.com/) today! **Leaderboard:** https://zilliz.com/benchmark ## Quick Start @@ -30,12 +30,12 @@ pip install vectordb-bench **Install all database clients** ``` shell -pip install vectordb-bench[all] +pip install 'vectordb-bench[all]' ``` **Install the specific database client** ```shell -pip install vectordb-bench[pinecone] +pip install 'vectordb-bench[pinecone]' ``` All the database client supported @@ -52,11 +52,17 @@ All the database client supported | redis | `pip install vectordb-bench[redis]` | | memorydb | `pip install vectordb-bench[memorydb]` | | chromadb | `pip install vectordb-bench[chromadb]` | +| cockroachdb | `pip install vectordb-bench[cockroachdb]` | | awsopensearch | `pip install vectordb-bench[opensearch]` | | aliyun_opensearch | `pip install vectordb-bench[aliyun_opensearch]` | | mongodb | `pip install vectordb-bench[mongodb]` | | tidb | `pip install vectordb-bench[tidb]` | | vespa | `pip install vectordb-bench[vespa]` | +| oceanbase | `pip install vectordb-bench[oceanbase]` | +| hologres | `pip install vectordb-bench[hologres]` | +| tencent_es | `pip install vectordb-bench[tencent_es]` | +| alisql | `pip install 'vectordb-bench[alisql]'` | +| doris | `pip install vectordb-bench[doris]` | ### Run @@ -114,6 +120,10 @@ Options: --num-concurrency TEXT Comma-separated list of concurrency values to test during concurrent search [default: 1,10,20] + --concurrency-timeout INTEGER Timeout (in seconds) to wait for a + concurrency slot before failing. Set to a + negative value to wait indefinitely. + [default: 3600] --user-name TEXT Db username [required] --password TEXT Db password [required] --host TEXT Db host [required] @@ -169,7 +179,7 @@ vectordbbench awsopensearch --db-label awsopensearch \ --m 16 --ef-construction 256 \ --host search-vector-db-prod-h4f6m4of6x7yp2rz7gdmots7w4.us-west-2.es.amazonaws.com --port 443 \ --user vector --password '' \ ---case-type Performance1536D5M --num-insert-workers 10 \ +--case-type Performance1536D5M --number-of-indexing-clients 10 \ --skip-load --num-concurrency 75 ``` @@ -184,7 +194,7 @@ Options: --number-of-shards INTEGER Number of primary shards for the index --number-of-replicas INTEGER Number of replica copies for each primary shard - # Indexing Performance + # Indexing Performance --index-thread-qty INTEGER Thread count for native engine indexing --index-thread-qty-during-force-merge INTEGER Thread count during force merge operations @@ -197,17 +207,254 @@ Options: --force-merge-enabled BOOLEAN Whether to perform force merge operation --flush-threshold-size TEXT Size threshold for flushing the transaction log + --engine TEXT type of engine to use valid values [faiss, lucene, s3vector] # Memory Management --cb-threshold TEXT k-NN Memory circuit breaker threshold - --help Show this message and exit.``` + --ondisk Ondisk mode with binary quantization(32x compression) + --oversample-factor Controls the degree of oversampling applied to minority classes in imbalanced datasets to improve model performance by balancing class distributions.(default 1.0) + + + # Quantization Type + --quantization-type TEXT which type of quantization to use valid values [fp32, fp16, bq] + --help Show this message and exit. + ``` +### Run Elastic Cloud from command line + +Elastic Cloud supports multiple index types: HNSW, HNSW_INT8, HNSW_INT4, and HNSW_BBQ. + +**Example: Run HNSW index test** + +```shell +vectordbbench elasticcloudhnsw --db-label elastic-cloud-test \ +--cloud-id --password '' \ +--m 16 --ef-construction 100 --num-candidates 100 \ +--case-type Performance768D1M --number-of-shards 1 \ +--number-of-replicas 0 --refresh-interval 30s +``` + +**Example: Run HNSW_INT8 index test** + +```shell +vectordbbench elasticcloudhnswint8 --db-label elastic-cloud-int8 \ +--cloud-id --password '' \ +--m 16 --ef-construction 200 --num-candidates 200 \ +--case-type Performance1536D50K --element-type float +``` + +**Example: Run HNSW_INT4 index test** + +```shell +vectordbbench elasticcloudhnswint4 --db-label elastic-cloud-int4 \ +--cloud-id --password '' \ +--m 16 --ef-construction 200 --num-candidates 200 \ +--case-type Performance768D10M --use-rescore --oversample-ratio 2.0 +``` + +**Example: Run HNSW_BBQ index test** + +```shell +vectordbbench elasticcloudhnswbbq --db-label elastic-cloud-bbq \ +--cloud-id --password '' \ +--m 16 --ef-construction 200 --num-candidates 200 \ +--case-type Performance1536D5M --use-routing --use-force-merge +``` + +**Example: Run Label Filter Performance test** + +```shell +vectordbbench elasticcloudhnsw --db-label elastic-cloud-label-filter \ +--cloud-id --password '' \ +--case-type LabelFilterPerformanceCase \ +--dataset-with-size-type "Medium OpenAI (1536dim, 500K)" \ +--label-percentage 0.001 \ +--m 16 --ef-construction 128 --num-candidates 100 \ +--num-concurrency 1,5 --number-of-shards 1 +``` + +To list all options for Elastic Cloud, execute `vectordbbench elasticcloudhnsw --help`. The following are Elastic Cloud-specific command-line options: + +```text +$ vectordbbench elasticcloudhnsw --help +Usage: vectordbbench elasticcloudhnsw [OPTIONS] + +Options: + # Connection + --cloud-id TEXT Elastic Cloud ID [required] + --password TEXT Elastic Cloud password [required] + + # HNSW Index Parameters + --m INTEGER HNSW M parameter [default: 16] + --ef-construction INTEGER HNSW efConstruction parameter [default: 100] + --num-candidates INTEGER Number of candidates for search [default: 100] + --element-type [float|byte] Element type for vectors (float: 4 bytes, byte: 1 byte) [default: float] + + # Index Configuration + --number-of-shards INTEGER Number of shards [default: 1] + --number-of-replicas INTEGER Number of replicas [default: 0] + --refresh-interval TEXT Index refresh interval [default: 30s] + --merge-max-thread-count INTEGER + Maximum thread count for merge [default: 8] + --use-force-merge BOOLEAN Whether to use force merge [default: True] + --use-routing BOOLEAN Whether to use routing [default: False] + --use-rescore BOOLEAN Whether to use rescore [default: False] + --oversample-ratio FLOAT Oversample ratio for rescore [default: 2.0] + + # Common Options + --case-type [CapacityDim128|CapacityDim960|Performance768D100M|...] + Case type + --db-label TEXT Db label, default: date in ISO format + --k INTEGER K value for number of nearest neighbors to search [default: 100] + --num-concurrency TEXT Comma-separated list of concurrency values [default: 1,5,10,20,30,40,60,80] + --help Show this message and exit. +``` + +### Run OceanBase from command line + +Execute tests for the index types: HNSW, HNSW_SQ, or HNSW_BQ. + +```shell +vectordbbench oceanbasehnsw --host xxx --port xxx --user root@mysql_tenant --database test \ +--m 16 --ef-construction 200 --case-type Performance1536D50K \ +--index-type HNSW --ef-search 100 +``` + +To list the options for oceanbase, execute `vectordbbench oceanbasehnsw --help`, The following are some OceanBase-specific command-line options. + +```text +$ vectordbbench oceanbasehnsw --help +Usage: vectordbbench oceanbasehnsw [OPTIONS] + +Options: + [...] + --host TEXT OceanBase host + --user TEXT OceanBase username [required] + --password TEXT OceanBase database password + --database TEXT DataBase name [required] + --port INTEGER OceanBase port [required] + --m INTEGER hnsw m [required] + --ef-construction INTEGER hnsw ef-construction [required] + --ef-search INTEGER hnsw ef-search [required] + --index-type [HNSW|HNSW_SQ|HNSW_BQ] + Type of index to use. Supported values: + HNSW, HNSW_SQ, HNSW_BQ [required] + --help Show this message and exit. + ``` + +Execute tests for the index types: IVF_FLAT, IVF_SQ8, or IVF_PQ. + +```shell +vectordbbench oceanbaseivf --host xxx --port xxx --user root@mysql_tenant --database test \ +--nlist 1000 --sample_per_nlist 256 --case-type Performance768D1M \ +--index-type IVF_FLAT --ivf_nprobes 100 +``` + +To list the options for oceanbase, execute `vectordbbench oceanbaseivf --help`, The following are some OceanBase-specific command-line options. + +```text +$ vectordbbench oceanbaseivf --help +Usage: vectordbbench oceanbaseivf [OPTIONS] + +Options: + [...] + --host TEXT OceanBase host + --user TEXT OceanBase username [required] + --password TEXT OceanBase database password + --database TEXT DataBase name [required] + --port INTEGER OceanBase port [required] + --index-type [IVF_FLAT|IVF_SQ8|IVF_PQ] + Type of index to use. Supported values: + IVF_FLAT, IVF_SQ8, IVF_PQ [required] + --nlist INTEGER Number of cluster centers [required] + --sample_per_nlist INTEGER The cluster centers are calculated by total + sampling sample_per_nlist * nlist vectors + [required] + --ivf_nprobes TEXT How many clustering centers to search during + the query [required] + --m INTEGER The number of sub-vectors that each data + vector is divided into during IVF-PQ + --help Show this message and exit. Show this message and exit. + ``` + +### Run Hologres from command line + +It is recommended to use the following code for installation. +```shell +pip install 'vectordb-bench[hologres]' 'psycopg[binary]' pgvector +``` + +Execute tests for the index types: HGraph. + +```shell +NUM_PER_BATCH=10000 vectordbbench hologreshgraph --host Hologres_Endpoint --port 80 \ +--user ACCESS_ID --password ACCESS_KEY --database DATABASE_NAME \ +--m 64 --ef-construction 400 --case-type Performance768D10M \ +--index-type HGraph --ef-search 400 --k 10 --num-concurrency 1,60,70,75,80,90,95,100,110,120 +``` + +To list the options for Hologres, execute `vectordbbench hologreshgraph --help`, The following are some Hologres-specific command-line options. + +```text +$ vectordbbench hologreshgraph --help +Usage: vectordbbench hologreshgraph [OPTIONS] + +Options: + [...] + --host TEXT Hologres host + --user TEXT Hologres username [required] + --password TEXT Hologres database password + --database TEXT Hologres database name [required] + --port INTEGER Hologres port [required] + --m INTEGER hnsw m [required] + --ef-construction INTEGER hnsw ef-construction [required] + --ef-search INTEGER hnsw ef-search [required] + --index-type [HGraph] Type of index to use. Supported values: + HGraph [required] + --help Show this message and exit. + ``` + +### Run Doris from command line + +Doris supports ann index with type hnsw from version 4.0.x + +```shell +NUM_PER_BATCH=1000000 vectordbbench doris --http-port=8030 --port=9030 --db-name=vector_test --case-type=Performance768D1M --stream-load-rows-per-batch=500000 +``` + +Using flag `--session-var`, if you want to test doris with some customized session variables. For example: +```shell +NUM_PER_BATCH=1000000 vectordbbench doris --http-port=8030 --port=9030 --db-name=vector_test --case-type=Performance768D1M --stream-load-rows-per-batch=500000 --session-var enable_profile=True +``` + +Mote options: + +```text +--m INTEGER hnsw m +--ef-construction INTEGER hnsw ef-construction +--username TEXT Username [default: root; required] +--password TEXT Password [default: ""] +--host TEXT Db host [default: 127.0.0.1; required] +--port INTEGER Query Port [default: 9030; required] +--http-port INTEGER Http Port [default: 8030; required] +--db-name TEXT Db name [default: test; required] +--ssl / --no-ssl Enable or disable SSL, for Doris Serverless + SSL must be enabled [default: no-ssl] +--index-prop TEXT Extra index PROPERTY as key=value + (repeatable) +--session-var TEXT Session variable key=value applied to each + SQL session (repeatable) +--stream-load-rows-per-batch INTEGER + Rows per single stream load request; default + uses NUM_PER_BATCH +--no-index Create table without ANN index +``` #### Using a configuration file. The vectordbbench command can optionally read some or all the options from a yaml formatted configuration file. -By default, configuration files are expected to be in vectordb_bench/config-files/, this can be overridden by setting -the environment variable CONFIG_LOCAL_DIR or by passing the full path to the file. +By default, configuration files are expected to be in vectordb_bench/config-files/, this can be overridden by setting +the environment variable CONFIG_LOCAL_DIR or by passing the full path to the file. The required format is: ```yaml @@ -235,10 +482,85 @@ milvushnsw: ef_search: 128 drop_old: False load: False +elasticcloudhnsw: + db_label: elastic-cloud-hnsw + cloud_id: + password: + case_type: Performance768D1M + m: 16 + ef_construction: 100 + num_candidates: 100 + number_of_shards: 1 + number_of_replicas: 0 + refresh_interval: 30s + element_type: float ``` -> Notes: +> Notes: > - Options passed on the command line will override the configuration file* > - Parameter names use an _ not - +> - For `LabelFilterPerformanceCase` and `NewIntFilterPerformanceCase`, you must specify `dataset_with_size_type` in addition to `case_type` + +#### Using a batch configuration file. + +The vectordbbench command can read a batch configuration file to run all the test cases in the yaml formatted configuration file. + +By default, configuration files are expected to be in vectordb_bench/config-files/, this can be overridden by setting +the environment variable CONFIG_LOCAL_DIR or by passing the full path to the file. + +The required format is: +```yaml +commandname: + - parameter_name: parameter_value + another_parameter_name: parameter_value +``` +Example: +```yaml +pgvectorhnsw: + - db_label: pgConfigTest + user_name: vectordbbench + password: vectordbbench + db_name: vectordbbench + host: localhost + m: 16 + ef_construction: 128 + ef_search: 128 +milvushnsw: + - skip_search_serial: True + case_type: Performance1536D50K + uri: http://localhost:19530 + m: 16 + ef_construction: 128 + ef_search: 128 + drop_old: False + load: False +elasticcloudhnsw: + - db_label: elastic-cloud-hnsw-test-1 + cloud_id: + password: + case_type: Performance768D1M + m: 16 + ef_construction: 100 + num_candidates: 100 + - db_label: elastic-cloud-label-filter-0.1 + cloud_id: + password: + case_type: LabelFilterPerformanceCase + dataset_with_size_type: "Medium OpenAI (1536dim, 500K)" + label_percentage: 0.001 + m: 16 + ef_construction: 128 + num_candidates: 100 + num_concurrency: "1,5" +``` +> Notes: +> - Options can only be passed through configuration files +> - Parameter names use an _ not - +> - For `LabelFilterPerformanceCase` and `NewIntFilterPerformanceCase`, you must specify `dataset_with_size_type` in addition to `case_type` + +How to use? +```shell +vectordbbench batchcli --batch-config-file +``` ## Leaderboard ### Introduction @@ -246,11 +568,11 @@ To facilitate the presentation of test results and provide a comprehensive perfo ### Scoring Rules -1. For each case, select a base value and score each system based on relative values. - - For QPS and QP$, we use the highest value as the reference, denoted as `base_QPS` or `base_QP$`, and the score of each system is `(QPS/base_QPS) * 100` or `(QP$/base_QP$) * 100`. - - For Latency, we use the lowest value as the reference, that is, `base_Latency`, and the score of each system is `(base_Latency + 10ms)/(Latency + 10ms) * 100`. +1. For each case, select a base value and score each system based on relative values. + - For QPS and QP$, we use the highest value as the reference, denoted as `base_QPS` or `base_QP$`, and the score of each system is `(QPS/base_QPS) * 100` or `(QP$/base_QP$) * 100`. + - For Latency, we use the lowest value as the reference, that is, `base_Latency`, and the score of each system is `(base_Latency + 10ms)/(Latency + 10ms) * 100`. - We want to give equal weight to different cases, and not let a case with high absolute result values become the sole reason for the overall scoring. Therefore, when scoring different systems in each case, we need to use relative values. + We want to give equal weight to different cases, and not let a case with high absolute result values become the sole reason for the overall scoring. Therefore, when scoring different systems in each case, we need to use relative values. Also, for Latency, we add 10ms to the numerator and denominator to ensure that if every system performs particularly well in a case, its advantage will not be infinitely magnified when latency tends to 0. @@ -267,13 +589,13 @@ pip install -e '.[pinecone]' ``` ### Run test server ``` -$ python -m vectordb_bench +python -m vectordb_bench ``` OR: ```shell -$ init_bench +init_bench ``` OR: @@ -290,70 +612,53 @@ After reopen the repository in container, run `python -m vectordb_bench` in the ### Check coding styles ```shell -$ make lint +make lint ``` To fix the coding styles automatically ```shell -$ make format +make format ``` ## How does it work? ### Result Page ![image](https://github.com/zilliztech/VectorDBBench/assets/105927039/8a981327-c1c6-4796-8a85-c86154cb5472) -This is the main page of VectorDBBench, which displays the standard benchmark results we provide. Additionally, results of all tests performed by users themselves will also be shown here. We also offer the ability to select and compare results from multiple tests simultaneously. +This is the main page of VDBBench, which displays the standard benchmark results we provide. Additionally, results of all tests performed by users themselves will also be shown here. We also offer the ability to select and compare results from multiple tests simultaneously. The standard benchmark results displayed here include all 15 cases that we currently support for 6 of our clients (Milvus, Zilliz Cloud, Elastic Search, Qdrant Cloud, Weaviate Cloud and PgVector). However, as some systems may not be able to complete all the tests successfully due to issues like Out of Memory (OOM) or timeouts, not all clients are included in every case. All standard benchmark results are generated by a client running on an 8 core, 32 GB host, which is located in the same region as the server being tested. The client host is equipped with an `Intel(R) Xeon(R) Platinum 8375C CPU @ 2.90GHz` processor. Also all the servers for the open-source systems tested in our benchmarks run on hosts with the same type of processor. ### Run Test Page -![image](https://github.com/zilliztech/VectorDBBench/assets/105927039/f3135a29-8f12-4aac-bbb3-f2f55e2a2ff0) -This is the page to run a test: 1. Initially, you select the systems to be tested - multiple selections are allowed. Once selected, corresponding forms will pop up to gather necessary information for using the chosen databases. The db_label is used to differentiate different instances of the same system. We recommend filling in the host size or instance type here (as we do in our standard results). 2. The next step is to select the test cases you want to perform. You can select multiple cases at once, and a form to collect corresponding parameters will appear. 3. Finally, you'll need to provide a task label to distinguish different test results. Using the same label for different tests will result in the previous results being overwritten. -Now we can only run one task at the same time. +Now we can only run one task at the same time. +![image](vectordb_bench/fig/run_test_select_db.png) +![image](vectordb_bench/fig/run_test_select_case.png) +![image](vectordb_bench/fig/run_test_submit.png) + ## Module ### Code Structure ![image](https://github.com/zilliztech/VectorDBBench/assets/105927039/8c06512e-5419-4381-b084-9c93aed59639) ### Client -Our client module is designed with flexibility and extensibility in mind, aiming to integrate APIs from different systems seamlessly. As of now, it supports Milvus, Zilliz Cloud, Elastic Search, Pinecone, Qdrant Cloud, Weaviate Cloud, PgVector, Redis, and Chroma. Stay tuned for more options, as we are consistently working on extending our reach to other systems. +Our client module is designed with flexibility and extensibility in mind, aiming to integrate APIs from different systems seamlessly. As of now, it supports Milvus, Zilliz Cloud, Elastic Search, Pinecone, Qdrant Cloud, Weaviate Cloud, PgVector, Redis, Chroma, CockroachDB, etc. Stay tuned for more options, as we are consistently working on extending our reach to other systems. ### Benchmark Cases -We've developed an array of 15 comprehensive benchmark cases to test vector databases' various capabilities, each designed to give you a different piece of the puzzle. These cases are categorized into three main types: +We've developed lots of comprehensive benchmark cases to test vector databases' various capabilities, each designed to give you a different piece of the puzzle. These cases are categorized into four main types: #### Capacity Case - **Large Dim:** Tests the database's loading capacity by inserting large-dimension vectors (GIST 100K vectors, 960 dimensions) until fully loaded. The final number of inserted vectors is reported. - **Small Dim:** Similar to the Large Dim case but uses small-dimension vectors (SIFT 500K vectors, 128 dimensions). #### Search Performance Case - **XLarge Dataset:** Measures search performance with a massive dataset (LAION 100M vectors, 768 dimensions) at varying parallel levels. The results include index building time, recall, latency, and maximum QPS. -- **Large Dataset:** Similar to the XLarge Dataset case, but uses a slightly smaller dataset (10M-768dim, 5M-1536dim). -- **Medium Dataset:** A case using a medium dataset (1M-768dim, 500K-1536dim). +- **Large Dataset:** Similar to the XLarge Dataset case, but uses a slightly smaller dataset (10M-1024dim, 10M-768dim, 5M-1536dim). +- **Medium Dataset:** A case using a medium dataset (1M-1024dim, 1M-768dim, 500K-1536dim). +- **Small Dataset:** For development (100K-768dim, 50K-1536dim). #### Filtering Search Performance Case -- **Large Dataset, Low Filtering Rate:** Evaluates search performance with a large dataset (10M-768dim, 5M-1536dim) under a low filtering rate (1% vectors) at different parallel levels. -- **Medium Dataset, Low Filtering Rate:** This case uses a medium dataset (1M-768dim, 500K-1536dim) with a similar low filtering rate. -- **Large Dataset, High Filtering Rate:** It tests with a large dataset (10M-768dim, 5M-1536dim) but under a high filtering rate (99% vectors). -- **Medium Dataset, High Filtering Rate:** This case uses a medium dataset (1M-768dim, 500K-1536dim) with a high filtering rate. -For a quick reference, here is a table summarizing the key aspects of each case: - -Case No. | Case Type | Dataset Size | Filtering Rate | Results | -|----------|-----------|--------------|----------------|---------| -1 | Capacity Case | SIFT 500K vectors, 128 dimensions | N/A | Number of inserted vectors | -2 | Capacity Case | GIST 100K vectors, 960 dimensions | N/A | Number of inserted vectors | -3 | Search Performance Case | LAION 100M vectors, 768 dimensions | N/A | Index building time, recall, latency, maximum QPS | -4 | Search Performance Case | Cohere 10M vectors, 768 dimensions | N/A | Index building time, recall, latency, maximum QPS | -5 | Search Performance Case | Cohere 1M vectors, 768 dimensions | N/A | Index building time, recall, latency, maximum QPS | -6 | Filtering Search Performance Case | Cohere 10M vectors, 768 dimensions | 1% vectors | Index building time, recall, latency, maximum QPS | -7 | Filtering Search Performance Case | Cohere 1M vectors, 768 dimensions | 1% vectors | Index building time, recall, latency, maximum QPS | -8 | Filtering Search Performance Case | Cohere 10M vectors, 768 dimensions | 99% vectors | Index building time, recall, latency, maximum QPS | -9 | Filtering Search Performance Case | Cohere 1M vectors, 768 dimensions | 99% vectors | Index building time, recall, latency, maximum QPS | -10 | Search Performance Case | OpenAI generated 500K vectors, 1536 dimensions | N/A | Index building time, recall, latency, maximum QPS | -11 | Search Performance Case | OpenAI generated 5M vectors, 1536 dimensions | N/A | Index building time, recall, latency, maximum QPS | -12 | Filtering Search Performance Case | OpenAI generated 500K vectors, 1536 dimensions | 1% vectors | Index building time, recall, latency, maximum QPS | -13 | Filtering Search Performance Case | OpenAI generated 5M vectors, 1536 dimensions | 1% vectors | Index building time, recall, latency, maximum QPS | -14 | Filtering Search Performance Case | OpenAI generated 500K vectors, 1536 dimensions | 99% vectors | Index building time, recall, latency, maximum QPS | -15 | Filtering Search Performance Case | OpenAI generated 5M vectors, 1536 dimensions | 99% vectors | Index building time, recall, latency, maximum QPS | - +- **Int-Filter Cases:** Evaluates search performance with int-based filter expression (e.g. "id >= 2,000"). +- **Label-Filter Cases:** Evaluates search performance with label-based filter expressions (e.g., "color == 'red'"). The test includes randomly generated labels to simulate real-world filtering scenarios. +#### Streaming Cases +- **Insertion-Under-Load Case:** Evaluates search performance while maintaining a constant insertion workload. VDBBench applies a steady stream of insert requests at a fixed rate to simulate real-world scenarios where search operations must perform reliably under continuous data ingestion. Each case provides an in-depth examination of a vector database's abilities, providing you a comprehensive view of the database's performance. @@ -361,33 +666,33 @@ Each case provides an in-depth examination of a vector database's abilities, pro Through the `/custom` page, users can customize their own performance case using local datasets. After saving, the corresponding case can be selected from the `/run_test` page to perform the test. -![image](fig/custom_dataset.png) -![image](fig/custom_case_run_test.png) +![image](vectordb_bench/fig/custom_dataset.png) +![image](vectordb_bench/fig/custom_case_run_test.png) We have strict requirements for the data set format, please follow them. - `Folder Path` - The path to the folder containing all the files. Please ensure that all files in the folder are in the `Parquet` format. - Vectors data files: The file must be named `train.parquet` and should have two columns: `id` as an incrementing `int` and `emb` as an array of `float32`. - Query test vectors: The file must be named `test.parquet` and should have two columns: `id` as an incrementing `int` and `emb` as an array of `float32`. - We recommend limiting the number of test query vectors, like 1,000. - When conducting concurrent query tests, Vdbbench creates a large number of processes. - To minimize additional communication overhead during testing, + When conducting concurrent query tests, Vdbbench creates a large number of processes. + To minimize additional communication overhead during testing, we prepare a complete set of test queries for each process, allowing them to run independently. - However, this means that as the number of concurrent processes increases, - the number of copied query vectors also increases significantly, + However, this means that as the number of concurrent processes increases, + the number of copied query vectors also increases significantly, which can place substantial pressure on memory resources. - Ground truth file: The file must be named `neighbors.parquet` and should have two columns: `id` corresponding to query vectors and `neighbors_id` as an array of `int`. - `Train File Count` - If the vector file is too large, you can consider splitting it into multiple files. The naming format for the split files should be `train-[index]-of-[file_count].parquet`. For example, `train-01-of-10.parquet` represents the second file (0-indexed) among 10 split files. -- `Use Shuffled Data` - If you check this option, the vector data files need to be modified. VectorDBBench will load the data labeled with `shuffle`. For example, use `shuffle_train.parquet` instead of `train.parquet` and `shuffle_train-04-of-10.parquet` instead of `train-04-of-10.parquet`. The `id` column in the shuffled data can be in any order. +- `Use Shuffled Data` - If you check this option, the vector data files need to be modified. VDBBench will load the data labeled with `shuffle`. For example, use `shuffle_train.parquet` instead of `train.parquet` and `shuffle_train-04-of-10.parquet` instead of `train-04-of-10.parquet`. The `id` column in the shuffled data can be in any order. ## Goals Our goals of this benchmark are: ### Reproducibility & Usability -One of the primary goals of VectorDBBench is to enable users to reproduce benchmark results swiftly and easily, or to test their customized scenarios. We believe that lowering the barriers to entry for conducting these tests will enhance the community's understanding and improvement of vector databases. We aim to create an environment where any user, regardless of their technical expertise, can quickly set up and run benchmarks, and view and analyze results in an intuitive manner. +One of the primary goals of VDBBench is to enable users to reproduce benchmark results swiftly and easily, or to test their customized scenarios. We believe that lowering the barriers to entry for conducting these tests will enhance the community's understanding and improvement of vector databases. We aim to create an environment where any user, regardless of their technical expertise, can quickly set up and run benchmarks, and view and analyze results in an intuitive manner. ### Representability & Realism -VectorDBBench aims to provide a more comprehensive, multi-faceted testing environment that accurately represents the complexity of vector databases. By moving beyond a simple speed test for algorithms, we hope to contribute to a better understanding of vector databases in real-world scenarios. By incorporating as many complex scenarios as possible, including a variety of test cases and datasets, we aim to reflect realistic conditions and offer tangible significance to our community. Our goal is to deliver benchmarking results that can drive tangible improvements in the development and usage of vector databases. +VDBBench aims to provide a more comprehensive, multi-faceted testing environment that accurately represents the complexity of vector databases. By moving beyond a simple speed test for algorithms, we hope to contribute to a better understanding of vector databases in real-world scenarios. By incorporating as many complex scenarios as possible, including a variety of test cases and datasets, we aim to reflect realistic conditions and offer tangible significance to our community. Our goal is to deliver benchmarking results that can drive tangible improvements in the development and usage of vector databases. ## Contribution ### General Guidelines @@ -403,10 +708,10 @@ VectorDBBench aims to provide a more comprehensive, multi-faceted testing enviro **Step 2: Implement new_client.py and config.py** -1. Open new_client.py and define the NewClient class, which should inherit from the clients/api.py file's VectorDB abstract class. The VectorDB class serves as the API for benchmarking, and all DB clients must implement this abstract class. +1. Open new_client.py and define the NewClient class, which should inherit from the clients/api.py file's VectorDB abstract class. The VectorDB class serves as the API for benchmarking, and all DB clients must implement this abstract class. Example implementation in new_client.py: new_client.py -```python +```python from ..api import VectorDB class NewClient(VectorDB): # Implement the abstract methods defined in the VectorDB class @@ -435,7 +740,7 @@ class NewDBCaseConfig(DBCaseConfig): In this final step, you will import your DB client into clients/__init__.py and update the initialization process. 1. Open clients/__init__.py and import your NewClient from new_client.py. -2. Add your NewClient to the DB enum. +2. Add your NewClient to the DB enum. 3. Update the db2client dictionary by adding an entry for your NewClient. Example implementation in clients/__init__.py: @@ -533,14 +838,14 @@ def ZillizAutoIndex(**parameters: Unpack[ZillizTypedDict]): ) ``` 3. Update cli by adding: - 1. Add database specific options as an Annotated TypedDict, see ZillizTypedDict above. + 1. Add database specific options as an Annotated TypedDict, see ZillizTypedDict above. 2. Add index configuration specific options as an Annotated TypedDict. (example: vectordb_bench/backend/clients/pgvector/cli.py) 1. May not be needed if there is only one index config. - 2. Repeat for each index configuration, nesting them if possible. + 2. Repeat for each index configuration, nesting them if possible. 2. Add a index config specific function for each index type, see Zilliz above. The function name, in lowercase, will be the command name passed to the vectordbbench command. 3. Update db_config and db_case_config to match client requirements 4. Continue to add new functions for each index config. - 5. Import the client cli module and command to vectordb_bench/cli/vectordbbench.py (for databases with multiple commands (index configs), this only needs to be done for one command) + 5. Import the client cli module and command to vectordb_bench/cli/vectordbbench.py (for databases with multiple commands (index configs), this only needs to be done for one command) 6. Import the `get_custom_case_config` function from `vectordb_bench/cli/cli.py` and use it to add a new key `custom_case` to the `parameters` variable within the command. @@ -558,7 +863,7 @@ For the system under test, we use the default server-side configuration to maint For the Client, we welcome any parameter tuning to obtain better results. ### Incomplete Results Many databases may not be able to complete all test cases due to issues such as Out of Memory (OOM), crashes, or timeouts. In these scenarios, we will clearly state these occurrences in the test results. -### Mistake Or Misrepresentation +### Mistake Or Misrepresentation We strive for accuracy in learning and supporting various vector databases, yet there might be oversights or misapplications. For any such occurrences, feel free to [raise an issue](https://github.com/zilliztech/VectorDBBench/issues/new) or make amendments on our GitHub page. ## Timeout In our pursuit to ensure that our benchmark reflects the reality of a production environment while guaranteeing the practicality of the system, we have implemented a timeout plan based on our experiences for various tests. @@ -620,4 +925,4 @@ This multi-tiered timeout approach allows our benchmark to be more representativ -**Note:** Some datapoints in the standard benchmark results that voilate this timeout will be kept for now for reference. We will remove them in the future. +**Note:** Some datapoints in the standard benchmark results that violate this timeout will be kept for now for reference. We will remove them in the future. diff --git a/install/requirements_py3.11.txt b/install/requirements_py3.11.txt index 86958ada2..c6b5d1cc3 100644 --- a/install/requirements_py3.11.txt +++ b/install/requirements_py3.11.txt @@ -3,7 +3,7 @@ grpcio-tools==1.53.0 qdrant-client pinecone-client weaviate-client -elasticsearch +elasticsearch==8.16.0 pgvector pgvecto_rs[psycopg3]>=0.2.1 sqlalchemy @@ -24,3 +24,6 @@ scikit-learn pymilvus clickhouse_connect pyvespa +mysql-connector-python +packaging +hdrhistogram>=0.10.1 \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 09fa66972..63c585c55 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,6 +4,7 @@ build-backend = "setuptools.build_meta" [tool.setuptools.package-data] "vectordb_bench.results" = ["*.json"] +"vectordb_bench.fig.homepage" = ["*.png"] [tool.setuptools.packages.find] where = ["."] @@ -12,7 +13,7 @@ include = ["vectordb_bench", "vectordb_bench.cli"] [project] name = "vectordb-bench" authors = [ - {name="XuanYang-cn", email="xuan.yang@zilliz.com"}, + {name="VDBBench teams", email="pymilvus@zilliz.com"}, ] description = "VectorDBBench is not just an offering of benchmark results for mainstream vector databases and cloud services, it's your go-to tool for the ultimate performance and cost-effectiveness comparison. Designed with ease-of-use in mind, VectorDBBench is devised to help users, even non-professionals, reproduce results or test new systems, making the hunt for the optimal choice amongst a plethora of cloud services and open-source vector databases a breeze." @@ -35,10 +36,12 @@ dependencies = [ "psutil", "polars", "plotly", - "environs<14.1.0", + "environs", "pydantic=0.10.1", ] dynamic = ["version"] @@ -48,6 +51,7 @@ test = [ "ruff", "pytest", ] +restful = [ "flask" ] all = [ "grpcio==1.53.0", # for qdrant-client and pymilvus @@ -67,11 +71,13 @@ all = [ "opensearch-py", "memorydb", "alibabacloud_ha3engine_vector", - "alibabacloud_searchengine20211025", "mariadb", "PyMySQL", "clickhouse-connect", "pyvespa", + "lancedb", + "mysql-connector-python", + "turbopuffer[fast]", ] qdrant = [ "qdrant-client" ] @@ -88,18 +94,26 @@ redis = [ "redis" ] memorydb = [ "memorydb" ] chromadb = [ "chromadb" ] opensearch = [ "opensearch-py" ] -aliyun_opensearch = [ "alibabacloud_ha3engine_vector", "alibabacloud_searchengine20211025"] +aliyun_opensearch = [ "alibabacloud_ha3engine_vector" ] mongodb = [ "pymongo" ] mariadb = [ "mariadb" ] tidb = [ "PyMySQL" ] +cockroachdb = [ "psycopg[binary,pool]", "pgvector" ] clickhouse = [ "clickhouse-connect" ] vespa = [ "pyvespa" ] +lancedb = [ "lancedb" ] +oceanbase = [ "mysql-connector-python" ] +alisql = [ "mysql-connector-python" ] +doris = [ "doris-vector-search" ] +turbopuffer = [ "turbopuffer" ] [project.urls] -"repository" = "https://github.com/zilliztech/VectorDBBench" +Repository = "https://github.com/zilliztech/VectorDBBench" +Leaderborad = "https://zilliz.com/vdbbench-leaderboard" [project.scripts] init_bench = "vectordb_bench.__main__:main" +init_bench_rest = "vectordb_bench.restful.app:main" vectordbbench = "vectordb_bench.cli.vectordbbench:cli" [tool.setuptools_scm] @@ -216,3 +230,9 @@ builtins-ignorelist = [ # "dict", # TODO # "filter", ] + +[tool.ruff.lint.per-file-ignores] +"vectordb_bench/backend/clients/*" = ["PLC0415"] +"vectordb_bench/cli/batch_cli.py" = ["PLC0415"] +"vectordb_bench/backend/data_source.py" = ["PLC0415"] + diff --git a/tests/test_cockroachdb.py b/tests/test_cockroachdb.py new file mode 100644 index 000000000..b55cded88 --- /dev/null +++ b/tests/test_cockroachdb.py @@ -0,0 +1,128 @@ +""" +Tests for CockroachDB vector database client. + +Assumes CockroachDB is running on localhost:26257. + +To start CockroachDB locally: + cockroach start-single-node --insecure --listen-addr=localhost:26257 +""" + +import logging + +import numpy as np + +from vectordb_bench.models import DB + +log = logging.getLogger(__name__) + + +class TestCockroachDB: + """Test suite for CockroachDB vector operations.""" + + def test_insert_and_search(self): + """Test basic insert and search operations.""" + assert DB.CockroachDB.value == "CockroachDB" + + dbcls = DB.CockroachDB.init_cls + dbConfig = DB.CockroachDB.config_cls + + # Connection config (matches your local CockroachDB instance) + config = { + "host": "localhost", + "port": 26257, + "user_name": "root", + "password": "", + "db_name": "defaultdb", + "table_name": "test_cockroachdb", + } + + # Note: sslmode=disable is handled in the client's connect_config options + + dim = 128 + count = 1000 + + # Initialize CockroachDB client + cockroachdb = dbcls( + dim=dim, + db_config=config, + db_case_config=None, + collection_name="test_cockroachdb", + drop_old=True, + ) + + embeddings = [[np.random.random() for _ in range(dim)] for _ in range(count)] + + # Test insert + with cockroachdb.init(): + res = cockroachdb.insert_embeddings(embeddings=embeddings, metadata=list(range(count))) + + assert res[0] == count, f"Insert count mismatch: {res[0]} != {count}" + assert res[1] is None, f"Insert failed with error: {res[1]}" + + # Test search + with cockroachdb.init(): + test_id = np.random.randint(count) + q = embeddings[test_id] + + res = cockroachdb.search_embedding(query=q, k=10) + + assert len(res) > 0, "Search returned no results" + assert res[0] == int(test_id), f"Top result {res[0]} != query id {test_id}" + + log.info("CockroachDB insert and search test passed") + + def test_search_with_filter(self): + """Test search with filters.""" + assert DB.CockroachDB.value == "CockroachDB" + + dbcls = DB.CockroachDB.init_cls + + config = { + "host": "localhost", + "port": 26257, + "user_name": "root", + "password": "", + "db_name": "defaultdb", + "table_name": "test_cockroachdb_filter", + } + + dim = 128 + count = 1000 + filter_value = 0.9 + + cockroachdb = dbcls( + dim=dim, + db_config=config, + db_case_config=None, + collection_name="test_cockroachdb_filter", + drop_old=True, + ) + + embeddings = [[np.random.random() for _ in range(dim)] for _ in range(count)] + + # Insert data + with cockroachdb.init(): + res = cockroachdb.insert_embeddings(embeddings=embeddings, metadata=list(range(count))) + assert res[0] == count, f"Insert count mismatch" + + # Search with filter + with cockroachdb.init(): + filter_id = int(count * filter_value) + test_id = np.random.randint(filter_id, count) + q = embeddings[test_id] + + from vectordb_bench.backend.filter import IntFilter + + filters = IntFilter(int_value=filter_id, filter_rate=0.9) + cockroachdb.prepare_filter(filters) + + res = cockroachdb.search_embedding(query=q, k=10) + + assert len(res) > 0, "Filtered search returned no results" + assert res[0] == int(test_id), f"Top result {res[0]} != query id {test_id}" + + # Verify all results are >= filter_value + for result_id in res: + assert int(result_id) >= filter_id, f"Result {result_id} < filter threshold {filter_id}" + + log.info("CockroachDB filter test passed") diff --git a/vectordb_bench/__init__.py b/vectordb_bench/__init__.py index c07fc855d..07f77bb02 100644 --- a/vectordb_bench/__init__.py +++ b/vectordb_bench/__init__.py @@ -6,7 +6,7 @@ from . import log_util env = environs.Env() -env.read_env(".env", False) +env.read_env(path=".env", recurse=False) class config: @@ -14,44 +14,27 @@ class config: AWS_S3_URL = "assets.zilliz.com/benchmark/" LOG_LEVEL = env.str("LOG_LEVEL", "INFO") + LOG_FILE = env.str("LOG_FILE", "logs/vectordb_bench.log") DEFAULT_DATASET_URL = env.str("DEFAULT_DATASET_URL", AWS_S3_URL) + DATASET_SOURCE = env.str("DATASET_SOURCE", "S3") # Options "S3" or "AliyunOSS" DATASET_LOCAL_DIR = env.path("DATASET_LOCAL_DIR", "/tmp/vectordb_bench/dataset") NUM_PER_BATCH = env.int("NUM_PER_BATCH", 100) + TIME_PER_BATCH = 1 # 1s. for streaming insertion. + MAX_INSERT_RETRY = 5 + MAX_SEARCH_RETRY = 5 + + LOAD_MAX_TRY_COUNT = 10 DROP_OLD = env.bool("DROP_OLD", True) USE_SHUFFLED_DATA = env.bool("USE_SHUFFLED_DATA", True) - NUM_CONCURRENCY = env.list( - "NUM_CONCURRENCY", - [ - 1, - 5, - 10, - 15, - 20, - 25, - 30, - 35, - 40, - 45, - 50, - 55, - 60, - 65, - 70, - 75, - 80, - 85, - 90, - 95, - 100, - ], - subcast=int, - ) + NUM_CONCURRENCY = env.list("NUM_CONCURRENCY", [1, 5, 10, 20, 30, 40, 60, 80], subcast=int) CONCURRENCY_DURATION = 30 + CONCURRENCY_TIMEOUT = 3600 + RESULTS_LOCAL_DIR = env.path( "RESULTS_LOCAL_DIR", pathlib.Path(__file__).parent.joinpath("results"), @@ -66,6 +49,7 @@ class config: CAPACITY_TIMEOUT_IN_SECONDS = 24 * 3600 # 24h LOAD_TIMEOUT_DEFAULT = 24 * 3600 # 24h + LOAD_TIMEOUT_768D_100K = 24 * 3600 # 24h LOAD_TIMEOUT_768D_1M = 24 * 3600 # 24h LOAD_TIMEOUT_768D_10M = 240 * 3600 # 10d LOAD_TIMEOUT_768D_100M = 2400 * 3600 # 100d @@ -73,7 +57,11 @@ class config: LOAD_TIMEOUT_1536D_500K = 24 * 3600 # 24h LOAD_TIMEOUT_1536D_5M = 240 * 3600 # 10d + LOAD_TIMEOUT_1024D_1M = 24 * 3600 # 24h + LOAD_TIMEOUT_1024D_10M = 240 * 3600 # 10d + OPTIMIZE_TIMEOUT_DEFAULT = 24 * 3600 # 24h + OPTIMIZE_TIMEOUT_768D_100K = 24 * 3600 # 24h OPTIMIZE_TIMEOUT_768D_1M = 24 * 3600 # 24h OPTIMIZE_TIMEOUT_768D_10M = 240 * 3600 # 10d OPTIMIZE_TIMEOUT_768D_100M = 2400 * 3600 # 100d @@ -81,6 +69,9 @@ class config: OPTIMIZE_TIMEOUT_1536D_500K = 24 * 3600 # 24h OPTIMIZE_TIMEOUT_1536D_5M = 240 * 3600 # 10d + OPTIMIZE_TIMEOUT_1024D_1M = 24 * 3600 # 24h + OPTIMIZE_TIMEOUT_1024D_10M = 240 * 3600 # 10d + def display(self) -> str: return [ i @@ -89,4 +80,4 @@ def display(self) -> str: ] -log_util.init(config.LOG_LEVEL) +log_util.init(config.LOG_LEVEL, pathlib.Path(config.LOG_FILE)) diff --git a/vectordb_bench/__main__.py b/vectordb_bench/__main__.py index 6663731f5..53a372422 100644 --- a/vectordb_bench/__main__.py +++ b/vectordb_bench/__main__.py @@ -17,7 +17,7 @@ def run_streamlit(): cmd = [ "streamlit", "run", - f"{pathlib.Path(__file__).parent}/frontend/vdb_benchmark.py", + f"{pathlib.Path(__file__).parent}/frontend/vdbbench.py", "--logger.level", "info", "--theme.base", diff --git a/vectordb_bench/backend/assembler.py b/vectordb_bench/backend/assembler.py index b28a7847d..3268cde35 100644 --- a/vectordb_bench/backend/assembler.py +++ b/vectordb_bench/backend/assembler.py @@ -1,7 +1,8 @@ import logging -from vectordb_bench.backend.clients import EmptyDBCaseConfig +from vectordb_bench.backend.clients import DB, EmptyDBCaseConfig from vectordb_bench.backend.data_source import DatasetSource +from vectordb_bench.backend.filter import FilterOp from vectordb_bench.models import TaskConfig from .cases import CaseLabel @@ -10,6 +11,13 @@ log = logging.getLogger(__name__) +class FilterNotSupportedError(ValueError): + """Raised when a filter type is not supported by a vector database.""" + + def __init__(self, db_name: str, filter_type: FilterOp): + super().__init__(f"{filter_type} Filter test is not supported by {db_name}.") + + class Assembler: @classmethod def assemble(cls, run_id: str, task: TaskConfig, source: DatasetSource) -> CaseRunner: @@ -39,25 +47,30 @@ def assemble_all( runners = [cls.assemble(run_id, task, source) for task in tasks] load_runners = [r for r in runners if r.ca.label == CaseLabel.Load] perf_runners = [r for r in runners if r.ca.label == CaseLabel.Performance] + streaming_runners = [r for r in runners if r.ca.label == CaseLabel.Streaming] # group by db - db2runner = {} + db2runner: dict[DB, list[CaseRunner]] = {} for r in perf_runners: db = r.config.db if db not in db2runner: db2runner[db] = [] db2runner[db].append(r) - # check dbclient installed - for k in db2runner: - _ = k.init_cls + # check + for db, runners in db2runner.items(): + db_instance = db.init_cls + for runner in runners: + if not db_instance.filter_supported(runner.ca.filters): + raise FilterNotSupportedError(db.value, runner.ca.filters.type) # sort by dataset size for _, runner in db2runner.items(): - runner.sort(key=lambda x: x.ca.dataset.data.size) + runner.sort(key=lambda x: (x.ca.dataset.data.size, 0 if x.ca.filters.type == FilterOp.StrEqual else 1)) all_runners = [] all_runners.extend(load_runners) + all_runners.extend(streaming_runners) for v in db2runner.values(): all_runners.extend(v) diff --git a/vectordb_bench/backend/cases.py b/vectordb_bench/backend/cases.py index 15fc069cc..edfcdea19 100644 --- a/vectordb_bench/backend/cases.py +++ b/vectordb_bench/backend/cases.py @@ -1,14 +1,14 @@ +import json import logging from enum import Enum, auto from vectordb_bench import config from vectordb_bench.backend.clients.api import MetricType +from vectordb_bench.backend.filter import Filter, FilterOp, IntFilter, LabelFilter, NewIntFilter, NonFilter, non_filter from vectordb_bench.base import BaseModel -from vectordb_bench.frontend.components.custom.getCustomConfig import ( - CustomDatasetConfig, -) +from vectordb_bench.frontend.components.custom.getCustomConfig import CustomDatasetConfig -from .dataset import CustomDataset, Dataset, DatasetManager +from .dataset import CustomDataset, Dataset, DatasetManager, DatasetWithSizeType log = logging.getLogger(__name__) @@ -42,11 +42,21 @@ class CaseType(Enum): Performance1536D500K99P = 14 Performance1536D5M99P = 15 + Performance1024D1M = 17 + Performance1024D10M = 20 + Performance1536D50K = 50 Custom = 100 PerformanceCustomDataset = 101 + StreamingPerformanceCase = 200 + StreamingCustomDataset = 201 + + LabelFilterPerformanceCase = 300 + + NewIntFilterPerformanceCase = 400 + def case_cls(self, custom_configs: dict | None = None) -> type["Case"]: if custom_configs is None: return type2case.get(self)() @@ -68,6 +78,7 @@ def case_description(self, custom_configs: dict | None = None) -> str: class CaseLabel(Enum): Load = auto() Performance = auto() + Streaming = auto() class Case(BaseModel): @@ -87,35 +98,42 @@ class Case(BaseModel): description: str dataset: DatasetManager - load_timeout: float | int + load_timeout: float | int | None = None optimize_timeout: float | int | None = None filter_rate: float | None = None @property - def filters(self) -> dict | None: - if self.filter_rate is not None: - target_id = round(self.filter_rate * self.dataset.data.size) - return { - "metadata": f">={target_id}", - "id": target_id, - } + def filters(self) -> Filter: + return non_filter + + @property + def with_scalar_labels(self) -> bool: + return self.filters.type == FilterOp.StrEqual - return None + def check_scalar_labels(self) -> None: + if self.with_scalar_labels and not self.dataset.data.with_scalar_labels: + msg = f"Case init failed: no scalar_labels data in current dataset ({self.dataset.data.full_name})" + raise ValueError(msg) + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.check_scalar_labels() -class CapacityCase(Case, BaseModel): + +class CapacityCase(Case): label: CaseLabel = CaseLabel.Load filter_rate: float | None = None load_timeout: float | int = config.CAPACITY_TIMEOUT_IN_SECONDS optimize_timeout: float | int | None = None -class PerformanceCase(Case, BaseModel): +class PerformanceCase(Case): label: CaseLabel = CaseLabel.Performance filter_rate: float | None = None load_timeout: float | int = config.LOAD_TIMEOUT_DEFAULT optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_DEFAULT + int_value: float | None = None class CapacityDim960(CapacityCase): @@ -147,6 +165,14 @@ class Performance768D10M(PerformanceCase): optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_768D_10M +class IntFilterPerformanceCase(PerformanceCase): + @property + def filters(self) -> Filter: + int_field = self.dataset.data.train_id_field + int_value = int(self.dataset.data.size * self.filter_rate) + return IntFilter(filter_rate=self.filter_rate, int_field=int_field, int_value=int_value) + + class Performance768D1M(PerformanceCase): case_id: CaseType = CaseType.Performance768D1M dataset: DatasetManager = Dataset.COHERE.manager(1_000_000) @@ -158,7 +184,7 @@ class Performance768D1M(PerformanceCase): optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_768D_1M -class Performance768D10M1P(PerformanceCase): +class Performance768D10M1P(IntFilterPerformanceCase): case_id: CaseType = CaseType.Performance768D10M1P filter_rate: float | int | None = 0.01 dataset: DatasetManager = Dataset.COHERE.manager(10_000_000) @@ -170,7 +196,7 @@ class Performance768D10M1P(PerformanceCase): optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_768D_10M -class Performance768D1M1P(PerformanceCase): +class Performance768D1M1P(IntFilterPerformanceCase): case_id: CaseType = CaseType.Performance768D1M1P filter_rate: float | int | None = 0.01 dataset: DatasetManager = Dataset.COHERE.manager(1_000_000) @@ -182,7 +208,7 @@ class Performance768D1M1P(PerformanceCase): optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_768D_1M -class Performance768D10M99P(PerformanceCase): +class Performance768D10M99P(IntFilterPerformanceCase): case_id: CaseType = CaseType.Performance768D10M99P filter_rate: float | int | None = 0.99 dataset: DatasetManager = Dataset.COHERE.manager(10_000_000) @@ -194,7 +220,7 @@ class Performance768D10M99P(PerformanceCase): optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_768D_10M -class Performance768D1M99P(PerformanceCase): +class Performance768D1M99P(IntFilterPerformanceCase): case_id: CaseType = CaseType.Performance768D1M99P filter_rate: float | int | None = 0.99 dataset: DatasetManager = Dataset.COHERE.manager(1_000_000) @@ -242,7 +268,7 @@ class Performance1536D5M(PerformanceCase): optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1536D_5M -class Performance1536D500K1P(PerformanceCase): +class Performance1536D500K1P(IntFilterPerformanceCase): case_id: CaseType = CaseType.Performance1536D500K1P filter_rate: float | int | None = 0.01 dataset: DatasetManager = Dataset.OPENAI.manager(500_000) @@ -254,7 +280,7 @@ class Performance1536D500K1P(PerformanceCase): optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1536D_500K -class Performance1536D5M1P(PerformanceCase): +class Performance1536D5M1P(IntFilterPerformanceCase): case_id: CaseType = CaseType.Performance1536D5M1P filter_rate: float | int | None = 0.01 dataset: DatasetManager = Dataset.OPENAI.manager(5_000_000) @@ -266,7 +292,7 @@ class Performance1536D5M1P(PerformanceCase): optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1536D_5M -class Performance1536D500K99P(PerformanceCase): +class Performance1536D500K99P(IntFilterPerformanceCase): case_id: CaseType = CaseType.Performance1536D500K99P filter_rate: float | int | None = 0.99 dataset: DatasetManager = Dataset.OPENAI.manager(500_000) @@ -278,7 +304,7 @@ class Performance1536D500K99P(PerformanceCase): optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1536D_500K -class Performance1536D5M99P(PerformanceCase): +class Performance1536D5M99P(IntFilterPerformanceCase): case_id: CaseType = CaseType.Performance1536D5M99P filter_rate: float | int | None = 0.99 dataset: DatasetManager = Dataset.OPENAI.manager(5_000_000) @@ -290,6 +316,30 @@ class Performance1536D5M99P(PerformanceCase): optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1536D_5M +class Performance1024D1M(PerformanceCase): + case_id: CaseType = CaseType.Performance1024D1M + filter_rate: float | int | None = None + dataset: DatasetManager = Dataset.BIOASQ.manager(1_000_000) + name: str = "Search Performance Test (1M Dataset, 1024 Dim)" + description: str = """This case tests the search performance of a vector database with a medium 1M dataset + (Bioasq 1M vectors, 1024 dimensions), at varying parallel levels. Results will show index building time, + recall, and maximum QPS.""" + load_timeout: float | int = config.LOAD_TIMEOUT_1024D_1M + optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1024D_1M + + +class Performance1024D10M(PerformanceCase): + case_id: CaseType = CaseType.Performance1024D10M + filter_rate: float | int | None = None + dataset: DatasetManager = Dataset.BIOASQ.manager(10_000_000) + name: str = "Search Performance Test (10M Dataset, 1024 Dim)" + description: str = """This case tests the search performance of a vector database with a large 10M dataset + (Bioasq 10M vectors, 1024 dimensions), at varying parallel levels. Results will show index building time, + recall, and maximum QPS.""" + load_timeout: float | int = config.LOAD_TIMEOUT_1024D_10M + optimize_timeout: float | int | None = config.OPTIMIZE_TIMEOUT_1024D_10M + + class Performance1536D50K(PerformanceCase): case_id: CaseType = CaseType.Performance1536D50K filter_rate: float | int | None = None @@ -318,7 +368,10 @@ class PerformanceCustomDataset(PerformanceCase): case_id: CaseType = CaseType.PerformanceCustomDataset name: str = "Performance With Custom Dataset" description: str = "" + gt_file: str dataset: DatasetManager + label_percentage: float | None = None + use_filter: bool def __init__( self, @@ -327,6 +380,8 @@ def __init__( load_timeout: float, optimize_timeout: float, dataset_config: dict, + label_percentage: float | None = None, + use_filter: bool = False, **kwargs, ): dataset_config = CustomDatasetConfig(**dataset_config) @@ -339,15 +394,242 @@ def __init__( with_gt=dataset_config.with_gt, dir=dataset_config.dir, file_num=dataset_config.file_count, + train_file=dataset_config.train_name, + test_file=f"{dataset_config.test_name}.parquet", + train_id_field=dataset_config.train_id_name, + train_vector_field=dataset_config.train_col_name, + test_vector_field=dataset_config.test_col_name, + gt_neighbors_field=dataset_config.gt_col_name, + scalar_labels_file=f"{dataset_config.scalar_labels_name}.parquet", ) super().__init__( name=name, description=description, load_timeout=load_timeout, optimize_timeout=optimize_timeout, + gt_file=f"{dataset_config.gt_name}.parquet", dataset=DatasetManager(data=dataset), + use_filter=use_filter, + label_percentage=label_percentage, ) + @property + def filters(self) -> Filter: + if self.use_filter is True: + return LabelFilter(label_percentage=self.label_percentage) + return NonFilter(gt_file_name=self.gt_file) + + +class StreamingPerformanceCase(Case): + case_id: CaseType = CaseType.StreamingPerformanceCase + label: CaseLabel = CaseLabel.Streaming + dataset_with_size_type: DatasetWithSizeType + insert_rate: int + search_stages: list[float] + concurrencies: list[int] + optimize_after_write: bool = True + read_dur_after_write: int = 30 + + def __init__( + self, + dataset_with_size_type: DatasetWithSizeType | str = DatasetWithSizeType.CohereSmall.value, + insert_rate: int = 500, + search_stages: list[float] | str = (0.5, 0.8), + concurrencies: list[int] | str = (5, 10), + **kwargs, + ): + num_per_batch = config.NUM_PER_BATCH + if insert_rate % config.NUM_PER_BATCH != 0: + _insert_rate = max( + num_per_batch, + insert_rate // num_per_batch * num_per_batch, + ) + log.warning( + f"[streaming_case init] insert_rate(={insert_rate}) should be " + f"divisible by NUM_PER_BATCH={num_per_batch}), reset to {_insert_rate}", + ) + insert_rate = _insert_rate + if not isinstance(dataset_with_size_type, DatasetWithSizeType): + dataset_with_size_type = DatasetWithSizeType(dataset_with_size_type) + dataset = dataset_with_size_type.get_manager() + name = f"Streaming-Perf - {dataset_with_size_type.value}, {insert_rate} rows/s" + description = ( + "This case tests the search performance of vector database while maintaining " + f"a fixed insertion speed. (dataset: {dataset_with_size_type.value})" + ) + + if isinstance(search_stages, str): + search_stages = json.loads(search_stages) + if isinstance(concurrencies, str): + concurrencies = json.loads(concurrencies) + + super().__init__( + name=name, + description=description, + dataset=dataset, + dataset_with_size_type=dataset_with_size_type, + insert_rate=insert_rate, + search_stages=search_stages, + concurrencies=concurrencies, + **kwargs, + ) + + +class StreamingCustomDataset(Case): + case_id: CaseType = CaseType.StreamingCustomDataset + label: CaseLabel = CaseLabel.Streaming + name: str = "Streaming Performance With Custom Dataset" + description: str = "" + dataset: DatasetManager + insert_rate: int + search_stages: list[float] + concurrencies: list[int] + optimize_after_write: bool = True + read_dur_after_write: int = 30 + + def __init__( + self, + description: str, + dataset_config: dict, + insert_rate: int = 500, + search_stages: list[float] | str = (0.5, 0.8), + concurrencies: list[int] | str = (5, 10), + optimize_after_write: bool = True, + read_dur_after_write: int = 30, + **kwargs, + ): + num_per_batch = config.NUM_PER_BATCH + if insert_rate % config.NUM_PER_BATCH != 0: + _insert_rate = max( + num_per_batch, + insert_rate // num_per_batch * num_per_batch, + ) + log.warning( + f"[streaming_case init] insert_rate(={insert_rate}) should be " + f"divisible by NUM_PER_BATCH={num_per_batch}), reset to {_insert_rate}", + ) + insert_rate = _insert_rate + + dataset_config = CustomDatasetConfig(**dataset_config) + dataset = CustomDataset( + name=dataset_config.name, + size=dataset_config.size, + dim=dataset_config.dim, + metric_type=metric_type_map(dataset_config.metric_type), + use_shuffled=dataset_config.use_shuffled, + with_gt=dataset_config.with_gt, + dir=dataset_config.dir, + file_num=dataset_config.file_count, + train_file=dataset_config.train_name, + test_file=f"{dataset_config.test_name}.parquet", + train_id_field=dataset_config.train_id_name, + train_vector_field=dataset_config.train_col_name, + test_vector_field=dataset_config.test_col_name, + gt_neighbors_field=dataset_config.gt_col_name, + scalar_labels_file=f"{dataset_config.scalar_labels_name}.parquet", + ) + name = f"Streaming-Perf - Custom - {dataset_config.name}, {insert_rate} rows/s" + description = ( + description + if description + else f"This case tests the search performance of vector database while maintaining " + f"a fixed insertion speed. (dataset: Custom - {dataset_config.name})" + ) + + if isinstance(search_stages, str): + search_stages = json.loads(search_stages) + if isinstance(concurrencies, str): + concurrencies = json.loads(concurrencies) + + super().__init__( + name=name, + description=description, + dataset=DatasetManager(data=dataset), + insert_rate=insert_rate, + search_stages=search_stages, + concurrencies=concurrencies, + optimize_after_write=optimize_after_write, + read_dur_after_write=read_dur_after_write, + **kwargs, + ) + + +class NewIntFilterPerformanceCase(PerformanceCase): + case_id: CaseType = CaseType.NewIntFilterPerformanceCase + dataset_with_size_type: DatasetWithSizeType + filter_rate: float + + def __init__( + self, + dataset_with_size_type: DatasetWithSizeType | str, + filter_rate: float, + int_value: float | None = 0, + **kwargs, + ): + if not isinstance(dataset_with_size_type, DatasetWithSizeType): + dataset_with_size_type = DatasetWithSizeType(dataset_with_size_type) + name = f"Int-Filter-{filter_rate*100:.1f}% - {dataset_with_size_type.value}" + description = f"Int-Filter-{filter_rate*100:.1f}% Performance Test ({dataset_with_size_type.value})" + dataset = dataset_with_size_type.get_manager() + load_timeout = dataset_with_size_type.get_load_timeout() + optimize_timeout = dataset_with_size_type.get_optimize_timeout() + filters = IntFilter(filter_rate=filter_rate, int_value=int_value) + filter_rate = filters.filter_rate + super().__init__( + name=name, + description=description, + dataset=dataset, + load_timeout=load_timeout, + optimize_timeout=optimize_timeout, + filter_rate=filter_rate, + int_value=int_value, + dataset_with_size_type=dataset_with_size_type, + **kwargs, + ) + + @property + def filters(self) -> Filter: + int_field = self.dataset.data.train_id_field + int_value = int(self.dataset.data.size * self.filter_rate) + return NewIntFilter(filter_rate=self.filter_rate, int_field=int_field, int_value=int_value) + + +class LabelFilterPerformanceCase(PerformanceCase): + case_id: CaseType = CaseType.LabelFilterPerformanceCase + dataset_with_size_type: DatasetWithSizeType + label_percentage: float + + def __init__( + self, + dataset_with_size_type: DatasetWithSizeType | str, + label_percentage: float, + **kwargs, + ): + if not isinstance(dataset_with_size_type, DatasetWithSizeType): + dataset_with_size_type = DatasetWithSizeType(dataset_with_size_type) + name = f"Label-Filter-{label_percentage*100:.1f}% - {dataset_with_size_type.value}" + description = f"Label-Filter-{label_percentage*100:.1f}% Performance Test ({dataset_with_size_type.value})" + dataset = dataset_with_size_type.get_manager() + load_timeout = dataset_with_size_type.get_load_timeout() + optimize_timeout = dataset_with_size_type.get_optimize_timeout() + filters = LabelFilter(label_percentage=label_percentage) + filter_rate = filters.filter_rate + super().__init__( + name=name, + description=description, + dataset=dataset, + load_timeout=load_timeout, + optimize_timeout=optimize_timeout, + filter_rate=filter_rate, + dataset_with_size_type=dataset_with_size_type, + label_percentage=label_percentage, + **kwargs, + ) + + @property + def filters(self) -> Filter: + return LabelFilter(label_percentage=self.label_percentage) + type2case = { CaseType.CapacityDim960: CapacityDim960, @@ -365,6 +647,12 @@ def __init__( CaseType.Performance1536D5M1P: Performance1536D5M1P, CaseType.Performance1536D500K99P: Performance1536D500K99P, CaseType.Performance1536D5M99P: Performance1536D5M99P, + CaseType.Performance1024D1M: Performance1024D1M, + CaseType.Performance1024D10M: Performance1024D10M, CaseType.Performance1536D50K: Performance1536D50K, CaseType.PerformanceCustomDataset: PerformanceCustomDataset, + CaseType.StreamingPerformanceCase: StreamingPerformanceCase, + CaseType.StreamingCustomDataset: StreamingCustomDataset, + CaseType.NewIntFilterPerformanceCase: NewIntFilterPerformanceCase, + CaseType.LabelFilterPerformanceCase: LabelFilterPerformanceCase, } diff --git a/vectordb_bench/backend/clients/__init__.py b/vectordb_bench/backend/clients/__init__.py index cfef0283f..d69c54504 100644 --- a/vectordb_bench/backend/clients/__init__.py +++ b/vectordb_bench/backend/clients/__init__.py @@ -27,6 +27,7 @@ class DB(Enum): Pinecone = "Pinecone" ElasticCloud = "ElasticCloud" QdrantCloud = "QdrantCloud" + QdrantLocal = "QdrantLocal" WeaviateCloud = "WeaviateCloud" PgVector = "PgVector" PgVectoRS = "PgVectoRS" @@ -37,17 +38,27 @@ class DB(Enum): MemoryDB = "MemoryDB" Chroma = "Chroma" AWSOpenSearch = "OpenSearch" + OSSOpenSearch = "OSSOpenSearch" AliyunElasticsearch = "AliyunElasticsearch" MariaDB = "MariaDB" Test = "test" AliyunOpenSearch = "AliyunOpenSearch" MongoDB = "MongoDB" TiDB = "TiDB" + CockroachDB = "CockroachDB" Clickhouse = "Clickhouse" Vespa = "Vespa" + LanceDB = "LanceDB" + OceanBase = "OceanBase" + S3Vectors = "S3Vectors" + Hologres = "Alibaba Cloud Hologres" + TencentElasticsearch = "TencentElasticsearch" + AliSQL = "AlibabaCloudRDSMySQL" + Doris = "Doris" + TurboPuffer = "TurboPuffer" @property - def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901 + def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901, PLR0915 """Import while in use""" if self == DB.Milvus: from .milvus.milvus import Milvus @@ -74,6 +85,11 @@ def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901 return QdrantCloud + if self == DB.QdrantLocal: + from .qdrant_local.qdrant_local import QdrantLocal + + return QdrantLocal + if self == DB.WeaviateCloud: from .weaviate_cloud.weaviate_cloud import WeaviateCloud @@ -119,6 +135,11 @@ def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901 return AWSOpenSearch + if self == DB.OSSOpenSearch: + from .oss_opensearch.oss_opensearch import OSSOpenSearch + + return OSSOpenSearch + if self == DB.Clickhouse: from .clickhouse.clickhouse import Clickhouse @@ -144,6 +165,11 @@ def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901 return MongoDB + if self == DB.OceanBase: + from .oceanbase.oceanbase import OceanBase + + return OceanBase + if self == DB.MariaDB: from .mariadb.mariadb import MariaDB @@ -154,6 +180,19 @@ def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901 return TiDB + if self == DB.CockroachDB: + from .cockroachdb.cockroachdb import CockroachDB + + return CockroachDB + if self == DB.Doris: + from .doris.doris import Doris + + return Doris + if self == DB.TurboPuffer: + from .turbopuffer.turbopuffer import TurboPuffer + + return TurboPuffer + if self == DB.Test: from .test.test import Test @@ -164,11 +203,36 @@ def init_cls(self) -> type[VectorDB]: # noqa: PLR0911, PLR0912, C901 return Vespa + if self == DB.LanceDB: + from .lancedb.lancedb import LanceDB + + return LanceDB + + if self == DB.S3Vectors: + from .s3_vectors.s3_vectors import S3Vectors + + return S3Vectors + + if self == DB.Hologres: + from .hologres.hologres import Hologres + + return Hologres + + if self == DB.TencentElasticsearch: + from .tencent_elasticsearch.tencent_elasticsearch import TencentElasticsearch + + return TencentElasticsearch + + if self == DB.AliSQL: + from .alisql.alisql import AliSQL + + return AliSQL + msg = f"Unknown DB: {self.name}" raise ValueError(msg) @property - def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912, C901 + def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912, C901, PLR0915 """Import while in use""" if self == DB.Milvus: from .milvus.config import MilvusConfig @@ -195,6 +259,11 @@ def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912, C901 return QdrantConfig + if self == DB.QdrantLocal: + from .qdrant_local.config import QdrantLocalConfig + + return QdrantLocalConfig + if self == DB.WeaviateCloud: from .weaviate_cloud.config import WeaviateConfig @@ -240,6 +309,11 @@ def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912, C901 return AWSOpenSearchConfig + if self == DB.OSSOpenSearch: + from .oss_opensearch.config import OSSOpenSearchConfig + + return OSSOpenSearchConfig + if self == DB.Clickhouse: from .clickhouse.config import ClickhouseConfig @@ -265,6 +339,11 @@ def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912, C901 return MongoDBConfig + if self == DB.OceanBase: + from .oceanbase.config import OceanBaseConfig + + return OceanBaseConfig + if self == DB.MariaDB: from .mariadb.config import MariaDBConfig @@ -275,6 +354,19 @@ def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912, C901 return TiDBConfig + if self == DB.CockroachDB: + from .cockroachdb.config import CockroachDBConfig + + return CockroachDBConfig + if self == DB.Doris: + from .doris.config import DorisConfig + + return DorisConfig + if self == DB.TurboPuffer: + from .turbopuffer.config import TurboPufferConfig + + return TurboPufferConfig + if self == DB.Test: from .test.config import TestConfig @@ -285,10 +377,35 @@ def config_cls(self) -> type[DBConfig]: # noqa: PLR0911, PLR0912, C901 return VespaConfig + if self == DB.LanceDB: + from .lancedb.config import LanceDBConfig + + return LanceDBConfig + + if self == DB.S3Vectors: + from .s3_vectors.config import S3VectorsConfig + + return S3VectorsConfig + + if self == DB.Hologres: + from .hologres.config import HologresConfig + + return HologresConfig + + if self == DB.TencentElasticsearch: + from .tencent_elasticsearch.config import TencentElasticsearchConfig + + return TencentElasticsearchConfig + + if self == DB.AliSQL: + from .alisql.config import AliSQLConfig + + return AliSQLConfig + msg = f"Unknown DB: {self.name}" raise ValueError(msg) - def case_config_cls( # noqa: C901, PLR0911, PLR0912 + def case_config_cls( # noqa: C901, PLR0911, PLR0912, PLR0915 self, index_type: IndexType | None = None, ) -> type[DBCaseConfig]: @@ -312,6 +429,11 @@ def case_config_cls( # noqa: C901, PLR0911, PLR0912 return QdrantIndexConfig + if self == DB.QdrantLocal: + from .qdrant_local.config import QdrantLocalIndexConfig + + return QdrantLocalIndexConfig + if self == DB.WeaviateCloud: from .weaviate_cloud.config import WeaviateIndexConfig @@ -332,6 +454,11 @@ def case_config_cls( # noqa: C901, PLR0911, PLR0912 return AWSOpenSearchIndexConfig + if self == DB.OSSOpenSearch: + from .oss_opensearch.config import OSSOpenSearchIndexConfig + + return OSSOpenSearchIndexConfig + if self == DB.Clickhouse: from .clickhouse.config import ClickhouseHNSWConfig @@ -367,6 +494,11 @@ def case_config_cls( # noqa: C901, PLR0911, PLR0912 return MongoDBIndexConfig + if self == DB.OceanBase: + from .oceanbase.config import _oceanbase_case_config + + return _oceanbase_case_config.get(index_type) + if self == DB.MariaDB: from .mariadb.config import _mariadb_case_config @@ -377,12 +509,56 @@ def case_config_cls( # noqa: C901, PLR0911, PLR0912 return TiDBIndexConfig + if self == DB.CockroachDB: + from .cockroachdb.config import _cockroachdb_case_config + + return _cockroachdb_case_config.get(index_type) + if self == DB.Vespa: from .vespa.config import VespaHNSWConfig return VespaHNSWConfig - # DB.Pinecone, DB.Chroma, DB.Redis + if self == DB.LanceDB: + from .lancedb.config import _lancedb_case_config + + return _lancedb_case_config.get(index_type) + + if self == DB.S3Vectors: + from .s3_vectors.config import S3VectorsIndexConfig + + return S3VectorsIndexConfig + if self == DB.Hologres: + from .hologres.config import HologresIndexConfig + + return HologresIndexConfig + + if self == DB.TencentElasticsearch: + from .tencent_elasticsearch.config import TencentElasticsearchIndexConfig + + return TencentElasticsearchIndexConfig + + if self == DB.AliSQL: + from .alisql.alisql import AliSQLIndexConfig + + return AliSQLIndexConfig + + if self == DB.Doris: + from .doris.config import DorisCaseConfig + + return DorisCaseConfig + + if self == DB.TurboPuffer: + from .turbopuffer.config import TurboPufferIndexConfig + + return TurboPufferIndexConfig + + if self == DB.Chroma: + from .chroma.config import ChromaIndexConfig + + return ChromaIndexConfig + + # DB.Pinecone, DB.Redis return EmptyDBCaseConfig diff --git a/vectordb_bench/backend/clients/alisql/alisql.py b/vectordb_bench/backend/clients/alisql/alisql.py new file mode 100644 index 000000000..f88cf9d88 --- /dev/null +++ b/vectordb_bench/backend/clients/alisql/alisql.py @@ -0,0 +1,209 @@ +import logging +from contextlib import contextmanager + +import mysql.connector as mysql +import numpy as np + +from ..api import VectorDB +from .config import AliSQLConfigDict, AliSQLIndexConfig + +log = logging.getLogger(__name__) + + +class AliSQL(VectorDB): + def __init__( + self, + dim: int, + db_config: AliSQLConfigDict, + db_case_config: AliSQLIndexConfig, + collection_name: str = "vec_collection", + drop_old: bool = False, + **kwargs, + ): + self.name = "AliSQL" + self.db_config = db_config + self.case_config = db_case_config + self.table_name = collection_name + self.dim = dim + + # construct basic units + self.conn, self.cursor = self._create_connection() + + if drop_old: + self._drop_db() + self._create_db_table(dim) + + self.cursor.close() + self.conn.close() + self.cursor = None + self.conn = None + + def _create_connection(self): + conn = mysql.connect( + host=self.db_config["host"], + user=self.db_config["user"], + port=self.db_config["port"], + password=self.db_config["password"], + buffered=True, + ) + cursor = conn.cursor() + + assert conn is not None, "Connection is not initialized" + assert cursor is not None, "Cursor is not initialized" + + return conn, cursor + + def _drop_db(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + log.info(f'{self.name} client drop db : {self.db_config["database"]}') + + # flush tables before dropping database to avoid some locking issue + self.cursor.execute("FLUSH TABLES") + self.cursor.execute(f'DROP DATABASE IF EXISTS {self.db_config["database"]}') + self.cursor.execute("COMMIT") + self.cursor.execute("FLUSH TABLES") + + def _create_db_table(self, dim: int): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + try: + log.info(f'{self.name} client create database : {self.db_config["database"]}') + self.cursor.execute(f'CREATE DATABASE {self.db_config["database"]}') + + log.info(f"{self.name} client create table : {self.table_name}") + self.cursor.execute(f'USE {self.db_config["database"]}') + + self.cursor.execute(f""" + CREATE TABLE {self.table_name} ( + id INT PRIMARY KEY, + v VECTOR({self.dim}) NOT NULL + ) + """) + self.cursor.execute("COMMIT") + + except Exception as e: + log.warning(f"Failed to create table: {self.table_name} error: {e}") + raise e from None + + @contextmanager + def init(self): + """create and destory connections to database. + + Examples: + >>> with self.init(): + >>> self.insert_embeddings() + """ + self.conn, self.cursor = self._create_connection() + + index_param = self.case_config.index_param() + search_param = self.case_config.search_param() + + self.cursor.execute("SET sql_mode = ''") + + if index_param["index_type"] == "HNSW": + if search_param["ef_search"] is not None: + self.cursor.execute(f"SET SESSION vidx_hnsw_ef_search = {search_param['ef_search']}") + self.cursor.execute("COMMIT") + + self.insert_sql = ( + f'INSERT INTO {self.db_config["database"]}.{self.table_name} (id, v) VALUES (%s, %s)' # noqa: S608 + ) + self.select_sql = ( + f'SELECT id FROM {self.db_config["database"]}.{self.table_name} ' # noqa: S608 + f"ORDER by vec_distance_{search_param['metric_type']}(v, %s) LIMIT %s" + ) + self.select_sql_with_filter = ( + f'SELECT id FROM {self.db_config["database"]}.{self.table_name} WHERE id >= %s ' # noqa: S608 + f"ORDER by vec_distance_{search_param['metric_type']}(v, %s) LIMIT %s" + ) + + try: + yield + finally: + self.cursor.close() + self.conn.close() + self.cursor = None + self.conn = None + + def ready_to_load(self) -> bool: + pass + + def optimize(self, data_size: int) -> None: + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + index_param = self.case_config.index_param() + + try: + index_options = f"DISTANCE={index_param['metric_type']}" + if index_param["index_type"] == "HNSW" and index_param["M"] is not None: + index_options += f" M={index_param['M']}" + + self.cursor.execute(f""" + ALTER TABLE {self.db_config["database"]}.{self.table_name} + ADD VECTOR KEY v(v) {index_options} + """) + self.cursor.execute("COMMIT") + + except Exception as e: + log.warning(f"Failed to create index: {self.table_name} error: {e}") + raise e from None + + @staticmethod + def vector_to_hex(v): # noqa: ANN001 + return np.array(v, "float32").tobytes() + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + **kwargs, + ) -> tuple[int, Exception]: + """Insert embeddings into the database. + Should call self.init() first. + """ + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + try: + metadata_arr = np.array(metadata) + embeddings_arr = np.array(embeddings) + + batch_data = [] + for i, row in enumerate(metadata_arr): + batch_data.append((int(row), self.vector_to_hex(embeddings_arr[i]))) + + self.cursor.executemany(self.insert_sql, batch_data) + self.cursor.execute("COMMIT") + self.cursor.execute("FLUSH TABLES") + + return len(metadata), None + except Exception as e: + log.warning(f"Failed to insert data into Vector table ({self.table_name}), error: {e}") + return 0, e + + def search_embedding( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + timeout: int | None = None, + **kwargs, + ) -> list[int]: + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + search_param = self.case_config.search_param() # noqa: F841 + + try: + if filters: + self.cursor.execute(self.select_sql_with_filter, (filters.get("id"), self.vector_to_hex(query), k)) + else: + self.cursor.execute(self.select_sql, (self.vector_to_hex(query), k)) + return [row[0] for row in self.cursor.fetchall()] + + except mysql.Error: + log.exception("Failed to execute search query") + raise diff --git a/vectordb_bench/backend/clients/alisql/cli.py b/vectordb_bench/backend/clients/alisql/cli.py new file mode 100644 index 000000000..aee3ec2b2 --- /dev/null +++ b/vectordb_bench/backend/clients/alisql/cli.py @@ -0,0 +1,111 @@ +from typing import Annotated, Unpack + +import click +from pydantic import SecretStr + +from vectordb_bench.backend.clients import DB + +from ....cli.cli import ( + CommonTypedDict, + cli, + click_parameter_decorators_from_typed_dict, + run, +) + + +class AliSQLTypedDict(CommonTypedDict): + user_name: Annotated[ + str, + click.option( + "--username", + type=str, + help="Username", + required=True, + ), + ] + password: Annotated[ + str, + click.option( + "--password", + type=str, + help="Password", + required=True, + ), + ] + + host: Annotated[ + str, + click.option( + "--host", + type=str, + help="Db host", + default="127.0.0.1", + ), + ] + + port: Annotated[ + int, + click.option( + "--port", + type=int, + default=3306, + help="Db Port", + ), + ] + + database: Annotated[ + str, + click.option( + "--database", + type=str, + help="Database name", + default="vectordbbench", + ), + ] + + +class AliSQLHNSWTypedDict(AliSQLTypedDict): + m: Annotated[ + int | None, + click.option( + "--m", + type=int, + help="M parameter in HNSW vector indexing", + required=False, + ), + ] + + ef_search: Annotated[ + int | None, + click.option( + "--ef-search", + type=int, + help="AliSQL system variable vidx_hnsw_ef_search", + required=False, + ), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(AliSQLHNSWTypedDict) +def AliSQLHNSW( + **parameters: Unpack[AliSQLHNSWTypedDict], +): + from .config import AliSQLConfig, AliSQLHNSWConfig + + run( + db=DB.AliSQL, + db_config=AliSQLConfig( + db_label=parameters["db_label"], + user_name=parameters["username"], + password=SecretStr(parameters["password"]), + host=parameters["host"], + port=parameters["port"], + database=parameters["database"], + ), + db_case_config=AliSQLHNSWConfig( + M=parameters["m"], + ef_search=parameters["ef_search"], + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/alisql/config.py b/vectordb_bench/backend/clients/alisql/config.py new file mode 100644 index 000000000..f942c30f1 --- /dev/null +++ b/vectordb_bench/backend/clients/alisql/config.py @@ -0,0 +1,72 @@ +from typing import TypedDict + +from pydantic import BaseModel, SecretStr + +from ..api import DBCaseConfig, DBConfig, IndexType, MetricType + + +class AliSQLConfigDict(TypedDict): + """These keys will be directly used as kwargs in alisql connection string, + so the names must match exactly alisql API""" + + user: str + password: str + host: str + port: int + database: str + + +class AliSQLConfig(DBConfig): + user_name: str = "root" + password: SecretStr + host: str = "127.0.0.1" + port: int = 3306 + database: str = "vectordbbench" + + def to_dict(self) -> AliSQLConfigDict: + pwd_str = self.password.get_secret_value() + return { + "host": self.host, + "port": self.port, + "user": self.user_name, + "password": pwd_str, + "database": self.database, + } + + +class AliSQLIndexConfig(BaseModel): + """Base config for AliSQL""" + + metric_type: MetricType | None = None + + def parse_metric(self) -> str: + if self.metric_type == MetricType.L2: + return "euclidean" + if self.metric_type == MetricType.COSINE: + return "cosine" + msg = f"Metric type {self.metric_type} is not supported!" + raise ValueError(msg) + + +class AliSQLHNSWConfig(AliSQLIndexConfig, DBCaseConfig): + M: int | None + ef_search: int | None + index: IndexType = IndexType.HNSW + + def index_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "M": self.M, + } + + def search_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "ef_search": self.ef_search, + } + + +_alisql_case_config = { + IndexType.HNSW: AliSQLHNSWConfig, +} diff --git a/vectordb_bench/backend/clients/aliyun_opensearch/aliyun_opensearch.py b/vectordb_bench/backend/clients/aliyun_opensearch/aliyun_opensearch.py index 324871934..138e10bba 100644 --- a/vectordb_bench/backend/clients/aliyun_opensearch/aliyun_opensearch.py +++ b/vectordb_bench/backend/clients/aliyun_opensearch/aliyun_opensearch.py @@ -3,11 +3,9 @@ import time from contextlib import contextmanager -from alibabacloud_ha3engine_vector import client, models +from alibabacloud_ha3engine_vector import models +from alibabacloud_ha3engine_vector.client import Client from alibabacloud_ha3engine_vector.models import QueryRequest -from alibabacloud_searchengine20211025 import models as searchengine_models -from alibabacloud_searchengine20211025.client import Client as searchengineClient -from alibabacloud_tea_openapi import models as open_api_models from ..api import MetricType, VectorDB from .config import AliyunOpenSearchIndexConfig @@ -28,18 +26,25 @@ def __init__( drop_old: bool = False, **kwargs, ): - self.control_client = None self.dim = dim self.db_config = db_config self.case_config = db_case_config self.collection_name = collection_name self.instance_id = db_config["host"].split(".")[0].replace("http://", "").replace("https://", "") + self.config = models.Config( + endpoint=self.db_config["host"], + protocol="http", + access_user_name=self.db_config["user"], + access_pass_word=self.db_config["password"], + ) self._primary_field = "id" self._scalar_field = "int_id" self._vector_field = "vector" self._index_name = "vector_idx" + client = Client(self.config) + self.batch_size = int( min( ALIYUN_OPENSEARCH_MAX_SIZE_PER_BATCH / (dim * 25), @@ -48,22 +53,16 @@ def __init__( ) log.info(f"Aliyun_OpenSearch client config: {self.db_config}") - control_config = open_api_models.Config( - access_key_id=self.db_config["ak"], - access_key_secret=self.db_config["sk"], - endpoint=self.db_config["control_host"], - ) - self.control_client = searchengineClient(control_config) if drop_old: log.info(f"aliyun_OpenSearch client drop old index: {self.collection_name}") - if self._index_exists(self.control_client): - self._modify_index(self.control_client) + if self._index_exists(client): + self._modify_index(client) else: - self._create_index(self.control_client) + self._create_index(client) - def _create_index(self, client: searchengineClient): - create_table_request = searchengine_models.CreateTableRequest() + def _create_index(self, client: Client): + create_table_request = models.CreateTableRequest() create_table_request.name = self.collection_name create_table_request.primary_key = self._primary_field create_table_request.partition_count = 1 @@ -72,14 +71,14 @@ def _create_index(self, client: searchengineClient): self._vector_field: "MULTI_FLOAT", self._scalar_field: "INT64", } - vector_index = searchengine_models.ModifyTableRequestVectorIndex() + vector_index = models.ModifyTableRequestVectorIndex() vector_index.index_name = self._index_name vector_index.dimension = self.dim vector_index.distance_type = self.case_config.distance_type() vector_index.vector_field = self._vector_field vector_index.vector_index_type = "HNSW" - advance_params = searchengine_models.ModifyTableRequestVectorIndexAdvanceParams() + advance_params = models.ModifyTableRequestVectorIndexAdvanceParams() str_max_neighbor_count = f'"proxima.hnsw.builder.max_neighbor_count":{self.case_config.M}' str_efc = f'"proxima.hnsw.builder.efconstruction":{self.case_config.ef_construction}' str_enable_adsampling = '"proxima.hnsw.builder.enable_adsampling":true' @@ -95,7 +94,7 @@ def _create_index(self, client: searchengineClient): str_thread_count, ], ) - advance_params.build_index_params = params + advance_params.build_index_params = "{" + params + "}" advance_params.search_index_params = ( '{"proxima.hnsw.searcher.ef":400,"proxima.hnsw.searcher.dynamic_termination.prob_threshold":0.7}' ) @@ -103,7 +102,7 @@ def _create_index(self, client: searchengineClient): create_table_request.vector_index = [vector_index] try: - response = client.create_table(self.instance_id, create_table_request) + response = client.create_table(create_table_request) log.info(f"create table success: {response.body}") except Exception as error: log.info(error.message) @@ -115,20 +114,20 @@ def _create_index(self, client: searchengineClient): self._active_index(client) # check if index create success - def _active_index(self, client: searchengineClient) -> None: + def _active_index(self, client: Client) -> None: retry_times = 0 while True: time.sleep(10) log.info(f"begin to {retry_times} times get table") retry_times += 1 - response = client.get_table(self.instance_id, self.collection_name) + response = client.get_table(self.collection_name) if response.body.result.status == "IN_USE": log.info(f"{self.collection_name} table begin to use.") return - def _index_exists(self, client: searchengineClient) -> bool: + def _index_exists(self, client: Client) -> bool: try: - client.get_table(self.instance_id, self.collection_name) + client.get_table(self.collection_name) except Exception as err: log.warning(f"get table from searchengine error, err={err}") return False @@ -136,7 +135,7 @@ def _index_exists(self, client: searchengineClient) -> bool: return True # check if index build success, Insert the embeddings to the vector database after index build success - def _index_build_success(self, client: searchengineClient) -> None: + def _index_build_success(self, client: Client) -> None: log.info("begin to check if table build success.") time.sleep(50) @@ -145,10 +144,10 @@ def _index_build_success(self, client: searchengineClient) -> None: time.sleep(10) log.info(f"begin to {retry_times} times get table fsm") retry_times += 1 - request = searchengine_models.ListTasksRequest() - request.start = (int(time.time()) - 3600) * 1000 - request.end = int(time.time()) * 1000 - response = client.list_tasks(self.instance_id, request) + request = models.ListTasksRequest() + request.start = int(time.time()) - 3600 + request.end = int(time.time()) + response = client.list_tasks(request) fsms = response.body.result cur_fsm = None for fsm in fsms: @@ -164,11 +163,11 @@ def _index_build_success(self, client: searchengineClient) -> None: if cur_fsm["status"] == "success": return - def _modify_index(self, client: searchengineClient) -> None: + def _modify_index(self, client: Client) -> None: # check if index create success self._active_index(client) - modify_table_request = searchengine_models.ModifyTableRequest() + modify_table_request = models.ModifyTableRequest() modify_table_request.partition_count = 1 modify_table_request.primary_key = self._primary_field modify_table_request.field_schema = { @@ -176,13 +175,13 @@ def _modify_index(self, client: searchengineClient) -> None: self._vector_field: "MULTI_FLOAT", self._scalar_field: "INT64", } - vector_index = searchengine_models.ModifyTableRequestVectorIndex() + vector_index = models.ModifyTableRequestVectorIndex() vector_index.index_name = self._index_name vector_index.dimension = self.dim vector_index.distance_type = self.case_config.distance_type() vector_index.vector_field = self._vector_field vector_index.vector_index_type = "HNSW" - advance_params = searchengine_models.ModifyTableRequestVectorIndexAdvanceParams() + advance_params = models.ModifyTableRequestVectorIndexAdvanceParams() str_max_neighbor_count = f'"proxima.hnsw.builder.max_neighbor_count":{self.case_config.M}' str_efc = f'"proxima.hnsw.builder.efconstruction":{self.case_config.ef_construction}' @@ -199,7 +198,7 @@ def _modify_index(self, client: searchengineClient) -> None: str_thread_count, ], ) - advance_params.build_index_params = params + advance_params.build_index_params = "{" + params + "}" advance_params.search_index_params = ( '{"proxima.hnsw.searcher.ef":400,"proxima.hnsw.searcher.dynamic_termination.prob_threshold":0.7}' ) @@ -209,7 +208,6 @@ def _modify_index(self, client: searchengineClient) -> None: try: response = client.modify_table( - self.instance_id, self.collection_name, modify_table_request, ) @@ -240,14 +238,8 @@ def _get_total_count(self): @contextmanager def init(self) -> None: """connect to aliyun opensearch""" - config = models.Config( - endpoint=self.db_config["host"], - protocol="http", - access_user_name=self.db_config["user"], - access_pass_word=self.db_config["password"], - ) - self.client = client.Client(config) + self.client = Client(self.config) yield self.client = None diff --git a/vectordb_bench/backend/clients/aliyun_opensearch/config.py b/vectordb_bench/backend/clients/aliyun_opensearch/config.py index e215b7d68..0d29be382 100644 --- a/vectordb_bench/backend/clients/aliyun_opensearch/config.py +++ b/vectordb_bench/backend/clients/aliyun_opensearch/config.py @@ -12,18 +12,11 @@ class AliyunOpenSearchConfig(DBConfig, BaseModel): user: str = "" password: SecretStr = "" - ak: str = "" - sk: SecretStr = "" - control_host: str = "searchengine.cn-hangzhou.aliyuncs.com" - def to_dict(self) -> dict: return { "host": self.host, "user": self.user, "password": self.password.get_secret_value(), - "ak": self.ak, - "sk": self.sk.get_secret_value(), - "control_host": self.control_host, } diff --git a/vectordb_bench/backend/clients/api.py b/vectordb_bench/backend/clients/api.py index ce2b05650..822821b34 100644 --- a/vectordb_bench/backend/clients/api.py +++ b/vectordb_bench/backend/clients/api.py @@ -4,6 +4,8 @@ from pydantic import BaseModel, SecretStr, validator +from vectordb_bench.backend.filter import Filter, FilterOp + class MetricType(str, Enum): L2 = "L2" @@ -16,19 +18,41 @@ class MetricType(str, Enum): class IndexType(str, Enum): HNSW = "HNSW" + HNSW_SQ = "HNSW_SQ" + HNSW_BQ = "HNSW_BQ" + HNSW_PQ = "HNSW_PQ" + HNSW_PRQ = "HNSW_PRQ" DISKANN = "DISKANN" STREAMING_DISKANN = "DISKANN" IVFFlat = "IVF_FLAT" + IVFPQ = "IVF_PQ" IVFSQ8 = "IVF_SQ8" + IVF_RABITQ = "IVF_RABITQ" Flat = "FLAT" AUTOINDEX = "AUTOINDEX" ES_HNSW = "hnsw" + ES_HNSW_INT8 = "int8_hnsw" + ES_HNSW_INT4 = "int4_hnsw" + ES_HNSW_BBQ = "bbq_hnsw" + TES_VSEARCH = "vsearch" ES_IVFFlat = "ivfflat" GPU_IVF_FLAT = "GPU_IVF_FLAT" GPU_BRUTE_FORCE = "GPU_BRUTE_FORCE" GPU_IVF_PQ = "GPU_IVF_PQ" GPU_CAGRA = "GPU_CAGRA" SCANN = "scann" + SCANN_MILVUS = "SCANN_MILVUS" + Hologres_HGraph = "HGraph" + Hologres_Graph = "Graph" + NONE = "NONE" + + +class SQType(str, Enum): + SQ6 = "SQ6" + SQ8 = "SQ8" + BF16 = "BF16" + FP16 = "FP16" + FP32 = "FP32" class DBConfig(ABC, BaseModel): @@ -111,6 +135,22 @@ class VectorDB(ABC): >>> milvus.search_embedding() """ + "The filtering types supported by the VectorDB Client, default only non-filter" + supported_filter_types: list[FilterOp] = [FilterOp.NonFilter] + name: str = "" + + @classmethod + def filter_supported(cls, filters: Filter) -> bool: + """Ensure that the filters are supported before testing filtering cases.""" + return filters.type in cls.supported_filter_types + + def prepare_filter(self, filters: Filter): + """The vector database is allowed to pre-prepare different filter conditions + to reduce redundancy during the testing process. + + (All search tests in a case use consistent filtering conditions.)""" + return + @abstractmethod def __init__( self, @@ -161,6 +201,7 @@ def insert_embeddings( self, embeddings: list[list[float]], metadata: list[int], + labels_data: list[str] | None = None, **kwargs, ) -> tuple[int, Exception]: """Insert the embeddings to the vector database. The default number of embeddings for @@ -181,7 +222,6 @@ def search_embedding( self, query: list[float], k: int = 100, - filters: dict | None = None, ) -> list[int]: """Get k most similar embeddings to query vector. diff --git a/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py b/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py index adb766300..8d77d263b 100644 --- a/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py +++ b/vectordb_bench/backend/clients/aws_opensearch/aws_opensearch.py @@ -5,8 +5,10 @@ from opensearchpy import OpenSearch -from ..api import IndexType, VectorDB -from .config import AWSOpenSearchConfig, AWSOpenSearchIndexConfig, AWSOS_Engine +from vectordb_bench.backend.filter import Filter, FilterOp + +from ..api import VectorDB +from .config import AWSOpenSearchIndexConfig, AWSOS_Engine log = logging.getLogger(__name__) @@ -16,6 +18,12 @@ class AWSOpenSearch(VectorDB): + supported_filter_types: list[FilterOp] = [ + FilterOp.NonFilter, + FilterOp.NumGE, + FilterOp.StrEqual, + ] + def __init__( self, dim: int, @@ -23,8 +31,10 @@ def __init__( db_case_config: AWSOpenSearchIndexConfig, index_name: str = "vdb_bench_index", # must be lowercase id_col_name: str = "_id", + label_col_name: str = "label", vector_col_name: str = "embedding", drop_old: bool = False, + with_scalar_labels: bool = False, **kwargs, ): self.dim = dim @@ -32,10 +42,12 @@ def __init__( self.case_config = db_case_config self.index_name = index_name self.id_col_name = id_col_name - self.category_col_names = [f"scalar-{categoryCount}" for categoryCount in [2, 5, 10, 100, 1000]] + self.label_col_name = label_col_name self.vector_col_name = vector_col_name + self.with_scalar_labels = with_scalar_labels log.info(f"AWS_OpenSearch client config: {self.db_config}") + log.info(f"AWS_OpenSearch db case config : {self.case_config}") client = OpenSearch(**self.db_config) if drop_old: log.info(f"AWS_OpenSearch client drop old index: {self.index_name}") @@ -43,57 +55,139 @@ def __init__( if is_existed: client.indices.delete(index=self.index_name) self._create_index(client) + else: + is_existed = client.indices.exists(index=self.index_name) + if not is_existed: + self._create_index(client) + log.info(f"AWS_OpenSearch client create index: {self.index_name}") - @classmethod - def config_cls(cls) -> AWSOpenSearchConfig: - return AWSOpenSearchConfig + self._update_ef_search_before_search(client) + self._load_graphs_to_memory(client) - @classmethod - def case_config_cls(cls, index_type: IndexType | None = None) -> AWSOpenSearchIndexConfig: - return AWSOpenSearchIndexConfig + def _create_index(self, client: OpenSearch) -> None: + self._log_index_creation_info() + self._configure_cluster_settings(client) + settings = self._build_index_settings() + vector_field_config = self._build_vector_field_config() + mappings = self._build_mappings(vector_field_config) + self._create_opensearch_index(client, settings, mappings) - def _create_index(self, client: OpenSearch): + def _log_index_creation_info(self) -> None: + log.info(f"Creating index with ef_search: {self.case_config.ef_search}") + log.info(f"Creating index with number_of_replicas: {self.case_config.number_of_replicas}") + log.info(f"Creating index with engine: {self.case_config.engine}") + log.info(f"Creating index with metric type: {self.case_config.metric_type_name}") + log.info(f"All case_config parameters: {self.case_config.__dict__}") + + def _configure_cluster_settings(self, client: OpenSearch) -> None: cluster_settings_body = { "persistent": { "knn.algo_param.index_thread_qty": self.case_config.index_thread_qty, "knn.memory.circuit_breaker.limit": self.case_config.cb_threshold, } } - client.cluster.put_settings(cluster_settings_body) - settings = { + client.cluster.put_settings(body=cluster_settings_body) + + def _build_index_settings(self) -> dict: + return { "index": { "knn": True, "number_of_shards": self.case_config.number_of_shards, - "number_of_replicas": 0, + "number_of_replicas": self.case_config.number_of_replicas, "translog.flush_threshold_size": self.case_config.flush_threshold_size, - # Setting trans log threshold to 5GB - **( - {"knn.algo_param.ef_search": self.case_config.ef_search} - if self.case_config.engine == AWSOS_Engine.nmslib - else {} - ), + "knn.advanced.approximate_threshold": "-1", + "knn.algo_param.ef_search": self.case_config.ef_search, }, "refresh_interval": self.case_config.refresh_interval, } - mappings = { - "properties": { - **{categoryCol: {"type": "keyword"} for categoryCol in self.category_col_names}, - self.vector_col_name: { - "type": "knn_vector", - "dimension": self.dim, - "method": self.case_config.index_param(), + + def _build_vector_field_config(self) -> dict: + method_config = self.case_config.index_param() + log.info(f"Raw method config from index_param(): {method_config}") + + if self.case_config.engine == AWSOS_Engine.s3vector: + method_config = {"engine": "s3vector"} + + if self.case_config.on_disk: + space_type = self.case_config.parse_metric() + vector_field_config = { + "type": "knn_vector", + "dimension": self.dim, + "space_type": space_type, + "data_type": "float", + "mode": "on_disk", + "compression_level": "32x", + "method": method_config, + } + log.info("Using on-disk vector configuration with compression_level: 32x") + else: + vector_field_config = { + "type": "knn_vector", + "dimension": self.dim, + "method": method_config, + } + + if self.case_config.on_disk: + log.info(f"Final on-disk vector field config: {vector_field_config}") + elif self.case_config.engine == AWSOS_Engine.s3vector: + space_type = self.case_config.parse_metric() + vector_field_config["space_type"] = space_type + vector_field_config["method"] = {"engine": "s3vector"} + log.info(f"Final vector field config for s3vector: {vector_field_config}") + else: + log.info(f"Standard vector field config: {vector_field_config}") + + return vector_field_config + + def _build_mappings(self, vector_field_config: dict) -> dict: + if self.case_config.engine == AWSOS_Engine.s3vector: + mappings = { + "properties": { + self.label_col_name: {"type": "keyword"}, + self.vector_col_name: vector_field_config, }, - }, - } + } + log.info("Using simplified mappings for s3vector engine (no _source configuration)") + else: + mappings = { + "_source": {"excludes": [self.vector_col_name], "recovery_source_excludes": [self.vector_col_name]}, + "properties": { + self.label_col_name: {"type": "keyword"}, + self.vector_col_name: vector_field_config, + }, + } + log.info("Using standard mappings with _source configuration for non-s3vector engines") + return mappings + + def _create_opensearch_index(self, client: OpenSearch, settings: dict, mappings: dict) -> None: try: + log.info(f"Creating index with settings: {settings}") + log.info(f"Creating index with mappings: {mappings}") + + if self.case_config.engine == AWSOS_Engine.s3vector: + method_in_mappings = mappings["properties"][self.vector_col_name]["method"] + log.info(f"Final method config being sent to OpenSearch: {method_in_mappings}") + client.indices.create( index=self.index_name, body={"settings": settings, "mappings": mappings}, ) + + if self.case_config.engine == AWSOS_Engine.s3vector: + self._verify_s3vector_index_config(client) + except Exception as e: log.warning(f"Failed to create index: {self.index_name} error: {e!s}") raise e from None + def _verify_s3vector_index_config(self, client: OpenSearch) -> None: + try: + actual_mapping = client.indices.get_mapping(index=self.index_name) + actual_method = actual_mapping[self.index_name]["mappings"]["properties"][self.vector_col_name]["method"] + log.info(f"Actual method config in created index: {actual_method}") + except Exception as e: + log.warning(f"Failed to verify index configuration: {e}") + @contextmanager def init(self) -> None: """connect to opensearch""" @@ -107,53 +201,206 @@ def insert_embeddings( self, embeddings: Iterable[list[float]], metadata: list[int], + labels_data: list[str] | None = None, **kwargs, ) -> tuple[int, Exception]: """Insert the embeddings to the opensearch.""" assert self.client is not None, "should self.init() first" + num_clients = self.case_config.number_of_indexing_clients or 1 + log.info(f"Number of indexing clients from case_config: {num_clients}") + + if num_clients <= 1: + log.info("Using single client for data insertion") + return self._insert_with_single_client(embeddings, metadata, labels_data) + log.info(f"Using {num_clients} parallel clients for data insertion") + return self._insert_with_multiple_clients(embeddings, metadata, num_clients, labels_data) + + def _insert_with_single_client( + self, + embeddings: Iterable[list[float]], + metadata: list[int], + labels_data: list[str] | None = None, + ) -> tuple[int, Exception]: insert_data = [] for i in range(len(embeddings)): - insert_data.append( - {"index": {"_index": self.index_name, self.id_col_name: metadata[i]}}, - ) - insert_data.append({self.vector_col_name: embeddings[i]}) + index_data = {"index": {"_index": self.index_name, self.id_col_name: metadata[i]}} + if self.with_scalar_labels and self.case_config.use_routing and labels_data is not None: + index_data["routing"] = labels_data[i] + insert_data.append(index_data) + + other_data = {self.vector_col_name: embeddings[i]} + if self.with_scalar_labels and labels_data is not None: + other_data[self.label_col_name] = labels_data[i] + insert_data.append(other_data) + try: - resp = self.client.bulk(insert_data) - log.info(f"AWS_OpenSearch adding documents: {len(resp['items'])}") - resp = self.client.indices.stats(self.index_name) - log.info( - f"Total document count in index: {resp['_all']['primaries']['indexing']['index_total']}", - ) - return (len(embeddings), None) + self.client.bulk(body=insert_data) + return len(embeddings), None except Exception as e: log.warning(f"Failed to insert data: {self.index_name} error: {e!s}") time.sleep(10) - return self.insert_embeddings(embeddings, metadata) + return self._insert_with_single_client(embeddings, metadata, labels_data) + + def _insert_with_multiple_clients( + self, + embeddings: Iterable[list[float]], + metadata: list[int], + num_clients: int, + labels_data: list[str] | None = None, + ) -> tuple[int, Exception]: + import concurrent.futures + from concurrent.futures import ThreadPoolExecutor + + embeddings_list = list(embeddings) + chunk_size = max(1, len(embeddings_list) // num_clients) + chunks = [] + + for i in range(0, len(embeddings_list), chunk_size): + end = min(i + chunk_size, len(embeddings_list)) + chunk_labels = labels_data[i:end] if labels_data is not None else None + chunks.append((embeddings_list[i:end], metadata[i:end], chunk_labels)) + + clients = [] + for _ in range(min(num_clients, len(chunks))): + client = OpenSearch(**self.db_config) + clients.append(client) + + log.info(f"AWS_OpenSearch using {len(clients)} parallel clients for data insertion") + + def insert_chunk(client_idx: int, chunk_idx: int): + chunk_embeddings, chunk_metadata, chunk_labels_data = chunks[chunk_idx] + client = clients[client_idx] + + insert_data = [] + for i in range(len(chunk_embeddings)): + index_data = {"index": {"_index": self.index_name, self.id_col_name: chunk_metadata[i]}} + if self.with_scalar_labels and self.case_config.use_routing and chunk_labels_data is not None: + index_data["routing"] = chunk_labels_data[i] + insert_data.append(index_data) + + other_data = {self.vector_col_name: chunk_embeddings[i]} + if self.with_scalar_labels and chunk_labels_data is not None: + other_data[self.label_col_name] = chunk_labels_data[i] + insert_data.append(other_data) + + max_retries = 10 + for attempt in range(max_retries): + try: + client.bulk(body=insert_data) + return len(chunk_embeddings), None + except Exception as e: + if "429" in str(e) and attempt < max_retries - 1: + log.warning(f"Client {client_idx} got 429 error, retry {attempt + 1}/{max_retries} after 10s") + time.sleep(10) + else: + log.warning(f"Client {client_idx} failed to insert data: {e!s}") + return 0, e + return 0, Exception("Max retries exceeded") + + results = [] + with ThreadPoolExecutor(max_workers=len(clients)) as executor: + futures = [] + + for chunk_idx in range(len(chunks)): + client_idx = chunk_idx % len(clients) + futures.append(executor.submit(insert_chunk, client_idx, chunk_idx)) + + for future in concurrent.futures.as_completed(futures): + count, error = future.result() + results.append((count, error)) + + from contextlib import suppress + + for client in clients: + with suppress(Exception): + client.close() + + total_count = sum(count for count, _ in results) + errors = [error for _, error in results if error is not None] + + if errors: + log.warning("Some clients failed to insert data, retrying with single client") + time.sleep(10) + return self._insert_with_single_client(embeddings, metadata) + + resp = self.client.indices.stats(index=self.index_name) + log.info( + f"""Total document count in index after parallel insertion: + {resp['_all']['primaries']['indexing']['index_total']}""", + ) + + return (total_count, None) + + def _update_ef_search_before_search(self, client: OpenSearch): + ef_search_value = self.case_config.ef_search + try: + index_settings = client.indices.get_settings(index=self.index_name) + current_ef_search = ( + index_settings.get(self.index_name, {}) + .get("settings", {}) + .get("index", {}) + .get("knn.algo_param", {}) + .get("ef_search") + ) + + if current_ef_search != str(ef_search_value): + log.info(f"Updating ef_search before search from {current_ef_search} to {ef_search_value}") + settings_body = {"index": {"knn.algo_param.ef_search": ef_search_value}} + client.indices.put_settings(index=self.index_name, body=settings_body) + log.info(f"Successfully updated ef_search to {ef_search_value} before search") + + log.info(f"Current engine: {self.case_config.engine}") + log.info(f"Current metric_type: {self.case_config.metric_type_name}") + + except Exception as e: + log.warning(f"Failed to update ef_search parameter before search: {e}") def search_embedding( self, query: list[float], k: int = 100, - filters: dict | None = None, + **kwargs, ) -> list[int]: """Get k most similar embeddings to query vector. Args: query(list[float]): query embedding to look up documents similar to. k(int): Number of most similar embeddings to return. Defaults to 100. - filters(dict, optional): filtering expression to filter the data while searching. Returns: - list[tuple[int, float]]: list of k most similar embeddings in (id, score) tuple to the query embedding. + list[int]: list of k most similar ids to the query embedding. """ assert self.client is not None, "should self.init() first" + # Configure query based on engine type + if self.case_config.engine == AWSOS_Engine.s3vector: + # For s3vector engine, use simplified query without method_parameters + knn_query = { + "vector": query, + "k": k, + **({"filter": self.filter} if self.filter else {}), + } + log.debug("Using simplified knn query for s3vector engine (no method_parameters)") + else: + # For other engines (faiss, lucene), use standard query with method_parameters + knn_query = { + "vector": query, + "k": k, + "method_parameters": self.case_config.search_param(), + **({"filter": self.filter} if self.filter else {}), + "rescore": {"oversample_factor": self.case_config.oversample_factor} + # if self.case_config.use_quant + # else {} + , + } + log.debug("Using standard knn query with method_parameters for non-s3vector engines") + body = { "size": k, - "query": {"knn": {self.vector_col_name: {"vector": query, "k": k}}}, - **({"filter": {"range": {self.id_col_name: {"gt": filters["id"]}}}} if filters else {}), + "query": {"knn": {self.vector_col_name: knn_query}}, } + try: resp = self.client.search( index=self.index_name, @@ -162,17 +409,38 @@ def search_embedding( _source=False, docvalue_fields=[self.id_col_name], stored_fields="_none_", + preference="_only_local" if self.case_config.number_of_shards == 1 else None, + routing=self.routing_key, ) log.debug(f"Search took: {resp['took']}") log.debug(f"Search shards: {resp['_shards']}") log.debug(f"Search hits total: {resp['hits']['total']}") - return [int(h["fields"][self.id_col_name][0]) for h in resp["hits"]["hits"]] + try: + return [int(h["fields"][self.id_col_name][0]) for h in resp["hits"]["hits"]] + except Exception: + # empty results + return [] except Exception as e: log.warning(f"Failed to search: {self.index_name} error: {e!s}") raise e from None + def prepare_filter(self, filters: Filter): + self.routing_key = None + if filters.type == FilterOp.NonFilter: + self.filter = None + elif filters.type == FilterOp.NumGE: + self.filter = {"range": {self.id_col_name: {"gt": filters.int_value}}} + elif filters.type == FilterOp.StrEqual: + self.filter = {"term": {self.label_col_name: filters.label_value}} + if self.case_config.use_routing: + self.routing_key = filters.label_value + else: + msg = f"Not support Filter for OpenSearch - {filters}" + raise ValueError(msg) + def optimize(self, data_size: int | None = None): """optimize will be called between insertion and search in performance cases.""" + self._update_ef_search() # Call refresh first to ensure that all segments are created self._refresh_index() if self.case_config.force_merge_enabled: @@ -182,7 +450,22 @@ def optimize(self, data_size: int | None = None): # Call refresh again to ensure that the index is ready after force merge. self._refresh_index() # ensure that all graphs are loaded in memory and ready for search - self._load_graphs_to_memory() + self._load_graphs_to_memory(self.client) + + def _update_ef_search(self): + ef_search_value = ( + self.case_config.ef_search if self.case_config.ef_search is not None else self.case_config.efSearch + ) + log.info(f"Updating ef_search parameter to: {ef_search_value}") + + settings_body = {"index": {"knn.algo_param.ef_search": ef_search_value}} + try: + self.client.indices.put_settings(index=self.index_name, body=settings_body) + log.info(f"Successfully updated ef_search to {ef_search_value}") + log.info(f"Current engine: {self.case_config.engine}") + log.info(f"Current metric_type: {self.case_config.metric_type}") + except Exception as e: + log.warning(f"Failed to update ef_search parameter: {e}") def _update_replicas(self): index_settings = self.client.indices.get_settings(index=self.index_name) @@ -200,7 +483,7 @@ def _wait_till_green(self): while True: res = self.client.cat.indices(index=self.index_name, h="health", format="json") health = res[0]["health"] - if health != "green": + if health == "green": break log.info(f"The index {self.index_name} has health : {health} and is not green. Retrying") time.sleep(SECONDS_WAITING_FOR_REPLICAS_TO_BE_ENABLED_SEC) @@ -227,19 +510,27 @@ def _do_force_merge(self): cluster_settings_body = { "persistent": {"knn.algo_param.index_thread_qty": self.case_config.index_thread_qty_during_force_merge} } - self.client.cluster.put_settings(cluster_settings_body) - log.debug(f"Starting force merge for index {self.index_name}") - force_merge_endpoint = f"/{self.index_name}/_forcemerge?max_num_segments=1&wait_for_completion=false" + self.client.cluster.put_settings(body=cluster_settings_body) + + log.info("Updating the graph threshold to ensure that during merge we can do graph creation.") + output = self.client.indices.put_settings( + index=self.index_name, body={"index.knn.advanced.approximate_threshold": "0"} + ) + log.info(f"response of updating setting is: {output}") + + log.info(f"Starting force merge for index {self.index_name}") + segments = self.case_config.number_of_segments + force_merge_endpoint = f"/{self.index_name}/_forcemerge?max_num_segments={segments}&wait_for_completion=false" force_merge_task_id = self.client.transport.perform_request("POST", force_merge_endpoint)["task"] while True: time.sleep(WAITING_FOR_FORCE_MERGE_SEC) task_status = self.client.tasks.get(task_id=force_merge_task_id) if task_status["completed"]: break - log.debug(f"Completed force merge for index {self.index_name}") + log.info(f"Completed force merge for index {self.index_name}") - def _load_graphs_to_memory(self): + def _load_graphs_to_memory(self, client: OpenSearch): if self.case_config.engine != AWSOS_Engine.lucene: log.info("Calling warmup API to load graphs into memory") warmup_endpoint = f"/_plugins/_knn/warmup/{self.index_name}" - self.client.transport.perform_request("GET", warmup_endpoint) + client.transport.perform_request("GET", warmup_endpoint) diff --git a/vectordb_bench/backend/clients/aws_opensearch/cli.py b/vectordb_bench/backend/clients/aws_opensearch/cli.py index fa457154d..a3ddb8712 100644 --- a/vectordb_bench/backend/clients/aws_opensearch/cli.py +++ b/vectordb_bench/backend/clients/aws_opensearch/cli.py @@ -1,3 +1,4 @@ +import logging from typing import Annotated, TypedDict, Unpack import click @@ -5,18 +6,21 @@ from ....cli.cli import ( CommonTypedDict, - HNSWFlavor2, + HNSWFlavor1, cli, click_parameter_decorators_from_typed_dict, run, ) from .. import DB +from .config import AWSOS_Engine, AWSOSQuantization + +log = logging.getLogger(__name__) class AWSOpenSearchTypedDict(TypedDict): host: Annotated[str, click.option("--host", type=str, help="Db host", required=True)] - port: Annotated[int, click.option("--port", type=int, default=443, help="Db Port")] - user: Annotated[str, click.option("--user", type=str, default="admin", help="Db User")] + port: Annotated[int, click.option("--port", type=int, default=80, help="Db Port")] + user: Annotated[str, click.option("--user", type=str, help="Db User")] password: Annotated[str, click.option("--password", type=str, help="Db password")] number_of_shards: Annotated[ int, @@ -38,23 +42,23 @@ class AWSOpenSearchTypedDict(TypedDict): ), ] - index_thread_qty_during_force_merge: Annotated[ - int, + engine: Annotated[ + str, click.option( - "--index-thread-qty-during-force-merge", - type=int, - help="Thread count during force merge operations", - default=4, + "--engine", + type=click.Choice(["faiss", "lucene", "s3vector"], case_sensitive=False), + help="HNSW algorithm implementation to use", + default="faiss", ), ] - number_of_indexing_clients: Annotated[ - int, + metric_type: Annotated[ + str, click.option( - "--number-of-indexing-clients", - type=int, - help="Number of concurrent indexing clients", - default=1, + "--metric-type", + type=click.Choice(["l2", "cosine", "ip"], case_sensitive=False), + help="Distance metric type for vector similarity", + default="l2", ), ] @@ -64,26 +68,26 @@ class AWSOpenSearchTypedDict(TypedDict): ] refresh_interval: Annotated[ - int, + str, click.option( "--refresh-interval", type=str, help="How often to make new data available for search", default="60s" ), ] force_merge_enabled: Annotated[ - int, + bool, click.option("--force-merge-enabled", type=bool, help="Whether to perform force merge operation", default=True), ] flush_threshold_size: Annotated[ - int, + str, click.option( "--flush-threshold-size", type=str, help="Size threshold for flushing the transaction log", default="5120mb" ), ] cb_threshold: Annotated[ - int, + str, click.option( "--cb-threshold", type=str, @@ -92,8 +96,70 @@ class AWSOpenSearchTypedDict(TypedDict): ), ] + index_thread_qty_during_force_merge: Annotated[ + int, + click.option( + "--index-thread-qty-during-force-merge", + type=int, + help="Thread count during force merge operations", + default=8, + ), + ] + + number_of_indexing_clients: Annotated[ + int, + click.option( + "--number-of-indexing-clients", + type=int, + help="Number of concurrent indexing clients", + default=1, + ), + ] + + ef_construction: Annotated[ + int | None, + click.option( + "--ef-construction", + type=int, + help="ef parameter for HNSW construction (not used for s3vector engine)", + default=None, + required=False, + ), + ] + + quantization_type: Annotated[ + str | None, + click.option( + "--quantization-type", + type=click.Choice(["fp32", "fp16", "bq"]), + help="quantization type for vectors (in index)", + default="fp32", + required=False, + ), + ] + + oversample_factor: Annotated[ + float, + click.option( + "--oversample-factor", + type=float, + help="Oversample factor for vector search", + default=1.0, + ), + ] + + on_disk: Annotated[ + bool, + click.option( + "--on-disk", + is_flag=True, + help="Enable on-disk vector storage mode", + default=False, + ), + ] + -class AWSOpenSearchHNSWTypedDict(CommonTypedDict, AWSOpenSearchTypedDict, HNSWFlavor2): ... +class AWSOpenSearchHNSWTypedDict(CommonTypedDict, AWSOpenSearchTypedDict, HNSWFlavor1): ... @cli.command() @@ -101,6 +167,21 @@ class AWSOpenSearchHNSWTypedDict(CommonTypedDict, AWSOpenSearchTypedDict, HNSWFl def AWSOpenSearch(**parameters: Unpack[AWSOpenSearchHNSWTypedDict]): from .config import AWSOpenSearchConfig, AWSOpenSearchIndexConfig + # Set default values for HNSW parameters if not provided and not using s3vector + engine = AWSOS_Engine(parameters["engine"]) + ef_construction = parameters.get("ef_construction") + ef_search = parameters.get("ef_search") + m = parameters.get("m") + + # For non-s3vector engines, provide defaults if None + if engine != AWSOS_Engine.s3vector: + if ef_construction is None: + ef_construction = 200 + if ef_search is None: + ef_search = 100 + if m is None: + m = 16 + run( db=DB.AWSOpenSearch, db_config=AWSOpenSearchConfig( @@ -117,9 +198,17 @@ def AWSOpenSearch(**parameters: Unpack[AWSOpenSearchHNSWTypedDict]): refresh_interval=parameters["refresh_interval"], force_merge_enabled=parameters["force_merge_enabled"], flush_threshold_size=parameters["flush_threshold_size"], - number_of_indexing_clients=parameters["number_of_indexing_clients"], index_thread_qty_during_force_merge=parameters["index_thread_qty_during_force_merge"], + number_of_indexing_clients=parameters["number_of_indexing_clients"], cb_threshold=parameters["cb_threshold"], + efConstruction=ef_construction, + ef_search=ef_search, + M=m, + engine=engine, + quantization_type=AWSOSQuantization(parameters["quantization_type"]), + metric_type_name=parameters["metric_type"], + on_disk=parameters["on_disk"], + oversample_factor=parameters["oversample_factor"], ), **parameters, ) diff --git a/vectordb_bench/backend/clients/aws_opensearch/config.py b/vectordb_bench/backend/clients/aws_opensearch/config.py index dd51b266d..ff87f66bf 100644 --- a/vectordb_bench/backend/clients/aws_opensearch/config.py +++ b/vectordb_bench/backend/clients/aws_opensearch/config.py @@ -10,17 +10,21 @@ class AWSOpenSearchConfig(DBConfig, BaseModel): host: str = "" - port: int = 443 + port: int = 80 user: str = "" password: SecretStr = "" def to_dict(self) -> dict: + use_ssl = self.port == 443 + http_auth = ( + (self.user, self.password.get_secret_value()) if len(self.user) != 0 and len(self.password) != 0 else () + ) return { "hosts": [{"host": self.host, "port": self.port}], - "http_auth": (self.user, self.password.get_secret_value()), - "use_ssl": True, + "http_auth": http_auth, + "use_ssl": use_ssl, "http_compress": True, - "verify_certs": True, + "verify_certs": use_ssl, "ssl_assert_hostname": False, "ssl_show_warn": False, "timeout": 600, @@ -28,17 +32,25 @@ def to_dict(self) -> dict: class AWSOS_Engine(Enum): - nmslib = "nmslib" faiss = "faiss" - lucene = "Lucene" + lucene = "lucene" + s3vector = "s3vector" + + +class AWSOSQuantization(Enum): + fp32 = "fp32" + fp16 = "fp16" + bq = "bq" class AWSOpenSearchIndexConfig(BaseModel, DBCaseConfig): metric_type: MetricType = MetricType.L2 engine: AWSOS_Engine = AWSOS_Engine.faiss - efConstruction: int = 256 - efSearch: int = 256 - M: int = 16 + efConstruction: int | None = 256 + ef_search: int | None = 100 + engine_name: str | None = None + metric_type_name: str | None = None + M: int | None = 16 index_thread_qty: int | None = 4 number_of_shards: int | None = 1 number_of_replicas: int | None = 0 @@ -46,33 +58,97 @@ class AWSOpenSearchIndexConfig(BaseModel, DBCaseConfig): refresh_interval: str | None = "60s" force_merge_enabled: bool | None = True flush_threshold_size: str | None = "5120mb" - number_of_indexing_clients: int | None = 1 - index_thread_qty_during_force_merge: int + index_thread_qty_during_force_merge: int = 8 cb_threshold: str | None = "50%" + number_of_indexing_clients: int | None = 1 + use_routing: bool = False # for label-filter cases + oversample_factor: float = 1.0 + quantization_type: AWSOSQuantization = AWSOSQuantization.fp32 + on_disk: bool = False + + def __eq__(self, obj: any): + return ( + self.engine == obj.engine + and self.M == obj.M + and self.efConstruction == obj.efConstruction + and self.number_of_shards == obj.number_of_shards + and self.number_of_replicas == obj.number_of_replicas + and self.number_of_segments == obj.number_of_segments + and self.use_routing == obj.use_routing + and self.quantization_type == obj.quantization_type + and self.on_disk == obj.on_disk + ) + + def __hash__(self) -> int: + return hash( + ( + self.engine, + self.M, + self.efConstruction, + self.number_of_shards, + self.number_of_replicas, + self.number_of_segments, + self.use_routing, + self.quantization_type, + self.on_disk, + ) + ) def parse_metric(self) -> str: + log.info(f"User specified metric_type: {self.metric_type_name}") + + # Handle None or empty metric_type_name + if self.metric_type_name is None or self.metric_type_name == "": + log.info("No metric_type_name specified, defaulting to l2") + self.metric_type = MetricType.L2 + return "l2" + + self.metric_type = MetricType[self.metric_type_name.upper()] if self.metric_type == MetricType.IP: return "innerproduct" if self.metric_type == MetricType.COSINE: - if self.engine == AWSOS_Engine.faiss: - log.info( - "Using innerproduct because faiss doesn't support cosine as metric type for Opensearch", - ) - return "innerproduct" return "cosinesimil" + if self.metric_type == MetricType.L2: + log.info("Using l2 as specified by user") + return "l2" return "l2" + @property + def use_quant(self) -> bool: + return self.quantization_type is not AWSOSQuantization.fp32 + def index_param(self) -> dict: + log.info(f"Using engine: {self.engine} for index creation") + log.info(f"Using metric_type: {self.metric_type_name} for index creation") + log.info(f"Using on_disk mode: {self.on_disk} for index creation") + space_type = self.parse_metric() + log.info(f"Resulting space_type: {space_type} for index creation") + + # Handle s3vector engine with simplified configuration + # For s3vector, space_type should be set at the vector field level, not in method + if self.engine == AWSOS_Engine.s3vector: + return {"engine": "s3vector"} + + parameters = {"ef_construction": self.efConstruction, "m": self.M} + + # Add encoder configuration based on quantization type + if self.engine == AWSOS_Engine.faiss and self.use_quant: + if self.quantization_type == AWSOSQuantization.fp16: + parameters["encoder"] = {"name": "sq", "parameters": {"type": "fp16"}} + elif self.quantization_type == AWSOSQuantization.bq: + parameters["encoder"] = {"name": "binary", "parameters": {"bits": 1}} + + # For other engines (faiss, lucene), space_type is set at method level return { "name": "hnsw", - "space_type": self.parse_metric(), "engine": self.engine.value, - "parameters": { - "ef_construction": self.efConstruction, - "m": self.M, - "ef_search": self.efSearch, - }, + "space_type": space_type, + "parameters": parameters, } def search_param(self) -> dict: - return {} + # s3vector engine doesn't use ef_search parameter + if self.engine == AWSOS_Engine.s3vector: + return {} + + return {"ef_search": self.ef_search} diff --git a/vectordb_bench/backend/clients/chroma/chroma.py b/vectordb_bench/backend/clients/chroma/chroma.py index 26a810065..6942f4fc9 100644 --- a/vectordb_bench/backend/clients/chroma/chroma.py +++ b/vectordb_bench/backend/clients/chroma/chroma.py @@ -1,10 +1,10 @@ import logging from contextlib import contextmanager -from typing import Any import chromadb -from ..api import DBCaseConfig, VectorDB +from ..api import VectorDB +from .config import ChromaIndexConfig log = logging.getLogger(__name__) @@ -21,35 +21,34 @@ def __init__( self, dim: int, db_config: dict, - db_case_config: DBCaseConfig, + db_case_config: ChromaIndexConfig, + collection_name: str = "VectorDBBenchCollection", drop_old: bool = False, **kwargs, ): self.db_config = db_config self.case_config = db_case_config - self.collection_name = "example2" + self.collection_name = collection_name - client = chromadb.HttpClient(host=self.db_config["host"], port=self.db_config["port"]) + client = chromadb.HttpClient(**db_config) assert client.heartbeat() is not None + if drop_old: try: - client.reset() # Reset the database + client.reset() except Exception: drop_old = False log.info(f"Chroma client drop_old collection: {self.collection_name}") - @contextmanager - def init(self) -> None: - """create and destory connections to database. - - Examples: - >>> with self.init(): - >>> self.insert_embeddings() - """ - # create connection - self.client = chromadb.HttpClient(host=self.db_config["host"], port=self.db_config["port"]) + self.client = None + self.collection = None - self.collection = self.client.get_or_create_collection("example2") + @contextmanager + def init(self): + self.client = chromadb.HttpClient(**self.db_config) + self.collection = self.client.get_or_create_collection( + name=self.collection_name, configuration=self.case_config.index_param() + ) yield self.client = None self.collection = None @@ -58,58 +57,38 @@ def ready_to_search(self) -> bool: pass def optimize(self, data_size: int | None = None): - pass + assert self.collection is not None, "Please call self.init() before" + try: + self.collection.modify(configuration=self.case_config.search_param()) + except Exception as e: + log.warning(f"Optimize error: {e}") + raise def insert_embeddings( self, embeddings: list[list[float]], metadata: list[int], - **kwargs: Any, + **kwargs, ) -> tuple[int, Exception]: - """Insert embeddings into the database. - - Args: - embeddings(list[list[float]]): list of embeddings - metadata(list[int]): list of metadata - kwargs: other arguments - - Returns: - tuple[int, Exception]: number of embeddings inserted and exception if any - """ - ids = [str(i) for i in metadata] - metadata = [{"id": int(i)} for i in metadata] - if len(embeddings) > 0: - self.collection.add(embeddings=embeddings, ids=ids, metadatas=metadata) - return len(embeddings), None + assert self.collection is not None, "Please call self.init() before" + ids = [f"{idx}" for idx in metadata] + metadata = [{"index": mid} for mid in metadata] + try: + self.collection.add(ids=ids, embeddings=embeddings, metadatas=metadata) + except Exception as e: + log.warning(f"Failed to insert data: {e}") + return 0, e + + return len(metadata), None def search_embedding( - self, - query: list[float], - k: int = 100, - filters: dict | None = None, - timeout: int | None = None, - **kwargs: Any, - ) -> dict: - """Search embeddings from the database. - Args: - embedding(list[float]): embedding to search - k(int): number of results to return - kwargs: other arguments - - Returns: - Dict {ids: list[list[int]], - embedding: list[list[float]] - distance: list[list[float]]} - """ + self, query: list[float], k: int = 100, filters: dict | None = None, timeout: int | None = None + ) -> list[int]: + assert self.client is not None, "Please call self.init() before" if filters: - # assumes benchmark test filters of format: {'metadata': '>=10000', 'id': 10000} - id_value = filters.get("id") results = self.collection.query( - query_embeddings=query, - n_results=k, - where={"id": {"$gt": id_value}}, + query_embeddings=[query], n_results=k, where={"id": {"$gt": filters.get("id")}} ) - # return list of id's in results - return [int(i) for i in results.get("ids")[0]] - results = self.collection.query(query_embeddings=query, n_results=k) - return [int(i) for i in results.get("ids")[0]] + else: + results = self.collection.query(query_embeddings=[query], n_results=k) + return [int(idx) for idx in results["ids"][0]] diff --git a/vectordb_bench/backend/clients/chroma/cli.py b/vectordb_bench/backend/clients/chroma/cli.py new file mode 100644 index 000000000..64a2f972f --- /dev/null +++ b/vectordb_bench/backend/clients/chroma/cli.py @@ -0,0 +1,64 @@ +from typing import Annotated, Unpack + +import click +from pydantic import SecretStr + +from vectordb_bench.backend.clients import DB +from vectordb_bench.cli.cli import ( + CommonTypedDict, + cli, + click_parameter_decorators_from_typed_dict, + run, +) + +DBTYPE = DB.Chroma + + +class ChromaTypeDict(CommonTypedDict): + user: Annotated[ + str | None, + click.option("--user", type=str, help="Db username", required=False), + ] + password: Annotated[ + str | None, + click.option("--password", type=str, help="Db password", required=False), + ] + host: Annotated[ + str, + click.option("--host", type=str, help="Chroma host", default="localhost"), + ] + port: Annotated[int, click.option("--port", type=int, help="Chroma port", default=8000)] + m: Annotated[ + int, + click.option("--m", type=int, help="HNSW Maximum Neighbors", default=16), + ] + ef_construct: Annotated[ + int, + click.option("--ef-construct", type=int, help="HNSW efConstruct", default=256), + ] + ef_search: Annotated[ + int, + click.option("--ef-search", type=int, help="HNSW efSearch", default=256), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(ChromaTypeDict) +def Chroma(**parameters: Unpack[ChromaTypeDict]): + from .config import ChromaConfig, ChromaIndexConfig + + run( + db=DBTYPE, + db_config=ChromaConfig( + user=parameters["user"], + password=SecretStr(parameters["password"]) if parameters["password"] else None, + host=SecretStr(parameters["host"]), + port=parameters["port"], + ), + db_case_config=ChromaIndexConfig( + m=parameters["m"], + ef_construct=parameters["ef_construct"], + ef_search=parameters["ef_search"], + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/chroma/config.py b/vectordb_bench/backend/clients/chroma/config.py index af34cf513..cd3e01ecc 100644 --- a/vectordb_bench/backend/clients/chroma/config.py +++ b/vectordb_bench/backend/clients/chroma/config.py @@ -1,16 +1,54 @@ +from chromadb.config import Settings from pydantic import SecretStr -from ..api import DBConfig +from ..api import DBCaseConfig, DBConfig, MetricType class ChromaConfig(DBConfig): - password: SecretStr - host: SecretStr - port: int + user: str | None = None + password: SecretStr | None + host: SecretStr = "localhost" + port: int = 8000 def to_dict(self) -> dict: - return { + config = { "host": self.host.get_secret_value(), "port": self.port, - "password": self.password.get_secret_value(), } + if self.password and self.user: + config["settings"] = Settings( + settings=Settings( + chroma_client_auth_provider="chromadb.auth.token_authn.TokenAuthClientProvider", + chroma_client_auth_credentials=f"{self.user}:{self.password}", + ) + ) + return config + + +class ChromaIndexConfig(ChromaConfig, DBCaseConfig): + metric_type: MetricType = "cosine" + m: int = 16 + ef_construct: int = 100 + ef_search: int | None = 100 + + def parse_metric(self) -> str: + if self.metric_type == MetricType.L2: + return "l2" + if self.metric_type == MetricType.IP: + return "ip" + if self.metric_type == MetricType.COSINE: + return "cosine" + raise ValueError("Unsupported metric type: %s" % self.metric_type) + + def index_param(self): + return { + "hnsw": { + "space": self.parse_metric(), + "max_neighbors": self.m, + "ef_construction": self.ef_construct, + "ef_search": self.search_param().get("ef_search", 100), + } + } + + def search_param(self) -> dict: + return {"ef_search": self.ef_search} diff --git a/vectordb_bench/backend/clients/clickhouse/cli.py b/vectordb_bench/backend/clients/clickhouse/cli.py index 6fc1a84d7..4b50bc55b 100644 --- a/vectordb_bench/backend/clients/clickhouse/cli.py +++ b/vectordb_bench/backend/clients/clickhouse/cli.py @@ -51,6 +51,7 @@ def Clickhouse(**parameters: Unpack[ClickhouseHNSWTypedDict]): db=DB.Clickhouse, db_config=ClickhouseConfig( db_label=parameters["db_label"], + user=parameters["user"], password=SecretStr(parameters["password"]) if parameters["password"] else None, host=parameters["host"], port=parameters["port"], diff --git a/vectordb_bench/backend/clients/clickhouse/clickhouse.py b/vectordb_bench/backend/clients/clickhouse/clickhouse.py index e241cdb8d..de09895a8 100644 --- a/vectordb_bench/backend/clients/clickhouse/clickhouse.py +++ b/vectordb_bench/backend/clients/clickhouse/clickhouse.py @@ -5,8 +5,11 @@ from typing import Any import clickhouse_connect +from clickhouse_connect.driver import Client -from ..api import DBCaseConfig, VectorDB +from .. import IndexType +from ..api import VectorDB +from .config import ClickhouseConfigDict, ClickhouseIndexConfig log = logging.getLogger(__name__) @@ -17,8 +20,8 @@ class Clickhouse(VectorDB): def __init__( self, dim: int, - db_config: dict, - db_case_config: DBCaseConfig, + db_config: ClickhouseConfigDict, + db_case_config: ClickhouseIndexConfig, collection_name: str = "CHVectorCollection", drop_old: bool = False, **kwargs, @@ -28,29 +31,29 @@ def __init__( self.table_name = collection_name self.dim = dim + self.index_param = self.case_config.index_param() + self.search_param = self.case_config.search_param() + self.session_param = self.case_config.session_param() + self._index_name = "clickhouse_index" self._primary_field = "id" self._vector_field = "embedding" # construct basic units - self.conn = clickhouse_connect.get_client( - host=self.db_config["host"], - port=self.db_config["port"], - username=self.db_config["user"], - password=self.db_config["password"], - database=self.db_config["dbname"], - ) + self.conn = self._create_connection(**self.db_config, settings=self.session_param) if drop_old: log.info(f"Clickhouse client drop table : {self.table_name}") self._drop_table() self._create_table(dim) + if self.case_config.create_index_before_load: + self._create_index() self.conn.close() self.conn = None @contextmanager - def init(self): + def init(self) -> None: """ Examples: >>> with self.init(): @@ -58,13 +61,7 @@ def init(self): >>> self.search_embedding() """ - self.conn = clickhouse_connect.get_client( - host=self.db_config["host"], - port=self.db_config["port"], - username=self.db_config["user"], - password=self.db_config["password"], - database=self.db_config["dbname"], - ) + self.conn = self._create_connection(**self.db_config, settings=self.session_param) try: yield @@ -72,10 +69,61 @@ def init(self): self.conn.close() self.conn = None + def _create_connection(self, settings: dict | None, **kwargs) -> Client: + return clickhouse_connect.get_client(**self.db_config, settings=settings) + + def _drop_index(self): + assert self.conn is not None, "Connection is not initialized" + try: + self.conn.command( + f'ALTER TABLE {self.db_config["database"]}.{self.table_name} DROP INDEX {self._index_name}' + ) + except Exception as e: + log.warning(f"Failed to drop index on table {self.db_config['database']}.{self.table_name}: {e}") + raise e from None + def _drop_table(self): assert self.conn is not None, "Connection is not initialized" - self.conn.command(f'DROP TABLE IF EXISTS {self.db_config["dbname"]}.{self.table_name}') + try: + self.conn.command(f'DROP TABLE IF EXISTS {self.db_config["database"]}.{self.table_name}') + except Exception as e: + log.warning(f"Failed to drop table {self.db_config['database']}.{self.table_name}: {e}") + raise e from None + + def _perfomance_tuning(self): + self.conn.command("SET materialize_skip_indexes_on_insert = 1") + + def _create_index(self): + assert self.conn is not None, "Connection is not initialized" + try: + if self.index_param["index_type"] == IndexType.HNSW.value: + if ( + self.index_param["quantization"] + and self.index_param["params"]["M"] + and self.index_param["params"]["efConstruction"] + ): + query = f""" + ALTER TABLE {self.db_config["database"]}.{self.table_name} + ADD INDEX {self._index_name} {self._vector_field} + TYPE vector_similarity('hnsw', '{self.index_param["metric_type"]}',{self.dim}, + '{self.index_param["quantization"]}', + {self.index_param["params"]["M"]}, {self.index_param["params"]["efConstruction"]}) + GRANULARITY {self.index_param["granularity"]} + """ + else: + query = f""" + ALTER TABLE {self.db_config["database"]}.{self.table_name} + ADD INDEX {self._index_name} {self._vector_field} + TYPE vector_similarity('hnsw', '{self.index_param["metric_type"]}', {self.dim}) + GRANULARITY {self.index_param["granularity"]} + """ + self.conn.command(cmd=query) + else: + log.warning("HNSW is only avaliable method in clickhouse now") + except Exception as e: + log.warning(f"Failed to create Clickhouse vector index on table: {self.table_name} error: {e}") + raise e from None def _create_table(self, dim: int): assert self.conn is not None, "Connection is not initialized" @@ -83,21 +131,22 @@ def _create_table(self, dim: int): try: # create table self.conn.command( - f'CREATE TABLE IF NOT EXISTS {self.db_config["dbname"]}.{self.table_name} \ - (id UInt32, embedding Array(Float64)) ENGINE = MergeTree() ORDER BY id;' + f'CREATE TABLE IF NOT EXISTS {self.db_config["database"]}.{self.table_name} ' + f"({self._primary_field} UInt32, " + f'{self._vector_field} Array({self.index_param["vector_data_type"]}) CODEC(NONE), ' + f"CONSTRAINT same_length CHECK length(embedding) = {dim}) " + f"ENGINE = MergeTree() " + f"ORDER BY {self._primary_field}" ) except Exception as e: log.warning(f"Failed to create Clickhouse table: {self.table_name} error: {e}") raise e from None - def ready_to_load(self): - pass - def optimize(self, data_size: int | None = None): pass - def ready_to_search(self): + def _post_insert(self): pass def insert_embeddings( @@ -105,7 +154,7 @@ def insert_embeddings( embeddings: list[list[float]], metadata: list[int], **kwargs: Any, - ) -> tuple[int, Exception]: + ) -> (int, Exception): assert self.conn is not None, "Connection is not initialized" try: @@ -116,7 +165,7 @@ def insert_embeddings( table=self.table_name, data=items, column_names=["id", "embedding"], - column_type_names=["UInt32", "Array(Float64)"], + column_type_names=["UInt32", f'Array({self.index_param["vector_data_type"]})'], column_oriented=True, ) return len(metadata), None @@ -132,25 +181,52 @@ def search_embedding( timeout: int | None = None, ) -> list[int]: assert self.conn is not None, "Connection is not initialized" - - index_param = self.case_config.index_param() # noqa: F841 - search_param = self.case_config.search_param() - - if filters: - gt = filters.get("id") - filter_sql = ( - f'SELECT id, {search_param["metric_type"]}(embedding,{query}) AS score ' # noqa: S608 - f'FROM {self.db_config["dbname"]}.{self.table_name} ' - f"WHERE id > {gt} " - f"ORDER BY score LIMIT {k};" - ) - result = self.conn.query(filter_sql).result_rows + parameters = { + "primary_field": self._primary_field, + "vector_field": self._vector_field, + "schema": self.db_config["database"], + "table": self.table_name, + "gt": 0 if filters is None else filters.get("id", 0), + "k": k, + "metric_type": self.search_param["metric_type"], + "query": query, + } + if self.case_config.metric_type == "COSINE": + if filters: + result = self.conn.query( + "SELECT {primary_field:Identifier}, {vector_field:Identifier} " + "FROM {schema:Identifier}.{table:Identifier} " + "WHERE {primary_field:Identifier} > {gt:UInt32} " + "ORDER BY cosineDistance(embedding,{query:Array(Float64)}) " + "LIMIT {k:UInt32}", + parameters=parameters, + ).result_rows + return [int(row[0]) for row in result] + + result = self.conn.query( + "SELECT {primary_field:Identifier}, {vector_field:Identifier} " + "FROM {schema:Identifier}.{table:Identifier} " + "ORDER BY cosineDistance(embedding,{query:Array(Float64)}) " + "LIMIT {k:UInt32}", + parameters=parameters, + ).result_rows return [int(row[0]) for row in result] - else: # noqa: RET505 - select_sql = ( - f'SELECT id, {search_param["metric_type"]}(embedding,{query}) AS score ' # noqa: S608 - f'FROM {self.db_config["dbname"]}.{self.table_name} ' - f"ORDER BY score LIMIT {k};" - ) - result = self.conn.query(select_sql).result_rows + if filters: + result = self.conn.query( + "SELECT {primary_field:Identifier}, {vector_field:Identifier} " + "FROM {schema:Identifier}.{table:Identifier} " + "WHERE {primary_field:Identifier} > {gt:UInt32} " + "ORDER BY L2Distance(embedding,{query:Array(Float64)}) " + "LIMIT {k:UInt32}", + parameters=parameters, + ).result_rows return [int(row[0]) for row in result] + + result = self.conn.query( + "SELECT {primary_field:Identifier}, {vector_field:Identifier} " + "FROM {schema:Identifier}.{table:Identifier} " + "ORDER BY L2Distance(embedding,{query:Array(Float64)}) " + "LIMIT {k:UInt32}", + parameters=parameters, + ).result_rows + return [int(row[0]) for row in result] diff --git a/vectordb_bench/backend/clients/clickhouse/config.py b/vectordb_bench/backend/clients/clickhouse/config.py index fad446049..f9e09812b 100644 --- a/vectordb_bench/backend/clients/clickhouse/config.py +++ b/vectordb_bench/backend/clients/clickhouse/config.py @@ -1,29 +1,46 @@ +from abc import abstractmethod +from typing import TypedDict + from pydantic import BaseModel, SecretStr from ..api import DBCaseConfig, DBConfig, IndexType, MetricType +class ClickhouseConfigDict(TypedDict): + user: str + password: str + host: str + port: int + database: str + secure: bool + + class ClickhouseConfig(DBConfig): - user_name: str = "clickhouse" + user: str = "clickhouse" password: SecretStr host: str = "localhost" port: int = 8123 db_name: str = "default" + secure: bool = False - def to_dict(self) -> dict: + def to_dict(self) -> ClickhouseConfigDict: pwd_str = self.password.get_secret_value() return { "host": self.host, "port": self.port, - "dbname": self.db_name, - "user": self.user_name, + "database": self.db_name, + "user": self.user, "password": pwd_str, + "secure": self.secure, } -class ClickhouseIndexConfig(BaseModel): +class ClickhouseIndexConfig(BaseModel, DBCaseConfig): metric_type: MetricType | None = None + vector_data_type: str | None = "Float32" # Data type of vectors. Can be Float32 or Float64 or BFloat16 + create_index_before_load: bool = True + create_index_after_load: bool = False def parse_metric(self) -> str: if not self.metric_type: @@ -35,26 +52,38 @@ def parse_metric_str(self) -> str: return "L2Distance" if self.metric_type == MetricType.COSINE: return "cosineDistance" - msg = f"Not Support for {self.metric_type}" - raise RuntimeError(msg) - return None + return "cosineDistance" + + @abstractmethod + def session_param(self): + pass -class ClickhouseHNSWConfig(ClickhouseIndexConfig, DBCaseConfig): - M: int | None - efConstruction: int | None +class ClickhouseHNSWConfig(ClickhouseIndexConfig): + M: int | None # Default in clickhouse in 32 + efConstruction: int | None # Default in clickhouse in 128 ef: int | None = None index: IndexType = IndexType.HNSW + quantization: str | None = "bf16" # Default is bf16. Possible values are f64, f32, f16, bf16, or i8 + granularity: int | None = 10_000_000 # Size of the index granules. By default, in CH it's equal 10.000.000 def index_param(self) -> dict: return { + "vector_data_type": self.vector_data_type, "metric_type": self.parse_metric_str(), "index_type": self.index.value, + "quantization": self.quantization, + "granularity": self.granularity, "params": {"M": self.M, "efConstruction": self.efConstruction}, } def search_param(self) -> dict: return { - "met˝ric_type": self.parse_metric_str(), + "metric_type": self.parse_metric_str(), "params": {"ef": self.ef}, } + + def session_param(self) -> dict: + return { + "allow_experimental_vector_similarity_index": 1, + } diff --git a/vectordb_bench/backend/clients/cockroachdb/cli.py b/vectordb_bench/backend/clients/cockroachdb/cli.py new file mode 100644 index 000000000..a32dfe255 --- /dev/null +++ b/vectordb_bench/backend/clients/cockroachdb/cli.py @@ -0,0 +1,131 @@ +"""CLI parameter definitions for CockroachDB.""" + +from typing import Annotated, Unpack + +import click +from pydantic import SecretStr + +from vectordb_bench.backend.clients import DB + +from ....cli.cli import ( + CommonTypedDict, + cli, + click_parameter_decorators_from_typed_dict, + get_custom_case_config, + run, +) + + +class CockroachDBTypedDict(CommonTypedDict): + """Type definition for CockroachDB CLI parameters.""" + + user_name: Annotated[ + str, + click.option("--user-name", type=str, help="CockroachDB username", default="root", show_default=True), + ] + password: Annotated[ + str, + click.option("--password", type=str, help="CockroachDB password", default="", show_default=False), + ] + host: Annotated[ + str, + click.option("--host", type=str, help="CockroachDB host", required=True), + ] + port: Annotated[ + int, + click.option("--port", type=int, help="CockroachDB port", default=26257, show_default=True), + ] + db_name: Annotated[ + str, + click.option("--db-name", type=str, help="Database name", required=True), + ] + sslmode: Annotated[ + str, + click.option( + "--sslmode", + type=str, + help="SSL mode (disable, require, verify-ca, verify-full)", + default="disable", + show_default=True, + ), + ] + sslrootcert: Annotated[ + str | None, + click.option( + "--sslrootcert", + type=str, + help="Path to SSL root certificate (required for verify-ca, verify-full)", + default=None, + ), + ] + min_partition_size: Annotated[ + int | None, + click.option( + "--min-partition-size", + type=int, + help="Minimum vectors per partition (default: 16, range: 1-1024)", + default=16, + show_default=True, + ), + ] + max_partition_size: Annotated[ + int | None, + click.option( + "--max-partition-size", + type=int, + help="Maximum vectors per partition (default: 128, range: 4x min-4096)", + default=128, + show_default=True, + ), + ] + vector_search_beam_size: Annotated[ + int | None, + click.option( + "--vector-search-beam-size", + type=int, + help="Partitions explored during search (default: 32)", + default=32, + show_default=True, + ), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(CockroachDBTypedDict) +def CockroachDB( + **parameters: Unpack[CockroachDBTypedDict], +): + """Run CockroachDB vector benchmark.""" + from .config import CockroachDBConfig, CockroachDBVectorIndexConfig + + parameters["custom_case"] = get_custom_case_config(parameters) + + from vectordb_bench.backend.clients.api import MetricType + + # Use provided metric_type or default to COSINE + metric_type = parameters.get("metric_type") + if metric_type is None: + metric_type = MetricType.COSINE + elif isinstance(metric_type, str): + metric_type = MetricType(metric_type) + + run( + db=DB.CockroachDB, + db_config=CockroachDBConfig( + db_label=parameters["db_label"], + user_name=SecretStr(parameters["user_name"]), + password=SecretStr(parameters["password"]) if parameters["password"] else None, + host=parameters["host"], + port=parameters["port"], + db_name=parameters["db_name"], + sslmode=parameters.get("sslmode", "disable"), + sslrootcert=parameters.get("sslrootcert"), + ), + db_case_config=CockroachDBVectorIndexConfig( + metric_type=metric_type, + min_partition_size=parameters.get("min_partition_size", 16), + max_partition_size=parameters.get("max_partition_size", 128), + vector_search_beam_size=parameters.get("vector_search_beam_size", 32), + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/cockroachdb/cockroachdb.py b/vectordb_bench/backend/clients/cockroachdb/cockroachdb.py new file mode 100644 index 000000000..89fdf2826 --- /dev/null +++ b/vectordb_bench/backend/clients/cockroachdb/cockroachdb.py @@ -0,0 +1,606 @@ +"""CockroachDB vector database client with connection pooling and retry logic.""" + +import logging +from collections.abc import Generator +from contextlib import contextmanager +from typing import Any + +import numpy as np +import psycopg +from pgvector.psycopg import register_vector +from psycopg import Connection, Cursor, sql +from psycopg_pool import ConnectionPool + +from vectordb_bench.backend.filter import Filter, FilterOp + +from ..api import VectorDB +from .config import CockroachDBIndexConfig +from .db_retry import db_retry + +log = logging.getLogger(__name__) + + +class CockroachDB(VectorDB): + """ + CockroachDB vector database client: + - Connection pooling (100+ connections for high throughput) + - Automatic retry for serialization errors (40001, 40003) + - Vector index support (C-SPANN algorithm) + - Multi-region resilience + """ + + supported_filter_types: list[FilterOp] = [ + FilterOp.NonFilter, + FilterOp.NumGE, + FilterOp.StrEqual, + ] + + def __init__( # noqa: PLR0915 + self, + dim: int, + db_config: dict, + db_case_config: CockroachDBIndexConfig | None, + collection_name: str = "vdbbench_cockroachdb", + drop_old: bool = False, + with_scalar_labels: bool = False, + **kwargs, + ): + self.name = "CockroachDB" + self.case_config = db_case_config + self.table_name = collection_name + + # Handle both dict-style config (from to_dict()) and direct dict + if "connect_config" in db_config: + self.connect_config = db_config["connect_config"] + self.pool_size = db_config.get("pool_size", 100) + self.max_overflow = db_config.get("max_overflow", 100) + self.pool_recycle = db_config.get("pool_recycle", 3600) + else: + # Direct connection config for tests + conn_params = { + "host": db_config.get("host", "localhost"), + "port": db_config.get("port", 26257), + "dbname": db_config.get("db_name", "defaultdb"), + "user": db_config.get("user_name", "root"), + "password": db_config.get("password", ""), + } + # Add SSL configuration if specified + conn_params["sslmode"] = db_config.get("sslmode", "disable") + if db_config.get("sslrootcert"): + conn_params["sslrootcert"] = db_config["sslrootcert"] + if db_config.get("sslcert"): + conn_params["sslcert"] = db_config["sslcert"] + if db_config.get("sslkey"): + conn_params["sslkey"] = db_config["sslkey"] + + self.connect_config = conn_params + self.pool_size = db_config.get("pool_size", 100) + self.max_overflow = db_config.get("max_overflow", 100) + self.pool_recycle = db_config.get("pool_recycle", 3600) + + self.dim = dim + self.with_scalar_labels = with_scalar_labels + + self._index_name = f"{self.table_name}_vector_idx" + self._primary_field = "id" # UUID for distribution + self._metadata_field = "metadata_id" # BIGINT for framework compatibility + self._vector_field = "embedding" + self._scalar_label_field = "label" + + self.pool: ConnectionPool | None = None + self.conn: Connection | None = None + self.cursor: Cursor | None = None + + log.info(f"{self.name} config: {self.connect_config}, pool_size={self.pool_size}") + + # Allow manual index creation (both flags can be False) + # This is useful when CREATE INDEX times out in subprocess on multi-node clusters + if self.case_config is not None and not any( + (self.case_config.create_index_before_load, self.case_config.create_index_after_load) + ): + log.warning(f"{self.name}: Both create_index flags are False - expecting manually created index") + + # Initialize with temporary connection for setup + conn, cursor = self._create_connection(**self.connect_config) + try: + # Enable pgvector extension (in transaction) + cursor.execute("CREATE EXTENSION IF NOT EXISTS vector") + conn.commit() + cursor.close() + + # Enable vector indexes at cluster level (requires autocommit, not in transaction) + conn.autocommit = True + cursor = conn.cursor() + try: + cursor.execute("SET CLUSTER SETTING feature.vector_index.enabled = true") + except Exception as e: + # May already be enabled or permission issue, log and continue + log.warning(f"Could not enable vector indexes: {e}") + cursor.close() + + # Reset to transaction mode for remaining operations + conn.autocommit = False + cursor = conn.cursor() + + if drop_old: + # DROP TABLE CASCADE will automatically drop indexes, no need to drop separately + self._drop_table(cursor, conn) + self._create_table(cursor, conn, dim) + if self.case_config is not None and self.case_config.create_index_before_load: + self._create_index() # Use SQLAlchemy + finally: + cursor.close() + conn.close() + + @staticmethod + def _create_connection(**kwargs) -> tuple[Connection, Cursor]: + """Create a single connection with pgvector support.""" + conn = psycopg.connect(**kwargs) + register_vector(conn) + conn.autocommit = False + cursor = conn.cursor() + return conn, cursor + + def _create_connection_pool(self) -> ConnectionPool: + """Create connection pool with production settings.""" + # Build connection info without 'options' parameter (not supported by psycopg_pool) + # Include vector_search_beam_size in connection options for performance + beam_size = 32 + if self.case_config is not None: + search_param = self.case_config.search_param() + beam_size = search_param.get("vector_search_beam_size", 32) + + conninfo = ( + f"host={self.connect_config['host']} " + f"port={self.connect_config['port']} " + f"dbname={self.connect_config['dbname']} " + f"user={self.connect_config['user']} " + f"password={self.connect_config['password']}" + ) + + # Add SSL configuration if present + if "sslmode" in self.connect_config: + conninfo += f" sslmode={self.connect_config['sslmode']}" + if "sslrootcert" in self.connect_config: + conninfo += f" sslrootcert={self.connect_config['sslrootcert']}" + if "sslcert" in self.connect_config: + conninfo += f" sslcert={self.connect_config['sslcert']}" + if "sslkey" in self.connect_config: + conninfo += f" sslkey={self.connect_config['sslkey']}" + + # Add all settings in connection options to avoid per-connection overhead + conninfo += f" options='-c statement_timeout=600s -c vector_search_beam_size={beam_size}'" + + # Configure each connection with vector support (lightweight operation) + def configure_connection(conn: Connection) -> None: + register_vector(conn) + # No need to set beam_size here - it's in connection options + + return ConnectionPool( + conninfo=conninfo, + min_size=self.pool_size, + max_size=self.pool_size + self.max_overflow, + max_lifetime=self.pool_recycle, + max_idle=300, + reconnect_timeout=10.0, + configure=configure_connection, + ) + + @contextmanager + def init(self) -> Generator[None, None, None]: + """Initialize connection pool for benchmark operations.""" + self.pool = self._create_connection_pool() + + try: + with self.pool.connection() as conn: + conn.autocommit = False + self.conn = conn + self.cursor = conn.cursor() + + # Set session parameters (only if case_config is provided) + if self.case_config is not None: + session_options = self.case_config.session_param()["session_options"] + for setting in session_options: + param = setting["parameter"] + command = sql.SQL("SET {setting_name} = {val};").format( + setting_name=sql.Identifier(param["setting_name"]), + val=sql.Literal(int(param["val"])), + ) + log.debug(command.as_string(self.cursor)) + self.cursor.execute(command) + conn.commit() + + yield + finally: + if self.cursor: + self.cursor.close() + if self.conn: + self.conn.close() + if self.pool: + self.pool.close() + self.cursor = None + self.conn = None + self.pool = None + + def _cancel_running_schema_jobs(self): + """Cancel any running schema change jobs for this table. + CockroachDB-specific: Running CREATE INDEX jobs block DROP TABLE.""" + import psycopg + + try: + conn = psycopg.connect(**self.connect_config) + conn.autocommit = True + cursor = conn.cursor() + + # Find running schema change jobs for our table + cursor.execute( + """ + SELECT job_id + FROM [SHOW JOBS] + WHERE status IN ('running', 'pending') + AND job_type = 'NEW SCHEMA CHANGE' + AND description LIKE %s + """, + (f"%{self.table_name}%",), + ) + jobs = cursor.fetchall() + + for job in jobs: + job_id = job[0] + log.warning(f"{self.name} canceling schema job {job_id} before dropping table") + try: + cursor.execute(f"CANCEL JOB {job_id}") + log.info(f"Canceled job {job_id}") + except Exception as e: + log.warning(f"Failed to cancel job {job_id}: {e}") + + cursor.close() + conn.close() + except Exception as e: + log.warning(f"Failed to check/cancel running jobs: {e}") + + @db_retry(max_attempts=3, initial_delay=0.5, backoff_factor=2.0) + def _drop_table(self, cursor: Cursor, conn: Connection): + """Drop table with retry logic. + Note: CockroachDB-specific - must cancel running schema jobs first.""" + # Cancel any running schema change jobs that would block DROP + self._cancel_running_schema_jobs() + + log.info(f"{self.name} dropping table: {self.table_name}") + cursor.execute( + sql.SQL("DROP TABLE IF EXISTS {table_name} CASCADE").format( + table_name=sql.Identifier(self.table_name), + ), + ) + conn.commit() + + def _drop_index(self): + """Drop CockroachDB vector index if it exists (DDL with autocommit). + Note: This is typically not needed as DROP TABLE CASCADE handles it.""" + log.info(f"{self.name} dropping index: {self._index_name}") + conn = psycopg.connect(**self.connect_config) + conn.autocommit = True + try: + cursor = conn.cursor() + cursor.execute(f"DROP INDEX IF EXISTS {self._index_name}") + cursor.close() + finally: + conn.close() + + @db_retry(max_attempts=3, initial_delay=0.5, backoff_factor=2.0) + def _create_table(self, cursor: Cursor, conn: Connection, dim: int): + """Create table with VECTOR column.""" + log.info(f"{self.name} creating table: {self.table_name}") + + # CockroachDB best practice: Use UUID primary key to avoid hotspots in distributed deployments + # Keep metadata_id as BIGINT for framework compatibility + if self.with_scalar_labels: + cursor.execute( + sql.SQL( + """ + CREATE TABLE IF NOT EXISTS {table_name} + ({primary_field} UUID PRIMARY KEY DEFAULT gen_random_uuid(), + {metadata_field} BIGINT NOT NULL, + {vector_field} VECTOR({dim}), + {label_field} VARCHAR(64)); + """, + ).format( + table_name=sql.Identifier(self.table_name), + primary_field=sql.Identifier(self._primary_field), + metadata_field=sql.Identifier(self._metadata_field), + vector_field=sql.Identifier(self._vector_field), + label_field=sql.Identifier(self._scalar_label_field), + dim=dim, + ) + ) + else: + cursor.execute( + sql.SQL(""" + CREATE TABLE IF NOT EXISTS {table_name} + ({primary_field} UUID PRIMARY KEY DEFAULT gen_random_uuid(), + {metadata_field} BIGINT NOT NULL, + {vector_field} VECTOR({dim})); + """).format( + table_name=sql.Identifier(self.table_name), + primary_field=sql.Identifier(self._primary_field), + metadata_field=sql.Identifier(self._metadata_field), + vector_field=sql.Identifier(self._vector_field), + dim=dim, + ) + ) + + # Note: CockroachDB doesn't support SET STORAGE PLAIN (PostgreSQL-specific) + # Vector columns are handled automatically + conn.commit() + + def _create_index(self): + """Create CockroachDB vector index (DDL with autocommit).""" + log.info(f"{self.name} creating vector index: {self._index_name}") + + index_param = self.case_config.index_param() + + # Build WITH clause for index parameters + options_list = [] + for option in index_param["index_creation_with_options"]: + if option["val"] is not None: + options_list.append(f"{option['option_name']} = {option['val']}") + + with_clause = f" WITH ({', '.join(options_list)})" if options_list else "" + + # Build SQL string (DDL - no need for parameterization) + sql_str = ( + f"CREATE VECTOR INDEX IF NOT EXISTS {self._index_name} " + f"ON {self.table_name} ({self._vector_field} {index_param['metric']})" + f"{with_clause}" + ) + + log.info(f"Creating index with SQL: {sql_str}") + + # Use autocommit for DDL + conn = psycopg.connect(**self.connect_config) + conn.autocommit = True + try: + cursor = conn.cursor() + cursor.execute(sql_str) + cursor.close() + finally: + conn.close() + + def _wait_for_index_creation(self, start_time: float) -> None: + """Wait for background index creation to complete after connection timeout.""" + import time + + max_wait = 300 # 5 minutes max + poll_interval = 5 + waited = 0 + + while waited < max_wait: + time.sleep(poll_interval) + waited += poll_interval + + # Create fresh connection to check status + try: + check_conn = psycopg.connect(**self.connect_config) + check_cursor = check_conn.cursor() + try: + # Check if index exists + check_cursor.execute( + "SELECT 1 FROM pg_indexes WHERE tablename = %s AND indexname = %s", + (self.table_name, self._index_name), + ) + if check_cursor.fetchone(): + # Index exists! Verify it's usable by doing a quick test query + try: + from psycopg import sql + + check_cursor.execute( + sql.SQL("SELECT 1 FROM {} LIMIT 1").format(sql.Identifier(self.table_name)) + ) + check_cursor.fetchone() + total_time = time.time() - start_time + log.info(f"Index {self._index_name} created successfully (total time: {total_time:.1f}s)") + except Exception as query_error: + # Index not yet usable + log.info(f"Index exists but not yet usable... ({waited}s elapsed, error: {query_error})") + else: + return + finally: + check_cursor.close() + check_conn.close() + except Exception as check_error: + log.warning(f"Error checking index status: {check_error}") + # Continue waiting + + # Timeout waiting for index + msg = f"Timeout waiting for index {self._index_name} after {waited}s" + log.error(msg) + raise RuntimeError(msg) + + def optimize(self, data_size: int | None = None): + """Post-insert optimization: create index if needed. + + Note: On multi-node clusters, CockroachDB v25.4 may close connections + at 30s during CREATE VECTOR INDEX from subprocess contexts. The index + creation continues in background. We handle this gracefully by checking + if the index was created successfully after timeout. + """ + log.info(f"{self.name} post-insert optimization") + if self.case_config is not None and self.case_config.create_index_after_load: + import time + + # Build CREATE INDEX SQL + index_param = self.case_config.index_param() + options_list = [] + for option in index_param["index_creation_with_options"]: + if option["val"] is not None: + options_list.append(f"{option['option_name']} = {option['val']}") + + with_clause = f" WITH ({', '.join(options_list)})" if options_list else "" + sql_str = ( + f"CREATE VECTOR INDEX IF NOT EXISTS {self._index_name} " + f"ON {self.table_name} ({self._vector_field} {index_param['metric']})" + f"{with_clause}" + ) + + log.info(f"{self.name} creating vector index: {self._index_name}") + log.info(f"Index SQL: {sql_str}") + + start_time = time.time() + connection_closed = False + + # Try to create index - use a connection without statement_timeout + try: + # Create connection without statement_timeout for long-running index creation + import psycopg + + conninfo_no_timeout = ( + f"host={self.connect_config['host']} " + f"port={self.connect_config['port']} " + f"dbname={self.connect_config['dbname']} " + f"user={self.connect_config['user']} " + f"password={self.connect_config['password']}" + ) + if "sslmode" in self.connect_config: + conninfo_no_timeout += f" sslmode={self.connect_config['sslmode']}" + if "sslrootcert" in self.connect_config: + conninfo_no_timeout += f" sslrootcert={self.connect_config['sslrootcert']}" + if "sslcert" in self.connect_config: + conninfo_no_timeout += f" sslcert={self.connect_config['sslcert']}" + if "sslkey" in self.connect_config: + conninfo_no_timeout += f" sslkey={self.connect_config['sslkey']}" + + with psycopg.connect(conninfo_no_timeout, autocommit=True) as conn: + register_vector(conn) + with conn.cursor() as cursor: + cursor.execute(sql_str) + elapsed = time.time() - start_time + log.info(f"{self.name} index created successfully in {elapsed:.1f}s") + return # Success! + except Exception as e: + elapsed = time.time() - start_time + error_msg = str(e) + # Check for timeout or connection issues + if ( + "server closed the connection" in error_msg + or "statement timeout" in error_msg.lower() + or "query execution canceled" in error_msg.lower() + or "connection" in error_msg.lower() + ): + log.warning(f"Timeout/connection issue after {elapsed:.1f}s during index creation: {e}") + log.info("This is expected on large datasets - checking if index is being created in background...") + connection_closed = True + else: + # Unexpected error, re-raise + raise + + # Connection closed - wait for background index creation to complete + if connection_closed: + self._wait_for_index_creation(start_time) + + @db_retry(max_attempts=3, initial_delay=0.5, backoff_factor=2.0) + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + labels_data: list[str] | None = None, + **kwargs: Any, + ) -> tuple[int, Exception | None]: + """Insert embeddings with COPY for performance.""" + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + if self.with_scalar_labels: + assert labels_data is not None, "labels_data required when with_scalar_labels=True" + + try: + metadata_arr = np.array(metadata) + embeddings_arr = np.array(embeddings) + + # UUID primary key is auto-generated, we only insert metadata_id and embedding + with self.cursor.copy( + sql.SQL( + "COPY {table_name} ({metadata_field}, {vector_field}{label_field}) FROM STDIN (FORMAT BINARY)" + ).format( + table_name=sql.Identifier(self.table_name), + metadata_field=sql.Identifier(self._metadata_field), + vector_field=sql.Identifier(self._vector_field), + label_field=sql.SQL(f", {self._scalar_label_field}") if self.with_scalar_labels else sql.SQL(""), + ) + ) as copy: + for i, row in enumerate(metadata_arr): + if self.with_scalar_labels: + copy.set_types(["bigint", "vector", "varchar"]) + copy.write_row((row, embeddings_arr[i], labels_data[i])) + else: + copy.set_types(["bigint", "vector"]) + copy.write_row((row, embeddings_arr[i])) + + self.conn.commit() + return len(metadata), None + + except Exception as e: + log.warning(f"Failed to insert data into {self.table_name}: {e}") + return 0, e + + def prepare_filter(self, filters: Filter): + """Prepare WHERE clause for filtered queries.""" + if filters.type == FilterOp.NonFilter: + self.where_clause = "" + elif filters.type == FilterOp.NumGE: + # Filter on metadata_id, not UUID primary key + self.where_clause = f"WHERE {self._metadata_field} >= {filters.int_value}" + elif filters.type == FilterOp.StrEqual: + self.where_clause = f"WHERE {self._scalar_label_field} = '{filters.label_value}'" + else: + msg = f"Unsupported filter for CockroachDB: {filters}" + raise ValueError(msg) + + def ready_to_load(self) -> bool: + """Check if ready to load data.""" + + @db_retry(max_attempts=3, initial_delay=0.5, backoff_factor=2.0) + def search_embedding( + self, + query: list[float], + k: int = 100, + timeout: int | None = None, + **kwargs: Any, + ) -> list[int]: + """Search for k nearest neighbors using vector index.""" + assert self.pool is not None, "Connection pool is not initialized" + + # Use default L2 distance if no case_config provided + if self.case_config is not None: + search_param = self.case_config.search_param() + metric_op = search_param["metric_fun_op"] + else: + metric_op = "<->" # Default to L2 distance + + q = np.asarray(query) + + # Build search query - return metadata_id for framework compatibility + search_sql = sql.SQL("SELECT {metadata_field} FROM {table_name} {where_clause} ORDER BY {vector_field}").format( + metadata_field=sql.Identifier(self._metadata_field), + table_name=sql.Identifier(self.table_name), + where_clause=sql.SQL(getattr(self, "where_clause", "")), + vector_field=sql.Identifier(self._vector_field), + ) + + # Add distance operator and limit + full_sql = search_sql + sql.SQL(" {metric_op} %s LIMIT %s").format( + metric_op=sql.SQL(metric_op), + ) + + # Get a connection from the pool for this query (enables true concurrency) + # Pool returns already-configured connections with vector support + with self.pool.connection() as conn, conn.cursor() as cursor: + try: + result = cursor.execute(full_sql, (q, k), prepare=True, binary=True) + return [int(i[0]) for i in result.fetchall()] + except Exception as e: + # If transaction is aborted, rollback and retry + if "transaction is aborted" in str(e).lower(): + conn.rollback() + result = cursor.execute(full_sql, (q, k), prepare=True, binary=True) + return [int(i[0]) for i in result.fetchall()] + raise diff --git a/vectordb_bench/backend/clients/cockroachdb/config.py b/vectordb_bench/backend/clients/cockroachdb/config.py new file mode 100644 index 000000000..0d608da8f --- /dev/null +++ b/vectordb_bench/backend/clients/cockroachdb/config.py @@ -0,0 +1,216 @@ +"""Configuration classes for CockroachDB vector database integration.""" + +from abc import abstractmethod +from collections.abc import Mapping, Sequence +from typing import Any, LiteralString, TypedDict + +from pydantic import BaseModel, SecretStr + +from ..api import DBCaseConfig, DBConfig, IndexType, MetricType + + +class CockroachDBConfigDict(TypedDict): + """Connection configuration for CockroachDB using psycopg.""" + + user: str + password: str + host: str + port: int + dbname: str + sslmode: str + + +class CockroachDBConfig(DBConfig): + """Main configuration for CockroachDB connection.""" + + user_name: SecretStr = "root" + password: SecretStr | None = None + host: str = "localhost" + port: int = 26257 + db_name: str = "defaultdb" + table_name: str = "vdbbench_cockroachdb" + isolation_level: str = "serializable" + pool_size: int = 100 + max_overflow: int = 100 + pool_recycle: int = 3600 + connect_timeout: int = 10 + sslmode: str = "disable" # Options: disable, require, verify-ca, verify-full + sslrootcert: str | None = None # Path to CA cert (for verify-ca, verify-full) + sslcert: str | None = None # Path to client cert (for mutual TLS) + sslkey: str | None = None # Path to client key (for mutual TLS) + + def to_dict(self) -> CockroachDBConfigDict: + user_str = self.user_name.get_secret_value() if isinstance(self.user_name, SecretStr) else self.user_name + pwd_str = self.password.get_secret_value() if self.password else "" + + connect_config = { + "host": self.host, + "port": self.port, + "dbname": self.db_name, + "user": user_str, + "password": pwd_str, + "sslmode": self.sslmode, + } + + # Add SSL certificate paths if provided + if self.sslrootcert: + connect_config["sslrootcert"] = self.sslrootcert + if self.sslcert: + connect_config["sslcert"] = self.sslcert + if self.sslkey: + connect_config["sslkey"] = self.sslkey + + return { + "connect_config": connect_config, + "table_name": self.table_name, + "pool_size": self.pool_size, + "max_overflow": self.max_overflow, + "pool_recycle": self.pool_recycle, + "connect_timeout": self.connect_timeout, + } + + +class CockroachDBIndexParam(TypedDict): + """Index parameters for CockroachDB vector indexes.""" + + metric: str + index_creation_with_options: Sequence[dict[str, Any]] + min_partition_size: int | None + max_partition_size: int | None + build_beam_size: int | None + + +class CockroachDBSearchParam(TypedDict): + """Search parameters for CockroachDB vector queries.""" + + metric_fun_op: LiteralString + vector_search_beam_size: int | None + + +class CockroachDBSessionCommands(TypedDict): + """Session-level commands for CockroachDB.""" + + session_options: Sequence[dict[str, Any]] + + +class CockroachDBIndexConfig(BaseModel, DBCaseConfig): + """Base configuration for CockroachDB vector indexes.""" + + metric_type: MetricType | None = None + create_index_before_load: bool = False + create_index_after_load: bool = True + min_partition_size: int | None = 16 + max_partition_size: int | None = 128 + build_beam_size: int | None = 8 + vector_search_beam_size: int | None = 32 + + def parse_metric(self) -> str: + """Parse metric type to CockroachDB opclass.""" + metric_map = { + MetricType.L2: "vector_l2_ops", + MetricType.IP: "vector_ip_ops", + MetricType.COSINE: "vector_cosine_ops", + } + metric = metric_map.get(self.metric_type) + if metric is None: + msg = f"Unsupported metric type: {self.metric_type}" + raise ValueError(msg) + return metric + + def parse_metric_fun_op(self) -> LiteralString: + """Parse metric type to distance operator.""" + if self.metric_type == MetricType.L2: + return "<->" + if self.metric_type == MetricType.IP: + return "<#>" + return "<=>" + + @abstractmethod + def index_param(self) -> CockroachDBIndexParam: ... + + @abstractmethod + def search_param(self) -> CockroachDBSearchParam: ... + + @abstractmethod + def session_param(self) -> CockroachDBSessionCommands: ... + + @staticmethod + def _optionally_build_with_options(with_options: Mapping[str, Any]) -> Sequence[dict[str, Any]]: + """Build WITH options for index creation.""" + options = [] + for option_name, value in with_options.items(): + if value is not None: + options.append( + { + "option_name": option_name, + "val": str(value), + }, + ) + return options + + @staticmethod + def _optionally_build_set_options(set_mapping: Mapping[str, Any]) -> Sequence[dict[str, Any]]: + """Build SET options for session configuration.""" + session_options = [] + for setting_name, value in set_mapping.items(): + if value is not None: + session_options.append( + { + "parameter": { + "setting_name": setting_name, + "val": str(value), + }, + }, + ) + return session_options + + +class CockroachDBVectorIndexConfig(CockroachDBIndexConfig): + """ + CockroachDB Vector Index Configuration using C-SPANN algorithm. + + Available since CockroachDB v25.2. Uses hierarchical k-means clustering + for efficient approximate nearest neighbor (ANN) search. + + Tunable parameters: + - min_partition_size: Minimum vectors per partition (default: 16, range: 1-1024) + - max_partition_size: Maximum vectors per partition (default: 128, range: 4x min-4096) + - vector_search_beam_size: Partitions explored during search (default: 32) + """ + + index: IndexType = IndexType.Flat + + def index_param(self) -> CockroachDBIndexParam: + """Get index creation parameters.""" + index_parameters = { + "min_partition_size": self.min_partition_size, + "max_partition_size": self.max_partition_size, + "build_beam_size": self.build_beam_size, + } + + return { + "metric": self.parse_metric(), + "index_creation_with_options": self._optionally_build_with_options(index_parameters), + "min_partition_size": self.min_partition_size, + "max_partition_size": self.max_partition_size, + "build_beam_size": self.build_beam_size, + } + + def search_param(self) -> CockroachDBSearchParam: + """Get search parameters.""" + return { + "metric_fun_op": self.parse_metric_fun_op(), + "vector_search_beam_size": self.vector_search_beam_size, + } + + def session_param(self) -> CockroachDBSessionCommands: + """Get session parameters.""" + session_parameters = {"vector_search_beam_size": self.vector_search_beam_size} + return {"session_options": self._optionally_build_set_options(session_parameters)} + + +_cockroachdb_case_config = { + IndexType.Flat: CockroachDBVectorIndexConfig, + IndexType.HNSW: CockroachDBVectorIndexConfig, + IndexType.IVFFlat: CockroachDBVectorIndexConfig, +} diff --git a/vectordb_bench/backend/clients/cockroachdb/db_retry.py b/vectordb_bench/backend/clients/cockroachdb/db_retry.py new file mode 100644 index 000000000..a838c029c --- /dev/null +++ b/vectordb_bench/backend/clients/cockroachdb/db_retry.py @@ -0,0 +1,113 @@ +""" +Database retry utilities for handling CockroachDB transient connection failures. + +Implements retry logic for CockroachDB-specific error codes: +- 40001: Serialization failure +- 40003: Statement completion unknown (multi-region ambiguous results) +""" + +import functools +import logging +import time +from collections.abc import Callable + +from psycopg import InterfaceError, OperationalError + +log = logging.getLogger(__name__) + +TRANSIENT_ERRORS = ( + OperationalError, + InterfaceError, + Exception, # Catch all for transient detection +) + +TRANSIENT_ERROR_MESSAGES = ( + "server closed the connection unexpectedly", + "connection already closed", + "SSL connection has been closed unexpectedly", + "could not receive data from server", + "connection timed out", + "Connection refused", + "connection reset by peer", + "broken pipe", + "restart transaction", + "TransactionRetryError", + "SerializationFailure", + "StatementCompletionUnknown", + "result is ambiguous", + "failed to connect", + "no such host", + "initial connection heartbeat failed", + "sending to all replicas failed", + "transaction is aborted", + "commands ignored until end of transaction block", + "40001", + "40003", +) + + +def is_transient_error(error: Exception) -> bool: + """Check if an error is transient and should be retried.""" + if not isinstance(error, TRANSIENT_ERRORS): + return False + + error_msg = str(error).lower() + return any(msg.lower() in error_msg for msg in TRANSIENT_ERROR_MESSAGES) + + +def db_retry( + max_attempts: int = 3, + initial_delay: float = 0.5, + backoff_factor: float = 2.0, + max_delay: float = 10.0, + exceptions: tuple[type[Exception], ...] = TRANSIENT_ERRORS, +): + """ + Decorator to retry database operations on transient failures. + + Uses exponential backoff with configurable parameters per CockroachDB best practices. + + Args: + max_attempts: Maximum number of retry attempts (default: 3) + initial_delay: Initial retry delay in seconds (default: 0.5) + backoff_factor: Multiplier for exponential backoff (default: 2.0) + max_delay: Maximum delay between retries (default: 10.0) + exceptions: Tuple of exception types to catch and retry + """ + + def decorator(func: Callable) -> Callable: + @functools.wraps(func) + def wrapper(*args, **kwargs): + attempt = 0 + delay = initial_delay + + while attempt < max_attempts: + try: + return func(*args, **kwargs) + + except exceptions as e: + attempt += 1 + + if not is_transient_error(e): + log.warning(f"Non-transient error in {func.__name__}: {e}") + raise + + if attempt >= max_attempts: + log.warning(f"Max retry attempts ({max_attempts}) reached for {func.__name__}") + raise + + current_delay = min(delay * (backoff_factor ** (attempt - 1)), max_delay) + + log.info( + f"Transient DB error in {func.__name__} (attempt {attempt}/{max_attempts}): {e}. " + f"Retrying in {current_delay:.2f}s..." + ) + + time.sleep(current_delay) + + msg = f"Unexpected state in retry logic for {func.__name__}" + raise RuntimeError(msg) + + return wrapper + + return decorator diff --git a/vectordb_bench/backend/clients/doris/cli.py b/vectordb_bench/backend/clients/doris/cli.py new file mode 100644 index 000000000..8153b412d --- /dev/null +++ b/vectordb_bench/backend/clients/doris/cli.py @@ -0,0 +1,199 @@ +from typing import Annotated, Unpack + +import click +from pydantic import SecretStr + +from vectordb_bench.backend.clients import DB + +from ....cli.cli import ( + CommonTypedDict, + HNSWBaseTypedDict, + cli, + click_parameter_decorators_from_typed_dict, + run, +) + + +def _parse_kv_list(_ctx, _param, values): # noqa: ANN001 + """Parse repeatable or comma-separated key=value items into a dict. + + Accepts any of the following forms (and mixtures thereof): + --index-prop a=1 --index-prop b=2 + --index-prop a=1,b=2 + --index-prop a=1,b=2 --index-prop c=3 + """ + parsed: dict[str, str] = {} + if not values: + return parsed + for item in values: + # allow comma-separated list in a single occurrence + parts = [p.strip() for p in str(item).split(",") if p and p.strip()] + for part in parts: + if "=" not in part: + msg = f"Expect key=value, got: {part}" + raise click.BadParameter(msg) + k, v = part.split("=", 1) + k = k.strip() + v = v.strip() + if not k: + msg = f"Empty key in: {part}" + raise click.BadParameter(msg) + parsed[k] = v + return parsed + + +class DorisTypedDict(CommonTypedDict, HNSWBaseTypedDict): + user_name: Annotated[ + str, + click.option( + "--username", + type=str, + help="Username", + default="root", + show_default=True, + required=True, + ), + ] + password: Annotated[ + str, + click.option( + "--password", + type=str, + default="", + show_default=True, + help="Password", + ), + ] + host: Annotated[ + str, + click.option( + "--host", + type=str, + default="127.0.0.1", + show_default=True, + required=True, + help="Db host", + ), + ] + port: Annotated[ + int, + click.option( + "--port", + type=int, + default=9030, + show_default=True, + required=True, + help="Query Port", + ), + ] + http_port: Annotated[ + int, + click.option( + "--http-port", + type=int, + default=8030, + show_default=True, + required=True, + help="Http Port", + ), + ] + db_name: Annotated[ + str, + click.option( + "--db-name", + type=str, + default="test", + show_default=True, + required=True, + help="Db name", + ), + ] + ssl: Annotated[ + bool, + click.option( + "--ssl/--no-ssl", + default=False, + show_default=True, + is_flag=True, + help="Enable or disable SSL, for Doris Serverless SSL must be enabled", + ), + ] + index_prop: Annotated[ + dict, + click.option( + "--index-prop", + type=str, + multiple=True, + help="Extra index PROPERTY as key=value (repeatable or comma-separated, e.g. a=1,b=2)", + callback=_parse_kv_list, + ), + ] + session_var: Annotated[ + dict, + click.option( + "--session-var", + type=str, + multiple=True, + help="Session variable key=value applied to each SQL session (repeatable or comma-separated)", + callback=_parse_kv_list, + ), + ] + stream_load_rows_per_batch: Annotated[ + int | None, + click.option( + "--stream-load-rows-per-batch", + type=int, + required=False, + help="Rows per single stream load request; default uses NUM_PER_BATCH", + ), + ] + no_index: Annotated[ + bool, + click.option( + "--no-index", + is_flag=True, + default=False, + show_default=True, + help="Create table without ANN index", + ), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(DorisTypedDict) +def Doris( + **parameters: Unpack[DorisTypedDict], +): + from .config import DorisCaseConfig, DorisConfig + + # Merge explicit HNSW params into index properties using Doris naming + index_properties: dict[str, str] = {} + index_properties.update(parameters.get("index_prop", {}) or {}) + if parameters.get("m") is not None: + index_properties.setdefault("max_degree", str(parameters["m"])) + if parameters.get("ef_construction") is not None: + index_properties.setdefault("ef_construction", str(parameters["ef_construction"])) + + session_vars: dict[str, str] = parameters.get("session_var", {}) or {} + + run( + db=DB.Doris, + db_config=DorisConfig( + db_label=parameters["db_label"], + user_name=parameters["username"], + password=SecretStr(parameters["password"]), + host=parameters["host"], + port=parameters["port"], + http_port=parameters["http_port"], + db_name=parameters["db_name"], + ssl=parameters["ssl"], + ), + # metric_type should come from the dataset; Assembler will set it on the case config. + db_case_config=DorisCaseConfig( + index_properties=index_properties, + session_vars=session_vars, + stream_load_rows_per_batch=parameters.get("stream_load_rows_per_batch"), + no_index=parameters.get("no_index", False), + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/doris/config.py b/vectordb_bench/backend/clients/doris/config.py new file mode 100644 index 000000000..a15309922 --- /dev/null +++ b/vectordb_bench/backend/clients/doris/config.py @@ -0,0 +1,99 @@ +import logging + +from pydantic import BaseModel, SecretStr, validator + +from ..api import DBCaseConfig, DBConfig, MetricType + +log = logging.getLogger(__name__) + + +class DorisConfig(DBConfig): + user_name: str = "root" + password: SecretStr + host: str = "127.0.0.1" + port: int = 9030 + # Doris FE HTTP port for stream load. Default 8030 (8040 for HTTPS if enabled). + http_port: int = 8030 + db_name: str = "test" + ssl: bool = False + + @validator("*") + def not_empty_field(cls, v: any, field: any): + return v + + def to_dict(self) -> dict: + pwd_str = self.password.get_secret_value() + return { + "host": self.host, + "port": self.port, + "http_port": self.http_port, + "user": self.user_name, + "password": pwd_str, + "database": self.db_name, + } + + +class DorisCaseConfig(BaseModel, DBCaseConfig): + metric_type: MetricType | None = None + # Optional explicit HNSW/IVF params for convenience + index_type: str | None = None + m: int | None = None + ef_construction: int | None = None + nlist: int | None = None + # Arbitrary index properties and session variables + index_properties: dict[str, str] | None = None + session_vars: dict[str, str] | None = None + # Control rows per single stream load request + stream_load_rows_per_batch: int | None = None + # Create table without ANN index + no_index: bool = False + + def get_metric_fn(self) -> str: + if self.metric_type == MetricType.L2: + return "l2_distance_approximate" + if self.metric_type == MetricType.IP: + return "inner_product_approximate" + if self.metric_type == MetricType.COSINE: + log.debug("Using inner_product_approximate because doris doesn't support cosine as metric type") + return "inner_product_approximate" + msg = f"Unsupported metric type: {self.metric_type}" + raise ValueError(msg) + + def index_param(self) -> dict: + # Use exact metric function name for index creation by removing '_approximate' suffix + metric_type = self.get_metric_fn() + if metric_type.endswith("_approximate"): + metric_type = metric_type[: -len("_approximate")] + props = {"metric_type": metric_type} + + if self.index_type is not None: + props.setdefault("index_type", self.index_type) + else: + props.setdefault("index_type", "hnsw") + + # Merge optional HNSW params + props["index_type"] = str.lower(props["index_type"]) + if props["index_type"] == "hnsw": + if self.m is not None: + props.setdefault("max_degree", str(self.m)) + if self.ef_construction is not None: + props.setdefault("ef_construction", str(self.ef_construction)) + elif props["index_type"] == "ivf": + if self.nlist is not None: + props.setdefault("nlist", str(self.nlist)) + else: + msg = f"Unsupported index type: {props['index_type']}" + raise ValueError(msg) + + # Merge user provided index_properties + if self.index_properties: + props.update(self.index_properties) + return props + + def search_param(self) -> dict: + return { + "metric_fn": self.get_metric_fn(), + } + + def session_param(self) -> dict: + return self.session_vars or {} diff --git a/vectordb_bench/backend/clients/doris/doris.py b/vectordb_bench/backend/clients/doris/doris.py new file mode 100644 index 000000000..82b3a12da --- /dev/null +++ b/vectordb_bench/backend/clients/doris/doris.py @@ -0,0 +1,442 @@ +import logging +import os +from contextlib import contextmanager +from typing import Any + +import pandas as pd +from doris_vector_search import AuthOptions, DorisVectorClient, IndexOptions, LoadOptions + +from ..api import MetricType, VectorDB +from .config import DorisCaseConfig + +log = logging.getLogger(__name__) + + +class Doris(VectorDB): + def __init__( + self, + dim: int, + db_config: dict, + db_case_config: DorisCaseConfig, + collection_name: str | None = None, + drop_old: bool = False, + **kwargs, + ): + self.name = "Doris" + self.db_config = db_config + self.case_config = db_case_config + self.dim = dim + self.search_fn = db_case_config.search_param()["metric_fn"] + # Prefer provided collection_name; otherwise fallback to a simple default + # e.g. l2_distance128, inner_product128 + self.table_name = collection_name if collection_name else (self.search_fn + str(dim)) + + # Store connection configuration for lazy initialization + self.auth_options = AuthOptions( + host=db_config.get("host", "127.0.0.1"), + query_port=db_config.get("port", 9030), + http_port=db_config.get("http_port", 8030), + user=db_config.get("user", "root"), + password=db_config.get("password", ""), + ) + + # Configure load options + self.load_options = None + if hasattr(db_case_config, "stream_load_rows_per_batch") and db_case_config.stream_load_rows_per_batch: + self.load_options = LoadOptions(batch_size=db_case_config.stream_load_rows_per_batch) + + # Store database name for lazy initialization + self.database_name = db_config.get("database", "test") + + # Initialize client and table as None (lazy initialization) + self.client = None + self.table = None + self._client_pid: int | None = None + + if drop_old: + self._drop_table() + self._create_table() + + def _ensure_client_initialized(self): + """Ensure the client is initialized and bound to the current process. + + Multiprocessing will pickle the DB wrapper. Any existing mysql connection or + table cursor cached from a different PID must be discarded and recreated here. + """ + cur_pid = os.getpid() + + need_new_client = False + if self.client is None: + need_new_client = True + else: + # If client was created in another process or connection is not usable, recreate it + try: + # different process + if self._client_pid is None or self._client_pid != cur_pid: + need_new_client = True + else: + # check connection health if available + conn = getattr(self.client, "connection", None) + if conn is None or not getattr(conn, "is_connected", lambda: False)(): + need_new_client = True + except Exception: + need_new_client = True + + if need_new_client: + # Drop any table cached from another PID (its cursors are not valid across processes) + self.table = None + + # Recreate client and set sessions + self.client = DorisVectorClient( + database=self.database_name, + auth_options=self.auth_options, + load_options=self.load_options, + ) + + if hasattr(self.case_config, "session_vars") and self.case_config.session_vars: + self.client.with_sessions(self.case_config.session_vars) + + self._client_pid = cur_pid + + # Re-open table in this process to ensure fresh cursors + try: + self.table = self.client.open_table(self.table_name) + if hasattr(self.table, "index_options") and self.table.index_options: + self.table.index_options.dim = self.dim + if self.search_fn.startswith("inner_product"): + self.table.index_options.metric_type = "inner_product" + else: + self.table.index_options.metric_type = "l2_distance" + except Exception: + # Table might not exist yet; leave it to ready_to_load + self.table = None + + @contextmanager + def init(self): + try: + self._ensure_client_initialized() + # Open or create the table + if not self.table: + try: + # Try to open existing table + self.table = self.client.open_table(self.table_name) + # Avoid SHOW CREATE TABLE parsing in SDK by setting dim/metric directly + try: + if hasattr(self.table, "index_options") and self.table.index_options: + self.table.index_options.dim = self.dim + # Set metric_type according to current case + if self.search_fn.startswith("inner_product"): + self.table.index_options.metric_type = "inner_product" + else: + self.table.index_options.metric_type = "l2_distance" + except Exception: + log.exception("Failed to update index options for table: %s", self.table_name) + except Exception: + # Table doesn't exist, will be created in ready_to_load + self.table = None + yield + finally: + # Clean up if needed + pass + + def _drop_table(self): + try: + self._ensure_client_initialized() + self.client.drop_table(self.table_name) + except Exception: + log.exception("Failed to drop table: %s", self.table_name) + raise + + def _create_table(self): + """Create the table using doris-vector-search library""" + try: + self._ensure_client_initialized() + sample_data = pd.DataFrame([{"id": 1, "embedding": [0.0] * self.dim}]) + + index_options = None + if not getattr(self.case_config, "no_index", False): + index_options = self._build_index_options() + + self._create_table_with_options(sample_data, index_options) + log.info("Successfully created table %s", self.table_name) + + except Exception: + log.exception("Failed to create table: %s", self.table_name) + raise + + def _build_index_options(self) -> IndexOptions | None: + index_param = self.case_config.index_param() + index_options = IndexOptions() + + applied, not_applied = {}, {} + for key, value in index_param.items(): + attr_name = key + if hasattr(index_options, attr_name): + try: + setattr(index_options, attr_name, value) + applied[key] = value + except Exception: + not_applied[key] = value + else: + not_applied[key] = value + + log.info( + "Index options prepared: applied_props=%s not_applied_props=%s", + applied, + not_applied, + ) + + log.info("Creating table %s with index %s", self.table_name, index_param) + return index_options + + def _create_table_with_options(self, sample_data: pd.DataFrame, index_options: IndexOptions | None) -> None: + create_index = not getattr(self.case_config, "no_index", False) + if not create_index: + log.info("Creating table %s without ANN index", self.table_name) + + self.table = self.client.create_table( + self.table_name, + sample_data, + create_index=create_index, + index_options=index_options, + overwrite=True, + insert_data=False, + ) + + try: + if hasattr(self.table, "index_options") and self.table.index_options: + self.table.index_options.dim = self.dim + if self.search_fn.startswith("inner_product"): + self.table.index_options.metric_type = "inner_product" + else: + self.table.index_options.metric_type = "l2_distance" + if ( + index_options + and hasattr(index_options, "properties") + and isinstance(index_options.properties, dict) + ): + for key, value in index_options.properties.items(): + if hasattr(self.table.index_options, key): + try: + setattr(self.table.index_options, key, value) + except Exception: + log.debug("Skip setting index_options.%s at runtime", key) + except Exception: + log.exception("Failed to adjust index options for table: %s", self.table_name) + + def ready_to_load(self) -> bool: + self._ensure_client_initialized() + if not self.table: + self._create_table() + return True + + def optimize(self, data_size: int | None = None) -> None: + log.info("Optimization completed using doris-vector-search library") + + def need_normalize_cosine(self) -> bool: + """Wheather this database need to normalize dataset to support COSINE""" + if self.case_config.metric_type == MetricType.COSINE: + log.info("cosine dataset need normalize.") + return True + + return False + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + **kwargs: Any, + ) -> tuple[int, Exception | None]: + """Insert embeddings using doris-vector-search library.""" + try: + self._ensure_client_initialized() + # Prepare data in pandas DataFrame format + data = pd.DataFrame([{"id": metadata[i], "embedding": embeddings[i]} for i in range(len(embeddings))]) + + msg = f"Inserting {len(embeddings)} embeddings into table {self.table_name}" + log.info(msg) + + # Add data to the table + self.table.add(data) + + return len(metadata), None + + except Exception as e: + msg = "Failed to insert embeddings" + log.exception(msg) + return 0, e + + def search_embedding( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + timeout: int | None = None, + **kwargs: Any, + ) -> list[int]: + try: + self._ensure_client_initialized() + # Map metric functions to doris-vector-search metric types + metric_type = "l2_distance" + if self.search_fn.startswith("inner_product"): + metric_type = "inner_product" + elif self.search_fn.startswith("l2_distance"): + metric_type = "l2_distance" + + # Perform search using doris-vector-search + search_query = self.table.search(query, metric_type=metric_type).limit(k).select(["id"]) + + # Apply filters if provided + if filters and "id" in filters: + if self.search_fn.startswith("inner_product"): + where_clause = f"id >= {filters['id']}" + search_query = search_query.where(where_clause) + else: + where_clause = f"id < {filters['id']}" + search_query = search_query.where(where_clause) + + # Execute and get results + results_df = search_query.to_pandas() + return results_df["id"].tolist() + + except Exception: + msg = "Search embedding failed" + log.exception(msg) + return [] + + def search_embedding_range( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + distance: float | None = None, + timeout: int | None = None, + **kwargs: Any, + ) -> list[int]: + try: + self._ensure_client_initialized() + # Map metric functions to doris-vector-search metric types + metric_type = "l2_distance" + if self.search_fn.startswith("inner_product"): + metric_type = "inner_product" + elif self.search_fn.startswith("l2_distance"): + metric_type = "l2_distance" + + # Perform range search using doris-vector-search + search_query = self.table.search(query, metric_type=metric_type).select(["id"]) + + # Apply distance range + if distance is not None: + if self.search_fn.startswith("inner_product"): + adjusted_distance = distance - 0.000001 + search_query = search_query.distance_range(lower_bound=adjusted_distance) + else: + adjusted_distance = distance + 0.000001 + search_query = search_query.distance_range(upper_bound=adjusted_distance) + + # Execute and get results + results_df = search_query.to_pandas() + return results_df["id"].tolist() + + except Exception: + msg = "Range search failed" + log.exception(msg) + return [] + + def search_embedding_compound( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + distance: float | None = None, + timeout: int | None = None, + **kwargs: Any, + ) -> list[int]: + try: + self._ensure_client_initialized() + # Map metric functions to doris-vector-search metric types + metric_type = "l2_distance" + if self.search_fn.startswith("inner_product"): + metric_type = "inner_product" + elif self.search_fn.startswith("l2_distance"): + metric_type = "l2_distance" + + # Perform compound search using doris-vector-search + search_query = self.table.search(query, metric_type=metric_type).limit(k).select(["id"]) + + # Apply distance range + if distance is not None: + if self.search_fn.startswith("inner_product"): + adjusted_distance = distance - 0.000001 + search_query = search_query.distance_range(lower_bound=adjusted_distance) + else: + adjusted_distance = distance + 0.000001 + search_query = search_query.distance_range(upper_bound=adjusted_distance) + + # Execute and get results + results_df = search_query.to_pandas() + return results_df["id"].tolist() + + except Exception: + msg = "Compound search failed" + log.exception(msg) + return [] + + def search_distance(self, query: list[float], target_id: int | None = None): + try: + self._ensure_client_initialized() + metric_type = self.search_fn + if metric_type.endswith("_approximate"): + metric_type = metric_type.replace("_approximate", "") + + search_metric = "inner_product" if metric_type.startswith("inner_product") else "l2_distance" + where_clause = f"id = {target_id}" + search_query = self.table.search(query, metric_type=search_metric).where(where_clause).select(["id"]) + results_df = search_query.to_pandas() + except Exception: + msg = "Distance search failed" + log.exception(msg) + return [] + + # For now, return a placeholder distance + # The exact distance calculation would need custom SQL or library support + return [0.0] if not results_df.empty else [] + + def search_embedding_exact( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + timeout: int | None = None, + **kwargs: Any, + ) -> list[int]: + try: + self._ensure_client_initialized() + # Use exact search by removing approximate suffix + metric_type = self.search_fn + if metric_type.endswith("_approximate"): + metric_type = metric_type.replace("_approximate", "") + + # Map to doris-vector-search metric types + search_metric = "inner_product" if metric_type.startswith("inner_product") else "l2_distance" + + # Perform exact search + search_query = self.table.search(query, metric_type=search_metric).limit(k).select(["id"]) + + # Apply filters if provided + if filters and "id" in filters: + if metric_type.startswith("inner_product"): + where_clause = f"id >= {filters['id']}" + search_query = search_query.where(where_clause) + else: + where_clause = f"id < {filters['id']}" + search_query = search_query.where(where_clause) + + # Execute and get results + results_df = search_query.to_pandas() + return results_df["id"].tolist() + + except Exception: + msg = "Exact search failed" + log.exception(msg) + return [] diff --git a/vectordb_bench/backend/clients/elastic_cloud/cli.py b/vectordb_bench/backend/clients/elastic_cloud/cli.py new file mode 100644 index 000000000..55e521d8f --- /dev/null +++ b/vectordb_bench/backend/clients/elastic_cloud/cli.py @@ -0,0 +1,288 @@ +from typing import Annotated, TypedDict, Unpack + +import click +from pydantic import SecretStr + +from vectordb_bench.backend.clients import DB +from vectordb_bench.cli.cli import ( + CommonTypedDict, + cli, + click_parameter_decorators_from_typed_dict, + run, +) + +DBTYPE = DB.ElasticCloud + + +class ElasticCloudTypedDict(TypedDict): + cloud_id: Annotated[ + str, + click.option("--cloud-id", type=str, help="Elastic Cloud ID", required=True), + ] + password: Annotated[ + str, + click.option("--password", type=str, help="Elastic Cloud password", required=True), + ] + number_of_shards: Annotated[ + int, + click.option( + "--number-of-shards", + type=int, + help="Number of shards", + required=False, + default=1, + show_default=True, + ), + ] + number_of_replicas: Annotated[ + int, + click.option( + "--number-of-replicas", + type=int, + help="Number of replicas", + required=False, + default=0, + show_default=True, + ), + ] + refresh_interval: Annotated[ + str, + click.option( + "--refresh-interval", + type=str, + help="Index refresh interval", + required=False, + default="30s", + show_default=True, + ), + ] + merge_max_thread_count: Annotated[ + int, + click.option( + "--merge-max-thread-count", + type=int, + help="Maximum thread count for merge", + required=False, + default=8, + show_default=True, + ), + ] + use_force_merge: Annotated[ + bool, + click.option( + "--use-force-merge", + type=bool, + help="Whether to use force merge", + required=False, + default=True, + show_default=True, + ), + ] + use_routing: Annotated[ + bool, + click.option( + "--use-routing", + type=bool, + help="Whether to use routing", + required=False, + default=False, + show_default=True, + ), + ] + use_rescore: Annotated[ + bool, + click.option( + "--use-rescore", + type=bool, + help="Whether to use rescore", + required=False, + default=False, + show_default=True, + ), + ] + oversample_ratio: Annotated[ + float, + click.option( + "--oversample-ratio", + type=float, + help="Oversample ratio for rescore", + required=False, + default=2.0, + show_default=True, + ), + ] + + +class ElasticCloudHNSWTypedDict(CommonTypedDict, ElasticCloudTypedDict): + m: Annotated[ + int, + click.option( + "--m", + type=int, + help="HNSW M parameter", + required=False, + default=16, + show_default=True, + ), + ] + ef_construction: Annotated[ + int, + click.option( + "--ef-construction", + type=int, + help="HNSW efConstruction parameter", + required=False, + default=100, + show_default=True, + ), + ] + num_candidates: Annotated[ + int, + click.option( + "--num-candidates", + type=int, + help="Number of candidates for search", + required=False, + default=100, + show_default=True, + ), + ] + element_type: Annotated[ + str, + click.option( + "--element-type", + type=click.Choice(["float", "byte"], case_sensitive=False), + help="Element type for vectors (float: 4 bytes, byte: 1 byte)", + required=False, + default="float", + show_default=True, + ), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(ElasticCloudHNSWTypedDict) +def ElasticCloudHNSW(**parameters: Unpack[ElasticCloudHNSWTypedDict]): + from ..api import IndexType + from .config import ElasticCloudConfig, ElasticCloudIndexConfig, ESElementType + + run( + db=DBTYPE, + db_config=ElasticCloudConfig( + db_label=parameters["db_label"], + cloud_id=SecretStr(parameters["cloud_id"]), + password=SecretStr(parameters["password"]), + ), + db_case_config=ElasticCloudIndexConfig( + index=IndexType.ES_HNSW, + M=parameters["m"], + efConstruction=parameters["ef_construction"], + num_candidates=parameters["num_candidates"], + element_type=ESElementType(parameters["element_type"]), + number_of_shards=parameters["number_of_shards"], + number_of_replicas=parameters["number_of_replicas"], + refresh_interval=parameters["refresh_interval"], + merge_max_thread_count=parameters["merge_max_thread_count"], + use_force_merge=parameters["use_force_merge"], + use_routing=parameters["use_routing"], + use_rescore=parameters["use_rescore"], + oversample_ratio=parameters["oversample_ratio"], + ), + **parameters, + ) + + +@cli.command() +@click_parameter_decorators_from_typed_dict(ElasticCloudHNSWTypedDict) +def ElasticCloudHNSWInt8(**parameters: Unpack[ElasticCloudHNSWTypedDict]): + from ..api import IndexType + from .config import ElasticCloudConfig, ElasticCloudIndexConfig, ESElementType + + run( + db=DBTYPE, + db_config=ElasticCloudConfig( + db_label=parameters["db_label"], + cloud_id=SecretStr(parameters["cloud_id"]), + password=SecretStr(parameters["password"]), + ), + db_case_config=ElasticCloudIndexConfig( + index=IndexType.ES_HNSW_INT8, + M=parameters["m"], + efConstruction=parameters["ef_construction"], + num_candidates=parameters["num_candidates"], + element_type=ESElementType(parameters["element_type"]), + number_of_shards=parameters["number_of_shards"], + number_of_replicas=parameters["number_of_replicas"], + refresh_interval=parameters["refresh_interval"], + merge_max_thread_count=parameters["merge_max_thread_count"], + use_force_merge=parameters["use_force_merge"], + use_routing=parameters["use_routing"], + use_rescore=parameters["use_rescore"], + oversample_ratio=parameters["oversample_ratio"], + ), + **parameters, + ) + + +@cli.command() +@click_parameter_decorators_from_typed_dict(ElasticCloudHNSWTypedDict) +def ElasticCloudHNSWInt4(**parameters: Unpack[ElasticCloudHNSWTypedDict]): + from ..api import IndexType + from .config import ElasticCloudConfig, ElasticCloudIndexConfig, ESElementType + + run( + db=DBTYPE, + db_config=ElasticCloudConfig( + db_label=parameters["db_label"], + cloud_id=SecretStr(parameters["cloud_id"]), + password=SecretStr(parameters["password"]), + ), + db_case_config=ElasticCloudIndexConfig( + index=IndexType.ES_HNSW_INT4, + M=parameters["m"], + efConstruction=parameters["ef_construction"], + num_candidates=parameters["num_candidates"], + element_type=ESElementType(parameters["element_type"]), + number_of_shards=parameters["number_of_shards"], + number_of_replicas=parameters["number_of_replicas"], + refresh_interval=parameters["refresh_interval"], + merge_max_thread_count=parameters["merge_max_thread_count"], + use_force_merge=parameters["use_force_merge"], + use_routing=parameters["use_routing"], + use_rescore=parameters["use_rescore"], + oversample_ratio=parameters["oversample_ratio"], + ), + **parameters, + ) + + +@cli.command() +@click_parameter_decorators_from_typed_dict(ElasticCloudHNSWTypedDict) +def ElasticCloudHNSWBBQ(**parameters: Unpack[ElasticCloudHNSWTypedDict]): + from ..api import IndexType + from .config import ElasticCloudConfig, ElasticCloudIndexConfig, ESElementType + + run( + db=DBTYPE, + db_config=ElasticCloudConfig( + db_label=parameters["db_label"], + cloud_id=SecretStr(parameters["cloud_id"]), + password=SecretStr(parameters["password"]), + ), + db_case_config=ElasticCloudIndexConfig( + index=IndexType.ES_HNSW_BBQ, + M=parameters["m"], + efConstruction=parameters["ef_construction"], + num_candidates=parameters["num_candidates"], + element_type=ESElementType(parameters["element_type"]), + number_of_shards=parameters["number_of_shards"], + number_of_replicas=parameters["number_of_replicas"], + refresh_interval=parameters["refresh_interval"], + merge_max_thread_count=parameters["merge_max_thread_count"], + use_force_merge=parameters["use_force_merge"], + use_routing=parameters["use_routing"], + use_rescore=parameters["use_rescore"], + oversample_ratio=parameters["oversample_ratio"], + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/elastic_cloud/config.py b/vectordb_bench/backend/clients/elastic_cloud/config.py index d35ee68ec..13826a61e 100644 --- a/vectordb_bench/backend/clients/elastic_cloud/config.py +++ b/vectordb_bench/backend/clients/elastic_cloud/config.py @@ -23,13 +23,44 @@ class ESElementType(str, Enum): class ElasticCloudIndexConfig(BaseModel, DBCaseConfig): element_type: ESElementType = ESElementType.float - index: IndexType = IndexType.ES_HNSW # ES only support 'hnsw' + index: IndexType = IndexType.ES_HNSW + number_of_shards: int = 1 + number_of_replicas: int = 0 + refresh_interval: str = "30s" + merge_max_thread_count: int = 8 + use_rescore: bool = False + oversample_ratio: float = 2.0 + use_routing: bool = False + use_force_merge: bool = True metric_type: MetricType | None = None efConstruction: int | None = None M: int | None = None num_candidates: int | None = None + def __eq__(self, obj: any): + return ( + self.index == obj.index + and self.number_of_shards == obj.number_of_shards + and self.number_of_replicas == obj.number_of_replicas + and self.use_routing == obj.use_routing + and self.efConstruction == obj.efConstruction + and self.M == obj.M + ) + + def __hash__(self) -> int: + return hash( + ( + self.index, + self.number_of_shards, + self.number_of_replicas, + self.use_routing, + self.efConstruction, + self.M, + 2, + ) + ) + def parse_metric(self) -> str: if self.metric_type == MetricType.L2: return "l2_norm" diff --git a/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py b/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py index 7d201729f..ae3350ddf 100644 --- a/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py +++ b/vectordb_bench/backend/clients/elastic_cloud/elastic_cloud.py @@ -5,6 +5,8 @@ from elasticsearch.helpers import bulk +from vectordb_bench.backend.filter import Filter, FilterOp + from ..api import VectorDB from .config import ElasticCloudIndexConfig @@ -18,6 +20,12 @@ class ElasticCloud(VectorDB): + supported_filter_types: list[FilterOp] = [ + FilterOp.NonFilter, + FilterOp.NumGE, + FilterOp.StrEqual, + ] + def __init__( self, dim: int, @@ -25,8 +33,10 @@ def __init__( db_case_config: ElasticCloudIndexConfig, indice: str = "vdb_bench_indice", # must be lowercase id_col_name: str = "id", + label_col_name: str = "label", vector_col_name: str = "vector", drop_old: bool = False, + with_scalar_labels: bool = False, **kwargs, ): self.dim = dim @@ -34,7 +44,9 @@ def __init__( self.case_config = db_case_config self.indice = indice self.id_col_name = id_col_name + self.label_col_name = label_col_name self.vector_col_name = vector_col_name + self.with_scalar_labels = with_scalar_labels from elasticsearch import Elasticsearch @@ -69,9 +81,17 @@ def _create_indice(self, client: any) -> None: }, }, } + settings = { + "index": { + "number_of_shards": self.case_config.number_of_shards, + "number_of_replicas": self.case_config.number_of_replicas, + "refresh_interval": self.case_config.refresh_interval, + "merge.scheduler.max_thread_count": self.case_config.merge_max_thread_count, + } + } try: - client.indices.create(index=self.indice, mappings=mappings) + client.indices.create(index=self.indice, mappings=mappings, settings=settings) except Exception as e: log.warning(f"Failed to create indice: {self.indice} error: {e!s}") raise e from None @@ -80,21 +100,48 @@ def insert_embeddings( self, embeddings: Iterable[list[float]], metadata: list[int], + labels_data: list[str] | None = None, **kwargs, ) -> tuple[int, Exception]: """Insert the embeddings to the elasticsearch.""" assert self.client is not None, "should self.init() first" - insert_data = [ - { - "_index": self.indice, - "_source": { - self.id_col_name: metadata[i], - self.vector_col_name: embeddings[i], - }, - } - for i in range(len(embeddings)) - ] + insert_data = ( + [ + ( + { + "_index": self.indice, + "_source": { + self.id_col_name: metadata[i], + self.label_col_name: labels_data[i], + self.vector_col_name: embeddings[i], + }, + "_routing": labels_data[i], + } + if self.case_config.use_routing + else { + "_index": self.indice, + "_source": { + self.id_col_name: metadata[i], + self.label_col_name: labels_data[i], + self.vector_col_name: embeddings[i], + }, + } + ) + for i in range(len(embeddings)) + ] + if self.with_scalar_labels + else [ + { + "_index": self.indice, + "_source": { + self.id_col_name: metadata[i], + self.vector_col_name: embeddings[i], + }, + } + for i in range(len(embeddings)) + ] + ) try: bulk_insert_res = bulk(self.client, insert_data) return (bulk_insert_res[0], None) @@ -102,18 +149,31 @@ def insert_embeddings( log.warning(f"Failed to insert data: {self.indice} error: {e!s}") return (0, e) + def prepare_filter(self, filters: Filter): + self.routing_key = None + if filters.type == FilterOp.NonFilter: + self.filter = [] + elif filters.type == FilterOp.NumGE: + self.filter = {"range": {self.id_col_name: {"gt": filters.int_value}}} + elif filters.type == FilterOp.StrEqual: + self.filter = {"term": {self.label_col_name: filters.label_value}} + if self.case_config.use_routing: + self.routing_key = filters.label_value + else: + msg = f"Not support Filter for Milvus - {filters}" + raise ValueError(msg) + def search_embedding( self, query: list[float], k: int = 100, - filters: dict | None = None, + **kwargs, ) -> list[int]: """Get k most similar embeddings to query vector. Args: query(list[float]): query embedding to look up documents similar to. k(int): Number of most similar embeddings to return. Defaults to 100. - filters(dict, optional): filtering expression to filter the data while searching. Returns: list[tuple[int, float]]: list of k most similar embeddings in (id, score) tuple to the query embedding. @@ -124,37 +184,38 @@ def search_embedding( "field": self.vector_col_name, "k": k, "num_candidates": self.case_config.num_candidates, - "filter": [{"range": {self.id_col_name: {"gt": filters["id"]}}}] if filters else [], + "filter": self.filter, "query_vector": query, } + if self.case_config.use_rescore: + knn["rescore_vector"] = {"oversample": self.case_config.oversample_ratio} size = k - try: - res = self.client.search( - index=self.indice, - knn=knn, - size=size, - _source=False, - docvalue_fields=[self.id_col_name], - stored_fields="_none_", - filter_path=[f"hits.hits.fields.{self.id_col_name}"], - ) - return [h["fields"][self.id_col_name][0] for h in res["hits"]["hits"]] - except Exception as e: - log.warning(f"Failed to search: {self.indice} error: {e!s}") - raise e from None + + res = self.client.search( + index=self.indice, + knn=knn, + routing=self.routing_key, + size=size, + _source=False, + docvalue_fields=[self.id_col_name], + stored_fields="_none_", + filter_path=[f"hits.hits.fields.{self.id_col_name}"], + ) + return [h["fields"][self.id_col_name][0] for h in res["hits"]["hits"]] def optimize(self, data_size: int | None = None): """optimize will be called between insertion and search in performance cases.""" assert self.client is not None, "should self.init() first" self.client.indices.refresh(index=self.indice) - force_merge_task_id = self.client.indices.forcemerge( - index=self.indice, - max_num_segments=1, - wait_for_completion=False, - )["task"] - log.info(f"Elasticsearch force merge task id: {force_merge_task_id}") - while True: - time.sleep(SECONDS_WAITING_FOR_FORCE_MERGE_API_CALL_SEC) - task_status = self.client.tasks.get(task_id=force_merge_task_id) - if task_status["completed"]: - return + if self.case_config.use_force_merge: + force_merge_task_id = self.client.indices.forcemerge( + index=self.indice, + max_num_segments=1, + wait_for_completion=False, + )["task"] + log.info(f"Elasticsearch force merge task id: {force_merge_task_id}") + while True: + time.sleep(SECONDS_WAITING_FOR_FORCE_MERGE_API_CALL_SEC) + task_status = self.client.tasks.get(task_id=force_merge_task_id) + if task_status["completed"]: + return diff --git a/vectordb_bench/backend/clients/hologres/cli.py b/vectordb_bench/backend/clients/hologres/cli.py new file mode 100644 index 000000000..7b90b006f --- /dev/null +++ b/vectordb_bench/backend/clients/hologres/cli.py @@ -0,0 +1,50 @@ +from typing import Annotated, Unpack + +import click +from pydantic import SecretStr + +from vectordb_bench.backend.clients import DB +from vectordb_bench.cli.cli import ( + CommonTypedDict, + HNSWFlavor5, + cli, + click_parameter_decorators_from_typed_dict, + run, +) + + +class HologresTypedDict(CommonTypedDict): + host: Annotated[str, click.option("--host", type=str, help="Hologres host", required=True)] + user: Annotated[str, click.option("--user", type=str, help="Hologres username", required=True)] + password: Annotated[str, click.option("--password", type=str, help="Hologres password", required=True)] + database: Annotated[str, click.option("--database", type=str, help="Hologres database name", required=True)] + port: Annotated[int, click.option("--port", type=int, help="Hologres port", required=True)] + + +class HologresHGraphTypedDict(CommonTypedDict, HologresTypedDict, HNSWFlavor5): ... + + +@cli.command() +@click_parameter_decorators_from_typed_dict(HologresHGraphTypedDict) +def HologresHGraph(**parameters: Unpack[HologresHGraphTypedDict]): + from .config import HologresConfig, HologresIndexConfig + + run( + db=DB.Hologres, + db_config=HologresConfig( + db_label=parameters["db_label"], + user_name=SecretStr(parameters["user"]), + password=SecretStr(parameters["password"]), + host=parameters["host"], + port=parameters["port"], + db_name=parameters["database"], + ), + db_case_config=HologresIndexConfig( + index=parameters["index_type"], + max_degree=parameters["m"], + ef_construction=parameters["ef_construction"], + ef_search=parameters["ef_search"], + use_reorder=parameters["use_reorder"], + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/hologres/config.py b/vectordb_bench/backend/clients/hologres/config.py new file mode 100644 index 000000000..ccd476556 --- /dev/null +++ b/vectordb_bench/backend/clients/hologres/config.py @@ -0,0 +1,120 @@ +from pydantic import BaseModel, SecretStr + +from ..api import DBCaseConfig, DBConfig, IndexType, MetricType + + +class HologresConfig(DBConfig): + user_name: SecretStr = SecretStr("hologres") + password: SecretStr + host: str = "localhost" + port: int = 5432 + db_name: str + + def to_dict(self) -> dict: + user_str = self.user_name.get_secret_value() + pwd_str = self.password.get_secret_value() + return { + "host": self.host, + "port": self.port, + "dbname": self.db_name, + "user": user_str, + "password": pwd_str, + } + + +class HologresIndexConfig(BaseModel, DBCaseConfig): + index: IndexType = IndexType.Hologres_HGraph + metric_type: MetricType | None = None + + create_index_before_load: bool = False + create_index_after_load: bool = True + + min_flush_proxima_row_count: int = 1000 + min_compaction_proxima_row_count: int = 1000 + max_total_size_to_merge_mb: int = 4096 + full_compact_max_file_size_mb: int = 4096 + + base_quantization_type: str = "sq8_uniform" + precise_quantization_type: str = "fp32" + use_reorder: bool = True + build_thread_count: int = 16 + max_degree: int = 64 + ef_construction: int = 400 + + ef_search: int = 51 + + def index_param(self) -> dict: + return { + "algorithm": self.algorithm(), + "distance_method": self.distance_method(), + "builder_params": self.builder_params(), + "full_compact_max_file_size_mb": self.full_compact_max_file_size_mb, + } + + def search_param(self) -> dict: + return { + "distance_function": self.distance_function(), + "order_direction": self.order_direction(), + "searcher_params": self.search_params(), + } + + def algorithm(self) -> str: + return self.index.value + + def is_proxima(self) -> bool: + return self.index == IndexType.Hologres_Graph + + def distance_method(self) -> str: + if self.metric_type == MetricType.L2: + if self.index == IndexType.Hologres_Graph: + return "SquaredEuclidean" + return "Euclidean" + if self.metric_type == MetricType.IP: + return "InnerProduct" + if self.metric_type == MetricType.COSINE: + if self.index == IndexType.Hologres_Graph: + return "InnerProduct" + return "Cosine" + return "Euclidean" + + def distance_function(self) -> str: + if self.metric_type == MetricType.L2: + if self.index == IndexType.Hologres_Graph: + return "approx_squared_euclidean_distance" + return "approx_euclidean_distance" + if self.metric_type == MetricType.IP: + return "approx_inner_product_distance" + if self.metric_type == MetricType.COSINE: + if self.index == IndexType.Hologres_Graph: + return "approx_inner_product_distance" + return "approx_cosine_distance" + return "approx_euclidean_distance" + + def order_direction(self) -> str: + if self.metric_type == MetricType.L2: + return "ASC" + if self.metric_type in {MetricType.IP, MetricType.COSINE}: + return "DESC" + return "ASC" + + def builder_params(self) -> dict: + if self.use_reorder: + self.base_quantization_type = "sq8_uniform" + else: + self.base_quantization_type = "fp32" + + return { + "max_total_size_to_merge_mb": self.max_total_size_to_merge_mb, + "build_thread_count": self.build_thread_count, + "base_quantization_type": self.base_quantization_type, + "max_degree": self.max_degree, + "ef_construction": self.ef_construction, + "precise_quantization_type": self.precise_quantization_type, + "use_reorder": self.use_reorder, + "precise_io_type": "reader_io", + } + + def searcher_params(self) -> dict: + return { + "ef_search": self.ef_search, + } diff --git a/vectordb_bench/backend/clients/hologres/hologres.py b/vectordb_bench/backend/clients/hologres/hologres.py new file mode 100644 index 000000000..9fecc5c01 --- /dev/null +++ b/vectordb_bench/backend/clients/hologres/hologres.py @@ -0,0 +1,373 @@ +"""Wrapper around the Hologres vector database over VectorDB""" + +import json +import logging +from collections.abc import Generator +from contextlib import contextmanager +from io import StringIO +from typing import Any + +import psycopg +from psycopg import Connection, Cursor, sql + +from ..api import VectorDB +from .config import HologresConfig, HologresIndexConfig + +log = logging.getLogger(__name__) + + +class Hologres(VectorDB): + """Use psycopg instructions""" + + conn: psycopg.Connection[Any] | None = None + cursor: psycopg.Cursor[Any] | None = None + + _tg_name: str = "vdb_bench_tg_1" + + def __init__( + self, + dim: int, + db_config: HologresConfig, + db_case_config: HologresIndexConfig, + collection_name: str = "vector_collection", + drop_old: bool = False, + **kwargs, + ): + self.name = "Alibaba Cloud Hologres" + self.db_config = db_config + self.case_config = db_case_config + self.table_name = collection_name + self.dim = dim + + self._primary_field = "id" + self._vector_field = "embedding" + + # construct basic units + self.conn, self.cursor = self._create_connection(**self.db_config) + + # create vector extension + if self.case_config.is_proxima(): + self.cursor.execute("CREATE EXTENSION proxima;") + self.conn.commit() + + log.info(f"{self.name} config values: {self.db_config}\n{self.case_config}") + if not any( + ( + self.case_config.create_index_before_load, + self.case_config.create_index_after_load, + ), + ): + msg = ( + f"{self.name} config must create an index using create_index_before_load or create_index_after_load" + f"{self.name} config values: {self.db_config}\n{self.case_config}" + ) + log.error(msg) + raise RuntimeError(msg) + + if drop_old: + self._drop_table() + self._create_table(dim) + if self.case_config.create_index_before_load: + self._create_index() + + self.cursor.close() + self.conn.close() + self.cursor = None + self.conn = None + + @staticmethod + def _create_connection(**kwargs) -> tuple[Connection, Cursor]: + conn = psycopg.connect(**kwargs) + conn.autocommit = True + cursor = conn.cursor() + + assert conn is not None, "Connection is not initialized" + assert cursor is not None, "Cursor is not initialized" + + return conn, cursor + + @contextmanager + def init(self) -> Generator[None, None, None]: + """ + Examples: + >>> with self.init(): + >>> self.insert_embeddings() + >>> self.search_embedding() + """ + + self.conn, self.cursor = self._create_connection(**self.db_config) + + self._set_search_guc() + + try: + yield + finally: + self.cursor.close() + self.conn.close() + self.cursor = None + self.conn = None + + def _set_search_guc(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + sql_guc = sql.SQL(f"SET hg_vector_ef_search = {self.case_config.ef_search};") + log.info(f"{self.name} client set search guc: {sql_guc.as_string()}") + self.cursor.execute(sql_guc) + self.conn.commit() + + def _drop_table(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + log.info(f"{self.name} client drop table : {self.table_name}") + self.cursor.execute( + sql.SQL("DROP TABLE IF EXISTS {table_name};").format( + table_name=sql.Identifier(self.table_name), + ), + ) + self.conn.commit() + + try: + log.info(f"{self.name} client purge table recycle bin: {self.table_name}") + self.cursor.execute( + sql.SQL("purge TABLE {table_name};").format( + table_name=sql.Identifier(self.table_name), + ), + ) + except Exception as e: + log.info(f"{self.name} client purge table {self.table_name} recycle bin failed, error: {e}, ignore.") + finally: + self.conn.commit() + + try: + log.info(f"{self.name} client drop table group : {self._tg_name}") + self.cursor.execute(sql.SQL(f"CALL HG_DROP_TABLE_GROUP('{self._tg_name}');")) + except Exception as e: + log.info(f"{self.name} client drop table group : {self._tg_name} failed, error: {e}, ignore.") + finally: + self.conn.commit() + + try: + log.info(f"{self.name} client free cache") + self.cursor.execute("select hg_admin_command('freecache');") + except Exception as e: + log.info(f"{self.name} client free cache failed, error: {e}, ignore.") + finally: + self.conn.commit() + + def optimize(self, data_size: int | None = None): + if self.case_config.create_index_after_load: + self._create_index() + self._full_compact() + self._analyze() + + def _vacuum(self): + log.info(f"{self.name} client vacuum table : {self.table_name}") + try: + # VACUUM cannot run inside a transaction block + # it's better to new a connection + self.conn.autocommit = True + with self.conn.cursor() as cursor: + cursor.execute( + sql.SQL(""" + VACUUM {table_name}; + """).format( + table_name=sql.Identifier(self.table_name), + ) + ) + log.info(f"{self.name} client vacuum table : {self.table_name} done") + except Exception as e: + log.warning(f"Failed to vacuum table: {self.table_name} error: {e}") + raise e from None + finally: + self.conn.autocommit = True + + def _analyze(self): + log.info(f"{self.name} client analyze table : {self.table_name}") + self.cursor.execute(sql.SQL(f"ANALYZE {self.table_name};")) + log.info(f"{self.name} client analyze table : {self.table_name} done") + + def _full_compact(self): + log.info(f"{self.name} client full compact table : {self.table_name}") + self.cursor.execute( + sql.SQL(""" + SELECT hologres.hg_full_compact_table( + '{table_name}', + 'max_file_size_mb={full_compact_max_file_size_mb}' + ); + """).format( + table_name=sql.SQL(self.table_name), + full_compact_max_file_size_mb=sql.SQL(str(self.case_config.full_compact_max_file_size_mb)), + ) + ) + log.info(f"{self.name} client full compact table : {self.table_name} done") + + def _create_index(self): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + sql_index = sql.SQL(""" + CALL set_table_property ('{table_name}', 'vectors', '{{ + "embedding": {{ + "algorithm": "{algorithm}", + "distance_method": "{distance_method}", + "builder_params": {builder_params} + }} + }}'); + """).format( + table_name=sql.Identifier(self.table_name), + algorithm=sql.SQL(self.case_config.algorithm()), + distance_method=sql.SQL(self.case_config.distance_method()), + builder_params=sql.SQL(json.dumps(self.case_config.builder_params())), + ) + + log.info(f"{self.name} client create index on table : {self.table_name}, with sql: {sql_index.as_string()}") + try: + self.cursor.execute(sql_index) + self.conn.commit() + except Exception as e: + log.warning(f"Failed to create index on table: {self.table_name} error: {e}") + raise e from None + + def _set_replica_count(self, replica_count: int = 2): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + try: + # non-warehouse mode by default + sql_tg_replica = sql.SQL( + f"CALL hg_set_table_group_property('{self._tg_name}', 'replica_count', '{replica_count}');" + ) + + # check warehouse mode + sql_check = sql.SQL("select count(*) from hologres.hg_warehouses;") + log.info(f"check warehouse mode with sql: {sql_check}") + self.cursor.execute(sql_check) + result_check = self.cursor.fetchone()[0] + if result_check > 0: + # get warehouse name + sql_get_warehouse_name = sql.SQL("select current_warehouse();") + log.info(f"get warehouse name with sql: {sql_get_warehouse_name}") + self.cursor.execute(sql_get_warehouse_name) + sql_tg_replica = sql.SQL(""" + CALL hg_table_group_set_warehouse_replica_count ( + '{dbname}.{tg_name}', + {replica_count}, + '{warehouse_name}' + ); + """).format( + tg_name=sql.SQL(self._tg_name), + warehouse_name=sql.SQL(self.cursor.fetchone()[0]), + dbname=sql.SQL(self.db_config["dbname"]), + replica_count=replica_count, + ) + log.info(f"{self.name} client set table group replica: {self._tg_name}, with sql: {sql_tg_replica}") + self.cursor.execute(sql_tg_replica) + except Exception as e: + log.warning(f"Failed to set replica count, error: {e}, ignore") + finally: + self.conn.commit() + + def _create_table(self, dim: int): + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + sql_tg = sql.SQL(f"CALL HG_CREATE_TABLE_GROUP ('{self._tg_name}', 1);") + log.info(f"{self.name} client create table group : {self._tg_name}, with sql: {sql_tg}") + try: + self.cursor.execute(sql_tg) + except Exception as e: + log.warning(f"Failed to create table group : {self._tg_name} error: {e}, ignore") + finally: + self.conn.commit() + + self._set_replica_count(replica_count=2) + + sql_table = sql.SQL(""" + CREATE TABLE IF NOT EXISTS {table_name} ( + id BIGINT PRIMARY KEY, + embedding FLOAT4[] CHECK (array_ndims(embedding) = 1 AND array_length(embedding, 1) = {dim}) + ) + WITH (table_group = {tg_name}); + """).format( + table_name=sql.Identifier(self.table_name), + dim=dim, + tg_name=sql.SQL(self._tg_name), + ) + log.info(f"{self.name} client create table : {self.table_name}, with sql: {sql_table.as_string()}") + try: + self.cursor.execute(sql_table) + self.conn.commit() + except Exception as e: + log.warning(f"Failed to create table : {self.table_name} error: {e}") + raise e from None + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + **kwargs: Any, + ) -> tuple[int, Exception | None]: + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + try: + buffer = StringIO() + for i in range(len(metadata)): + buffer.write("%d\t%s\n" % (metadata[i], "{" + ",".join("%f" % x for x in embeddings[i]) + "}")) + buffer.seek(0) + + with self.cursor.copy( + sql.SQL("COPY {table_name} FROM STDIN").format(table_name=sql.Identifier(self.table_name)) + ) as copy: + copy.write(buffer.getvalue()) + self.conn.commit() + + return len(metadata), None + except Exception as e: + log.warning(f"Failed to insert data into table ({self.table_name}), error: {e}") + return 0, e + + def _compose_query_and_params(self, vec: list[float], topk: int, ge_id: int | None = None): + params = [] + + where_clause = sql.SQL("") + if ge_id is not None: + where_clause = sql.SQL(" WHERE id >= %s ") + params.append(ge_id) + + vec_float4 = [psycopg._wrappers.Float4(i) for i in vec] + params.append(vec_float4) + params.append(topk) + + query = sql.SQL(""" + SELECT id + FROM {table_name} + {where_clause} + ORDER BY {distance_function}(embedding, %b) + {order_direction} + LIMIT %s; + """).format( + table_name=sql.Identifier(self.table_name), + distance_function=sql.SQL(self.case_config.distance_function()), + where_clause=where_clause, + order_direction=sql.SQL(self.case_config.order_direction()), + ) + + return query, params + + def search_embedding( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + timeout: int | None = None, + ) -> list[int]: + assert self.conn is not None, "Connection is not initialized" + assert self.cursor is not None, "Cursor is not initialized" + + ge = filters.get("id") if filters else None + q, params = self._compose_query_and_params(query, k, ge) + result = self.cursor.execute(q, params, prepare=True, binary=True) + return [int(i[0]) for i in result.fetchall()] diff --git a/vectordb_bench/backend/clients/lancedb/cli.py b/vectordb_bench/backend/clients/lancedb/cli.py new file mode 100644 index 000000000..219ae114e --- /dev/null +++ b/vectordb_bench/backend/clients/lancedb/cli.py @@ -0,0 +1,146 @@ +from typing import Annotated, Unpack + +import click +from pydantic import SecretStr + +from ....cli.cli import ( + CommonTypedDict, + cli, + click_parameter_decorators_from_typed_dict, + run, +) +from .. import DB +from ..api import IndexType + + +class LanceDBTypedDict(CommonTypedDict): + uri: Annotated[ + str, + click.option("--uri", type=str, help="URI connection string", required=True), + ] + token: Annotated[ + str | None, + click.option("--token", type=str, help="Authentication token", required=False), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(LanceDBTypedDict) +def LanceDB(**parameters: Unpack[LanceDBTypedDict]): + from .config import LanceDBConfig, _lancedb_case_config + + run( + db=DB.LanceDB, + db_config=LanceDBConfig( + db_label=parameters["db_label"], + uri=parameters["uri"], + token=SecretStr(parameters["token"]) if parameters.get("token") else None, + ), + db_case_config=_lancedb_case_config.get("NONE")(), + **parameters, + ) + + +@cli.command() +@click_parameter_decorators_from_typed_dict(LanceDBTypedDict) +def LanceDBAutoIndex(**parameters: Unpack[LanceDBTypedDict]): + from .config import LanceDBConfig, _lancedb_case_config + + run( + db=DB.LanceDB, + db_config=LanceDBConfig( + db_label=parameters["db_label"], + uri=parameters["uri"], + token=SecretStr(parameters["token"]) if parameters.get("token") else None, + ), + db_case_config=_lancedb_case_config.get(IndexType.AUTOINDEX)(), + **parameters, + ) + + +class LanceDBIVFPQTypedDict(CommonTypedDict, LanceDBTypedDict): + num_partitions: Annotated[ + int, + click.option( + "--num-partitions", + type=int, + default=0, + help="Number of partitions for IVFPQ index, unset = use LanceDB default", + ), + ] + num_sub_vectors: Annotated[ + int, + click.option( + "--num-sub-vectors", + type=int, + default=0, + help="Number of sub-vectors for IVFPQ index, unset = use LanceDB default", + ), + ] + nbits: Annotated[ + int, + click.option( + "--nbits", + type=int, + default=8, + help="Number of bits for IVFPQ index (must be 4 or 8), unset = use LanceDB default", + ), + ] + nprobes: Annotated[ + int, + click.option( + "--nprobes", type=int, default=0, help="Number of probes for IVFPQ search, unset = use LanceDB default" + ), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(LanceDBIVFPQTypedDict) +def LanceDBIVFPQ(**parameters: Unpack[LanceDBIVFPQTypedDict]): + from .config import LanceDBConfig, LanceDBIndexConfig + + run( + db=DB.LanceDB, + db_config=LanceDBConfig( + db_label=parameters["db_label"], + uri=parameters["uri"], + token=SecretStr(parameters["token"]) if parameters.get("token") else None, + ), + db_case_config=LanceDBIndexConfig( + index=IndexType.IVFPQ, + num_partitions=parameters["num_partitions"], + num_sub_vectors=parameters["num_sub_vectors"], + nbits=parameters["nbits"], + nprobes=parameters["nprobes"], + ), + **parameters, + ) + + +class LanceDBHNSWTypedDict(CommonTypedDict, LanceDBTypedDict): + m: Annotated[int, click.option("--m", type=int, default=0, help="HNSW parameter m")] + ef_construction: Annotated[ + int, click.option("--ef-construction", type=int, default=0, help="HNSW parameter ef_construction") + ] + ef: Annotated[int, click.option("--ef", type=int, default=0, help="HNSW search parameter ef")] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(LanceDBHNSWTypedDict) +def LanceDBHNSW(**parameters: Unpack[LanceDBHNSWTypedDict]): + from .config import LanceDBConfig, LanceDBHNSWIndexConfig + + run( + db=DB.LanceDB, + db_config=LanceDBConfig( + db_label=parameters["db_label"], + uri=parameters["uri"], + token=SecretStr(parameters["token"]) if parameters.get("token") else None, + ), + db_case_config=LanceDBHNSWIndexConfig( + m=parameters["m"], + ef_construction=parameters["ef_construction"], + ef=parameters["ef"], + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/lancedb/config.py b/vectordb_bench/backend/clients/lancedb/config.py new file mode 100644 index 000000000..b0621c22e --- /dev/null +++ b/vectordb_bench/backend/clients/lancedb/config.py @@ -0,0 +1,116 @@ +from pydantic import BaseModel, SecretStr + +from ..api import DBCaseConfig, DBConfig, IndexType, MetricType + + +class LanceDBConfig(DBConfig): + """LanceDB connection configuration.""" + + db_label: str + uri: str + token: SecretStr | None = None + + def to_dict(self) -> dict: + return { + "uri": self.uri, + "token": self.token.get_secret_value() if self.token else None, + } + + +class LanceDBIndexConfig(BaseModel, DBCaseConfig): + index: IndexType = IndexType.IVFPQ + metric_type: MetricType = MetricType.L2 + num_partitions: int = 0 + num_sub_vectors: int = 0 + nbits: int = 8 # Must be 4 or 8 + sample_rate: int = 256 + max_iterations: int = 50 + nprobes: int = 0 + + def index_param(self) -> dict: + if self.index not in [ + IndexType.IVFPQ, + IndexType.HNSW, + IndexType.AUTOINDEX, + IndexType.NONE, + ]: + msg = f"Index type {self.index} is not supported for LanceDB!" + raise ValueError(msg) + + # See https://lancedb.github.io/lancedb/python/python/#lancedb.table.Table.create_index + params = { + "metric": self.parse_metric(), + "num_bits": self.nbits, + "sample_rate": self.sample_rate, + "max_iterations": self.max_iterations, + } + + if self.num_partitions > 0: + params["num_partitions"] = self.num_partitions + if self.num_sub_vectors > 0: + params["num_sub_vectors"] = self.num_sub_vectors + + return params + + def search_param(self) -> dict: + params = {} + if self.nprobes > 0: + params["nprobes"] = self.nprobes + + return params + + def parse_metric(self) -> str: + if self.metric_type in [MetricType.L2, MetricType.COSINE]: + return self.metric_type.value.lower() + if self.metric_type in [MetricType.IP, MetricType.DP]: + return "dot" + msg = f"Metric type {self.metric_type} is not supported for LanceDB!" + raise ValueError(msg) + + +class LanceDBNoIndexConfig(LanceDBIndexConfig): + index: IndexType = IndexType.NONE + + def index_param(self) -> dict: + return {} + + +class LanceDBAutoIndexConfig(LanceDBIndexConfig): + index: IndexType = IndexType.AUTOINDEX + + def index_param(self) -> dict: + return {} + + +class LanceDBHNSWIndexConfig(LanceDBIndexConfig): + index: IndexType = IndexType.HNSW + m: int = 0 + ef_construction: int = 0 + ef: int = 0 + + def index_param(self) -> dict: + params = LanceDBIndexConfig.index_param(self) + + # See https://lancedb.github.io/lancedb/python/python/#lancedb.index.HnswSq + params["index_type"] = "IVF_HNSW_SQ" + if self.m > 0: + params["m"] = self.m + if self.ef_construction > 0: + params["ef_construction"] = self.ef_construction + + return params + + def search_param(self) -> dict: + params = {} + if self.ef != 0: + params = {"ef": self.ef} + + return params + + +_lancedb_case_config = { + IndexType.IVFPQ: LanceDBIndexConfig, + IndexType.AUTOINDEX: LanceDBAutoIndexConfig, + IndexType.HNSW: LanceDBHNSWIndexConfig, + IndexType.NONE: LanceDBNoIndexConfig, +} diff --git a/vectordb_bench/backend/clients/lancedb/lancedb.py b/vectordb_bench/backend/clients/lancedb/lancedb.py new file mode 100644 index 000000000..65330e2cb --- /dev/null +++ b/vectordb_bench/backend/clients/lancedb/lancedb.py @@ -0,0 +1,110 @@ +import logging +from contextlib import contextmanager + +import lancedb +import pyarrow as pa +from lancedb.pydantic import LanceModel + +from ..api import IndexType, VectorDB +from .config import LanceDBConfig, LanceDBIndexConfig + +log = logging.getLogger(__name__) + + +class VectorModel(LanceModel): + id: int + vector: list[float] + + +class LanceDB(VectorDB): + def __init__( + self, + dim: int, + db_config: LanceDBConfig, + db_case_config: LanceDBIndexConfig, + collection_name: str = "vector_bench_test", + drop_old: bool = False, + **kwargs, + ): + self.name = "LanceDB" + self.db_config = db_config + self.case_config = db_case_config + self.table_name = collection_name + self.dim = dim + self.uri = db_config["uri"] + # avoid the search_param being called every time during the search process + self.search_config = db_case_config.search_param() + + log.info(f"Search config: {self.search_config}") + + db = lancedb.connect(self.uri) + + if drop_old: + try: + db.drop_table(self.table_name) + except Exception as e: + log.warning(f"Failed to drop table {self.table_name}: {e}") + + try: + db.open_table(self.table_name) + except Exception: + schema = pa.schema( + [pa.field("id", pa.int64()), pa.field("vector", pa.list_(pa.float32(), list_size=self.dim))] + ) + db.create_table(self.table_name, schema=schema, mode="overwrite") + + @contextmanager + def init(self): + self.db = lancedb.connect(self.uri) + self.table = self.db.open_table(self.table_name) + yield + self.db = None + self.table = None + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + **kwargs, + ) -> tuple[int, Exception | None]: + try: + data = [{"id": meta, "vector": emb} for meta, emb in zip(metadata, embeddings, strict=False)] + self.table.add(data) + return len(metadata), None + except Exception as e: + log.warning(f"Failed to insert data into LanceDB table ({self.table_name}), error: {e}") + return 0, e + + def search_embedding( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + ) -> list[int]: + if filters: + results = self.table.search(query).select(["id"]).where(f"id >= {filters['id']}", prefilter=True).limit(k) + if self.case_config.index == IndexType.IVFPQ and "nprobes" in self.search_config: + results = results.nprobes(self.search_config["nprobes"]).to_list() + elif self.case_config.index == IndexType.HNSW and "ef" in self.search_config: + results = results.ef(self.search_config["ef"]).to_list() + else: + results = results.to_list() + else: + results = self.table.search(query).select(["id"]).limit(k) + if self.case_config.index == IndexType.IVFPQ and "nprobes" in self.search_config: + results = results.nprobes(self.search_config["nprobes"]).to_list() + elif self.case_config.index == IndexType.HNSW and "ef" in self.search_config: + results = results.ef(self.search_config["ef"]).to_list() + else: + results = results.to_list() + + return [int(result["id"]) for result in results] + + def optimize(self, data_size: int | None = None): + if self.table and hasattr(self, "case_config") and self.case_config.index != IndexType.NONE: + log.info(f"Creating index for LanceDB table ({self.table_name})") + log.info(f"Index parameters: {self.case_config.index_param()}") + self.table.create_index(**self.case_config.index_param()) + # Better recall with IVF_PQ (though still bad) but breaks HNSW: https://github.com/lancedb/lancedb/issues/2369 + if self.case_config.index in (IndexType.IVFPQ, IndexType.AUTOINDEX): + self.table.optimize() diff --git a/vectordb_bench/backend/clients/mariadb/mariadb.py b/vectordb_bench/backend/clients/mariadb/mariadb.py index 5ccddfe7a..db3863c85 100644 --- a/vectordb_bench/backend/clients/mariadb/mariadb.py +++ b/vectordb_bench/backend/clients/mariadb/mariadb.py @@ -73,14 +73,12 @@ def _create_db_table(self, dim: int): log.info(f"{self.name} client create table : {self.table_name}") self.cursor.execute(f"USE {self.db_name}") - self.cursor.execute( - f""" + self.cursor.execute(f""" CREATE TABLE {self.table_name} ( id INT PRIMARY KEY, v VECTOR({self.dim}) NOT NULL ) ENGINE={index_param["storage_engine"]} - """ - ) + """) self.cursor.execute("COMMIT") except Exception as e: @@ -142,12 +140,10 @@ def optimize(self) -> None: if index_param["index_type"] == "HNSW" and index_param["M"] is not None: index_options += f" M={index_param['M']}" - self.cursor.execute( - f""" + self.cursor.execute(f""" ALTER TABLE {self.db_name}.{self.table_name} ADD VECTOR KEY v(v) {index_options} - """ - ) + """) self.cursor.execute("COMMIT") except Exception as e: diff --git a/vectordb_bench/backend/clients/memorydb/memorydb.py b/vectordb_bench/backend/clients/memorydb/memorydb.py index 9d077f5df..7e7a8650b 100644 --- a/vectordb_bench/backend/clients/memorydb/memorydb.py +++ b/vectordb_bench/backend/clients/memorydb/memorydb.py @@ -9,10 +9,10 @@ from redis import Redis from redis.cluster import RedisCluster from redis.commands.search.field import NumericField, TagField, VectorField -from redis.commands.search.indexDefinition import IndexDefinition +from redis.commands.search.indexDefinition import IndexDefinition, IndexType from redis.commands.search.query import Query -from ..api import IndexType, VectorDB +from ..api import VectorDB from .config import MemoryDBIndexConfig log = logging.getLogger(__name__) diff --git a/vectordb_bench/backend/clients/milvus/cli.py b/vectordb_bench/backend/clients/milvus/cli.py index 24a61566f..af31fe50d 100644 --- a/vectordb_bench/backend/clients/milvus/cli.py +++ b/vectordb_bench/backend/clients/milvus/cli.py @@ -29,6 +29,28 @@ class MilvusTypedDict(TypedDict): str | None, click.option("--password", type=str, help="Db password", required=False), ] + num_shards: Annotated[ + int, + click.option( + "--num-shards", + type=int, + help="Number of shards", + required=False, + default=1, + show_default=True, + ), + ] + replica_number: Annotated[ + int, + click.option( + "--replica-number", + type=int, + help="Number of replicas", + required=False, + default=1, + show_default=True, + ), + ] class MilvusAutoIndexTypedDict(CommonTypedDict, MilvusTypedDict): ... @@ -45,7 +67,9 @@ def MilvusAutoIndex(**parameters: Unpack[MilvusAutoIndexTypedDict]): db_label=parameters["db_label"], uri=SecretStr(parameters["uri"]), user=parameters["user_name"], - password=SecretStr(parameters["password"]), + password=SecretStr(parameters["password"]) if parameters["password"] else None, + num_shards=int(parameters["num_shards"]), + replica_number=int(parameters["replica_number"]), ), db_case_config=AutoIndexConfig(), **parameters, @@ -63,7 +87,9 @@ def MilvusFlat(**parameters: Unpack[MilvusAutoIndexTypedDict]): db_label=parameters["db_label"], uri=SecretStr(parameters["uri"]), user=parameters["user_name"], - password=SecretStr(parameters["password"]), + password=SecretStr(parameters["password"]) if parameters["password"] else None, + num_shards=int(parameters["num_shards"]), + replica_number=int(parameters["replica_number"]), ), db_case_config=FLATConfig(), **parameters, @@ -85,6 +111,8 @@ def MilvusHNSW(**parameters: Unpack[MilvusHNSWTypedDict]): uri=SecretStr(parameters["uri"]), user=parameters["user_name"], password=SecretStr(parameters["password"]) if parameters["password"] else None, + num_shards=int(parameters["num_shards"]), + replica_number=int(parameters["replica_number"]), ), db_case_config=HNSWConfig( M=parameters["m"], @@ -95,6 +123,160 @@ def MilvusHNSW(**parameters: Unpack[MilvusHNSWTypedDict]): ) +class MilvusRefineTypedDict(TypedDict): + refine: Annotated[ + bool, + click.option( + "--refine", + type=bool, + required=True, + help="Whether refined data is reserved during index building.", + ), + ] + refine_type: Annotated[ + str | None, + click.option( + "--refine-type", + type=click.Choice(["SQ6", "SQ8", "BF16", "FP16", "FP32"], case_sensitive=False), + help="The data type of the refine index to use. Supported values: SQ6,SQ8,BF16,FP16,FP32", + required=True, + ), + ] + refine_k: Annotated[ + float, + click.option( + "--refine-k", + type=float, + help="The magnification factor of refine compared to k.", + required=True, + ), + ] + + +class MilvusHNSWPQTypedDict(CommonTypedDict, MilvusTypedDict, MilvusHNSWTypedDict, MilvusRefineTypedDict): + nbits: Annotated[ + int, + click.option( + "--nbits", + type=int, + required=True, + ), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(MilvusHNSWPQTypedDict) +def MilvusHNSWPQ(**parameters: Unpack[MilvusHNSWPQTypedDict]): + from .config import HNSWPQConfig, MilvusConfig + + run( + db=DBTYPE, + db_config=MilvusConfig( + db_label=parameters["db_label"], + uri=SecretStr(parameters["uri"]), + user=parameters["user_name"], + password=SecretStr(parameters["password"]) if parameters["password"] else None, + num_shards=int(parameters["num_shards"]), + replica_number=int(parameters["replica_number"]), + ), + db_case_config=HNSWPQConfig( + M=parameters["m"], + efConstruction=parameters["ef_construction"], + ef=parameters["ef_search"], + nbits=parameters["nbits"], + refine=parameters["refine"], + refine_type=parameters["refine_type"], + refine_k=parameters["refine_k"], + ), + **parameters, + ) + + +class MilvusHNSWPRQTypedDict( + CommonTypedDict, + MilvusTypedDict, + MilvusHNSWPQTypedDict, +): + nrq: Annotated[ + int, + click.option( + "--nrq", + type=int, + help="The number of residual subquantizers.", + required=True, + ), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(MilvusHNSWPRQTypedDict) +def MilvusHNSWPRQ(**parameters: Unpack[MilvusHNSWPRQTypedDict]): + from .config import HNSWPRQConfig, MilvusConfig + + run( + db=DBTYPE, + db_config=MilvusConfig( + db_label=parameters["db_label"], + uri=SecretStr(parameters["uri"]), + user=parameters["user_name"], + password=SecretStr(parameters["password"]) if parameters["password"] else None, + num_shards=int(parameters["num_shards"]), + replica_number=int(parameters["replica_number"]), + ), + db_case_config=HNSWPRQConfig( + M=parameters["m"], + efConstruction=parameters["ef_construction"], + ef=parameters["ef_search"], + nbits=parameters["nbits"], + refine=parameters["refine"], + refine_type=parameters["refine_type"], + refine_k=parameters["refine_k"], + nrq=parameters["nrq"], + ), + **parameters, + ) + + +class MilvusHNSWSQTypedDict(CommonTypedDict, MilvusTypedDict, MilvusHNSWTypedDict, MilvusRefineTypedDict): + sq_type: Annotated[ + str | None, + click.option( + "--sq-type", + type=click.Choice(["SQ6", "SQ8", "BF16", "FP16", "FP32"], case_sensitive=False), + help="Scalar quantizer type. Supported values: SQ6,SQ8,BF16,FP16,FP32", + required=True, + ), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(MilvusHNSWSQTypedDict) +def MilvusHNSWSQ(**parameters: Unpack[MilvusHNSWSQTypedDict]): + from .config import HNSWSQConfig, MilvusConfig + + run( + db=DBTYPE, + db_config=MilvusConfig( + db_label=parameters["db_label"], + uri=SecretStr(parameters["uri"]), + user=parameters["user_name"], + password=SecretStr(parameters["password"]) if parameters["password"] else None, + num_shards=int(parameters["num_shards"]), + replica_number=int(parameters["replica_number"]), + ), + db_case_config=HNSWSQConfig( + M=parameters["m"], + efConstruction=parameters["ef_construction"], + ef=parameters["ef_search"], + sq_type=parameters["sq_type"], + refine=parameters["refine"], + refine_type=parameters["refine_type"], + refine_k=parameters["refine_k"], + ), + **parameters, + ) + + class MilvusIVFFlatTypedDict(CommonTypedDict, MilvusTypedDict, IVFFlatTypedDictN): ... @@ -109,7 +291,9 @@ def MilvusIVFFlat(**parameters: Unpack[MilvusIVFFlatTypedDict]): db_label=parameters["db_label"], uri=SecretStr(parameters["uri"]), user=parameters["user_name"], - password=SecretStr(parameters["password"]), + password=SecretStr(parameters["password"]) if parameters["password"] else None, + num_shards=int(parameters["num_shards"]), + replica_number=int(parameters["replica_number"]), ), db_case_config=IVFFlatConfig( nlist=parameters["nlist"], @@ -130,7 +314,9 @@ def MilvusIVFSQ8(**parameters: Unpack[MilvusIVFFlatTypedDict]): db_label=parameters["db_label"], uri=SecretStr(parameters["uri"]), user=parameters["user_name"], - password=SecretStr(parameters["password"]), + password=SecretStr(parameters["password"]) if parameters["password"] else None, + num_shards=int(parameters["num_shards"]), + replica_number=int(parameters["replica_number"]), ), db_case_config=IVFSQ8Config( nlist=parameters["nlist"], @@ -140,6 +326,72 @@ def MilvusIVFSQ8(**parameters: Unpack[MilvusIVFFlatTypedDict]): ) +class MilvusIVFRABITQTypedDict(CommonTypedDict, MilvusTypedDict, MilvusIVFFlatTypedDict): + rbq_bits_query: Annotated[ + int, + click.option( + "--rbq-bits-query", + type=int, + help="The level of quantization of a query vector. Use 1…8 for the SQ1…SQ8 and 0 to disable.", + required=True, + ), + ] + refine: Annotated[ + bool, + click.option( + "--refine", + type=bool, + required=True, + help="Whether refined data is reserved during index building.", + ), + ] + refine_type: Annotated[ + str | None, + click.option( + "--refine-type", + type=click.Choice(["SQ6", "SQ8", "BF16", "FP16", "FP32"], case_sensitive=False), + help="The data type of the refine index to use. Supported values: SQ6,SQ8,BF16,FP16,FP32", + required=True, + ), + ] + refine_k: Annotated[ + float, + click.option( + "--refine-k", + type=float, + help="The magnification factor of refine compared to k.", + required=True, + ), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(MilvusIVFRABITQTypedDict) +def MilvusIVFRabitQ(**parameters: Unpack[MilvusIVFRABITQTypedDict]): + from .config import IVFRABITQConfig, MilvusConfig + + run( + db=DBTYPE, + db_config=MilvusConfig( + db_label=parameters["db_label"], + uri=SecretStr(parameters["uri"]), + user=parameters["user_name"], + password=SecretStr(parameters["password"]) if parameters["password"] else None, + num_shards=int(parameters["num_shards"]), + replica_number=int(parameters["replica_number"]), + ), + db_case_config=IVFRABITQConfig( + nlist=parameters["nlist"], + nprobe=parameters["nprobe"], + rbq_bits_query=parameters["rbq_bits_query"], + refine=parameters["refine"], + refine_type=parameters["refine_type"], + refine_k=parameters["refine_k"], + ), + **parameters, + ) + + class MilvusDISKANNTypedDict(CommonTypedDict, MilvusTypedDict): search_list: Annotated[str, click.option("--search-list", type=int, required=True)] @@ -155,7 +407,9 @@ def MilvusDISKANN(**parameters: Unpack[MilvusDISKANNTypedDict]): db_label=parameters["db_label"], uri=SecretStr(parameters["uri"]), user=parameters["user_name"], - password=SecretStr(parameters["password"]), + password=SecretStr(parameters["password"]) if parameters["password"] else None, + num_shards=int(parameters["num_shards"]), + replica_number=int(parameters["replica_number"]), ), db_case_config=DISKANNConfig( search_list=parameters["search_list"], @@ -183,7 +437,9 @@ def MilvusGPUIVFFlat(**parameters: Unpack[MilvusGPUIVFTypedDict]): db_label=parameters["db_label"], uri=SecretStr(parameters["uri"]), user=parameters["user_name"], - password=SecretStr(parameters["password"]), + password=SecretStr(parameters["password"]) if parameters["password"] else None, + num_shards=int(parameters["num_shards"]), + replica_number=int(parameters["replica_number"]), ), db_case_config=GPUIVFFlatConfig( nlist=parameters["nlist"], @@ -217,7 +473,9 @@ def MilvusGPUBruteForce(**parameters: Unpack[MilvusGPUBruteForceTypedDict]): db_label=parameters["db_label"], uri=SecretStr(parameters["uri"]), user=parameters["user_name"], - password=SecretStr(parameters["password"]), + password=SecretStr(parameters["password"]) if parameters["password"] else None, + num_shards=int(parameters["num_shards"]), + replica_number=int(parameters["replica_number"]), ), db_case_config=GPUBruteForceConfig( metric_type=parameters["metric_type"], @@ -248,7 +506,9 @@ def MilvusGPUIVFPQ(**parameters: Unpack[MilvusGPUIVFPQTypedDict]): db_label=parameters["db_label"], uri=SecretStr(parameters["uri"]), user=parameters["user_name"], - password=SecretStr(parameters["password"]), + password=SecretStr(parameters["password"]) if parameters["password"] else None, + num_shards=int(parameters["num_shards"]), + replica_number=int(parameters["replica_number"]), ), db_case_config=GPUIVFPQConfig( nlist=parameters["nlist"], @@ -287,7 +547,9 @@ def MilvusGPUCAGRA(**parameters: Unpack[MilvusGPUCAGRATypedDict]): db_label=parameters["db_label"], uri=SecretStr(parameters["uri"]), user=parameters["user_name"], - password=SecretStr(parameters["password"]), + password=SecretStr(parameters["password"]) if parameters["password"] else None, + num_shards=int(parameters["num_shards"]), + replica_number=int(parameters["replica_number"]), ), db_case_config=GPUCAGRAConfig( intermediate_graph_degree=parameters["intermediate_graph_degree"], diff --git a/vectordb_bench/backend/clients/milvus/config.py b/vectordb_bench/backend/clients/milvus/config.py index e3a3f9b19..9ffbdcece 100644 --- a/vectordb_bench/backend/clients/milvus/config.py +++ b/vectordb_bench/backend/clients/milvus/config.py @@ -1,18 +1,22 @@ from pydantic import BaseModel, SecretStr, validator -from ..api import DBCaseConfig, DBConfig, IndexType, MetricType +from ..api import DBCaseConfig, DBConfig, IndexType, MetricType, SQType class MilvusConfig(DBConfig): uri: SecretStr = "http://localhost:19530" user: str | None = None password: SecretStr | None = None + num_shards: int = 1 + replica_number: int = 1 def to_dict(self) -> dict: return { "uri": self.uri.get_secret_value(), "user": self.user if self.user else None, "password": self.password.get_secret_value() if self.password else None, + "num_shards": self.num_shards, + "replica_number": self.replica_number, } @validator("*") @@ -33,6 +37,7 @@ class MilvusIndexConfig(BaseModel): index: IndexType metric_type: MetricType | None = None + use_partition_key: bool = False # for label-filter @property def is_gpu_index(self) -> bool: @@ -88,6 +93,88 @@ def search_param(self) -> dict: } +class HNSWSQConfig(HNSWConfig, DBCaseConfig): + index: IndexType = IndexType.HNSW_SQ + sq_type: SQType = SQType.SQ8 + refine: bool = True + refine_type: SQType = SQType.FP32 + refine_k: float = 1 + + def index_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": { + "M": self.M, + "efConstruction": self.efConstruction, + "sq_type": self.sq_type.value, + "refine": self.refine, + "refine_type": self.refine_type.value, + }, + } + + def search_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "params": {"ef": self.ef, "refine_k": self.refine_k}, + } + + +class HNSWPQConfig(HNSWConfig): + index: IndexType = IndexType.HNSW_PQ + m: int = 32 + nbits: int = 8 + refine: bool = True + refine_type: SQType = SQType.FP32 + refine_k: float = 1 + + def index_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": { + "M": self.M, + "efConstruction": self.efConstruction, + "m": self.m, + "nbits": self.nbits, + "refine": self.refine, + "refine_type": self.refine_type.value, + }, + } + + def search_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "params": {"ef": self.ef, "refine_k": self.refine_k}, + } + + +class HNSWPRQConfig(HNSWPQConfig): + index: IndexType = IndexType.HNSW_PRQ + nrq: int = 2 + + def index_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": { + "M": self.M, + "efConstruction": self.efConstruction, + "m": self.m, + "nbits": self.nbits, + "nrq": self.nrq, + "refine": self.refine, + "refine_type": self.refine_type.value, + }, + } + + def search_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "params": {"ef": self.ef, "refine_k": self.refine_k}, + } + + class DISKANNConfig(MilvusIndexConfig, DBCaseConfig): search_list: int | None = None index: IndexType = IndexType.DISKANN @@ -125,6 +212,27 @@ def search_param(self) -> dict: } +class IVFPQConfig(MilvusIndexConfig, DBCaseConfig): + nlist: int + nprobe: int | None = None + m: int = 32 + nbits: int = 8 + index: IndexType = IndexType.IVFPQ + + def index_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": {"nlist": self.nlist, "m": self.m, "nbits": self.nbits}, + } + + def search_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "params": {"nprobe": self.nprobe}, + } + + class IVFSQ8Config(MilvusIndexConfig, DBCaseConfig): nlist: int nprobe: int | None = None @@ -144,6 +252,31 @@ def search_param(self) -> dict: } +class IVFRABITQConfig(IVFSQ8Config): + index: IndexType = IndexType.IVF_RABITQ + rbq_bits_query: int = 0 # 0, 1, 2, ..., 8 + refine: bool = True + refine_type: SQType = SQType.FP32 + refine_k: float = 1 + + def index_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": { + "nlist": self.nlist, + "refine": self.refine, + "refine_type": self.refine_type.value, + }, + } + + def search_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "params": {"nprobe": self.nprobe, "rbq_bits_query": self.rbq_bits_query, "refine_k": self.refine_k}, + } + + class FLATConfig(MilvusIndexConfig, DBCaseConfig): index: IndexType = IndexType.Flat @@ -187,7 +320,6 @@ def search_param(self) -> dict: class GPUBruteForceConfig(MilvusIndexConfig, DBCaseConfig): limit: int = 10 # Default top-k for search - metric_type: str # Metric type (e.g., 'L2', 'IP', etc.) index: IndexType = IndexType.GPU_BRUTE_FORCE # Index type set to GPU_BRUTE_FORCE def index_param(self) -> dict: @@ -282,15 +414,48 @@ def search_param(self) -> dict: } +class SCANNConfig(MilvusIndexConfig, DBCaseConfig): + nlist: int = 1024 + with_raw_data: bool = False + nprobe: int = 64 + reorder_k: int | None = 100 + index: IndexType = IndexType.SCANN_MILVUS + + def index_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "index_type": "SCANN", + "params": { + "nlist": self.nlist, + "with_raw_data": self.with_raw_data, + }, + } + + def search_param(self) -> dict: + return { + "metric_type": self.parse_metric(), + "params": { + "nprobe": self.nprobe, + "reorder_k": self.reorder_k, + }, + } + + _milvus_case_config = { IndexType.AUTOINDEX: AutoIndexConfig, IndexType.HNSW: HNSWConfig, + IndexType.HNSW_SQ: HNSWSQConfig, + IndexType.HNSW_PQ: HNSWPQConfig, + IndexType.HNSW_PRQ: HNSWPRQConfig, IndexType.DISKANN: DISKANNConfig, IndexType.IVFFlat: IVFFlatConfig, + IndexType.IVFPQ: IVFPQConfig, IndexType.IVFSQ8: IVFSQ8Config, + IndexType.IVF_RABITQ: IVFRABITQConfig, IndexType.Flat: FLATConfig, IndexType.GPU_IVF_FLAT: GPUIVFFlatConfig, IndexType.GPU_IVF_PQ: GPUIVFPQConfig, IndexType.GPU_CAGRA: GPUCAGRAConfig, IndexType.GPU_BRUTE_FORCE: GPUBruteForceConfig, + IndexType.SCANN_MILVUS: SCANNConfig, } diff --git a/vectordb_bench/backend/clients/milvus/milvus.py b/vectordb_bench/backend/clients/milvus/milvus.py index c812698fe..b177af332 100644 --- a/vectordb_bench/backend/clients/milvus/milvus.py +++ b/vectordb_bench/backend/clients/milvus/milvus.py @@ -7,6 +7,8 @@ from pymilvus import Collection, CollectionSchema, DataType, FieldSchema, MilvusException, utility +from vectordb_bench.backend.filter import Filter, FilterOp + from ..api import VectorDB from .config import MilvusIndexConfig @@ -16,14 +18,21 @@ class Milvus(VectorDB): + supported_filter_types: list[FilterOp] = [ + FilterOp.NonFilter, + FilterOp.NumGE, + FilterOp.StrEqual, + ] + def __init__( self, dim: int, db_config: dict, db_case_config: MilvusIndexConfig, - collection_name: str = "VectorDBBenchCollection", + collection_name: str = "VDBBench", drop_old: bool = False, name: str = "Milvus", + with_scalar_labels: bool = False, **kwargs, ): """Initialize wrapper around the milvus vector database.""" @@ -32,15 +41,24 @@ def __init__( self.case_config = db_case_config self.collection_name = collection_name self.batch_size = int(MILVUS_LOAD_REQS_SIZE / (dim * 4)) + self.with_scalar_labels = with_scalar_labels self._primary_field = "pk" - self._scalar_field = "id" + self._scalar_id_field = "id" + self._scalar_label_field = "label" self._vector_field = "vector" - self._index_name = "vector_idx" + self._vector_index_name = "vector_idx" + self._scalar_id_index_name = "id_sort_idx" + self._scalar_labels_index_name = "labels_idx" from pymilvus import connections - connections.connect(**self.db_config, timeout=30) + connections.connect( + uri=self.db_config.get("uri"), + user=self.db_config.get("user"), + password=self.db_config.get("password"), + timeout=30, + ) if drop_old and utility.has_collection(self.collection_name): log.info(f"{self.name} client drop_old collection: {self.collection_name}") utility.drop_collection(self.collection_name) @@ -48,9 +66,20 @@ def __init__( if not utility.has_collection(self.collection_name): fields = [ FieldSchema(self._primary_field, DataType.INT64, is_primary=True), - FieldSchema(self._scalar_field, DataType.INT64), + FieldSchema(self._scalar_id_field, DataType.INT64), FieldSchema(self._vector_field, DataType.FLOAT_VECTOR, dim=dim), ] + if self.with_scalar_labels: + is_partition_key = db_case_config.use_partition_key + log.info(f"with_scalar_labels, add a new varchar field, as partition_key: {is_partition_key}") + fields.append( + FieldSchema( + self._scalar_label_field, + DataType.VARCHAR, + max_length=256, + is_partition_key=is_partition_key, + ) + ) log.info(f"{self.name} create collection: {self.collection_name}") @@ -59,19 +88,42 @@ def __init__( name=self.collection_name, schema=CollectionSchema(fields), consistency_level="Session", + num_shards=self.db_config.get("num_shards", 1), ) - col.create_index( - self._vector_field, - self.case_config.index_param(), - index_name=self._index_name, - ) - col.load() + self.create_index() + col.load(replica_number=self.db_config.get("replica_number", 1)) connections.disconnect("default") + def create_index(self): + col = Collection(self.collection_name) + # vector index + col.create_index( + self._vector_field, + self.case_config.index_param(), + index_name=self._vector_index_name, + ) + # scalar index for range-expr (int-filter) + col.create_index( + self._scalar_id_field, + index_params={ + "index_type": "STL_SORT", + }, + index_name=self._scalar_id_index_name, + ) + # scalar index for varchar (label-filter) + if self.with_scalar_labels: + col.create_index( + self._scalar_label_field, + index_params={ + "index_type": "BITMAP", + }, + index_name=self._scalar_labels_index_name, + ) + @contextmanager - def init(self) -> None: + def init(self): """ Examples: >>> with self.init(): @@ -102,17 +154,13 @@ def _post_insert(self): try: self.col.flush() # wait for index done and load refresh - self.col.create_index( - self._vector_field, - self.case_config.index_param(), - index_name=self._index_name, - ) + self.create_index() - utility.wait_for_index_building_complete(self.collection_name) + utility.wait_for_index_building_complete(self.collection_name, index_name=self._vector_index_name) def wait_index(): while True: - progress = utility.index_building_progress(self.collection_name) + progress = utility.index_building_progress(self.collection_name, index_name=self._vector_index_name) if progress.get("pending_index_rows", -1) == 0: break time.sleep(5) @@ -126,6 +174,7 @@ def wait_index(): try: self.col.compact() self.col.wait_for_compaction_completed() + log.info("compactation completed. waiting for the rest of index buliding.") except Exception as e: log.warning(f"{self.name} compact error: {e}") if hasattr(e, "code"): @@ -154,6 +203,7 @@ def insert_embeddings( self, embeddings: Iterable[list[float]], metadata: list[int], + labels_data: list[str] | None = None, **kwargs, ) -> tuple[int, Exception]: """Insert embeddings into Milvus. should call self.init() first""" @@ -169,32 +219,42 @@ def insert_embeddings( metadata[batch_start_offset:batch_end_offset], embeddings[batch_start_offset:batch_end_offset], ] + if self.with_scalar_labels: + insert_data.append(labels_data[batch_start_offset:batch_end_offset]) res = self.col.insert(insert_data) insert_count += len(res.primary_keys) except MilvusException as e: log.info(f"Failed to insert data: {e}") - return (insert_count, e) - return (insert_count, None) + return insert_count, e + return insert_count, None + + def prepare_filter(self, filters: Filter): + if filters.type == FilterOp.NonFilter: + self.expr = "" + elif filters.type == FilterOp.NumGE: + self.expr = f"{self._scalar_id_field} >= {filters.int_value}" + elif filters.type == FilterOp.StrEqual: + self.expr = f"{self._scalar_label_field} == '{filters.label_value}'" + else: + msg = f"Not support Filter for Milvus - {filters}" + raise ValueError(msg) def search_embedding( self, query: list[float], k: int = 100, - filters: dict | None = None, timeout: int | None = None, ) -> list[int]: """Perform a search on a query embedding and return results.""" assert self.col is not None - expr = f"{self._scalar_field} {filters.get('metadata')}" if filters else "" - # Perform the search. res = self.col.search( data=[query], anns_field=self._vector_field, param=self.case_config.search_param(), limit=k, - expr=expr, + expr=self.expr, ) # Organize results. diff --git a/vectordb_bench/backend/clients/oceanbase/cli.py b/vectordb_bench/backend/clients/oceanbase/cli.py new file mode 100644 index 000000000..61583cc82 --- /dev/null +++ b/vectordb_bench/backend/clients/oceanbase/cli.py @@ -0,0 +1,101 @@ +import os +from typing import Annotated, Unpack + +import click +from pydantic import SecretStr + +from vectordb_bench.backend.clients import DB +from vectordb_bench.cli.cli import ( + CommonTypedDict, + HNSWFlavor4, + OceanBaseIVFTypedDict, + cli, + click_parameter_decorators_from_typed_dict, + run, +) + +from ..api import IndexType + + +class OceanBaseTypedDict(CommonTypedDict): + host: Annotated[str, click.option("--host", type=str, help="OceanBase host", default="")] + user: Annotated[str, click.option("--user", type=str, help="OceanBase username", required=True)] + password: Annotated[ + str, + click.option( + "--password", + type=str, + help="OceanBase database password", + default=lambda: os.environ.get("OB_PASSWORD", ""), + ), + ] + database: Annotated[str, click.option("--database", type=str, help="DataBase name", required=True)] + port: Annotated[int, click.option("--port", type=int, help="OceanBase port", required=True)] + + +class OceanBaseHNSWTypedDict(CommonTypedDict, OceanBaseTypedDict, HNSWFlavor4): ... + + +@cli.command() +@click_parameter_decorators_from_typed_dict(OceanBaseHNSWTypedDict) +def OceanBaseHNSW(**parameters: Unpack[OceanBaseHNSWTypedDict]): + from .config import OceanBaseConfig, OceanBaseHNSWConfig + + run( + db=DB.OceanBase, + db_config=OceanBaseConfig( + db_label=parameters["db_label"], + user=SecretStr(parameters["user"]), + password=SecretStr(parameters["password"]), + host=parameters["host"], + port=parameters["port"], + database=parameters["database"], + ), + db_case_config=OceanBaseHNSWConfig( + m=parameters["m"], + efConstruction=parameters["ef_construction"], + ef_search=parameters["ef_search"], + index=parameters["index_type"], + ), + **parameters, + ) + + +class OceanBaseIVFTypedDict(CommonTypedDict, OceanBaseTypedDict, OceanBaseIVFTypedDict): ... + + +@cli.command() +@click_parameter_decorators_from_typed_dict(OceanBaseIVFTypedDict) +def OceanBaseIVF(**parameters: Unpack[OceanBaseIVFTypedDict]): + from .config import OceanBaseConfig, OceanBaseIVFConfig + + type_str = parameters["index_type"] + if type_str == "IVF_FLAT": + input_index_type = IndexType.IVFFlat + elif type_str == "IVF_PQ": + input_index_type = IndexType.IVFPQ + elif type_str == "IVF_SQ8": + input_index_type = IndexType.IVFSQ8 + + input_m = 0 if parameters["m"] is None else parameters["m"] + + run( + db=DB.OceanBase, + db_config=OceanBaseConfig( + db_label=parameters["db_label"], + user=SecretStr(parameters["user"]), + password=SecretStr(parameters["password"]), + host=parameters["host"], + port=parameters["port"], + database=parameters["database"], + ), + db_case_config=OceanBaseIVFConfig( + m=input_m, + nlist=parameters["nlist"], + sample_per_nlist=parameters["sample_per_nlist"], + nbits=parameters["nbits"], + index=input_index_type, + ivf_nprobes=parameters["ivf_nprobes"], + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/oceanbase/config.py b/vectordb_bench/backend/clients/oceanbase/config.py new file mode 100644 index 000000000..1f37cfc75 --- /dev/null +++ b/vectordb_bench/backend/clients/oceanbase/config.py @@ -0,0 +1,119 @@ +from typing import TypedDict + +from pydantic import BaseModel, SecretStr + +from ..api import DBCaseConfig, DBConfig, IndexType, MetricType + + +class OceanBaseConfigDict(TypedDict): + user: str + host: str + port: str + password: str + database: str + + +class OceanBaseConfig(DBConfig): + user: SecretStr = SecretStr("root@perf") + password: SecretStr + host: str + port: int + database: str + + def to_dict(self) -> OceanBaseConfigDict: + user_str = self.user.get_secret_value() + pwd_str = self.password.get_secret_value() + return { + "user": user_str, + "host": self.host, + "port": self.port, + "password": pwd_str, + "database": self.database, + } + + +class OceanBaseIndexConfig(BaseModel): + index: IndexType + metric_type: MetricType | None = None + lib: str = "vsag" + + def parse_metric(self) -> str: + if self.metric_type == MetricType.L2 or ( + self.index == IndexType.HNSW_BQ and self.metric_type == MetricType.COSINE + ): + return "l2" + if self.metric_type == MetricType.IP: + return "inner_product" + return "cosine" + + def parse_metric_func_str(self) -> str: + if self.metric_type == MetricType.L2 or ( + self.index == IndexType.HNSW_BQ and self.metric_type == MetricType.COSINE + ): + return "l2_distance" + if self.metric_type == MetricType.IP: + return "negative_inner_product" + return "cosine_distance" + + +class OceanBaseHNSWConfig(OceanBaseIndexConfig, DBCaseConfig): + m: int + efConstruction: int + ef_search: int | None = None + index: IndexType + + def index_param(self) -> dict: + return { + "lib": self.lib, + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": {"m": self.m, "ef_construction": self.efConstruction}, + } + + def search_param(self) -> dict: + return {"metric_type": self.parse_metric_func_str(), "params": {"ef_search": self.ef_search}} + + +class OceanBaseIVFConfig(OceanBaseIndexConfig, DBCaseConfig): + m: int | None = None + sample_per_nlist: int + nbits: int | None = None + nlist: int + index: IndexType + ivf_nprobes: int | None = None + + def index_param(self) -> dict: + if self.index == IndexType.IVFPQ: + return { + "lib": "OB", + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": { + "m": self.m, + "sample_per_nlist": self.sample_per_nlist, + "nbits": self.nbits, + "nlist": self.nlist, + }, + } + return { + "lib": "OB", + "metric_type": self.parse_metric(), + "index_type": self.index.value, + "params": { + "sample_per_nlist": self.sample_per_nlist, + "nlist": self.nlist, + }, + } + + def search_param(self) -> dict: + return {"metric_type": self.metric_type, "params": {"ivf_nprobes": self.ivf_nprobes}} + + +_oceanbase_case_config = { + IndexType.HNSW_SQ: OceanBaseHNSWConfig, + IndexType.HNSW: OceanBaseHNSWConfig, + IndexType.HNSW_BQ: OceanBaseHNSWConfig, + IndexType.IVFFlat: OceanBaseIVFConfig, + IndexType.IVFPQ: OceanBaseIVFConfig, + IndexType.IVFSQ8: OceanBaseIVFConfig, +} diff --git a/vectordb_bench/backend/clients/oceanbase/oceanbase.py b/vectordb_bench/backend/clients/oceanbase/oceanbase.py new file mode 100644 index 000000000..93c42aac1 --- /dev/null +++ b/vectordb_bench/backend/clients/oceanbase/oceanbase.py @@ -0,0 +1,231 @@ +import logging +import struct +import time +from collections.abc import Generator +from contextlib import contextmanager +from typing import Any + +import mysql.connector as mysql + +from vectordb_bench.backend.filter import Filter, FilterOp + +from ..api import IndexType, VectorDB +from .config import OceanBaseConfigDict, OceanBaseHNSWConfig + +log = logging.getLogger(__name__) + +OCEANBASE_DEFAULT_LOAD_BATCH_SIZE = 256 + + +class OceanBase(VectorDB): + supported_filter_types: list[FilterOp] = [ + FilterOp.NonFilter, + FilterOp.NumGE, + FilterOp.StrEqual, + ] + + def __init__( + self, + dim: int, + db_config: OceanBaseConfigDict, + db_case_config: OceanBaseHNSWConfig, + collection_name: str = "items", + drop_old: bool = False, + **kwargs, + ): + self.name = "OceanBase" + self.dim = dim + self.db_config = db_config + self.db_case_config = db_case_config + self.table_name = collection_name + self.load_batch_size = OCEANBASE_DEFAULT_LOAD_BATCH_SIZE + self._index_name = "vidx" + self._primary_field = "id" + self._vector_field = "embedding" + + log.info( + f"{self.name} initialized with config:\nDatabase: {self.db_config}\nCase Config: {self.db_case_config}" + ) + + self._conn = None + self._cursor = None + + try: + self._connect() + if drop_old: + self._drop_table() + self._create_table() + finally: + self._disconnect() + + def _connect(self): + try: + self._conn = mysql.connect( + host=self.db_config["host"], + user=self.db_config["user"], + port=self.db_config["port"], + password=self.db_config["password"], + database=self.db_config["database"], + ) + self._cursor = self._conn.cursor() + except mysql.Error: + log.exception("Failed to connect to the database") + raise + + def _disconnect(self): + if self._cursor: + self._cursor.close() + self._cursor = None + if self._conn: + self._conn.close() + self._conn = None + + @contextmanager + def init(self) -> Generator[None, None, None]: + try: + self._connect() + self._cursor.execute("SET autocommit=1") + + if self.db_case_config.index in {IndexType.HNSW, IndexType.HNSW_SQ, IndexType.HNSW_BQ}: + self._cursor.execute( + f"SET ob_hnsw_ef_search={(self.db_case_config.search_param())['params']['ef_search']}" + ) + else: + self._cursor.execute( + f"SET ob_ivf_nprobes={(self.db_case_config.search_param())['params']['ivf_nprobes']}" + ) + yield + finally: + self._disconnect() + + def _drop_table(self): + if not self._cursor: + raise ValueError("Cursor is not initialized") + + log.info(f"Dropping table {self.table_name}") + self._cursor.execute(f"DROP TABLE IF EXISTS {self.table_name}") + + def _create_table(self): + if not self._cursor: + raise ValueError("Cursor is not initialized") + + log.info(f"Creating table {self.table_name}") + create_table_query = f""" + CREATE TABLE {self.table_name} ( + id INT PRIMARY KEY, + embedding VECTOR({self.dim}) + ); + """ + self._cursor.execute(create_table_query) + + def optimize(self, data_size: int): + index_params = self.db_case_config.index_param() + index_args = ", ".join(f"{k}={v}" for k, v in index_params["params"].items()) + index_query = ( + f"CREATE /*+ PARALLEL(18) */ VECTOR INDEX idx1 " + f"ON {self.table_name}(embedding) " + f"WITH (distance={self.db_case_config.parse_metric()}, " + f"type={index_params['index_type']}, lib={index_params['lib']}, {index_args}" + ) + + if self.db_case_config.index in {IndexType.HNSW, IndexType.HNSW_SQ, IndexType.HNSW_BQ}: + index_query += ", extra_info_max_size=32" + + index_query += ")" + + log.info("Create index query: %s", index_query) + + try: + log.info("Creating index...") + start_time = time.time() + self._cursor.execute(index_query) + log.info(f"Index created in {time.time() - start_time:.2f} seconds") + + log.info("Performing major freeze...") + self._cursor.execute("ALTER SYSTEM MAJOR FREEZE;") + time.sleep(10) + self._wait_for_major_compaction() + + log.info("Gathering schema statistics...") + self._cursor.execute("CALL dbms_stats.gather_schema_stats('test', degree => 96);") + except mysql.Error: + log.exception("Failed to optimize index") + raise + + def need_normalize_cosine(self) -> bool: + if self.db_case_config.index == IndexType.HNSW_BQ: + log.info("current HNSW_BQ only supports L2, cosine dataset need normalize.") + return True + + return False + + def _wait_for_major_compaction(self): + while True: + self._cursor.execute( + "SELECT IF(COUNT(*) = COUNT(STATUS = 'IDLE' OR NULL), 'TRUE', 'FALSE') " + "AS all_status_idle FROM oceanbase.DBA_OB_ZONE_MAJOR_COMPACTION;" + ) + all_status_idle = self._cursor.fetchone()[0] + if all_status_idle == "TRUE": + break + time.sleep(10) + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + **kwargs: Any, + ) -> tuple[int, Exception | None]: + if not self._cursor: + raise ValueError("Cursor is not initialized") + + insert_count = 0 + try: + for batch_start in range(0, len(embeddings), self.load_batch_size): + batch_end = min(batch_start + self.load_batch_size, len(embeddings)) + batch = [(metadata[i], embeddings[i]) for i in range(batch_start, batch_end)] + values = ", ".join(f"({item_id}, '[{','.join(map(str, embedding))}]')" for item_id, embedding in batch) + self._cursor.execute( + f"INSERT /*+ ENABLE_PARALLEL_DML PARALLEL(32) */ INTO {self.table_name} VALUES {values}" # noqa: S608 + ) + insert_count += len(batch) + except mysql.Error: + log.exception("Failed to insert embeddings") + raise + + return insert_count, None + + def prepare_filter(self, filters: Filter): + if filters.type == FilterOp.NonFilter: + self.expr = "" + elif filters.type == FilterOp.NumGE: + self.expr = f"WHERE id >= {filters.int_value}" + elif filters.type == FilterOp.StrEqual: + self.expr = f"WHERE id == '{filters.label_value}'" + else: + msg = f"Not support Filter for Oceanbase - {filters}" + raise ValueError(msg) + + def search_embedding( + self, + query: list[float], + k: int = 100, + ) -> list[int]: + if not self._cursor: + raise ValueError("Cursor is not initialized") + + packed = struct.pack(f"<{len(query)}f", *query) + hex_vec = packed.hex() + query_str = ( + f"SELECT id FROM {self.table_name} " # noqa: S608 + f"{self.expr} ORDER BY " + f"{self.db_case_config.parse_metric_func_str()}(embedding, X'{hex_vec}') " + f"APPROXIMATE LIMIT {k}" + ) + + try: + self._cursor.execute(query_str) + return [row[0] for row in self._cursor.fetchall()] + except mysql.Error: + log.exception("Failed to execute search query") + raise diff --git a/vectordb_bench/backend/clients/oss_opensearch/cli.py b/vectordb_bench/backend/clients/oss_opensearch/cli.py new file mode 100644 index 000000000..0c4b694ea --- /dev/null +++ b/vectordb_bench/backend/clients/oss_opensearch/cli.py @@ -0,0 +1,179 @@ +import logging +from typing import Annotated, TypedDict, Unpack + +import click +from pydantic import SecretStr + +from ....cli.cli import ( + CommonTypedDict, + HNSWFlavor1, + cli, + click_parameter_decorators_from_typed_dict, + run, +) +from .. import DB +from .config import OSSOpenSearchQuantization, OSSOS_Engine + +log = logging.getLogger(__name__) + + +class OSSOpenSearchTypedDict(TypedDict): + host: Annotated[str, click.option("--host", type=str, help="Db host", required=True)] + port: Annotated[int, click.option("--port", type=int, default=80, help="Db Port")] + user: Annotated[str, click.option("--user", type=str, help="Db User")] + password: Annotated[str, click.option("--password", type=str, help="Db password")] + number_of_shards: Annotated[ + int, + click.option("--number-of-shards", type=int, help="Number of primary shards for the index", default=1), + ] + number_of_replicas: Annotated[ + int, + click.option( + "--number-of-replicas", type=int, help="Number of replica copies for each primary shard", default=1 + ), + ] + index_thread_qty: Annotated[ + int, + click.option( + "--index-thread-qty", + type=int, + help="Thread count for native engine indexing", + default=4, + ), + ] + + index_thread_qty_during_force_merge: Annotated[ + int, + click.option( + "--index_thread_qty_during_force_merge", + type=int, + help="Thread count for native engine indexing during force merge", + default=4, + ), + ] + + metric_type: Annotated[ + str, + click.option( + "--metric-type", + type=click.Choice(["l2", "cosine", "ip"], case_sensitive=False), + help="Distance metric type for vector similarity", + default="l2", + ), + ] + + number_of_segments: Annotated[ + int, + click.option("--number-of-segments", type=int, help="Target number of segments after merging", default=1), + ] + + refresh_interval: Annotated[ + str, + click.option( + "--refresh-interval", type=str, help="How often to make new data available for search", default="60s" + ), + ] + + force_merge_enabled: Annotated[ + bool, + click.option("--force-merge-enabled", type=bool, help="Whether to perform force merge operation", default=True), + ] + + flush_threshold_size: Annotated[ + str, + click.option( + "--flush-threshold-size", type=str, help="Size threshold for flushing the transaction log", default="5120mb" + ), + ] + + cb_threshold: Annotated[ + str, + click.option( + "--cb-threshold", + type=str, + help="k-NN Memory circuit breaker threshold", + default="50%", + ), + ] + + quantization_type: Annotated[ + str | None, + click.option( + "--quantization-type", + type=click.Choice(["None", "LuceneSQ", "FaissSQfp16"]), + help="quantization type for vectors (in index)", + default="None", + required=False, + ), + ] + + confidence_interval: Annotated[ + float | None, + click.option( + "--confidence-interval", + type=float, + help="Confidence interval for Lucene SQ (0.0-1.0, optional)", + default=None, + required=False, + ), + ] + + clip: Annotated[ + bool, + click.option( + "--clip", + type=bool, + help="Clip vectors to [-65504, 65504] for FAISS FP16", + default=False, + required=False, + ), + ] + + engine: Annotated[ + str | None, + click.option( + "--engine", + type=click.Choice(["faiss", "lucene"]), + help="quantization type for vectors (in index)", + default="faiss", + required=False, + ), + ] + + +class OSSOpenSearchHNSWTypedDict(CommonTypedDict, OSSOpenSearchTypedDict, HNSWFlavor1): ... + + +@cli.command() +@click_parameter_decorators_from_typed_dict(OSSOpenSearchHNSWTypedDict) +def OSSOpenSearch(**parameters: Unpack[OSSOpenSearchHNSWTypedDict]): + from .config import OSSOpenSearchConfig, OSSOpenSearchIndexConfig + + run( + db=DB.OSSOpenSearch, + db_config=OSSOpenSearchConfig( + host=parameters["host"], + port=parameters["port"], + user=parameters["user"], + password=SecretStr(parameters["password"]), + ), + db_case_config=OSSOpenSearchIndexConfig( + number_of_shards=parameters["number_of_shards"], + number_of_replicas=parameters["number_of_replicas"], + index_thread_qty=parameters["index_thread_qty"], + number_of_segments=parameters["number_of_segments"], + refresh_interval=parameters["refresh_interval"], + force_merge_enabled=parameters["force_merge_enabled"], + flush_threshold_size=parameters["flush_threshold_size"], + index_thread_qty_during_force_merge=parameters["index_thread_qty_during_force_merge"], + cb_threshold=parameters["cb_threshold"], + efConstruction=parameters["ef_construction"], + efSearch=parameters["ef_search"], + M=parameters["m"], + engine=OSSOS_Engine(parameters["engine"]), + quantization_type=OSSOpenSearchQuantization(parameters["quantization_type"]), + confidence_interval=parameters["confidence_interval"], + clip=parameters["clip"], + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/oss_opensearch/config.py b/vectordb_bench/backend/clients/oss_opensearch/config.py new file mode 100644 index 000000000..83fed3d58 --- /dev/null +++ b/vectordb_bench/backend/clients/oss_opensearch/config.py @@ -0,0 +1,254 @@ +import logging +from enum import Enum + +from pydantic import BaseModel, SecretStr, root_validator, validator + +from ..api import DBCaseConfig, DBConfig, MetricType + +log = logging.getLogger(__name__) + + +class OSSOpenSearchConfig(DBConfig, BaseModel): + host: str = "" + port: int = 80 + user: str | None = None + password: SecretStr | None = None + + def to_dict(self) -> dict: + use_ssl = self.port == 443 + http_auth = ( + (self.user, self.password.get_secret_value()) + if self.user is not None and self.password is not None and len(self.user) != 0 and len(self.password) != 0 + else () + ) + return { + "hosts": [{"host": self.host, "port": self.port}], + "http_auth": http_auth, + "use_ssl": use_ssl, + "http_compress": True, + "verify_certs": use_ssl, + "ssl_assert_hostname": False, + "ssl_show_warn": False, + "timeout": 600, + } + + @validator("*") + def not_empty_field(cls, v: any, field: any): + if ( + field.name in cls.common_short_configs() + or field.name in cls.common_long_configs() + or field.name in ["user", "password", "host"] + ): + return v + if isinstance(v, str | SecretStr) and len(v) == 0: + raise ValueError("Empty string!") + return v + + +class OSSOS_Engine(Enum): + faiss = "faiss" + lucene = "lucene" + + +class OSSOpenSearchQuantization(Enum): + """In-memory scalar quantization types""" + + NONE = "None" + LUCENE_SQ = "LuceneSQ" + FAISS_SQFP16 = "FaissSQfp16" + + +# Compression level constants for disk-based mode +class CompressionLevel: + """Valid compression levels for disk-based vector search""" + + LEVEL_1X = "1x" + LEVEL_2X = "2x" + LEVEL_4X = "4x" + LEVEL_8X = "8x" + LEVEL_16X = "16x" + LEVEL_32X = "32x" + + ALL = [LEVEL_1X, LEVEL_2X, LEVEL_4X, LEVEL_8X, LEVEL_16X, LEVEL_32X] + + # Lucene: 1x, 4x | FAISS: 2x, 8x, 16x, 32x + ENGINE_MAP = { + LEVEL_1X: OSSOS_Engine.lucene, + LEVEL_2X: OSSOS_Engine.faiss, + LEVEL_4X: OSSOS_Engine.lucene, + LEVEL_8X: OSSOS_Engine.faiss, + LEVEL_16X: OSSOS_Engine.faiss, + LEVEL_32X: OSSOS_Engine.faiss, + } + + +class OSSOpenSearchIndexConfig(BaseModel, DBCaseConfig): + metric_type: MetricType = MetricType.L2 + engine: OSSOS_Engine = OSSOS_Engine.faiss + efConstruction: int = 256 + efSearch: int = 100 + engine_name: str | None = None + metric_type_name: str | None = None + M: int = 16 + index_thread_qty: int | None = 4 + number_of_shards: int | None = 1 + number_of_replicas: int | None = 0 + number_of_segments: int | None = 1 + refresh_interval: str | None = "60s" + force_merge_enabled: bool | None = True + flush_threshold_size: str | None = "5120mb" + index_thread_qty_during_force_merge: int = 8 + cb_threshold: str | None = "50%" + number_of_indexing_clients: int | None = 1 + use_routing: bool = False # for label-filter cases + quantization_type: OSSOpenSearchQuantization = OSSOpenSearchQuantization.NONE + confidence_interval: float | None = None + clip: bool = False + replication_type: str | None = "DOCUMENT" + knn_derived_source_enabled: bool = False + memory_optimized_search: bool = False + on_disk: bool = False + compression_level: str = CompressionLevel.LEVEL_32X + oversample_factor: float = 1.0 + + @validator("quantization_type", pre=True, always=True) + def validate_quantization_type(cls, value: any): + """Convert string values to enum""" + if not value: + return OSSOpenSearchQuantization.NONE + + if isinstance(value, OSSOpenSearchQuantization): + return value + + mapping = { + "None": OSSOpenSearchQuantization.NONE, + "LuceneSQ": OSSOpenSearchQuantization.LUCENE_SQ, + "FaissSQfp16": OSSOpenSearchQuantization.FAISS_SQFP16, + } + + return mapping.get(value, OSSOpenSearchQuantization.NONE) + + @root_validator + def validate_engine_name(cls, values: dict): + """Map engine_name string from UI to engine enum""" + if values.get("engine_name"): + engine_name = values["engine_name"].lower() + if engine_name == "faiss": + values["engine"] = OSSOS_Engine.faiss + elif engine_name == "lucene": + values["engine"] = OSSOS_Engine.lucene + else: + log.warning(f"Unknown engine_name: {engine_name}, defaulting to faiss") + values["engine"] = OSSOS_Engine.faiss + return values + + def __eq__(self, obj: any): + return ( + self.engine == obj.engine + and self.M == obj.M + and self.efConstruction == obj.efConstruction + and self.number_of_shards == obj.number_of_shards + and self.number_of_replicas == obj.number_of_replicas + and self.number_of_segments == obj.number_of_segments + and self.use_routing == obj.use_routing + and self.quantization_type == obj.quantization_type + and self.confidence_interval == obj.confidence_interval + and self.clip == obj.clip + and self.replication_type == obj.replication_type + and self.knn_derived_source_enabled == obj.knn_derived_source_enabled + and self.memory_optimized_search == obj.memory_optimized_search + and self.on_disk == obj.on_disk + and self.compression_level == obj.compression_level + and self.oversample_factor == obj.oversample_factor + ) + + def __hash__(self) -> int: + return hash( + ( + self.engine, + self.M, + self.efConstruction, + self.number_of_shards, + self.number_of_replicas, + self.number_of_segments, + self.use_routing, + self.quantization_type, + self.confidence_interval, + self.clip, + self.replication_type, + self.knn_derived_source_enabled, + self.memory_optimized_search, + self.on_disk, + self.compression_level, + self.oversample_factor, + ) + ) + + def parse_metric(self) -> str: + log.info(f"User specified metric_type: {self.metric_type_name}") + self.metric_type = MetricType[self.metric_type_name.upper()] + if self.metric_type == MetricType.IP: + return "innerproduct" + if self.metric_type == MetricType.COSINE: + return "cosinesimil" + if self.metric_type == MetricType.L2: + log.info("Using l2 as specified by user") + return "l2" + return "l2" + + @property + def use_quant(self) -> bool: + """Only use in-memory quantization when NOT in disk mode""" + return not self.on_disk and self.quantization_type != OSSOpenSearchQuantization.NONE + + @property + def resolved_engine(self) -> OSSOS_Engine: + """Return engine based on mode: auto-selected for disk, configured for in-memory.""" + if self.on_disk: + return CompressionLevel.ENGINE_MAP.get(self.compression_level, OSSOS_Engine.faiss) + return self.engine + + def index_param(self) -> dict: + resolved_engine = self.resolved_engine + space_type = self.parse_metric() + + log.info( + f"Index configuration - " + f"mode: {'disk' if self.on_disk else 'in-memory'}, " + f"configured_engine: {self.engine.value}, " + f"resolved_engine: {resolved_engine.value}, " + f"metric_type: {self.metric_type_name}, " + f"space_type: {space_type}" + f"{', ' if self.on_disk else ''}" + f"{'compression_level: ' + self.compression_level if self.on_disk else ''}" + ) + + method_config = { + "name": "hnsw", + "engine": resolved_engine.value, + "space_type": space_type, + "parameters": { + "ef_construction": self.efConstruction, + "m": self.M, + }, + } + + # Add encoder for in-memory quantization + if self.use_quant: + encoder_config = {"name": "sq"} + + if self.quantization_type == OSSOpenSearchQuantization.LUCENE_SQ: + # Lucene SQ: optional confidence_interval + if self.confidence_interval is not None: + encoder_config["parameters"] = {"confidence_interval": self.confidence_interval} + + elif self.quantization_type == OSSOpenSearchQuantization.FAISS_SQFP16 and self.clip: + # FAISS SQfp16: optional clip parameter + encoder_config["parameters"] = {"type": "fp16", "clip": True} + + method_config["parameters"]["encoder"] = encoder_config + + return method_config + + def search_param(self) -> dict: + return {"ef_search": self.efSearch} diff --git a/vectordb_bench/backend/clients/oss_opensearch/oss_opensearch.py b/vectordb_bench/backend/clients/oss_opensearch/oss_opensearch.py new file mode 100644 index 000000000..f71850a17 --- /dev/null +++ b/vectordb_bench/backend/clients/oss_opensearch/oss_opensearch.py @@ -0,0 +1,677 @@ +import logging +import time +from collections.abc import Iterable +from contextlib import contextmanager, suppress +from typing import Any, Final + +from opensearchpy import OpenSearch +from packaging.version import Version +from packaging.version import parse as parse_version + +from vectordb_bench.backend.filter import Filter, FilterOp + +from ..api import VectorDB +from .config import OSSOpenSearchIndexConfig, OSSOS_Engine + +log = logging.getLogger(__name__) + +WAITING_FOR_REFRESH_SEC: Final[int] = 30 +WAITING_FOR_FORCE_MERGE_SEC: Final[int] = 30 +SECONDS_WAITING_FOR_REPLICAS_TO_BE_ENABLED_SEC: Final[int] = 30 + +# Central registry for version-dependent OpenSearch index settings. +# Add new rules here to automatically support future versions. +VERSION_SPECIFIC_SETTING_RULES = [ + { + "name": "knn.advanced.approximate_threshold", + "applies": lambda version, _: version >= Version("3.0"), + "value": lambda _: "-1", + }, + { + "name": "knn.derived_source.enabled", + "applies": lambda version, _: version >= Version("3.0"), + "value": lambda case_config: case_config.knn_derived_source_enabled, + }, + { + "name": "knn.memory_optimized_search", + "applies": lambda version, case_config: ( + version >= Version("3.1") + and case_config.engine == OSSOS_Engine.faiss + and case_config.memory_optimized_search + ), + "value": lambda case_config: case_config.memory_optimized_search, + }, +] + + +class OpenSearchError(Exception): + """Custom exception for OpenSearch operations.""" + + +class OpenSearchSettingsManager: + """Manages OpenSearch cluster and index settings.""" + + def __init__(self, client: OpenSearch, index_name: str) -> None: + self.client = client + self.index_name = index_name + + def apply_cluster_settings(self, settings: dict[str, Any], log_message: str = "Applied cluster settings") -> dict: + """Apply cluster-level settings.""" + try: + response = self.client.cluster.put_settings(body={"persistent": settings}) + log.info(log_message) + except Exception as e: + log.warning(f"Failed to apply cluster settings: {e}") + error_msg = f"Cluster settings application failed: {e}" + raise OpenSearchError(error_msg) from e + else: + return response + + def apply_index_settings(self, settings: dict[str, Any], log_message: str = "Applied index settings") -> dict: + """Apply index-level settings.""" + try: + response = self.client.indices.put_settings(index=self.index_name, body={"index": settings}) + log.info(log_message) + except Exception as e: + log.warning(f"Failed to apply index settings: {e}") + error_msg = f"Index settings application failed: {e}" + raise OpenSearchError(error_msg) from e + else: + return response + + +class BulkInsertManager: + """Manages bulk insertion operations with chunking and parallelization.""" + + def __init__(self, client: OpenSearch, index_name: str, case_config: OSSOpenSearchIndexConfig) -> None: + self.client = client + self.index_name = index_name + self.case_config = case_config + + def prepare_bulk_data( + self, + embeddings: list[list[float]], + metadata: list[int], + labels_data: list[str] | None, + id_col_name: str, + vector_col_name: str, + label_col_name: str, + with_scalar_labels: bool, + ) -> list[dict[str, Any]]: + """Prepare bulk actions for OpenSearch bulk insert.""" + if len(embeddings) != len(metadata): + error_msg = f"Embeddings ({len(embeddings)}) and metadata ({len(metadata)}) length mismatch" + raise ValueError(error_msg) + + if with_scalar_labels and labels_data and len(labels_data) != len(embeddings): + error_msg = f"Labels data ({len(labels_data)}) and embeddings ({len(embeddings)}) length mismatch" + raise ValueError(error_msg) + + insert_data: list[dict[str, Any]] = [] + for i in range(len(embeddings)): + index_data = {"index": {"_index": self.index_name, id_col_name: metadata[i]}} + if with_scalar_labels and self.case_config.use_routing and labels_data: + index_data["routing"] = labels_data[i] + insert_data.append(index_data) + + other_data = {vector_col_name: embeddings[i]} + if with_scalar_labels and labels_data: + other_data[label_col_name] = labels_data[i] + insert_data.append(other_data) + return insert_data + + def execute_single_client_insert(self, insert_data: list[dict[str, Any]]) -> tuple[int, Exception | None]: + """Execute bulk insert with single client and retry logic.""" + try: + response = self.client.bulk(body=insert_data) + if response.get("errors"): + log.warning(f"Bulk insert had errors: {response}") + return len(insert_data) // 2, None + except Exception as e: + log.warning(f"Failed to insert data: {self.index_name} error: {e!s}") + time.sleep(10) + return self.execute_single_client_insert(insert_data) + + +class SearchQueryBuilder: + """Builds OpenSearch KNN queries with proper configuration.""" + + def __init__(self, case_config: OSSOpenSearchIndexConfig, vector_col_name: str) -> None: + self.case_config = case_config + self.vector_col_name = vector_col_name + + def build_knn_query( + self, query_vector: list[float], k: int, filter_clause: dict[str, Any] | None = None + ) -> dict[str, Any]: + """Build a KNN query with optional filtering.""" + knn_config: dict[str, Any] = { + "vector": query_vector, + "k": k, + "method_parameters": self.case_config.search_param(), + } + + if filter_clause: + knn_config["filter"] = filter_clause + + # Handle rescoring for both in-memory quantization and disk-based modes + if self.case_config.use_quant or self.case_config.on_disk: + knn_config["rescore"] = {"oversample_factor": self.case_config.oversample_factor} + + return {"size": k, "query": {"knn": {self.vector_col_name: knn_config}}} + + def build_search_kwargs( + self, index_name: str, body: dict[str, Any], k: int, id_col_name: str, routing_key: str | None = None + ) -> dict[str, Any]: + """Build search kwargs with proper field selection.""" + search_kwargs: dict[str, Any] = { + "index": index_name, + "body": body, + "size": k, + "_source": False, + "preference": "_only_local" if self.case_config.number_of_shards == 1 else None, + "routing": routing_key, + } + + if id_col_name == "_id": + search_kwargs["stored_fields"] = "_id" + else: + search_kwargs["docvalue_fields"] = [id_col_name] + search_kwargs["stored_fields"] = "_none_" + + return search_kwargs + + +class OSSOpenSearch(VectorDB): + """OpenSearch client implementation for VectorDBBench.""" + + supported_filter_types: list[FilterOp] = [ + FilterOp.NonFilter, + FilterOp.NumGE, + FilterOp.StrEqual, + ] + + def __init__( + self, + dim: int, + db_config: dict[str, Any], + db_case_config: OSSOpenSearchIndexConfig, + index_name: str = "vdb_bench_index", # must be lowercase + id_col_name: str = "_id", + label_col_name: str = "label", + vector_col_name: str = "embedding", + drop_old: bool = False, + with_scalar_labels: bool = False, + **kwargs: Any, + ) -> None: + """Initialize the OpenSearch client.""" + self.dim = dim + self.db_config = db_config + self.case_config = db_case_config + self.index_name = index_name + self.id_col_name = id_col_name + self.label_col_name = label_col_name + self.vector_col_name = vector_col_name + self.with_scalar_labels = with_scalar_labels + + # Initialize client state + self.client: OpenSearch | None = None + self.filter: dict[str, Any] | None = None + self.routing_key: str | None = None + + log.info(f"OSS_OpenSearch client config: {self.db_config}") + log.info(f"OSS_OpenSearch db case config: {self.case_config}") + client = OpenSearch(**self.db_config) + self._handle_index_initialization(client, drop_old) + + def _handle_index_initialization(self, client: OpenSearch, drop_old: bool) -> None: + """Check, drop, create index, and perform post-creation setup.""" + if drop_old: + log.info(f"OSS_OpenSearch client drop old index: {self.index_name}") + is_existed = client.indices.exists(index=self.index_name) + if is_existed: + client.indices.delete(index=self.index_name) + self._create_index(client) + else: + is_existed = client.indices.exists(index=self.index_name) + if not is_existed: + self._create_index(client) + log.info(f"OSS_OpenSearch client create index: {self.index_name}") + self._update_ef_search_before_search(client) + self._load_graphs_to_memory(client) + + def need_normalize_cosine(self) -> bool: + """Whether this database needs to normalize dataset to support COSINE metric.""" + return True + + def _get_cluster_version(self, client: OpenSearch) -> Version: + """ + Return the OpenSearch cluster version as a comparable Version object. + Raises an exception if the version cannot be determined. + """ + try: + info = client.info() + raw_version_str = info.get("version", {}).get("number", "") + if not raw_version_str: + raise ValueError("Received empty version string from OpenSearch") # noqa: TRY301 + cluster_version = parse_version(raw_version_str) + log.debug(f"Detected OpenSearch version: {cluster_version}") + return cluster_version # noqa: TRY300 + except Exception: + log.exception("Failed to determine OpenSearch version") + raise + + def _get_settings_manager(self, client: OpenSearch) -> OpenSearchSettingsManager: + """Get settings manager for the given client.""" + return OpenSearchSettingsManager(client, self.index_name) + + def _get_version_specific_settings(self, cluster_version: Version) -> dict: + """ + Builds and returns a dictionary of applicable version-specific settings. + """ + version_specific_settings = {} + for setting in VERSION_SPECIFIC_SETTING_RULES: + if setting["applies"](cluster_version, self.case_config): + name = setting["name"] + value = setting["value"](self.case_config) + version_specific_settings[name] = value + return version_specific_settings + + def _build_vector_field_mapping(self) -> dict[str, Any]: + """Build vector field mapping configuration based on storage mode.""" + vector_field = { + "type": "knn_vector", + "dimension": self.dim, + "method": self.case_config.index_param(), + } + + if self.case_config.on_disk: + vector_field.update( + { + "space_type": self.case_config.parse_metric(), + "data_type": "float", + "mode": "on_disk", + "compression_level": self.case_config.compression_level, + } + ) + log.info( + f"Creating disk-based index - " + f"compression_level: {self.case_config.compression_level}, " + f"resolved_engine: {self.case_config.resolved_engine.value}" + ) + else: + log.info(f"Creating in-memory index with engine: {self.case_config.engine.value}") + + return vector_field + + def _get_bulk_manager(self, client: OpenSearch) -> BulkInsertManager: + """Get bulk insert manager for the given client.""" + return BulkInsertManager(client, self.index_name, self.case_config) + + def _create_index(self, client: OpenSearch) -> None: + cluster_version = self._get_cluster_version(client) + + if self.case_config.on_disk and cluster_version < Version("2.17"): + error_msg = f"Disk-based vector search requires OpenSearch 2.17+, but cluster is running {cluster_version}" + raise OpenSearchError(error_msg) + + ef_search_value = self.case_config.efSearch + log.info(f"Creating index with ef_search: {ef_search_value}") + log.info(f"Creating index with number_of_replicas: {self.case_config.number_of_replicas}") + log.info(f"Creating index with replication_type: {self.case_config.replication_type}") + log.info(f"Creating index with knn_derived_source_enabled: {self.case_config.knn_derived_source_enabled}") + log.info(f"Creating index with engine: {self.case_config.engine}") + log.info(f"Creating index with metric type: {self.case_config.metric_type_name}") + log.info(f"Creating index with memory_optimized_search: {self.case_config.memory_optimized_search}") + log.info(f"All case_config parameters: {self.case_config.__dict__}") + + settings_manager = self._get_settings_manager(client) + cluster_settings = { + "knn.algo_param.index_thread_qty": self.case_config.index_thread_qty, + "knn.memory.circuit_breaker.limit": self.case_config.cb_threshold, + } + settings_manager.apply_cluster_settings( + cluster_settings, "Successfully updated cluster settings for index creation" + ) + # Base settings that are safe for all versions + settings = { + "index": { + "knn": True, + "number_of_shards": self.case_config.number_of_shards, + "number_of_replicas": self.case_config.number_of_replicas, + "translog.flush_threshold_size": self.case_config.flush_threshold_size, + "replication.type": self.case_config.replication_type, + }, + "refresh_interval": self.case_config.refresh_interval, + } + settings["index"]["knn.algo_param.ef_search"] = ef_search_value + + version_specific_settings = self._get_version_specific_settings(self._get_cluster_version(client)) + if version_specific_settings: + log.info(f"Applying version-dependent settings: {version_specific_settings}") + settings["index"].update(version_specific_settings) + + # Build properties mapping, excluding _id which is automatically handled by OpenSearch + properties = {} + + # Only add id field to properties if it's not the special _id field + if self.id_col_name != "_id": + properties[self.id_col_name] = {"type": "integer", "store": True} + + properties[self.label_col_name] = {"type": "keyword"} + properties[self.vector_col_name] = self._build_vector_field_mapping() + + mappings = { + "properties": properties, + } + try: + log.info(f"Creating index with settings: {settings}") + log.info(f"Creating index with mappings: {mappings}") + client.indices.create( + index=self.index_name, + body={"settings": settings, "mappings": mappings}, + ) + except Exception as e: + log.warning(f"Failed to create index: {self.index_name} error: {e!s}") + raise e from None + + @contextmanager + def init(self) -> None: + """Connect to OpenSearch""" + self.client = OpenSearch(**self.db_config) + + yield + self.client = None + del self.client + + def _prepare_bulk_data( + self, + embeddings: Iterable[list[float]], + metadata: list[int], + labels_data: list[str] | None = None, + ) -> list[dict]: + """Prepare the list of bulk actions for OpenSearch bulk insert.""" + bulk_manager = self._get_bulk_manager(self.client) + return bulk_manager.prepare_bulk_data( + list(embeddings), + metadata, + labels_data, + self.id_col_name, + self.vector_col_name, + self.label_col_name, + self.with_scalar_labels, + ) + + def insert_embeddings( + self, + embeddings: Iterable[list[float]], + metadata: list[int], + labels_data: list[str] | None = None, + **kwargs: Any, + ) -> tuple[int, Exception | None]: + """Insert embeddings into the OpenSearch index.""" + assert self.client is not None, "should self.init() first" + + num_clients = self.case_config.number_of_indexing_clients or 1 + log.info(f"Number of indexing clients from case_config: {num_clients}") + + if num_clients <= 1: + log.info("Using single client for data insertion") + return self._insert_with_single_client(embeddings, metadata, labels_data) + log.info(f"Using {num_clients} parallel clients for data insertion") + return self._insert_with_multiple_clients(embeddings, metadata, num_clients, labels_data) + + def _insert_with_single_client( + self, + embeddings: Iterable[list[float]], + metadata: list[int], + labels_data: list[str] | None = None, + ) -> tuple[int, Exception | None]: + """Insert data using a single client with retry logic.""" + insert_data = self._prepare_bulk_data(embeddings, metadata, labels_data) + bulk_manager = self._get_bulk_manager(self.client) + return bulk_manager.execute_single_client_insert(insert_data) + + def _insert_with_multiple_clients( + self, + embeddings: Iterable[list[float]], + metadata: list[int], + num_clients: int, + labels_data: list[str] | None = None, + ) -> tuple[int, Exception | None]: + """Insert data using multiple parallel clients for better performance.""" + import concurrent.futures + from concurrent.futures import ThreadPoolExecutor + + embeddings_list = list(embeddings) + chunk_size = max(1, len(embeddings_list) // num_clients) + chunks = [] + + for i in range(0, len(embeddings_list), chunk_size): + end = min(i + chunk_size, len(embeddings_list)) + chunks.append((embeddings_list[i:end], metadata[i:end], labels_data[i:end])) + clients = [OpenSearch(**self.db_config) for _ in range(min(num_clients, len(chunks)))] + log.info(f"OSS_OpenSearch using {len(clients)} parallel clients for data insertion") + + def insert_chunk(client_idx: int, chunk_idx: int): + chunk_embeddings, chunk_metadata, chunk_labels_data = chunks[chunk_idx] + client = clients[client_idx] + insert_data = self._prepare_bulk_data(chunk_embeddings, chunk_metadata, chunk_labels_data) + try: + response = client.bulk(body=insert_data) + log.info(f"Client {client_idx} added {len(response['items'])} documents") + return len(chunk_embeddings), None + except Exception as e: + log.warning(f"Client {client_idx} failed to insert data: {e!s}") + return 0, e + + results = [] + with ThreadPoolExecutor(max_workers=len(clients)) as executor: + futures = [ + executor.submit(insert_chunk, chunk_idx % len(clients), chunk_idx) for chunk_idx in range(len(chunks)) + ] + for future in concurrent.futures.as_completed(futures): + count, error = future.result() + results.append((count, error)) + + for client in clients: + with suppress(Exception): + client.close() + + total_count = sum(count for count, _ in results) + errors = [error for _, error in results if error is not None] + + if errors: + log.warning("Some clients failed to insert data, retrying with single client") + time.sleep(10) + return self._insert_with_single_client(embeddings, metadata, labels_data) + + response = self.client.indices.stats(self.index_name) + log.info( + f"""Total document count in index after parallel insertion: + {response['_all']['primaries']['indexing']['index_total']}""", + ) + + return (total_count, None) + + def _update_ef_search_before_search(self, client: OpenSearch): + ef_search_value = self.case_config.efSearch + + try: + index_settings = client.indices.get_settings(index=self.index_name) + current_ef_search = ( + index_settings.get(self.index_name, {}) + .get("settings", {}) + .get("index", {}) + .get("knn.algo_param", {}) + .get("ef_search") + ) + + if current_ef_search != str(ef_search_value): + settings_manager = self._get_settings_manager(client) + log_message = f"Successfully updated ef_search to {ef_search_value} before search" + settings_manager.apply_index_settings({"knn.algo_param.ef_search": ef_search_value}, log_message) + log.info(f"Current engine: {self.case_config.engine}") + log.info(f"Current metric_type: {self.case_config.metric_type_name}") + + except Exception as e: + log.warning(f"Failed to update ef_search parameter before search: {e}") + + def search_embedding( + self, + query: list[float], + k: int = 100, + filters: Filter | None = None, + **kwargs, + ) -> list[int]: + """Get k most similar embeddings to query vector. + + Args: + query(list[float]): query embedding to look up documents similar to. + k(int): Number of most similar embeddings to return. Defaults to 100. + filters(Filter, optional): filtering expression to filter the data while searching. + + Returns: + list[int]: list of k most similar ids to the query embedding. + """ + assert self.client is not None, "should self.init() first" + + search_query_builder = SearchQueryBuilder(self.case_config, self.vector_col_name) + body = search_query_builder.build_knn_query(query, k, self.filter) + + try: + search_kwargs = search_query_builder.build_search_kwargs( + self.index_name, body, k, self.id_col_name, self.routing_key + ) + response = self.client.search(**search_kwargs) + + log.debug(f"Search took: {response['took']}") + log.debug(f"Search shards: {response['_shards']}") + log.debug(f"Search hits total: {response['hits']['total']}") + try: + if self.id_col_name == "_id": + # Get _id directly from hit metadata + result_ids = [] + for h in response["hits"]["hits"]: + if (doc_id := h.get("_id")) is not None: + result_ids.append(int(doc_id)) + else: + log.warning(f"Hit missing _id in final extraction: {h}") + else: + # Get custom id field from docvalue fields + result_ids = [int(h["fields"][self.id_col_name][0]) for h in response["hits"]["hits"]] + + except Exception: + # empty results + return [] + else: + return result_ids + except Exception as e: + log.warning(f"Failed to search: {self.index_name} error: {e!s}") + raise e from None + + def prepare_filter(self, filters: Filter) -> None: + """Prepare filter conditions for search operations.""" + self.routing_key = None + if filters.type == FilterOp.NonFilter: + self.filter = None + elif filters.type == FilterOp.NumGE: + self.filter = {"range": {self.id_col_name: {"gt": filters.int_value}}} + elif filters.type == FilterOp.StrEqual: + self.filter = {"term": {self.label_col_name: filters.label_value}} + if self.case_config.use_routing: + self.routing_key = filters.label_value + else: + msg = f"Filter type {filters.type} not supported for OpenSearch" + log.error(f"Unsupported filter type: {filters.type}") + raise ValueError(msg) + + def optimize(self, data_size: int | None = None) -> None: + """Optimize the index for better search performance.""" + self._update_ef_search() + # Call refresh first to ensure that all segments are created + self._refresh_index() + if self.case_config.force_merge_enabled: + self._do_force_merge() + self._refresh_index() + self._update_replicas() + # Call refresh again to ensure that the index is ready after force merge. + self._refresh_index() + # ensure that all graphs are loaded in memory and ready for search + self._load_graphs_to_memory(self.client) + + def _update_ef_search(self): + ef_search_value = self.case_config.efSearch + settings_manager = self._get_settings_manager(self.client) + log_message = f"Successfully updated ef_search to {ef_search_value}" + settings_manager.apply_index_settings({"knn.algo_param.ef_search": ef_search_value}, log_message) + log.info(f"Current engine: {self.case_config.engine}") + log.info(f"Current metric_type: {self.case_config.metric_type}") + + def _update_replicas(self): + index_settings = self.client.indices.get_settings(index=self.index_name) + current_number_of_replicas = int(index_settings[self.index_name]["settings"]["index"]["number_of_replicas"]) + log.info( + f"Current Number of replicas are {current_number_of_replicas}" + f" and changing the replicas to {self.case_config.number_of_replicas}" + ) + settings_manager = self._get_settings_manager(self.client) + log_message = f"Successfully updated number of replicas to {self.case_config.number_of_replicas}" + settings_manager.apply_index_settings({"number_of_replicas": self.case_config.number_of_replicas}, log_message) + self._wait_till_green() + + def _wait_till_green(self): + log.info("Wait for index to become green..") + while True: + res = self.client.cat.indices(index=self.index_name, h="health", format="json") + health = res[0]["health"] + if health == "green": + break + log.info(f"The index {self.index_name} has health : {health} and is not green. Retrying") + time.sleep(SECONDS_WAITING_FOR_REPLICAS_TO_BE_ENABLED_SEC) + log.info(f"Index {self.index_name} is green..") + + def _refresh_index(self): + log.debug(f"Starting refresh for index {self.index_name}") + while True: + try: + log.info("Starting the Refresh Index..") + self.client.indices.refresh(index=self.index_name) + break + except Exception as e: + log.info( + f"Refresh errored out. Sleeping for {WAITING_FOR_REFRESH_SEC} sec and then Retrying : {e}", + ) + time.sleep(WAITING_FOR_REFRESH_SEC) + continue + log.debug(f"Completed refresh for index {self.index_name}") + + def _do_force_merge(self): + log.info(f"Updating the Index thread qty to {self.case_config.index_thread_qty_during_force_merge}.") + + settings_manager = self._get_settings_manager(self.client) + cluster_settings = {"knn.algo_param.index_thread_qty": self.case_config.index_thread_qty_during_force_merge} + log_message_cluster = ( + f"Successfully updated cluster index thread qty to {self.case_config.index_thread_qty_during_force_merge}" + ) + settings_manager.apply_cluster_settings(cluster_settings, log_message_cluster) + log.info("Updating the graph threshold to ensure that during merge we can do graph creation.") + log_message_index = "Successfully updated index approximate threshold to 0" + output = settings_manager.apply_index_settings({"knn.advanced.approximate_threshold": "0"}, log_message_index) + log.info(f"response of updating setting is: {output}") + + log.info(f"Starting force merge for index {self.index_name}") + segments = self.case_config.number_of_segments + force_merge_endpoint = f"/{self.index_name}/_forcemerge?max_num_segments={segments}&wait_for_completion=false" + force_merge_task_id = self.client.transport.perform_request("POST", force_merge_endpoint)["task"] + while True: + time.sleep(WAITING_FOR_FORCE_MERGE_SEC) + task_status = self.client.tasks.get(task_id=force_merge_task_id) + if task_status["completed"]: + break + log.info(f"Completed force merge for index {self.index_name}") + + def _load_graphs_to_memory(self, client: OpenSearch): + if self.case_config.engine != OSSOS_Engine.lucene: + log.info("Calling warmup API to load graphs into memory") + warmup_endpoint = f"/_plugins/_knn/warmup/{self.index_name}" + client.transport.perform_request("GET", warmup_endpoint) diff --git a/vectordb_bench/backend/clients/oss_opensearch/run.py b/vectordb_bench/backend/clients/oss_opensearch/run.py new file mode 100644 index 000000000..68aa200a5 --- /dev/null +++ b/vectordb_bench/backend/clients/oss_opensearch/run.py @@ -0,0 +1,166 @@ +import logging +import random +import time + +from opensearchpy import OpenSearch + +log = logging.getLogger(__name__) + +_HOST = "xxxxxx.us-west-2.es.amazonaws.com" +_PORT = 443 +_AUTH = ("admin", "xxxxxx") # For testing only. Don't store credentials in code. + +_INDEX_NAME = "my-dsl-index" +_BATCH = 100 +_ROWS = 100 +_DIM = 128 +_TOPK = 10 + + +def create_client(): + return OpenSearch( + hosts=[{"host": _HOST, "port": _PORT}], + http_compress=True, # enables gzip compression for request bodies + http_auth=_AUTH, + use_ssl=True, + verify_certs=True, + ssl_assert_hostname=False, + ssl_show_warn=False, + ) + + +def create_index(client: OpenSearch, index_name: str): + settings = { + "index": { + "knn": True, + "number_of_shards": 1, + "refresh_interval": "5s", + }, + } + mappings = { + "properties": { + "embedding": { + "type": "knn_vector", + "dimension": _DIM, + "method": { + "engine": "faiss", + "name": "hnsw", + "space_type": "l2", + "parameters": { + "ef_construction": 256, + "m": 16, + }, + }, + }, + }, + } + + response = client.indices.create( + index=index_name, + body={"settings": settings, "mappings": mappings}, + ) + log.info("\nCreating index:") + log.info(response) + + +def delete_index(client: OpenSearch, index_name: str): + response = client.indices.delete(index=index_name) + log.info("\nDeleting index:") + log.info(response) + + +def bulk_insert(client: OpenSearch, index_name: str): + # Perform bulk operations + ids = list(range(_ROWS)) + vec = [[random.random() for _ in range(_DIM)] for _ in range(_ROWS)] + + docs = [] + for i in range(0, _ROWS, _BATCH): + docs.clear() + for j in range(_BATCH): + docs.append({"index": {"_index": index_name, "_id": ids[i + j]}}) + docs.append({"embedding": vec[i + j]}) + response = client.bulk(docs) + log.info(f"Adding documents: {len(response['items'])}, {response['errors']}") + response = client.indices.stats(index_name) + log.info( + f'Total document count in index: { response["_all"]["primaries"]["indexing"]["index_total"] }', + ) + + +def search(client: OpenSearch, index_name: str): + # Search for the document. + search_body = { + "size": _TOPK, + "query": { + "knn": { + "embedding": { + "vector": [random.random() for _ in range(_DIM)], + "k": _TOPK, + }, + }, + }, + } + while True: + response = client.search(index=index_name, body=search_body) + log.info(f'\nSearch took: {response["took"]}') + log.info(f'\nSearch shards: {response["_shards"]}') + log.info(f'\nSearch hits total: {response["hits"]["total"]}') + result = response["hits"]["hits"] + if len(result) != 0: + log.info("\nSearch results:") + for hit in response["hits"]["hits"]: + log.info(hit["_id"], hit["_score"]) + break + log.info("\nSearch not ready, sleep 1s") + time.sleep(1) + + +SECONDS_WAITING_FOR_FORCE_MERGE_API_CALL_SEC = 30 +WAITINT_FOR_REFRESH_SEC = 30 + + +def optimize_index(client: OpenSearch, index_name: str): + log.info(f"Starting force merge for index {index_name}") + force_merge_endpoint = f"/{index_name}/_forcemerge?max_num_segments=1&wait_for_completion=false" + force_merge_task_id = client.transport.perform_request("POST", force_merge_endpoint)["task"] + while True: + time.sleep(SECONDS_WAITING_FOR_FORCE_MERGE_API_CALL_SEC) + task_status = client.tasks.get(task_id=force_merge_task_id) + if task_status["completed"]: + break + log.info(f"Completed force merge for index {index_name}") + + +def refresh_index(client: OpenSearch, index_name: str): + log.info(f"Starting refresh for index {index_name}") + while True: + try: + log.info("Starting the Refresh Index..") + client.indices.refresh(index=index_name) + break + except Exception as e: + log.info( + f"Refresh errored out. Sleeping for {WAITINT_FOR_REFRESH_SEC} sec and then Retrying : {e}", + ) + time.sleep(WAITINT_FOR_REFRESH_SEC) + continue + log.info(f"Completed refresh for index {index_name}") + + +def main(): + client = create_client() + try: + create_index(client, _INDEX_NAME) + bulk_insert(client, _INDEX_NAME) + optimize_index(client, _INDEX_NAME) + refresh_index(client, _INDEX_NAME) + search(client, _INDEX_NAME) + delete_index(client, _INDEX_NAME) + except Exception as e: + log.info(e) + delete_index(client, _INDEX_NAME) + + +if __name__ == "__main__": + main() diff --git a/vectordb_bench/backend/clients/pgdiskann/cli.py b/vectordb_bench/backend/clients/pgdiskann/cli.py index 19f47988f..f1e44e2f2 100644 --- a/vectordb_bench/backend/clients/pgdiskann/cli.py +++ b/vectordb_bench/backend/clients/pgdiskann/cli.py @@ -5,6 +5,7 @@ from pydantic import SecretStr from vectordb_bench.backend.clients import DB +from vectordb_bench.backend.clients.api import MetricType from ....cli.cli import ( CommonTypedDict, @@ -48,6 +49,15 @@ class PgDiskAnnTypedDict(CommonTypedDict): help="PgDiskAnn l_value_ib", ), ] + pq_param_num_chunks: Annotated[ + int, + click.option( + "--pq-param-num-chunks", + type=int, + help="PgDiskAnn pq_param_num_chunks", + required=False, + ), + ] l_value_is: Annotated[ float, click.option( @@ -56,6 +66,37 @@ class PgDiskAnnTypedDict(CommonTypedDict): help="PgDiskAnn l_value_is", ), ] + reranking: Annotated[ + bool | None, + click.option( + "--reranking/--skip-reranking", + type=bool, + help="Enable reranking for PQ search", + default=False, + ), + ] + reranking_metric: Annotated[ + str | None, + click.option( + "--reranking-metric", + type=click.Choice( + [metric.value for metric in MetricType if metric.value not in ["HAMMING", "JACCARD", "DP"]], + ), + help="Distance metric for reranking", + default="COSINE", + show_default=True, + required=False, + ), + ] + quantized_fetch_limit: Annotated[ + int | None, + click.option( + "--quantized-fetch-limit", + type=int, + help="Limit of inner query in case of reranking", + required=False, + ), + ] maintenance_work_mem: Annotated[ str | None, click.option( @@ -98,7 +139,11 @@ def PgDiskAnn( db_case_config=PgDiskANNImplConfig( max_neighbors=parameters["max_neighbors"], l_value_ib=parameters["l_value_ib"], + pq_param_num_chunks=parameters["pq_param_num_chunks"], l_value_is=parameters["l_value_is"], + reranking=parameters["reranking"], + reranking_metric=parameters["reranking_metric"], + quantized_fetch_limit=parameters["quantized_fetch_limit"], max_parallel_workers=parameters["max_parallel_workers"], maintenance_work_mem=parameters["maintenance_work_mem"], ), diff --git a/vectordb_bench/backend/clients/pgdiskann/config.py b/vectordb_bench/backend/clients/pgdiskann/config.py index ed478acc2..7f83a05c8 100644 --- a/vectordb_bench/backend/clients/pgdiskann/config.py +++ b/vectordb_bench/backend/clients/pgdiskann/config.py @@ -60,6 +60,13 @@ def parse_metric_fun_op(self) -> LiteralString: return "<#>" return "<=>" + def parse_reranking_metric_fun_op(self) -> LiteralString: + if self.reranking_metric == MetricType.L2: + return "<->" + if self.reranking_metric == MetricType.IP: + return "<#>" + return "<=>" + def parse_metric_fun_str(self) -> str: if self.metric_type == MetricType.L2: return "l2_distance" @@ -115,7 +122,11 @@ class PgDiskANNImplConfig(PgDiskANNIndexConfig): index: IndexType = IndexType.DISKANN max_neighbors: int | None l_value_ib: int | None + pq_param_num_chunks: int | None l_value_is: float | None + reranking: bool | None = None + reranking_metric: str | None = None + quantized_fetch_limit: int | None = None maintenance_work_mem: str | None = None max_parallel_workers: int | None = None @@ -126,6 +137,8 @@ def index_param(self) -> dict: "options": { "max_neighbors": self.max_neighbors, "l_value_ib": self.l_value_ib, + "pq_param_num_chunks": self.pq_param_num_chunks, + "product_quantized": str(self.reranking), }, "maintenance_work_mem": self.maintenance_work_mem, "max_parallel_workers": self.max_parallel_workers, @@ -135,6 +148,9 @@ def search_param(self) -> dict: return { "metric": self.parse_metric(), "metric_fun_op": self.parse_metric_fun_op(), + "reranking": self.reranking, + "reranking_metric_fun_op": self.parse_reranking_metric_fun_op(), + "quantized_fetch_limit": self.quantized_fetch_limit, } def session_param(self) -> dict: diff --git a/vectordb_bench/backend/clients/pgdiskann/pgdiskann.py b/vectordb_bench/backend/clients/pgdiskann/pgdiskann.py index 8bede0f01..5f069ace5 100644 --- a/vectordb_bench/backend/clients/pgdiskann/pgdiskann.py +++ b/vectordb_bench/backend/clients/pgdiskann/pgdiskann.py @@ -90,38 +90,79 @@ def _create_connection(**kwargs) -> tuple[Connection, Cursor]: def init(self) -> Generator[None, None, None]: self.conn, self.cursor = self._create_connection(**self.db_config) - # index configuration may have commands defined that we should set during each client session session_options: dict[str, Any] = self.case_config.session_param() if len(session_options) > 0: for setting_name, setting_val in session_options.items(): - command = sql.SQL("SET {setting_name} " + "= {setting_val};").format( - setting_name=sql.Identifier(setting_name), - setting_val=sql.Identifier(str(setting_val)), + command = sql.SQL("SET {setting_name} = {setting_val};").format( + setting_name=sql.Identifier(setting_name), setting_val=sql.Literal(setting_val) ) log.debug(command.as_string(self.cursor)) self.cursor.execute(command) self.conn.commit() - self._filtered_search = sql.Composed( - [ - sql.SQL( - "SELECT id FROM public.{table_name} WHERE id >= %s ORDER BY embedding ", - ).format(table_name=sql.Identifier(self.table_name)), - sql.SQL(self.case_config.search_param()["metric_fun_op"]), - sql.SQL(" %s::vector LIMIT %s::int"), - ], - ) + search_params = self.case_config.search_param() + + if search_params.get("reranking"): + # Reranking-enabled queries + self._filtered_search = sql.SQL(""" + SELECT i.id + FROM ( + SELECT id, embedding + FROM public.{table_name} + WHERE id >= %s + ORDER BY embedding {metric_fun_op} %s::vector + LIMIT {quantized_fetch_limit}::int + ) i + ORDER BY i.embedding {reranking_metric_fun_op} %s::vector + LIMIT %s::int + """).format( + table_name=sql.Identifier(self.table_name), + metric_fun_op=sql.SQL(search_params["metric_fun_op"]), + reranking_metric_fun_op=sql.SQL(search_params["reranking_metric_fun_op"]), + quantized_fetch_limit=sql.Literal(search_params["quantized_fetch_limit"]), + ) - self._unfiltered_search = sql.Composed( - [ - sql.SQL("SELECT id FROM public.{} ORDER BY embedding ").format( - sql.Identifier(self.table_name), - ), - sql.SQL(self.case_config.search_param()["metric_fun_op"]), - sql.SQL(" %s::vector LIMIT %s::int"), - ], - ) + self._unfiltered_search = sql.SQL(""" + SELECT i.id + FROM ( + SELECT id, embedding + FROM public.{table_name} + ORDER BY embedding {metric_fun_op} %s::vector + LIMIT {quantized_fetch_limit}::int + ) i + ORDER BY i.embedding {reranking_metric_fun_op} %s::vector + LIMIT %s::int + """).format( + table_name=sql.Identifier(self.table_name), + metric_fun_op=sql.SQL(search_params["metric_fun_op"]), + reranking_metric_fun_op=sql.SQL(search_params["reranking_metric_fun_op"]), + quantized_fetch_limit=sql.Literal(search_params["quantized_fetch_limit"]), + ) + + else: + self._filtered_search = sql.Composed( + [ + sql.SQL( + "SELECT id FROM public.{table_name} WHERE id >= %s ORDER BY embedding ", + ).format(table_name=sql.Identifier(self.table_name)), + sql.SQL(search_params["metric_fun_op"]), + sql.SQL(" %s::vector LIMIT %s::int"), + ] + ) + + self._unfiltered_search = sql.Composed( + [ + sql.SQL("SELECT id FROM public.{table_name} ORDER BY embedding ").format( + table_name=sql.Identifier(self.table_name) + ), + sql.SQL(search_params["metric_fun_op"]), + sql.SQL(" %s::vector LIMIT %s::int"), + ] + ) + + log.debug(f"Unfiltered search query={self._unfiltered_search.as_string(self.conn)}") + log.debug(f"Filtered search query={self._filtered_search.as_string(self.conn)}") try: yield @@ -234,7 +275,7 @@ def _create_index(self): options.append( sql.SQL("{option_name} = {val}").format( option_name=sql.Identifier(option_name), - val=sql.Identifier(str(option_val)), + val=sql.Literal(option_val), ), ) @@ -314,16 +355,39 @@ def search_embedding( assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" + search_params = self.case_config.search_param() + is_reranking = search_params.get("reranking", False) + q = np.asarray(query) if filters: gt = filters.get("id") + if is_reranking: + result = self.cursor.execute( + self._filtered_search, + (gt, q, q, k), + prepare=True, + binary=True, + ) + else: + result = self.cursor.execute( + self._filtered_search, + (gt, q, k), + prepare=True, + binary=True, + ) + elif is_reranking: result = self.cursor.execute( - self._filtered_search, - (gt, q, k), + self._unfiltered_search, + (q, q, k), prepare=True, binary=True, ) else: - result = self.cursor.execute(self._unfiltered_search, (q, k), prepare=True, binary=True) + result = self.cursor.execute( + self._unfiltered_search, + (q, k), + prepare=True, + binary=True, + ) return [int(i[0]) for i in result.fetchall()] diff --git a/vectordb_bench/backend/clients/pgvector/config.py b/vectordb_bench/backend/clients/pgvector/config.py index abfddc0cf..98e82f1c2 100644 --- a/vectordb_bench/backend/clients/pgvector/config.py +++ b/vectordb_bench/backend/clients/pgvector/config.py @@ -21,21 +21,25 @@ class PgVectorConfigDict(TypedDict): class PgVectorConfig(DBConfig): - user_name: SecretStr = SecretStr("postgres") + user_name: SecretStr = "postgres" password: SecretStr host: str = "localhost" port: int = 5432 - db_name: str + db_name: str = "vectordb" + table_name: str = "vdbbench_table_test" def to_dict(self) -> PgVectorConfigDict: - user_str = self.user_name.get_secret_value() + user_str = self.user_name.get_secret_value() if isinstance(self.user_name, SecretStr) else self.user_name pwd_str = self.password.get_secret_value() return { - "host": self.host, - "port": self.port, - "dbname": self.db_name, - "user": user_str, - "password": pwd_str, + "connect_config": { + "host": self.host, + "port": self.port, + "dbname": self.db_name, + "user": user_str, + "password": pwd_str, + }, + "table_name": self.table_name, } @@ -59,6 +63,10 @@ class PgVectorIndexConfig(BaseModel, DBCaseConfig): metric_type: MetricType | None = None create_index_before_load: bool = False create_index_after_load: bool = True + # Scan more of the index to get enough results for filter-cases. + # Options: "strict_order" (order by distance), "relaxed_order" (slightly out of order but better recall) + # See: https://github.com/pgvector/pgvector?tab=readme-ov-file#iterative-index-scans + iterative_scan: str = "relaxed_order" def parse_metric(self) -> str: d = { @@ -205,7 +213,7 @@ def search_param(self) -> PgVectorSearchParam: } def session_param(self) -> PgVectorSessionCommands: - session_parameters = {"ivfflat.probes": self.probes} + session_parameters = {"ivfflat.probes": self.probes, "ivfflat.iterative_scan": self.iterative_scan} return {"session_options": self._optionally_build_set_options(session_parameters)} @@ -255,7 +263,7 @@ def search_param(self) -> PgVectorSearchParam: } def session_param(self) -> PgVectorSessionCommands: - session_parameters = {"hnsw.ef_search": self.ef_search} + session_parameters = {"hnsw.ef_search": self.ef_search, "hnsw.iterative_scan": self.iterative_scan} return {"session_options": self._optionally_build_set_options(session_parameters)} diff --git a/vectordb_bench/backend/clients/pgvector/pgvector.py b/vectordb_bench/backend/clients/pgvector/pgvector.py index 7d06a2ba5..42fa7533d 100644 --- a/vectordb_bench/backend/clients/pgvector/pgvector.py +++ b/vectordb_bench/backend/clients/pgvector/pgvector.py @@ -10,6 +10,8 @@ from pgvector.psycopg import register_vector from psycopg import Connection, Cursor, sql +from vectordb_bench.backend.filter import Filter, FilterOp + from ..api import VectorDB from .config import PgVectorConfigDict, PgVectorIndexConfig @@ -19,39 +21,46 @@ class PgVector(VectorDB): """Use psycopg instructions""" + supported_filter_types: list[FilterOp] = [ + FilterOp.NonFilter, + FilterOp.NumGE, + FilterOp.StrEqual, + ] + conn: psycopg.Connection[Any] | None = None cursor: psycopg.Cursor[Any] | None = None - _filtered_search: sql.Composed - _unfiltered_search: sql.Composed + _search: sql.Composed def __init__( self, dim: int, db_config: PgVectorConfigDict, db_case_config: PgVectorIndexConfig, - collection_name: str = "pg_vector_collection", drop_old: bool = False, + with_scalar_labels: bool = False, **kwargs, ): self.name = "PgVector" - self.db_config = db_config self.case_config = db_case_config - self.table_name = collection_name + self.table_name = db_config["table_name"] + self.connect_config = db_config["connect_config"] self.dim = dim + self.with_scalar_labels = with_scalar_labels self._index_name = "pgvector_index" self._primary_field = "id" self._vector_field = "embedding" + self._scalar_label_field = "label" # construct basic units - self.conn, self.cursor = self._create_connection(**self.db_config) + self.conn, self.cursor = self._create_connection(**self.connect_config) # create vector extension self.cursor.execute("CREATE EXTENSION IF NOT EXISTS vector") self.conn.commit() - log.info(f"{self.name} config values: {self.db_config}\n{self.case_config}") + log.info(f"{self.name} config values: {self.connect_config}\n{self.case_config}") if not any( ( self.case_config.create_index_before_load, @@ -60,7 +69,7 @@ def __init__( ): msg = ( f"{self.name} config must create an index using create_index_before_load or create_index_after_load" - f"{self.name} config values: {self.db_config}\n{self.case_config}" + f"{self.name} config values: {self.connect_config}\n{self.case_config}" ) log.error(msg) raise RuntimeError(msg) @@ -89,13 +98,13 @@ def _create_connection(**kwargs) -> tuple[Connection, Cursor]: return conn, cursor - def _generate_search_query(self, filtered: bool = False) -> sql.Composed: + def _generate_search_query(self) -> sql.Composed: index_param = self.case_config.index_param() reranking = self.case_config.search_param()["reranking"] column_name = ( - sql.SQL("binary_quantize({0})").format(sql.Identifier("embedding")) + sql.SQL("binary_quantize({0})").format(sql.Identifier(self._vector_field)) if index_param["quantization_type"] == "bit" and index_param["table_quantization_type"] != "bit" - else sql.SQL("embedding") + else sql.SQL(self._vector_field) ) search_vector = ( sql.SQL("binary_quantize({0})").format(sql.Placeholder()) @@ -114,12 +123,14 @@ def _generate_search_query(self, filtered: bool = False) -> sql.Composed: """ SELECT i.id FROM ( - SELECT id, embedding {reranking_metric_fun_op} %s::{table_quantization_type} AS distance + SELECT {primary_field}, {vector_field} {reranking_metric_fun_op} %s::{table_quantization_type} AS distance FROM public.{table_name} {where_clause} ORDER BY {column_name}::{quantization_type}({dim}) - """, + """, # noqa: E501 ).format( table_name=sql.Identifier(self.table_name), + primary_field=sql.Identifier(self._primary_field), + vector_field=sql.Identifier(self._vector_field), column_name=column_name, reranking_metric_fun_op=sql.SQL( self.case_config.search_param()["reranking_metric_fun_op"], @@ -128,7 +139,7 @@ def _generate_search_query(self, filtered: bool = False) -> sql.Composed: table_quantization_type=sql.SQL(index_param["table_quantization_type"]), quantization_type=sql.SQL(index_param["quantization_type"]), dim=sql.Literal(self.dim), - where_clause=sql.SQL("WHERE id >= %s") if filtered else sql.SQL(""), + where_clause=sql.SQL(self.where_clause), ), sql.SQL(self.case_config.search_param()["metric_fun_op"]), sql.SQL( @@ -154,15 +165,16 @@ def _generate_search_query(self, filtered: bool = False) -> sql.Composed: [ sql.SQL( """ - SELECT id FROM public.{table_name} + SELECT {primary_field} FROM public.{table_name} {where_clause} ORDER BY {column_name}::{quantization_type}({dim}) """, ).format( table_name=sql.Identifier(self.table_name), + primary_field=sql.Identifier(self._primary_field), column_name=column_name, quantization_type=sql.SQL(index_param["quantization_type"]), dim=sql.Literal(self.dim), - where_clause=sql.SQL("WHERE id >= %s") if filtered else sql.SQL(""), + where_clause=sql.SQL(self.where_clause), ), sql.SQL(self.case_config.search_param()["metric_fun_op"]), sql.SQL(" {search_vector}::{quantization_type}({dim}) LIMIT %s::int").format( @@ -176,10 +188,12 @@ def _generate_search_query(self, filtered: bool = False) -> sql.Composed: search_query = sql.Composed( [ sql.SQL( - "SELECT id FROM public.{table_name} {where_clause} ORDER BY embedding ", + "SELECT {primary_field} FROM public.{table_name} {where_clause} ORDER BY {vector_field}", ).format( table_name=sql.Identifier(self.table_name), - where_clause=sql.SQL("WHERE id >= %s") if filtered else sql.SQL(""), + primary_field=sql.Identifier(self._primary_field), + vector_field=sql.Identifier(self._vector_field), + where_clause=sql.SQL(self.where_clause), ), sql.SQL(self.case_config.search_param()["metric_fun_op"]), sql.SQL(" {search_vector}::{quantization_type}({dim}) LIMIT %s::int").format( @@ -201,7 +215,7 @@ def init(self) -> Generator[None, None, None]: >>> self.search_embedding() """ - self.conn, self.cursor = self._create_connection(**self.db_config) + self.conn, self.cursor = self._create_connection(**self.connect_config) # index configuration may have commands defined that we should set during each client session session_options: Sequence[dict[str, Any]] = self.case_config.session_param()["session_options"] @@ -216,9 +230,6 @@ def init(self) -> Generator[None, None, None]: self.cursor.execute(command) self.conn.commit() - self._filtered_search = self._generate_search_query(filtered=True) - self._unfiltered_search = self._generate_search_query() - try: yield finally: @@ -274,7 +285,7 @@ def _set_parallel_index_build_param(self): ) self.cursor.execute( sql.SQL("ALTER USER {} SET maintenance_work_mem TO {};").format( - sql.Identifier(self.db_config["user"]), + sql.Identifier(self.connect_config["user"]), index_param["maintenance_work_mem"], ), ) @@ -288,7 +299,7 @@ def _set_parallel_index_build_param(self): ) self.cursor.execute( sql.SQL("ALTER USER {} SET max_parallel_maintenance_workers TO '{}';").format( - sql.Identifier(self.db_config["user"]), + sql.Identifier(self.connect_config["user"]), index_param["max_parallel_workers"], ), ) @@ -299,7 +310,7 @@ def _set_parallel_index_build_param(self): ) self.cursor.execute( sql.SQL("ALTER USER {} SET max_parallel_workers TO '{}';").format( - sql.Identifier(self.db_config["user"]), + sql.Identifier(self.connect_config["user"]), index_param["max_parallel_workers"], ), ) @@ -382,18 +393,34 @@ def _create_table(self, dim: int): log.info(f"{self.name} client create table : {self.table_name}") # create table - self.cursor.execute( - sql.SQL( - """ - CREATE TABLE IF NOT EXISTS public.{table_name} - (id BIGINT PRIMARY KEY, embedding {table_quantization_type}({dim})); - """ - ).format( - table_name=sql.Identifier(self.table_name), - table_quantization_type=sql.SQL(index_param["table_quantization_type"]), - dim=dim, + if self.with_scalar_labels: + self.cursor.execute( + sql.SQL( + """ + CREATE TABLE IF NOT EXISTS public.{table_name} + ({primary_field} BIGINT PRIMARY KEY, embedding {table_quantization_type}({dim}), {label_field} VARCHAR(64)); + """, # noqa: E501 + ).format( + table_name=sql.Identifier(self.table_name), + table_quantization_type=sql.SQL(index_param["table_quantization_type"]), + dim=dim, + primary_field=sql.Identifier(self._primary_field), + label_field=sql.Identifier(self._scalar_label_field), + ) ) - ) + else: + self.cursor.execute( + sql.SQL(""" + CREATE TABLE IF NOT EXISTS public.{table_name} + ({primary_field} BIGINT PRIMARY KEY, embedding {table_quantization_type}({dim})); + """).format( + table_name=sql.Identifier(self.table_name), + table_quantization_type=sql.SQL(index_param["table_quantization_type"]), + dim=dim, + primary_field=sql.Identifier(self._primary_field), + ) + ) + self.cursor.execute( sql.SQL( "ALTER TABLE public.{table_name} ALTER COLUMN embedding SET STORAGE PLAIN;", @@ -404,14 +431,17 @@ def _create_table(self, dim: int): log.warning(f"Failed to create pgvector table: {self.table_name} error: {e}") raise e from None - def insert_embeddings( + def insert_embeddings( # noqa: PLR0912 self, embeddings: list[list[float]], metadata: list[int], + labels_data: list[str] | None = None, **kwargs: Any, ) -> tuple[int, Exception | None]: assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" + if self.with_scalar_labels: + assert labels_data is not None, "labels_data should be provided if with_scalar_labels is set to True" index_param = self.case_config.index_param() @@ -433,7 +463,10 @@ def insert_embeddings( embeddings_bit += "1" else: embeddings_bit += "0" - copy.write_row((str(row), embeddings_bit)) + if self.with_scalar_labels: + copy.write_row((str(row), embeddings_bit, labels_data[i])) + else: + copy.write_row((str(row), embeddings_bit)) else: with self.cursor.copy( sql.SQL("COPY public.{table_name} FROM STDIN (FORMAT BINARY)").format( @@ -441,29 +474,47 @@ def insert_embeddings( ) ) as copy: if index_param["table_quantization_type"] == "halfvec": - copy.set_types(["bigint", "halfvec"]) for i, row in enumerate(metadata_arr): - copy.write_row((row, np.float16(embeddings_arr[i]))) + if self.with_scalar_labels: + copy.set_types(["bigint", "halfvec", "varchar"]) + copy.write_row((row, np.float16(embeddings_arr[i]), labels_data[i])) + else: + copy.set_types(["bigint", "halfvec"]) + copy.write_row((row, np.float16(embeddings_arr[i]))) else: - copy.set_types(["bigint", "vector"]) for i, row in enumerate(metadata_arr): - copy.write_row((row, embeddings_arr[i])) + if self.with_scalar_labels: + copy.set_types(["bigint", "vector", "varchar"]) + copy.write_row((row, embeddings_arr[i], labels_data[i])) + else: + copy.set_types(["bigint", "vector"]) + copy.write_row((row, embeddings_arr[i])) self.conn.commit() - if kwargs.get("last_batch"): - self._post_insert() - return len(metadata), None except Exception as e: log.warning(f"Failed to insert data into pgvector table ({self.table_name}), error: {e}") return 0, e + def prepare_filter(self, filters: Filter): + if filters.type == FilterOp.NonFilter: + self.where_clause = "" + elif filters.type == FilterOp.NumGE: + self.where_clause = f"WHERE {self._primary_field} >= {filters.int_value}" + elif filters.type == FilterOp.StrEqual: + self.where_clause = f"WHERE {self._scalar_label_field} = '{filters.label_value}'" + else: + msg = f"Not support Filter for PgVector - {filters}" + raise ValueError(msg) + + self._search = self._generate_search_query() + def search_embedding( self, query: list[float], k: int = 100, - filters: dict | None = None, timeout: int | None = None, + **kwargs: Any, ) -> list[int]: assert self.conn is not None, "Connection is not initialized" assert self.cursor is not None, "Cursor is not initialized" @@ -471,36 +522,10 @@ def search_embedding( index_param = self.case_config.index_param() search_param = self.case_config.search_param() q = np.asarray(query) - if filters: - gt = filters.get("id") - if index_param["quantization_type"] == "bit" and search_param["reranking"]: - result = self.cursor.execute( - self._filtered_search, - (q, gt, q, k), - prepare=True, - binary=True, - ) - else: - result = self.cursor.execute( - self._filtered_search, - (gt, q, k), - prepare=True, - binary=True, - ) - - elif index_param["quantization_type"] == "bit" and search_param["reranking"]: - result = self.cursor.execute( - self._unfiltered_search, - (q, q, k), - prepare=True, - binary=True, - ) - else: - result = self.cursor.execute( - self._unfiltered_search, - (q, k), - prepare=True, - binary=True, - ) - + result = self.cursor.execute( + self._search, + (q, q, k) if index_param["quantization_type"] == "bit" and search_param["reranking"] else (q, k), + prepare=True, + binary=True, + ) return [int(i[0]) for i in result.fetchall()] diff --git a/vectordb_bench/backend/clients/pinecone/pinecone.py b/vectordb_bench/backend/clients/pinecone/pinecone.py index 9fd0afc7c..9c2b38888 100644 --- a/vectordb_bench/backend/clients/pinecone/pinecone.py +++ b/vectordb_bench/backend/clients/pinecone/pinecone.py @@ -5,8 +5,9 @@ import pinecone -from ..api import DBCaseConfig, DBConfig, EmptyDBCaseConfig, IndexType, VectorDB -from .config import PineconeConfig +from vectordb_bench.backend.filter import Filter, FilterOp + +from ..api import DBCaseConfig, VectorDB log = logging.getLogger(__name__) @@ -15,12 +16,19 @@ class Pinecone(VectorDB): + supported_filter_types: list[FilterOp] = [ + FilterOp.NonFilter, + FilterOp.NumGE, + FilterOp.StrEqual, + ] + def __init__( self, dim: int, db_config: dict, db_case_config: DBCaseConfig, drop_old: bool = False, + with_scalar_labels: bool = False, **kwargs, ): """Initialize wrapper around the milvus vector database.""" @@ -33,6 +41,7 @@ def __init__( pc = pinecone.Pinecone(api_key=self.api_key) index = pc.Index(self.index_name) + self.with_scalar_labels = with_scalar_labels if drop_old: index_stats = index.describe_index_stats() index_dim = index_stats["dimension"] @@ -43,15 +52,8 @@ def __init__( log.info(f"Pinecone index delete namespace: {namespace}") index.delete(delete_all=True, namespace=namespace) - self._metadata_key = "meta" - - @classmethod - def config_cls(cls) -> type[DBConfig]: - return PineconeConfig - - @classmethod - def case_config_cls(cls, index_type: IndexType | None = None) -> type[DBCaseConfig]: - return EmptyDBCaseConfig + self._scalar_id_field = "meta" + self._scalar_label_field = "label" @contextmanager def init(self): @@ -66,6 +68,7 @@ def insert_embeddings( self, embeddings: list[list[float]], metadata: list[int], + labels_data: list[str] | None = None, **kwargs, ) -> tuple[int, Exception]: assert len(embeddings) == len(metadata) @@ -75,33 +78,44 @@ def insert_embeddings( batch_end_offset = min(batch_start_offset + self.batch_size, len(embeddings)) insert_datas = [] for i in range(batch_start_offset, batch_end_offset): + metadata_dict = {self._scalar_id_field: metadata[i]} + if self.with_scalar_labels: + metadata_dict[self._scalar_label_field] = labels_data[i] insert_data = ( str(metadata[i]), embeddings[i], - {self._metadata_key: metadata[i]}, + metadata_dict, ) insert_datas.append(insert_data) self.index.upsert(insert_datas) insert_count += batch_end_offset - batch_start_offset except Exception as e: - return (insert_count, e) - return (len(embeddings), None) + return insert_count, e + return len(embeddings), None def search_embedding( self, query: list[float], k: int = 100, - filters: dict | None = None, timeout: int | None = None, ) -> list[int]: - pinecone_filters = {} if filters is None else {self._metadata_key: {"$gte": filters["id"]}} - try: - res = self.index.query( - top_k=k, - vector=query, - filter=pinecone_filters, - )["matches"] - except Exception as e: - log.warning(f"Error querying index: {e}") - raise e from e + pinecone_filters = self.expr + res = self.index.query( + top_k=k, + vector=query, + filter=pinecone_filters, + )["matches"] return [int(one_res["id"]) for one_res in res] + + def prepare_filter(self, filters: Filter): + if filters.type == FilterOp.NonFilter: + self.expr = None + elif filters.type == FilterOp.NumGE: + self.expr = {self._scalar_id_field: {"$gte": filters.int_value}} + elif filters.type == FilterOp.StrEqual: + # both "in" and "==" are supported + # for example, self.expr = {self._scalar_label_field: {"$in": [filters.label_value]}} + self.expr = {self._scalar_label_field: {"$eq": filters.label_value}} + else: + msg = f"Not support Filter for Pinecone - {filters}" + raise ValueError(msg) diff --git a/vectordb_bench/backend/clients/qdrant_cloud/cli.py b/vectordb_bench/backend/clients/qdrant_cloud/cli.py new file mode 100644 index 000000000..32353472b --- /dev/null +++ b/vectordb_bench/backend/clients/qdrant_cloud/cli.py @@ -0,0 +1,43 @@ +from typing import Annotated, Unpack + +import click +from pydantic import SecretStr + +from ....cli.cli import ( + CommonTypedDict, + cli, + click_parameter_decorators_from_typed_dict, + run, +) +from .. import DB + + +class QdrantTypedDict(CommonTypedDict): + url: Annotated[ + str, + click.option("--url", type=str, help="URL connection string", required=True), + ] + api_key: Annotated[ + str | None, + click.option("--api-key", type=str, help="API key for authentication", required=False), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(QdrantTypedDict) +def QdrantCloud(**parameters: Unpack[QdrantTypedDict]): + from .config import QdrantConfig, QdrantIndexConfig + + config_params = { + "db_label": parameters["db_label"], + "url": SecretStr(parameters["url"]), + } + + config_params["api_key"] = SecretStr(parameters["api_key"]) if parameters["api_key"] else None + + run( + db=DB.QdrantCloud, + db_config=QdrantConfig(**config_params), + db_case_config=QdrantIndexConfig(), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/qdrant_cloud/config.py b/vectordb_bench/backend/clients/qdrant_cloud/config.py index d4e27cb3c..b2eeb2ce6 100644 --- a/vectordb_bench/backend/clients/qdrant_cloud/config.py +++ b/vectordb_bench/backend/clients/qdrant_cloud/config.py @@ -1,28 +1,81 @@ -from pydantic import BaseModel, SecretStr +from typing import TypeVar + +from pydantic import BaseModel, SecretStr, validator from ..api import DBCaseConfig, DBConfig, MetricType +# define type "SearchParams" +SearchParams = TypeVar("SearchParams") + # Allowing `api_key` to be left empty, to ensure compatibility with the open-source Qdrant. class QdrantConfig(DBConfig): url: SecretStr - api_key: SecretStr + api_key: SecretStr | None = None def to_dict(self) -> dict: - api_key = self.api_key.get_secret_value() - if len(api_key) > 0: + api_key_value = self.api_key.get_secret_value() if self.api_key else None + if api_key_value: return { "url": self.url.get_secret_value(), - "api_key": self.api_key.get_secret_value(), + "api_key": api_key_value, "prefer_grpc": True, } return { "url": self.url.get_secret_value(), } + @validator("*") + def not_empty_field(cls, v: any, field: any): + if field.name in ["api_key"]: + return v + return super().not_empty_field(v, field) + class QdrantIndexConfig(BaseModel, DBCaseConfig): metric_type: MetricType | None = None + m: int = 16 + payload_m: int = 16 # only for label_filter cases + create_payload_int_index: bool = False + create_payload_keyword_index: bool = False + is_tenant: bool = False + use_scalar_quant: bool = False + sq_quantile: float = 0.99 + default_segment_number: int = 0 + + use_rescore: bool = False + oversampling: float = 1.0 + indexed_only: bool = False + hnsw_ef: int | None = 100 + exact: bool = False + + with_payload: bool = False + + def __eq__(self, obj: any): + return ( + self.m == obj.m + and self.payload_m == obj.payload_m + and self.create_payload_int_index == obj.create_payload_int_index + and self.create_payload_keyword_index == obj.create_payload_keyword_index + and self.is_tenant == obj.is_tenant + and self.use_scalar_quant == obj.use_scalar_quant + and self.sq_quantile == obj.sq_quantile + and self.default_segment_number == obj.default_segment_number + ) + + def __hash__(self) -> int: + return hash( + ( + self.m, + self.payload_m, + self.create_payload_int_index, + self.create_payload_keyword_index, + self.is_tenant, + self.use_scalar_quant, + self.sq_quantile, + self.default_segment_number, + ) + ) def parse_metric(self) -> str: if self.metric_type == MetricType.L2: @@ -36,5 +89,22 @@ def parse_metric(self) -> str: def index_param(self) -> dict: return {"distance": self.parse_metric()} - def search_param(self) -> dict: - return {} + def search_param(self) -> SearchParams: + # Import while in use + from qdrant_client.http.models import QuantizationSearchParams, SearchParams + + quantization = ( + QuantizationSearchParams( + ignore=False, + rescore=True, + oversampling=self.oversampling, + ) + if self.use_rescore + else None + ) + return SearchParams( + hnsw_ef=self.hnsw_ef, + exact=self.exact, + indexed_only=self.indexed_only, + quantization=quantization, + ) diff --git a/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py b/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py index f618c3ba4..4eeda649d 100644 --- a/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py +++ b/vectordb_bench/backend/clients/qdrant_cloud/qdrant_cloud.py @@ -9,13 +9,24 @@ Batch, CollectionStatus, FieldCondition, - Filter, + HnswConfigDiff, + KeywordIndexParams, + OptimizersConfigDiff, PayloadSchemaType, Range, + ScalarQuantization, + ScalarQuantizationConfig, + ScalarType, VectorParams, ) +from qdrant_client.http.models import ( + Filter as QdrantFilter, +) -from ..api import DBCaseConfig, VectorDB +from vectordb_bench.backend.clients.qdrant_cloud.config import QdrantIndexConfig +from vectordb_bench.backend.filter import Filter, FilterOp + +from ..api import VectorDB log = logging.getLogger(__name__) @@ -25,24 +36,33 @@ class QdrantCloud(VectorDB): + supported_filter_types: list[FilterOp] = [ + FilterOp.NonFilter, + FilterOp.NumGE, + FilterOp.StrEqual, + ] + def __init__( self, dim: int, db_config: dict, - db_case_config: DBCaseConfig, + db_case_config: QdrantIndexConfig, collection_name: str = "QdrantCloudCollection", drop_old: bool = False, + with_scalar_labels: bool = False, **kwargs, ): """Initialize wrapper around the QdrantCloud vector database.""" self.db_config = db_config - self.case_config = db_case_config + self.db_case_config = db_case_config self.collection_name = collection_name self._primary_field = "pk" + self._scalar_label_field = "label" self._vector_field = "vector" tmp_client = QdrantClient(**self.db_config) + self.with_scalar_labels = with_scalar_labels if drop_old: log.info(f"QdrantCloud client drop_old collection: {self.collection_name}") tmp_client.delete_collection(self.collection_name) @@ -50,7 +70,7 @@ def __init__( tmp_client = None @contextmanager - def init(self) -> None: + def init(self): """ Examples: >>> with self.init(): @@ -74,7 +94,7 @@ def optimize(self, data_size: int | None = None): if info.status == CollectionStatus.GREEN: msg = ( f"Stored vectors: {info.vectors_count}, Indexed vectors: {info.indexed_vectors_count}, " - f"Collection status: {info.indexed_vectors_count}" + f"Collection status: {info.status}, Segment counts: {info.segments_count}" ) log.info(msg) return @@ -86,19 +106,48 @@ def _create_collection(self, dim: int, qdrant_client: QdrantClient): log.info(f"Create collection: {self.collection_name}") try: + # whether to use quant (SQ8) + quantization_config = None + if self.db_case_config.use_scalar_quant: + quantization_config = ScalarQuantization( + scalar=ScalarQuantizationConfig( + type=ScalarType.INT8, + quantile=self.db_case_config.sq_quantile, + always_ram=True, + ) + ) + + # create collection qdrant_client.create_collection( collection_name=self.collection_name, vectors_config=VectorParams( size=dim, - distance=self.case_config.index_param()["distance"], + distance=self.db_case_config.parse_metric(), + ), + hnsw_config=HnswConfigDiff(m=self.db_case_config.m, payload_m=self.db_case_config.payload_m), + optimizers_config=OptimizersConfigDiff( + default_segment_number=self.db_case_config.default_segment_number ), + quantization_config=quantization_config, ) - qdrant_client.create_payload_index( - collection_name=self.collection_name, - field_name=self._primary_field, - field_schema=PayloadSchemaType.INTEGER, - ) + # create payload_index for int-field + if self.db_case_config.create_payload_int_index: + qdrant_client.create_payload_index( + collection_name=self.collection_name, + field_name=self._primary_field, + field_schema=PayloadSchemaType.INTEGER, + ) + + # create payload_index for str-field + if self.with_scalar_labels and self.db_case_config.create_payload_keyword_index: + qdrant_client.create_payload_index( + collection_name=self.collection_name, + field_name=self._scalar_label_field, + field_schema=KeywordIndexParams( + type=PayloadSchemaType.KEYWORD, is_tenant=self.db_case_config.is_tenant + ), + ) except Exception as e: if "already exists!" in str(e): @@ -110,16 +159,22 @@ def insert_embeddings( self, embeddings: list[list[float]], metadata: list[int], + labels_data: list[str] | None = None, **kwargs, ) -> tuple[int, Exception]: """Insert embeddings into Milvus. should call self.init() first""" assert self.qdrant_client is not None try: - # TODO: counts for offset in range(0, len(embeddings), QDRANT_BATCH_SIZE): vectors = embeddings[offset : offset + QDRANT_BATCH_SIZE] ids = metadata[offset : offset + QDRANT_BATCH_SIZE] - payloads = [{self._primary_field: v} for v in ids] + if self.with_scalar_labels: + labels = labels_data[offset : offset + QDRANT_BATCH_SIZE] + payloads = [ + {self._primary_field: pk, self._scalar_label_field: labels[i]} for i, pk in enumerate(ids) + ] + else: + payloads = [{self._primary_field: pk} for i, pk in enumerate(ids)] _ = self.qdrant_client.upsert( collection_name=self.collection_name, wait=True, @@ -135,34 +190,46 @@ def search_embedding( self, query: list[float], k: int = 100, - filters: dict | None = None, timeout: int | None = None, + **kwargs, ) -> list[int]: """Perform a search on a query embedding and return results with score. Should call self.init() first. """ assert self.qdrant_client is not None - f = None - if filters: - f = Filter( + res = self.qdrant_client.search( + collection_name=self.collection_name, + query_vector=query, + limit=k, + query_filter=self.query_filter, + search_params=self.db_case_config.search_param(), + with_payload=self.db_case_config.with_payload, + ) + + return [r.id for r in res] + + def prepare_filter(self, filters: Filter): + if filters.type == FilterOp.NonFilter: + self.query_filter = None + elif filters.type == FilterOp.NumGE: + self.query_filter = QdrantFilter( must=[ FieldCondition( key=self._primary_field, - range=Range( - gt=filters.get("id"), - ), + range=Range(gte=filters.int_value), ), - ], + ] ) - - res = ( - self.qdrant_client.search( - collection_name=self.collection_name, - query_vector=query, - limit=k, - query_filter=f, - ), - ) - - return [result.id for result in res[0]] + elif filters.type == FilterOp.StrEqual: + self.query_filter = QdrantFilter( + must=[ + FieldCondition( + key=self._scalar_label_field, + match={"value": filters.label_value}, + ), + ] + ) + else: + msg = f"Not support Filter for Qdrant - {filters}" + raise ValueError(msg) diff --git a/vectordb_bench/backend/clients/qdrant_local/cli.py b/vectordb_bench/backend/clients/qdrant_local/cli.py new file mode 100644 index 000000000..7995b99b3 --- /dev/null +++ b/vectordb_bench/backend/clients/qdrant_local/cli.py @@ -0,0 +1,60 @@ +from typing import Annotated, Unpack + +import click +from pydantic import SecretStr + +from vectordb_bench.backend.clients import DB +from vectordb_bench.cli.cli import ( + CommonTypedDict, + cli, + click_parameter_decorators_from_typed_dict, + run, +) + +DBTYPE = DB.QdrantLocal + + +class QdrantLocalTypedDict(CommonTypedDict): + url: Annotated[ + str, + click.option("--url", type=str, help="Qdrant url", required=True), + ] + on_disk: Annotated[ + bool, + click.option("--on-disk", type=bool, default=False, help="Store the vectors and the HNSW index on disk"), + ] + m: Annotated[ + int, + click.option("--m", type=int, default=16, help="HNSW index parameter m, set 0 to disable the index"), + ] + ef_construct: Annotated[ + int, + click.option("--ef-construct", type=int, default=200, help="HNSW index parameter ef_construct"), + ] + hnsw_ef: Annotated[ + int, + click.option( + "--hnsw-ef", + type=int, + default=0, + help="HNSW index parameter hnsw_ef, set 0 to use ef_construct for search", + ), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(QdrantLocalTypedDict) +def QdrantLocal(**parameters: Unpack[QdrantLocalTypedDict]): + from .config import QdrantLocalConfig, QdrantLocalIndexConfig + + run( + db=DBTYPE, + db_config=QdrantLocalConfig(url=SecretStr(parameters["url"])), + db_case_config=QdrantLocalIndexConfig( + on_disk=parameters["on_disk"], + m=parameters["m"], + ef_construct=parameters["ef_construct"], + hnsw_ef=parameters["hnsw_ef"], + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/qdrant_local/config.py b/vectordb_bench/backend/clients/qdrant_local/config.py new file mode 100644 index 000000000..ebdf99dc4 --- /dev/null +++ b/vectordb_bench/backend/clients/qdrant_local/config.py @@ -0,0 +1,47 @@ +from pydantic import BaseModel, SecretStr + +from ..api import DBCaseConfig, DBConfig, MetricType + + +class QdrantLocalConfig(DBConfig): + url: SecretStr + + def to_dict(self) -> dict: + return { + "url": self.url.get_secret_value(), + } + + +class QdrantLocalIndexConfig(BaseModel, DBCaseConfig): + metric_type: MetricType | None = None + m: int + ef_construct: int + hnsw_ef: int | None = 0 + on_disk: bool | None = False + + def parse_metric(self) -> str: + if self.metric_type == MetricType.L2: + return "Euclid" + + if self.metric_type == MetricType.IP: + return "Dot" + + return "Cosine" + + def index_param(self) -> dict: + return { + "distance": self.parse_metric(), + "m": self.m, + "ef_construct": self.ef_construct, + "on_disk": self.on_disk, + } + + def search_param(self) -> dict: + search_params = { + "exact": False, # Force to use ANNs + } + + if self.hnsw_ef != 0: + search_params["hnsw_ef"] = self.hnsw_ef + + return search_params diff --git a/vectordb_bench/backend/clients/qdrant_local/qdrant_local.py b/vectordb_bench/backend/clients/qdrant_local/qdrant_local.py new file mode 100644 index 000000000..1340be614 --- /dev/null +++ b/vectordb_bench/backend/clients/qdrant_local/qdrant_local.py @@ -0,0 +1,232 @@ +"""Wrapper around the Qdrant over VectorDB""" + +import logging +import time +from collections.abc import Iterable +from contextlib import contextmanager + +from qdrant_client import QdrantClient +from qdrant_client.http.models import ( + Batch, + CollectionStatus, + FieldCondition, + Filter, + HnswConfigDiff, + OptimizersConfigDiff, + PayloadSchemaType, + Range, + SearchParams, + VectorParams, +) + +from ..api import VectorDB +from .config import QdrantLocalIndexConfig + +log = logging.getLogger(__name__) + +SECONDS_WAITING_FOR_INDEXING_API_CALL = 5 +QDRANT_BATCH_SIZE = 100 + + +def qdrant_collection_exists(client: QdrantClient, collection_name: str) -> bool: + collection_exists = True + + try: + client.get_collection(collection_name) + except Exception: + collection_exists = False + + return collection_exists + + +class QdrantLocal(VectorDB): + def __init__( + self, + dim: int, + db_config: dict, + db_case_config: QdrantLocalIndexConfig, + collection_name: str = "QdrantLocalCollection", + drop_old: bool = False, + name: str = "QdrantLocal", + **kwargs, + ): + """Initialize wrapper around the qdrant.""" + self.name = name + self.db_config = db_config + self.case_config = db_case_config + self.search_parameter = self.case_config.search_param() + self.collection_name = collection_name + self.client = None + + self._primary_field = "pk" + self._vector_field = "vector" + + client = QdrantClient(**self.db_config) + + # Lets just print the parameters here for double check + log.info(f"Case config: {self.case_config.index_param()}") + log.info(f"Search parameter: {self.search_parameter}") + + if drop_old and qdrant_collection_exists(client, self.collection_name): + log.info(f"{self.name} client drop_old collection: {self.collection_name}") + client.delete_collection(self.collection_name) + + if not qdrant_collection_exists(client, self.collection_name): + log.info(f"{self.name} create collection: {self.collection_name}") + self._create_collection(dim, client) + + client = None + + @contextmanager + def init(self): + """ + Examples: + >>> with self.init(): + >>> self.insert_embeddings() + >>> self.search_embedding() + """ + # create connection + self.client = QdrantClient(**self.db_config) + yield + self.client = None + del self.client + + def _create_collection(self, dim: int, qdrant_client: QdrantClient): + log.info(f"Create collection: {self.collection_name}") + log.info( + f"Index parameters: m={self.case_config.index_param()['m']}, " + f"ef_construct={self.case_config.index_param()['ef_construct']}, " + f"on_disk={self.case_config.index_param()['on_disk']}" + ) + + # If the on_disk is true, we enable both on disk index and vectors. + try: + qdrant_client.create_collection( + collection_name=self.collection_name, + vectors_config=VectorParams( + size=dim, + distance=self.case_config.index_param()["distance"], + on_disk=self.case_config.index_param()["on_disk"], + ), + hnsw_config=HnswConfigDiff( + m=self.case_config.index_param()["m"], + ef_construct=self.case_config.index_param()["ef_construct"], + on_disk=self.case_config.index_param()["on_disk"], + ), + ) + + qdrant_client.create_payload_index( + collection_name=self.collection_name, + field_name=self._primary_field, + field_schema=PayloadSchemaType.INTEGER, + ) + + except Exception as e: + if "already exists!" in str(e): + return + log.warning(f"Failed to create collection: {self.collection_name} error: {e}") + raise e from None + + def optimize(self, data_size: int | None = None): + assert self.client, "Please call self.init() before" + # wait for vectors to be fully indexed + try: + while True: + info = self.client.get_collection(self.collection_name) + time.sleep(SECONDS_WAITING_FOR_INDEXING_API_CALL) + if info.status != CollectionStatus.GREEN: + continue + if info.status == CollectionStatus.GREEN: + log.info(f"Finishing building index for collection: {self.collection_name}") + msg = ( + f"Stored vectors: {info.vectors_count}, Indexed vectors: {info.indexed_vectors_count}, " + f"Collection status: {info.indexed_vectors_count}" + ) + log.info(msg) + return + + except Exception as e: + log.warning(f"QdrantCloud ready to search error: {e}") + raise e from None + + def insert_embeddings( + self, + embeddings: Iterable[list[float]], + metadata: list[int], + **kwargs, + ) -> tuple[int, Exception]: + """Insert embeddings into the database. + + Args: + embeddings(list[list[float]]): list of embeddings + metadata(list[int]): list of metadata + kwargs: other arguments + + Returns: + tuple[int, Exception]: number of embeddings inserted and exception if any + """ + assert self.client is not None + assert len(embeddings) == len(metadata) + insert_count = 0 + + # disable indexing for quick insertion + self.client.update_collection( + collection_name=self.collection_name, + optimizer_config=OptimizersConfigDiff(indexing_threshold=0), + ) + try: + for offset in range(0, len(embeddings), QDRANT_BATCH_SIZE): + vectors = embeddings[offset : offset + QDRANT_BATCH_SIZE] + ids = metadata[offset : offset + QDRANT_BATCH_SIZE] + payloads = [{self._primary_field: v} for v in ids] + _ = self.client.upsert( + collection_name=self.collection_name, + wait=True, + points=Batch(ids=ids, payloads=payloads, vectors=vectors), + ) + insert_count += QDRANT_BATCH_SIZE + # enable indexing after insertion + self.client.update_collection( + collection_name=self.collection_name, + optimizer_config=OptimizersConfigDiff(indexing_threshold=100), + ) + + except Exception as e: + log.info(f"Failed to insert data, {e}") + return insert_count, e + else: + return insert_count, None + + def search_embedding( + self, + query: list[float], + k: int = 100, + filters: dict | None = None, + timeout: int | None = None, + ) -> list[int]: + """Perform a search on a query embedding and return results with score. + Should call self.init() first. + """ + assert self.client is not None + + f = None + if filters: + f = Filter( + must=[ + FieldCondition( + key=self._primary_field, + range=Range( + gt=filters.get("id"), + ), + ), + ], + ) + res = self.client.query_points( + collection_name=self.collection_name, + query=query, + limit=k, + query_filter=f, + search_params=SearchParams(**self.search_parameter), + ).points + + return [result.id for result in res] diff --git a/vectordb_bench/backend/clients/s3_vectors/cli.py b/vectordb_bench/backend/clients/s3_vectors/cli.py new file mode 100644 index 000000000..ff362f7c1 --- /dev/null +++ b/vectordb_bench/backend/clients/s3_vectors/cli.py @@ -0,0 +1,67 @@ +from typing import Annotated, TypedDict, Unpack + +import click +from pydantic import SecretStr + +from ....cli.cli import ( + CommonTypedDict, + cli, + click_parameter_decorators_from_typed_dict, + get_custom_case_config, + run, +) +from .. import DB +from ..api import MetricType +from .config import S3VectorsIndexConfig + + +class S3VectorsTypedDict(TypedDict): + region_name: Annotated[ + str, click.option("--region", type=str, help="AWS region for S3 bucket (eg. us-east-1)", default="us-east-1") + ] + access_key_id: Annotated[str, click.option("--access_key_id", type=str, help="AWS access key ID", required=True)] + secret_access_key: Annotated[ + str, click.option("--secret_access_key", type=str, help="AWS secret access key", required=True) + ] + + bucket: Annotated[str, click.option("--bucket", type=str, help="S3 bucket name", required=True)] + index: Annotated[str, click.option("--index", type=str, help="Unique vector index name", default="vdbbench-index")] + + metric: Annotated[ + str, + click.option( + "--metric", + type=str, + help="Distance metric for vector similarity (e.g., 'cosine', 'euclidean').", + default=None, + ), + ] + + +class S3VectorsIndexTypedDict(CommonTypedDict, S3VectorsTypedDict): ... + + +@cli.command() +@click_parameter_decorators_from_typed_dict(S3VectorsIndexTypedDict) +def S3Vectors(**parameters: Unpack[S3VectorsIndexTypedDict]): + from .config import S3VectorsConfig + + parameters["custom_case"] = get_custom_case_config(parameters) + run( + db=DB.S3Vectors, + db_config=S3VectorsConfig( + region_name=parameters["region"], + access_key_id=SecretStr(parameters["access_key_id"]), + secret_access_key=SecretStr(parameters["secret_access_key"]), + bucket_name=parameters["bucket"], + index_name=parameters["index"] if parameters["index"] else "vdbbench-index", + ), + db_case_config=S3VectorsIndexConfig( + metric_type=( + MetricType.COSINE + if parameters["metric"] == "cosine" + else MetricType.L2 if parameters["metric"] == "l2" else None + ) + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/s3_vectors/config.py b/vectordb_bench/backend/clients/s3_vectors/config.py new file mode 100644 index 000000000..f08b93a3a --- /dev/null +++ b/vectordb_bench/backend/clients/s3_vectors/config.py @@ -0,0 +1,41 @@ +from pydantic import BaseModel, SecretStr + +from ..api import DBCaseConfig, DBConfig, MetricType + + +class S3VectorsConfig(DBConfig): + region_name: str = "us-west-2" + access_key_id: SecretStr + secret_access_key: SecretStr + bucket_name: str + index_name: str = "vdbbench-index" + + def to_dict(self) -> dict: + return { + "region_name": self.region_name, + "access_key_id": self.access_key_id.get_secret_value() if self.access_key_id else "", + "secret_access_key": self.secret_access_key.get_secret_value() if self.secret_access_key else "", + "bucket_name": self.bucket_name, + "index_name": self.index_name, + } + + +class S3VectorsIndexConfig(DBCaseConfig, BaseModel): + """Base config for s3-vectors""" + + metric_type: MetricType | None = None + data_type: str = "float32" + + def parse_metric(self) -> str: + if self.metric_type == MetricType.COSINE: + return "cosine" + if self.metric_type == MetricType.L2: + return "euclidean" + msg = f"Unsupported metric type: {self.metric_type}" + raise ValueError(msg) + + def index_param(self) -> dict: + return {} + + def search_param(self) -> dict: + return {} diff --git a/vectordb_bench/backend/clients/s3_vectors/s3_vectors.py b/vectordb_bench/backend/clients/s3_vectors/s3_vectors.py new file mode 100644 index 000000000..b05134f6b --- /dev/null +++ b/vectordb_bench/backend/clients/s3_vectors/s3_vectors.py @@ -0,0 +1,171 @@ +"""Wrapper around the Milvus vector database over VectorDB""" + +import logging +from collections.abc import Iterable +from contextlib import contextmanager + +import boto3 + +from vectordb_bench.backend.filter import Filter, FilterOp + +from ..api import VectorDB +from .config import S3VectorsIndexConfig + +log = logging.getLogger(__name__) + + +class S3Vectors(VectorDB): + supported_filter_types: list[FilterOp] = [ + FilterOp.NonFilter, + FilterOp.NumGE, + FilterOp.StrEqual, + ] + + def __init__( + self, + dim: int, + db_config: dict, + db_case_config: S3VectorsIndexConfig, + drop_old: bool = False, + with_scalar_labels: bool = False, + **kwargs, + ): + """Initialize wrapper around the s3-vectors client.""" + self.db_config = db_config + self.case_config = db_case_config + self.with_scalar_labels = with_scalar_labels + + self.batch_size = 500 + + self._scalar_id_field = "id" + self._scalar_label_field = "label" + self._vector_field = "vector" + + self.region_name = self.db_config.get("region_name") + self.access_key_id = self.db_config.get("access_key_id") + self.secret_access_key = self.db_config.get("secret_access_key") + self.bucket_name = self.db_config.get("bucket_name") + self.index_name = self.db_config.get("index_name") + + client = boto3.client( + service_name="s3vectors", + region_name=self.region_name, + aws_access_key_id=self.access_key_id, + aws_secret_access_key=self.secret_access_key, + ) + + if drop_old: + # delete old index if exists + response = client.list_indexes(vectorBucketName=self.bucket_name) + index_names = [index["indexName"] for index in response["indexes"]] + if self.index_name in index_names: + log.info(f"drop old index: {self.index_name}") + client.delete_index(vectorBucketName=self.bucket_name, indexName=self.index_name) + + # create the index + client.create_index( + vectorBucketName=self.bucket_name, + indexName=self.index_name, + dataType=self.case_config.data_type, + dimension=dim, + distanceMetric=self.case_config.parse_metric(), + ) + + client.close() + + @contextmanager + def init(self): + """ + Examples: + >>> with self.init(): + >>> self.insert_embeddings() + >>> self.search_embedding() + """ + self.client = boto3.client( + service_name="s3vectors", + region_name=self.region_name, + aws_access_key_id=self.access_key_id, + aws_secret_access_key=self.secret_access_key, + ) + + yield + self.client.close() + + def optimize(self, **kwargs): + return + + def need_normalize_cosine(self) -> bool: + """Wheather this database need to normalize dataset to support COSINE""" + return False + + def insert_embeddings( + self, + embeddings: Iterable[list[float]], + metadata: list[int], + labels_data: list[str] | None = None, + **kwargs, + ) -> tuple[int, Exception]: + """Insert embeddings into s3-vectors. should call self.init() first""" + # use the first insert_embeddings to init collection + assert self.client is not None + assert len(embeddings) == len(metadata) + insert_count = 0 + try: + for batch_start_offset in range(0, len(embeddings), self.batch_size): + batch_end_offset = min(batch_start_offset + self.batch_size, len(embeddings)) + insert_data = [ + { + "key": str(metadata[i]), + "data": {self.case_config.data_type: embeddings[i]}, + "metadata": ( + {self._scalar_label_field: labels_data[i], self._scalar_id_field: metadata[i]} + if self.with_scalar_labels + else {self._scalar_id_field: metadata[i]} + ), + } + for i in range(batch_start_offset, batch_end_offset) + ] + self.client.put_vectors( + vectorBucketName=self.bucket_name, + indexName=self.index_name, + vectors=insert_data, + ) + insert_count += len(insert_data) + except Exception as e: + log.info(f"Failed to insert data: {e}") + return insert_count, e + return insert_count, None + + def prepare_filter(self, filters: Filter): + if filters.type == FilterOp.NonFilter: + self.filter = None + elif filters.type == FilterOp.NumGE: + self.filter = {self._scalar_id_field: {"$gte": filters.int_value}} + elif filters.type == FilterOp.StrEqual: + self.filter = {self._scalar_label_field: filters.label_value} + else: + msg = f"Not support Filter for S3Vectors - {filters}" + raise ValueError(msg) + + def search_embedding( + self, + query: list[float], + k: int = 100, + timeout: int | None = None, + ) -> list[int]: + """Perform a search on a query embedding and return results.""" + assert self.client is not None + + # Perform the search. + res = self.client.query_vectors( + vectorBucketName=self.bucket_name, + indexName=self.index_name, + queryVector={"float32": query}, + topK=k, + filter=self.filter, + returnDistance=False, + returnMetadata=False, + ) + + # Organize results. + return [int(result["key"]) for result in res["vectors"]] diff --git a/vectordb_bench/backend/clients/tencent_elasticsearch/cli.py b/vectordb_bench/backend/clients/tencent_elasticsearch/cli.py new file mode 100644 index 000000000..40d78e02c --- /dev/null +++ b/vectordb_bench/backend/clients/tencent_elasticsearch/cli.py @@ -0,0 +1,96 @@ +import os +from typing import Annotated, Unpack + +import click +from pydantic import SecretStr + +from vectordb_bench.backend.clients import DB +from vectordb_bench.cli.cli import ( + CommonTypedDict, + cli, + click_parameter_decorators_from_typed_dict, + run, +) + + +class TencentElasticsearchTypedDict(CommonTypedDict): + scheme: Annotated[ + str, + click.option( + "--scheme", + type=str, + help="Protocol in use to connect to the node", + default="http", + show_default=True, + ), + ] + host: Annotated[ + str, + click.option("--host", type=str, help="shot connection string", required=True), + ] + port: Annotated[ + int, + click.option("--port", type=int, help="Port to connect to", default=9200, show_default=True), + ] + user: Annotated[ + str, + click.option("--user", type=str, help="Db username", required=True), + ] + password: Annotated[ + str, + click.option( + "--password", + type=str, + help="TencentElasticsearch password", + default=lambda: os.environ.get("TES_PASSWORD", ""), + show_default="$TES_PASSWORD", + ), + ] + m: Annotated[ + int, + click.option("--m", type=int, help="HNSW M parameter", default=16, show_default=True), + ] + ef_construction: Annotated[ + int, + click.option( + "--ef_construction", + type=int, + help="HNSW efConstruction parameter", + default=200, + show_default=True, + ), + ] + num_candidates: Annotated[ + int, + click.option( + "--num_candidates", + type=int, + help="Number of candidates to consider during searching", + default=200, + show_default=True, + ), + ] + + +@cli.command() +@click_parameter_decorators_from_typed_dict(TencentElasticsearchTypedDict) +def TencentElasticsearch(**parameters: Unpack[TencentElasticsearchTypedDict]): + from .config import TencentElasticsearchConfig, TencentElasticsearchIndexConfig + + run( + db=DB.TencentElasticsearch, + db_config=TencentElasticsearchConfig( + db_label=parameters["db_label"], + scheme=parameters["scheme"], + host=parameters["host"], + port=parameters["port"], + user=parameters["user"], + password=SecretStr(parameters["password"]), + ), + db_case_config=TencentElasticsearchIndexConfig( + M=parameters["m"], + efConstruction=parameters["ef_construction"], + num_candidates=parameters["num_candidates"], + ), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/tencent_elasticsearch/config.py b/vectordb_bench/backend/clients/tencent_elasticsearch/config.py new file mode 100644 index 000000000..53a42c058 --- /dev/null +++ b/vectordb_bench/backend/clients/tencent_elasticsearch/config.py @@ -0,0 +1,92 @@ +from enum import Enum + +from pydantic import BaseModel, SecretStr + +from ..api import DBCaseConfig, DBConfig, IndexType, MetricType + + +class TencentElasticsearchConfig(DBConfig, BaseModel): + #: Protocol in use to connect to the node + scheme: str = "http" + host: str = "" + port: int = 9200 + user: str = "elastic" + password: SecretStr + + def to_dict(self) -> dict: + return { + "hosts": [{"scheme": self.scheme, "host": self.host, "port": self.port}], + "basic_auth": (self.user, self.password.get_secret_value()), + } + + +class ESElementType(str, Enum): + float = "float" # 4 byte + byte = "byte" # 1 byte, -128 to 127 + + +class TencentElasticsearchIndexConfig(BaseModel, DBCaseConfig): + element_type: ESElementType = ESElementType.float + index: IndexType = IndexType.TES_VSEARCH + number_of_shards: int = 1 + number_of_replicas: int = 0 + refresh_interval: str = "3s" + merge_max_thread_count: int = 8 + use_rescore: bool = False + oversample_ratio: float = 2.0 + use_routing: bool = False + use_force_merge: bool = True + + metric_type: MetricType | None = None + efConstruction: int | None = None + M: int | None = None + num_candidates: int | None = None + + def __eq__(self, obj: any): + return ( + self.index == obj.index + and self.number_of_shards == obj.number_of_shards + and self.number_of_replicas == obj.number_of_replicas + and self.use_routing == obj.use_routing + and self.efConstruction == obj.efConstruction + and self.M == obj.M + ) + + def __hash__(self) -> int: + return hash( + ( + self.index, + self.number_of_shards, + self.number_of_replicas, + self.use_routing, + self.efConstruction, + self.M, + 2, + ) + ) + + def parse_metric(self) -> str: + if self.metric_type == MetricType.L2: + return "l2_norm" + if self.metric_type == MetricType.IP: + return "dot_product" + return "cosine" + + def index_param(self) -> dict: + return { + "type": "dense_vector", + "index": True, + "element_type": self.element_type.value, + "similarity": self.parse_metric(), + "index_options": { + "type": self.index.value, + "index": "hnsw", + "m": self.M, + "ef_construction": self.efConstruction, + }, + } + + def search_param(self) -> dict: + return { + "num_candidates": self.num_candidates, + } diff --git a/vectordb_bench/backend/clients/tencent_elasticsearch/tencent_elasticsearch.py b/vectordb_bench/backend/clients/tencent_elasticsearch/tencent_elasticsearch.py new file mode 100644 index 000000000..eb69a730d --- /dev/null +++ b/vectordb_bench/backend/clients/tencent_elasticsearch/tencent_elasticsearch.py @@ -0,0 +1,52 @@ +import logging +import time +from contextlib import contextmanager + +from vectordb_bench.backend.filter import FilterOp + +from ..elastic_cloud.elastic_cloud import ElasticCloud + +for logger in ("elasticsearch", "elastic_transport"): + logging.getLogger(logger).setLevel(logging.WARNING) + +log = logging.getLogger(__name__) + + +SECONDS_WAITING_FOR_FORCE_MERGE_API_CALL_SEC = 30 + + +class TencentElasticsearch(ElasticCloud): + supported_filter_types: list[FilterOp] = [ + FilterOp.NonFilter, + FilterOp.NumGE, + FilterOp.StrEqual, + ] + + @contextmanager + def init(self) -> None: + """connect to elasticsearch""" + from elasticsearch import Elasticsearch + + self.client = Elasticsearch(**self.db_config, request_timeout=1800) + + yield + self.client = None + del self.client + + def optimize(self, data_size: int | None = None): + """optimize will be called between insertion and search in performance cases.""" + assert self.client is not None, "should self.init() first" + self.client.indices.refresh(index=self.indice) + time.sleep(SECONDS_WAITING_FOR_FORCE_MERGE_API_CALL_SEC) + if self.case_config.use_force_merge: + force_merge_task_id = self.client.indices.forcemerge( + index=self.indice, + max_num_segments=1, + wait_for_completion=False, + )["task"] + log.info(f"Elasticsearch force merge task id: {force_merge_task_id}") + while True: + time.sleep(SECONDS_WAITING_FOR_FORCE_MERGE_API_CALL_SEC) + task_status = self.client.tasks.get(task_id=force_merge_task_id) + if task_status["completed"]: + return diff --git a/vectordb_bench/backend/clients/tidb/cli.py b/vectordb_bench/backend/clients/tidb/cli.py index cdfcbe432..20fe3d1ef 100644 --- a/vectordb_bench/backend/clients/tidb/cli.py +++ b/vectordb_bench/backend/clients/tidb/cli.py @@ -17,7 +17,6 @@ class TiDBTypedDict(CommonTypedDict): help="Username", default="root", show_default=True, - required=True, ), ] password: Annotated[ @@ -37,7 +36,6 @@ class TiDBTypedDict(CommonTypedDict): type=str, default="127.0.0.1", show_default=True, - required=True, help="Db host", ), ] @@ -48,7 +46,6 @@ class TiDBTypedDict(CommonTypedDict): type=int, default=4000, show_default=True, - required=True, help="Db Port", ), ] @@ -59,7 +56,6 @@ class TiDBTypedDict(CommonTypedDict): type=str, default="test", show_default=True, - required=True, help="Db name", ), ] diff --git a/vectordb_bench/backend/clients/tidb/config.py b/vectordb_bench/backend/clients/tidb/config.py index 693551045..71fdbad66 100644 --- a/vectordb_bench/backend/clients/tidb/config.py +++ b/vectordb_bench/backend/clients/tidb/config.py @@ -1,8 +1,20 @@ -from pydantic import BaseModel, SecretStr +from typing import TypedDict + +from pydantic import BaseModel, SecretStr, validator from ..api import DBCaseConfig, DBConfig, MetricType +class TiDBConfigDict(TypedDict): + host: str + port: int + user: str + password: str + database: str + ssl_verify_cert: bool + ssl_verify_identity: bool + + class TiDBConfig(DBConfig): user_name: str = "root" password: SecretStr @@ -11,7 +23,7 @@ class TiDBConfig(DBConfig): db_name: str = "test" ssl: bool = False - def to_dict(self) -> dict: + def to_dict(self) -> TiDBConfigDict: pwd_str = self.password.get_secret_value() return { "host": self.host, @@ -23,6 +35,14 @@ def to_dict(self) -> dict: "ssl_verify_identity": self.ssl, } + @validator("*") + def not_empty_field(cls, v: any, field: any): + if field.name in ["password", "db_label"]: + return v + if isinstance(v, str | SecretStr) and len(v) == 0: + raise ValueError("Empty string!") + return v + class TiDBIndexConfig(BaseModel, DBCaseConfig): metric_type: MetricType | None = None diff --git a/vectordb_bench/backend/clients/tidb/tidb.py b/vectordb_bench/backend/clients/tidb/tidb.py index b75605eda..a5c99bbe4 100644 --- a/vectordb_bench/backend/clients/tidb/tidb.py +++ b/vectordb_bench/backend/clients/tidb/tidb.py @@ -68,15 +68,13 @@ def _create_table(self): try: index_param = self.case_config.index_param() with self._get_connection() as (conn, cursor): - cursor.execute( - f""" + cursor.execute(f""" CREATE TABLE {self.table_name} ( id BIGINT PRIMARY KEY, embedding VECTOR({self.dim}) NOT NULL, VECTOR INDEX (({index_param["metric_fn"]}(embedding))) ); - """ - ) + """) conn.commit() except Exception as e: log.warning("Failed to create table: %s error: %s", self.table_name, e) @@ -118,12 +116,10 @@ def _optimize_check_tiflash_replica_progress(self): try: database = self.db_config["database"] with self._get_connection() as (_, cursor): - cursor.execute( - f""" + cursor.execute(f""" SELECT PROGRESS FROM information_schema.tiflash_replica WHERE TABLE_SCHEMA = "{database}" AND TABLE_NAME = "{self.table_name}" - """ # noqa: S608 - ) + """) # noqa: S608 result = cursor.fetchone() return result[0] except Exception as e: @@ -155,13 +151,11 @@ def _optimize_get_tiflash_index_pending_rows(self): try: database = self.db_config["database"] with self._get_connection() as (_, cursor): - cursor.execute( - f""" + cursor.execute(f""" SELECT SUM(ROWS_STABLE_NOT_INDEXED) FROM information_schema.tiflash_indexes WHERE TIDB_DATABASE = "{database}" AND TIDB_TABLE = "{self.table_name}" - """ # noqa: S608 - ) + """) # noqa: S608 result = cursor.fetchone() return result[0] except Exception as e: @@ -223,11 +217,9 @@ def search_embedding( timeout: int | None = None, **kwargs: Any, ) -> list[int]: - self.cursor.execute( - f""" + self.cursor.execute(f""" SELECT id FROM {self.table_name} ORDER BY {self.search_fn}(embedding, "{query!s}") LIMIT {k}; - """ # noqa: S608 - ) + """) # noqa: S608 result = self.cursor.fetchall() return [int(i[0]) for i in result] diff --git a/vectordb_bench/backend/clients/turbopuffer/cli.py b/vectordb_bench/backend/clients/turbopuffer/cli.py new file mode 100644 index 000000000..6fd91f2a8 --- /dev/null +++ b/vectordb_bench/backend/clients/turbopuffer/cli.py @@ -0,0 +1,62 @@ +from typing import Annotated, TypedDict, Unpack + +import click +from pydantic import SecretStr + +from ....cli.cli import ( + CommonTypedDict, + cli, + click_parameter_decorators_from_typed_dict, + run, +) +from .. import DB + + +class TurboPufferTypedDict(TypedDict): + api_key: Annotated[ + str, + click.option("--api-key", type=str, help="TurboPuffer API key", required=True), + ] + api_base_url: Annotated[ + str, + click.option( + "--api-base-url", + type=str, + help="TurboPuffer API base URL", + required=False, + default="https://api.turbopuffer.com", + show_default=True, + ), + ] + namespace: Annotated[ + str, + click.option( + "--namespace", + type=str, + help="TurboPuffer namespace", + required=False, + default="vdbbench_test", + show_default=True, + ), + ] + + +class TurboPufferIndexTypedDict(CommonTypedDict, TurboPufferTypedDict): ... + + +@cli.command() +@click_parameter_decorators_from_typed_dict(TurboPufferIndexTypedDict) +def TurboPuffer(**parameters: Unpack[TurboPufferIndexTypedDict]): + from .config import TurboPufferConfig, TurboPufferIndexConfig + + run( + db=DB.TurboPuffer, + db_config=TurboPufferConfig( + db_label=parameters["db_label"], + api_key=SecretStr(parameters["api_key"]), + api_base_url=parameters["api_base_url"], + namespace=parameters["namespace"], + ), + db_case_config=TurboPufferIndexConfig(), + **parameters, + ) diff --git a/vectordb_bench/backend/clients/turbopuffer/config.py b/vectordb_bench/backend/clients/turbopuffer/config.py new file mode 100644 index 000000000..c552299e3 --- /dev/null +++ b/vectordb_bench/backend/clients/turbopuffer/config.py @@ -0,0 +1,37 @@ +from pydantic import BaseModel, SecretStr + +from ..api import DBCaseConfig, DBConfig, MetricType + + +class TurboPufferConfig(DBConfig): + api_key: SecretStr + api_base_url: str + namespace: str = "vdbbench_test" + + def to_dict(self) -> dict: + return { + "api_key": self.api_key.get_secret_value(), + "api_base_url": self.api_base_url, + "namespace": self.namespace, + } + + +class TurboPufferIndexConfig(BaseModel, DBCaseConfig): + metric_type: MetricType | None = None + use_multi_ns_for_filter: bool = False + time_wait_warmup: int = 60 * 1 # 1min + + def parse_metric(self) -> str: + if self.metric_type == MetricType.COSINE: + return "cosine_distance" + if self.metric_type == MetricType.L2: + return "euclidean_squared" + + msg = f"Not Support {self.metric_type}" + raise ValueError(msg) + + def index_param(self) -> dict: + return {} + + def search_param(self) -> dict: + return {} diff --git a/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py b/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py new file mode 100644 index 000000000..241551792 --- /dev/null +++ b/vectordb_bench/backend/clients/turbopuffer/turbopuffer.py @@ -0,0 +1,122 @@ +"""Wrapper around the Pinecone vector database over VectorDB""" + +import logging +import time +from contextlib import contextmanager + +import turbopuffer as tpuf + +from vectordb_bench.backend.clients.turbopuffer.config import TurboPufferIndexConfig +from vectordb_bench.backend.filter import Filter, FilterOp + +from ..api import VectorDB + +log = logging.getLogger(__name__) + + +class TurboPuffer(VectorDB): + supported_filter_types: list[FilterOp] = [ + FilterOp.NonFilter, + FilterOp.NumGE, + FilterOp.StrEqual, + ] + + def __init__( + self, + dim: int, + db_config: dict, + db_case_config: TurboPufferIndexConfig, + drop_old: bool = False, + with_scalar_labels: bool = False, + **kwargs, + ): + """Initialize wrapper around the milvus vector database.""" + self.api_key = db_config.get("api_key", "") + self.api_base_url = db_config.get("api_base_url", "") + self.namespace = db_config.get("namespace", "") + self.db_case_config = db_case_config + self.metric = db_case_config.parse_metric() + + self._vector_field = "vector" + self._scalar_id_field = "id" + self._scalar_label_field = "label" + + self.with_scalar_labels = with_scalar_labels + + # Initialize client with new SDK pattern + self.client = tpuf.Turbopuffer(api_key=self.api_key, base_url=self.api_base_url) + + if drop_old: + log.info(f"Drop old. delete the namespace: {self.namespace}") + ns = self.client.namespace(self.namespace) + try: + ns.delete_all() + except Exception as e: + log.warning(f"Failed to delete all. Error: {e}") + + @contextmanager + def init(self): + self.ns = self.client.namespace(self.namespace) + yield + + def optimize(self, data_size: int | None = None): + # turbopuffer responds to the request + # once the cache warming operation has been started. + # It does not wait for the operation to complete, + # which can take multiple minutes for large namespaces. + self.ns.hint_cache_warm() + log.info(f"warming up but no api waiting for complete. just sleep {self.db_case_config.time_wait_warmup}s") + time.sleep(self.db_case_config.time_wait_warmup) + + def insert_embeddings( + self, + embeddings: list[list[float]], + metadata: list[int], + labels_data: list[str] | None = None, + **kwargs, + ) -> tuple[int, Exception]: + try: + if self.with_scalar_labels: + self.ns.write( + columns={ + self._scalar_id_field: metadata, + self._vector_field: embeddings, + self._scalar_label_field: labels_data, + }, + distance_metric=self.metric, + ) + else: + self.ns.write( + columns={ + self._scalar_id_field: metadata, + self._vector_field: embeddings, + }, + distance_metric=self.metric, + ) + except Exception as e: + log.warning(f"Failed to insert. Error: {e}") + return len(embeddings), None + + def search_embedding( + self, + query: list[float], + k: int = 100, + timeout: int | None = None, + ) -> list[int]: + res = self.ns.query( + rank_by=("vector", "ANN", query), + top_k=k, + filters=self.expr, + ) + return [row.id for row in res.rows] if res.rows is not None else [] + + def prepare_filter(self, filters: Filter): + if filters.type == FilterOp.NonFilter: + self.expr = None + elif filters.type == FilterOp.NumGE: + self.expr = (self._scalar_id_field, "Gte", filters.int_value) + elif filters.type == FilterOp.StrEqual: + self.expr = (self._scalar_label_field, "Eq", filters.label_value) + else: + msg = f"Not support Filter for TurboPuffer - {filters}" + raise ValueError(msg) diff --git a/vectordb_bench/backend/clients/weaviate_cloud/cli.py b/vectordb_bench/backend/clients/weaviate_cloud/cli.py index 181898c74..cba3c2377 100644 --- a/vectordb_bench/backend/clients/weaviate_cloud/cli.py +++ b/vectordb_bench/backend/clients/weaviate_cloud/cli.py @@ -15,12 +15,33 @@ class WeaviateTypedDict(CommonTypedDict): api_key: Annotated[ str, - click.option("--api-key", type=str, help="Weaviate api key", required=True), + click.option("--api-key", type=str, help="Weaviate api key", required=False, default=""), ] url: Annotated[ str, click.option("--url", type=str, help="Weaviate url", required=True), ] + no_auth: Annotated[ + bool, + click.option( + "--no-auth", + is_flag=True, + help="Do not use api-key, set it to true if you are using a local setup. Default is False.", + default=False, + ), + ] + m: Annotated[ + int, + click.option("--m", type=int, default=16, help="HNSW index parameter m."), + ] + ef_construct: Annotated[ + int, + click.option("--ef-construction", type=int, default=256, help="HNSW index parameter ef_construction"), + ] + ef: Annotated[ + int, + click.option("--ef", type=int, default=256, help="HNSW index parameter ef for search"), + ] @cli.command() @@ -32,9 +53,14 @@ def Weaviate(**parameters: Unpack[WeaviateTypedDict]): db=DB.WeaviateCloud, db_config=WeaviateConfig( db_label=parameters["db_label"], - api_key=SecretStr(parameters["api_key"]), + api_key=SecretStr(parameters["api_key"]) if parameters["api_key"] != "" else SecretStr("-"), url=SecretStr(parameters["url"]), + no_auth=parameters["no_auth"], + ), + db_case_config=WeaviateIndexConfig( + efConstruction=parameters["ef_construction"], + maxConnections=parameters["m"], + ef=parameters["ef"], ), - db_case_config=WeaviateIndexConfig(ef=256, efConstruction=256, maxConnections=16), **parameters, ) diff --git a/vectordb_bench/backend/clients/weaviate_cloud/config.py b/vectordb_bench/backend/clients/weaviate_cloud/config.py index 4c58167d4..f29a307a3 100644 --- a/vectordb_bench/backend/clients/weaviate_cloud/config.py +++ b/vectordb_bench/backend/clients/weaviate_cloud/config.py @@ -6,11 +6,13 @@ class WeaviateConfig(DBConfig): url: SecretStr api_key: SecretStr + no_auth: bool | None = False def to_dict(self) -> dict: return { "url": self.url.get_secret_value(), "auth_client_secret": self.api_key.get_secret_value(), + "no_auth": self.no_auth, } diff --git a/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py b/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py index c31104d8b..d6111c8da 100644 --- a/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py +++ b/vectordb_bench/backend/clients/weaviate_cloud/weaviate_cloud.py @@ -38,6 +38,11 @@ def __init__( self._vector_field = "vector" self._index_name = "vector_idx" + # If local setup is used, we + if db_config["no_auth"]: + del db_config["auth_client_secret"] + del db_config["no_auth"] + from weaviate import Client client = Client(**db_config) diff --git a/vectordb_bench/backend/clients/zilliz_cloud/cli.py b/vectordb_bench/backend/clients/zilliz_cloud/cli.py index 810a4175e..a8d177ee5 100644 --- a/vectordb_bench/backend/clients/zilliz_cloud/cli.py +++ b/vectordb_bench/backend/clients/zilliz_cloud/cli.py @@ -36,6 +36,28 @@ class ZillizTypedDict(CommonTypedDict): str, click.option("--level", type=str, help="Zilliz index level", required=False), ] + num_shards: Annotated[ + int, + click.option( + "--num-shards", + type=int, + help="Number of shards", + required=False, + default=1, + show_default=True, + ), + ] + collection_name: Annotated[ + str, + click.option( + "--collection-name", + type=str, + help="Collection name for Zilliz", + required=False, + default="ZillizCloudVDBBench", + show_default=True, + ), + ] @cli.command() @@ -50,9 +72,12 @@ def ZillizAutoIndex(**parameters: Unpack[ZillizTypedDict]): uri=SecretStr(parameters["uri"]), user=parameters["user_name"], password=SecretStr(parameters["password"]), + num_shards=parameters["num_shards"], + collection_name=parameters["collection_name"], ), db_case_config=AutoIndexConfig( - params={parameters["level"]}, + level=int(parameters["level"]) if parameters["level"] else 1, + num_shards=parameters["num_shards"], ), **parameters, ) diff --git a/vectordb_bench/backend/clients/zilliz_cloud/config.py b/vectordb_bench/backend/clients/zilliz_cloud/config.py index 9f113dbda..f0b3fd000 100644 --- a/vectordb_bench/backend/clients/zilliz_cloud/config.py +++ b/vectordb_bench/backend/clients/zilliz_cloud/config.py @@ -8,24 +8,29 @@ class ZillizCloudConfig(DBConfig): uri: SecretStr user: str password: SecretStr + num_shards: int = 1 + collection_name: str = "ZillizCloudVDBBench" def to_dict(self) -> dict: return { "uri": self.uri.get_secret_value(), "user": self.user, "password": self.password.get_secret_value(), + "num_shards": self.num_shards, + "collection_name": self.collection_name, } class AutoIndexConfig(MilvusIndexConfig, DBCaseConfig): index: IndexType = IndexType.AUTOINDEX level: int = 1 + num_shards: int = 1 def index_param(self) -> dict: return { "metric_type": self.parse_metric(), "index_type": self.index.value, - "params": {}, + "params": {"shardsNum": self.num_shards}, } def search_param(self) -> dict: diff --git a/vectordb_bench/backend/clients/zilliz_cloud/zilliz_cloud.py b/vectordb_bench/backend/clients/zilliz_cloud/zilliz_cloud.py index 4ce15545a..6a3694ebb 100644 --- a/vectordb_bench/backend/clients/zilliz_cloud/zilliz_cloud.py +++ b/vectordb_bench/backend/clients/zilliz_cloud/zilliz_cloud.py @@ -10,7 +10,7 @@ def __init__( dim: int, db_config: dict, db_case_config: DBCaseConfig, - collection_name: str = "ZillizCloudVectorDBBench", + collection_name: str = "ZillizCloudVDBBench", drop_old: bool = False, name: str = "ZillizCloud", **kwargs, diff --git a/vectordb_bench/backend/dataset.py b/vectordb_bench/backend/dataset.py index 62700b0fa..552b73417 100644 --- a/vectordb_bench/backend/dataset.py +++ b/vectordb_bench/backend/dataset.py @@ -20,6 +20,7 @@ from . import utils from .clients import MetricType from .data_source import DatasetReader, DatasetSource +from .filter import Filter, FilterOp, non_filter log = logging.getLogger(__name__) @@ -39,6 +40,22 @@ class BaseDataset(BaseModel): with_gt: bool = False _size_label: dict[int, SizeLabel] = PrivateAttr() is_custom: bool = False + with_remote_resource: bool = True + # for label filter cases + with_scalar_labels: bool = False + # if True, scalar_labels will be retrieved from a separate parquet file; + # otherwise, they will be obtained from train.parquet. + scalar_labels_file_separated: bool = True + scalar_labels_file: str = "scalar_labels.parquet" + scalar_label_percentages: list[float] = [] + scalar_int_rates: list[float] = [] + train_id_field: str = "id" + train_vector_field: str = "emb" + test_file: str = "test.parquet" + test_id_field: str = "id" + test_vector_field: str = "emb" + gt_id_field: str = "id" + gt_neighbors_field: str = "neighbors_id" @validator("size") def verify_size(cls, v: int): @@ -51,6 +68,10 @@ def verify_size(cls, v: int): def label(self) -> str: return self._size_label.get(self.size).label + @property + def full_name(self) -> str: + return f"{self.name.capitalize()} ({self.label.capitalize()})" + @property def dir_name(self) -> str: return f"{self.name}_{self.label}_{utils.numerize(self.size)}".lower() @@ -59,11 +80,27 @@ def dir_name(self) -> str: def file_count(self) -> int: return self._size_label.get(self.size).file_count + @property + def train_files(self) -> list[str]: + return utils.compose_train_files(self.file_count, self.use_shuffled) + class CustomDataset(BaseDataset): dir: str file_num: int is_custom: bool = True + with_remote_resource: bool = False + train_file: str = "train" + train_id_field: str = "id" + train_vector_field: str = "emb" + test_file: str = "test.parquet" + gt_file: str = "neighbors.parquet" + test_vector_field: str = "emb" + gt_neighbors_field: str = "neighbors_id" + with_scalar_labels: bool = True + scalar_labels_file_separated: bool = True + scalar_labels_file: str = "scalar_labels.parquet" + label_percentages: list[float] = [] @validator("size") def verify_size(cls, v: int): @@ -81,6 +118,17 @@ def dir_name(self) -> str: def file_count(self) -> int: return self.file_num + @property + def train_files(self) -> list[str]: + train_file = self.train_file + prefix = f"{train_file}" + train_files = [] + prefix_s = [item.strip() for item in prefix.split(",") if item.strip()] + for i in range(len(prefix_s)): + sub_file = f"{prefix_s[i]}.parquet" + train_files.append(sub_file) + return train_files + class LAION(BaseDataset): name: str = "LAION" @@ -109,12 +157,74 @@ class Cohere(BaseDataset): dim: int = 768 metric_type: MetricType = MetricType.COSINE use_shuffled: bool = config.USE_SHUFFLED_DATA - with_gt: bool = (True,) + with_gt: bool = True _size_label: dict = { 100_000: SizeLabel(100_000, "SMALL", 1), 1_000_000: SizeLabel(1_000_000, "MEDIUM", 1), 10_000_000: SizeLabel(10_000_000, "LARGE", 10), } + with_scalar_labels: bool = True + scalar_label_percentages: list[float] = [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5] + scalar_int_rates: list[float] = [ + 0.001, + 0.002, + 0.005, + 0.01, + 0.02, + 0.05, + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.8, + 0.9, + 0.95, + 0.98, + 0.99, + 0.995, + 0.998, + 0.999, + ] + + +class Bioasq(BaseDataset): + name: str = "Bioasq" + dim: int = 1024 + metric_type: MetricType = MetricType.COSINE + use_shuffled: bool = config.USE_SHUFFLED_DATA + with_gt: bool = True + _size_label: dict = { + 1_000_000: SizeLabel(1_000_000, "MEDIUM", 1), + 10_000_000: SizeLabel(10_000_000, "LARGE", 10), + } + with_scalar_labels: bool = True + scalar_label_percentages: list[float] = [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5] + scalar_int_rates: list[float] = [ + 0.001, + 0.002, + 0.005, + 0.01, + 0.02, + 0.05, + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.8, + 0.9, + 0.95, + 0.98, + 0.99, + 0.995, + 0.998, + 0.999, + ] class Glove(BaseDataset): @@ -146,12 +256,37 @@ class OpenAI(BaseDataset): dim: int = 1536 metric_type: MetricType = MetricType.COSINE use_shuffled: bool = config.USE_SHUFFLED_DATA - with_gt: bool = (True,) + with_gt: bool = True _size_label: dict = { 50_000: SizeLabel(50_000, "SMALL", 1), 500_000: SizeLabel(500_000, "MEDIUM", 1), 5_000_000: SizeLabel(5_000_000, "LARGE", 10), } + with_scalar_labels: bool = True + scalar_label_percentages: list[float] = [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5] + scalar_int_rates: list[float] = [ + 0.001, + 0.002, + 0.005, + 0.01, + 0.02, + 0.05, + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.8, + 0.9, + 0.95, + 0.98, + 0.99, + 0.995, + 0.998, + 0.999, + ] class DatasetManager(BaseModel): @@ -166,8 +301,9 @@ class DatasetManager(BaseModel): """ data: BaseDataset - test_data: pd.DataFrame | None = None - gt_data: pd.DataFrame | None = None + test_data: list[list[float]] | None = None + gt_data: list[list[int]] | None = None + scalar_labels: pl.DataFrame | None = None train_files: list[str] = [] reader: DatasetReader | None = None @@ -176,6 +312,9 @@ def __eq__(self, obj: any): return self.data.name == obj.data.name and self.data.label == obj.data.label return False + def __hash__(self) -> int: + return hash((self.data.name, self.data.label)) + def set_reader(self, reader: DatasetReader): self.reader = reader @@ -191,7 +330,7 @@ def data_dir(self) -> pathlib.Path: return pathlib.Path( config.DATASET_LOCAL_DIR, self.data.name.lower(), - self.data.dir_name.lower(), + self.data.dir_name, ) def __iter__(self): @@ -201,54 +340,60 @@ def __iter__(self): def prepare( self, source: DatasetSource = DatasetSource.S3, - filters: float | str | None = None, + filters: Filter = non_filter, ) -> bool: """Download the dataset from DatasetSource url = f"{source}/{self.data.dir_name}" Args: source(DatasetSource): S3 or AliyunOSS, default as S3 - filters(Optional[int | float | str]): combined with dataset's with_gt to + filters(Filter): combined with dataset's with_gt to compose the correct ground_truth file Returns: bool: whether the dataset is successfully prepared """ - file_count, use_shuffled = self.data.file_count, self.data.use_shuffled - - train_files = utils.compose_train_files(file_count, use_shuffled) - all_files = train_files - + self.train_files = self.data.train_files gt_file, test_file = None, None if self.data.with_gt: - gt_file, test_file = utils.compose_gt_file(filters), "test.parquet" - all_files.extend([gt_file, test_file]) - - if not self.data.is_custom: + gt_file, test_file = filters.groundtruth_file, self.data.test_file + + if self.data.with_remote_resource: + download_files = [file for file in self.train_files] + download_files.extend([gt_file, test_file]) + if self.data.with_scalar_labels and self.data.scalar_labels_file_separated: + download_files.append(self.data.scalar_labels_file) + download_files = [file for file in download_files if file is not None] source.reader().read( dataset=self.data.dir_name.lower(), - files=all_files, + files=download_files, local_ds_root=self.data_dir, ) + # read scalar_labels_file if separated + if ( + filters.type == FilterOp.StrEqual + and self.data.with_scalar_labels + and self.data.scalar_labels_file_separated + ): + self.scalar_labels = self._read_file(self.data.scalar_labels_file) + if gt_file is not None and test_file is not None: - self.test_data = self._read_file(test_file) - self.gt_data = self._read_file(gt_file) + self.test_data = self._read_file(test_file)[self.data.test_vector_field].to_list() + self.gt_data = self._read_file(gt_file)[self.data.gt_neighbors_field].to_list() - prefix = "shuffle_train" if use_shuffled else "train" - self.train_files = sorted([f.name for f in self.data_dir.glob(f"{prefix}*.parquet")]) log.debug(f"{self.data.name}: available train files {self.train_files}") return True - def _read_file(self, file_name: str) -> pd.DataFrame: + def _read_file(self, file_name: str) -> pl.DataFrame: """read one file from disk into memory""" log.info(f"Read the entire file into memory: {file_name}") p = pathlib.Path(self.data_dir, file_name) if not p.exists(): log.warning(f"No such file: {p}") - return pd.DataFrame() + return pl.DataFrame() return pl.read_parquet(p) @@ -304,6 +449,7 @@ class Dataset(Enum): LAION = LAION GIST = GIST COHERE = Cohere + BIOASQ = Bioasq GLOVE = Glove SIFT = SIFT OPENAI = OpenAI @@ -313,3 +459,51 @@ def get(self, size: int) -> BaseDataset: def manager(self, size: int) -> DatasetManager: return DatasetManager(data=self.get(size)) + + +class DatasetWithSizeType(Enum): + CohereSmall = "Small Cohere (768dim, 100K)" + CohereMedium = "Medium Cohere (768dim, 1M)" + CohereLarge = "Large Cohere (768dim, 10M)" + BioasqMedium = "Medium Bioasq (1024dim, 1M)" + BioasqLarge = "Large Bioasq (1024dim, 10M)" + OpenAISmall = "Small OpenAI (1536dim, 50K)" + OpenAIMedium = "Medium OpenAI (1536dim, 500K)" + OpenAILarge = "Large OpenAI (1536dim, 5M)" + + def get_manager(self) -> DatasetManager: + if self not in DatasetWithSizeMap: + msg = f"wrong ScalarDatasetWithSizeType: {self.name}" + raise ValueError(msg) + return DatasetWithSizeMap.get(self) + + def get_load_timeout(self) -> float: + if "small" in self.value.lower(): + return config.LOAD_TIMEOUT_768D_100K + if "medium" in self.value.lower(): + return config.LOAD_TIMEOUT_768D_1M + if "large" in self.value.lower(): + return config.LOAD_TIMEOUT_768D_10M + msg = f"No load_timeout for {self.value}" + raise KeyError(msg) + + def get_optimize_timeout(self) -> float: + if "small" in self.value.lower(): + return config.OPTIMIZE_TIMEOUT_768D_100K + if "medium" in self.value.lower(): + return config.OPTIMIZE_TIMEOUT_768D_1M + if "large" in self.value.lower(): + return config.OPTIMIZE_TIMEOUT_768D_10M + return config.OPTIMIZE_TIMEOUT_DEFAULT + + +DatasetWithSizeMap = { + DatasetWithSizeType.CohereSmall: Dataset.COHERE.manager(100_000), + DatasetWithSizeType.CohereMedium: Dataset.COHERE.manager(1_000_000), + DatasetWithSizeType.CohereLarge: Dataset.COHERE.manager(10_000_000), + DatasetWithSizeType.BioasqMedium: Dataset.BIOASQ.manager(1_000_000), + DatasetWithSizeType.BioasqLarge: Dataset.BIOASQ.manager(10_000_000), + DatasetWithSizeType.OpenAISmall: Dataset.OPENAI.manager(50_000), + DatasetWithSizeType.OpenAIMedium: Dataset.OPENAI.manager(500_000), + DatasetWithSizeType.OpenAILarge: Dataset.OPENAI.manager(5_000_000), +} diff --git a/vectordb_bench/backend/filter.py b/vectordb_bench/backend/filter.py new file mode 100644 index 000000000..7520539c4 --- /dev/null +++ b/vectordb_bench/backend/filter.py @@ -0,0 +1,93 @@ +from enum import StrEnum + +from ..base import BaseModel + + +class FilterOp(StrEnum): + NumGE = "NumGE" # test ">=" + StrEqual = "Label" # test "==" + NonFilter = "NonFilter" + + +class Filter(BaseModel): + type: FilterOp + filter_rate: float = 0.0 + + @property + def groundtruth_file(self) -> str: + raise NotImplementedError + + +class NonFilter(Filter): + type: FilterOp = FilterOp.NonFilter + filter_rate: float = 0.0 + gt_file_name: str = "neighbors.parquet" + + @property + def groundtruth_file(self) -> str: + return self.gt_file_name + + +non_filter = NonFilter() + + +class IntFilter(Filter): + """ + compatible with older int-filter cases + filter expr: int_field >= int_value (dataset_size * filter_rate) + """ + + type: FilterOp = FilterOp.NumGE + int_field: str = "id" + int_value: int + + @property + def groundtruth_file(self) -> str: + if self.filter_rate == 0.01: + return "neighbors_head_1p.parquet" + if self.filter_rate == 0.99: + return "neighbors_tail_1p.parquet" + msg = f"Not Support Int Filter - {self.filter_rate}" + raise RuntimeError(msg) + + +class NewIntFilter(Filter): + type: FilterOp = FilterOp.NumGE + int_field: str = "id" + int_value: int + + @property + def int_rate(self) -> str: + r = self.filter_rate * 100 + if 1 <= r <= 99: + return f"int_{int(r)}p" + return f"int_{r:.1f}p" + + @property + def groundtruth_file(self) -> str: + return f"neighbors_{self.int_rate}.parquet" + + +class LabelFilter(Filter): + """ + filter expr: label_field == label_value, like `color == "red"` + """ + + type: FilterOp = FilterOp.StrEqual + label_field: str = "labels" + label_percentage: float + + @property + def label_value(self) -> str: + p = self.label_percentage * 100 + if p >= 1: + return f"label_{int(p)}p" # such as 5p, 20p, 1p, ... + return f"label_{p:.1f}p" # such as 0.1p, 0.5p, ... + + def __init__(self, label_percentage: float, **kwargs): + filter_rate = 1.0 - label_percentage + super().__init__(filter_rate=filter_rate, label_percentage=label_percentage, **kwargs) + + @property + def groundtruth_file(self) -> str: + return f"neighbors_{self.label_field}_{self.label_value}.parquet" diff --git a/vectordb_bench/backend/runner/__init__.py b/vectordb_bench/backend/runner/__init__.py index b83df6f99..4af583773 100644 --- a/vectordb_bench/backend/runner/__init__.py +++ b/vectordb_bench/backend/runner/__init__.py @@ -1,10 +1,10 @@ -from .mp_runner import ( - MultiProcessingSearchRunner, -) +from .mp_runner import MultiProcessingSearchRunner +from .read_write_runner import ReadWriteRunner from .serial_runner import SerialInsertRunner, SerialSearchRunner __all__ = [ "MultiProcessingSearchRunner", + "ReadWriteRunner", "SerialInsertRunner", "SerialSearchRunner", ] diff --git a/vectordb_bench/backend/runner/mp_runner.py b/vectordb_bench/backend/runner/mp_runner.py index 687a0ecd7..9133e407a 100644 --- a/vectordb_bench/backend/runner/mp_runner.py +++ b/vectordb_bench/backend/runner/mp_runner.py @@ -5,15 +5,26 @@ import time import traceback from collections.abc import Iterable +from multiprocessing.queues import Queue import numpy as np +from hdrh.histogram import HdrHistogram + +from vectordb_bench.backend.filter import Filter, non_filter from ... import config +from ...models import ConcurrencySlotTimeoutError from ..clients import api NUM_PER_BATCH = config.NUM_PER_BATCH log = logging.getLogger(__name__) +# HDR Histogram constants +HDR_HISTOGRAM_MIN_US = 1 +HDR_HISTOGRAM_MAX_US = 60_000_000 # 60 seconds +HDR_HISTOGRAM_SIGNIFICANT_DIGITS = 3 # ±0.1% accuracy +US_TO_SECONDS = 1_000_000 + class MultiProcessingSearchRunner: """multiprocessing search runner @@ -28,16 +39,18 @@ def __init__( self, db: api.VectorDB, test_data: list[list[float]], - k: int = 100, - filters: dict | None = None, + k: int = config.K_DEFAULT, + filters: Filter = non_filter, concurrencies: Iterable[int] = config.NUM_CONCURRENCY, - duration: int = 30, + duration: int = config.CONCURRENCY_DURATION, + concurrency_timeout: int = config.CONCURRENCY_TIMEOUT, ): self.db = db self.k = k self.filters = filters self.concurrencies = concurrencies self.duration = duration + self.concurrency_timeout = concurrency_timeout self.test_data = test_data log.debug(f"test dataset columns: {len(test_data)}") @@ -54,6 +67,7 @@ def search( cond.wait() with self.db.init(): + self.db.prepare_filter(self.filters) num, idx = len(test_data), random.randint(0, len(test_data) - 1) start_time = time.perf_counter() @@ -62,18 +76,12 @@ def search( while time.perf_counter() < start_time + self.duration: s = time.perf_counter() try: - self.db.search_embedding( - test_data[idx], - self.k, - self.filters, - ) + self.db.search_embedding(test_data[idx], self.k) + count += 1 + latencies.append(time.perf_counter() - s) except Exception as e: log.warning(f"VectorDB search_embedding error: {e}") - traceback.print_exc(chain=True) - raise e from None - latencies.append(time.perf_counter() - s) - count += 1 # loop through the test data idx = idx + 1 if idx < num - 1 else 0 @@ -102,6 +110,7 @@ def _run_all_concurrencies_mem_efficient(self): conc_num_list = [] conc_qps_list = [] conc_latency_p99_list = [] + conc_latency_p95_list = [] conc_latency_avg_list = [] try: for conc in self.concurrencies: @@ -114,9 +123,7 @@ def _run_all_concurrencies_mem_efficient(self): log.info(f"Start search {self.duration}s in concurrency {conc}, filters: {self.filters}") future_iter = [executor.submit(self.search, self.test_data, q, cond) for i in range(conc)] # Sync all processes - while q.qsize() < conc: - sleep_t = conc if conc < 10 else 10 - time.sleep(sleep_t) + self._wait_for_queue_fill(q, size=conc) with cond: cond.notify_all() @@ -126,6 +133,7 @@ def _run_all_concurrencies_mem_efficient(self): all_count = sum([r.result()[0] for r in future_iter]) latencies = sum([r.result()[2] for r in future_iter], start=[]) latency_p99 = np.percentile(latencies, 99) + latency_p95 = np.percentile(latencies, 95) latency_avg = np.mean(latencies) cost = time.perf_counter() - start @@ -133,6 +141,7 @@ def _run_all_concurrencies_mem_efficient(self): conc_num_list.append(conc) conc_qps_list.append(qps) conc_latency_p99_list.append(latency_p99) + conc_latency_p95_list.append(latency_p95) conc_latency_avg_list.append(latency_avg) log.info(f"End search in concurrency {conc}: dur={cost}s, total_count={all_count}, qps={qps}") @@ -157,9 +166,19 @@ def _run_all_concurrencies_mem_efficient(self): conc_num_list, conc_qps_list, conc_latency_p99_list, + conc_latency_p95_list, conc_latency_avg_list, ) + def _wait_for_queue_fill(self, q: Queue, size: int): + wait_t = 0 + while q.qsize() < size: + sleep_t = size if size < 10 else 10 + wait_t += sleep_t + if wait_t > self.concurrency_timeout > 0: + raise ConcurrencySlotTimeoutError + time.sleep(sleep_t) + def run(self) -> float: """ Returns: @@ -170,11 +189,56 @@ def run(self) -> float: def stop(self) -> None: pass - def run_by_dur(self, duration: int) -> float: + def _aggregate_latency_stats(self, res: list) -> tuple[float, float, float]: + """Aggregate latency stats from worker processes. + + Returns: + tuple: (p99, p95, avg) latencies in seconds + """ + latency_stats_list = [r[2] for r in res if r[2] and r[2].get("count", 0) > 0] + + if not latency_stats_list: + return 0, 0, 0 + + total_query_count = sum(stats["count"] for stats in latency_stats_list) + + if total_query_count == 0: + return 0, 0, 0 + + # Use max for conservative percentile estimate + latency_p99 = max(stats["p99"] for stats in latency_stats_list) + latency_p95 = max(stats["p95"] for stats in latency_stats_list) + + # Weighted average + latency_avg = sum(stats["avg"] * stats["count"] for stats in latency_stats_list) / total_query_count + + return latency_p99, latency_p95, latency_avg + + def run_by_dur(self, duration: int) -> tuple[float, float]: + """ + Returns: + float: largest qps + float: failed rate + """ return self._run_by_dur(duration) - def _run_by_dur(self, duration: int) -> float: + def _run_by_dur(self, duration: int) -> tuple[float, float, list, list, list, list, list]: + """ + Returns: + float: largest qps + float: failed rate + list: concurrency numbers + list: qps values at each concurrency + list: p99 latencies at each concurrency + list: p95 latencies at each concurrency + list: avg latencies at each concurrency + """ max_qps = 0 + conc_num_list = [] + conc_qps_list = [] + conc_latency_p99_list = [] + conc_latency_p95_list = [] + conc_latency_avg_list = [] try: for conc in self.concurrencies: with mp.Manager() as m: @@ -197,12 +261,27 @@ def _run_by_dur(self, duration: int) -> float: log.info(f"Syncing all process and start concurrency search, concurrency={conc}") start = time.perf_counter() - all_count = sum([r.result() for r in future_iter]) + res = [r.result() for r in future_iter] + all_success_count = sum([r[0] for r in res]) + all_failed_count = sum([r[1] for r in res]) + failed_rate = all_failed_count / (all_failed_count + all_success_count) cost = time.perf_counter() - start - qps = round(all_count / cost, 4) - log.info(f"End search in concurrency {conc}: dur={cost}s, total_count={all_count}, qps={qps}") + qps = round(all_success_count / cost, 4) + + latency_p99, latency_p95, latency_avg = self._aggregate_latency_stats(res) + + conc_num_list.append(conc) + conc_qps_list.append(qps) + conc_latency_p99_list.append(latency_p99) + conc_latency_p95_list.append(latency_p95) + conc_latency_avg_list.append(latency_avg) + log.info( + f"End search in concurrency {conc}: dur={cost}s, failed_rate={failed_rate}, " + f"all_success_count={all_success_count}, all_failed_count={all_failed_count}, qps={qps}, " + f"p99={latency_p99:.4f}s, p95={latency_p95:.4f}s, avg={latency_avg:.4f}s", + ) if qps > max_qps: max_qps = qps log.info(f"Update largest qps with concurrency {conc}: current max_qps={max_qps}") @@ -219,52 +298,76 @@ def _run_by_dur(self, duration: int) -> float: finally: self.stop() - return max_qps + return ( + max_qps, + failed_rate, + conc_num_list, + conc_qps_list, + conc_latency_p99_list, + conc_latency_p95_list, + conc_latency_avg_list, + ) def search_by_dur( - self, - dur: int, - test_data: list[list[float]], - q: mp.Queue, - cond: mp.Condition, - ) -> int: + self, dur: int, test_data: list[list[float]], q: mp.Queue, cond: mp.Condition + ) -> tuple[int, int, dict]: + """ + Returns: + int: successful requests count + int: failed requests count + dict: latency statistics with p99, p95, avg, count (computed via HDR Histogram) + """ # sync all process q.put(1) with cond: cond.wait() with self.db.init(): + self.db.prepare_filter(self.filters) num, idx = len(test_data), random.randint(0, len(test_data) - 1) + # Memory-efficient latency tracking + histogram = HdrHistogram(HDR_HISTOGRAM_MIN_US, HDR_HISTOGRAM_MAX_US, HDR_HISTOGRAM_SIGNIFICANT_DIGITS) + start_time = time.perf_counter() - count = 0 + success_count = 0 + failed_cnt = 0 while time.perf_counter() < start_time + dur: s = time.perf_counter() try: - self.db.search_embedding( - test_data[idx], - self.k, - self.filters, - ) + self.db.search_embedding(test_data[idx], self.k) + success_count += 1 + latency_us = int((time.perf_counter() - s) * US_TO_SECONDS) + histogram.record_value(min(latency_us, HDR_HISTOGRAM_MAX_US)) except Exception as e: - log.warning(f"VectorDB search_embedding error: {e}") - traceback.print_exc(chain=True) - raise e from None + failed_cnt += 1 + # reduce log + if failed_cnt <= 3: + log.warning(f"VectorDB search_embedding error: {e}") + else: + log.debug(f"VectorDB search_embedding error: {e}") - count += 1 # loop through the test data idx = idx + 1 if idx < num - 1 else 0 - if count % 500 == 0: + if success_count % 500 == 0: log.debug( - f"({mp.current_process().name:16}) search_count: {count}, " - f"latest_latency={time.perf_counter()-s}" + f"({mp.current_process().name:16}) search_count: {success_count}", ) total_dur = round(time.perf_counter() - start_time, 4) log.debug( f"{mp.current_process().name:16} search {self.duration}s: " - f"actual_dur={total_dur}s, count={count}, qps in this process: {round(count / total_dur, 4):3}" + f"actual_dur={total_dur}s, count={success_count}, failed_cnt={failed_cnt}, " + f"qps (successful) in this process: {round(success_count / total_dur, 4):3}", ) - return count + # Pre-computed stats to avoid large data transfer + latency_stats = { + "p99": histogram.get_value_at_percentile(99) / US_TO_SECONDS, + "p95": histogram.get_value_at_percentile(95) / US_TO_SECONDS, + "avg": histogram.get_mean_value() / US_TO_SECONDS, + "count": histogram.get_total_count(), + } + + return success_count, failed_cnt, latency_stats diff --git a/vectordb_bench/backend/runner/rate_runner.py b/vectordb_bench/backend/runner/rate_runner.py index 4b32bcd9f..c56c8ca61 100644 --- a/vectordb_bench/backend/runner/rate_runner.py +++ b/vectordb_bench/backend/runner/rate_runner.py @@ -3,6 +3,7 @@ import multiprocessing as mp import time from concurrent.futures import ThreadPoolExecutor +from copy import deepcopy from vectordb_bench import config from vectordb_bench.backend.clients import api @@ -30,78 +31,116 @@ def __init__( self.insert_rate = rate self.batch_rate = rate // config.NUM_PER_BATCH + self.executing_futures = [] + self.sig_idx = 0 + def send_insert_task(self, db: api.VectorDB, emb: list[list[float]], metadata: list[str]): - db.insert_embeddings(emb, metadata) + def _insert_embeddings(db: api.VectorDB, emb: list[list[float]], metadata: list[str], retry_idx: int = 0): + _, error = db.insert_embeddings(emb, metadata) + if error is not None: + log.warning(f"Insert Failed, try_idx={retry_idx}, Exception: {error}") + retry_idx += 1 + if retry_idx <= config.MAX_INSERT_RETRY: + time.sleep(retry_idx) + _insert_embeddings(db, emb=emb, metadata=metadata, retry_idx=retry_idx) + else: + msg = f"Insert failed and retried more than {config.MAX_INSERT_RETRY} times" + raise RuntimeError(msg) from None + + if db.name == "PgVector": + # pgvector is not thread-safe for concurrent insert, + # so we need to copy the db object, make sure each thread has its own connection + db_copy = deepcopy(db) + with db_copy.init(): + _insert_embeddings(db_copy, emb, metadata, retry_idx=0) + elif db.name == "Doris": + # DorisVectorClient is not thread-safe. Similar to pgvector, create a per-thread client + # by deep-copying the wrapper and forcing lazy re-init inside the thread. + db_copy = deepcopy(db) + # Ensure a fresh client/table will be created in this thread + try: + db_copy.client = None + db_copy.table = None + except Exception: + log.debug("Failed to reset Doris client or table on thread-local copy", exc_info=True) + with db_copy.init(): + _insert_embeddings(db_copy, emb, metadata, retry_idx=0) + else: + _insert_embeddings(db, emb, metadata, retry_idx=0) @time_it def run_with_rate(self, q: mp.Queue): with ThreadPoolExecutor(max_workers=mp.cpu_count()) as executor: - executing_futures = [] @time_it def submit_by_rate() -> bool: rate = self.batch_rate for data in self.dataset: emb, metadata = get_data(data, self.normalize) - executing_futures.append( - executor.submit(self.send_insert_task, self.db, emb, metadata), - ) + self.executing_futures.append(executor.submit(self.send_insert_task, self.db, emb, metadata)) rate -= 1 if rate == 0: return False return rate == self.batch_rate + def check_and_send_signal(wait_interval: float, finished: bool = False): + try: + done, not_done = concurrent.futures.wait( + self.executing_futures, + timeout=wait_interval, + return_when=concurrent.futures.FIRST_EXCEPTION, + ) + _ = [fut.result() for fut in done] + if len(not_done) > 0: + self.executing_futures = list(not_done) + else: + self.executing_futures = [] + + self.sig_idx += len(done) + while self.sig_idx >= self.batch_rate: + self.sig_idx -= self.batch_rate + if self.sig_idx < self.batch_rate and len(not_done) == 0 and finished: + q.put(True, block=True) + else: + q.put(False, block=False) + + except Exception as e: + log.warning(f"task error, terminating, err={e}") + q.put(None, block=True) + executor.shutdown(wait=True, cancel_futures=True) + raise e from None + + time_per_batch = config.TIME_PER_BATCH with self.db.init(): + start_time = time.perf_counter() + round_idx = 0 + while True: - start_time = time.perf_counter() - finished, elapsed_time = submit_by_rate() - if finished is True: - q.put(True, block=True) - log.info(f"End of dataset, left unfinished={len(executing_futures)}") - break - - q.put(False, block=False) - wait_interval = 1 - elapsed_time if elapsed_time < 1 else 0.001 - - try: - done, not_done = concurrent.futures.wait( - executing_futures, - timeout=wait_interval, - return_when=concurrent.futures.FIRST_EXCEPTION, - ) - - if len(not_done) > 0: - log.warning( - f"Failed to finish all tasks in 1s, [{len(not_done)}/{len(executing_futures)}] " - f"tasks are not done, waited={wait_interval:.2f}, trying to wait in the next round" + if len(self.executing_futures) > 200: + log.warning("Skip data insertion this round. There are 200+ unfinished insertion tasks.") + else: + finished, elapsed_time = submit_by_rate() + if finished is True: + log.info( + f"End of dataset, left unfinished={len(self.executing_futures)}, num_round={round_idx}" ) - executing_futures = list(not_done) - else: - log.debug( - f"Finished {len(executing_futures)} insert-{config.NUM_PER_BATCH} " - f"task in 1s, wait_interval={wait_interval:.2f}" + break + if elapsed_time >= 1.5: + log.warning( + f"Submit insert tasks took {elapsed_time}s, expected 1s, " + f"indicating potential resource limitations on the client machine.", ) - executing_futures = [] - except Exception as e: - log.warning(f"task error, terminating, err={e}") - q.put(None, block=True) - executor.shutdown(wait=True, cancel_futures=True) - raise e from e - dur = time.perf_counter() - start_time - if dur < 1: - time.sleep(1 - dur) + check_and_send_signal(wait_interval=0.001, finished=False) + dur = time.perf_counter() - start_time - round_idx * time_per_batch + if dur < time_per_batch: + time.sleep(time_per_batch - dur) + round_idx += 1 # wait for all tasks in executing_futures to complete - if len(executing_futures) > 0: - try: - done, _ = concurrent.futures.wait( - executing_futures, - return_when=concurrent.futures.FIRST_EXCEPTION, - ) - except Exception as e: - log.warning(f"task error, terminating, err={e}") - q.put(None, block=True) - executor.shutdown(wait=True, cancel_futures=True) - raise e from e + while len(self.executing_futures) > 0: + check_and_send_signal(wait_interval=1, finished=True) + round_idx += 1 + + log.info(f"Finish all streaming insertion, num_round={round_idx}") diff --git a/vectordb_bench/backend/runner/read_write_runner.py b/vectordb_bench/backend/runner/read_write_runner.py index eaba51f5f..d3d1df2fa 100644 --- a/vectordb_bench/backend/runner/read_write_runner.py +++ b/vectordb_bench/backend/runner/read_write_runner.py @@ -1,13 +1,18 @@ import concurrent +import concurrent.futures import logging import math import multiprocessing as mp +import time from collections.abc import Iterable import numpy as np from vectordb_bench.backend.clients import api from vectordb_bench.backend.dataset import DatasetManager +from vectordb_bench.backend.filter import Filter, non_filter +from vectordb_bench.backend.utils import time_it +from vectordb_bench.metric import Metric from .mp_runner import MultiProcessingSearchRunner from .rate_runner import RatedMultiThreadingInsertRunner @@ -24,35 +29,39 @@ def __init__( insert_rate: int = 1000, normalize: bool = False, k: int = 100, - filters: dict | None = None, + filters: Filter = non_filter, concurrencies: Iterable[int] = (1, 15, 50), - search_stage: Iterable[float] = ( + search_stages: Iterable[float] = ( 0.5, 0.6, 0.7, 0.8, 0.9, ), # search from insert portion, 0.0 means search from the start + optimize_after_write: bool = True, read_dur_after_write: int = 300, # seconds, search duration when insertion is done timeout: float | None = None, ): self.insert_rate = insert_rate self.data_volume = dataset.data.size - for stage in search_stage: + for stage in search_stages: assert 0.0 <= stage < 1.0, "each search stage should be in [0.0, 1.0)" - self.search_stage = sorted(search_stage) + self.search_stages = sorted(search_stages) + self.optimize_after_write = optimize_after_write self.read_dur_after_write = read_dur_after_write log.info( - f"Init runner, concurencys={concurrencies}, search_stage={search_stage}, " - f"stage_search_dur={read_dur_after_write}" + f"Init runner, concurencys={concurrencies}, search_stages={self.search_stages}, " + f"stage_search_dur={read_dur_after_write}", ) - test_emb = np.stack(dataset.test_data["emb"]) if normalize: + test_emb = np.array(dataset.test_data) test_emb = test_emb / np.linalg.norm(test_emb, axis=1)[:, np.newaxis] - test_emb = test_emb.tolist() + test_emb = test_emb.tolist() + else: + test_emb = dataset.test_data MultiProcessingSearchRunner.__init__( self, @@ -74,8 +83,10 @@ def __init__( test_data=test_emb, ground_truth=dataset.gt_data, k=k, + filters=filters, ) + @time_it def run_optimize(self): """Optimize needs to run in differenct process for pymilvus schema recursion problem""" with self.db.init(): @@ -83,49 +94,133 @@ def run_optimize(self): self.db.optimize(data_size=self.data_volume) log.info("Search after write - Optimize finished") - def run_search(self): + def run_search(self, perc: int): log.info("Search after write - Serial search start") + test_time = round(time.perf_counter(), 4) res, ssearch_dur = self.serial_search_runner.run() - recall, ndcg, p99_latency = res + recall, ndcg, p99_latency, p95_latency = res + log.info( + f"Search after write - Serial search - recall={recall}, ndcg={ndcg}, " + f"p99={p99_latency}, p95={p95_latency}, dur={ssearch_dur:.4f}", + ) log.info( - f"Search after write - Serial search - recall={recall}, ndcg={ndcg}, p99={p99_latency}, " - f"dur={ssearch_dur:.4f}", + f"Search after wirte - Conc search start, dur for each conc={self.read_dur_after_write}", ) - log.info(f"Search after wirte - Conc search start, dur for each conc={self.read_dur_after_write}") - max_qps = self.run_by_dur(self.read_dur_after_write) + result = self.run_by_dur(self.read_dur_after_write) + max_qps = result[0] + conc_failed_rate = result[1] + conc_num_list = result[2] + conc_qps_list = result[3] + conc_latency_p99_list = result[4] + conc_latency_p95_list = result[5] + conc_latency_avg_list = result[6] log.info(f"Search after wirte - Conc search finished, max_qps={max_qps}") - return (max_qps, recall, ndcg, p99_latency) + return [ + ( + perc, + test_time, + max_qps, + recall, + ndcg, + p99_latency, + p95_latency, + conc_failed_rate, + conc_num_list, + conc_qps_list, + conc_latency_p99_list, + conc_latency_p95_list, + conc_latency_avg_list, + ) + ] - def run_read_write(self): - with mp.Manager() as m: - q = m.Queue() - with concurrent.futures.ProcessPoolExecutor( - mp_context=mp.get_context("spawn"), - max_workers=2, - ) as executor: - read_write_futures = [] - read_write_futures.append(executor.submit(self.run_with_rate, q)) - read_write_futures.append(executor.submit(self.run_search_by_sig, q)) + def run_read_write(self) -> Metric: + """ + Test search performance with a fixed insert rate. + - Insert requests are sent to VectorDB at a fixed rate within a dedicated insert process pool. + - if the database cannot promptly process these requests, the process pool will accumulate insert tasks. + - Search Tests are categorized into three types: + - streaming_search: Initiates a new search test upon receiving a signal that the inserted data has + reached the search_stage. + - streaming_end_search: initiates a new search test after all data has been inserted. + - optimized_search (optional): After the streaming_end_search, optimizes and initiates a search test. + """ + m = Metric() + with mp.Manager() as mp_manager: + q = mp_manager.Queue() + with concurrent.futures.ProcessPoolExecutor(mp_context=mp.get_context("spawn"), max_workers=2) as executor: + insert_future = executor.submit(self.run_with_rate, q) + streaming_search_future = executor.submit(self.run_search_by_sig, q) try: - for f in concurrent.futures.as_completed(read_write_futures): - res = f.result() - log.info(f"Result = {res}") + start_time = time.perf_counter() + _, m.insert_duration = insert_future.result() + streaming_search_res = streaming_search_future.result() + if streaming_search_res is None: + streaming_search_res = [] + + streaming_end_search_future = executor.submit(self.run_search, 100) + streaming_end_search_res = streaming_end_search_future.result() # Wait for read_write_futures finishing and do optimize and search - op_future = executor.submit(self.run_optimize) - op_future.result() + if self.optimize_after_write: + op_future = executor.submit(self.run_optimize) + _, m.optimize_duration = op_future.result() + log.info(f"Optimize cost {m.optimize_duration}s") + optimized_search_future = executor.submit(self.run_search, 110) + optimized_search_res = optimized_search_future.result() + else: + log.info("Skip optimization and search") + optimized_search_res = [] - search_future = executor.submit(self.run_search) - last_res = search_future.result() + r = [*streaming_search_res, *streaming_end_search_res, *optimized_search_res] + m.st_search_stage_list = [d[0] for d in r] + m.st_search_time_list = [round(d[1] - start_time, 4) for d in r] + m.st_max_qps_list_list = [d[2] for d in r] + m.st_recall_list = [d[3] for d in r] + m.st_ndcg_list = [d[4] for d in r] + m.st_serial_latency_p99_list = [d[5] for d in r] + m.st_serial_latency_p95_list = [d[6] for d in r] + m.st_conc_failed_rate_list = [d[7] for d in r] + + # Extract concurrent latency data + m.st_conc_num_list_list = [d[8] for d in r] + m.st_conc_qps_list_list = [d[9] for d in r] + m.st_conc_latency_p99_list_list = [d[10] for d in r] + m.st_conc_latency_p95_list_list = [d[11] for d in r] + m.st_conc_latency_avg_list_list = [d[12] for d in r] - log.info(f"Max QPS after optimze and search: {last_res}") except Exception as e: log.warning(f"Read and write error: {e}") executor.shutdown(wait=True, cancel_futures=True) - raise e from e - log.info("Concurrent read write all done") + # raise e + m.st_ideal_insert_duration = math.ceil(self.data_volume / self.insert_rate) + log.info(f"Concurrent read write all done, results: {m}") + return m + + def get_each_conc_search_dur(self, ssearch_dur: float, cur_stage: float, next_stage: float) -> float: + # Search duration for non-last search stage is carefully calculated. + # If duration for each concurrency is less than 30s, runner will raise error. + total_dur_between_stages = self.data_volume * (next_stage - cur_stage) // self.insert_rate + csearch_dur = total_dur_between_stages - ssearch_dur + + # Try to leave room for init process executors + if csearch_dur > 60: + csearch_dur -= 30 + elif csearch_dur > 30: + csearch_dur -= 15 + else: + csearch_dur /= 2 + + each_conc_search_dur = round(csearch_dur / len(self.concurrencies), 4) + if each_conc_search_dur < 30: + warning_msg = ( + f"Results might be inaccurate, duration[{csearch_dur:.4f}] left for conc-search is too short, " + f"total available dur={total_dur_between_stages}, serial_search_cost={ssearch_dur}, " + f"each_conc_search_dur={each_conc_search_dur}." + ) + log.warning(warning_msg) + return each_conc_search_dur def run_search_by_sig(self, q: mp.Queue): """ @@ -137,7 +232,7 @@ def run_search_by_sig(self, q: mp.Queue): """ result, start_batch = [], 0 total_batch = math.ceil(self.data_volume / self.insert_rate) - recall, ndcg, p99_latency = None, None, None + recall, ndcg, p99_latency, p95_latency = None, None, None, None def wait_next_target(start: int, target_batch: int) -> bool: """Return False when receive True or None""" @@ -149,7 +244,7 @@ def wait_next_target(start: int, target_batch: int) -> bool: start += 1 return True - for idx, stage in enumerate(self.search_stage): + for idx, stage in enumerate(self.search_stages): target_batch = int(total_batch * stage) perc = int(stage * 100) @@ -159,41 +254,59 @@ def wait_next_target(start: int, target_batch: int) -> bool: return None log.info(f"Insert {perc}% done, total batch={total_batch}") - log.info(f"[{target_batch}/{total_batch}] Serial search - {perc}% start") - res, ssearch_dur = self.serial_search_runner.run() - recall, ndcg, p99_latency = res - log.info( - f"[{target_batch}/{total_batch}] Serial search - {perc}% done, recall={recall}, " - f"ndcg={ndcg}, p99={p99_latency}, dur={ssearch_dur:.4f}" - ) - - # Search duration for non-last search stage is carefully calculated. - # If duration for each concurrency is less than 30s, runner will raise error. - if idx < len(self.search_stage) - 1: - total_dur_between_stages = self.data_volume * (self.search_stage[idx + 1] - stage) // self.insert_rate - csearch_dur = total_dur_between_stages - ssearch_dur + test_time = round(time.perf_counter(), 4) + max_qps, recall, ndcg, p99_latency, p95_latency, conc_failed_rate = 0, 0, 0, 0, 0, 0 + conc_num_list, conc_qps_list = [], [] + conc_latency_p99_list, conc_latency_p95_list, conc_latency_avg_list = [], [], [] + try: + log.info(f"[{target_batch}/{total_batch}] Serial search - {perc}% start") + res, ssearch_dur = self.serial_search_runner.run() + ssearch_dur = round(ssearch_dur, 4) + recall, ndcg, p99_latency, p95_latency = res + log.info( + f"[{target_batch}/{total_batch}] Serial search - {perc}% done, " + f"recall={recall}, ndcg={ndcg}, p99={p99_latency}, p95={p95_latency}, dur={ssearch_dur}" + ) - # Try to leave room for init process executors - csearch_dur = csearch_dur - 30 if csearch_dur > 60 else csearch_dur - - each_conc_search_dur = csearch_dur / len(self.concurrencies) - if each_conc_search_dur < 30: - warning_msg = ( - f"Results might be inaccurate, duration[{csearch_dur:.4f}] left for conc-search is too short, " - f"total available dur={total_dur_between_stages}, serial_search_cost={ssearch_dur}." + each_conc_search_dur = self.get_each_conc_search_dur( + ssearch_dur, + cur_stage=stage, + next_stage=self.search_stages[idx + 1] if idx < len(self.search_stages) - 1 else 1.0, + ) + if each_conc_search_dur > 10: + log.info( + f"[{target_batch}/{total_batch}] Concurrent search - {perc}% start, " + f"dur={each_conc_search_dur:.4f}" ) - log.warning(warning_msg) - - # The last stage - else: - each_conc_search_dur = 60 - - log.info( - f"[{target_batch}/{total_batch}] Concurrent search - {perc}% start, dur={each_conc_search_dur:.4f}" + conc_result = self.run_by_dur(each_conc_search_dur) + max_qps = conc_result[0] + conc_failed_rate = conc_result[1] + conc_num_list = conc_result[2] + conc_qps_list = conc_result[3] + conc_latency_p99_list = conc_result[4] + conc_latency_p95_list = conc_result[5] + conc_latency_avg_list = conc_result[6] + else: + log.warning(f"Skip concurrent tests, each_conc_search_dur={each_conc_search_dur} less than 10s.") + except Exception as e: + log.warning(f"Streaming Search Failed at stage={stage}. Exception: {e}") + result.append( + ( + perc, + test_time, + max_qps, + recall, + ndcg, + p99_latency, + p95_latency, + conc_failed_rate, + conc_num_list, + conc_qps_list, + conc_latency_p99_list, + conc_latency_p95_list, + conc_latency_avg_list, + ) ) - max_qps = self.run_by_dur(each_conc_search_dur) - result.append((perc, max_qps, recall, ndcg, p99_latency)) - start_batch = target_batch # Drain the queue diff --git a/vectordb_bench/backend/runner/serial_runner.py b/vectordb_bench/backend/runner/serial_runner.py index 365641132..300553a4e 100644 --- a/vectordb_bench/backend/runner/serial_runner.py +++ b/vectordb_bench/backend/runner/serial_runner.py @@ -6,10 +6,10 @@ import traceback import numpy as np -import pandas as pd import psutil from vectordb_bench.backend.dataset import DatasetManager +from vectordb_bench.backend.filter import Filter, FilterOp, non_filter from ... import config from ...metric import calc_ndcg, calc_recall, get_ideal_dcg @@ -18,8 +18,7 @@ from ..clients import api NUM_PER_BATCH = config.NUM_PER_BATCH -LOAD_MAX_TRY_COUNT = 10 -WAITTING_TIME = 60 +LOAD_MAX_TRY_COUNT = config.LOAD_MAX_TRY_COUNT log = logging.getLogger(__name__) @@ -30,12 +29,26 @@ def __init__( db: api.VectorDB, dataset: DatasetManager, normalize: bool, + filters: Filter = non_filter, timeout: float | None = None, ): self.timeout = timeout if isinstance(timeout, int | float) else None self.dataset = dataset self.db = db self.normalize = normalize + self.filters = filters + + def retry_insert(self, db: api.VectorDB, retry_idx: int = 0, **kwargs): + _, error = db.insert_embeddings(**kwargs) + if error is not None: + log.warning(f"Insert Failed, try_idx={retry_idx}, Exception: {error}") + retry_idx += 1 + if retry_idx <= config.MAX_INSERT_RETRY: + time.sleep(retry_idx) + self.retry_insert(db, retry_idx=retry_idx, **kwargs) + else: + msg = f"Insert failed and retried more than {config.MAX_INSERT_RETRY} times" + raise RuntimeError(msg) from None def task(self) -> int: count = 0 @@ -43,9 +56,9 @@ def task(self) -> int: log.info(f"({mp.current_process().name:16}) Start inserting embeddings in batch {config.NUM_PER_BATCH}") start = time.perf_counter() for data_df in self.dataset: - all_metadata = data_df["id"].tolist() + all_metadata = data_df[self.dataset.data.train_id_field].tolist() - emb_np = np.stack(data_df["emb"]) + emb_np = np.stack(data_df[self.dataset.data.train_vector_field]) if self.normalize: log.debug("normalize the 100k train data") all_embeddings = (emb_np / np.linalg.norm(emb_np, axis=1)[:, np.newaxis]).tolist() @@ -54,12 +67,25 @@ def task(self) -> int: del emb_np log.debug(f"batch dataset size: {len(all_embeddings)}, {len(all_metadata)}") + labels_data = None + if self.filters.type == FilterOp.StrEqual: + if self.dataset.data.scalar_labels_file_separated: + labels_data = self.dataset.scalar_labels[self.filters.label_field][all_metadata].to_list() + else: + labels_data = data_df[self.filters.label_field].tolist() + insert_count, error = self.db.insert_embeddings( embeddings=all_embeddings, metadata=all_metadata, + labels_data=labels_data, ) if error is not None: - raise error + self.retry_insert( + self.db, + embeddings=all_embeddings, + metadata=all_metadata, + labels_data=labels_data, + ) assert insert_count == len(all_metadata) count += insert_count @@ -101,7 +127,7 @@ def endless_insert_data(self, all_embeddings: list, all_metadata: list, left_id: already_insert_count += insert_count if error is not None: retry_count += 1 - time.sleep(WAITTING_TIME) + time.sleep(10) log.info(f"Failed to insert data, try {retry_count} time") if retry_count >= LOAD_MAX_TRY_COUNT: @@ -149,8 +175,8 @@ def run_endlessness(self) -> int: # only 1 file data_df = next(iter(self.dataset)) all_embeddings, all_metadata = ( - np.stack(data_df["emb"]).tolist(), - data_df["id"].tolist(), + np.stack(data_df[self.dataset.data.train_vector_field]).tolist(), + data_df[self.dataset.data.train_id_field].tolist(), ) start_time = time.perf_counter() @@ -179,7 +205,7 @@ def run_endlessness(self) -> int: raise LoadTimeoutError(self.timeout) def run(self) -> int: - count, dur = self._insert_all_batches() + count, _ = self._insert_all_batches() return count @@ -188,9 +214,9 @@ def __init__( self, db: api.VectorDB, test_data: list[list[float]], - ground_truth: pd.DataFrame, + ground_truth: list[list[int]], k: int = 100, - filters: dict | None = None, + filters: Filter = non_filter, ): self.db = db self.k = k @@ -202,35 +228,47 @@ def __init__( self.test_data = test_data self.ground_truth = ground_truth - def search(self, args: tuple[list, pd.DataFrame]) -> tuple[float, float, float]: + def _get_db_search_res(self, emb: list[float], retry_idx: int = 0) -> list[int]: + try: + results = self.db.search_embedding(emb, self.k) + except Exception as e: + log.warning(f"Serial search failed, retry_idx={retry_idx}, Exception: {e}") + if retry_idx < config.MAX_SEARCH_RETRY: + return self._get_db_search_res(emb=emb, retry_idx=retry_idx + 1) + + msg = f"Serial search failed and retried more than {config.MAX_SEARCH_RETRY} times" + raise RuntimeError(msg) from e + + return results + + def search(self, args: tuple[list, list[list[int]]]) -> tuple[float, float, float, float]: log.info(f"{mp.current_process().name:14} start search the entire test_data to get recall and latency") with self.db.init(): + self.db.prepare_filter(self.filters) test_data, ground_truth = args ideal_dcg = get_ideal_dcg(self.k) log.debug(f"test dataset size: {len(test_data)}") - log.debug(f"ground truth size: {ground_truth.columns}, shape: {ground_truth.shape}") + log.debug(f"ground truth size: {len(ground_truth)}") latencies, recalls, ndcgs = [], [], [] for idx, emb in enumerate(test_data): s = time.perf_counter() try: - results = self.db.search_embedding( - emb, - self.k, - self.filters, - ) - + results = self._get_db_search_res(emb) except Exception as e: log.warning(f"VectorDB search_embedding error: {e}") - traceback.print_exc(chain=True) raise e from None latencies.append(time.perf_counter() - s) - gt = ground_truth["neighbors_id"][idx] - recalls.append(calc_recall(self.k, gt[: self.k], results)) - ndcgs.append(calc_ndcg(gt[: self.k], results, ideal_dcg)) + if ground_truth is not None: + gt = ground_truth[idx] + recalls.append(calc_recall(self.k, gt[: self.k], results)) + ndcgs.append(calc_ndcg(gt[: self.k], results, ideal_dcg)) + else: + recalls.append(0) + ndcgs.append(0) if len(latencies) % 100 == 0: log.debug( @@ -243,27 +281,43 @@ def search(self, args: tuple[list, pd.DataFrame]) -> tuple[float, float, float]: avg_ndcg = round(np.mean(ndcgs), 4) cost = round(np.sum(latencies), 4) p99 = round(np.percentile(latencies, 99), 4) + p95 = round(np.percentile(latencies, 95), 4) log.info( f"{mp.current_process().name:14} search entire test_data: " f"cost={cost}s, " f"queries={len(latencies)}, " f"avg_recall={avg_recall}, " - f"avg_ndcg={avg_ndcg}," + f"avg_ndcg={avg_ndcg}, " f"avg_latency={avg_latency}, " - f"p99={p99}" + f"p99={p99}, " + f"p95={p95}" ) - return (avg_recall, avg_ndcg, p99) + return (avg_recall, avg_ndcg, p99, p95) - def _run_in_subprocess(self) -> tuple[float, float]: + def _run_in_subprocess(self) -> tuple[float, float, float, float]: with concurrent.futures.ProcessPoolExecutor(max_workers=1) as executor: future = executor.submit(self.search, (self.test_data, self.ground_truth)) return future.result() @utils.time_it - def run(self) -> tuple[float, float, float]: + def run(self) -> tuple[float, float, float, float]: + log.info(f"{mp.current_process().name:14} start serial search") + if self.test_data is None: + msg = "empty test_data" + raise RuntimeError(msg) + + return self._run_in_subprocess() + + @utils.time_it + def run_with_cost(self) -> tuple[tuple[float, float, float, float], float]: """ + Search all test data in serial. Returns: - tuple[tuple[float, float, float], float]: (avg_recall, avg_ndcg, p99_latency), cost - + tuple[tuple[float, float, float, float], float]: (avg_recall, avg_ndcg, p99_latency, p95_latency), cost """ + log.info(f"{mp.current_process().name:14} start serial search") + if self.test_data is None: + msg = "empty test_data" + raise RuntimeError(msg) + return self._run_in_subprocess() diff --git a/vectordb_bench/backend/task_runner.py b/vectordb_bench/backend/task_runner.py index 2a583b4f5..8224a0415 100644 --- a/vectordb_bench/backend/task_runner.py +++ b/vectordb_bench/backend/task_runner.py @@ -1,20 +1,21 @@ import concurrent +import hashlib import logging +import re import traceback from enum import Enum, auto import numpy as np import psutil -from vectordb_bench.base import BaseModel -from vectordb_bench.metric import Metric -from vectordb_bench.models import PerformanceTimeoutError, TaskConfig, TaskStage - +from ..base import BaseModel +from ..metric import Metric +from ..models import PerformanceTimeoutError, TaskConfig, TaskStage from . import utils -from .cases import Case, CaseLabel -from .clients import MetricType, api +from .cases import Case, CaseLabel, StreamingPerformanceCase +from .clients import DB, MetricType, api from .data_source import DatasetSource -from .runner import MultiProcessingSearchRunner, SerialInsertRunner, SerialSearchRunner +from .runner import MultiProcessingSearchRunner, ReadWriteRunner, SerialInsertRunner, SerialSearchRunner log = logging.getLogger(__name__) @@ -48,6 +49,7 @@ class CaseRunner(BaseModel): serial_search_runner: SerialSearchRunner | None = None search_runner: MultiProcessingSearchRunner | None = None final_search_runner: MultiProcessingSearchRunner | None = None + read_write_runner: ReadWriteRunner | None = None def __eq__(self, obj: any): if isinstance(obj, CaseRunner): @@ -59,10 +61,22 @@ def __eq__(self, obj: any): ) return False + def __hash__(self) -> int: + """Hash method to maintain consistency with __eq__ method.""" + return hash( + ( + self.ca.label, + self.config.db, + self.config.db_case_config, + self.ca.dataset, + ) + ) + def display(self) -> dict: c_dict = self.ca.dict( include={ "label": True, + "name": True, "filters": True, "dataset": { "data": { @@ -85,18 +99,42 @@ def normalize(self) -> bool: def init_db(self, drop_old: bool = True) -> None: db_cls = self.config.db.init_cls + # Compose a compact, case-unique collection/table name for Doris to avoid cross-case interference + collection_name = None + try: + if self.config.db == DB.Doris: + # Primary identifier = case-type enum name from CLI (e.g., Performance768D10M) + case_type_name = self.config.case_config.case_id.name + base = f"{case_type_name.lower()}" + # Sanitize to [a-z0-9_] + base = re.sub(r"[^a-z0-9_]+", "_", base).strip("_") + # Cap to 63 chars; add short hash if truncated + if len(base) > 63: + h = hashlib.md5(base.encode(), usedforsecurity=False).hexdigest()[:6] + base = f"{base[:(63-7)]}_{h}" + collection_name = base + except Exception: + # If anything goes wrong, fall back silently; Doris will use its default name logic + collection_name = None + + # Check if collection_name is in the db_config (e.g., for Zilliz, Milvus) + db_config_dict = self.config.db_config.to_dict() + if "collection_name" in db_config_dict and not collection_name: + collection_name = db_config_dict.pop("collection_name") self.db = db_cls( dim=self.ca.dataset.data.dim, - db_config=self.config.db_config.to_dict(), + db_config=db_config_dict, db_case_config=self.config.db_case_config, drop_old=drop_old, + with_scalar_labels=self.ca.with_scalar_labels, + **({"collection_name": collection_name} if collection_name else {}), ) def _pre_run(self, drop_old: bool = True): try: self.init_db(drop_old) - self.ca.dataset.prepare(self.dataset_source, filters=self.ca.filter_rate) + self.ca.dataset.prepare(self.dataset_source, filters=self.ca.filters) except ModuleNotFoundError as e: log.warning(f"pre run case error: please install client for db: {self.config.db}, error={e}") raise e from None @@ -110,6 +148,8 @@ def run(self, drop_old: bool = True) -> Metric: return self._run_capacity_case() if self.ca.label == CaseLabel.Performance: return self._run_perf_case(drop_old) + if self.ca.label == CaseLabel.Streaming: + return self._run_streaming_case() msg = f"unknown case type: {self.ca.label}" log.warning(msg) raise ValueError(msg) @@ -127,6 +167,7 @@ def _run_capacity_case(self) -> Metric: self.db, self.ca.dataset, self.normalize, + self.ca.filters, self.ca.load_timeout, ) count = runner.run_endlessness() @@ -151,6 +192,8 @@ def _run_perf_case(self, drop_old: bool = True) -> Metric: if TaskStage.LOAD in self.config.stages: _, load_dur = self._load_train_data() build_dur = self._optimize() + m.insert_duration = round(load_dur, 4) + m.optimize_duration = round(build_dur, 4) m.load_duration = round(load_dur + build_dur, 4) log.info( f"Finish loading the entire dataset into VectorDB," @@ -168,15 +211,12 @@ def _run_perf_case(self, drop_old: bool = True) -> Metric: m.conc_num_list, m.conc_qps_list, m.conc_latency_p99_list, + m.conc_latency_p95_list, m.conc_latency_avg_list, ) = search_results if TaskStage.SEARCH_SERIAL in self.config.stages: search_results = self._serial_search() - """ - m.recall = search_results.recall - m.serial_latencies = search_results.serial_latencies - """ - m.recall, m.ndcg, m.serial_latency_p99 = search_results + m.recall, m.ndcg, m.serial_latency_p99, m.serial_latency_p95 = search_results except Exception as e: log.warning(f"Failed to run performance case, reason = {e}") @@ -186,6 +226,19 @@ def _run_perf_case(self, drop_old: bool = True) -> Metric: log.info(f"Performance case got result: {m}") return m + def _run_streaming_case(self) -> Metric: + log.info("Start streaming case") + try: + self._init_read_write_runner() + m = self.read_write_runner.run_read_write() + except Exception as e: + log.warning(f"Failed to run streaming case, reason = {e}") + traceback.print_exc() + raise e from None + else: + log.info(f"Streaming case got result: {m}") + return m + @utils.time_it def _load_train_data(self): """Insert train data and get the insert_duration""" @@ -194,6 +247,7 @@ def _load_train_data(self): self.db, self.ca.dataset, self.normalize, + self.ca.filters, self.ca.load_timeout, ) runner.run() @@ -202,12 +256,12 @@ def _load_train_data(self): finally: runner = None - def _serial_search(self) -> tuple[float, float, float]: + def _serial_search(self) -> tuple[float, float, float, float]: """Performance serial tests, search the entire test data once, - calculate the recall, serial_latency_p99 + calculate the recall, serial_latency_p99, serial_latency_p95 Returns: - tuple[float, float]: recall, serial_latency_p99 + tuple[float, float, float, float]: recall, ndcg, serial_latency_p99, serial_latency_p95 """ try: results, _ = self.serial_search_runner.run() @@ -253,10 +307,12 @@ def _optimize(self) -> float: raise e from None def _init_search_runner(self): - test_emb = np.stack(self.ca.dataset.test_data["emb"]) if self.normalize: + test_emb = np.stack(self.ca.dataset.test_data) test_emb = test_emb / np.linalg.norm(test_emb, axis=1)[:, np.newaxis] - self.test_emb = test_emb.tolist() + self.test_emb = test_emb.tolist() + else: + self.test_emb = self.ca.dataset.test_data gt_df = self.ca.dataset.gt_data @@ -275,9 +331,24 @@ def _init_search_runner(self): filters=self.ca.filters, concurrencies=self.config.case_config.concurrency_search_config.num_concurrency, duration=self.config.case_config.concurrency_search_config.concurrency_duration, + concurrency_timeout=self.config.case_config.concurrency_search_config.concurrency_timeout, k=self.config.case_config.k, ) + def _init_read_write_runner(self): + ca: StreamingPerformanceCase = self.ca + self.read_write_runner = ReadWriteRunner( + db=self.db, + dataset=ca.dataset, + insert_rate=ca.insert_rate, + search_stages=ca.search_stages, + optimize_after_write=ca.optimize_after_write, + read_dur_after_write=ca.read_dur_after_write, + concurrencies=ca.concurrencies, + k=self.config.case_config.k, + normalize=self.normalize, + ) + def stop(self): if self.search_runner: self.search_runner.stop() @@ -315,12 +386,7 @@ def display(self) -> None: fmt.append(DATA_FORMAT % ("-" * 11, "-" * 12, "-" * 20, "-" * 7, "-" * 7)) for f in self.case_runners: - if f.ca.filter_rate != 0.0: - filters = f.ca.filter_rate - elif f.ca.filter_size != 0: - filters = f.ca.filter_size - else: - filters = "None" + filters = f.ca.filters.filter_rate ds_str = f"{f.ca.dataset.data.name}-{f.ca.dataset.data.label}-{utils.numerize(f.ca.dataset.data.size)}" fmt.append( diff --git a/vectordb_bench/cli/batch_cli.py b/vectordb_bench/cli/batch_cli.py new file mode 100644 index 000000000..5ac2b1cf1 --- /dev/null +++ b/vectordb_bench/cli/batch_cli.py @@ -0,0 +1,121 @@ +import logging +import time +from collections.abc import MutableMapping +from concurrent.futures import wait +from pathlib import Path +from typing import Annotated, Any, TypedDict + +import click +from click.testing import CliRunner +from yaml import Loader, load + +from .. import config +from ..cli.cli import ( + cli, + click_parameter_decorators_from_typed_dict, +) + +log = logging.getLogger(__name__) + + +def click_get_defaults_from_file(ctx, param, value): # noqa: ANN001, ARG001 + if not value: + raise click.MissingParameter + path = Path(value) + input_file = path if path.exists() else Path(config.CONFIG_LOCAL_DIR, path) + try: + with input_file.open() as f: + _config: dict[str, list[dict[str, Any]]] = load(f.read(), Loader=Loader) # noqa: S506 + ctx.default_map = _config + except Exception as e: + msg = f"Failed to load batch config file: {e}" + raise click.BadParameter(msg) from e + return value + + +class BatchCliTypedDict(TypedDict): + batch_config_file: Annotated[ + bool, + click.option( + "--batch-config-file", + type=click.Path(), + callback=click_get_defaults_from_file, + is_eager=True, + expose_value=False, + help="Read batch configuration from yaml file", + ), + ] + + +def build_sub_cmd_args(batch_config: MutableMapping[str, Any] | None): + bool_options = { + "drop_old": True, + "load": True, + "search_serial": True, + "search_concurrent": True, + "dry_run": False, + "custom_dataset_use_shuffled": True, + "custom_dataset_with_gt": True, + } + + def format_option(key: str, value: Any): + opt_name = key.replace("_", "-") + + if key in bool_options: + return format_bool_option(opt_name, value, skip=False) + + if key.startswith("skip_"): + raw_key = key[5:] + raw_opt = raw_key.replace("_", "-") + return format_bool_option(raw_opt, value, skip=True, raw_key=raw_key) + + return [f"--{opt_name}", str(value)] + + def format_bool_option(opt_name: str, value: Any, skip: bool = False, raw_key: str | None = None): + if isinstance(value, bool): + if skip: + if bool_options.get(raw_key, False): + return [f"--skip-{opt_name}"] if value else [f"--{opt_name}"] + return [f"--{opt_name}", str(value)] + if value: + return [f"--{opt_name}"] + if bool_options.get(opt_name.replace("-", "_"), False): + return [f"--skip-{opt_name}"] + return [] + return [f"--{opt_name}", str(value)] + + args_arr = [] + for sub_cmd_key, sub_cmd_config_list in batch_config.items(): + for sub_cmd_args in sub_cmd_config_list: + args = [sub_cmd_key] + for k, v in sub_cmd_args.items(): + args.extend(format_option(k, v)) + args_arr.append(args) + + return args_arr + + +@cli.command() +@click_parameter_decorators_from_typed_dict(BatchCliTypedDict) +def BatchCli(): + ctx = click.get_current_context() + batch_config = ctx.default_map + + runner = CliRunner() + + args_arr = build_sub_cmd_args(batch_config) + + for args in args_arr: + log.info(f"got batch config: {' '.join(args)}") + + for args in args_arr: + result = runner.invoke(cli, args) + time.sleep(5) + + from ..interface import global_result_future + + if global_result_future: + wait([global_result_future]) + + if result.exception: + log.exception(f"failed to run sub command: {args[0]}", exc_info=result.exception) diff --git a/vectordb_bench/cli/cli.py b/vectordb_bench/cli/cli.py index 4b42a912c..12bb4be9b 100644 --- a/vectordb_bench/cli/cli.py +++ b/vectordb_bench/cli/cli.py @@ -17,10 +17,9 @@ import click from yaml import load -from vectordb_bench.backend.clients.api import MetricType - from .. import config from ..backend.clients import DB +from ..backend.clients.api import MetricType from ..interface import benchmark_runner, global_result_future from ..models import ( CaseConfig, @@ -111,7 +110,7 @@ def deco(f): # noqa: ANN001 return deco -def click_arg_split(ctx: click.Context, param: click.core.Option, value): # noqa: ANN001, ARG001 +def click_arg_split(ctx: click.Context, param: click.core.Option, value: any): # noqa: ARG001 """Will split a comma-separated list input into an actual list. Args: @@ -184,6 +183,16 @@ def get_custom_case_config(parameters: dict) -> dict: "with_gt": parameters["custom_dataset_with_gt"], }, } + elif parameters["case_type"] == "NewIntFilterPerformanceCase": + custom_case_config = { + "dataset_with_size_type": parameters["dataset_with_size_type"], + "filter_rate": parameters["filter_rate"], + } + elif parameters["case_type"] == "LabelFilterPerformanceCase": + custom_case_config = { + "dataset_with_size_type": parameters["dataset_with_size_type"], + "label_percentage": parameters["label_percentage"], + } return custom_case_config @@ -303,6 +312,17 @@ class CommonTypedDict(TypedDict): callback=lambda *args: list(map(int, click_arg_split(*args))), ), ] + concurrency_timeout: Annotated[ + int, + click.option( + "--concurrency-timeout", + type=int, + default=config.CONCURRENCY_TIMEOUT, + show_default=True, + help="Timeout (in seconds) to wait for a concurrency slot before failing. " + "Set to a negative value to wait indefinitely.", + ), + ] custom_case_name: Annotated[ str, click.option( @@ -405,6 +425,36 @@ class CommonTypedDict(TypedDict): show_default=True, ), ] + task_label: Annotated[str, click.option("--task-label", help="Task label")] + dataset_with_size_type: Annotated[ + str, + click.option( + "--dataset-with-size-type", + help="Dataset with size type for NewIntFilterPerformanceCase/LabelFilterPerformanceCase, you can use " + "Medium Cohere (768dim, 1M)|Large Cohere (768dim, 10M)|Medium Bioasq (1024dim, 1M)|" + "Large Bioasq (1024dim, 10M)|Large OpenAI (1536dim, 5M)|Medium OpenAI (1536dim, 500K)", + default="Medium Cohere (768dim, 1M)", + show_default=True, + ), + ] + filter_rate: Annotated[ + float, + click.option( + "--filter-rate", + help="Filter rate for NewIntFilterPerformanceCase", + default=0.01, + show_default=True, + ), + ] + label_percentage: Annotated[ + float, + click.option( + "--label-percentage", + help="Filter rate for LabelFilterPerformanceCase", + default=0.01, + show_default=True, + ), + ] class HNSWBaseTypedDict(TypedDict): @@ -444,6 +494,49 @@ class HNSWFlavor3(HNSWBaseRequiredTypedDict): ] +class HNSWFlavor4(HNSWBaseRequiredTypedDict): + ef_search: Annotated[ + int | None, + click.option("--ef-search", type=int, help="hnsw ef-search", required=True), + ] + index_type: Annotated[ + str | None, + click.option( + "--index-type", + type=click.Choice(["HNSW", "HNSW_SQ", "HNSW_BQ"], case_sensitive=False), + help="Type of index to use. Supported values: HNSW, HNSW_SQ, HNSW_BQ", + required=True, + ), + ] + + +class HNSWFlavor5(HNSWBaseRequiredTypedDict): + ef_search: Annotated[ + int | None, + click.option("--ef-search", type=int, help="hnsw ef-search", required=True), + ] + index_type: Annotated[ + str | None, + click.option( + "--index-type", + type=click.Choice(["HGraph"], case_sensitive=True), + help="Type of index to use. Supported values: HGraph", + required=True, + ), + ] + use_reorder: Annotated[ + bool, + click.option( + "--use-reorder/--no-use-reorder", + is_flag=True, + type=bool, + help="use reorder index", + default=True, + show_default=True, + ), + ] + + class IVFFlatTypedDict(TypedDict): lists: Annotated[int | None, click.option("--lists", type=int, help="ivfflat lists")] probes: Annotated[int | None, click.option("--probes", type=int, help="ivfflat probes")] @@ -460,6 +553,57 @@ class IVFFlatTypedDictN(TypedDict): ] +class OceanBaseIVFTypedDict(TypedDict): + index_type: Annotated[ + str | None, + click.option( + "--index-type", + type=click.Choice(["IVF_FLAT", "IVF_SQ8", "IVF_PQ"], case_sensitive=False), + help="Type of index to use. Supported values: IVF_FLAT, IVF_SQ8, IVF_PQ", + required=True, + ), + ] + nlist: Annotated[ + int | None, + click.option("--nlist", "nlist", type=int, help="Number of cluster centers", required=True), + ] + nbits: Annotated[ + int | None, + click.option( + "--nbits", + "nbits", + type=int, + help="Number of bits used to encode the index of a sub-vector's centroid in the compressed representation", + ), + ] + sample_per_nlist: Annotated[ + int | None, + click.option( + "--sample_per_nlist", + "sample_per_nlist", + type=int, + help="The cluster centers are calculated by total sampling sample_per_nlist * nlist vectors", + required=True, + ), + ] + ivf_nprobes: Annotated[ + int | None, + click.option( + "--ivf_nprobes", + "ivf_nprobes", + type=str, + help="How many clustering centers to search during the query", + required=True, + ), + ] + m: Annotated[ + int | None, + click.option( + "--m", "m", type=int, help="The number of sub-vectors that each data vector is divided into during IVF-PQ" + ), + ] + + @click.group() def cli(): ... @@ -489,6 +633,7 @@ def run( concurrency_search_config=ConcurrencySearchConfig( concurrency_duration=parameters["concurrency_duration"], num_concurrency=[int(s) for s in parameters["num_concurrency"]], + concurrency_timeout=parameters["concurrency_timeout"], ), custom_case=get_custom_case_config(parameters), ), @@ -499,10 +644,14 @@ def run( parameters["search_concurrent"], ), ) + task_label = parameters["task_label"] log.info(f"Task:\n{pformat(task)}\n") if not parameters["dry_run"]: - benchmark_runner.run([task]) + benchmark_runner.run([task], task_label) time.sleep(5) if global_result_future: wait([global_result_future]) + + while benchmark_runner.has_running(): + time.sleep(1) diff --git a/vectordb_bench/cli/vectordbbench.py b/vectordb_bench/cli/vectordbbench.py index 1c6ff1260..76e9534f9 100644 --- a/vectordb_bench/cli/vectordbbench.py +++ b/vectordb_bench/cli/vectordbbench.py @@ -1,19 +1,39 @@ +from ..backend.clients.alisql.cli import AliSQLHNSW from ..backend.clients.alloydb.cli import AlloyDBScaNN from ..backend.clients.aws_opensearch.cli import AWSOpenSearch +from ..backend.clients.chroma.cli import Chroma from ..backend.clients.clickhouse.cli import Clickhouse +from ..backend.clients.cockroachdb.cli import CockroachDB as CockroachDBCli +from ..backend.clients.doris.cli import Doris +from ..backend.clients.elastic_cloud.cli import ( + ElasticCloudHNSW, + ElasticCloudHNSWBBQ, + ElasticCloudHNSWInt4, + ElasticCloudHNSWInt8, +) +from ..backend.clients.hologres.cli import HologresHGraph +from ..backend.clients.lancedb.cli import LanceDB from ..backend.clients.mariadb.cli import MariaDBHNSW from ..backend.clients.memorydb.cli import MemoryDB from ..backend.clients.milvus.cli import MilvusAutoIndex +from ..backend.clients.oceanbase.cli import OceanBaseHNSW, OceanBaseIVF +from ..backend.clients.oss_opensearch.cli import OSSOpenSearch from ..backend.clients.pgdiskann.cli import PgDiskAnn from ..backend.clients.pgvecto_rs.cli import PgVectoRSHNSW, PgVectoRSIVFFlat from ..backend.clients.pgvector.cli import PgVectorHNSW from ..backend.clients.pgvectorscale.cli import PgVectorScaleDiskAnn +from ..backend.clients.qdrant_cloud.cli import QdrantCloud +from ..backend.clients.qdrant_local.cli import QdrantLocal from ..backend.clients.redis.cli import Redis +from ..backend.clients.s3_vectors.cli import S3Vectors +from ..backend.clients.tencent_elasticsearch.cli import TencentElasticsearch from ..backend.clients.test.cli import Test from ..backend.clients.tidb.cli import TiDB +from ..backend.clients.turbopuffer.cli import TurboPuffer from ..backend.clients.vespa.cli import Vespa from ..backend.clients.weaviate_cloud.cli import Weaviate from ..backend.clients.zilliz_cloud.cli import ZillizAutoIndex +from .batch_cli import BatchCli from .cli import cli cli.add_command(PgVectorHNSW) @@ -26,13 +46,32 @@ cli.add_command(ZillizAutoIndex) cli.add_command(MilvusAutoIndex) cli.add_command(AWSOpenSearch) +cli.add_command(OSSOpenSearch) cli.add_command(PgVectorScaleDiskAnn) cli.add_command(PgDiskAnn) cli.add_command(AlloyDBScaNN) +cli.add_command(OceanBaseHNSW) +cli.add_command(OceanBaseIVF) cli.add_command(MariaDBHNSW) cli.add_command(TiDB) +cli.add_command(CockroachDBCli) cli.add_command(Clickhouse) cli.add_command(Vespa) +cli.add_command(LanceDB) +cli.add_command(HologresHGraph) +cli.add_command(QdrantCloud) +cli.add_command(QdrantLocal) +cli.add_command(ElasticCloudHNSW) +cli.add_command(ElasticCloudHNSWInt8) +cli.add_command(ElasticCloudHNSWInt4) +cli.add_command(ElasticCloudHNSWBBQ) +cli.add_command(BatchCli) +cli.add_command(S3Vectors) +cli.add_command(TencentElasticsearch) +cli.add_command(AliSQLHNSW) +cli.add_command(Doris) +cli.add_command(TurboPuffer) +cli.add_command(Chroma) if __name__ == "__main__": diff --git a/vectordb_bench/config-files/batch_sample_config.yml b/vectordb_bench/config-files/batch_sample_config.yml new file mode 100644 index 000000000..3edaab160 --- /dev/null +++ b/vectordb_bench/config-files/batch_sample_config.yml @@ -0,0 +1,17 @@ +pgvectorhnsw: + - db_label: pgConfigTest + user_name: vectordbbench + db_name: vectordbbench + host: localhost + m: 16 + ef_construction: 128 + ef_search: 128 +milvushnsw: + - skip_search_serial: True + case_type: Performance1536D50K + uri: http://localhost:19530 + m: 16 + ef_construction: 128 + ef_search: 128 + drop_old: False + load: False diff --git a/vectordb_bench/custom/custom_case.json b/vectordb_bench/custom/custom_case.json index 48ca8d8c4..12ca6597b 100644 --- a/vectordb_bench/custom/custom_case.json +++ b/vectordb_bench/custom/custom_case.json @@ -14,5 +14,18 @@ "use_shuffled": false, "with_gt": true } + }, + { + "case_type": "streaming", + "description": "This is a custom streaming dataset.", + "dataset_config": { + "name": "My Streaming Dataset", + "dir": "/my_dataset_path", + "size": 1000000, + "dim": 1024, + "file_count": 1, + "train_name": "shuffle_train", + "with_gt": true + } } ] \ No newline at end of file diff --git a/fig/custom_case_run_test.png b/vectordb_bench/fig/custom_case_run_test.png similarity index 100% rename from fig/custom_case_run_test.png rename to vectordb_bench/fig/custom_case_run_test.png diff --git a/fig/custom_dataset.png b/vectordb_bench/fig/custom_dataset.png similarity index 100% rename from fig/custom_dataset.png rename to vectordb_bench/fig/custom_dataset.png diff --git a/vectordb_bench/fig/homepage/bar-chart.png b/vectordb_bench/fig/homepage/bar-chart.png new file mode 100644 index 000000000..a78c67eca Binary files /dev/null and b/vectordb_bench/fig/homepage/bar-chart.png differ diff --git a/vectordb_bench/fig/homepage/concurrent.png b/vectordb_bench/fig/homepage/concurrent.png new file mode 100644 index 000000000..9192abcef Binary files /dev/null and b/vectordb_bench/fig/homepage/concurrent.png differ diff --git a/vectordb_bench/fig/homepage/custom.png b/vectordb_bench/fig/homepage/custom.png new file mode 100644 index 000000000..30f3f9fb1 Binary files /dev/null and b/vectordb_bench/fig/homepage/custom.png differ diff --git a/vectordb_bench/fig/homepage/label_filter.png b/vectordb_bench/fig/homepage/label_filter.png new file mode 100644 index 000000000..3bb056ede Binary files /dev/null and b/vectordb_bench/fig/homepage/label_filter.png differ diff --git a/vectordb_bench/fig/homepage/qp$.png b/vectordb_bench/fig/homepage/qp$.png new file mode 100644 index 000000000..4a1edd281 Binary files /dev/null and b/vectordb_bench/fig/homepage/qp$.png differ diff --git a/vectordb_bench/fig/homepage/run_test.png b/vectordb_bench/fig/homepage/run_test.png new file mode 100644 index 000000000..950f1f440 Binary files /dev/null and b/vectordb_bench/fig/homepage/run_test.png differ diff --git a/vectordb_bench/fig/homepage/streaming.png b/vectordb_bench/fig/homepage/streaming.png new file mode 100644 index 000000000..582c1206b Binary files /dev/null and b/vectordb_bench/fig/homepage/streaming.png differ diff --git a/vectordb_bench/fig/homepage/table.png b/vectordb_bench/fig/homepage/table.png new file mode 100644 index 000000000..e6d60a9f0 Binary files /dev/null and b/vectordb_bench/fig/homepage/table.png differ diff --git a/vectordb_bench/fig/run_test_select_case.png b/vectordb_bench/fig/run_test_select_case.png new file mode 100644 index 000000000..c186d8413 Binary files /dev/null and b/vectordb_bench/fig/run_test_select_case.png differ diff --git a/vectordb_bench/fig/run_test_select_db.png b/vectordb_bench/fig/run_test_select_db.png new file mode 100644 index 000000000..8b08dee24 Binary files /dev/null and b/vectordb_bench/fig/run_test_select_db.png differ diff --git a/vectordb_bench/fig/run_test_submit.png b/vectordb_bench/fig/run_test_submit.png new file mode 100644 index 000000000..04b105a00 Binary files /dev/null and b/vectordb_bench/fig/run_test_submit.png differ diff --git a/vectordb_bench/frontend/components/check_results/data.py b/vectordb_bench/frontend/components/check_results/data.py index 94d1b4eab..289453f7a 100644 --- a/vectordb_bench/frontend/components/check_results/data.py +++ b/vectordb_bench/frontend/components/check_results/data.py @@ -1,6 +1,6 @@ from collections import defaultdict from dataclasses import asdict -from vectordb_bench.metric import isLowerIsBetterMetric +from vectordb_bench.metric import QPS_METRIC, isLowerIsBetterMetric from vectordb_bench.models import CaseResult, ResultLabel @@ -22,8 +22,7 @@ def getFilterTasks( filterTasks = [ task for task in tasks - if task.task_config.db_name in dbNames - and task.task_config.case_config.case_id.case_cls(task.task_config.case_config.custom_case).name in caseNames + if task.task_config.db_name in dbNames and task.task_config.case_config.case_name in caseNames ] return filterTasks @@ -35,17 +34,22 @@ def mergeTasks(tasks: list[CaseResult]): db = task.task_config.db.value db_label = task.task_config.db_config.db_label or "" version = task.task_config.db_config.version or "" - case = task.task_config.case_config.case_id.case_cls(task.task_config.case_config.custom_case) + case = task.task_config.case_config.case + case_name = case.name + dataset_name = case.dataset.data.full_name + filter_rate = case.filter_rate dbCaseMetricsMap[db_name][case.name] = { "db": db, "db_label": db_label, "version": version, + "dataset_name": dataset_name, + "filter_rate": filter_rate, "metrics": mergeMetrics( - dbCaseMetricsMap[db_name][case.name].get("metrics", {}), + dbCaseMetricsMap[db_name][case_name].get("metrics", {}), asdict(task.metrics), ), "label": getBetterLabel( - dbCaseMetricsMap[db_name][case.name].get("label", ResultLabel.FAILED), + dbCaseMetricsMap[db_name][case_name].get("label", ResultLabel.FAILED), task.label, ), } @@ -59,12 +63,16 @@ def mergeTasks(tasks: list[CaseResult]): db_label = metricInfo["db_label"] version = metricInfo["version"] label = metricInfo["label"] + dataset_name = metricInfo["dataset_name"] + filter_rate = metricInfo["filter_rate"] if label == ResultLabel.NORMAL: mergedTasks.append( { "db_name": db_name, "db": db, "db_label": db_label, + "dataset_name": dataset_name, + "filter_rate": filter_rate, "version": version, "case_name": case_name, "metricsSet": set(metrics.keys()), @@ -77,12 +85,9 @@ def mergeTasks(tasks: list[CaseResult]): return mergedTasks, failedTasks +# for same db-label, we use the results with the highest qps def mergeMetrics(metrics_1: dict, metrics_2: dict) -> dict: - metrics = {**metrics_1} - for key, value in metrics_2.items(): - metrics[key] = getBetterMetric(key, value, metrics[key]) if key in metrics else value - - return metrics + return metrics_1 if metrics_1.get(QPS_METRIC, 0) > metrics_2.get(QPS_METRIC, 0) else metrics_2 def getBetterMetric(metric, value_1, value_2): diff --git a/vectordb_bench/frontend/components/check_results/filters.py b/vectordb_bench/frontend/components/check_results/filters.py index 129c1d5ae..6016c0040 100644 --- a/vectordb_bench/frontend/components/check_results/filters.py +++ b/vectordb_bench/frontend/components/check_results/filters.py @@ -1,14 +1,19 @@ from vectordb_bench.backend.cases import Case +from vectordb_bench.backend.dataset import DatasetWithSizeType +from vectordb_bench.backend.filter import FilterOp from vectordb_bench.frontend.components.check_results.data import getChartData -from vectordb_bench.frontend.components.check_results.expanderStyle import initSidebarExanderStyle +from vectordb_bench.frontend.components.check_results.expanderStyle import ( + initSidebarExanderStyle, +) from vectordb_bench.frontend.config.dbCaseConfigs import CASE_NAME_ORDER -from vectordb_bench.frontend.config.styles import * +from vectordb_bench.frontend.config.styles import SIDEBAR_CONTROL_COLUMNS import streamlit as st +from typing import Callable from vectordb_bench.models import CaseResult, TestResult -def getshownData(results: list[TestResult], st): +def getshownData(st, results: list[TestResult], filter_type: FilterOp = FilterOp.NonFilter, **kwargs): # hide the nav st.markdown( "", @@ -17,15 +22,21 @@ def getshownData(results: list[TestResult], st): st.header("Filters") - shownResults = getshownResults(results, st) - showDBNames, showCaseNames = getShowDbsAndCases(shownResults, st) + shownResults = getshownResults(st, results, **kwargs) + showDBNames, showCaseNames = getShowDbsAndCases(st, shownResults, filter_type) shownData, failedTasks = getChartData(shownResults, showDBNames, showCaseNames) return shownData, failedTasks, showCaseNames -def getshownResults(results: list[TestResult], st) -> list[CaseResult]: +def getshownResults( + st, + results: list[TestResult], + case_results_filter: Callable[[CaseResult], bool] = lambda x: True, + default_selected_task_labels: list[str] = [], + **kwargs, +) -> list[CaseResult]: resultSelectOptions = [ result.task_label if result.task_label != result.run_id else f"res-{result.run_id[:4]}" for result in results ] @@ -37,27 +48,22 @@ def getshownResults(results: list[TestResult], st) -> list[CaseResult]: "Select the task results you need to analyze.", resultSelectOptions, # label_visibility="hidden", - default=resultSelectOptions, + default=default_selected_task_labels or resultSelectOptions, ) selectedResult: list[CaseResult] = [] for option in selectedResultSelectedOptions: - result = results[resultSelectOptions.index(option)].results - selectedResult += result + case_results = results[resultSelectOptions.index(option)].results + selectedResult += [r for r in case_results if case_results_filter(r)] return selectedResult -def getShowDbsAndCases(result: list[CaseResult], st) -> tuple[list[str], list[str]]: +def getShowDbsAndCases(st, result: list[CaseResult], filter_type: FilterOp) -> tuple[list[str], list[str]]: initSidebarExanderStyle(st) - allDbNames = list(set({res.task_config.db_name for res in result})) + case_results = [res for res in result if res.task_config.case_config.case.filters.type == filter_type] + allDbNames = list(set({res.task_config.db_name for res in case_results})) allDbNames.sort() - allCases: list[Case] = [ - res.task_config.case_config.case_id.case_cls(res.task_config.case_config.custom_case) for res in result - ] - allCaseNameSet = set({case.name for case in allCases}) - allCaseNames = [case_name for case_name in CASE_NAME_ORDER if case_name in allCaseNameSet] + [ - case_name for case_name in allCaseNameSet if case_name not in CASE_NAME_ORDER - ] + allCases: list[Case] = [res.task_config.case_config.case for res in case_results] # DB Filter dbFilterContainer = st.container() @@ -67,15 +73,35 @@ def getShowDbsAndCases(result: list[CaseResult], st) -> tuple[list[str], list[st allDbNames, col=1, ) + showCaseNames = [] + + if filter_type == FilterOp.NonFilter: + allCaseNameSet = set({case.name for case in allCases}) + allCaseNames = [case_name for case_name in CASE_NAME_ORDER if case_name in allCaseNameSet] + [ + case_name for case_name in allCaseNameSet if case_name not in CASE_NAME_ORDER + ] + + # Case Filter + caseFilterContainer = st.container() + showCaseNames = filterView( + caseFilterContainer, + "Case Filter", + [caseName for caseName in allCaseNames], + col=1, + ) - # Case Filter - caseFilterContainer = st.container() - showCaseNames = filterView( - caseFilterContainer, - "Case Filter", - [caseName for caseName in allCaseNames], - col=1, - ) + if filter_type == FilterOp.StrEqual or filter_type == FilterOp.NumGE: + container = st.container() + datasetWithSizeTypes = [dataset_with_size_type for dataset_with_size_type in DatasetWithSizeType] + showDatasetWithSizeTypes = filterView( + container, + "Case Filter", + datasetWithSizeTypes, + col=1, + optionLables=[v.value for v in datasetWithSizeTypes], + ) + datasets = [dataset_with_size_type.get_manager() for dataset_with_size_type in showDatasetWithSizeTypes] + showCaseNames = list(set([case.name for case in allCases if case.dataset in datasets])) return showDBNames, showCaseNames diff --git a/vectordb_bench/frontend/components/check_results/headerIcon.py b/vectordb_bench/frontend/components/check_results/headerIcon.py index 4715701f6..048228280 100644 --- a/vectordb_bench/frontend/components/check_results/headerIcon.py +++ b/vectordb_bench/frontend/components/check_results/headerIcon.py @@ -4,19 +4,24 @@ def drawHeaderIcon(st): st.markdown( f""" -
+ +
+ - + .headerIconContainer {{ + position: relative; + top: 0px; + height: 50px; + width: 100%; + border-bottom: 2px solid #E8EAEE; + background-image: url({HEADER_ICON}); + background-size: contain; + background-position: left top; + background-repeat: no-repeat; + cursor: pointer; + }} + + """, unsafe_allow_html=True, ) diff --git a/vectordb_bench/frontend/components/check_results/nav.py b/vectordb_bench/frontend/components/check_results/nav.py index f95e43d7a..ba4fa99c7 100644 --- a/vectordb_bench/frontend/components/check_results/nav.py +++ b/vectordb_bench/frontend/components/check_results/nav.py @@ -19,4 +19,26 @@ def NavToQuriesPerDollar(st): def NavToResults(st, key="nav-to-results"): navClick = st.button("<   Back to Results", key=key) if navClick: - switch_page("vdb benchmark") + switch_page("results") + + +def NavToPages(st): + options = [ + {"name": "Run Test", "link": "run_test"}, + {"name": "Results", "link": "results"}, + {"name": "Qps & Recall", "link": "qps_recall"}, + {"name": "Quries Per Dollar", "link": "quries_per_dollar"}, + {"name": "Concurrent", "link": "concurrent"}, + {"name": "Label Filter", "link": "label_filter"}, + {"name": "Int Filter", "link": "int_filter"}, + {"name": "Streaming", "link": "streaming"}, + {"name": "Tables", "link": "tables"}, + {"name": "Custom Dataset", "link": "custom"}, + ] + + html = "" + for i, option in enumerate(options): + html += f'{option["name"]}' + if i < len(options) - 1: + html += '|' + st.markdown(html, unsafe_allow_html=True) diff --git a/vectordb_bench/frontend/components/concurrent/charts.py b/vectordb_bench/frontend/components/concurrent/charts.py index 83e2961d6..004fcb261 100644 --- a/vectordb_bench/frontend/components/concurrent/charts.py +++ b/vectordb_bench/frontend/components/concurrent/charts.py @@ -20,6 +20,11 @@ def drawChartsByCase(allData, showCaseNames: list[str], st, latency_type: str): if 0 <= i < len(caseData["conc_latency_p99_list"]) else 0 ), + "latency_p95": ( + caseData["conc_latency_p95_list"][i] * 1000 + if "conc_latency_p95_list" in caseData and 0 <= i < len(caseData["conc_latency_p95_list"]) + else 0 + ), "latency_avg": ( caseData["conc_latency_avg_list"][i] * 1000 if 0 <= i < len(caseData["conc_latency_avg_list"]) diff --git a/vectordb_bench/frontend/components/custom/displayCustomCase.py b/vectordb_bench/frontend/components/custom/displayCustomCase.py index ac111c883..1aa03f96b 100644 --- a/vectordb_bench/frontend/components/custom/displayCustomCase.py +++ b/vectordb_bench/frontend/components/custom/displayCustomCase.py @@ -12,7 +12,7 @@ def displayCustomCase(customCase: CustomCaseConfig, st, key): "Folder Path", key=f"{key}_dir", value=customCase.dataset_config.dir ) - columns = st.columns(4) + columns = st.columns(3) customCase.dataset_config.dim = columns[0].number_input( "dim", key=f"{key}_dim", value=customCase.dataset_config.dim ) @@ -22,16 +22,51 @@ def displayCustomCase(customCase: CustomCaseConfig, st, key): customCase.dataset_config.metric_type = columns[2].selectbox( "metric type", key=f"{key}_metric_type", options=["L2", "Cosine", "IP"] ) - customCase.dataset_config.file_count = columns[3].number_input( - "train file count", key=f"{key}_file_count", value=customCase.dataset_config.file_count + + columns = st.columns(3) + customCase.dataset_config.train_name = columns[0].text_input( + "train file name", + key=f"{key}_train_name", + value=customCase.dataset_config.train_name, + ) + customCase.dataset_config.test_name = columns[1].text_input( + "test file name", key=f"{key}_test_name", value=customCase.dataset_config.test_name + ) + customCase.dataset_config.gt_name = columns[2].text_input( + "ground truth file name", key=f"{key}_gt_name", value=customCase.dataset_config.gt_name + ) + + columns = st.columns([1, 1, 2, 2]) + customCase.dataset_config.train_id_name = columns[0].text_input( + "train id name", key=f"{key}_train_id_name", value=customCase.dataset_config.train_id_name + ) + customCase.dataset_config.train_col_name = columns[1].text_input( + "train emb name", key=f"{key}_train_col_name", value=customCase.dataset_config.train_col_name + ) + customCase.dataset_config.test_col_name = columns[2].text_input( + "test emb name", key=f"{key}_test_col_name", value=customCase.dataset_config.test_col_name + ) + customCase.dataset_config.gt_col_name = columns[3].text_input( + "ground truth emb name", key=f"{key}_gt_col_name", value=customCase.dataset_config.gt_col_name ) - columns = st.columns(4) - customCase.dataset_config.use_shuffled = columns[0].checkbox( - "use shuffled data", key=f"{key}_use_shuffled", value=customCase.dataset_config.use_shuffled + columns = st.columns(2) + customCase.dataset_config.scalar_labels_name = columns[0].text_input( + "scalar labels file name", + key=f"{key}_scalar_labels_file_name", + value=customCase.dataset_config.scalar_labels_name, ) - customCase.dataset_config.with_gt = columns[1].checkbox( - "with groundtruth", key=f"{key}_with_gt", value=customCase.dataset_config.with_gt + default_label_percentages = ",".join(map(str, customCase.dataset_config.with_label_percentages)) + label_percentage_input = columns[1].text_input( + "label percentages", + key=f"{key}_label_percantages", + value=default_label_percentages, ) + try: + customCase.dataset_config.label_percentages = [ + float(item.strip()) for item in label_percentage_input.split(",") if item.strip() + ] + except ValueError as e: + st.write(f"{e},please input correct number", unsafe_allow_html=True) customCase.description = st.text_area("description", key=f"{key}_description", value=customCase.description) diff --git a/vectordb_bench/frontend/components/custom/displayCustomStreamingCase.py b/vectordb_bench/frontend/components/custom/displayCustomStreamingCase.py new file mode 100644 index 000000000..e861ca1a6 --- /dev/null +++ b/vectordb_bench/frontend/components/custom/displayCustomStreamingCase.py @@ -0,0 +1,49 @@ +from vectordb_bench.frontend.components.custom.getCustomConfig import CustomStreamingCaseConfig + + +def displayCustomStreamingCase(streamingCase: CustomStreamingCaseConfig, st, key): + + columns = st.columns([1, 2]) + streamingCase.dataset_config.name = columns[0].text_input( + "Name", key=f"{key}_name", value=streamingCase.dataset_config.name + ) + streamingCase.dataset_config.dir = columns[1].text_input( + "Folder Path", key=f"{key}_dir", value=streamingCase.dataset_config.dir + ) + + columns = st.columns(2) + streamingCase.dataset_config.dim = columns[0].number_input( + "dim", key=f"{key}_dim", value=streamingCase.dataset_config.dim + ) + streamingCase.dataset_config.size = columns[1].number_input( + "size", key=f"{key}_size", value=streamingCase.dataset_config.size + ) + + columns = st.columns(3) + streamingCase.dataset_config.train_name = columns[0].text_input( + "train file name", + key=f"{key}_train_name", + value=streamingCase.dataset_config.train_name, + ) + streamingCase.dataset_config.test_name = columns[1].text_input( + "test file name", key=f"{key}_test_name", value=streamingCase.dataset_config.test_name + ) + streamingCase.dataset_config.gt_name = columns[2].text_input( + "ground truth file name", key=f"{key}_gt_name", value=streamingCase.dataset_config.gt_name + ) + + columns = st.columns([1, 1, 2, 2]) + streamingCase.dataset_config.train_id_name = columns[0].text_input( + "train id name", key=f"{key}_train_id_name", value=streamingCase.dataset_config.train_id_name + ) + streamingCase.dataset_config.train_col_name = columns[1].text_input( + "train emb name", key=f"{key}_train_col_name", value=streamingCase.dataset_config.train_col_name + ) + streamingCase.dataset_config.test_col_name = columns[2].text_input( + "test emb name", key=f"{key}_test_col_name", value=streamingCase.dataset_config.test_col_name + ) + streamingCase.dataset_config.gt_col_name = columns[3].text_input( + "ground truth emb name", key=f"{key}_gt_col_name", value=streamingCase.dataset_config.gt_col_name + ) + + streamingCase.description = st.text_area("description", key=f"{key}_description", value=streamingCase.description) diff --git a/vectordb_bench/frontend/components/custom/displaypPrams.py b/vectordb_bench/frontend/components/custom/displaypPrams.py index 712b57b00..80a694308 100644 --- a/vectordb_bench/frontend/components/custom/displaypPrams.py +++ b/vectordb_bench/frontend/components/custom/displaypPrams.py @@ -1,16 +1,19 @@ def displayParams(st): - st.markdown( - """ + st.markdown(""" - `Folder Path` - The path to the folder containing all the files. Please ensure that all files in the folder are in the `Parquet` format. - - Vectors data files: The file must be named `train.parquet` and should have two columns: `id` as an incrementing `int` and `emb` as an array of `float32`. - - Query test vectors: The file must be named `test.parquet` and should have two columns: `id` as an incrementing `int` and `emb` as an array of `float32`. - - Ground truth file: The file must be named `neighbors.parquet` and should have two columns: `id` corresponding to query vectors and `neighbors_id` as an array of `int`. + - Vectors data files: The file should have two kinds of columns: `id` as an incrementing `int` and `emb` as an array of `float32`. The name of two columns could be defined on your own. + - Query test vectors: The file could be named on your own and should have two kinds of columns: `id` as an incrementing `int` and `emb` as an array of `float32`. The `id` column must be named as `id`, and `emb` column could be defined on your own. + - Ground truth file: The file could be named on your own and should have two kinds of columns: `id` corresponding to query vectors and `neighbors_id` as an array of `int`. The `id` column must be named as `id`, and `neighbors_id` column could be defined on your own. -- `Train File Count` - If the vector file is too large, you can consider splitting it into multiple files. The naming format for the split files should be `train-[index]-of-[file_count].parquet`. For example, `train-01-of-10.parquet` represents the second file (0-indexed) among 10 split files. +- `Train File Name` - If the number of train file is `more than one`, please input all your train file name and `split with ','` without the `.parquet` file extensionthe. For example, if there are two train file and the name of them are `train1.parquet` and `train2.parquet`, then input `train1,train2`. -- `Use Shuffled Data` - If you check this option, the vector data files need to be modified. VectorDBBench will load the data labeled with `shuffle`. For example, use `shuffle_train.parquet` instead of `train.parquet` and `shuffle_train-04-of-10.parquet` instead of `train-04-of-10.parquet`. The `id` column in the shuffled data can be in any order. -""" - ) +- `Ground Truth Emb Name` - No matter whether filter file is applied or not, the `neighbors_id` column in ground truth file must have the same name. + +- `Scalar Labels File Name ` - If there is a scalar labels file, please input the filename without the .parquet extension. The file should have two columns: `id` as an incrementing `int` and `labels` as an array of `string`. The `id` column must correspond one-to-one with the `id` column in train file.. + +- `Label percentages` - If you have filter file, please input label percentage you want to real run and `split with ','` when it's `more than one`. If you `don't have` filter file, than `keep the text vacant.` + +""") st.caption( """We recommend limiting the number of test query vectors, like 1,000.""", help=""" diff --git a/vectordb_bench/frontend/components/custom/getCustomConfig.py b/vectordb_bench/frontend/components/custom/getCustomConfig.py index 668bfb6d5..a1ddfb737 100644 --- a/vectordb_bench/frontend/components/custom/getCustomConfig.py +++ b/vectordb_bench/frontend/components/custom/getCustomConfig.py @@ -14,6 +14,16 @@ class CustomDatasetConfig(BaseModel): file_count: int = 1 use_shuffled: bool = False with_gt: bool = True + train_name: str = "train" + test_name: str = "test" + gt_name: str = "neighbors" + train_id_name: str = "id" + train_col_name: str = "emb" + test_col_name: str = "emb" + gt_col_name: str = "neighbors_id" + scalar_labels_name: str = "scalar_labels" + label_percentages: list[str] = [] + with_label_percentages: list[float] = [0.001, 0.02, 0.5] class CustomCaseConfig(BaseModel): @@ -24,10 +34,30 @@ class CustomCaseConfig(BaseModel): dataset_config: CustomDatasetConfig = CustomDatasetConfig() +class CustomStreamingCaseConfig(BaseModel): + case_type: str = "streaming" + description: str = "" + dataset_config: CustomDatasetConfig = CustomDatasetConfig() + + def get_custom_configs(): with open(config.CUSTOM_CONFIG_DIR, "r") as f: custom_configs = json.load(f) - return [CustomCaseConfig(**custom_config) for custom_config in custom_configs] + return [ + CustomCaseConfig(**custom_config) + for custom_config in custom_configs + if custom_config.get("case_type") != "streaming" + ] + + +def get_custom_streaming_configs(): + with open(config.CUSTOM_CONFIG_DIR, "r") as f: + custom_configs = json.load(f) + return [ + CustomStreamingCaseConfig(**custom_config) + for custom_config in custom_configs + if custom_config.get("case_type") == "streaming" + ] def save_custom_configs(custom_configs: list[CustomDatasetConfig]): @@ -35,5 +65,18 @@ def save_custom_configs(custom_configs: list[CustomDatasetConfig]): json.dump([custom_config.dict() for custom_config in custom_configs], f, indent=4) +def save_all_custom_configs( + performance_configs: list[CustomCaseConfig], streaming_configs: list[CustomStreamingCaseConfig] +): + """Save both performance and streaming configs to the same JSON file""" + all_configs = [config.dict() for config in performance_configs] + [config.dict() for config in streaming_configs] + with open(config.CUSTOM_CONFIG_DIR, "w") as f: + json.dump(all_configs, f, indent=4) + + def generate_custom_case(): return CustomCaseConfig() + + +def generate_custom_streaming_case(): + return CustomStreamingCaseConfig() diff --git a/vectordb_bench/frontend/components/int_filter/charts.py b/vectordb_bench/frontend/components/int_filter/charts.py new file mode 100644 index 000000000..881681031 --- /dev/null +++ b/vectordb_bench/frontend/components/int_filter/charts.py @@ -0,0 +1,60 @@ +import plotly.express as px +from vectordb_bench.metric import metric_unit_map + + +def drawCharts(st, allData, **kwargs): + dataset_names = list(set([data["dataset_name"] for data in allData])) + dataset_names.sort() + for dataset_name in dataset_names: + container = st.container() + container.subheader(dataset_name) + data = [d for d in allData if d["dataset_name"] == dataset_name] + drawChartByMetric(container, data, **kwargs) + + +def drawChartByMetric(st, data, metrics=("qps", "recall"), **kwargs): + columns = st.columns(len(metrics)) + for i, metric in enumerate(metrics): + container = columns[i] + container.markdown(f"#### {metric}") + drawChart(container, data, metric) + + +def getRange(metric, data, padding_multipliers): + minV = min([d.get(metric, 0) for d in data]) + maxV = max([d.get(metric, 0) for d in data]) + padding = maxV - minV + rangeV = [ + minV - padding * padding_multipliers[0], + maxV + padding * padding_multipliers[1], + ] + return rangeV + + +def drawChart(st, data: list[object], metric): + unit = metric_unit_map.get(metric, "") + x = "filter_rate" + xrange = getRange(x, data, [0.05, 0.1]) + + y = metric + yrange = getRange(y, data, [0.2, 0.1]) + + data.sort(key=lambda a: a[x]) + + fig = px.line( + data, + x=x, + y=y, + color="db_name", + line_group="db_name", + text=metric, + markers=True, + ) + fig.update_xaxes(range=xrange) + fig.update_yaxes(range=yrange) + fig.update_traces(textposition="bottom right", texttemplate="%{y:,.4~r}" + unit) + fig.update_layout( + margin=dict(l=0, r=0, t=40, b=0, pad=8), + legend=dict(orientation="h", yanchor="bottom", y=1, xanchor="right", x=1, title=""), + ) + st.plotly_chart(fig, use_container_width=True) diff --git a/vectordb_bench/frontend/components/label_filter/charts.py b/vectordb_bench/frontend/components/label_filter/charts.py new file mode 100644 index 000000000..881681031 --- /dev/null +++ b/vectordb_bench/frontend/components/label_filter/charts.py @@ -0,0 +1,60 @@ +import plotly.express as px +from vectordb_bench.metric import metric_unit_map + + +def drawCharts(st, allData, **kwargs): + dataset_names = list(set([data["dataset_name"] for data in allData])) + dataset_names.sort() + for dataset_name in dataset_names: + container = st.container() + container.subheader(dataset_name) + data = [d for d in allData if d["dataset_name"] == dataset_name] + drawChartByMetric(container, data, **kwargs) + + +def drawChartByMetric(st, data, metrics=("qps", "recall"), **kwargs): + columns = st.columns(len(metrics)) + for i, metric in enumerate(metrics): + container = columns[i] + container.markdown(f"#### {metric}") + drawChart(container, data, metric) + + +def getRange(metric, data, padding_multipliers): + minV = min([d.get(metric, 0) for d in data]) + maxV = max([d.get(metric, 0) for d in data]) + padding = maxV - minV + rangeV = [ + minV - padding * padding_multipliers[0], + maxV + padding * padding_multipliers[1], + ] + return rangeV + + +def drawChart(st, data: list[object], metric): + unit = metric_unit_map.get(metric, "") + x = "filter_rate" + xrange = getRange(x, data, [0.05, 0.1]) + + y = metric + yrange = getRange(y, data, [0.2, 0.1]) + + data.sort(key=lambda a: a[x]) + + fig = px.line( + data, + x=x, + y=y, + color="db_name", + line_group="db_name", + text=metric, + markers=True, + ) + fig.update_xaxes(range=xrange) + fig.update_yaxes(range=yrange) + fig.update_traces(textposition="bottom right", texttemplate="%{y:,.4~r}" + unit) + fig.update_layout( + margin=dict(l=0, r=0, t=40, b=0, pad=8), + legend=dict(orientation="h", yanchor="bottom", y=1, xanchor="right", x=1, title=""), + ) + st.plotly_chart(fig, use_container_width=True) diff --git a/vectordb_bench/frontend/components/qps_recall/charts.py b/vectordb_bench/frontend/components/qps_recall/charts.py new file mode 100644 index 000000000..ab57dd0ce --- /dev/null +++ b/vectordb_bench/frontend/components/qps_recall/charts.py @@ -0,0 +1,118 @@ +from vectordb_bench.frontend.components.check_results.expanderStyle import ( + initMainExpanderStyle, +) +from vectordb_bench.metric import metric_order, isLowerIsBetterMetric, metric_unit_map +from vectordb_bench.frontend.config.styles import * +import plotly.express as px +import pandas as pd +import plotly.graph_objects as go +import matplotlib.pyplot as plt + + +def drawCharts(st, allData, caseNames: list[str]): + initMainExpanderStyle(st) + for caseName in caseNames: + chartContainer = st.expander(caseName, True) + data = [data for data in allData if data["case_name"] == caseName] + drawChart(data, chartContainer, key_prefix=caseName) + + +def drawChart(data, st, key_prefix: str): + metricsSet = set() + for d in data: + metricsSet = metricsSet.union(d["metricsSet"]) + showlineMetrics = [metric for metric in metric_order[:2] if metric in metricsSet] + + if showlineMetrics: + metric = showlineMetrics[0] + key = f"{key_prefix}-{metric}" + drawlinechart(st, data, metric, key=key) + + +def drawBestperformance(data, y, group): + all_filter_points = [] + data = pd.DataFrame(data) + grouped = data.groupby(group) + for name, group_df in grouped: + filter_points = [] + current_start = 0 + for _ in range(len(group_df)): + if current_start >= len(group_df): + break + max_index = group_df[y].iloc[current_start:].idxmax() + filter_points.append(group_df.loc[max_index]) + + current_start = group_df.index.get_loc(max_index) + 1 + all_filter_points.extend(filter_points) + + all_filter_df = pd.DataFrame(all_filter_points) + remaining_df = data[~data.isin(all_filter_df).any(axis=1)] + new_data = all_filter_df.to_dict(orient="records") + remain_data = remaining_df.to_dict(orient="records") + return new_data, remain_data + + +def drawlinechart(st, data: list[object], metric, key: str): + minV = min([d.get(metric, 0) for d in data]) + maxV = max([d.get(metric, 0) for d in data]) + padding = maxV - minV + rangeV = [ + minV - padding * 0.1, + maxV + padding * 0.1, + ] + x = "recall" + xrange = [0.8, 1.01] + y = "qps" + yrange = rangeV + data.sort(key=lambda a: a[x]) + group = "db_name" + new_data, new_remain_data = drawBestperformance(data, y, group) + unique_db_names = list(set(item["db_name"] for item in new_data + new_remain_data)) + + colors = plt.cm.get_cmap("tab10", len(unique_db_names)) + + color_map = { + db: f"rgb({int(colors(i)[0] * 255)}, {int(colors(i)[1] * 255)}, {int(colors(i)[2] * 255)})" + for i, db in enumerate(unique_db_names) + } + + fig = go.Figure() + + new_data_df = pd.DataFrame(new_data) + + for db in unique_db_names: + db_data = new_data_df[new_data_df["db_name"] == db] + fig.add_trace( + go.Scatter( + x=db_data["recall"], + y=db_data["qps"], + mode="lines+markers+text", + name=db, + line=dict(color=color_map[db]), + marker=dict(color=color_map[db]), + showlegend=True, + hovertemplate="QPS=%{y:.4g}, Recall=%{x:.2f}", + text=[f"{qps:.4g}@{recall:.2f}" for recall, qps in zip(db_data["recall"], db_data["qps"])], + textposition="top right", + ) + ) + + for item in new_remain_data: + fig.add_trace( + go.Scatter( + x=[item["recall"]], + y=[item["qps"]], + mode="markers", + name=item["db_name"], + marker=dict(color=color_map[item["db_name"]]), + showlegend=False, + ) + ) + + fig.update_xaxes(range=xrange, title_text="Recall") + fig.update_yaxes(range=yrange, title_text="QPS") + fig.update_layout( + margin=dict(l=0, r=0, t=40, b=0, pad=8), + legend=dict(orientation="h", yanchor="bottom", y=1, xanchor="right", x=1, title=""), + ) + st.plotly_chart(fig, use_container_width=True, key=key) diff --git a/vectordb_bench/frontend/components/qps_recall/data.py b/vectordb_bench/frontend/components/qps_recall/data.py new file mode 100644 index 000000000..b4cbcb1b5 --- /dev/null +++ b/vectordb_bench/frontend/components/qps_recall/data.py @@ -0,0 +1,58 @@ +from collections import defaultdict +from dataclasses import asdict +from vectordb_bench.backend.filter import FilterOp +from vectordb_bench.frontend.components.check_results.data import getFilterTasks +from vectordb_bench.frontend.components.check_results.filters import getShowDbsAndCases, getshownResults +from vectordb_bench.models import CaseResult, ResultLabel, TestResult + + +def getshownData(st, results: list[TestResult], filter_type: FilterOp = FilterOp.NonFilter, **kwargs): + # hide the nav + st.markdown( + "", + unsafe_allow_html=True, + ) + st.header("Filters") + shownResults = getshownResults(st, results, **kwargs) + showDBNames, showCaseNames = getShowDbsAndCases(st, shownResults, filter_type) + shownData, failedTasks = getChartData(shownResults, showDBNames, showCaseNames) + return shownData, failedTasks, showCaseNames + + +def getChartData( + tasks: list[CaseResult], + dbNames: list[str], + caseNames: list[str], +): + filterTasks = getFilterTasks(tasks, dbNames, caseNames) + failedTasks = defaultdict(lambda: defaultdict(str)) + nonemergedTasks = [] + for task in filterTasks: + db_name = task.task_config.db_name + db = task.task_config.db.value + db_label = task.task_config.db_config.db_label or "" + version = task.task_config.db_config.version or "" + case = task.task_config.case_config.case + case_name = case.name + dataset_name = case.dataset.data.full_name + filter_rate = case.filter_rate + metrics = asdict(task.metrics) + label = task.label + if label == ResultLabel.NORMAL: + nonemergedTasks.append( + { + "db_name": db_name, + "db": db, + "db_label": db_label, + "dataset_name": dataset_name, + "filter_rate": filter_rate, + "version": version, + "case_name": case_name, + "metricsSet": set(metrics.keys()), + **metrics, + } + ) + else: + failedTasks[case_name][db_name] = label + + return nonemergedTasks, failedTasks diff --git a/vectordb_bench/frontend/components/run_test/caseSelector.py b/vectordb_bench/frontend/components/run_test/caseSelector.py index e3d28238d..2e104ce54 100644 --- a/vectordb_bench/frontend/components/run_test/caseSelector.py +++ b/vectordb_bench/frontend/components/run_test/caseSelector.py @@ -1,8 +1,22 @@ -from vectordb_bench.frontend.config.styles import * -from vectordb_bench.frontend.config.dbCaseConfigs import * +from vectordb_bench.backend.clients import DB +from vectordb_bench.frontend.components.run_test.inputWidget import inputWidget from collections import defaultdict +from vectordb_bench.frontend.config.dbCaseConfigs import ( + UI_CASE_CLUSTERS, + UICaseItem, + UICaseItemCluster, + get_case_config_inputs, + get_custom_case_cluter, + get_custom_streaming_case_cluster, +) +from vectordb_bench.frontend.config.styles import ( + CASE_CONFIG_SETTING_COLUMNS, + CHECKBOX_INDENT, + DB_CASE_CONFIG_SETTING_COLUMNS, +) from vectordb_bench.frontend.utils import addHorizontalLine +from vectordb_bench.models import CaseConfig def caseSelector(st, activedDbList: list[DB]): @@ -19,12 +33,12 @@ def caseSelector(st, activedDbList: list[DB]): activedCaseList: list[CaseConfig] = [] dbToCaseClusterConfigs = defaultdict(lambda: defaultdict(dict)) dbToCaseConfigs = defaultdict(lambda: defaultdict(dict)) - caseClusters = UI_CASE_CLUSTERS + [get_custom_case_cluter()] + caseClusters = UI_CASE_CLUSTERS + [get_custom_case_cluter(), get_custom_streaming_case_cluster()] for caseCluster in caseClusters: activedCaseList += caseClusterExpander(st, caseCluster, dbToCaseClusterConfigs, activedDbList) for db in dbToCaseClusterConfigs: for uiCaseItem in dbToCaseClusterConfigs[db]: - for case in uiCaseItem.cases: + for case in uiCaseItem.get_cases(): dbToCaseConfigs[db][case] = dbToCaseClusterConfigs[db][uiCaseItem] return activedCaseList, dbToCaseConfigs @@ -48,15 +62,38 @@ def caseItemCheckbox(st, dbToCaseClusterConfigs, uiCaseItem: UICaseItem, actived unsafe_allow_html=True, ) + caseConfigSetting(st.container(), uiCaseItem) + if selected: - caseConfigSetting(st.container(), dbToCaseClusterConfigs, uiCaseItem, activedDbList) + dbCaseConfigSetting(st.container(), dbToCaseClusterConfigs, uiCaseItem, activedDbList) + + return uiCaseItem.get_cases() if selected else [] + + +def caseConfigSetting(st, uiCaseItem: UICaseItem): + config_inputs = uiCaseItem.extra_custom_case_config_inputs + if len(config_inputs) == 0: + return - return uiCaseItem.cases if selected else [] + columns = st.columns( + [ + 1, + *[DB_CASE_CONFIG_SETTING_COLUMNS / CASE_CONFIG_SETTING_COLUMNS] * CASE_CONFIG_SETTING_COLUMNS, + ] + ) + columns[0].markdown( + f"
Custom Config
", + unsafe_allow_html=True, + ) + for i, config_input in enumerate(config_inputs): + column = columns[1 + i % CASE_CONFIG_SETTING_COLUMNS] + key = f"custom-config-{uiCaseItem.label}-{config_input.label.value}" + uiCaseItem.tmp_custom_config[config_input.label.value] = inputWidget(column, config=config_input, key=key) -def caseConfigSetting(st, dbToCaseClusterConfigs, uiCaseItem: UICaseItem, activedDbList: list[DB]): +def dbCaseConfigSetting(st, dbToCaseClusterConfigs, uiCaseItem: UICaseItem, activedDbList: list[DB]): for db in activedDbList: - columns = st.columns(1 + CASE_CONFIG_SETTING_COLUMNS) + columns = st.columns(1 + DB_CASE_CONFIG_SETTING_COLUMNS) # column 0 - title dbColumn = columns[0] dbColumn.markdown( @@ -64,52 +101,12 @@ def caseConfigSetting(st, dbToCaseClusterConfigs, uiCaseItem: UICaseItem, active unsafe_allow_html=True, ) k = 0 - caseConfig = dbToCaseClusterConfigs[db][uiCaseItem] - for config in CASE_CONFIG_MAP.get(db, {}).get(uiCaseItem.caseLabel, []): - if config.isDisplayed(caseConfig): - column = columns[1 + k % CASE_CONFIG_SETTING_COLUMNS] + dbCaseConfig = dbToCaseClusterConfigs[db][uiCaseItem] + for config in get_case_config_inputs(db, uiCaseItem.caseLabel): + if config.isDisplayed(dbCaseConfig): + column = columns[1 + k % DB_CASE_CONFIG_SETTING_COLUMNS] key = "%s-%s-%s" % (db, uiCaseItem.label, config.label.value) - if config.inputType == InputType.Text: - caseConfig[config.label] = column.text_input( - config.displayLabel if config.displayLabel else config.label.value, - key=key, - help=config.inputHelp, - value=config.inputConfig["value"], - ) - elif config.inputType == InputType.Option: - caseConfig[config.label] = column.selectbox( - config.displayLabel if config.displayLabel else config.label.value, - config.inputConfig["options"], - key=key, - help=config.inputHelp, - ) - elif config.inputType == InputType.Number: - caseConfig[config.label] = column.number_input( - config.displayLabel if config.displayLabel else config.label.value, - # format="%d", - step=config.inputConfig.get("step", 1), - min_value=config.inputConfig["min"], - max_value=config.inputConfig["max"], - key=key, - value=config.inputConfig["value"], - help=config.inputHelp, - ) - elif config.inputType == InputType.Float: - caseConfig[config.label] = column.number_input( - config.displayLabel if config.displayLabel else config.label.value, - step=config.inputConfig.get("step", 0.1), - min_value=config.inputConfig["min"], - max_value=config.inputConfig["max"], - key=key, - value=config.inputConfig["value"], - help=config.inputHelp, - ) - elif config.inputType == InputType.Bool: - caseConfig[config.label] = column.checkbox( - config.displayLabel if config.displayLabel else config.label.value, - value=config.inputConfig["value"], - help=config.inputHelp, - ) + dbCaseConfig[config.label] = inputWidget(column, config, key) k += 1 if k == 0: columns[1].write("Auto") diff --git a/vectordb_bench/frontend/components/run_test/dbConfigSetting.py b/vectordb_bench/frontend/components/run_test/dbConfigSetting.py index 800e6dede..a2d2de77f 100644 --- a/vectordb_bench/frontend/components/run_test/dbConfigSetting.py +++ b/vectordb_bench/frontend/components/run_test/dbConfigSetting.py @@ -36,21 +36,27 @@ def dbConfigSettingItem(st, activeDb: DB): columns = st.columns(DB_CONFIG_SETTING_COLUMNS) dbConfigClass = activeDb.config_cls - properties = dbConfigClass.schema().get("properties") + schema = dbConfigClass.schema() + property_items = schema.get("properties").items() + required_fields = set(schema.get("required", [])) dbConfig = {} idx = 0 # db config (unique) - for key, property in properties.items(): + for key, property in property_items: if key not in dbConfigClass.common_short_configs() and key not in dbConfigClass.common_long_configs(): column = columns[idx % DB_CONFIG_SETTING_COLUMNS] idx += 1 - dbConfig[key] = column.text_input( + input_value = column.text_input( key, - key="%s-%s" % (activeDb.name, key), + key=f"{activeDb.name}-{key}", value=property.get("default", ""), type="password" if inputIsPassword(key) else "default", + placeholder="optional" if key not in required_fields else None, ) + if key in required_fields or input_value: + dbConfig[key] = input_value + # db config (common short labels) for key in dbConfigClass.common_short_configs(): column = columns[idx % DB_CONFIG_SETTING_COLUMNS] diff --git a/vectordb_bench/frontend/components/run_test/dbSelector.py b/vectordb_bench/frontend/components/run_test/dbSelector.py index e20ee5059..0864a4986 100644 --- a/vectordb_bench/frontend/components/run_test/dbSelector.py +++ b/vectordb_bench/frontend/components/run_test/dbSelector.py @@ -1,9 +1,10 @@ from streamlit.runtime.media_file_storage import MediaFileStorageError from vectordb_bench.frontend.config.styles import DB_SELECTOR_COLUMNS, DB_TO_ICON from vectordb_bench.frontend.config.dbCaseConfigs import DB_LIST +import streamlit as st -def dbSelector(st): +def dbSelector(st: st): st.markdown( "
", unsafe_allow_html=True, @@ -20,11 +21,14 @@ def dbSelector(st): for i, db in enumerate(DB_LIST): column = dbContainerColumns[i % DB_SELECTOR_COLUMNS] dbIsActived[db] = column.checkbox(db.name) - try: - column.image(DB_TO_ICON.get(db, "")) - except MediaFileStorageError: + image_src = DB_TO_ICON.get(db, None) + if image_src: + column.markdown( + f'', + unsafe_allow_html=True, + ) + else: column.warning(f"{db.name} image not available") - pass activedDbList = [db for db in DB_LIST if dbIsActived[db]] return activedDbList diff --git a/vectordb_bench/frontend/components/run_test/generateTasks.py b/vectordb_bench/frontend/components/run_test/generateTasks.py index d8a678ffc..725dea769 100644 --- a/vectordb_bench/frontend/components/run_test/generateTasks.py +++ b/vectordb_bench/frontend/components/run_test/generateTasks.py @@ -6,12 +6,17 @@ def generate_tasks(activedDbList: list[DB], dbConfigs, activedCaseList: list[Cas tasks = [] for db in activedDbList: for case in activedCaseList: + cfg = {key.value: value for key, value in allCaseConfigs[db][case].items()} + # Many DBCaseConfig models require an `index` field, while the UI stores the selection under `IndexType`. + # Passing both keeps backwards-compatibility (extra fields are ignored) and enables strict models (e.g. OceanBase). + if CaseConfigParamType.IndexType in allCaseConfigs[db][case] and "index" not in cfg: + cfg["index"] = allCaseConfigs[db][case][CaseConfigParamType.IndexType] task = TaskConfig( db=db.value, db_config=dbConfigs[db], case_config=case, db_case_config=db.case_config_cls(allCaseConfigs[db][case].get(CaseConfigParamType.IndexType, None))( - **{key.value: value for key, value in allCaseConfigs[db][case].items()} + **cfg ), ) tasks.append(task) diff --git a/vectordb_bench/frontend/components/run_test/inputWidget.py b/vectordb_bench/frontend/components/run_test/inputWidget.py new file mode 100644 index 000000000..da9fc4bb7 --- /dev/null +++ b/vectordb_bench/frontend/components/run_test/inputWidget.py @@ -0,0 +1,48 @@ +from vectordb_bench.frontend.config.dbCaseConfigs import CaseConfigInput, InputType + + +def inputWidget(st, config: CaseConfigInput, key: str): + if config.inputType == InputType.Text: + return st.text_input( + config.displayLabel if config.displayLabel else config.label.value, + key=key, + help=config.inputHelp, + value=config.inputConfig["value"], + ) + if config.inputType == InputType.Option: + return st.selectbox( + config.displayLabel if config.displayLabel else config.label.value, + config.inputConfig["options"], + key=key, + help=config.inputHelp, + ) + if config.inputType == InputType.Number: + return st.number_input( + config.displayLabel if config.displayLabel else config.label.value, + # format="%d", + step=config.inputConfig.get("step", 1), + min_value=config.inputConfig["min"], + max_value=config.inputConfig["max"], + key=key, + value=config.inputConfig["value"], + help=config.inputHelp, + ) + if config.inputType == InputType.Float: + return st.number_input( + config.displayLabel if config.displayLabel else config.label.value, + step=config.inputConfig.get("step", 0.1), + min_value=config.inputConfig["min"], + max_value=config.inputConfig["max"], + key=key, + value=config.inputConfig["value"], + help=config.inputHelp, + ) + if config.inputType == InputType.Bool: + return st.selectbox( + config.displayLabel if config.displayLabel else config.label.value, + options=[True, False], + index=0 if config.inputConfig["value"] else 1, + key=key, + help=config.inputHelp, + ) + raise Exception(f"Invalid InputType: {config.inputType}") diff --git a/vectordb_bench/frontend/components/run_test/submitTask.py b/vectordb_bench/frontend/components/run_test/submitTask.py index 5827efeb4..01d0c5876 100644 --- a/vectordb_bench/frontend/components/run_test/submitTask.py +++ b/vectordb_bench/frontend/components/run_test/submitTask.py @@ -55,11 +55,17 @@ def advancedSettings(st): "Concurrent Input", value=defaultconcurrentInput, label_visibility="collapsed" ) container[1].caption("num of concurrencies for search tests to get max-qps") - return index_already_exists, use_aliyun, k, concurrentInput + + container = st.columns([1, 2]) + concurrency_duration = container[0].number_input( + "Concurrency Duration", value=config.CONCURRENCY_DURATION, label_visibility="collapsed" + ) + container[1].caption("concurrency duration for each concurrency search test") + return index_already_exists, use_aliyun, k, concurrentInput, concurrency_duration def controlPanel(st, tasks: list[TaskConfig], taskLabel, isAllValid): - index_already_exists, use_aliyun, k, concurrentInput = advancedSettings(st) + index_already_exists, use_aliyun, k, concurrentInput, concurrency_duration = advancedSettings(st) def runHandler(): benchmark_runner.set_drop_old(not index_already_exists) @@ -73,7 +79,7 @@ def runHandler(): for task in tasks: task.case_config.k = k task.case_config.concurrency_search_config.num_concurrency = concurrentInput_list - + task.case_config.concurrency_search_config.concurrency_duration = concurrency_duration benchmark_runner.set_download_address(use_aliyun) benchmark_runner.run(tasks, taskLabel) @@ -86,7 +92,9 @@ def stopHandler(): currentTaskId = benchmark_runner.get_current_task_id() tasksCount = benchmark_runner.get_tasks_count() text = f":running: Running Task {currentTaskId} / {tasksCount}" - st.progress(currentTaskId / tasksCount, text=text) + + if tasksCount > 0: + st.progress(currentTaskId / tasksCount, text=text) columns = st.columns(6) columns[0].button( diff --git a/vectordb_bench/frontend/components/streaming/charts.py b/vectordb_bench/frontend/components/streaming/charts.py new file mode 100644 index 000000000..a05da9b25 --- /dev/null +++ b/vectordb_bench/frontend/components/streaming/charts.py @@ -0,0 +1,253 @@ +import plotly.graph_objects as go + +from vectordb_bench.frontend.components.streaming.data import ( + DisplayedMetric, + StreamingData, + get_streaming_data, +) +from vectordb_bench.frontend.config.styles import ( + COLORS_10, + COLORS_2, + SCATTER_LINE_WIDTH, + SCATTER_MAKER_SIZE, + STREAMING_CHART_COLUMNS, +) + + +def drawChartsByCase( + st, + allData, + showCaseNames: list[str], + **kwargs, +): + allData = [d for d in allData if len(d["st_search_stage_list"]) > 0] + for case_name in showCaseNames: + data = [d for d in allData if d["case_name"] == case_name] + if len(data) == 0: + continue + container = st.container() + container.write("") # blank line + container.subheader(case_name) + drawChartByMetric(container, data, case_name=case_name, **kwargs) + container.write("") # blank line + + +def drawChartByMetric( + st, + case_data, + case_name: str, + line_chart_displayed_y_metrics: list[tuple[DisplayedMetric, str]], + **kwargs, +): + columns = st.columns(STREAMING_CHART_COLUMNS) + streaming_data = get_streaming_data(case_data) + + # line chart + for i, metric_info in enumerate(line_chart_displayed_y_metrics): + metric, note = metric_info + container = columns[i % STREAMING_CHART_COLUMNS] + container.markdown(f"#### {metric.value.capitalize()}") + container.markdown(f"{note}") + key = f"{case_name}-{metric.value}" + drawLineChart(container, streaming_data, metric=metric, key=key, **kwargs) + + # bar chart + container = columns[len(line_chart_displayed_y_metrics) % STREAMING_CHART_COLUMNS] + container.markdown("#### Duration") + container.markdown( + "insert more than ideal-insert-duration (dash-line) means exceeding the maximum processing capacity.", + help="vectordb need more time to process accumulated insert requests.", + ) + key = f"{case_name}-duration" + drawBarChart(container, case_data, key=key, **kwargs) + # drawLineChart(container, data, line_x_displayed_label, label) + # drawTestChart(container) + + +def drawLineChart( + st, + streaming_data: list[StreamingData], + metric: DisplayedMetric, + key: str, + with_last_optimized_data=True, + **kwargs, +): + db_names = list({d.db_name for d in streaming_data}) + db_names.sort() + x_metric = kwargs.get("line_chart_displayed_x_metric", DisplayedMetric.search_stage) + fig = go.Figure() + if x_metric == DisplayedMetric.search_time: + ideal_insert_duration = streaming_data[0].ideal_insert_duration + fig.add_shape( + type="line", + y0=min([getattr(d, metric.value) for d in streaming_data]), + y1=max([getattr(d, metric.value) for d in streaming_data]), + x0=ideal_insert_duration, + x1=ideal_insert_duration, + line=dict(color="#999", width=SCATTER_LINE_WIDTH, dash="dot"), + showlegend=True, + name="insert 100% standard time", + ) + for i, db_name in enumerate(db_names): + data = [d for d in streaming_data if d.db_name == db_name] + color = COLORS_10[i] + if with_last_optimized_data: + fig.add_trace( + get_optimized_scatter( + data, + db_name=db_name, + metric=metric, + color=color, + **kwargs, + ) + ) + fig.add_trace( + get_normal_scatter( + data, + db_name=db_name, + metric=metric, + color=color, + **kwargs, + ) + ) + fig.update_layout( + margin=dict(l=0, r=0, t=40, b=0, pad=8), + legend=dict(orientation="h", yanchor="bottom", y=1, xanchor="left", x=0, title=""), + ) + + x_title = "Search Stages (%)" + if x_metric == DisplayedMetric.search_time: + x_title = "Actual Time (s)" + fig.update_layout(xaxis_title=x_title) + st.plotly_chart(fig, use_container_width=True, key=key) + + +def get_normal_scatter( + data: list[StreamingData], + db_name: str, + metric: DisplayedMetric, + color: str, + line_chart_displayed_x_metric: DisplayedMetric, + **kwargs, +): + unit = "" + if "latency" in metric.value: + unit = "ms" + data.sort(key=lambda x: getattr(x, line_chart_displayed_x_metric.value)) + data = [d for d in data if not d.optimized] + hovertemplate = f"%{{text}}% data inserted.
{metric.value}=%{{y:.4g}}{unit}" + if line_chart_displayed_x_metric == DisplayedMetric.search_time: + hovertemplate = f"%{{text}}% data inserted.
actual_time=%{{x:.4g}}s
{metric.value}=%{{y:.4g}}{unit}" + return go.Scatter( + x=[getattr(d, line_chart_displayed_x_metric.value) for d in data], + y=[getattr(d, metric.value) for d in data], + text=[d.search_stage for d in data], + mode="markers+lines", + name=db_name, + marker=dict(color=color, size=SCATTER_MAKER_SIZE), + line=dict(dash="solid", width=SCATTER_LINE_WIDTH, color=color), + legendgroup=db_name, + hovertemplate=hovertemplate, + ) + + +def get_optimized_scatter( + data: list[StreamingData], + db_name: str, + metric: DisplayedMetric, + color: str, + line_chart_displayed_x_metric: DisplayedMetric, + **kwargs, +): + unit = "" + if "latency" in metric.value: + unit = "ms" + data.sort(key=lambda x: x.search_stage) + if not data[-1].optimized or len(data) < 2: + return go.Scatter() + data = data[-2:] + hovertemplate = f"all data inserted and optimized.
{metric.value}=%{{y:.4g}}{unit}" + if line_chart_displayed_x_metric == DisplayedMetric.search_time: + hovertemplate = f"all data inserted and optimized.
actual_time=%{{x:.4g}}s
{metric.value}=%{{y:.4g}}{unit}" + return go.Scatter( + x=[getattr(d, line_chart_displayed_x_metric.value) for d in data], + y=[getattr(d, metric.value) for d in data], + text=[d.search_stage for d in data], + mode="markers+lines", + name=db_name, + legendgroup=db_name, + marker=dict(color=color, size=[0, SCATTER_MAKER_SIZE]), + line=dict(dash="dash", width=SCATTER_LINE_WIDTH, color=color), + hovertemplate=hovertemplate, + showlegend=False, + ) + + +def drawBarChart( + st, + data, + key: str, + with_last_optimized_data=True, + **kwargs, +): + if len(data) < 1: + return + fig = go.Figure() + + # ideal insert duration + ideal_insert_duration = data[0]["st_ideal_insert_duration"] + fig.add_shape( + type="line", + y0=-0.5, + y1=len(data) - 0.5, + x0=ideal_insert_duration, + x1=ideal_insert_duration, + line=dict(color="#999", width=SCATTER_LINE_WIDTH, dash="dot"), + showlegend=True, + name="insert 100% standard time", + ) + + # insert duration + fig.add_trace( + get_bar( + data, + metric=DisplayedMetric.insert_duration, + color=COLORS_2[0], + **kwargs, + ) + ) + + # optimized duration + if with_last_optimized_data: + fig.add_trace( + get_bar( + data, + metric=DisplayedMetric.optimize_duration, + color=COLORS_2[1], + **kwargs, + ) + ) + fig.update_layout( + margin=dict(l=0, r=0, t=40, b=0, pad=8), + legend=dict(orientation="h", yanchor="bottom", y=1, xanchor="left", x=0, title=""), + ) + fig.update_layout(xaxis_title="time (s)") + fig.update_layout(barmode="stack") + fig.update_traces(width=0.15) + st.plotly_chart(fig, use_container_width=True, key=key) + + +def get_bar( + data: list[StreamingData], + metric: DisplayedMetric, + color: str, + **kwargs, +): + return go.Bar( + x=[d[metric.value] for d in data], + y=[d["db_name"] for d in data], + name=metric, + marker_color=color, + orientation="h", + hovertemplate="%{y} %{x:.2f}s", + ) diff --git a/vectordb_bench/frontend/components/streaming/concurrent_detail.py b/vectordb_bench/frontend/components/streaming/concurrent_detail.py new file mode 100644 index 000000000..0580bad14 --- /dev/null +++ b/vectordb_bench/frontend/components/streaming/concurrent_detail.py @@ -0,0 +1,273 @@ +import plotly.graph_objects as go +import streamlit as st + + +def drawConcurrentPerformanceSection(container, case_data, case_name: str): + """Main section for concurrent performance detail in streaming tests""" + + # Check if data exists + if not case_data.get("st_conc_qps_list_list") or len(case_data["st_conc_qps_list_list"]) == 0: + container.info( + "Concurrent latency detail not available for this test. " + "Re-run the test with updated code to collect this data." + ) + return + + container.markdown("---") + container.subheader("Concurrent Search Performance") + container.markdown( + "Detailed Latency → QPS relationship at each stage. " "Displays how latency and QPS vary with concurrency." + ) + + # View mode selector + view_mode = container.radio( + "View Mode", options=["Single Stage", "Compare Stages"], horizontal=True, key=f"{case_name}-view-mode" + ) + + if view_mode == "Single Stage": + drawSingleStageView(container, case_data, case_name) + else: + drawCompareStagesView(container, case_data, case_name) + + +def drawSingleStageView(container, case_data, case_name: str): + """Show detailed Latency→QPS for one selected stage""" + + stages = case_data["st_search_stage_list"] + + # Find the last stage with data for default selection + default_stage_idx = len(stages) - 1 + for i in range(len(stages) - 1, -1, -1): + if case_data["st_conc_qps_list_list"][i]: + default_stage_idx = i + break + + # Stage selector (show all stages) + stage = container.selectbox( + "Select Stage", + options=stages, + index=default_stage_idx, + format_func=lambda x: f"{x}% data loaded", + key=f"{case_name}-stage-selector", + ) + + stage_idx = stages.index(stage) + + # Check if this stage has concurrent data + if not case_data["st_conc_qps_list_list"][stage_idx]: + container.warning( + f"No concurrent search data for {stage}% stage.\n\n" + f"**Reason:** Concurrent tests were skipped because there wasn't enough time " + f"between stages (< 10s per concurrency level).\n\n" + f"**Tip:** Use a larger dataset or slower insert rate to get data for all stages." + ) + return + + # Latency metric selector + latency_metric = container.radio( + "Latency Metric", options=["P99", "P95", "Average"], horizontal=True, key=f"{case_name}-latency-metric" + ) + + # Get data for selected stage + qps_values = case_data["st_conc_qps_list_list"][stage_idx] + conc_nums = case_data["st_conc_num_list_list"][stage_idx] + + # Get latency based on selection + if latency_metric == "P99": + latencies_sec = case_data["st_conc_latency_p99_list_list"][stage_idx] + elif latency_metric == "P95": + latencies_sec = case_data["st_conc_latency_p95_list_list"][stage_idx] + else: + latencies_sec = case_data["st_conc_latency_avg_list_list"][stage_idx] + + latencies_ms = [l * 1000 for l in latencies_sec] # Convert to ms + + # Draw chart + drawQPSLatencyChart(container, qps_values, latencies_ms, stage, latency_metric, case_name) + + # Draw table + drawMetricsTable( + container, + qps_values, + conc_nums, + case_data["st_conc_latency_p99_list_list"][stage_idx], + case_data["st_conc_latency_p95_list_list"][stage_idx], + case_data["st_conc_latency_avg_list_list"][stage_idx], + stage, + ) + + +def drawQPSLatencyChart(container, qps_values, latencies_ms, stage, metric_name, case_name): + """Draw Latency vs QPS scatter plot""" + + fig = go.Figure() + + fig.add_trace( + go.Scatter( + x=latencies_ms, + y=qps_values, + mode="lines+markers+text", + text=[f"{qps:.1f}" for qps in qps_values], + textposition="top center", + marker=dict(size=12, color="#1f77b4"), + line=dict(width=2, color="#1f77b4"), + hovertemplate=("QPS: %{y:.1f}
" f"{metric_name} Latency: %{{x:.1f}}ms
" ""), + ) + ) + + fig.update_layout( + title=f"Latency vs QPS at Stage {stage}%", + xaxis_title=f"Latency {metric_name} (ms)", + yaxis_title="Queries Per Second (QPS)", + height=500, + hovermode="closest", + showlegend=False, + ) + + container.plotly_chart(fig, use_container_width=True, key=f"{case_name}-chart-{stage}") + + +def drawMetricsTable(container, qps_values, conc_nums, p99_list, p95_list, avg_list, stage): + """Draw detailed metrics table""" + + container.markdown(f"**Detailed Metrics at Stage {stage}%**") + + # Build table data + table_data = [] + for i in range(len(qps_values)): + table_data.append( + { + "Concurrency": conc_nums[i], + "QPS": f"{qps_values[i]:.2f}", + "P99 (ms)": f"{p99_list[i] * 1000:.1f}", + "P95 (ms)": f"{p95_list[i] * 1000:.1f}", + "Avg (ms)": f"{avg_list[i] * 1000:.1f}", + } + ) + + container.table(table_data) + + +def drawCompareStagesView(container, case_data, case_name: str): + """Show QPS→Latency curves for multiple stages""" + + stages = case_data["st_search_stage_list"] + + # Find stages with data for default selection + stages_with_data = [stage for i, stage in enumerate(stages) if case_data["st_conc_qps_list_list"][i]] + + # Stage multi-selector (show all stages, but default to ones with data) + default_stages = [] + if stages_with_data: + default_stages = [stages_with_data[0], stages_with_data[-1]] if len(stages_with_data) >= 2 else stages_with_data + + selected_stages = container.multiselect( + "Select stages to compare", + options=stages, + default=default_stages, + format_func=lambda x: f"{x}%", + key=f"{case_name}-compare-stages", + help="Note: Some stages may have no concurrent data if test duration was too short", + ) + + if not selected_stages: + container.warning("Please select at least one stage to display.") + return + + # Latency metric selector + latency_metric = container.radio( + "Latency Metric", options=["P99", "P95", "Average"], horizontal=True, key=f"{case_name}-compare-metric" + ) + + # Draw comparison chart + drawComparisonChart(container, case_data, selected_stages, latency_metric, case_name) + + +def drawComparisonChart(container, case_data, selected_stages, metric_name, case_name): + """Draw multi-line comparison chart""" + + fig = go.Figure() + + # Color palette + colors = [ + "#1f77b4", + "#ff7f0e", + "#2ca02c", + "#d62728", + "#9467bd", + "#8c564b", + "#e377c2", + "#7f7f7f", + "#bcbd22", + "#17becf", + ] + + stages_plotted = 0 + stages_skipped = [] + + for idx, stage in enumerate(selected_stages): + stage_idx = case_data["st_search_stage_list"].index(stage) + + qps_values = case_data["st_conc_qps_list_list"][stage_idx] + + # Skip stages with no data + if not qps_values: + stages_skipped.append(stage) + continue + + # Get latency based on selection + if metric_name == "P99": + latencies_sec = case_data["st_conc_latency_p99_list_list"][stage_idx] + elif metric_name == "P95": + latencies_sec = case_data["st_conc_latency_p95_list_list"][stage_idx] + else: + latencies_sec = case_data["st_conc_latency_avg_list_list"][stage_idx] + + latencies_ms = [l * 1000 for l in latencies_sec] + stages_plotted += 1 + + fig.add_trace( + go.Scatter( + x=latencies_ms, + y=qps_values, + mode="lines+markers", + name=f"{stage}% loaded", + marker=dict(size=10), + line=dict(width=2, color=colors[idx % len(colors)]), + hovertemplate=( + f"Stage {stage}%
" + "QPS: %{y:.1f}
" + f"{metric_name} Latency: %{{x:.1f}}ms
" + "" + ), + ) + ) + + # Check if any data was plotted + if stages_plotted == 0: + container.warning( + f"None of the selected stages have concurrent search data.\n\n" + f"**Skipped stages:** {', '.join([f'{s}%' for s in stages_skipped])}\n\n" + f"**Reason:** Concurrent tests were skipped because there wasn't enough time " + f"between stages (< 10s per concurrency level).\n\n" + f"**Tip:** Use a larger dataset or slower insert rate to get data for all stages." + ) + return + + # Show warning for skipped stages + if stages_skipped: + container.info(f"Stages without data: {', '.join([f'{s}%' for s in stages_skipped])}") + + fig.update_layout( + title=f"Latency vs QPS Evolution Across Stages", + xaxis_title=f"Latency {metric_name} (ms)", + yaxis_title="Queries Per Second (QPS)", + height=600, + hovermode="closest", + legend=dict(yanchor="top", y=0.99, xanchor="right", x=0.99), + ) + + container.plotly_chart(fig, use_container_width=True, key=f"{case_name}-compare-chart") + + # Add insight + container.info("**Insight:** Compare curves across stages to understand how performance scales with data growth.") diff --git a/vectordb_bench/frontend/components/streaming/data.py b/vectordb_bench/frontend/components/streaming/data.py new file mode 100644 index 000000000..e3f2c8a1c --- /dev/null +++ b/vectordb_bench/frontend/components/streaming/data.py @@ -0,0 +1,69 @@ +from enum import StrEnum +from dataclasses import dataclass + + +class DisplayedMetric(StrEnum): + db = "db" + db_name = "db_name" + search_stage = "search_stage" + search_time = "search_time" + qps = "qps" + recall = "recall" + ndcg = "ndcg" + adjusted_recall = "adjusted_recall" + adjusted_ndcg = "adjusted_ndcg" + latency_p99 = "latency_p99" + latency_p95 = "latency_p95" + # st_ideal_insert_duration = "st_ideal_insert_duration" + # st_search_time_list = "st_search_time_list" + insert_duration = "insert_duration" + optimize_duration = "optimize_duration" + + +@dataclass +class StreamingData: + db: str + db_name: str + search_stage: int + search_time: float + qps: float + recall: float + ndcg: float + adjusted_recall: float + adjusted_ndcg: float + latency_p99: float + latency_p95: float + ideal_insert_duration: int + insert_duration: float + optimize_duration: float + + @property + def optimized(self) -> bool: + return self.search_stage > 100 + + +def get_streaming_data(data) -> list[StreamingData]: + return [ + StreamingData( + db=d["db"], + db_name=d["db_name"], + search_stage=search_stage, + search_time=d["st_search_time_list"][i], + qps=d["st_max_qps_list_list"][i], + recall=d["st_recall_list"][i], + ndcg=d["st_ndcg_list"][i], + adjusted_recall=round(d["st_recall_list"][i] / min(search_stage, 100) * 100, 4), + adjusted_ndcg=round(d["st_ndcg_list"][i] / min(search_stage, 100) * 100, 4), + latency_p99=round(d["st_serial_latency_p99_list"][i] * 1000, 2), + latency_p95=( + round(d["st_serial_latency_p95_list"][i] * 1000, 2) + if "st_serial_latency_p95_list" in d and i < len(d["st_serial_latency_p95_list"]) + else 0.0 + ), + ideal_insert_duration=d["st_ideal_insert_duration"], + insert_duration=d["insert_duration"], + optimize_duration=d["optimize_duration"], + ) + for d in data + for i, search_stage in enumerate(d["st_search_stage_list"]) + ] diff --git a/vectordb_bench/frontend/components/tables/data.py b/vectordb_bench/frontend/components/tables/data.py index fbe83f197..5a229e6e4 100644 --- a/vectordb_bench/frontend/components/tables/data.py +++ b/vectordb_bench/frontend/components/tables/data.py @@ -24,7 +24,7 @@ def formatData(caseResults: list[CaseResult]): db = caseResult.task_config.db.value db_label = caseResult.task_config.db_config.db_label case_config = caseResult.task_config.case_config - case = case_config.case_id.case_cls() + case = case_config.case filter_rate = case.filter_rate dataset = case.dataset.data.name metrics = asdict(caseResult.metrics) diff --git a/vectordb_bench/frontend/components/welcome/explainPrams.py b/vectordb_bench/frontend/components/welcome/explainPrams.py new file mode 100644 index 000000000..b7827d141 --- /dev/null +++ b/vectordb_bench/frontend/components/welcome/explainPrams.py @@ -0,0 +1,58 @@ +def explainPrams(st): + st.markdown("## descriptions") + st.markdown("### 1. Overview") + st.markdown(""" +- **VectorDBBench(VDBBench)** is an open-source benchmarking tool designed specifically for vector databases. Its main features include: + - (1) An easy-to-use **web UI** for configuration of tests and visual analysis of results. + - (2) A comprehensive set of **standards for testing and metric collection**. + - (3) Support for **various scenarios**, including additional support for **Filter** and **Streaming** based on standard tests. +- VDBBench embraces open-source and welcome contributions of code and test result submissions. The testing process and extended scenarios of VDBBench, as well as the intention behind our design will be introduced as follows. +""") + st.markdown("### 2. Dataset") + st.markdown(""" +- We provide two embedding datasets: + - (1)*[Cohere 768dim](https://huggingface.co/datasets/Cohere/wikipedia-22-12)*, generated using the **Cohere** model based on the Wikipedia corpus. + - (2)*[Cohere 1024dim](https://huggingface.co/datasets/Cohere/beir-embed-english-v3)*, generated using the **Cohere** embed-english-v3.0 model based on the bioasq corpus. + - (3)*OpenAI 1536dim*, generated using the **OpenAI** model based on the [C4 corpus](https://huggingface.co/datasets/legacy-datasets/c4). +""") + st.markdown("### 3. Standard Test") + st.markdown( + """ +The test is actually divided into 3 sub-processes +- **3.1 Test Part 1 - Load (Insert + Optimize)** + - (1) Use a single process to perform serial inserts until all data is inserted, and record the time taken as **insert_duration**. + - (2) For most vector databases, index construction requires additional time to optimize to achieve an optimal state, and record the time taken as **optimize_duration**. + - (3) **Load_duration (insert_duration + optimize_duration)** can be understood as the time from the start of insertion until the database is ready to query. + - load_duration can serve as a reference for the insert capability of a vector database to some extent. However, it should be noted that some vector databases may perform better under **concurrent insert operations**. +- **3.2 Test Part 2 - Serial Search Test** + - (1) Use a single process to perform serial searches, record the results and time taken for each search, and calculate **recall** and **latency**. + - (2) **Recall**: For vector databases, most searches are approximately nearest neighbor(ANN) searches rather than perfectly accurate results. In production environments, commonly targeted recall rates are 0.9 or 0.95. + - Note that there is a **trade-off** between **accuracy** and **search performance**. By adjusting parameters, it is possible to sacrifice some accuracy in exchange for better performance. We recommend comparing performance while ensuring that the recall rates remain reasonably close. + - (3) **Latency**:**p99** rather than average. **latency_p99** focuses on **the slowest 1% of requests**. In many high-demand applications, ensuring that most user requests stay within acceptable latency limits is critical, whereas **latency_avg** can be skewed by faster requests. + - **serial_latency** can serve as a reference for a database's search capability to some extent. However, serial_latency is significantly affected by network conditions. We recommend running the test client and database server within the same local network. +- **3.3 Test Part 3 - Concurrent Search Test** + - (1) Create multiple processes, each perform serial searches independently to test the database's **maximum throughput(max-qps)**. + - (2) Since different databases may reach peak throughput under different conditions, we conduct multiple test rounds. The number of processes **starts at 1 by default and gradually increases up to 80**, with each test group running for **30 seconds**. + - Detailed latency and QPS metrics at different concurrency levels can be viewed on the *concurrent* page. + - The highest recorded QPS value from these tests will be selected as the final max-qps. +""", + unsafe_allow_html=True, + ) + st.markdown("### 4. Filter Search Test") + st.markdown(""" +- Compared to the Standard Test, the **Filter Search** introduces additional scalar constraints (e.g. **color == red**) during the Search Test. Different **filter_ratios** present varying levels of challenge to the VectorDB's search performance. +- We provide an additional **string column** containing 10 labels with different distribution ratios (50%,20%,10%,5%,2%,1%,0.5%,0.2%,0.1%). For each label, we conduct both a **Serial Test** and a **Concurrency Test** to observe the VectorDB's performance in terms of **QPS, latency, and recall** under different filtering conditions. +""") + st.markdown("### 5. Streaming Search Test") + st.markdown(""" +Different from Standard's load and search separation, Streaming Search Test primarily focuses on **search performance during the insertion process**. +Different **base dataset sizes** and varying **insertion rates** set distinct challenges to the VectorDB's search capabilities. +VDBBench will send insert requests at a **fixed rate**, maintaining consistent insertion pressure. The search test consists of three steps as follows: +- 1.**Streaming Search** + - Users can configure **multiple search stages**. When the inserted data volume reaches a specified stage, a **Serial Test** and a **Concurrent Test** will be conducted, recording qps, latency, and recall performance. +- 2.**Streaming Final Search** + - After all of the data is inserted, a Serial Test and a Concurrent Test are immediately performed, recording qps, latency, and recall performance. + - Note: at this time, the insertion pressure drops to zero since data insertion is complete. +- 3.**Optimized Search (Optional)** + - Users can optionally perform an additional optimization step followed by a Serial Test and a Concurrent Test, recording qps, latency, and recall performance. This step **compares performance in Streaming section with the theoretically optimal performance**. +""") diff --git a/vectordb_bench/frontend/components/welcome/pagestyle.py b/vectordb_bench/frontend/components/welcome/pagestyle.py new file mode 100644 index 000000000..476de3fc6 --- /dev/null +++ b/vectordb_bench/frontend/components/welcome/pagestyle.py @@ -0,0 +1,106 @@ +def pagestyle(): + html_content = """ + + +
+ """ + return html_content diff --git a/vectordb_bench/frontend/components/welcome/welcomePrams.py b/vectordb_bench/frontend/components/welcome/welcomePrams.py new file mode 100644 index 000000000..48bf5995c --- /dev/null +++ b/vectordb_bench/frontend/components/welcome/welcomePrams.py @@ -0,0 +1,185 @@ +import base64 +from PIL import Image +from io import BytesIO +import os +from pathlib import Path +from importlib import resources + +from vectordb_bench.frontend.components.welcome.pagestyle import pagestyle + + +def get_image_as_base64(image_path): + try: + if image_path.startswith("http"): + return image_path + + # Try to load from package resources first (for pip installed package) + if image_path.startswith("fig/homepage/"): + try: + # Convert fig/homepage/xxx.png to vectordb_bench.fig.homepage + package_parts = ["vectordb_bench"] + image_path.split("/")[:-1] + package_name = ".".join(package_parts) + file_name = os.path.basename(image_path) + + # Get the resource content using importlib.resources + files = resources.files(package_name) + img_data = (files / file_name).read_bytes() + + img = Image.open(BytesIO(img_data)) + buffered = BytesIO() + img.save(buffered, format="PNG") + return f"data:image/png;base64,{base64.b64encode(buffered.getvalue()).decode()}" + except Exception: + # If package resource fails, try the original path + pass + + # Fallback to file system path (for development) + path = os.path.expanduser(image_path) + if not os.path.isabs(path): + # Try relative to the vectordb_bench package directory + package_dir = Path(__file__).parent.parent.parent + path = package_dir / path + + img = Image.open(path) + buffered = BytesIO() + img.save(buffered, format="PNG") + return f"data:image/png;base64,{base64.b64encode(buffered.getvalue()).decode()}" + except Exception: + return None + + +def welcomePrams(st): + st.title("Welcome to VDBBench!") + options = [ + { + "title": "Standard Test Results", + "description": ( + "" + "Select a specific run or compare all results side by side to view the results of previous tests." + "" + ), + "image": "fig/homepage/bar-chart.png", + "link": "results", + }, + { + "title": "Quries Per Dollar", + "description": ( + "" + "To view the results of quries per dollar.
" + "(similar to qps in Results) " + "" + ), + "image": "fig/homepage/qp$.png", + "link": "quries_per_dollar", + }, + { + "title": "Tables", + "description": ( + "" "To view the results of differnt datasets in tables." "" + ), + "image": "fig/homepage/table.png", + "link": "tables", + }, + { + "title": "Concurrent Performance", + "description": ( + "" + "To view the variation of qps with latency under different concurrent." + "" + ), + "image": "fig/homepage/concurrent.png", + "link": "concurrent", + }, + { + "title": "Label Filter Performance", + "description": ( + "" + "To view the perfomance of datasets under different label filter ratios " + "" + ), + "image": "fig/homepage/label_filter.png", + "link": "label_filter", + }, + { + "title": "Int Filter Performance", + "description": ( + "" + "To view the perfomance of datasets under different int filter ratios " + "" + ), + "image": "fig/homepage/label_filter.png", + "link": "int_filter", + }, + { + "title": "Streaming Performance", + "description": ( + "" + "To view the perfomance of datasets under different search stages and insertion rates. " + "" + ), + "image": "fig/homepage/streaming.png", + "link": "streaming", + }, + { + "title": "Run Test", + "description": ( + "" + "Select the databases and cases to test.
" + "The test results will be displayed in Results." + "" + ), + "image": "fig/homepage/run_test.png", + "link": "run_test", + }, + { + "title": "Custom Dataset", + "description": ( + "" + "Define users' own datasets with detailed descriptions of setting each parameter." + "" + ), + "image": "fig/homepage/custom.png", + "link": "custom", + }, + ] + + html_content = pagestyle() + + for option in options: + option["image"] = get_image_as_base64(option["image"]) + + for option in options[:7]: + html_content += f""" + +
+ {option['title']} +
{option['title']}
+
{option['description']}
+
+ + """ + + html_content += """ +
+
+

Run Your Own Test

+
+
+ """ + + for option in options[7:9]: + html_content += f""" + +
+ {option['title']} +
{option['title']}
+
{option['description']}
+
+ + """ + + html_content += """ +
+ """ + + st.html(html_content) diff --git a/vectordb_bench/frontend/config/dbCaseConfigs.py b/vectordb_bench/frontend/config/dbCaseConfigs.py index bb9cfa44b..6a32e5ff1 100644 --- a/vectordb_bench/frontend/config/dbCaseConfigs.py +++ b/vectordb_bench/frontend/config/dbCaseConfigs.py @@ -3,7 +3,8 @@ from pydantic import BaseModel from vectordb_bench.backend.cases import CaseLabel, CaseType from vectordb_bench.backend.clients import DB -from vectordb_bench.backend.clients.api import IndexType, MetricType +from vectordb_bench.backend.clients.api import IndexType, MetricType, SQType +from vectordb_bench.backend.dataset import DatasetWithSizeType from vectordb_bench.frontend.components.custom.getCustomConfig import get_custom_configs from vectordb_bench.models import CaseConfig, CaseConfigParamType @@ -23,32 +24,58 @@ class BatchCaseConfig(BaseModel): cases: list[CaseConfig] = [] +class InputType(IntEnum): + Text = 20001 + Number = 20002 + Option = 20003 + Float = 20004 + Bool = 20005 + + +class ConfigInput(BaseModel): + label: CaseConfigParamType + inputType: InputType = InputType.Text + inputConfig: dict = {} + inputHelp: str = "" + displayLabel: str = "" + + +class CaseConfigInput(ConfigInput): + # todo type should be a function + isDisplayed: typing.Any = lambda config: True + + class UICaseItem(BaseModel): isLine: bool = False + key: str = "" label: str = "" description: str = "" cases: list[CaseConfig] = [] caseLabel: CaseLabel = CaseLabel.Performance + extra_custom_case_config_inputs: list[ConfigInput] = [] + tmp_custom_config: dict = dict() def __init__( self, isLine: bool = False, - case_id: CaseType | None = None, - custom_case: dict | None = None, - cases: list[CaseConfig] | None = None, + cases: list[CaseConfig] = None, label: str = "", description: str = "", caseLabel: CaseLabel = CaseLabel.Performance, + **kwargs, ): if isLine is True: - super().__init__(isLine=True) - elif case_id is not None and isinstance(case_id, CaseType): - c = case_id.case_cls(custom_case) + super().__init__(isLine=True, **kwargs) + if cases is None: + cases = [] + elif len(cases) == 1: + c = cases[0].case super().__init__( - label=c.name, - description=c.description, - cases=[CaseConfig(case_id=case_id, custom_case=custom_case)], - caseLabel=c.label, + label=label if label else c.name, + description=description if description else c.description, + cases=cases, + caseLabel=caseLabel, + **kwargs, ) else: super().__init__( @@ -56,10 +83,26 @@ def __init__( description=description, cases=cases, caseLabel=caseLabel, + **kwargs, ) def __hash__(self) -> int: - return hash(self.json()) + return hash(self.key if self.key else self.label) + + def get_cases(self) -> list[CaseConfig]: + # return self.cases + if len(self.extra_custom_case_config_inputs) == 0: + return self.cases + cases = [ + CaseConfig( + case_id=c.case_id, + k=c.k, + concurrency_search_config=c.concurrency_search_config, + custom_case={**c.custom_case, **self.tmp_custom_config}, + ) + for c in self.cases + ] + return cases class UICaseItemCluster(BaseModel): @@ -70,47 +113,249 @@ class UICaseItemCluster(BaseModel): def get_custom_case_items() -> list[UICaseItem]: custom_configs = get_custom_configs() return [ - UICaseItem(case_id=CaseType.PerformanceCustomDataset, custom_case=custom_config.dict()) + UICaseItem( + label=f"{custom_config.dataset_config.name} - None Filter", + cases=[ + CaseConfig( + case_id=CaseType.PerformanceCustomDataset, + custom_case={ + **custom_config.dict(), + "use_filter": False, + }, + ) + ], + ) for custom_config in custom_configs + ] + [ + UICaseItem( + label=f"{custom_config.dataset_config.name} - Filter", + description=( + f'[Batch Cases] This case evaluate search performance under filtering constraints like "color==red."' + f"Vdbbench provides an additional column of randomly distributed labels with fixed proportions, " + f"such as [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5]. " + f"Essentially, vdbbench will test each filter label in your own dataset to" + " assess the vector database's search performance across different filtering conditions." + ), + cases=[ + CaseConfig( + case_id=CaseType.PerformanceCustomDataset, + custom_case={ + **custom_config.dict(), + "use_filter": True, + "label_percentage": label_percentage, + }, + ) + for label_percentage in custom_config.dataset_config.label_percentages + ], + ) + for custom_config in custom_configs + if custom_config.dataset_config.label_percentages ] +def generate_normal_cases(case_id: CaseType, custom_case: dict | None = None) -> list[CaseConfig]: + return [CaseConfig(case_id=case_id, custom_case=custom_case)] + + def get_custom_case_cluter() -> UICaseItemCluster: return UICaseItemCluster(label="Custom Search Performance Test", uiCaseItems=get_custom_case_items()) +def get_custom_streaming_case_items() -> list[UICaseItem]: + from vectordb_bench.frontend.components.custom.getCustomConfig import get_custom_streaming_configs + + custom_streaming_configs = get_custom_streaming_configs() + return [ + UICaseItem( + label=f"{custom_config.dataset_config.name} - Streaming", + description=f"Streaming test with custom dataset: {custom_config.dataset_config.name}", + cases=[ + CaseConfig( + case_id=CaseType.StreamingCustomDataset, + custom_case={ + "description": custom_config.description, + "dataset_config": custom_config.dataset_config.dict(), + }, + ) + ], + caseLabel=CaseLabel.Streaming, + extra_custom_case_config_inputs=custom_streaming_config_with_custom_dataset, + ) + for custom_config in custom_streaming_configs + ] + + +def get_custom_streaming_case_cluster() -> UICaseItemCluster: + return UICaseItemCluster(label="Custom Streaming Test", uiCaseItems=get_custom_streaming_case_items()) + + +def generate_custom_streaming_case() -> CaseConfig: + return CaseConfig( + case_id=CaseType.StreamingPerformanceCase, + custom_case=dict(), + ) + + +custom_streaming_config: list[ConfigInput] = [ + ConfigInput( + label=CaseConfigParamType.dataset_with_size_type, + displayLabel="dataset", + inputType=InputType.Option, + inputConfig=dict(options=[dataset.value for dataset in DatasetWithSizeType]), + ), + ConfigInput( + label=CaseConfigParamType.insert_rate, + inputType=InputType.Number, + inputConfig=dict(step=100, min=100, max=4_000, value=200), + inputHelp="fixed insertion rate (rows/s), must be divisible by 100", + ), + ConfigInput( + label=CaseConfigParamType.search_stages, + inputType=InputType.Text, + inputConfig=dict(value="[0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]"), + inputHelp="0<=stage<1.0; do search test when inserting a specified amount of data.", + ), + ConfigInput( + label=CaseConfigParamType.concurrencies, + inputType=InputType.Text, + inputConfig=dict(value="[5, 10, 20]"), + inputHelp="concurrent num of search test while insertion; record max-qps.", + ), + ConfigInput( + label=CaseConfigParamType.optimize_after_write, + inputType=InputType.Option, + inputConfig=dict(options=[True, False]), + inputHelp="whether to optimize after inserting all data", + ), + ConfigInput( + label=CaseConfigParamType.read_dur_after_write, + inputType=InputType.Number, + inputConfig=dict(step=10, min=30, max=360_000, value=30), + inputHelp="search test duration after inserting all data", + ), +] + +# Config for custom streaming tests (with custom dataset from JSON) +# Filter out the dataset_with_size_type from the existing config +custom_streaming_config_with_custom_dataset: list[ConfigInput] = [ + config for config in custom_streaming_config if config.label != CaseConfigParamType.dataset_with_size_type +] + + +def generate_label_filter_cases(dataset_with_size_type: DatasetWithSizeType) -> list[CaseConfig]: + label_percentages = dataset_with_size_type.get_manager().data.scalar_label_percentages + return [ + CaseConfig( + case_id=CaseType.LabelFilterPerformanceCase, + custom_case=dict(dataset_with_size_type=dataset_with_size_type, label_percentage=label_percentage), + ) + for label_percentage in label_percentages + ] + + +def generate_int_filter_cases(dataset_with_size_type: DatasetWithSizeType) -> list[CaseConfig]: + filter_rates = dataset_with_size_type.get_manager().data.scalar_int_rates + return [ + CaseConfig( + case_id=CaseType.NewIntFilterPerformanceCase, + custom_case=dict(dataset_with_size_type=dataset_with_size_type, filter_rate=filter_rate), + ) + for filter_rate in filter_rates + ] + + UI_CASE_CLUSTERS: list[UICaseItemCluster] = [ UICaseItemCluster( label="Search Performance Test", uiCaseItems=[ - UICaseItem(case_id=CaseType.Performance768D100M), - UICaseItem(case_id=CaseType.Performance768D10M), - UICaseItem(case_id=CaseType.Performance768D1M), + UICaseItem(cases=generate_normal_cases(CaseType.Performance768D100M)), + UICaseItem(cases=generate_normal_cases(CaseType.Performance768D10M)), + UICaseItem(cases=generate_normal_cases(CaseType.Performance768D1M)), + UICaseItem(isLine=True), + UICaseItem(cases=generate_normal_cases(CaseType.Performance1024D1M)), + UICaseItem(cases=generate_normal_cases(CaseType.Performance1024D10M)), UICaseItem(isLine=True), - UICaseItem(case_id=CaseType.Performance1536D5M), - UICaseItem(case_id=CaseType.Performance1536D500K), - UICaseItem(case_id=CaseType.Performance1536D50K), + UICaseItem(cases=generate_normal_cases(CaseType.Performance1536D5M)), + UICaseItem(cases=generate_normal_cases(CaseType.Performance1536D500K)), + UICaseItem(cases=generate_normal_cases(CaseType.Performance1536D50K)), ], ), UICaseItemCluster( - label="Filter Search Performance Test", + label="Int-Filter Search Performance Test", uiCaseItems=[ - UICaseItem(case_id=CaseType.Performance768D10M1P), - UICaseItem(case_id=CaseType.Performance768D10M99P), - UICaseItem(case_id=CaseType.Performance768D1M1P), - UICaseItem(case_id=CaseType.Performance768D1M99P), + UICaseItem(cases=generate_normal_cases(CaseType.Performance768D10M1P)), + UICaseItem(cases=generate_normal_cases(CaseType.Performance768D10M99P)), + UICaseItem(cases=generate_normal_cases(CaseType.Performance768D1M1P)), + UICaseItem(cases=generate_normal_cases(CaseType.Performance768D1M99P)), UICaseItem(isLine=True), - UICaseItem(case_id=CaseType.Performance1536D5M1P), - UICaseItem(case_id=CaseType.Performance1536D5M99P), - UICaseItem(case_id=CaseType.Performance1536D500K1P), - UICaseItem(case_id=CaseType.Performance1536D500K99P), + UICaseItem(cases=generate_normal_cases(CaseType.Performance1536D5M1P)), + UICaseItem(cases=generate_normal_cases(CaseType.Performance1536D5M99P)), + UICaseItem(cases=generate_normal_cases(CaseType.Performance1536D500K1P)), + UICaseItem(cases=generate_normal_cases(CaseType.Performance1536D500K99P)), + ], + ), + UICaseItemCluster( + label="New-Int-Filter Search Performance Test", + uiCaseItems=[ + UICaseItem( + label=f"Int-Filter Search Performance Test - {dataset_with_size_type.value}", + description=( + f"[Batch Cases]These cases test the search performance of a vector database " + f"with dataset {dataset_with_size_type.value}" + f"under filtering rates of {dataset_with_size_type.get_manager().data.scalar_int_rates}, at varying parallel levels." + f"Results will show index building time, recall, and maximum QPS." + ), + cases=generate_int_filter_cases(dataset_with_size_type), + ) + for dataset_with_size_type in [ + DatasetWithSizeType.CohereMedium, + DatasetWithSizeType.CohereLarge, + DatasetWithSizeType.OpenAIMedium, + DatasetWithSizeType.OpenAILarge, + DatasetWithSizeType.BioasqMedium, + DatasetWithSizeType.BioasqLarge, + ] + ], + ), + UICaseItemCluster( + label="Label-Filter Search Performance Test", + uiCaseItems=[ + UICaseItem( + label=f"Label-Filter Search Performance Test - {dataset_with_size_type.value}", + description=( + f'[Batch Cases] These cases evaluate search performance under filtering constraints like "color==red." ' + "Vdbbench provides an additional column of randomly distributed labels with fixed proportions, " + f"such as {dataset_with_size_type.get_manager().data.scalar_label_percentages}. " + f"Essentially, vdbbench will test each filter label in {dataset_with_size_type.value} to " + "assess the vector database's search performance across different filtering conditions. " + ), + cases=generate_label_filter_cases(dataset_with_size_type), + ) + for dataset_with_size_type in DatasetWithSizeType ], ), UICaseItemCluster( label="Capacity Test", uiCaseItems=[ - UICaseItem(case_id=CaseType.CapacityDim960), - UICaseItem(case_id=CaseType.CapacityDim128), + UICaseItem(cases=generate_normal_cases(CaseType.CapacityDim960)), + UICaseItem(cases=generate_normal_cases(CaseType.CapacityDim128)), + ], + ), + UICaseItemCluster( + label="Streaming Test", + uiCaseItems=[ + UICaseItem( + label="Customize Streaming Test", + description=( + "This case test the search performance during insertion. " + "VDBB will send insert requests to VectorDB at a fixed rate and " + "conduct a search test once the insert count reaches the search_stages. " + "After all data is inserted, optimization and search tests can be " + "optionally performed." + ), + cases=[generate_custom_streaming_case()], + extra_custom_case_config_inputs=custom_streaming_config, + ) ], ), ] @@ -123,14 +368,8 @@ def get_custom_case_cluter() -> UICaseItemCluster: CaseType.Performance1536D5M, CaseType.Performance1536D500K, CaseType.Performance1536D50K, - CaseType.Performance768D10M1P, - CaseType.Performance768D1M1P, - CaseType.Performance1536D5M1P, - CaseType.Performance1536D500K1P, - CaseType.Performance768D10M99P, - CaseType.Performance768D1M99P, - CaseType.Performance1536D5M99P, - CaseType.Performance1536D500K99P, + CaseType.Performance1024D1M, + CaseType.Performance1024D10M, CaseType.CapacityDim960, CaseType.CapacityDim128, ] @@ -146,6 +385,7 @@ class InputType(IntEnum): Option = 20003 Float = 20004 Bool = 20005 + Select = 20006 class CaseConfigInput(BaseModel): @@ -164,10 +404,15 @@ class CaseConfigInput(BaseModel): inputConfig={ "options": [ IndexType.HNSW.value, + IndexType.HNSW_SQ.value, + IndexType.HNSW_PQ.value, + IndexType.HNSW_PRQ.value, IndexType.IVFFlat.value, + IndexType.IVFPQ.value, IndexType.IVFSQ8.value, + IndexType.IVF_RABITQ.value, + IndexType.SCANN_MILVUS.value, IndexType.DISKANN.value, - IndexType.STREAMING_DISKANN.value, IndexType.Flat.value, IndexType.AUTOINDEX.value, IndexType.GPU_IVF_FLAT.value, @@ -178,6 +423,22 @@ class CaseConfigInput(BaseModel): }, ) +CaseConfigParamInput_IndexType_OceanBase = CaseConfigInput( + label=CaseConfigParamType.IndexType, + inputType=InputType.Option, + inputHelp="Select OceanBase index type", + inputConfig={ + "options": [ + IndexType.HNSW.value, + IndexType.HNSW_SQ.value, + IndexType.HNSW_BQ.value, + IndexType.IVFFlat.value, + IndexType.IVFSQ8.value, + IndexType.IVFPQ.value, + ], + }, +) + CaseConfigParamInput_IndexType_PgDiskANN = CaseConfigInput( label=CaseConfigParamType.IndexType, inputHelp="Select Index Type", @@ -213,8 +474,58 @@ class CaseConfigInput(BaseModel): }, ) -CaseConfigParamInput_max_neighbors = CaseConfigInput( +CaseConfigParamInput_reranking_PgDiskANN = CaseConfigInput( + label=CaseConfigParamType.reranking, + inputType=InputType.Bool, + displayLabel="Enable Reranking", + inputHelp="Enable if you want to use reranking while performing \ + similarity search with PQ", + inputConfig={ + "value": False, + }, +) + +CaseConfigParamInput_quantized_fetch_limit_PgDiskANN = CaseConfigInput( + label=CaseConfigParamType.quantized_fetch_limit, + displayLabel="Quantized Fetch Limit", + inputHelp="Limit top-k vectors using the quantized vector comparison", + inputType=InputType.Number, + inputConfig={ + "min": 20, + "max": 1000, + "value": 200, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.reranking, False), +) + +CaseConfigParamInput_pq_param_num_chunks_PgDiskANN = CaseConfigInput( + label=CaseConfigParamType.pq_param_num_chunks, + displayLabel="pq_param_num_chunks", + inputHelp="Number of chunks for product quantization (Defaults to 0). 0 means it is determined automatically, based on embedding dimensions.", + inputType=InputType.Number, + inputConfig={ + "min": 0, + "max": 1028, + "value": 0, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.reranking, False), +) + + +CaseConfigParamInput_reranking_metric_PgDiskANN = CaseConfigInput( + label=CaseConfigParamType.reranking_metric, + displayLabel="Reranking Metric", + inputType=InputType.Option, + inputConfig={ + "options": [metric.value for metric in MetricType if metric.value not in ["HAMMING", "JACCARD", "DP"]], + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.reranking, False), +) + + +CaseConfigParamInput_max_neighbors_PgDiskANN = CaseConfigInput( label=CaseConfigParamType.max_neighbors, + displayLabel="max_neighbors", inputType=InputType.Number, inputConfig={ "min": 10, @@ -246,6 +557,29 @@ class CaseConfigInput(BaseModel): isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.DISKANN.value, ) +CaseConfigParamInput_maintenance_work_mem_PgDiskANN = CaseConfigInput( + label=CaseConfigParamType.maintenance_work_mem, + inputHelp="Memory to use during index builds. Not to exceed the available free memory." + "Specify in gigabytes. e.g. 8GB", + inputType=InputType.Text, + inputConfig={ + "value": "8GB", + }, +) + +CaseConfigParamInput_max_parallel_workers_PgDiskANN = CaseConfigInput( + label=CaseConfigParamType.max_parallel_workers, + displayLabel="Max parallel workers", + inputHelp="Recommended value: (cpu cores - 1). This will set the parameters: max_parallel_maintenance_workers," + " max_parallel_workers & table(parallel_workers)", + inputType=InputType.Number, + inputConfig={ + "min": 0, + "max": 1024, + "value": 16, + }, +) + CaseConfigParamInput_num_neighbors = CaseConfigInput( label=CaseConfigParamType.num_neighbors, inputType=InputType.Number, @@ -344,11 +678,18 @@ class CaseConfigInput(BaseModel): inputConfig={ "min": 4, "max": 64, - "value": 30, + "value": 16, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [ + IndexType.HNSW.value, + IndexType.HNSW_SQ.value, + IndexType.HNSW_PQ.value, + IndexType.HNSW_PRQ.value, + ], ) + CaseConfigParamInput_m = CaseConfigInput( label=CaseConfigParamType.m, inputType=InputType.Number, @@ -360,6 +701,20 @@ class CaseConfigInput(BaseModel): isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value, ) +CaseConfigParamInput_m_OceanBase = CaseConfigInput( + label=CaseConfigParamType.m, + displayLabel="m", + inputHelp="HNSW graph degree (m) for OceanBase HNSW/HNSW_SQ/HNSW_BQ", + inputType=InputType.Number, + inputConfig={ + "min": 4, + "max": 128, + "value": 16, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [IndexType.HNSW.value, IndexType.HNSW_SQ.value, IndexType.HNSW_BQ.value], +) + CaseConfigParamInput_EFConstruction_Milvus = CaseConfigInput( label=CaseConfigParamType.EFConstruction, @@ -367,9 +722,78 @@ class CaseConfigInput(BaseModel): inputConfig={ "min": 8, "max": 512, - "value": 360, + "value": 256, }, - isDisplayed=lambda config: config[CaseConfigParamType.IndexType] == IndexType.HNSW.value, + isDisplayed=lambda config: config[CaseConfigParamType.IndexType] + in [ + IndexType.HNSW.value, + IndexType.HNSW_SQ.value, + IndexType.HNSW_PQ.value, + IndexType.HNSW_PRQ.value, + ], +) + +CaseConfigParamInput_EFConstruction_OceanBase = CaseConfigInput( + label=CaseConfigParamType.EFConstruction, + displayLabel="efConstruction", + inputHelp="HNSW efConstruction for OceanBase HNSW/HNSW_SQ/HNSW_BQ", + inputType=InputType.Number, + inputConfig={ + "min": 8, + "max": 65535, + "value": 256, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [IndexType.HNSW.value, IndexType.HNSW_SQ.value, IndexType.HNSW_BQ.value], +) + +CaseConfigParamInput_SQType = CaseConfigInput( + label=CaseConfigParamType.sq_type, + inputType=InputType.Option, + inputHelp="Scalar quantizer type.", + inputConfig={ + "options": [SQType.SQ6.value, SQType.SQ8.value, SQType.BF16.value, SQType.FP16.value, SQType.FP32.value] + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [IndexType.HNSW_SQ.value], +) + +CaseConfigParamInput_Refine = CaseConfigInput( + label=CaseConfigParamType.refine, + inputType=InputType.Option, + inputHelp="Whether refined data is reserved during index building.", + inputConfig={"options": [True, False]}, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [IndexType.HNSW_SQ.value, IndexType.HNSW_PQ.value, IndexType.HNSW_PRQ.value, IndexType.IVF_RABITQ.value], +) + +CaseConfigParamInput_RefineType = CaseConfigInput( + label=CaseConfigParamType.refine_type, + inputType=InputType.Option, + inputHelp="The data type of the refine index.", + inputConfig={ + "options": [SQType.FP32.value, SQType.FP16.value, SQType.BF16.value, SQType.SQ8.value, SQType.SQ6.value] + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [IndexType.HNSW_SQ.value, IndexType.HNSW_PQ.value, IndexType.HNSW_PRQ.value, IndexType.IVF_RABITQ.value] + and config.get(CaseConfigParamType.refine, True), +) + +CaseConfigParamInput_RefineK = CaseConfigInput( + label=CaseConfigParamType.refine_k, + inputType=InputType.Float, + inputHelp="The magnification factor of refine compared to k.", + inputConfig={"min": 1.0, "max": 10000.0, "value": 1.0}, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [IndexType.HNSW_SQ.value, IndexType.HNSW_PQ.value, IndexType.HNSW_PRQ.value, IndexType.IVF_RABITQ.value] + and config.get(CaseConfigParamType.refine, True), +) + +CaseConfigParamInput_RBQBitsQuery = CaseConfigInput( + label=CaseConfigParamType.rbq_bits_query, + inputType=InputType.Number, + inputHelp="The magnification factor of refine compared to k.", + inputConfig={"min": 0, "max": 8, "value": 0}, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [IndexType.IVF_RABITQ.value], ) CaseConfigParamInput_EFConstruction_Weaviate = CaseConfigInput( @@ -388,12 +812,13 @@ class CaseConfigInput(BaseModel): inputConfig={ "min": 8, "max": 512, - "value": 360, + "value": 128, }, ) CaseConfigParamInput_EFConstruction_AWSOpensearch = CaseConfigInput( label=CaseConfigParamType.EFConstruction, + displayLabel="EF Construction", inputType=InputType.Number, inputConfig={ "min": 100, @@ -404,6 +829,7 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_M_AWSOpensearch = CaseConfigInput( label=CaseConfigParamType.M, + displayLabel="M", inputType=InputType.Number, inputConfig={ "min": 4, @@ -414,9 +840,10 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_EF_SEARCH_AWSOpensearch = CaseConfigInput( label=CaseConfigParamType.ef_search, + displayLabel="EF Search", inputType=InputType.Number, inputConfig={ - "min": 100, + "min": 1, "max": 1024, "value": 256, }, @@ -479,6 +906,48 @@ class CaseConfigInput(BaseModel): isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value, ) +CaseConfigParamInput_ef_search_OceanBase = CaseConfigInput( + label=CaseConfigParamType.ef_search, + displayLabel="ef_search", + inputHelp="HNSW ef_search (session var ob_hnsw_ef_search) for OceanBase", + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 65535, + "value": 100, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [IndexType.HNSW.value, IndexType.HNSW_SQ.value, IndexType.HNSW_BQ.value], +) + +CaseConfigParamInput_sample_per_nlist_OceanBase = CaseConfigInput( + label=CaseConfigParamType.sample_per_nlist, + displayLabel="sample_per_nlist", + inputHelp="OceanBase IVF training sample multiplier (total samples = sample_per_nlist * nlist)", + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 1000000, + "value": 256, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [IndexType.IVFFlat.value, IndexType.IVFSQ8.value, IndexType.IVFPQ.value], +) + +CaseConfigParamInput_ivf_nprobes_OceanBase = CaseConfigInput( + label=CaseConfigParamType.ivf_nprobes, + displayLabel="ivf_nprobes", + inputHelp="OceanBase IVF search probes (session var ob_ivf_nprobes)", + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 65535, + "value": 10, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [IndexType.IVFFlat.value, IndexType.IVFSQ8.value, IndexType.IVFPQ.value], +) + CaseConfigParamInput_EFConstruction_PgVector = CaseConfigInput( label=CaseConfigParamType.ef_construction, inputType=InputType.Number, @@ -490,6 +959,68 @@ class CaseConfigInput(BaseModel): isDisplayed=lambda config: config[CaseConfigParamType.IndexType] == IndexType.HNSW.value, ) +CaseConfigParamInput_IndexType_ES = CaseConfigInput( + label=CaseConfigParamType.index, + inputType=InputType.Option, + inputConfig={ + "options": [ + IndexType.ES_HNSW.value, + IndexType.ES_HNSW_INT8.value, + IndexType.ES_HNSW_INT4.value, + IndexType.ES_HNSW_BBQ.value, + ], + }, +) + +CaseConfigParamInput_NumShards_ES = CaseConfigInput( + label=CaseConfigParamType.number_of_shards, + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 128, + "value": 1, + }, +) + +CaseConfigParamInput_NumReplica_ES = CaseConfigInput( + label=CaseConfigParamType.number_of_replicas, + inputType=InputType.Number, + inputConfig={ + "min": 0, + "max": 10, + "value": 0, + }, +) + +CaseConfigParamInput_RefreshInterval_ES = CaseConfigInput( + label=CaseConfigParamType.refresh_interval, + inputType=InputType.Text, + inputConfig={"value": "30s"}, +) + +CaseConfigParamInput_UseRescore_ES = CaseConfigInput( + label=CaseConfigParamType.use_rescore, + inputType=InputType.Bool, + inputConfig={"value": False}, + isDisplayed=lambda config: config.get(CaseConfigParamType.index, None) != IndexType.ES_HNSW.value, + inputHelp="Recalculating scores using the original (non-quantized) vectors.", +) + +CaseConfigParamInput_OversampleRatio_ES = CaseConfigInput( + label=CaseConfigParamType.oversample_ratio, + inputType=InputType.Float, + inputConfig={"min": 1.0, "max": 100.0, "value": 2.0}, + isDisplayed=lambda config: config.get(CaseConfigParamType.use_rescore, False), + inputHelp="Retrieving more candidates per shard for rescoring.", +) + +CaseConfigParamInput_UseRouting_ES = CaseConfigInput( + label=CaseConfigParamType.use_routing, + inputType=InputType.Bool, + inputConfig={"value": False}, + inputHelp="Using routing to improve label-filter case performance", +) + CaseConfigParamInput_M_ES = CaseConfigInput( label=CaseConfigParamType.M, @@ -497,10 +1028,11 @@ class CaseConfigInput(BaseModel): inputConfig={ "min": 4, "max": 64, - "value": 30, + "value": 16, }, ) + CaseConfigParamInput_NumCandidates_ES = CaseConfigInput( label=CaseConfigParamType.numCandidates, inputType=InputType.Number, @@ -519,7 +1051,13 @@ class CaseConfigInput(BaseModel): "max": MAX_STREAMLIT_INT, "value": 100, }, - isDisplayed=lambda config: config[CaseConfigParamType.IndexType] == IndexType.HNSW.value, + isDisplayed=lambda config: config[CaseConfigParamType.IndexType] + in [ + IndexType.HNSW.value, + IndexType.HNSW_SQ.value, + IndexType.HNSW_PQ.value, + IndexType.HNSW_PRQ.value, + ], ) CaseConfigParamInput_EF_Weaviate = CaseConfigInput( @@ -560,13 +1098,36 @@ class CaseConfigInput(BaseModel): isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [ IndexType.IVFFlat.value, + IndexType.IVFPQ.value, IndexType.IVFSQ8.value, + IndexType.IVF_RABITQ.value, + IndexType.SCANN_MILVUS.value, IndexType.GPU_IVF_FLAT.value, IndexType.GPU_IVF_PQ.value, IndexType.GPU_BRUTE_FORCE.value, ], ) +CaseConfigParamInput_with_raw_data = CaseConfigInput( + label=CaseConfigParamType.with_raw_data, + inputType=InputType.Option, + inputHelp="Whether to include raw data in the index. Setting to True enables reordering with original vectors.", + inputConfig={"options": [False, True]}, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.SCANN_MILVUS.value, +) + +CaseConfigParamInput_reorder_k = CaseConfigInput( + label=CaseConfigParamType.reorder_k, + inputType=InputType.Number, + inputHelp="Number of candidate vectors to reorder. Must be greater than or equal to k.", + inputConfig={ + "min": 1, + "max": 65536, + "value": 100, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.SCANN_MILVUS.value, +) + CaseConfigParamInput_Nprobe = CaseConfigInput( label=CaseConfigParamType.Nprobe, inputType=InputType.Number, @@ -578,7 +1139,10 @@ class CaseConfigInput(BaseModel): isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [ IndexType.IVFFlat.value, + IndexType.IVFPQ.value, IndexType.IVFSQ8.value, + IndexType.IVF_RABITQ.value, + IndexType.SCANN_MILVUS.value, IndexType.GPU_IVF_FLAT.value, IndexType.GPU_IVF_PQ.value, IndexType.GPU_BRUTE_FORCE.value, @@ -589,11 +1153,12 @@ class CaseConfigInput(BaseModel): label=CaseConfigParamType.m, inputType=InputType.Number, inputConfig={ - "min": 0, + "min": 1, "max": 65536, - "value": 0, + "value": 32, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [IndexType.GPU_IVF_PQ.value], + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [IndexType.GPU_IVF_PQ.value, IndexType.HNSW_PQ.value, IndexType.HNSW_PRQ.value, IndexType.IVFPQ.value], ) @@ -605,7 +1170,20 @@ class CaseConfigInput(BaseModel): "max": 65536, "value": 8, }, - isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [IndexType.GPU_IVF_PQ.value], + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) + in [IndexType.GPU_IVF_PQ.value, IndexType.HNSW_PQ.value, IndexType.HNSW_PRQ.value, IndexType.IVFPQ.value], +) + +CaseConfigParamInput_NRQ = CaseConfigInput( + label=CaseConfigParamType.nrq, + inputType=InputType.Number, + inputHelp="The number of residual subquantizers.", + inputConfig={ + "min": 1, + "max": 16, + "value": 2, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) in [IndexType.HNSW_PRQ.value], ) CaseConfigParamInput_intermediate_graph_degree = CaseConfigInput( @@ -853,7 +1431,7 @@ class CaseConfigInput(BaseModel): inputType=InputType.Number, inputConfig={ "min": 1, - "max": 3, + "max": 10, "value": 1, }, ) @@ -1058,6 +1636,102 @@ class CaseConfigInput(BaseModel): }, ) +CaseConfigParamInput_IndexType_TES = CaseConfigInput( + label=CaseConfigParamType.IndexType, + inputHelp="hnsw or vsearch", + inputType=InputType.Text, + inputConfig={ + "value": "hnsw", + }, +) + +CaseConfigParamInput_EFConstruction_TES = CaseConfigInput( + label=CaseConfigParamType.EFConstruction, + inputType=InputType.Number, + inputConfig={ + "min": 8, + "max": 512, + "value": 360, + }, +) + +CaseConfigParamInput_M_TES = CaseConfigInput( + label=CaseConfigParamType.M, + inputType=InputType.Number, + inputConfig={ + "min": 4, + "max": 64, + "value": 30, + }, +) +CaseConfigParamInput_NumCandidates_TES = CaseConfigInput( + label=CaseConfigParamType.numCandidates, + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 10000, + "value": 100, + }, +) + +# CockroachDB configs +CaseConfigParamInput_IndexType_CockroachDB = CaseConfigInput( + label=CaseConfigParamType.IndexType, + inputHelp="Select Index Type (all use C-SPANN vector index)", + inputType=InputType.Option, + inputConfig={ + "options": [ + IndexType.Flat.value, + IndexType.HNSW.value, + IndexType.IVFFlat.value, + ], + }, +) + +CaseConfigParamInput_MinPartitionSize_CockroachDB = CaseConfigInput( + label=CaseConfigParamType.min_partition_size, + inputHelp="Minimum partition size (1-1024)", + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 1024, + "value": 16, + }, +) + +CaseConfigParamInput_MaxPartitionSize_CockroachDB = CaseConfigInput( + label=CaseConfigParamType.max_partition_size, + inputHelp="Maximum partition size", + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 10000, + "value": 128, + }, +) + +CaseConfigParamInput_BuildBeamSize_CockroachDB = CaseConfigInput( + label=CaseConfigParamType.build_beam_size, + inputHelp="Build beam size for index creation", + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 100, + "value": 8, + }, +) + +CaseConfigParamInput_VectorSearchBeamSize_CockroachDB = CaseConfigInput( + label=CaseConfigParamType.vector_search_beam_size, + inputHelp="Vector search beam size", + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 1000, + "value": 32, + }, +) + CaseConfigParamInput_IndexType_MariaDB = CaseConfigInput( label=CaseConfigParamType.IndexType, inputHelp="Select Index Type", @@ -1078,6 +1752,41 @@ class CaseConfigInput(BaseModel): }, ) +CaseConfigParamInput_M_AliSQL = CaseConfigInput( + label=CaseConfigParamType.M, + inputHelp="M parameter in HNSW vector indexing", + inputType=InputType.Number, + inputConfig={ + "min": 3, + "max": 200, + "value": 6, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value, +) + +CaseConfigParamInput_EFSearch_AliSQL = CaseConfigInput( + label=CaseConfigParamType.ef_search, + inputHelp="vidx_hnsw_ef_search", + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 10000, + "value": 20, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value, +) + +CaseConfigParamInput_IndexType_AliSQL = CaseConfigInput( + label=CaseConfigParamType.IndexType, + inputHelp="Select Index Type", + inputType=InputType.Option, + inputConfig={ + "options": [ + IndexType.HNSW.value, + ], + }, +) + CaseConfigParamInput_M_MariaDB = CaseConfigInput( label=CaseConfigParamType.M, inputHelp="M parameter in MHNSW vector indexing", @@ -1113,6 +1822,13 @@ class CaseConfigInput(BaseModel): }, isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value, ) +CaseConfigParamInput_Milvus_use_partition_key = CaseConfigInput( + label=CaseConfigParamType.use_partition_key, + inputType=InputType.Option, + inputHelp="whether to use partition_key for label-filter cases. only works in label-filter cases", + inputConfig={"options": [False, True]}, +) + CaseConfigParamInput_MongoDBQuantizationType = CaseConfigInput( label=CaseConfigParamType.mongodb_quantization_type, @@ -1174,18 +1890,252 @@ class CaseConfigInput(BaseModel): isDisplayed=lambda config: config[CaseConfigParamType.IndexType] == IndexType.HNSW.value, ) +CaseConfigParamInput_INDEX_THREAD_QTY_DURING_FORCE_MERGE_AWSOpensearch = CaseConfigInput( + label=CaseConfigParamType.index_thread_qty_during_force_merge, + displayLabel="Index Thread Qty During Force Merge", + inputHelp="Thread count during force merge operations", + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 32, + "value": 4, + }, +) + +CaseConfigParamInput_NUMBER_OF_INDEXING_CLIENTS_AWSOpensearch = CaseConfigInput( + label=CaseConfigParamType.number_of_indexing_clients, + displayLabel="Number of Indexing Clients", + inputHelp="Number of concurrent clients for data insertion", + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 32, + "value": 1, + }, +) + +CaseConfigParamInput_NUMBER_OF_SHARDS_AWSOpensearch = CaseConfigInput( + label=CaseConfigParamType.number_of_shards, + displayLabel="Number of Shards", + inputHelp="Number of primary shards for the index", + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 32, + "value": 1, + }, +) + +CaseConfigParamInput_NUMBER_OF_REPLICAS_AWSOpensearch = CaseConfigInput( + label=CaseConfigParamType.number_of_replicas, + displayLabel="Number of Replicas", + inputHelp="Number of replica copies for each primary shard", + inputType=InputType.Number, + inputConfig={ + "min": 0, + "max": 10, + "value": 1, + }, +) + +CaseConfigParamInput_INDEX_THREAD_QTY_AWSOpensearch = CaseConfigInput( + label=CaseConfigParamType.index_thread_qty, + displayLabel="Index Thread Qty", + inputHelp="Thread count for native engine indexing", + inputType=InputType.Number, + inputConfig={ + "min": 1, + "max": 32, + "value": 4, + }, +) + +CaseConfigParamInput_ENGINE_NAME_AWSOpensearch = CaseConfigInput( + label=CaseConfigParamType.engine_name, + displayLabel="Engine", + inputHelp="HNSW algorithm implementation to use", + inputType=InputType.Option, + inputConfig={ + "options": ["faiss", "nmslib", "lucene"], + "default": "faiss", + }, +) + +CaseConfigParamInput_METRIC_TYPE_NAME_AWSOpensearch = CaseConfigInput( + label=CaseConfigParamType.metric_type_name, + displayLabel="Metric Type", + inputHelp="Distance metric type for vector similarity", + inputType=InputType.Option, + inputConfig={ + "options": ["l2", "cosine", "ip"], + "default": "l2", + }, +) + +CaseConfigParamInput_REFRESH_INTERVAL_AWSOpensearch = CaseConfigInput( + label=CaseConfigParamType.refresh_interval, + displayLabel="Refresh Interval", + inputHelp="How often to make new data searchable. (e.g., 30s, 1m).", + inputType=InputType.Text, + inputConfig={"value": "60s", "placeholder": "e.g. 30s, 1m"}, +) + +CaseConfigParamInput_REPLICATION_TYPE_AWSOpensearch = CaseConfigInput( + label=CaseConfigParamType.replication_type, + displayLabel="Replication Type", + inputHelp="Replication strategy: DOCUMENT (default) or SEGMENT", + inputType=InputType.Option, + inputConfig={ + "options": ["DOCUMENT", "SEGMENT"], + "default": "DOCUMENT", + }, +) + +CaseConfigParamInput_KNN_DERIVED_SOURCE_ENABLED_AWSOpensearch = CaseConfigInput( + label=CaseConfigParamType.knn_derived_source_enabled, + displayLabel="KNN Derived Source Enabled", + inputHelp="Enable KNN derived source (OpenSearch 3.x+ only). Ignored for 2.x versions.", + inputType=InputType.Bool, + inputConfig={ + "value": False, + }, +) + +CaseConfigParamInput_MEMORY_OPTIMIZED_SEARCH_AWSOpensearch = CaseConfigInput( + label=CaseConfigParamType.memory_optimized_search, + displayLabel="Memory Optimized Search", + inputHelp="Enable memory-optimized search (OpenSearch 3.1+ and FAISS engine only).", + inputType=InputType.Bool, + inputConfig={ + "value": False, + }, + isDisplayed=lambda config: (config.get(CaseConfigParamType.engine_name, "").lower() == "faiss"), +) + +CaseConfigParamInput_ON_DISK_OSSOpensearch = CaseConfigInput( + label=CaseConfigParamType.on_disk, + displayLabel="Disk-based Search", + inputHelp="Enable disk-based storage with Binary Quantization", + inputType=InputType.Bool, + inputConfig={ + "value": False, + }, +) + +CaseConfigParamInput_COMPRESSION_LEVEL_OSSOpensearch = CaseConfigInput( + label=CaseConfigParamType.compression_level, + displayLabel="Compression Level", + inputHelp="Binary quantization compression ratio for disk storage", + inputType=InputType.Option, + inputConfig={ + "options": ["32x", "16x", "8x", "4x", "2x", "1x"], + "default": "32x", + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.on_disk, False) == True, +) + +CaseConfigParamInput_OVERSAMPLE_FACTOR_OSSOpensearch = CaseConfigInput( + label=CaseConfigParamType.oversample_factor, + displayLabel="Oversample Factor", + inputHelp="Rescoring oversample factor for two-phase search", + inputType=InputType.Float, + inputConfig={ + "min": 1.0, + "max": 10.0, + "value": 3.0, + "step": 0.5, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.on_disk, False) == True, +) + +CaseConfigParamInput_ENGINE_NAME_OSSOpensearch = CaseConfigInput( + label=CaseConfigParamType.engine_name, + displayLabel="Engine", + inputHelp="HNSW algorithm implementation to use", + inputType=InputType.Option, + inputConfig={ + "options": ["faiss", "lucene"], + "default": "faiss", + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.on_disk, False) == False, +) + +CaseConfigParamInput_QUANTIZATION_TYPE_LUCENE_OSSOpensearch = CaseConfigInput( + label=CaseConfigParamType.quantizationType, + displayLabel="Quantization Type", + inputHelp="Scalar quantization for Lucene engine", + inputType=InputType.Option, + inputConfig={ + "options": ["None", "LuceneSQ"], + "default": "None", + }, + isDisplayed=lambda config: ( + not config.get(CaseConfigParamType.on_disk, False) and config.get(CaseConfigParamType.engine_name) == "lucene" + ), +) + +CaseConfigParamInput_QUANTIZATION_TYPE_FAISS_OSSOpensearch = CaseConfigInput( + label=CaseConfigParamType.quantizationType, + displayLabel="Quantization Type", + inputHelp="Scalar quantization for FAISS engine", + inputType=InputType.Option, + inputConfig={ + "options": ["None", "FaissSQfp16"], + "default": "None", + }, + isDisplayed=lambda config: ( + not config.get(CaseConfigParamType.on_disk, False) and config.get(CaseConfigParamType.engine_name) == "faiss" + ), +) + +CaseConfigParamInput_CONFIDENCE_INTERVAL_OSSOpensearch = CaseConfigInput( + label=CaseConfigParamType.confidence_interval, + displayLabel="Confidence Interval", + inputHelp="Quantile range for Lucene SQ (0.9-1.0, 0 for dynamic, or empty for auto)", + inputType=InputType.Float, + inputConfig={ + "min": 0.0, + "max": 1.0, + "value": None, + "step": 0.1, + }, + isDisplayed=lambda config: ( + not config.get(CaseConfigParamType.on_disk, False) + and config.get(CaseConfigParamType.quantizationType) == "LuceneSQ" + ), +) + +CaseConfigParamInput_CLIP_OSSOpensearch = CaseConfigInput( + label=CaseConfigParamType.clip, + displayLabel="Clip Vectors", + inputHelp="Clip out-of-range values to [-65504, 65504] for FP16", + inputType=InputType.Bool, + inputConfig={ + "value": False, + }, + isDisplayed=lambda config: ( + not config.get(CaseConfigParamType.on_disk, False) + and config.get(CaseConfigParamType.quantizationType) == "FaissSQfp16" + ), +) MilvusLoadConfig = [ CaseConfigParamInput_IndexType, CaseConfigParamInput_M, CaseConfigParamInput_EFConstruction_Milvus, CaseConfigParamInput_Nlist, + CaseConfigParamInput_with_raw_data, CaseConfigParamInput_M_PQ, CaseConfigParamInput_Nbits_PQ, CaseConfigParamInput_intermediate_graph_degree, CaseConfigParamInput_graph_degree, CaseConfigParamInput_build_algo, CaseConfigParamInput_cache_dataset_on_device, + CaseConfigParamInput_SQType, + CaseConfigParamInput_Refine, + CaseConfigParamInput_RefineType, + CaseConfigParamInput_NRQ, + CaseConfigParamInput_Milvus_use_partition_key, ] MilvusPerformanceConfig = [ CaseConfigParamInput_IndexType, @@ -1194,9 +2144,13 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_EF_Milvus, CaseConfigParamInput_SearchList, CaseConfigParamInput_Nlist, + CaseConfigParamInput_with_raw_data, + CaseConfigParamInput_reorder_k, CaseConfigParamInput_Nprobe, CaseConfigParamInput_M_PQ, CaseConfigParamInput_Nbits_PQ, + CaseConfigParamInput_RBQBitsQuery, + CaseConfigParamInput_NRQ, CaseConfigParamInput_intermediate_graph_degree, CaseConfigParamInput_graph_degree, CaseConfigParamInput_itopk_size, @@ -1207,6 +2161,11 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_build_algo, CaseConfigParamInput_cache_dataset_on_device, CaseConfigParamInput_refine_ratio, + CaseConfigParamInput_SQType, + CaseConfigParamInput_Refine, + CaseConfigParamInput_RefineType, + CaseConfigParamInput_RefineK, + CaseConfigParamInput_Milvus_use_partition_key, ] WeaviateLoadConfig = [ @@ -1219,21 +2178,39 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_EF_Weaviate, ] -ESLoadingConfig = [CaseConfigParamInput_EFConstruction_ES, CaseConfigParamInput_M_ES] +ESLoadingConfig = [ + CaseConfigParamInput_IndexType_ES, + CaseConfigParamInput_NumShards_ES, + CaseConfigParamInput_NumReplica_ES, + CaseConfigParamInput_RefreshInterval_ES, + CaseConfigParamInput_EFConstruction_ES, + CaseConfigParamInput_M_ES, +] ESPerformanceConfig = [ + CaseConfigParamInput_IndexType_ES, + CaseConfigParamInput_NumShards_ES, + CaseConfigParamInput_NumReplica_ES, + CaseConfigParamInput_RefreshInterval_ES, CaseConfigParamInput_EFConstruction_ES, CaseConfigParamInput_M_ES, CaseConfigParamInput_NumCandidates_ES, + CaseConfigParamInput_UseRescore_ES, + CaseConfigParamInput_OversampleRatio_ES, + CaseConfigParamInput_UseRouting_ES, ] AWSOpensearchLoadingConfig = [ CaseConfigParamInput_EFConstruction_AWSOpensearch, CaseConfigParamInput_M_AWSOpensearch, + CaseConfigParamInput_REPLICATION_TYPE_AWSOpensearch, + CaseConfigParamInput_KNN_DERIVED_SOURCE_ENABLED_AWSOpensearch, ] AWSOpenSearchPerformanceConfig = [ CaseConfigParamInput_EFConstruction_AWSOpensearch, CaseConfigParamInput_M_AWSOpensearch, CaseConfigParamInput_EF_SEARCH_AWSOpensearch, + CaseConfigParamInput_REPLICATION_TYPE_AWSOpensearch, + CaseConfigParamInput_KNN_DERIVED_SOURCE_ENABLED_AWSOpensearch, ] AliyunOpensearchLoadingConfig = [] @@ -1313,15 +2290,21 @@ class CaseConfigInput(BaseModel): PgDiskANNLoadConfig = [ CaseConfigParamInput_IndexType_PgDiskANN, - CaseConfigParamInput_max_neighbors, + CaseConfigParamInput_max_neighbors_PgDiskANN, CaseConfigParamInput_l_value_ib, ] PgDiskANNPerformanceConfig = [ CaseConfigParamInput_IndexType_PgDiskANN, - CaseConfigParamInput_max_neighbors, + CaseConfigParamInput_reranking_PgDiskANN, + CaseConfigParamInput_max_neighbors_PgDiskANN, CaseConfigParamInput_l_value_ib, CaseConfigParamInput_l_value_is, + CaseConfigParamInput_maintenance_work_mem_PgDiskANN, + CaseConfigParamInput_max_parallel_workers_PgDiskANN, + CaseConfigParamInput_pq_param_num_chunks_PgDiskANN, + CaseConfigParamInput_quantized_fetch_limit_PgDiskANN, + CaseConfigParamInput_reranking_metric_PgDiskANN, ] @@ -1351,13 +2334,35 @@ class CaseConfigInput(BaseModel): ] AliyunElasticsearchLoadingConfig = [ + CaseConfigParamInput_IndexType_ES, + CaseConfigParamInput_NumShards_ES, + CaseConfigParamInput_NumReplica_ES, + CaseConfigParamInput_RefreshInterval_ES, CaseConfigParamInput_EFConstruction_AliES, CaseConfigParamInput_M_AliES, ] AliyunElasticsearchPerformanceConfig = [ + CaseConfigParamInput_IndexType_ES, + CaseConfigParamInput_NumShards_ES, + CaseConfigParamInput_NumReplica_ES, + CaseConfigParamInput_RefreshInterval_ES, CaseConfigParamInput_EFConstruction_AliES, CaseConfigParamInput_M_AliES, CaseConfigParamInput_NumCandidates_AliES, + CaseConfigParamInput_UseRescore_ES, + CaseConfigParamInput_OversampleRatio_ES, + CaseConfigParamInput_UseRouting_ES, +] + +TencentElasticsearchLoadingConfig = [ + CaseConfigParamInput_EFConstruction_TES, + CaseConfigParamInput_M_TES, + CaseConfigParamInput_IndexType_TES, +] +TencentElasticsearchPerformanceConfig = [ + CaseConfigParamInput_EFConstruction_TES, + CaseConfigParamInput_M_TES, + CaseConfigParamInput_NumCandidates_TES, ] MongoDBLoadingConfig = [ @@ -1368,6 +2373,34 @@ class CaseConfigInput(BaseModel): CaseConfigParamInput_MongoDBNumCandidatesRatio, ] +CockroachDBLoadingConfig = [ + CaseConfigParamInput_IndexType_CockroachDB, + CaseConfigParamInput_MinPartitionSize_CockroachDB, + CaseConfigParamInput_MaxPartitionSize_CockroachDB, + CaseConfigParamInput_BuildBeamSize_CockroachDB, + CaseConfigParamInput_VectorSearchBeamSize_CockroachDB, +] +CockroachDBPerformanceConfig = [ + CaseConfigParamInput_IndexType_CockroachDB, + CaseConfigParamInput_MinPartitionSize_CockroachDB, + CaseConfigParamInput_MaxPartitionSize_CockroachDB, + CaseConfigParamInput_BuildBeamSize_CockroachDB, + CaseConfigParamInput_VectorSearchBeamSize_CockroachDB, +] + +OceanBaseLoadConfig = [ + CaseConfigParamInput_IndexType_OceanBase, + CaseConfigParamInput_m_OceanBase, + CaseConfigParamInput_EFConstruction_OceanBase, + CaseConfigParamInput_ef_search_OceanBase, + CaseConfigParamInput_Nlist, + CaseConfigParamInput_sample_per_nlist_OceanBase, + CaseConfigParamInput_Nbits_PQ, + CaseConfigParamInput_M_PQ, + CaseConfigParamInput_ivf_nprobes_OceanBase, +] +OceanBasePerformanceConfig = OceanBaseLoadConfig + MariaDBLoadingConfig = [ CaseConfigParamInput_IndexType_MariaDB, CaseConfigParamInput_StorageEngine_MariaDB, @@ -1391,10 +2424,221 @@ class CaseConfigInput(BaseModel): ] VespaPerformanceConfig = VespaLoadingConfig +AliSQLLoadingConfig = [ + CaseConfigParamInput_IndexType_AliSQL, + CaseConfigParamInput_M_AliSQL, +] +AliSQLPerformanceConfig = [ + CaseConfigParamInput_IndexType_AliSQL, + CaseConfigParamInput_M_AliSQL, + CaseConfigParamInput_EFSearch_AliSQL, +] + +CaseConfigParamInput_IndexType_LanceDB = CaseConfigInput( + label=CaseConfigParamType.IndexType, + inputHelp="AUTOINDEX = IVFPQ with default parameters", + inputType=InputType.Option, + inputConfig={ + "options": [ + IndexType.NONE.value, + IndexType.AUTOINDEX.value, + IndexType.IVFPQ.value, + IndexType.HNSW.value, + ], + }, +) + +CaseConfigParamInput_num_partitions_LanceDB = CaseConfigInput( + label=CaseConfigParamType.num_partitions, + displayLabel="Number of Partitions", + inputHelp="Number of partitions (clusters) for IVF_PQ. Default (when 0): sqrt(num_rows)", + inputType=InputType.Number, + inputConfig={ + "min": 0, + "max": 10000, + "value": 0, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.IVFPQ.value + or config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value, +) + +CaseConfigParamInput_num_sub_vectors_LanceDB = CaseConfigInput( + label=CaseConfigParamType.num_sub_vectors, + displayLabel="Number of Sub-vectors", + inputHelp="Number of sub-vectors for PQ. Default (when 0): dim/16 or dim/8", + inputType=InputType.Number, + inputConfig={ + "min": 0, + "max": 1000, + "value": 0, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.IVFPQ.value + or config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value, +) + +CaseConfigParamInput_num_bits_LanceDB = CaseConfigInput( + label=CaseConfigParamType.nbits, + displayLabel="Number of Bits", + inputHelp="Number of bits per sub-vector.", + inputType=InputType.Option, + inputConfig={ + "options": [4, 8], + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.IVFPQ.value + or config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value, +) + +CaseConfigParamInput_sample_rate_LanceDB = CaseConfigInput( + label=CaseConfigParamType.sample_rate, + displayLabel="Sample Rate", + inputHelp="Sample rate for training. Higher values are more accurate but slower", + inputType=InputType.Number, + inputConfig={ + "min": 16, + "max": 1024, + "value": 256, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.IVFPQ.value + or config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value, +) + +CaseConfigParamInput_max_iterations_LanceDB = CaseConfigInput( + label=CaseConfigParamType.max_iterations, + displayLabel="Max Iterations", + inputHelp="Maximum iterations for k-means clustering", + inputType=InputType.Number, + inputConfig={ + "min": 10, + "max": 200, + "value": 50, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.IVFPQ.value + or config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value, +) + +CaseConfigParamInput_m_LanceDB = CaseConfigInput( + label=CaseConfigParamType.m, + displayLabel="m", + inputHelp="m parameter in HNSW", + inputType=InputType.Number, + inputConfig={ + "min": 0, + "max": 1000, + "value": 0, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value, +) + +CaseConfigParamInput_ef_construction_LanceDB = CaseConfigInput( + label=CaseConfigParamType.ef_construction, + displayLabel="ef_construction", + inputHelp="ef_construction parameter in HNSW", + inputType=InputType.Number, + inputConfig={ + "min": 0, + "max": 1000, + "value": 0, + }, + isDisplayed=lambda config: config.get(CaseConfigParamType.IndexType, None) == IndexType.HNSW.value, +) + +LanceDBLoadConfig = [ + CaseConfigParamInput_IndexType_LanceDB, + CaseConfigParamInput_num_partitions_LanceDB, + CaseConfigParamInput_num_sub_vectors_LanceDB, + CaseConfigParamInput_num_bits_LanceDB, + CaseConfigParamInput_sample_rate_LanceDB, + CaseConfigParamInput_max_iterations_LanceDB, + CaseConfigParamInput_m_LanceDB, + CaseConfigParamInput_ef_construction_LanceDB, +] + +LanceDBPerformanceConfig = LanceDBLoadConfig + +AWSOpensearchLoadingConfig = [ + CaseConfigParamInput_REFRESH_INTERVAL_AWSOpensearch, + CaseConfigParamInput_ENGINE_NAME_AWSOpensearch, + CaseConfigParamInput_METRIC_TYPE_NAME_AWSOpensearch, + CaseConfigParamInput_M_AWSOpensearch, + CaseConfigParamInput_EFConstruction_AWSOpensearch, + CaseConfigParamInput_NUMBER_OF_SHARDS_AWSOpensearch, + CaseConfigParamInput_NUMBER_OF_REPLICAS_AWSOpensearch, + CaseConfigParamInput_KNN_DERIVED_SOURCE_ENABLED_AWSOpensearch, + CaseConfigParamInput_NUMBER_OF_INDEXING_CLIENTS_AWSOpensearch, + CaseConfigParamInput_INDEX_THREAD_QTY_AWSOpensearch, + CaseConfigParamInput_REPLICATION_TYPE_AWSOpensearch, + CaseConfigParamInput_MEMORY_OPTIMIZED_SEARCH_AWSOpensearch, + CaseConfigParamInput_INDEX_THREAD_QTY_DURING_FORCE_MERGE_AWSOpensearch, +] + +AWSOpenSearchPerformanceConfig = [ + CaseConfigParamInput_REFRESH_INTERVAL_AWSOpensearch, + CaseConfigParamInput_EF_SEARCH_AWSOpensearch, + CaseConfigParamInput_ENGINE_NAME_AWSOpensearch, + CaseConfigParamInput_METRIC_TYPE_NAME_AWSOpensearch, + CaseConfigParamInput_M_AWSOpensearch, + CaseConfigParamInput_EFConstruction_AWSOpensearch, + CaseConfigParamInput_NUMBER_OF_SHARDS_AWSOpensearch, + CaseConfigParamInput_NUMBER_OF_REPLICAS_AWSOpensearch, + CaseConfigParamInput_KNN_DERIVED_SOURCE_ENABLED_AWSOpensearch, + CaseConfigParamInput_NUMBER_OF_INDEXING_CLIENTS_AWSOpensearch, + CaseConfigParamInput_INDEX_THREAD_QTY_AWSOpensearch, + CaseConfigParamInput_REPLICATION_TYPE_AWSOpensearch, + CaseConfigParamInput_MEMORY_OPTIMIZED_SEARCH_AWSOpensearch, + CaseConfigParamInput_INDEX_THREAD_QTY_DURING_FORCE_MERGE_AWSOpensearch, +] + + +OSSOpensearchLoadingConfig = [ + CaseConfigParamInput_ON_DISK_OSSOpensearch, + CaseConfigParamInput_COMPRESSION_LEVEL_OSSOpensearch, + CaseConfigParamInput_ENGINE_NAME_OSSOpensearch, + CaseConfigParamInput_METRIC_TYPE_NAME_AWSOpensearch, + CaseConfigParamInput_M_AWSOpensearch, + CaseConfigParamInput_EFConstruction_AWSOpensearch, + CaseConfigParamInput_QUANTIZATION_TYPE_LUCENE_OSSOpensearch, + CaseConfigParamInput_QUANTIZATION_TYPE_FAISS_OSSOpensearch, + CaseConfigParamInput_CONFIDENCE_INTERVAL_OSSOpensearch, + CaseConfigParamInput_CLIP_OSSOpensearch, + CaseConfigParamInput_REFRESH_INTERVAL_AWSOpensearch, + CaseConfigParamInput_NUMBER_OF_SHARDS_AWSOpensearch, + CaseConfigParamInput_NUMBER_OF_REPLICAS_AWSOpensearch, + CaseConfigParamInput_NUMBER_OF_INDEXING_CLIENTS_AWSOpensearch, + CaseConfigParamInput_INDEX_THREAD_QTY_AWSOpensearch, + CaseConfigParamInput_REPLICATION_TYPE_AWSOpensearch, + CaseConfigParamInput_KNN_DERIVED_SOURCE_ENABLED_AWSOpensearch, + CaseConfigParamInput_MEMORY_OPTIMIZED_SEARCH_AWSOpensearch, +] + +OSSOpenSearchPerformanceConfig = [ + CaseConfigParamInput_ON_DISK_OSSOpensearch, + CaseConfigParamInput_COMPRESSION_LEVEL_OSSOpensearch, + CaseConfigParamInput_OVERSAMPLE_FACTOR_OSSOpensearch, + CaseConfigParamInput_EF_SEARCH_AWSOpensearch, + CaseConfigParamInput_ENGINE_NAME_OSSOpensearch, + CaseConfigParamInput_METRIC_TYPE_NAME_AWSOpensearch, + CaseConfigParamInput_M_AWSOpensearch, + CaseConfigParamInput_EFConstruction_AWSOpensearch, + CaseConfigParamInput_QUANTIZATION_TYPE_LUCENE_OSSOpensearch, + CaseConfigParamInput_QUANTIZATION_TYPE_FAISS_OSSOpensearch, + CaseConfigParamInput_CONFIDENCE_INTERVAL_OSSOpensearch, + CaseConfigParamInput_CLIP_OSSOpensearch, + CaseConfigParamInput_REFRESH_INTERVAL_AWSOpensearch, + CaseConfigParamInput_NUMBER_OF_SHARDS_AWSOpensearch, + CaseConfigParamInput_NUMBER_OF_REPLICAS_AWSOpensearch, + CaseConfigParamInput_NUMBER_OF_INDEXING_CLIENTS_AWSOpensearch, + CaseConfigParamInput_INDEX_THREAD_QTY_AWSOpensearch, + CaseConfigParamInput_REPLICATION_TYPE_AWSOpensearch, + CaseConfigParamInput_KNN_DERIVED_SOURCE_ENABLED_AWSOpensearch, + CaseConfigParamInput_MEMORY_OPTIMIZED_SEARCH_AWSOpensearch, +] + +# Map DB to config CASE_CONFIG_MAP = { DB.Milvus: { CaseLabel.Load: MilvusLoadConfig, CaseLabel.Performance: MilvusPerformanceConfig, + CaseLabel.Streaming: MilvusPerformanceConfig, }, DB.ZillizCloud: { CaseLabel.Performance: ZillizCloudPerformanceConfig, @@ -1411,6 +2655,10 @@ class CaseConfigInput(BaseModel): CaseLabel.Load: AWSOpensearchLoadingConfig, CaseLabel.Performance: AWSOpenSearchPerformanceConfig, }, + DB.OSSOpenSearch: { + CaseLabel.Load: OSSOpensearchLoadingConfig, + CaseLabel.Performance: OSSOpenSearchPerformanceConfig, + }, DB.PgVector: { CaseLabel.Load: PgVectorLoadingConfig, CaseLabel.Performance: PgVectorPerformanceConfig, @@ -1451,4 +2699,34 @@ class CaseConfigInput(BaseModel): CaseLabel.Load: VespaLoadingConfig, CaseLabel.Performance: VespaPerformanceConfig, }, + DB.LanceDB: { + CaseLabel.Load: LanceDBLoadConfig, + CaseLabel.Performance: LanceDBPerformanceConfig, + }, + DB.TencentElasticsearch: { + CaseLabel.Load: TencentElasticsearchLoadingConfig, + CaseLabel.Performance: TencentElasticsearchPerformanceConfig, + }, + DB.AliSQL: { + CaseLabel.Load: AliSQLLoadingConfig, + CaseLabel.Performance: AliSQLPerformanceConfig, + }, + DB.CockroachDB: { + CaseLabel.Load: CockroachDBLoadingConfig, + CaseLabel.Performance: CockroachDBPerformanceConfig, + }, + DB.OceanBase: { + CaseLabel.Load: OceanBaseLoadConfig, + CaseLabel.Performance: OceanBasePerformanceConfig, + }, } + + +def get_case_config_inputs(db: DB, case_label: CaseLabel) -> list[CaseConfigInput]: + if db not in CASE_CONFIG_MAP: + return [] + if case_label == CaseLabel.Load: + return CASE_CONFIG_MAP[db][CaseLabel.Load] + elif case_label == CaseLabel.Performance or case_label == CaseLabel.Streaming: + return CASE_CONFIG_MAP[db][CaseLabel.Performance] + return [] diff --git a/vectordb_bench/frontend/config/styles.py b/vectordb_bench/frontend/config/styles.py index 03bda0fec..21009b267 100644 --- a/vectordb_bench/frontend/config/styles.py +++ b/vectordb_bench/frontend/config/styles.py @@ -3,7 +3,8 @@ # style const DB_SELECTOR_COLUMNS = 6 DB_CONFIG_SETTING_COLUMNS = 3 -CASE_CONFIG_SETTING_COLUMNS = 4 +CASE_CONFIG_SETTING_COLUMNS = 2 +DB_CASE_CONFIG_SETTING_COLUMNS = 4 CHECKBOX_INDENT = 30 TASK_LABEL_INPUT_COLUMNS = 2 CHECKBOX_MAX_COLUMNS = 4 @@ -16,6 +17,9 @@ LEGEND_RECT_WIDTH = 24 LEGEND_RECT_HEIGHT = 16 LEGEND_TEXT_FONT_SIZE = 14 +STREAMING_CHART_COLUMNS = 2 +SCATTER_MAKER_SIZE = 8 +SCATTER_LINE_WIDTH = 2 PATTERN_SHAPES = ["", "+", "\\", "x", ".", "|", "/", "-"] @@ -29,8 +33,8 @@ def getPatternShape(i): MAX_AUTO_REFRESH_INTERVAL = 5000 # 5s PAGE_TITLE = "VectorDB Benchmark" -FAVICON = "https://assets.zilliz.com/favicon_f7f922fe27.png" -HEADER_ICON = "https://assets.zilliz.com/vdb_benchmark_db790b5387.png" +FAVICON = "https://assets.zilliz.com/VDB_Bench_icon_d3276bedc4.png" +HEADER_ICON = "https://assets.zilliz.com/VDB_Bench_text_icon_6c5f52a458.png" # RedisCloud icon: https://assets.zilliz.com/Redis_Cloud_74b8bfef39.png # Elasticsearch icon: https://assets.zilliz.com/elasticsearch_beffeadc29.png @@ -41,14 +45,32 @@ def getPatternShape(i): DB.ElasticCloud: "https://assets.zilliz.com/Elatic_Cloud_dad8d6a3a3.png", DB.Pinecone: "https://assets.zilliz.com/pinecone_94d8154979.png", DB.QdrantCloud: "https://assets.zilliz.com/qdrant_b691674fcd.png", + DB.QdrantLocal: "https://assets.zilliz.com/qdrant_b691674fcd.png", DB.WeaviateCloud: "https://assets.zilliz.com/weaviate_4f6f171ebe.png", DB.PgVector: "https://assets.zilliz.com/PG_Vector_d464f2ef5f.png", DB.PgVectoRS: "https://assets.zilliz.com/PG_Vector_d464f2ef5f.png", + DB.PgVectorScale: "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wCEAAkGBxAQEhUQEBAWEBAQEBUXEBIVFxgRERASGhUYFhkXGBgaHiggHR0nHRYVLTEhJSkrLjouFx8/OTMsNyg5MS0BCgoKDg0OGBAQGC0eHx8wLS0tLS03KzEtNysrLS0tLSs3Ky03LSstLS0tKyswNi0rNy0tMC4tLSstLS0tLS0tLf/AABEIAMgAyAMBIgACEQEDEQH/xAAcAAEAAgIDAQAAAAAAAAAAAAAABwgBBQMEBgL/xABFEAABAwICBgUJAg0EAwAAAAABAAIDBBEFIQYHEjFBUSIyYXGBExQjQlJikaGxCMEVJDM0Q3J0gpKTorLCFlVj0VNz0v/EABoBAQACAwEAAAAAAAAAAAAAAAACBQEDBgT/xAAqEQACAgIBAwIGAgMAAAAAAAAAAQIDBBExBRIhE0EiMlFSYfCBkTShsf/aAAwDAQACEQMRAD8Ag1ERAEREAREQBERAEREAREQBERAEREAREQBERAEREAREQBERAEREAREQBERAEREAWVutHtF62vds0sDpBexfbZjb3vOQUn4DqN3Oraq3OOAf5u/6Wi3Jrq+ZjRCqyrR4XqzwinGVI2V3tSkyk+By+S9FS4PTRC0VPFGPcja36BeKXVa18sdku0p3snksWVz/ACTfZHwXXqMLp5BaSnikHJzGu+oUF1aP2/7HaU4WFajE9XGE1A6VGxh9qK8JH8OS8Pjuo1hBdRVRaeEcw2m/xt3fAr0V9Spl4fgx2kIIvQaR6G1+Hk+cwOay+UrenEf3hl8V59e6MlJbT2YMIiLICIiAIiIAiIgCIiAIiIAiytjgWDz1szaenZtyPPg0cXOPADmjaS2wdWio5J3tiiY6SR5s1jRtOcewKa9B9TjGBs2J9N+8U7T0G/8AscOt3DLvXstAtBKfC48gJapw9LORn+qz2W/XivXqjyuot/DX4X1JKJwUtNHE0RxMbGxos1jQGtaOwBc6Ljlka0FziGtaCXOOQaBmSTyVX5kyR9oo7w3W3QTVjqU3jivsw1Dj0JHdo9Ucj8bKQwVsspnX8y0Nn0iItICIiA45YmvBa5oc1wsWkXaR2hRdptqfgnDpsPtTzb/JH8g89nFh+XcpUWVvpyJ1PcWOSnOK4XPSyOgqI3RSs3tcLeI5jtXSVsNM9EKbE4vJzN2ZGj0UwHTjP3jsVadKtG6jDpzT1DbHex46krODmldBi5cb19GQa0aREReswEREAREQBERAEREB2KKkfM9sUTS+SRwaxo3uccgFZzVzoVHhcFiA6qlAM8n+DfdHzXi9Reh4a04nM3pOu2lBHVbudJ45gdx5qY1SdRytv048LklFGURFUEgow18Y66CjZTMdsuq3u27bzEyxcPElnwKk9Qr9oqld+KTeoPKsJ5OOy4fEA/BezAinfFMw+CFlKWrPWg+j2aWuJfS7o5Os+Abre8zs3j5KLUXRW1Rtj2yRAuBhePUdV+b1MU3Yx7XO+G9bJUva4g3GRG48l6LCtOsUpreSrZbDc158q23c+4VVZ0r7Jf2S7i1yKAcK131rMqininHEtvC8+OY+S9vgeuPDqhzY5WSUznkDafsuiBPN4N7dpC8dmBdD22Z2iSEXy0g5jMHcea+l42tGTC89ptorDidOYJOi8XMMts4n8+7dcL0KKUJuElKPIKd4zhctJM+nnbsSxOs4fQjsIzXRVg9dmiAqqfz6Fvp6ZvpAN8kG8+Lb37rqvi6jGvV0FL3INaMIiLeYCIiAIiIAtxongrq6rhpW39LIA4j1Wb3u8BdahTL9nvBbunrXDqAQxHtPSf8ALY/iK05FvpVuRlEz0dKyGNsUbQ2ONgaxo4NAsFzrCyuVb29smERFEBaPTHR6LEaV9NLlfpRvGZjkG5w+fgSvJ6wNaUWHytp6djaiZrh5xc2bG32Lj1/ovX6P6QU+IU4qKd+00izmnrRvtm1w5r1Km2pRs1oFWMfwWaimdBMLOaciOq9vBwPJaxTTrKoWTAhwzabtdxaVDM0ZaS07wV0OPd6kE/cgzjREW8wEREBLeqjWV5vs0Nc/0GQgmcfyPJjj7Hbw7t07tcDmMwRkeapcpa1U6y/Ny2hrn3gOUMzj+R4Bjvc7eHduqs7B7tzgvJJMnhFhrgcxmCMjzWQqNokfMjA4EOAIIsQcwQqq6w9HvwfXSwAeiJ24O2J2YHhmP3Va1RH9oLBQ+CGsaOlC/wAm8/8AG/MX7nD+tWPTruy3t9mYkQOiIugIBERAEREBlWf1Q4cKfC4PamDpXfvnL+kNVYArg6OQCOkp4x6lNE34MAVZ1SWq0vqyUTZIiKhJGFptL4qt9JM2hcGVRZ6Mn5hp4Otex5rQaR6zqGiqmUj7yEutPIw3bT8BfmeYG75L2kEzZGh7HB7HAFrmm7XA5gg8lu9OdTjOSBTisikY9zZQ5sgcRIHdYO43vne63Wh2ldRhk3lYjdjspoj1JWfceRU260NXbMQaammAZWsbnwbUAeq73uR8D2V2qIHxucx7Sx7HEOaRZzXDIghdDRdDJh/1EGtEuY9jMNZD5eF12uGYPWY7i13aolruu7vXLQYg+EnZPRdk5vBwXXqXhzi4biVsqq9PwuAcSIi3GAiIgCIiAlvVRrLNPs0Nc+8BygmdmYeAa73Pp3bp3a4EXGYIyPNUuU56h9JqicSUMrttkEYfC45uY2+zsd2eXJVGfhrTsj/JJP2JfXntYWHipw6qiIufIOc39ZnpG/NoXoV8TxB7XMO5zS09xFlUVS7Zp/Qkyl6Lmqo9h7m+y4j4Gy4V1yNYREQBERAZCuXRfk2W3bDbfAKmgVwtH5xJSwSD16eJ3xYCqnqq+GP8komxXjtaeKVtLQvkomXde0sgzdBFY3e0ffw3r0Nbi8EEsMEsgZJUl4gB9ctsSL7r9Id67zm3FjmCMxwKqa36coyktokUwe8uJJNyTck5klSLqt1jOw9wpakl1E85He6nceI93mPEdux1r6tvNtqtom/i5N5oR+hPtN9z6d26J10i9PJr/BDgudBM2Roexwex7Q5rmm7XA53BUf60NXTcRaammAZWsbmNzaho9V3vcj4d0c6rtYzqBwpqkl1G45He6nJ4j3eY8e+w0EzXtD2ODmOALXNN2uBzBBVJZXZiWbXBLkptUwPjc5j2lj2OIc1ws5pG8ELiVjtaGrtuINNTTAMrWNz4NqGgdU+9yPh3V2nhdG4se0te1xDmkWLXDIg9qvMbJjfHa5ItaOFERbzAREQBERAZUxfZ4oJPK1NRb0QjbGDzeXB1h3AfMKJaCkfPIyGNpdJI8NYBxcTYK2Gh+Asw+kipWWuxt5HD15Dm53x+QC8HUblCrt92ZibtYWVxyyBrS47mgk9wzXPR5RMp7jP5xNbd5eS38ZXRXNWS7b3O9p7j8TdcK69cGsIiLICIiAyrR6psRE+F05vcxNMTuwsJA/p2fiquKa/s9YzlUUTjuImiH9D/APD5rwdRr7qdr2Mrk+vtFOI8xINiHVFjxB9Cu/qo1lCpDaKtfaoAtDKchP7rj7f1710ftFxOLaJ1jstdUBzuAJEVh/S74KE2PINwbEG4O4gqFFEbsWMX++TLemXDxcXjIOYIz7VW7T7RlsEjpYBaNxu5g9Q9nYve6vtZPnUYoq11qgC0MpyE43bLvf8Ar3ro6ZDetGNCzHscWH5IcUk6rNYr6Bwpakl1G92R3mnceI93mPHvjysaA8gbrrhCtbK42x7ZIiXRa4EXGYIyPNV7184M2CtZUMGyKuK77cZGHZcfEFinbAPzWD9ni/sCiL7RvWov1Z/rEqPAbjkdq/JN8EMLKwt/SaHYlMxskVDO+N4u1wjdsuHMdiv3JR5ZA0CL0f8AoTFv9vn/AJbk/wBCYt/t8/8ALco+rD7kDzqLb4nozXUzduopJYWX6zmODfiuHAcJkrKiOmiF3zPDRyA4uPYBc+Cl3x1vfgEoahtFdt7sSlb0Y7sp7je/13juBt+8eSnFdHBMLjo4I6aIWjhYGt5nm49pNz4rvLl8q/1rHI2LwFodPMQFNh1VKTa1O9rf1njYb83Bb9RR9oDGRHSxUjT0p5Nt4/42c/3iP4Uxa++2KDICREXUmsIiIAiIgMre6FY6aCshqR1WPtKB60RyePgStCsrEoqSafuC4GKYdTYhTmKVolgmYC09hF2vaeBzyKrXp9oVPhU2y676d5PkJrZOHsu5OHJSfqN0uE0P4Pmd6WAXgJ9eHi3vb9D2KSMbwiCthdT1DA+N4zHEHg5p4Ec1RV2zxLXCXH75J8lPmOINwbEHI7rL19HpMZ4/JTm8rRZrz+kHb2rh0+0KmwqbZdd9O8nyE1snD2XcnDkvKAq5+C2KkiPB2K/rldcLL3k5nMrAW1cGC4eAfmtP+zRf2BRH9o3rUXdP9YlLmAfmtP8As0X9gUR/aO61F3T/AFiXP4X+T/ZN8EMBXJwwAQxACwETLdnRCpsFcXBJ2yU8MjCHNdCwtI3EbIXq6r8sTETvIiKk2SPiRgcCHAOaRYg5ghanDtF6GnlNRBSxxTOBBe0Wy42G4eC3CKSnJLSYMoiKIPlxAzOXNVZ1maQ/hCvllabxRnycHLYbx8TtHxUw659LhR0ppYnfjNU0g23xw7nO8dw8eSrmrvpmPpOx+/BGRhERWxEIiIAiIgCIiA7uFYjLTSsnhdsSxODmOHP7x2K0WgulsOKU4mZZsrbCeLjG/wD+TwKqitxovpFUYdO2op3WcMntPUkZxa4cl5MvFV8fyjKei1eOYRBWwup6hgfG8ZjiDwc08COarPp9oVPhU2y676d5PkJrZOHsu5OHJWF0M0vpsUi8pC7ZkaB5aEnpxn728itnjeEQVsLqeoYHxvGYO8Hg5p4Ec1U4+RPGn2z4+hJrZTxAvV6faFT4VNsuu+neT5Ga2Th7LuThyXk1fxnGcVKL8EC4GjEofR0z27nUsRH8sKPdfeATVEENTE0vbSmTyoGZDHhvT7hs5966WpTTlhjbhlS7Ze0nzV53PaTfyffcm3fZTC5oIIIuCMxzXPS7sW/uaJ8opapJ1W6xnYe4UtSS6iecjvdTuPrD3eY8R27HWvq1832q6hZenJvNCP0HvN9z6d26Jldp15Nf4I8FzoJmyND2OD2PaC1wN2uBzBBXKq7ardYzqBwpapxdRvd0Xb3UxJ3j3eY8R22Fhma9oexwc1wBa4G7XA5ggrn8rGlTLT4Jp7OREReUGFpdLdI4MNp3VEx3ZRsHWlfwaP8AvgmlOk1NhsJmqH2/8cY/KSu9lo+/gq0aZ6WVGKTmaY2Y24iiB6ETOQ5nmeKsMPDdr7n8phvR0dIcamrp31M7rySG/Y1vBo5ALWIsLokklpEAiIgCIiAIiIAiIgCIiA72EYpPSStnp5DFKw5Ob9DzHYVOug+t2nqQ2Gu2aafcJN0Eh7/UPfl2qvqLRfjQuWpIynouDjGFU9dA6Cdolhlb32yyc08+1Vp0+0KmwqbZdd9O8nyE1snD2XcnDkuHRnTnEMPsKecmMb4X+kiPgd3hZSGzWxQV8BpcVpHNbILOfH6RoPtgHpNI7Lrx003Y0vHxRM7TIZY8ggg2INwRvCnvVRrKFUG0Va8CoAAhlOQnGXRd7/171CuP0UMMpFNUNqYHZxSAFrtnk9rgC1wWuY8g3BsQciN4K9l9EL4aZhPRc9zQQQRcEWIO4hQPrX1amnLq2hZenNzNCP0B4ub7n07l6TVTrKFUG0Va+1SMoZTkJxu2Xe/9e9SlKW2O0Rs2zvut2qkg7cS3X6yXJTBSXqr1jOoXCkqnF1G49F291O48R7nMePfw62NGKKmk84oamEskd6SmbI1z4nHO7Wg9Ts4d26OldtQyK/K8MjwXOima5oe1wcxzbtcDdpbvuCo9021r0lGHRUpFXU2t0TeGM7uk4b+4fEKCTpHW+bik85kFM29og6zc87do7N29aleKrpkYy3N7M9xs8fxyorpTPUyGSQ7r9Vo9lo3Adi1iLCtEklpEQiIgCIiAIiIAiIgCIiAIiIAiIgCIiAIiID7Y4g3BsRuI3hck1VI/rvc7vJcuBE0gEREAREQBERAEREAREQBERAEREAREQBERAEREAREQBERAEREAREQBERAEREAREQBERAEREB//2Q==", + DB.PgDiskANN: "https://assets.zilliz.com/PG_Vector_d464f2ef5f.png", + DB.AlloyDB: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAOEAAADhCAMAAAAJbSJIAAAAxlBMVEX///9DhfWvy/tmnvb7/f9DhPZBhPRmnvUbcOj1+/5lnfdim/NAhfL2+v9Xk/Cvy/3l8P/c6f9YkvW50f3x9/6gw/ukxflNkPOuzPlWkPdMjurh7P5kn/Q5gO+91/xCg/gxfPEzfugqeeh3q/mSufuGsvnp8/zQ4PquyP4WcuhkmvuuzPXO5f7I3/1Dhe06e/EVcdwYa+d/r/N2q/eUu/d0p/iqxvDP3PyowvHU6f+uyu7A0vmTt/q5zPeFre2bwPZmmugMa+1BOZ5TAAASGklEQVR4nO2de3eiOheHCxIUiNCO6HBaI5dpVepUx9bantPOe+Z8/y/17mBtETcJKF7a5e+PWWuqQh52srNzIfvs7KSTTjrppJNOOumkk0466aTjk3ahHboIO5U2iGg0+LqM2h0LfF0h4d3XZNTuIkZ1QhRFZ+wLMmq9MQPz6TqlgEjY+Isxaj0K9lMUolBOqCiUsK9UVw3gA+spaREwZthrHrpolcjogX8hCiIyYT3j0MXbWkZvHOr6GAOEuuqz8JMzcv8CfkWhOKECvofR3udtj2A/xvu/HLw3SPCrrc9pR63FAkpITgVdaY9B2Pp8dgQ+Jmd7FwtHn8yOrTAg4tqZqaokjLzPY0dtFAa6X4ZQ4TGAH40+CWNrMtGJD2XO86B5jDqNvEMXvoB6UUAXAXZZwqTziJ1DA0ikxqGkd5CIRLF6aAiBRr9IydaHiLKjZVQbfPynywhkjwAeEmu0Dw2DSI2h/9N1PZcQSk5JEE+mjFKZpQmLj83ndBqRIrWewqa2dqbZ0wk+1sh895js2JkG3H7CElOFxd8W/Z1hT3Wxn01ccTTtHJhrqU4jkJtPYY36R1xmWFCn5Z1JcAx21DrQ//nyskbWatxpWFOfj6vEopP438PGq1o7YlBMCaHPoof1chpWEFBxbeUfsqh9uFiueQ18UvORMOi6+O+7sbxz4f3j7DB21GYRgwImRhDUNhY94nxcLm+P8mcURAdgbF4HjEoiTwr9WvAjn4/L7QZMp9IGSfbN6EL99ImEkBA9fOhLr9Wvh9IIIDHktfhZVSn3NSA+VXQxoc7C+2JlcrtRgii4GozGiM/2xOi+RrK2wyfX/OgHZr9+D/3rA7RpaQeps2i2e0Z3FkriEW4MykL0efdbqjfyzrHr/siZNl65rq6HO2ZsXofUFzt4MIWvR2j9PG+pjuOpqtPC7Og+FOl79JxnV42gfkqCzwRxHKFlOFc91VG5HBBuRwbjDwkh0emOGDX3hscvUreuh9fIeot2PlKBcCHPc4bqCGHUwI7yjgN8+C4Y3ZvQ56M/8SOG+nmN9FvAt7TfApH/Z9hC7fgQgJWElLwYOl5PNpfxEI3l9oP7Yv2ydv5hvrSA8nw95NSMG7kdoSwsuqksBuD1k/fukvvqY5zv6reTtt87n8OxR1dIWN28n0AMQEQxK1QmQqObZhVBuebeRwXCYwX8OMrXQqyXlocxGtdM96W9EiHxg7s1I/cvBWIqFrRRPqx2ZoXa0bgJBXM+74g0vNmqPWr9mwmRTM+De4d+OIfPc5D6mTWi6mB21NzrUNYe+VK5Ht30N7Zj/z5M2oMwnIK7sH+x/uHqt+clza2I0Lrq3sDwTBpA6f6GdgT7BYxRIpmfh6fYRtoC2A88SXFCdYTVVa0PduSGFNoSnGBQ3o5a/5ExWaAI3k5nbdR+I68g2bsctK7COC3ZZiSZJ4H2+KMco/vAGDw8iYshwIfZ78JTsf5Bjoj6nKQ9ymI53fdDwVzCurRWmMxM5xLy3T8sxNrf2cUGdAkhH3ao2NZFYEy2GwlcK0QAZVfKmy02pnmVAzyYTgPcfqozHBZtfBgmxqi5M/7Ac9wBhcGxHrbK7z4y1Pye0KdhB6sTg0V85hXpBvPkeRfIld3ZmOGF4bNB1NssgjMchkz2Uj5+ULErDuZJBXXyCT3Pu1M9cTX2vOEcq6tGGx8/Ej1SNw9sNCfM1n5woKyD8klKzjW64l8cyfoQR33G7NjsMF6rVgpEx1Fnu8CNz2qv7LojrIP6lxE6fljR/OrtmgMYH6pCf+uN5ihje3Wug5AJFk2VlDGLwG6LyTCdRWj/N1AXg4VcSP7ZVeoHF07SPeQ/FC+nPTY74Ff1xHhUYdGsot2NrzHzeeiA75XQnp6TiQkhYZov0dUcGqXI7PCpM8BK04l4qAyliWfV4C0YG7rCptg+Ce2Og+USLmoiOr12MRcSQufoqc8o43AaED++qZCPgzyg65VQPxdwucZzvGdsSibR1bwj+m3ye5xRjbcc4beKfY23P0dICEaa5/FxXTnPMgflqSgjVuriTlWjca/At3qqsxg+5BDysufabyloj+IIAT4cFXlFw/s1LtFt2EogY4T29269HMKROpcvzADjsywGcnLaY0relPnfyhBC1D35dSfge3ovVe4gsIP6F0xXakeMCHcRMvamjPjkrzKEvMfRG9McxibYb6W/XiN0hp0ROpi58FCzXj3PJeEsjK2ecgh6cezzKdYyNvyWTB4QEqF21CDuEopXXpTv/NlxhrjrOe9I7Aj9I9oc76LwLR4pT8h7+biBtUftSRBywSc59hvyqHuoztFVi7P+XBzIoXy9aTB+m8mlZWrpN11ZvN0CTyeIMTs2n5zFnG6GDvrpTgvlGyTlX/hedS3EWTBCe8QiHT4lgtsvjmD8/zbtWIrQ1pfv7/Bx/mSKRog9rN14OfYbJP3e4pkk0c8IZTz/jYdyKN/fjWARLS8QNyVMFgSDBvat5u/1p/07h2+5dJH8y1cRh94/OGPmmmDxZ/ztoUb49g7VpoQrg8IpHsO7L/NUgbz5C8YHPScqR8UZ+95w+BbOJlNaL/i9jWlm5LodYSNvlOKm1iXQF3yaT0NBJ+ChL8321cVkJNTYUe7bX0YjO/mwFWGcPw5zX8A/ejl8xmCeLGznWTFnlvTM9ZIwwhO83VYtoSIghOKMHAetStoT4m7XKNEZRKgcUOlFd90Z4Tm6WwQrSnPgiUdWb20NqiLOiDXqq/eudGeEF9Z9oflzvDPJsaPn/SxyTe383son1EtF3gLCW8t6lTFq0DjLKSckW7no1U23myYkOyI0zdqtfS0cFrkvnnTGcJ0R79bfdX7TNU1TQFgqLhXWUriPWbfy9yY1e2+RS0lEJyf05NLOry2rVqvtjDDVH16YCWHd6rZRRrDfYhhUmjAxJM7Yf7Xq3dr+CGtmrV4HzG472x6h/b0vHKI7MCR8/MmsMWr9WTe55R4Jl7IydRXq58arTks56nyVEez3ccu9EFrL24Edb2/tdF39PdyWL6nc3ujjku6sa95+PNP9ENZS4t7tY/goGfoXlfcxM6ta1ur9Pgib+yHkVfVjcrUiQvWDsG2Z3TpOaMR7Iux2d0tY6+bYcD1q2xGhae6YsJ5LmFmh3gdh6S0mUsLZMRDWU4TVABYm3JOn+fKEZsrTVARYuB3uidDapQ1PhHtph+axEG41AhYQ1tOepiLEQxOauYSj3ROaeyA0M4RWxTaEi2xGuIvxYaIU4T8VAPLpxXk+ocjTVDSrnyVM+9Lz7QfAfPUjtYR6DITpmAYY55vvLeV0jteZp5dryhAqVREKorbkc+nOilzB6H6eWQIXEio7IszW0np2e9Fg0yEGssR/EBtKCZMFX/GuvHXB14fI8v5REJrYFrHBvMB+2lVCdPfUURAiNkwYSyDyOWR8Q9Axepp3aQNss0YOYd6GpzKeprLeoighZ+wUM2P+hq6D9IeC2USEcS6x4kh9Fi05HT0hZxyK9qvhS2off8oS1g5AWE/50v9hmxW0F0FVRe33ND4uwlQ7vLTQdwOMHn+9AKmh+E4u2/pejHBPnmaF0Ly10be9mi/ZrgPcLLY9Rftpdevdv46L0EoTmremjb5x4qYZebtEd8pcmJyom2/Dg/QWK4TdZDXqX+ytNr44/AbooTuBwH7JSqFZ24xwV/M05gphPVkD71rom23N3mKaw8POm9UGj+by0sdFWFtph0lxwJBda4basQcB2h32yc8/Fgc0PwNhstxes2oWupPBfUL+CvZ7Wwg1k6h3M1+6RxsmpgCn0/0X2VmEvXf6CBV7sRAqIzyIDbPtMFWcujWTvnChXdTBwaQx6vUNfeke5rwzhIkhMb+a4rt6hPqcqRRHRlgXEdagf4S+I5fv4tGCX3QzVzw2wkw7zMzE8S/gPudMO3+A6lnPft/c1NMckLDeRXbBaVc/uuBeShIexIaWjLAGIFamf9Su7q1bM0HMfh9+cFyENRnhsmh/UjH5+Q+rluz/w756bO3QLEa4iFcXV+nfJDs4a7XPQZjt8fMI+W7GuvWqnbn39beOPedRHFs7XCGs5xMmAts9WDx8AcJ6t4t+58gJxYDci9QXO0VNvl8M+wIfnaQJzcMTptrhtb3WfZcVH3pZw+MiTM/qN69tS2JFGaBpO6nRY9s8PGF9dd3CbdtWrYv6SJl4Dbas1VfOj8GG9eyctzGzBZ5SRGhadnbu9Rg8TX19RrjJ7VheJjK3fAjCgbXqSxBCflK7bYv6vDVB/LZmv8MRZgqHz+obzyXsCOGOhR9wsE74vsq/M0L3j706yM1btzA6xRiB4DaHb53Qfnin2BkhhJWPdqpnx9eAF1K/39ZMZICUxuvWbfsp9wppQtO6TB9PXC2hsvqWrPG/y3fziAjPzl7sb2JC0069zSAktC5Xjw/doQ253NdvPPrqygjPzlqX2EB+oW7N/i7ie9+rDw31Mns8ccWE62/JGrPLW4CEm79ILtb7jsfYdfOb/VPyW/6+BXzT/j5bK321hH6MzLRA1568MCcjhLp6icRypn0p43sjtL6t80E1qrYdKmjuHt4lmN0Ch/Wc3Vn2in+B8OVPkaOD2jDUsNAD51qRUi2hEsRYe9OGtiW3IdfT99QbaND//V3oV237O9rMven6+XvbEubl0tLU3wUvemcve1K7IN/ZWecFK7XaYMjhv1sTJrkJtssXNrDtWxgeWdhBIoWl5Z1YWQFhcia5s9Upd8D4Bz0Mo6ia7YgfLY4VrhJCSkkQoedCFlbRs6NQNYcBP8U0p3BVECaH2pMtGTeW0YkYP6klp2wVESrJ+dZb1tWN1GyHZJEfqhJC0fnShJ95g56RvEO5wMcrqODI5OpsyMWTv2LnfO5Ibjsi0uSRZQi1y5wze1OGJDqj6M6ZyqW57YDpY3HSC074vYQN+zch7pHT8n09xFcGK5U7i6AsktIQwth9kSMM36X1HyLqS7Ki8ZNp0TPZK5Q7G4/5geXJ0Vy5JdFZ2TPZzzjjfUQlaQP4se1sjO4OqkSaOwspPOiFA8UJ+dH3JNwsPwI/t9/3ZQe+K1A/2pXkClm7f3KmvuTu1Pcp2yL/g3sT6b4skYauj3fhV5Pz9KXOAOLILXN4uDcTKsnDmCRhiio4dTp9X+M6Eh7sryRp2ogSFjvTSSiw41iaEQ26jipzTRqvoc/bl9CG0ECjh0p8ALR3aS4d/rxpRXbk+ZCWhzbm3C75Ownuq4ur4J4F8mkSiueDKKfmQyRM9JDcifiMVZfviUtzXyN5DAANMt4ut53mPoSMyrIEUUJ3kQvxLY+GWOCUwi1Ognfvoa4QUe7r5DHuLrfcLJk+kLkdnodxo/YBsSLjbCIHOuZn/17vblzTBB8uDwE266PcHwGTJlzjAcaOczy6s0A27oBi6n7Z3HYQB/u+1MP4ZB+5SHnOF/mjVlhQoq/qP8pzK9JF/d8h2YfAjoGUUYd49bVYefrgP3VFmn1wb3xcPObXiS90OjwfTZGYI8l5rEgCNOL7u86xmpUxm/hF8tvGEp/jPkZBkdy07AB5q7W33D2SvJ1oXvWl3HrAG3WB/udA85ftOOATYNL2k8No8Py4kuiF+5fgQHxcmhNPdJkJICiPfqyXsWk22Fiac5wGaO7IPUrrNLifEBVS52sB8Z/VchrdiPl8CkLwU+rrrHFgvkTer4kkeR8XmdY/Qi3DbjCOLnGhkxjLbXMIqf9NFNlcBwwHptbCHobVmBCeXZiKpnnJ5L/2gbnS8qZRAZ9PY1s70+ypfKxJaIDmJjqkvJjJC05ZHP1i0gl6Qth/Q/kt967RrwK50XlmOCoDPD77LdVrBIKJ6Q9GwYd0DP4T2/dxLBrFTLbEwCWan0f3QxyTejGTrqPg5uOxOpkeY/vLqhdGVDzswK1HFcaOuX6m1YqYdLi+ZkKfxd5el5W3ktYKA1+aTzhVQXXKok/Ex6W1ggJ9x5ugfoZbZEs9lMCO0sWAhfQgHH0+Pi7jhTJZ/8i3dbCS2ZiPSVqPiusqVYIxmoLn80jrjUN8DEgW+wt6n9d+SzV7IcTkWHJowj67/ZbSeoytJTGH0SH7InxcnDGdjFmHCDu8+zp8XNrdmC3dKtV5/fz87S8r7WmZG51+Cf+CSRvw3Oh6ML77mnxcxs+IRYXSc31eaWiqvJNOOumkk0466aSTTjrppJMOrf8DUqHU5b2xAzsAAAAASUVORK5CYII=", + DB.MemoryDB: "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wCEAAkGBw4PEQ4PDw8VEA8WEhUVDw0YFRUaFw4gFxUXGBcVFhUeHDQsGRolGxUWIjEhJysrLi8uGCAzODcsNygvLi4BCgoKDg0OGxAQGjYlHiErLS0uMS0vKzItLSsvLS0tLS01LSstKy0tLS0tLS0tLS0rLS0tLS0rLS0tLS0tLS0rLf/AABEIAIkAiQMBEQACEQEDEQH/xAAcAAABBAMBAAAAAAAAAAAAAAACAQQFBgADBwj/xABFEAABAwEDBQ0FBQcEAwAAAAABAAIDBAURIQYSMUGyByIjUVJUYXFzkZOx0RMyNIGhJEJyksEUFzNidNLhFVOC8BYmQ//EABoBAAIDAQEAAAAAAAAAAAAAAAABAgMFBAb/xAA1EQACAQEFBQYFBQADAQAAAAAAAQIDBBExUXESITIzkQUTFEFSwRUiYbHwI4Gh0eElQlMk/9oADAMBAAIRAxEAPwCMtCpk9rLv3e+7WeMr2VOMdhbvI4G95pFTJy3d5U9iOQrwhUyct3eUbEchXsIVMnLd3lGxHIL2KKmTlu7ynsRyFewhUyct3eUbEcgvYX7TJy3d5RsRyFewhUyct3eUbEchXsIVMnLd3lGxHIL2KKh/Ld3lGxHIV7CFQ/lu7yjYjkF7CFQ/lu7ynsRyFewhUP5bu8o2I5BewhUP5bu8o2I5CvYQqH8s95RsRyFexRUP5Z7yjYjkF7HtlTv9tBvj/EbrPGFVXjHu5bvJjg3tI7IvGmuedbRPDTdo7zK9pT4FocDxNAKmIIFAggUAECgQQQIIFMBQUCCBQAQKBBAoAIFAggUxBAoAIFAgggB5ZB4eDtG7QVVflS0Y4cSO1LxZrnnO0Tw03aO8yvaU+BaHC8TQCpiCBQI2Rsc43NaXHkgEnuCTaW9iJ6hyOtKa4tp8wHQXnNB81yzt9nhjK/QmqU3giWi3Oaw+/NFH3u/ULmfa1JYJsl3EszeNzWfn0Phu/vUfjEP/ADfX/A8NL1Lp/oD9zmqHu1ETz1Fv6lSXa9J4xaDw8syOrMjLRixMOeONjs70XRDtGzz/AO12pB0ZryISSNzDmvaWu4iCCuxSUleipq4QFMQQKACBQIIFMQQQAQKBDyyP48HaM2gqq/KloyUOJHbF4s1jzjaP8abtHeZXs6fAtDheJoBVgiXyZsV9dUNgac1t2dJJyAP14lz2m0KhT23+xKMdp3F0rcoqCyr6eggbLO3B87sQCMDedJPQLgsyFlrWr560rk/Itc4w3RKzX5ZWjOTfUOY3kMuaB1EY/Vd9OwUIYRv13/4VOpJ+ZESVkz/fmkf+KR7vMrqVOCwSX7Ir3mpTvI3G+KpkZ7kj2fhe4eRUXCLxQ72S9BlXaEN2bUvcOS859/zdj9VzVLFQnjHpu+xJVJLzLTZ9v0dqXU1dC1kzsI524XnVcdLT13hZ9Sy1bJ+pRlelii5TjU3SRUrfsl9HO+B5vuxY/lg6D/3iWpZq6r01NHPODi7iPBV5AIFABAoEECmIIFAD2yDw8HaM2gqq/Kloxw4kdtXizWPN9onhpu0d5lezp8C0OJ4mgFTEX3cfF9TWX/7Me09ZXbHKhq/YsocT/MxjlRkXVUz5JImmanLi5r24lgJvucPnp1q6y2+nUioy3SFOk1gVfRpw6FoFQoKBBAoAIFAgggRackslaiokjle0xQNcHGQ4F92pvqs+2W6nTi4p3yLqdJt3+Q+3VPioewG29Vdj8l6+yHaOJFNBWsc4QKBBAoAIFAggUxD2yP48HaM2gqq/KloyUOJHb14s1TzbaJ4abtHeZXs6fAtDjeJoBUyJfNyH4iu7CPakWX2vy4av2LKPE/zMibFyxraMljJM+ION0L8QMdAOlvyXRWsNGtvaueaIxqSiT/8A5pZlT8bZ4zvvStAJPURj9VyeAtFPlVPz7E+8i8UJ/wCsy4j2kPWZBtEp/wDIRyfT2F+kwv8ATcm+eu6s8f2o763+j+AupZiiLJqPEvfL0Bzzf+UhG12hLyS6e4XUkGMqrJp/hKDOd917gAR83XlLwVqqc2p+fsHeQWCIq0Mr6ysfGxzvZxF7b4mYA4jSdJ6l007BSoxbSvd2LIOrKTJHdV+Kh7Abb1T2PyZa+yHaOJFMBWsc4QKACBQIIFABAoEPbHPDwdozaCqr8qWjHDiR3FeMNU82WkeGm7R3mV7OnwLQ43iaAVMRfNx/4iu/p49qRZfa/Lhq/Yso8TKQ8753WfNaiwKjAUxBAoEECgAgUCCBQA4oTwkX42+YUZ8L0BYlx3V/i4OwG29ZnZHJlr7IutHEimArWOcUFAggUAECgQQKAH1jnh4O0ZtBVV+XLRjjxI7kvGGoearSPDTdo7aK9nT4FocjxNAKmRL9uN41NaDo9hFtSLK7Y5UNX7FtHFjLKnIiqpnySwNNRTOcXNc3F0YJvzXDWBoBH+VbZe0KdRKM90vuRnTawwKocDccDracCPktD6lYoKYggUAECgQQKBDihPCxfjb5hRqcL0GsS5brHxcHYDbeszsjky19kW2jiKUFqnOGCmAoQIIFABAoEPrGPD0/aM2gqq/Lloxx4kd0XjDUPNFpHhpu0dtFezp8C0OR4mgFTEX3cdP2mv8A6ePakWX2vy4av2LKWLIWxMr66iJbFLnxBx4B++aMdWN7eoG7oXVWsVGtvkt+aIRm1gWVuXFm1IurrOF+t7Ljf06j9VxeAr0+VU6/jJ95F4oUNyYmxznwdG+F30KL+0IfUP02VC3oqZlQ5lG8yQb0RuN5LidIGAvx6FpWeVR006quZVK5PcTtiZBVk4Ek11LFpzn+9dx5mr/lcuWt2lSpvZj8z+n9/wBEo0m8SXNj2BSYT1Tp5Bpa03g/ID9Vzd/bavBG5EtmmsWY228nYyMyieSDeHZvFrxejw9uljP86BtU15Dy0corDrnNdURyB4bmiQtuzReTdg46ydShSstsoK6DVw5TpyxKtYdhNr56iOGURMbeYc7Eyb65o7sdC0K9pdCnGUle3jcUxhtN3Gm2sm6yjPDREs1TN3zD1nV87lOha6Vbhe/LzIzpyjiRYK6iAoKBBAoAfWMftFP2jNoKqvy5aMceJHdl4w0zzNaR4abtHbRXs6fAtDlZoBUxF+3G/ia/+ni2pVl9r8qGr9iyliyjvO+d+I+a1FgUmAoAe2RZk1XK2CBuc89zBrc46gq6tWFKO1N7gSbdyL+42dYLQLhVWiW33YcHf0/cb9TcdNyyf17c/TT/ADqy35af1ZWau17TtWTMBe/igjvDW9f+V3Qo2eyxv/llblKZL0O5tUkZ9RPHTt0luk9+gKifa1O+6EWySovzZudkdZDcJLYDXcWfAPMKKt9qfDR/iQnTgsZfY2Dc/glH2S0Wyn+bMds3JfE5x5lO7r7j7lPCRX7cycrLPzXy3BpdcyZjtJuJu6DcCu2ha6Vo3R6MrlCUMTJMp619O+lklz4nXXkjfAcnO4k1Y6KqKolc10E6kmrmNZLLqGQsqXRkQuNzZOP/AAf0Vir03N0096IuLuvGoKuIihAh9Yx+0U/as2gqq/Lloxx4kd4XjDTPMtpHhpu0dtFezp8C0OZjcFTInQNxcj9qrr9HsIr/AM8iyu2OVDV+xbSxHNu5CNqi6rsqZkrHkudTl12aSTfmu6796butRodoOn+nXVzXmKVO/fEo8tmVMcwppIXMqCQBCc0u3xw90kY3ca1FVg4bad8cyq533HRaqaKwKQRR3PtGYXvfyOn8I0Aa8SseKlbqu0+CJc/kX1K3knkvNaT31Ezyynzi6apJ30h15pPnoC7bVa42dKEVv8lkVwhtb2TdqZaU9G00tkxNaBg6puvzukX+8ekrlpWGdZ95aH+xJ1FHdEpFfaE9S7OnldKb798SQOoaB8lrU6cKaugriltvE0BTIhNwII0jQdY6igLh5VWnUTNYyWZ8jGXlgc4nNv6Sq4UoQbcVc2Dk3iW7IfI11Rm1VU0inGMcWufpPEzz6tOdb7eqf6dN/N5vL/fsW0qV+94DLLPKWWpkdTtaYYI3FoguuJLcL3DyGi64q6w2SNKKm98n5kalRyd2RWgV3lIQKYD6xfiKftWbQVVfly0Y48SO8rxhpHmO0zw03aO2ivZ0+BaHMxuCpiOgbjPxNof08W1KsrtflQ1fsWU/MptJXz073Pp5nwvzji04HHSWnAnrBWlOnCorpq/X8vKk2sCTszKeeKsFdK1tTNdcc7C/C68EDA/JU1LLCVLuou5DUmneOqGCe27Q35IzyXykf/Fg+6D8w0dd6hOULHQ3eWH1YXOciWy+yiaLrMoj7OliAbMW4e1I+4DyRr4zhqINFgsz59XfJ4fT665DqS/6rApIK0yoIFABApiCvQIvWS2S8UUf+oWlwcDRnMid9/iJGu/U3Wsq1WyUpdzQ3t+ZdCCS2pDS2MuaqWoZLAfZRRngoNTxrz+sdyto9nU403Ge9vF/0RlVk3eiWyso4rQpWWpTNueBdUM14ab+lpPcehc1jqSs9V2eph5fn1JVEpx20UIFbJzhBAh9Yp+0U/as2gq6/LloyUeJHe14w0TzDaZ4aftH7RXsqfAtDnY3BVgjoO4wftNeNZp47hx76RZXa/Khq/YnTxKLKCHPBwIe4EcRDiCD0ghaiwRUICmB0fJK6zrKq7RIHtpiWQnobe1jfz55+YWPav8A6LVGj5LH7v8Ai4sj8sWznl5xJN5JJc7W4nEk9JK2CkIFADmhoZ6h2ZBC+V/JaPMnAfMqM6kIK+buQJN4D+rybtGFufLRysbyt676NcSqoWqhN3Rmn197gcZLFFqyZyYipIxaNp3NY250FMdZ1OcNbuJurScbruG02yVWXcWfF4v8/l/jsjTS+aRA5U5TTWhJe7eQtPBQ6m/zHjcuuyWSNnjuxeLK5zciEBXWQLtuY2iBNJRyYxTscM3pAN/e3OWX2pSvgqscYl1F77sytW1RGmqJ4D9x5APGNS76FTvKcZ5opkrncNAVaRH1i/EU/as2gqq/LloxxxR31eNNE8v2meGn7R+0V7KnwLQ52NwVMRKZO21LQ1DKmLEi8OYdEjTpae7TqVVejGtTcJDTud50Koo7Gtu+aKX9jrHe+w3DPPSNDj0g39CyoztNk+WS2ok7oyKblVktPZroxK9jw+/2bmk43abwRhpC0bLa42hPZWBXKNxb8tqaVtl2TBDG94LWOkaxrnXcGHXm4colcFilF2mpOTu11uJzXypFBFmVXNpvCk9Fq97T9S6oquZtprHqnvYwU8oznNbnGN9zbzdecNA0pSr04pvaW76oNlnSbfrZbJiiorOp3OkLA6apEbnAX4aQMXEg6dAuwxWNQpxtUnVrS3eSvuLZPZ3RISxMrraikBqGTVMRO/Y6C4gfylrB9QV1V7FZZx+RqL+j/tshGc1iP91KiqJX0s8ftJYXs3sQa4+yOm8tAwvBGlV9l1IQUoO5NeeY6qbuZRhZtTzeXwn+i1e9p+pdUU3PIIWbU83l8J/ojvafqXVC2XkSFgU1VFVUsnsJRmzR3n2b9GcM7VxXqq0TpypSjtLB+aHFNNO4lt0+ENri4YZ8TDd1X4/Vc/Zcr6F2TJVl8xVAVolQ/sQ/aKftWbQVVfly0Y44o78vGmgeXrTPDT9o/aK9lT4FoUMbAqYggUCDadevjQBulqpH5ge9zw2/MziTm33X3X9QSUYq+5YiZcKbdLtCNjIwyLNa0NbeNQFw8lwS7Moybbb3k+8ZuG6haHIi/Ko/CqObDvGKN0+0ORF+VHwqjmxd6xf3n2hyIu5P4VRzYd7IIbptociL8qPhVHNh3shf3m2hyIu5Hwqjmxd7IUbpdfyIvyo+FUc2HfSF/eVX8iLuR8Ko5sXfSCG6TX8iLuR8Ko5sO+kQNvW5NXSNlmDQ4NzRmjBdlns8aEdmJXKTk72RwKvID+xPiKftWbQVVfly0Y44o9ALxpoHly1Dw8/aP2ivZU+BaFDG4KmIUFABAoEECmAQKBBAoEECgAgUCCBQAQKBCgoEECgAgUxBAoEECgB/YZ+0U/as2gqq/LloxxxR6CXjTvPNto5PV5mmIpJiDI64+zfjvj0L1lO0UtlfMsMyq5mgZOWhzOfwn+in4il6l1Fcwhk7X8zm8N/ojxFL1LqK5ijJ2v5nN4b/AER4il6l1C5hDJ6v5nN4b/RHiKXqXUVzCGT1fzSbw3+iPEUvUuobLFGT1fzSbw3+iPEUvUuotlhDJ+v5pN4b/RPxFL1LqGywhk/Xc0m8N/ojxFL1LqLZYoyfruaTeG/0R4il6l1FsvIIWBXc0m8N/ojxNL1LqGy8ghYFdzSbw3eiPEUvUuotl5CiwK7mk3hv9EeIpepdQ2XkELBruazeG70R4il6l1FsvIIWDW81m8N3ojxFL1LqGy8ghYVbzWbw3eifiaXqXUWw8h7Y9i1jZ6cuppQBIwkljrhvh0KqtaKTpySksMxxi78Dui8mdpiAMQBiAMQBiAMQBiAMQBiAMQBiAMQBiAMQBiAMQBiAMQB//9k=", + DB.AliyunElasticsearch: "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wCEAAkGBw8QEBUPDxAVFRAVFRUVFRUVFRUVFRYVFRUWFhUVFRUYHSggGBolHRUVITIhJSkrLi4uFx8zODMtNygtLisBCgoKDg0OGhAQGzAmHyUtLy0wLS03Ly4tLS0tLS0vLS0vLS8tLS0rLS0tLSsvLS0tNS0tLS0tLS0tLS0tLS0tLf/AABEIAL0BCgMBEQACEQEDEQH/xAAcAAEAAQUBAQAAAAAAAAAAAAAABgECBAUHAwj/xABLEAABAwIDAwcHCgIIBQUAAAABAAIDBBEFITEGEkEHEyJRYXHRMlSBkZOhsRQWFyNCUnKSosFighUzRFODssLSNXOz0/AkJUNjo//EABoBAQACAwEAAAAAAAAAAAAAAAACAwEEBQb/xAA7EQACAgEBBwAIBAQFBQEAAAAAAQIDEQQFEhMhMUFRFCJSYXGBkbEyocHwBhVT0TM1YsLxJDRCguEj/9oADAMBAAIRAxEAPwDuKAIAgCAIAgCAIAgCAIAgCAIAgCAIAgCAIAgCAIAgCAIAgCAIAgCAIAgCAIAgCAIAgCAIAgCAIAgCAIAgCAIAgCAIAgCAIAgCAIAgCAIAgCAIAgCAICl0AQBALoBdAQjbzlAbhz2wQxtlnPSe0uIbG0jK5A8o9XVnxF9qjTOxZfJFc7N3kRT6YqnzOL2j/BX+hR8lfGfgfTFU+Zxe0f4J6FHyOM/BUcsNT5nF7R/gnoUfI4z8HozldqT/AGSL87/BY9Cj5M8Z+DKh5Uap39kj/O7wWHpI+TPFfgz4uUKpIuaeMDr3nKD00fJniMwK3lbLMmQMkd2OcG/m4+hSjo89WRdxgfTHU+Zxe0f4KfoUfJjjPwPpjqfM4vaP8E9Cj5HGfgfTFU+Zxe0f4J6FHyOM/A+mKp8zi9o/wT0KPkcZ+B9MVT5nF7R/gnoUfI4z8D6YqnzOL2j/AAT0KPkcZ+B9MVT5nF7R/gnoUfI4z8GTScsDybS0rG9oc5w9Itf4qL0XhmVd5RthyhzObvRwxOb1h7lX6MujZLiGHNymVTf7LH+d3gprSxfcxxX4MOTlaqh/ZIvzv8FL0OPkxxX4PI8sFT5nF+d/gs+hR9oxxn4LfpiqfM4vaP8ABPQo+Rxn4H0xVPmcXtH+CehR8jjPwbPZ3lY56obDVwMijf0RI15Ia4+TvAjyTpfhlwuoWaPEcxeSUbsvmdPutEuF0AugF0AugF0AQBAEAQGg222lZh1K6Y2MruhEz7zyNT/CNT3dqtpqdksEJy3Vk+daqofK90sri6R7i5zjqXHMldlJJYRqnmhgqAgPeKO6w2ZNpR0d1ByJJGwnmipm3fm46NGp8B2qtJy6GeSI/iGJyTZONmcGDT09Z71dGCiQbyYakYCAIAgCAogKoAgKID3paqSJ29G4g8eo944rDin1Mp4JFQYnHUdB4DJf0u7uo9iplBx5omnksrqG3BZjINGmnhsrUyJjOasmCxDBRAdo5JdrTUR/IZ3XnibeNxOckQyt2ubp2i3UVzdXTuvfXRmxVPPJnRlplwQBAEAQBAEAQFHOAFybAanhZAfO23u0hxCsdI0/UMuyEfwA5v73HPu3RwXYoq4cMd+5qTlvPJG1cQLgEB7RMujMm1oaS6qlIkkZuIVzaZu62xlIyHBo63eCjGLl8DLeCMTSue4ucSXHUlXpYIFqGCiAIAgKoAgKICqAogKoAgCAkGEYvvWhnPY159wcf3VM4Y5ompdme+IUVuCxGRlo0c8VlcmRMZwWTBYhgycOrpKeZk8LrSRuDmntHA9YIuCOolYlFSWGZTxzR9KbO4xHW00dVHo9tyOLXDJzD2gghcWyDhJxZtxlvLJsVAkEAQBAEAQBAQTldx/5NR/J2G0tTdmWoiFucPpuG/zHqW1pK96e8+iKrZYWDhi6hrFUB6Mahk2VFT3KhJmUbWqqW00e9kXnJg7es9gVSW8yT5Ii0kjnEucbuJuSeJWwlgrLEBVAEAQFEBuaHZXEZxeKjmI6ywsB7i+wKrldXHqyShJ9jZDk5xjzM+1g/wB6h6VV5+5Lhy8GJV7E4rFm+ilsPubsn/TJUlfW+5jcl4NFNE5jix7XNcNWuBa4d4OYVqeeaIFqAIAgCAICSYHiHOt5iQ9MDoE/aA+ye0fBUTjjmiyLzyPPEKWylFmGjTSssrURPAhDAQHR+RnH+aqHULz0JrvjvwlaOkB+Jo/QOtaesrzHfXYuqlh4OzLmmwEAQBAEAQBAfO/KNjPyzEZXA3jjPMx9W7GSHEd7t436rLsaeG5Wvqak5ZkRpXECrQgMumjuVhskSChia1pe7JoFyewKiTJIjmI1hmkLzpo0dTRoFfGO6sEG8mMsmAgCAogJ/sjyZVFUBNVkwQGxDbfXPH4T5A7Tn2cVqW6uMeUeb/ItjU31OrYHstQ0QHyena13GQjekPe92foGS0J2zn+Jl6gl0NyqyQQBAYeJ4VT1LdyphZI3qe0G3aDqD2hSjOUXlMw0n1Oa7U8kwsZcOeb68xI64PYyQ5judfvC3atZ2n9SmVXg5ZVU8kT3RyscyRps5rgQ4HtBW+mmsopPNDAQBAVY8tIc02INwRwI0QEuimFRCJB5Wjh1OH7HX0rXxuvBb1Ro62GxVsWQZr3hTMFiGD1o6p8MjJozaSNzXtP8TSCPRksNJrDM9D6dwnEGVMEdRH5ErGvHZvC9j2jT0LiTi4ycWbieVky1EyEAQBAEBqNrsU+SUM9QDZzIzufjd0WfqIVlUN+aiRm8Js+aF2jTKoD0YEMm2w+G5VcmSR77RVG5G2Eau6TvwjQek/BRrWXkS8EdVxAIAgKoDsnJvyftha2srWXnNnRxOGUQ1DnDjJ/l71ztRqd71Y9DYrrxzZ0laRcEAQBAEAQBARnbXY2DEo87MqWj6uW3qY/7zPhe4V9N7rfuITgpHAMRoZaeV8E7C2Vh3XA/EHiCMweIK60ZKSyjVaxyZjLJgIAgNts5V7ku4fIk6J/F9k+vL0quyOVklF8zZYpT2uowZJoj8zbK1EDwKyYCA7VyLYpzlG+mJ6UEmX/Llu4fq5z3Lm6yGJ73k2aXywdDWmWhAEAQBAc45bq/cpIacHOWXeI62RNuf1OYt3RRzJy8FNz5JHGF0TXKhAZEDc1hmUSLCYVTNk0R7E6nnZnv4XsPwjIe4K2CwsEG8sxlIwEBRAdD5I9lhUzGtmbeGFwEYOj5hY37QzI95HUtTV3bq3V1ZbVDLyzsjq2EHdMrL3tbfbe/Va65PFhnGUbypsayovHwZ6c637w9YUt5EN1+AJG/eHrCZRjdfgvWTBjy18LDuvlY1w1Be0EccwSq5XVxeHJfUtjRbJZjFtfBln9KU/8Afx/nb4rHpFXtL6ol6Nd7D+jPaCpjkuY3tcBruuBt32UozjP8LyVzrnD8Sa+J6qZAtMjesesLGUZ3X4IBys7NMqaf5ZCB8ogb0rayRDNze0tzcP5hxW3pb1GW63yZXbU2spHEyLarqJp9DVaa5MIYKICoPVqgJjI/noWy/eaL94yd7wVrLk8FvVEdrGWKvRBmC5SIlEBO+Ruv5vETETlNE5tut7PrG+4SetausjmvPgtqeJHclyzZCAIAgCA4ry21e9WwxcI4N70yPdf3Maulol6jfvNa5+tg52twqLmoDNpG5qLJI35fzdO9/ENIHech7yqeskiXRERWwVlUAQFWMLiGtF3EgAdZJsB60B9M7NYS2ipIqVtugwBxH2nnN7vS4kriWT35ORuRWFgh1bSsklkaR9Y+s5sON+i033uziF5u2uM5yXdzxn3HrKbZwrg10VeceX2MuugwqOR7XxTXa6zi2+6D33VtsNHGclKL5dSime0Jwi4yjhrlnr9i7D8JpxiAY1p5tsbZW3JvvDccLn06LNWmqWrUUuSWV8eRi/V3PQuUnzbcX8OZN12zzpBsQgMmIzBtOJjutO4524B0I+lf3W7VxLYOWqmlDe6cm8dkejpsUNDW3Nw5vmlnu+Ri4fRPdNM0UTHlrhdhksI9cgb9K/7KqqqTsmlWnjtnp/cvvujGqtu5rPfHX+xudhBYTi1vrBl1a5Lc2ZyU/ic/bDy63nPqknlkaxpc4gNAuSdABqSunKSisvoceMXJqMVlsilZQYXLI6V1SN5xubSNtfsyXJsq0VknJz5v3o7dV+0KoKEa+S9zPL+iMJ85/wD0Z4KPo2h9v80WembR/p/k/wC5yPbWiMdZKWkPhLgI5Bm1zd0EC/EgZHtBXptm2adUqqmWce/L6nA2hDUOx23Rxn5LoaFdE54QFEBJ9nJN6B7PuOuO5w8QVRYsSyWR6GBiLM1OJFmqeFYYLEMG52Nq+ZxGlk6p42nueebPucVXcs1yXuJReJI+lVxTcCAIAgCA4Byrzb+KzC99xsTO76trrD83vXW0q/8AyRq2fiIitgrLmoZNhQjNQkZRtMbdu0tvvPaPi79lXD8ZKXQjKvKwgCAkPJ5RibFKZhGQk5w/4TXSD3tCp1Et2tk4LMkfRi45tkArSS2pibG90j6noFovulp49WRK4NuWrIJNty5YPT0pJ0zlJJKHPPdMvq2zvdWQRwueXvZvOaR0S3PQ63sVmxWSdsIxzlr5EanVGNFk5pYT5ecmzw9pGJ2Oopmg99mLYqWNZj/T/Y1LmnoMr23+pKV1TikLxgUr55HSU1U5+9YuYOid0Bt29mQXG1HBdsnKEm/d0PQaT0iNMVGyCXh9efM0tFHTmSQPgncwHoNZ5bRc+X26LTrVe/LejJrsl1XxOja7uHHdnFPu30fwJdsoYRzjYYZox0XHnRrqOiuroXBbyhGS+JwtpK17srJxfb1TeTwte0seLtcCCOsHULelFSTi+jObCcoSUo9UQnHMPgfOyko4gJL3kcCTu9hz0AzPoC4upprlYqaVz7vwej0eotjTLUaiXLsvP77fMz8bwumpIWvbTCSxaHuLnA2+9YHU+66v1NFVFakoZ8mrpNVfqbXF2bvXHJfQgHK1SQtFLLTZQStkNh95u7Y55g2eR6F3NlVUJOda69/ccjaN2olLh3Po+nLqc8XXOaEBRAb3ZR/Tkb1sv+VwH+pVW9EycC7FG5lIhmlkVpE80MF9PLuPa+9t1zXXGo3SDf3I1lYB9Ug3F1wTeKoAgCAID535Sv8Ai1V+OP8A6ES7Gn/wo/vuak/xMjSuIFzEMmzw/VQkZRnbR/1DPx/6SoV/iZmXQjiuIFEAQE35HR/7oOyGUjv6A/crW1n+H8yyr8R3dco2jV4Ph74XzueW2klL22J0N9bjVa2nplXKbfd5NzVaiNsK4x/8Y4Zl17JTGfk5a2XKxcOjrnf0XVtqnuvh9feUUutTXFTcfd1NXhOE1Dah1VUvYXlm4Ay9uGZuB1LUo09qtdtrWcY5G9qdXTKhUUp4znmbt4JBANjbI9RW8+hzV15keGE4j59+gLn+j6r+r+R1fS9F/R/Mx6fZysjc57KsB783EMzJvfP1lVx0V8ZOUbOb68i2e0dNOKjKrKXTmbPDKCsZIHTVXOMseju2z4G62aab4zzOeV4waeov004Yrr3X5ybgrcNAikexbQSTUPzJvuixzN8ySbrlrZiT/Gzty2y2kuGvme42Mp/tSyuP4m/7VL+WVd23+/gV/wA5uXSMV8n/AHIDyo4S+lpImueHNNS4sFs2tMbtT22bcdYXU2Pp5UucW8rsaG1NVDUOElHD7+85mu4ckICqA3Gyv9ef+W74tVdv4SUeplYrqVGBlmilVyInkhgsl8k9x+CyuofQ+rYPIb3D4LgvqbxesAIAgCA+eeUuM/0tVZaujI7uZjXX07XCj++5qTXrsjO4eo+oq7KI4fgqxZBuMOp3lvOBvQB1yWrZqao2Kpy9Z9jYjprZVu1R9VdzK2hF6dh6nj3tcrK/xFUuhG1cVhAEBLeSqp5vFYReweJIz6Yy4D1tC19Us1MsrfrH0AuSbQQBAYkOIwvkdC2QGRvlNzytr2HVVRurlNwT5rsXS09sa1Y4+q+5lq0pCAIAgCAIAgOVculV0aWG+plkI/CGNB/U5b+hX4mUXPojk63ygogKoDc7Kt+uceqN3vLVXb0JR6mRipzKxEyzRyaq0ieSGD1gopZg4RMLiBna3G9tSqbtVVRjiyxnoXU6a2/KqjnHU+pYR0R3D4Ljs2C9AEAQBAch20gLccc7Oz6ZjvUQzL8iq2m86D/2X6m/sb/vv/V/dEXq8UrmvfuxdAOdY827yQTY3v1KjT7P2dZXHes9Zpct5dX2xg3NRtDaELJKNfqpvD3X0+OSNl9yXHUkn1m69TGO6kl2PMSlvNvySjCj/wCid+I/Fq85qv8ANa/gv1PQ6b/KrPi/0LcTbvUjv4d13qIB9xK7seUzgPoaykwiNzGvkqWM3xdoyv6bkLn6jadsLJQqplLd6vt+SZ0tPs2qcIztujHe6Lv+bRjYthrqd+643BF2kZXHdwK2tBroauvfisNcmjW12inpLNyTyn0Z40NK6aQRs1PE6ADUlXanUQ09Tsn0RTptPPUWquHVm6pqVlJK2ojqWOlp3tkLLWJ3HAlozOZ09K5lW0brmoypajLkn8e/RfE6V2zqalJxuTlHqvh26v4H0J8ra6Hn4hvtMfOMDftgt3mgd+XrVVjcE3jmuxrVxU5JN4z38EMwvGJhVTSCCSQuuDGCbs6Q1FuFraLiU6mxXTlut57eD0Wo0VT09cd9LHfyTsLuHmzQYZUQOrpmMgDZQHb0m9fezbfo8L3HqWhTOt6mUVHD8/Q6mortjpISlPMX0Xjqb4kAXOi3zlpZI1JtW5znCmpnysbq4XHpsGn3rmPaDk3woOSXf9pnYWylGK41ii32/bRIaWUvY15aWlwB3Xai/A2XRhLeinjByrIqEnFPOO5q8Xx3mZWwRRGWZwvug2AGfGxzyJ9C1dRq+HNVwjmTN3S6Hi1u2ct2K7mTg+IvnDucgfE5hAIdoTr0Tlf1cQrNPdKxPei4tFOq08KWtyakn4/Uya2rZDG6WQ2a0Z/sB2lW2WRri5S6IpqqlbNQh1ZH27VvAEj6R7acmwkvf02tb3rn/wAxkvWlBqPk6j2VF5hGxOa7ftnKOVTF21OIuDHXjhY2JpGhIu55HpcR/KvUaSOKk/PM89dnfafbkaHD8MZIznJJ2Rsvu5639JC1tXr502cOupyeM8umPzN3SaGF1fEstUY5xz65/IriuEmBrZGvD43aOHdcdeR61jQbSWplKuUd2S7fvHTuZ12znpoxsjLeg+j/AH57M1zG3IHWQPXkulJ4TZzorLS8kvwfBzTukLnAgtaA61uJLhr2NXnntriwXDrbl48LznB3v5LwpPi2JR89MvxjJq8UdmV3YdDhSxkpDgl2CWeVsTTpe189L3It3Ll3bXxa6qK3Nrrj/hnWp2TmpW32KCfTP/1o1ddTiKQsDw8C3SGmea6OlvldUpyi4t9mc7VUqm1wUlLHdG92INpJCeAjPvcuJ/ELwqvi/wBDtfw8sytXuX6nYn7W7r279M9sLj0XuuCR94NtYjMcVovaOJLeg1F92WLZO9B7ticl2X2zn9CTrpnHCAIAgOecotFatpqgDWGaMn8L43NH6n+pau0Zf9G4/wCpfZnS2Ov+rz/pf3RzatmxAve1ofze84D6ttt25Azt1K7S1bMVcJScd7Cb9Z9fr5Jaq3absnGKlu5aXJdPp4I/Yg2ORGRHcvQqSkso8804vD7Eowq5oX2FzvHTPi0rzmsajtWtvwv1PR6SLlsqxLy/0PaiAexzDo5pb6xZdyXJ5PProYk2HMpo2WpjPK/UkEtByyIHf7tVw46uzV2zUruHCL6Lk317/v4Hfno69JVBxp4k5d3zS6dufy+5TbH/AOG4sbP/ANCx/D3W3Hu/3Gf4h6U/CX+002FPlbK10I3ni5trcWzB7F2tdGmVDje8Rff39jjaKV0b1KhZku3u7/v9TffJ4qtkj3wOhlaLlxBAJsdb2vpncXXBV1ugshCFqnBvGOrX3x7u3uO7wadfXOc6nXNc89E/tn35WfeT/kc2jE1OaGR31sAvHfV0JPD8JNu4tXX1lW7LeXR/c8/TPKwbXAJmR19TzjmtuX23iBf6y/Fec0s4w1Nm88f8npNbXKejp3Vnp0+BMV1zgkTwT/idR3O/zMXJ0/8A3k/n+h3NX/l9Xy+zJDi8bnU8rWeUY3gd5abBdDURcqpKPXDOVppRjdCUuia+5oNjMRp2U5Y+RrHhzi7eIbcHQi+uWXoWjs+6uNWG0nk6m1tPdK/eim00sY5knhla9oe0gtcAQRoQeK6cZKSTXQ40ouLcZLmjQY7QQSzgtqRDVNaLZ2uM7XzGeunArn6mmudmVPdmjp6PUW1VNSr3q2xsris0rpYZnB5j0kbocyNRkdLgpodROblCbzjuNpaWuqMLK1je7Pt++5ftvG51IS3Rr2l3dmPiQpbSTdHLyjGyJRWpWe6eCMYxO5lBzvywyizGtp2sG8XZWYbG7QLXvbRq16qXdBKV6Uffjl+ZuztVNzcdO3LL5pvn7+mOZx2rm35HPtbecXW6rm9l7DT1cKqNec4SX0PLX2cW2VmMZbf1NzQ0EbKYVDojM9xsGi9gLkZgd3auLqtZbZq3poz4cV1fd8s9/jy6HY02jqr0i1Mq+JJ9F2XPHb8+plY5f5Ey8YjO83oAWDcnZW4LV2YktozUZbyw+b79Da2nl7OhmO68rku3UjUHlt/E34hens/A/gzzVf44/Ffcn2IPs1eb2DFcGT9/6I7+32+NFf6f1ZE66TNejS5HANltTA+URyRNL2WPkgnyrEGw7F5vYd1dMrK7Woyz35dM5PSbbpsuVdlSco4fTn1xjkiNzwuYd17S12Rsdc8wvSV2wtjvQeV7jzllU6pbs1h+GSLYf+sk/wAP/M5cD+IelXxf6Hd/h/rb8F/uOvcoP9TH+M/5Sudtb/Dj8f0NjYf+LL4fqSiPQdwXUXQ4r6lyyYCAICIcqcEhw500RtJC5slwAejfcfr2Ov6FOuqu2Shasp/cyr7KfXreH+/JxH+nKr++P5Wf7VtfynRf0/zf9zP811v9R/SP9jAc4kknUkk95NyuhGKilFdEaMpOTcn1fMzsPxGWK4Y6wOosCL9ea1dToaNQ1KyOWvivsbWm1t+nTVbwn8za4VNaytmjWRbjtfUQv3WSERvG8MhccHAG19c/StNbN0tk+JKHP54fy6G7HaWqrhw4z5fLK+Zo6mskkDRI8uDRZt7ZDLjx0Gq26dNVS5OuOM9f38+xrW6m25RVks46fv5dy2Cd8bt5ji13WP8AzNTtphbHdsWV7yFVs6pb1bw/cZVVjFRI3cfJ0TqAAL99gtSjZmlpnvwjz9+Xj6m3ftPVXQ3Jy5e7CyWYTiUtLMyogduyMNx1HgWuHFpFwe9bs4qSwzRTw8o7tgUuH4vEKoRjnchKzeIc19tHAEbw6ncR6h57VbOqc8zj8/J19PtG+uG7XLl8nglYCtNUxocOhZI6ZrLSOvvOuc724XtwCqjTCM3NLmy6eosnWq5P1V0RlK0pNXVbO0kj+cdEN4m5sXNBPWQDZas9FROW848zdr2hqK47sZcvkzYQQtY0MY0NaMgBkAtiMVFbsVhGpOcpycpPLZh4hglNOd6WIF33gS0+kg5qm7S1W85o2KNbfQsQly8dT3oKCKBu5CwNGp6ye0nMqyqmFSxBYK7r7Lpb1jyytfVRRRPlnc1sTWkvLvJ3eN+vqtxVqjvergp3t3mfPm0O1Dpp3upW8zATZjQOlYfaJOhOthpp2rYr2NpY85Ry/nj6Fstr6trdU+Xyz9SOuN8zquqljkjmt55szKPFJ4Wlsb7NPCwNj1i+i09Rs/T6iSlZHL+n2NvT6/UaeLjXLC+v0yWT18z27j5C5t96xtrnne1+JU6tHRVPfhFJ4xy8fYhZrL7Ybk5trOefn7llFC6SRrG+UXCx6u30arYljDya6zlYJbi0/C60dNp66Vu1rCNrUaiy+W9Y8voRiqfcreRrMupcWniG6x/R6iAQO6+i09Rs7TaiW9ZHn56fY3NPtHU6eO7XLl46mLU1D5Hb8ji53Wfh2LYpprpjuVrCNa66y6W/Y8s6HyM4Myd9RLK3ejYI2AZi7yS69x1AD8y0dpVV2KKms45mxo77aXJ1vGep1yvw+GcBszN4A3AuRnpwK59tMLViayX06iylt1vBkgK0pKoAgCA8K6lZNE+GQXZIxzHDscCD8VmLaeUYayfMFdSPglfBJ5cb3Md3tJF+42uu5GSkk0aeMcjwWTBewoZNjQy2KhJGUbevp/lEFh5bek3t6x6R+yqT3ZEmsoii2CsIAgCAz8DxmoophPTP3XjIjVrhxa9vFv8A4LKM4RmsSJKTTyjt2yHKDSVwEbyIanQxuPRcf/rf9ru17OK5dumlDn1RsQsUiYLXLAgCAIAgNNtHtRR0DN6okAcRdsbelI/8LertNh2qyuqVj9UjKSj1OH7ZbZVOJPs/6unabshBuOxzz9p3uHDrPUpojWvea05uRG1cQCAogKoCRbM0m611Q7jdrP8AU79vWqbXn1ScV3LMSnuVmKDNLK5WoieSGAgPoLkwwn5LhsW8LSS3mf19O26D3MDAuTqZ71j93I2q1iJK1rlgQBAEAQBAcV5ZsE5qqbWMHQnG6/slYLfqbb8jl09HZmO74Na2OHk54tsqAQGRC+ywzJvcNqrKqSJpmFtDh+67nmDoPOf8Lj+x8VKuXZkZLuadWESiAqgCAoUBJ8C28xKkAayfnIx9iYc4Ldjrhw9Bt2Kmenrn2+hNWSRMaHljFrVFEb9cUgN/5XAW9a1paLwyxXeUbIcsFBxp6m/4Yf8AuKHoU/K/P+xnjLwYlXyxwAfU0cjjw5x7GD9O8pLRPuxxl4ItjHKhiU4LY3Mp2H+7F3+l7r+sAK+Gkrj15kHbJkMmlc9xe9xc85lziXOJ6yTmVspY5IrLEMFUAQBAZeF0Lp5AwZN1cepvj1KMpbqyZSySWumaxoY3JrRYDqAVMVnmWMjtVLcq5IgYbipGC1DBuNkMFNdWxU1ugXb0h6omZv8AWOj3uCrts3IORKMd54PpQAAWGQGg7FxTcKoAgCAIAgCA022GBtr6OSmNg8jejcfsyNzYe7gewlWVWcOakRnHeWD5tlicxzmPBa9pLXNOoc02IPaCCu0nnmjTLUBc0oZM2lmsotGTf0k7XtLHi7XCxBVMljmiSI/i2HOgd1xnyXfse1XQnvIi1gwVIiEBRAVQBAEAQFEBVAEAQBAEB60lM+V4YwXcfcOJJ4BYbSWWZxklkUTKaPm25nVzvvHw7Fr5cnllnQ0tbU3V0URZrJHKZE80MFEB2zke2e5imNZI362otuX1EI8n8x6Xdurm6uzelurt9zYqjhZOhLTLggCAIAgCAIAgOQcsWy+4/wDpGFvQeQ2cD7L9GydxyB7QOtdHSXZW4/ka9sf/ACOYrdKQEB6Mchkz6WosoNGTeQzskYY5Bdp1B+PYVU008ol1NFiuEOh6belF97i3sd4q2E88u5FxwaxTIlUBRAVQBAEAQBAEAQBAZNBQSTu3WDLi4+S3vP7KMpKPUylkk8EMdMzdZm4+U46u8B2KhtyfMs6Gqrau6tjEi2auWS6sIniShgtQEl2C2ZOI1YY4H5PHZ8x/hvkwdriLd1zwVN9vDjnv2JwjvM+h2MDQA0AAAAAZAAaABcc2y5AEAQBAEAQBAEB41lLHNG6KVodG9pa5p0IIsQsptPKMNZPnfbXZiTDakxOu6F13QyfeZ90/xNuAfQeK7FNqsjnuaso7rwaBWkACgPVj0MmZTVJCi0ZN1R1/AqqUSSZ41uCRy9KEhjvunyD3fd+CRsa6hxz0NBV0kkRtIwt6uo9x0KuUk+hBrB4rJgIAgCAIAgKIC+KNzjutBLjoALn1BG8A3lDs8fKqDuj7gPSPeeHo9yqlb7JNR8mzlqmRt3IwGtGgCrUW+bJZwaarrCVaokcmukkurMETxJQwUQGThtBLUSsggbvSvO60fEk8ABck8AFiUlFZZlLPJH0Vsjs7Hh1M2nZYv8qV9rF8hGZ7uAHAALj22uyWWbcI7qwbpVEggCAIAgKoAgCAIAgMTEsLp6loZUwxysB3g2RocAbEXAOhsSpRnKLzF4MNJ9TW/M3CvMKb2LPBT49ntMjw4+B8zcL8wp/Ys8E49ntMcOPgfM3C/MKf2TPBOPZ7THDj4KjY/C/MKf2TPBOPZ7TM7kfBeNlMNGlFB7Jngscaz2mNyPg9G7N0A0pIR/ht8E4s/I3V4Ln7PURG6aWItOoLGkeqyxxJ+TO6jF+ZuFeYU3sWeClx7PaZHhx8D5m4V5hTexZ4Jx7PaY4cfA+ZuF+YU/sWeCcez2mOHHwPmbhXmFN7FngnHs9pjhx8D5m4V5hTexZ4Jx7PaY4cfA+ZuFeYU3sWeCcez2mOHHwPmbhXmFN7FngnHs9pjhx8HvT7NUEYtHSQtB13Y2j4BYds31bMqMV2LnbO0J1pYj/I1Y4k/JndR5u2Ww460cHs2+CzxrPLMbkfBYdkMM8xp/ZM8Fnj2e0xuR8FvzOwvzCn9kzwTj2e0zHDj4HzNwvzCn9izwTj2e0xw4+B8zcK8wpvYs8E49ntMcOPgysO2foqZ/OU9LDE8jd3mRta6xsSLgaZD1KMrJyWJNsyoxXRGzUCQQBAEAQBAEAQBAEAQBAEAQBAEAQBAEAQBAEAQBAEAQBAEAQBAEAQBAEAQBAEAQBAEAQBAEAQBAEAQBAEAQBAEAQBAEAQBAEAQBAEAQBAEAQBAEAQBAEAQBAEAQH/2Q==", DB.Redis: "https://assets.zilliz.com/Redis_Cloud_74b8bfef39.png", + DB.MariaDB: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAPkAAADLCAMAAACbI8UEAAAAllBMVEX///8ANUUAIzcAKz0AJTkAMUIAFy+9w8YAJzoALT4AMkMAHTMAIDWKmJ4ALj+krrPz9fYAFC18ipBkeYFyhIuXo6jr7vCCkZi1vsLGztEAECsAFi/P1dff4+Xy9PWPnKLi5uhJYmyttroWP05Zbng9WGQxUFybp6zW3N4kR1VOZnBsfoYhRVMAACVecnsAABsAAA8AAB5kuVq4AAAQJUlEQVR4nO1d2WKrug4NGBwCDkOakWaAjCRtc875/5+7eIAw124hbrldT3sHClrYlmVZkgeD/zusx7fzcS1biufDUSygm/ZHJFuQJ8MZqaaCARayRXkmNmPAeCuKfZctzXPgzqPl5ABBwltR9FC2TB1iM3R20+NJQTCGpWZoY+Y32eJ1A3c4OXuWqhnA1nOEU/Syt7v70ENAryScYjSVLWbrCKaWajezJrp9JlvQljG/e+Bz2jG0pWxR28XC5+OtKFavLJm1YnDyVhRPtrBtYuFXq/Eq2D2azt1Q5eYdD3NHtrytYa7wjnACy5UtcFvYqJ/M33mAiWyB20IE+Yd4r5p8gwSJ92WUzw0x4qA3iv0gNMYVXZEtcFsIhbS6oqtz2RK3hIXIPB4TB4FsiVvC0BciDg59afG5JURc7Y1yG5w41uIpTH8sW97WMEMiPR30Z39h4wkQh31yvZ35Z3Ib9ckLc+Gf0FC4lS1ti5hDXt6md5EtbKu4jTiJA71XXrfBkLfJtf5M4hQm5wrNepMtacsYczpaYX+sFwpe9Wb1jfjgyKfetN7tn634mtw+yRa0dbzzWW+oL67GFA7fSgXtZQvaNlw+4mZvPG4ppnyuN6NfNmuMDacHCvbF85SC0xGj906xLzl9b/3r7CNOgx32a4U2GEw4DXZTly1py4h4fW+jF9mitowrr+9N65kZM+P2vcGNbFlbRcDvZzZky9ou+P3M9lG2rK1ix+9n7tdszq3XY1j92UqKoQiEhfiyhW0TR4HoCP0sW9oWMRPZK+9PyNtgsBKKjlD7s4G4eRUhrsBAtsBtYQ7Egt402QK3ha1g0Ftv7Bj3XSQeJobRkzyVrSJIXEEr2TK3gsgQ6+oxvF5sMSw9MeUWw7zKFroNTLiDQh4APdhIDK78CUkP9MAfM4PCQxzj19sx63exQOYEpi1b8u9hfve+1OC/3Y5xF1AsZD+D3+yPccfoK5qN4fdmnX6Pd2zBySbwRQRT+C3eiv07SymsQu5U6jr8xmE+nymW6OKkjN/ndl0doWDSXQ1z2UTEsBnr6PvNjfGrZvP55QoF3U31+D3J5Vvn5IuvwOvxS4z2uRP6Wou0f0kU3Nw5eVo7g/uBn7/HsJmdYZudPMEPd8FF40MntJWfPaetJsBqTZMX8XPntOEb6o52DPVHOqLc5R2h71rlnwD+wJS85d1SeVPMvoyfVx1o9WK1ZJw244d5XaOJhjpvbYqf1NmD2eHrHjVR/BynRHA5e62sPDnxQ3Kt55durLQm5rI5YyzDZ9P+ETa7uzPQs2krPyGYf/z6Pe/pFyE9CG5vSOEt3XKdnwSLlrUHuYHdzuszTLVq4jIzrt1QpLZNy5CZmbfSnmSkVkFmiMhOoORq+/ACacSPEnt63OTSsrTc69PWJZXwZY3yuS5NpxNIG+UbGcZqFlBS1GNkydRtMVRJIb4b2cTNgxzigTR7NYEnJ1jA1TsZ4wJTBZBUG+raiVa3J9zUTVMO8Xsna1K0XHGHfvpy+rojVnKVE/p1sNdqr5q5d6KdFOLBF8LPOYCWg1ltXzIP84xmAZI8zQL1NwVggsGgfpwbl0yKqi1pQht/LQ77M2An6rFWcXruIB0J+kjOrsparKA0N3DM7qHOSBjdB4NkjOlGIIW429H+CTHJajUnXKclAXVNDvHBqSMXjLFrUJ34s0T0s4xMSUvTaf288z1g98qwzs+hOvFFol60s6QF2qWTmVxhta52dapdjS/iud70ZC3JBcvlC4DsiIY1qp34lneGYoChJOIrkRLiQjDJjkHddEliA6aokzqu68s0PJ/Pp/tktl/XqZB1N7Ybhoa9DOuaoUS3S8/TDmbxHbDwyYsxbGCoFgTn6WVV4r9+5ZvPTN4ixJk/AfgFdbYr3U7oQKVHilYUVQcGgijmP4ySDx1NP2lxG2jIgp5nKIcD0HwPQgvhoyx5PgMkcQ81VrHR1eG2+4862TB/C/r69Xw2vcagNluDRrjYrzaZDrmdR6v9bHK8Gpalao1fQCNehrrqxV1VcHU+VdemqdccQcpoI/W+b6rS5UYrZ3G/Aoi/QPwJCg8zEV15OdWdHXUUtb/+55vGqAGPvFPNNhrux9PwalrxSLAQQTw+kv3QU2Vn19+7IT5wHYFDGMvQtPFXNM82iKIVxjoKkt9qqqF16XwZf3lT0FBbTBNbVCqSbvdR1qMv+ROB1apLqNKM6drR6J7Fe7zuTVvVuftK5l7n2QovorvB6NRyNFalU0J9wjn1O6G1CGg9JKlygWo/JeRryJ8danovrRsXVZVNTe05a/FtyFnawlDaH3zLqib3nxbeOLxyeFV1r4uxVzXKvWcGMV8+1fHqtYuGqDpICj41pbzamnjAhp3s1lcdRdDNm+owbPa5mFbYjd/zpfzBvadunH1yNqMBOhp46/IH959bPaGxZvaoE81GUKrpasLnehqnDept5N07c/Aviu999uFoy3pvE/CP3clSql9shc/dTKgf5AacBh2+uDCV208/BrDm3A9TU3edNkF+jJnWKejybRW4VC7WTNXuuAXyE6kBnp6csa0a5DpSuq7Zsc04vE1DRvTLW9mWsK33zg1nV9Efnxl0O6yqMS/ZEsALn1C84TpKmts7yUlCykdjmQChxTOOuTiRnmYb8LALnvC6KqQ7WqYNVGhOn1OrI4SGiqAaziQeJhL5lmUhCyIlXOyDJ710P5k5y9qN26dhHSOa96JA8B/+8Ic//OEPf/jDH/7whz+IY776MUdbjjEWdXG0DrnanrAvPvJEw/IX4wJ2zjKqfMaqdGt882W5rmQ3/wAxYI3Xb+sbDVfFscP+fCBYvs/CMmRhGKrlvU/KnhPHAyXENyPvfRyUbl7STeuaw7l2xE3T3rHadENBsD6CQZxVGQBgY2+ZpRf3mB0tFlbNQzOArpiGXwqpY8fI1wT7MP+UJyRpA2i0q+AptrEQ9n22e2A8uQFc99zUjLzYMXM9HC5z2F8mVxh/KaN4MBHbwqzOYHdYFENr5XPf8Kc0BWvSGlVluoOLgmLuVi5DM2ZeeR50cNNKgyxNkbKqtmyT3a7WyhNtgQqMD0EPfiXzAS7QNYpFy8YC1zGnYQkw136pp7lqLA+T/cX2qpC5l8lYdKaoYz5ww1j6UaYX1zN31WL07EuacldRXO4RcS312LJa5oPBMb5mPLjWM8dE8yP6EWCPkz7ziB6nViJZiVIYDcxJ7sejFzcwx5sJ2QlsQ4c5oe8V55o76Q+k3SvqLK6dycv9ZeGwyPM5QWoy0P/SnhI40/jGPX78lv6ef1KwHL/c79PZkP516Z4m5lhPPcL+G5jjYZ1l7tBhfsXUQSEEhm6zjUK9QvXPF2psX4xGI2BYr1M8cH0Y4yMZSdt/8X//wxbQJvS1+E4AcZ96+cC//5MT6d1TAb7BQP4Jd603cs+/fMxJCFuqnRuYXwptTppVv1FzxsoPZhK5op8jsqeeL8E2yx0aA7CVMMp1DZf0JXyQpeOzYAQVizclt2VmtbWiZiKSdOscMAsDcjLHGT/pQG1g/gbyetpgApLy4vkjbGhgHloOSNPnDt29FQMMtPMgPyhS5rN0f9Z3K5g7xUwKG0YLIeYkmTN5YANzLd+pIyJgbKcQK9UcZW+94IGAj2ojGn6UmRFCZtmZhuWpHlRtXPLhUMX8Qs40BhpCmkZmzQLzNGUKIGhBiHPCTPOoCzHH3Tg5fKd5Ps8azTRFKu7lbrlhidrD/ZV+lcfBF2PW4trBwQM8WIbQZHv/BeZgFptChvW2Hw73032ZeeQnDT1Zxz1iu5q8jphK5WeOszmTQt51zDe4sfzsM0hFB2LDkMGlZ4wCGnGNh/6KDvSAXQiYuJmQ1HVyoGuBuRITzyfQ5pmzdETjEQDmHpOoX27meKCDccpcv61XWQyHziK22201X/mZ2Gg4z5/xsR72HZGKMKEs0vrozPaB2Qe5h8o2V0iBhFrmzEQEOUNiogkyx5UnEuuMrNVQHqoWK7HDS95ipqUL6HLkhpvtUQucZhzThiYfIelFLm1yIz/Bszi6EnNYWOrkmNMaIsUDZFhgmhjztwdzs7hAxxmh73cn552ggey0tByjmtjU9EPQ4y6IsjV1eoGt54tpomNQxbxU0CrHnN5TPKqdGVdfZW6eJ3lMj9dYCdtGLmL3lg7zQdKy7BG087OuSvNq2Eehc06JEk2kLTIvnYSUZc4SzUs+kZOYbsevSt6LNVxF9TB3eI4pqJk1GWGUaAeq0ljRf6rw2MxPw+TYy2kXLTYUC5EuMC8v8bLMae8pH6PCeiIvc2y/GrsH8+pZ7QKTZGeMdWaYD5JpbPwQPDUK3/EVmyoi2iKolCdKYumKs1rJ2s8yp5Zz0WaO5xJViDnukUlDNFgyOy2TDERjofzkGjFdSAGP4hROCzlpGeZWaY09qWBePoa9zLzs9GCzKC/zsfEQp4E5bth0gU7WIpkeqaademQq2bcxs558MaIbKk4snAozp5mYFW0uxjy2GWiZhk+YxwoqbcvcMB+wJQq+SmV6qB6qvrRLylDRSuP8VDHOm5lThuVDNWhf4GWOh2zalk3M8TKVVUhd5Yd5siyNR/e7XuiGRAVQGdnoLGlQVKHbm5lvqWWgFu+hbgFe5ljjesn4bWKO9QGba6gvPetnIuaZfqKaL7supaHAr/ifbLotzues64owZxkzxWnCpZMdJ/MVzB481MQcD1nG/FQY5klKhU1Uea5VqZ+SKvT3SicNdeuIMacatuj0YHXx+Ji7WCP5qbptYo6LMtHezoTLKZgTsyeVwm4AbWg6a+7pUi2fWHSn4nIyZ4ensHB6IydrkkjDxdzF1Ucz8fBNzOP2ZBqOKpj8HsIqTWMpHF2V3WRiqxOYEebIlhmfMqcNmqzVWF1BLWPMOOlSh4N5cMB7JxkV2cAcb5ow8cYZoz1Fmkhj5Zca90wvTVbV6LwkfxzMDMB2aj5lTnWLfdzRHvNOVyfAmAXkr5bYyjR5mY9x2rSabaJ65nsskE/eQu30on2ZVDvQC5tQVKOzDrJn1HXVs89XhA+w1t8rfTIl5my9YyMq35ydG2oa0LqedQ/75IyzWcG8uO3nrsjB2YWCaXXM1zfcmS0qDvXClGoSjegXR4U30fVs6gDwH+kX5H4dBXxtPmcfLVFrgZm6Mk36KM+p8sPpp7eXDMIDqWZkqoUkeLJWGy/ymIQaPuHN9Jlw1DpWizsITN+Cws/Uh5H2kOiQK2VtmJta32sBN8bUY8tl9+5lE8VGvlPpe1X0URZ4ya0b8FDsCZXrc7blfEioLjy8ufxRjCZwffyzXxL5BvHv/6W3OweojeJWMm2AEO4KEMXX0763/Yc8phxxsFU0/Fejj/STr2++BvBvOtD8YzyjvBHJMj55DxW2xFWEIDztytthF794J4HlKW+PJl4NCUp/TH5flrbRInL3MmPdRM40vB7OxzF9Bn1cas4vyX+rsjkvt/fD6S3rItkuF8fz4RpOyDbMYE1f9bg+LGG1DiqeHM++5Vvx3VG2ff8HpeANMZ4rEKQAAAAASUVORK5CYII=", DB.Chroma: "https://assets.zilliz.com/chroma_ceb3f06ed7.png", + DB.AliyunOpenSearch: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAOAAAADgCAMAAAAt85rTAAAA51BMVEX/////agD/ZwD/XQD/7ubykWXy8Oz/7Ob/k0r/kUn08vD/h1DVgVL/ZAD/YQD/bQD/8uj/+vX/9+//ei//lV7/eRP/3cn/cgD/jEz/8OT/hDn/s4L/WgD/pW3/gTH/5NP/vIz/poH/g0X/nWH/t5L+j1THuavd1Mr/r4r8xKv/toj/59z/dyb/x573m3T/eAD/z7f/wJP/3cv/up/u5t/uhE7/zKv/chf/l1X/iz7/fyP/w6L/o3f/17zvwaH/q3vVyr//nG7inHLZr5jw08XgpYXaj2D/rXb/omT/nVjjwqnTuKP0roxxvdl9AAAG60lEQVR4nO2d/VuiShiGYaAy9wwIKn5LKm6mVua6fpFZu3uO2+7+/3/PAdu2rVQGZki6ruf+uXi5ZQZeEJ+RJAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAYF8Ypqmq6hEXKkshzhpHqpoxjdB29qA2nc0LVOGhWiwFlyoVq1xFNLkwn3YGthlCz3avy5QSQmQ+aIOlWoNylpEJoXL9zlmw2Y1uWlmd8ro94LIUdIWUIpRMcjenQcWMhaxoguxkkrVZBO2soHqepELHO6ejsRQlt67XZJr7RlNkTVrbMRmNpbhKfrE2i58ktQUKeiy3f6zHQgvJtMUm2OI+yzxntvkYGnZfcCG6YhNcia47sTZUGV3rYkeKLGs3bII3muDCRF8evqqyVARX8QRv2QRvRQvKslJ5WcQUPEzWVUZsgiPxn61MXsxDsy56fHpU2fwkqSq+Nik/MzSuY/CTKaug+CHqGf59tTAqkzhKHLMKzmL4eMnEeSpQ4W6rN6ExdaI+biyHUD5/3H4phjngoTHdDfqoMZxlPKqP07AWwxnU+wTzGVbBTD6OEfSn0TgU2ew+Qmh1ynwLak6rIpv8P/vQTK83v+gL3SqhlGpKvjIMcYttWpW8d5vm32SL3Jf+Q8vmCNsoIUTPtmsnH0/T7HKPpE8/ntTaWZ3/QcIT6xOpcSdmCnpHrp7ruEz3uNux3U6uTgUNWHrnXwttEU0MoVq+dRbqwc9WTPuslRfyWIH0h972BpwH0Jt0mlbv9tTQj+62Y6i9bt1/dsJpSQfrpxQcboTK/WbrnOHpYHhK561mX+Z6uEe9fi0zj/b/3pQr1IsHF44l8Mi9xLCci4NivRB1UhLv5v4oygEk3qCsnVl26TDC6TIc6cOebZ3V6poWxZGqUi9km+RPOWXm9jIxHrjXGJmeO4swKRVLGoRpdL0ply3WxkJOluExz2vFrBxqtNIr6YpZkGh6+c7hvM7xYjt3ZZ19tNKKVGGYg377pZCpa9lvOi43Y9jWp6mssLV1tCbdB/2Z1zZfpr58/abuaWBuwlS/ff2Sugxu0clUOg74G5KtJMjsOWYl6GsNkpJSu/+ETvc86XZjt3fPsGBBwnzXuh8yAbufChii2tW+DYLo7jyEwYLMz1X2xe7nOYFDlCRfcPf+Bx3BdyC4a/chCMH9A0EIJhwIQjDhQBCCCQeCEEw4EIRgwoEgBBMOBCGYcN5e0Gx8iE479Hc9by+YyfmvtkSDXoautw/B6C/uhHjHFIIQhOBWwegnGfIeTjLmv/9E57/Q777t4UKf5iF0NXQyEEw4EIRgwoEgBBMOBCGYcCAIwYQDQQgmHAhCMOFAEIIJB4IQTDgQfPc/r+MTpEf7FghC5fsFqNLbt0AQu6MqAgXpqxDEpLE76CD4V9hKN7G/ovcxu7uzRoIFZVocxBKHI4LSoBiQxMEgKBO96CwSeBjNhVOUA3c+JU2DX/ogcr85vUrU6aZ3NW32A/VkP8viO9NbLYRoynw1TEJgh6kOV3NFY4vOod9DJAgTqhWOP1mWPYo952gz6ZFtWZ+OCyEiyegqZPQloXq+/PmgM37zCA973Dn4XM6HDKnXXGkYNvpy/UbSJPthxZidL4LF6kN2si4ccme9RkWNFl5KCFX0+3Ev9rCSTG98rytRI+S8XjpqrpoP1ejcscQE/m3CtIfO3CsSeQdJ3ZQMvvhUb1KW75xFDFlPxsDPp+KLOPST8aQBb0qiv9JDs706+RG42AMrpz9OVu3mhHAHOJKBxPd+59OW/ARHRZlXhnwZjsb6Iqdo/LmN673KrWdPR1xgqXfmuVwOrIjNa8kaLAuKwDBc8pBE7erCtuhvlNJ8u+GGdiy5jXZeVHLqb/SHIOxboRm/8nq86tl2N0QvYHfb/jozooOM+7frradjSWkmSpk9pbmsxBFlToq/e8pFLDnbMmG+/+jFErMtkz/raizjiGKXtTGr4Die+k+PW9RCLIsV1FgFY4lqJ/O/2siB8OVQ1hVYBeNIgifPFn4xBF4Ln9BYBePI8ifOsxImS8xoWJTXK8ts5DAGQeq8PIenxBvucVGbDUs+jSbCRyn9ySb4U/hnS/obvlOxyqINtRM2wRPRgqS4sYuyZ4KHyr5WztKmW74TM1JVoQdxP2ufUaW2/Y7NbjM8TGWGNNkEhS7Pp+d2doiGWxZ3x8K4wKLVF1eRNAP7Q9udCRsxE6ZudCxs0Sc6PWNp8M1zUatkvugmtiBmSRZCFW338ph/c54r62S9xgwX2jVDReNa4yzj7+ikWQv3IDozdhqNi4sDPn4x3POavziLXFw0Ok60BQbS6fQhF0zf0vDWiPC7WAAAAAAAAAAAAAAAAAAAAAAAAAAAAMD74n+lRrmptwLV8gAAAABJRU5ErkJggg==", DB.AWSOpenSearch: "https://assets.zilliz.com/opensearch_1eee37584e.jpeg", + DB.OSSOpenSearch: "https://images.seeklogo.com/logo-png/50/1/opensearch-icon-logo-png_seeklogo-500356.png", + DB.MongoDB: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAOEAAADhCAMAAAAJbSJIAAAAolBMVEUAHisA7WQA8GUAHSsA8mYAACcAHCsAGioA9GYAGCoAACYADikAGSoAFyoADCkAFSoAESkA5GIABigAkEkAzlsA32AAt1QAqFAA1F0AJS0AMTAArFEAUjgA6mQANTEAok4Ax1kAaz4AnE0AXDoAOzIAQDMAKi4AiEcAdUEAzFsAv1cAgEQAYTsATDYAVzkAckAAACEAlksA/2oAf0YATjcARTQMLnchAAAHqklEQVR4nO2dWXuyPBCGIQQIoCzuG261q3b73vb//7WPgFRUYFDbiwlX7oMecZCnk0xmJpOoKBKJRCKRSCQSiUQikUgkEolEIpFIJBKJRCKRXIfma3UP4W/R3K7bbIl6979Hve5B/CXuTiXqnVP3MP6O1mREVDKatOoeyF+h6QOqqiod6E1divojFxhJbOpS9F8TgZHEV7/uwfwF1jxahAnRUrTqHs7vo7kDpqawgWPUPaBfx/6k6gHatese0G/TWanHrDp1D+l38TYBORJIgo1X96B+E9Po0RMb0p5h1j2s38Pwp0w9hQ0bFIPr3XOBkcRuYzZ+/YnkCIx4b4hE/y3IV0iCWSNiG9PsF5hQJf1WA7yN4S5P3WjGob644sc2+jrPy/x4m7XwmVTnvURf7G0Ej228+8JFmC7FTbvuQd6CoZcswv1SXAo9T/O3+pOlKHLG789AfdyK4u6K5sOofBHul2JvLOiuqOlDeI7G8/RT0HTYv6ukLyIUc54abqU5Gs/TkSdiaKN/QhvFAfopoD9tb6paMLbiXLh9X/MH1U3IQ3DhzhXdXTU/msJ2gp1IVdwKM9N0NBbL2VQJ146hYlVtWnMgpcgxYn8j0rGivrjEzSREyXDdw66ONb90jsYo4oSnnenlJoxW4lqYdN+aX7oIE8IHUYzYGV5jQp5jCGJE6z64SqCqBnMx3Onle+GPEcVwp+bkwnDmgCCBjfN03Srk0H8CRKeafrUJIyMuBZim3tvV+jgz/Gff+pVbRQIdojeipl+7VSSE6Cvg/oWZ7ylshbzsprkXFS/OoVPk/Qvm5LZJGsU1E9zBqb+6bZLin6b6VXlTFjpF7U0NuAAF/QeiyA3zNPXuwHimC1rxDfOm7z6Cy/Af9AF7dOuWUYL+Ai7DFfQBeUG8EDUDLiKCx8Kkb+DdEb23EBq/eg9+ESJeiO4W3ivG4Bd0izdJ1NewQh1WuMC7EO0lnPza4BeY0+AqmdMH/EmItnHBGIMmJP2PCmc2D1jrUd4MVjj6gMs45AvrkbcPV9nI88czrPAda3rhdmGFy48XUCHtYq3ud+BTQzr4gIsAdIFWIZwc0ulHhY+GSBVWqdHQhV3F0EhrNZoDZxZ0XUmhg1ShDzsR9ml/gikkwdo+pPlw0Ma6Npwkkxe0NuzBCh+rKHxGa0N4M2dbewsrXGJV6FTYzJ8Kr3llFOKdpbBCdaVDV0wwK6xyZrHTd2ClA+1+qLhwuBK+uXAth06x1hMr9NEEM3cGpsloozalA5dpgnsf7rbB2/7lwKW2/sab90GFaDsy4FML0n9oT8AyBkFbMG2B7fmkr7QUuFCD9pBUA2uhpG8ZPqwQb7eCDVWZyKhjuJBC0kNbTYR7acjIhv8NmHtqwONDbh4bykAo4gNED4rIeMFeh7LI8A6rK+XH+MAao4NIIRC9RjsK1pJ3BGQf3mgBLVayxOto4IXID86guxi4z/G9XbkNea96BypFoY1oOFA/Deu6ilveBk5GmJehArkRPgOBmcydEWa88uZL9uQozlP5J4j3Cg6wX7AnX/HfyxTi3isUXqsprWTwxsPy9kU6xZr9pniz0udodh4wkdk31vPfH+yyaRorLMuTeWiOnfKD4GivK72ugDnqTintgw5e20r7q+wDES7olbUJB6+WYn0XK8ScGh7wZoUpVHybuVXysFL4it7PcIrjGjKam4o5L4zs6KCDtkKTxfsqMiIZTQzFKL69R5DHMymaU2RE0htHCsdFZQxxHsdo3xfs6eTZjBQqRQrJtxgm5HXTAndKlp5WfNyP/KbFEdY8fyWSZeRJtE5RqUOk1+gLmoWTNvyCYo5Y7wwZSq6/LFMoyC3nH9xVnopkpeWvUrbCH5FmiXaMHH+a9KjnFtvY1BZkp0hpnz7gHcuID3fzim0kmAv30r6bU47hpbb8oqpoczTGPu9UZNtY4XlXFHvBn/ieY1hnOQQvtfF28FOFovnRFGd1Phd5i7p3VooKRXvKLEU/LWjwMg1vaDhRKO5PQZjWyVJk8ZGE93askL1YQs5RTuukK4HFKbz1faSQ9DsC1GaK8O7DrER2zzc96yi5inZCISoXRdhP2SyDxY/OtTZZhaHoD3rrjxkjhnM+H8151oRbkTKKPLTsq/PBhLsUc3KwK1uLFo6eYxwk7s+VMudTbK0L60YPGD/Z0j50Mcdp8kiHTRAYSWzva2+kp8Q2TEtRdOA3QmDkPK0kWSRLI1aoJUk+G1gCb4THmH78djnp+bEvTe5lsIXbGIF8LS5Y7FgiUaYbux620BskUOGyCOEd3G7b5XU4QtYN+P2VIwx9ywM4uvB50xcJt83wolk0fRdE2mjy5w5vG/ANOP5+HyQjRdCMFyItI2J++OI20hsneG+M3EpaKE2Kik3E2e7XIdo7MbfirfYpBfJH564n7dIIBXik9Dpam71CoV7QvwRNSXqFArNx8cye9PXWoJEBTUxyIQjzvaZbSZ7ko5gf1buR5DKJGO1519GJLyI0N6RJz0ZZY0MaZd+mz9A+dHU7ycuRmF9+vBXrO6CUBt/CtV1UxngYDgaDIfI7Izeh+Y7jNKXQnY/GqXsQEolEIpFIJBKJRCKRSCQSiUQikUgkEolE0lT+B4h2dnif2MTUAAAAAElFTkSuQmCC", DB.TiDB: "https://img2.pingcap.com/forms/3/d/3d7fd5f9767323d6f037795704211ac44b4923d6.png", + DB.Clickhouse: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAO4AAADUCAMAAACs0e/bAAAANlBMVEX/////zAD/yQD/AAD//fX/8Mf/9uD/ygD/6a7/0C3/CAj/rq7/0QD/hQz/67T/zhv/fwD/za9VhqZUAAABG0lEQVR4nO3auQ3DQBRDQa1un7L7b1YVUJkB4++8nMHkHFpq24fUPKbRuMTRmkdrHC15NMfRVbi4uLi4uLi4uLi4uLi4uLi4uLi4uLi4uLi4uLi4uLi4uLi4uLi4uLi4uLi4uLi4uLi4uLi4uLi4uLi4uLi4uLi4uLi4uLi4uLi4uLi4uLj/xn0ft1RF7h5HrSR3w8XFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXF/Rm3rytKZ0ejq3BxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXFxcXGLcZ+pVpL7Sn2+Fbn3KfXAxcXFxcXFxcXFxcXFxcXFxcXFxcXFxe2ZewKjx49mqHXf2AAAAABJRU5ErkJggg==", DB.Vespa: "https://vespa.ai/vespa-content/uploads/2025/01/Vespa-symbol-green-rgb.png.webp", + DB.LanceDB: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAMwAAADMCAMAAAAI/LzAAAAAM1BMVEX///8UFBRPT08xMTHExMQjIyPi4uKnp6fw8PBsbGyYmJiJiYl7e3teXl61tbVAQEDT09NsjTssAAAHbklEQVR4nO1d2ZKjMAwM9xEg+f+vXY6dBLAlNbGtSs2on11BsmVb6kbkdjMYDAaDwWAwGAwGg8FgMBgMBoPB8OfRlkNeZFlW5MPUUmP67lHNYx75vaR+p7yPxTymeoxNnchUCeWwGPlC57O1zPdDit5ja3s//EzeJDfcRd1lZ3RnU8vHeUh1Pw05ubK6rO5O49iw4GBqO/iGFAePn4VvzEDFbBrcfTYsZryH1M6y/F+c3cT7p+TscWJ453xF9/LFO+cr+pcv5BBFb6h12XnD+JJl0zZmYoYUWpFGT+h73kduSLXOe03E2IZcxxd20mc85zE9P2S1VPiZXjAjDtwz2bFU8jdrpPWdl08j0GrBiCwrRX+zSlyY0zmfCKKh89KIhmaTtDA6S8Nu2w3CjlkdzuUx6TMB7jyNjDG5M/R9GR1VcmeA8IiG5GmAvLfjgSyAYkHRl/QngDnzrc78qj1DlFxJ8EztzK+6Z+ScKhrSlzQtkJsBq/cduRlfQ26GAg43QDKqwAOUwIyKS1MAK9zJtoRDCpBCKu+ztSbmWJEVKgSNZOlyOwiWzv7eWuHG0ig0bxJbsRnBX0frpD9Ff1XAVc6PbQhLafTypCiygPSJ9iLvGG9eAUQHY/UN/Gz+JiFIb3abgfJGlWumzBj2Q2rvsVcdssfJ63GuLTl5TH2cs9zGNbU78UcemafS4TKPmI7u5JM7pD66Uw2epP7kTnWn6LJ6apqmTJVJH1SvOxEa5ZBvo4quIczcx9ro/5m2z3+eVY0JcjZHwHNUwPfQuqZ3QX+KxdGd+/qons7zEnlT9Z484INLu/ScAGdTXdkzK2Luq9afoF0+Uv2HYrFfHEJOJOLxA5B3yEVvyFTiPfGxHnXdl4tXN1Ma/Wxy5lFxvGHz3QuPYKueUvJlvtdiaB586QVX7jybsCV5fO00SI8INQIWIyWpcAQeFU6sSTQgqHiJClwpS6PBRY/MNUHXjSyN5sCjQpdG5ogg8k6kALJMKqyzYGpNnlFsvgDOGtG1wg40hNEEThmeAMARltbIHCC0+LFY3jByDVEBgE0Ti39/BDkDUM1IJCMLjCBMKYAeIac00URrc+YHXxZmYc5Amqb8M/LrRBjCDgBkSoEnAPIMhLAUAEhDkMNfVnkwhGkFiBUAF4Soicj2DMw0gUcgCROwaSb5UaE1gBxnUIohr3ABpAmhjJMcIBgLIN40DeBwMKchLQ2Y+0mWLhEkxWK4VCjUTDA9IwTRsrWFKIghFT7ZR3jEAD/4SdkmnV2+OPIaV4xcWHmOr/iJVS6mI6kBtDeXopj25v3iLO1NNO6cugIuPoDyZl93E4lPBYfzh2YUl2Wt1ndenXRA76Miy56OYEkLeBxcgcZt0jrrUSlkz2Z/7+U94UrdDF2e511PpVFlt4vZwj8jx0dFjLAXDgqtvyWxve8mtfJ2P56dIeKnnu7jMid3siM0BE7zW+GcZG4/YOG4UzppTWzBUob3jYXT5e+TPU8vAnjbH7UbNZGjmUhX9h4jR3Ny0EXvO9TIJPHtjb/ldEH6bpMXkHSGSXh/vEHSmeTgXwXctjhbJ2wSH5JopgdfAqzCmVCsrHtCKFaSvzu/AinOJHatxIqz5BDfjJ0tFQWLHGATNd7Ukum7AaA9S4TQSA/Z0AroGOwAqin9rkFIQITgQ1oKkjuD0LORkD7ONFsbk/cD/6rOJkVfrBvw7zoDaZqRYK2NVxBLWUWQ2pdoyioAhdZGxAp5SAFI8Apps2xpATh8B9KiL2ltlB2uZYdVSA3xLdBadrgDXtJKfpYtkCxdrjrhPY6Fn5GEXiV+BnhJW9A916udPxe12s5a7uL8bwS7fAMwKSpBtgDpNmA4jdfOZi5gxb4zpA+EjKIddU56o0qd1/4d/tgHOsEkH6hKohlIhwB8w5Hm3O9KetoWHTt9bVqj7scaN1OPdlSDe/48Twvo6lGuNprTy8L14QWjGV/9hpSkuY/HgtAiy/e05JTIW/fjNijlJ12fZdM0E/Xz5/ZHUuFry2n+mScVX0B/a3J4dMBPBEv3q6/Hk0YD/kZtKtZo+O9WLf1mA9JCD4DoCNXVPRnl44o3TJKn9NnQG/bZCQRcGqime/JZM5xF8vWoUgInfaoFvOOF0knnSxtiYxq4baSiVmXbyJ9rhZYmfWsjAKBjEAp3mRdRID2A7jgkQIA5Ufg+baQGS4Q8TV/FIUoBYAUyJ8nzAKihFLACmZOwziYAUKMPcH0jP5NckYa6Y+UTAOmiTi97mDPf6syv2jPQlALZGdISmj47Q85UoHpGWkLT15uIFZEygPQ8TaTcDFAT039rF+kohXIqOZ/RqJxF9RU7hBRaGwGISwMmu9LS6FAawt5FbwchZdWSCtk5xYkIflK0JClWob1wnnJsgh5Dy7CAlyaUvrNU+VnCm+picFCatC5zTvxp22VN3HvOX52ScLifa/2o/RH55z4NHFoSP+zkvC1C4eFnfF99VUHdjMtfUFbFSP8FJYB2ev3bZakvRhsMBoPBYDAYDAaDwWAwGAwGg8Fg+Dr8AypHSSkTIZFkAAAAAElFTkSuQmCC", + DB.OceanBase: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAOEAAADhCAMAAAAJbSJIAAABJlBMVEX///8Bgf0HyEb/oAUAe/2oy/+Vvf7/nAAAfv3/ngAAxjkAev0Ad/0Af/3/mgD/zJap6LgAx0AAxTYAg/0Adf0AxC//owD6/vuhxv//8uD/3rLz+f/h8P/7/f8AiP3k8P++3f9DkPSs0P//qyDH4v8zlf1fpf6As/70/fckzVhj2ILA78123JH99ej/5cb/vGX8tFb/y4n/+/T1x4r/qTL/1qH/rjT/4b7/xHTv6t//uFHzrEr/79be29Hv7ur/0J3/tENhne5IkOcjkP5CoP8yhOGGtv50rf7F1uvF3v//uWDs1bilwen/xX7tqlVipPDe5evI093h+OiT4aRO1XQ30WWv7MHS9NxQxG2O4qSt17ht2olZ1Xqyzex+3pfo++6G05rL49EXq/lGAAAGaElEQVR4nO3ba1faSACH8QIJJOSiCIIFEtJAgBjE2lW8rFVbKCzduqsi2lYr2+//JXaCtnJVgYSMnv/vVQsenefMDEnmHF69AgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoNOa1wNwlbJWXs++83oU7llLvo3LITnp9Tjc8nojLvN8IMCvez0SdyT3AiGSR/D8i1umifIfe5sh/raPkLe8HpGzyjuboZ48exIrXo/JOe+2u3mBAfJrrwfmBGW3vEW23nAeEVrwenSzer29UAnwA2uzd5n+6fUIZ7Gf3MmGRk/dfWHc61FOZ62c3Knwsvxw3e1GpOvObT+5XX6/e5BQht9SDg4OdsvbyeThzlGItI1dl3RvxAX5t0A8Wzk6OqoQ2Wz8/nX5kWU5tEwDXkf1WQjdj6zPJE0D5LLXVb16Ch0TOp5xUAqRymTStzKpVMp+haLCQGiye1MlldPYYL3x4eNHo1pVoz5OEGMx8R75j8D4oqparX36QUXhUx+hlIzG1vNG1A5gOIbjOEmSfA+QOJGlopDfe+zPpjJasFGLdssebBrEUFI4/t40lflLD9bzKmnjuEnSaCrk5fj6yMKMXq/5RJEh8zZFGy2FPC9nt0bc1GT0gsEJM6TRUUjuxzffJgc+SBOpTLNuRB2o87qQzF5lY38gL6MV3qg+h+q8LSSb7/Pg7KX1vFNT520hz4fk0NHg5kvrhk9wts6bQlIXX9/4e613+lK5Lw1X8uZf2M0bWptBlREmu4xTWdhdmpXj8kBeiqxNxq26uRWSNpnf/Od4cOe9S2sN1dW8ORSSC54sZ3eS5JZl8GGmmVejDn9uzrWQ77bJlZ3DkU+3TTJ57tdNWXh7TGEfwtw92/9+yrdf/XWQEa+sLyT3R/8KrTCvvKkKd7eTya2tw431vb2jSjYeD/CBeDybzVaOPu8sHG5tJZPb73cPxj1f54L/GhJ5OJ2fyQunl2MNQZjw6e7ZFCo5tiaI81uacyxUTk41tm6I3tS5WqikNbZQi8ZE+9xhzgtzxsLmiq7lUqOP6hQlk9Z0NtioqeK0pw5Om7wwGBMEMSZGq4ZRy+fzjUK90GiQf9RqRtUnkjD7rIiGtDtTFN591EuSxHUx3B0v1+J40xc+FyhEIf1QiEL6oRCF9EMhCumHQhTSD4UopB8KUUg/FKKQfihEIf1QiEL6oRCF9EMhCr0kcQwjEHff5Yoxvmg02v2aF/lP93te9ruxZ1nIMYIYixqN4Iqua7nTk5MTJdE/SkVJnZycnjY1nU0/r0LJ/h6eml/Rc9N+u5Dewu6aZIyCPvGcPIdCEhc18kE253acJ4USqcs/dTMlisVOMfH4z9FTyDGcEWyOD0qcWYuLrZvVc9MfiSwt/xKO+E2zfX3Tal1Yl0VqCxny6V8bNXkKKStai63SuX9peSkcDkciEf8w8ip5b2l5kc5CjlGDzfTQekt0Llqlr20zbJeN7BoSprGQE9RGMzX4h4uXrZJJykbP2PMplBiBrM3BvOLlt1WTxE2SRmWhxPiMwpf+rdf5+f3maso6ygoljqvWB654nYsr/3J46jqaCsmdZrUxcCfWsa4jS7PEUVPICZL6ppDr23vkQnftnz2PgkJyJ63m2Uyq77pQtFbb/pmWJi2F5HazOnxZ6Hwzn3ipo7nQfk4QyUNQZuiqZ636w47VeVTIMaLPqK/8GK7rfL9pO7U4PSrkujMXzKVGPr5a5KPF6b65FXL2soz5anU9M/7Z3Fp2PM/1wu5zuSj61E8fGkHtgbhbzk/gdIXCE2aMLEdBjHHVWn1FSz/5ROWrG4mTF66oPjF2d1InML8Jd+d6MdFXreULQfsLbpP+6m90FNqUdDqnabrOsuzKLVbXdU3LpR9diQ+xlqgpdMssJZF+lBa2Jr3Ydw8qbH6zfX1VKpVubFer523T9N++M/kphpvOnn69iNinGGb7qnVhnY07XetcWheLJWu+DY9oP/pZY88a+SHzetG6nPXY0AsXD37WREhb+8ayzg68Huf0iuPnMBL2n7cmP/ykzurok89wpF2ynn+dbfjTNBw2r//7fun1wBzzs38jRpbMmxewMnv1bkSy81ZfyNLs9WsjRsLmeavj9WjccNldpmGz9MIW572iaW++xZeaZystX7/Azder89PrEQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAuOF/FLjti7fsBDsAAAAASUVORK5CYII=", + DB.S3Vectors: "https://assets.zilliz.com/s3_vectors_daf370b4e5.png", + DB.Hologres: "https://img.alicdn.com/imgextra/i3/O1CN01d9qrry1i6lTNa2BRa_!!6000000004364-2-tps-218-200.png", + DB.Doris: "https://doris.apache.org/images/logo.svg", + DB.TurboPuffer: "https://turbopuffer.com/logo2.png", + DB.CockroachDB: "data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiPz4KPHN2ZyBpZD0iTGF5ZXJfMiIgZGF0YS1uYW1lPSJMYXllciAyIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnhsaW5rPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5L3hsaW5rIiB2aWV3Qm94PSIwIDAgMTIwLjA1IDE1MCI+CiAgPGRlZnM+CiAgICA8c3R5bGU+CiAgICAgIC5jbHMtMSB7CiAgICAgICAgY2xpcC1wYXRoOiB1cmwoI2NsaXBwYXRoKTsKICAgICAgfQoKICAgICAgLmNscy0yIHsKICAgICAgICBmaWxsOiBub25lOwogICAgICAgIHN0cm9rZS13aWR0aDogMHB4OwogICAgICB9CgogICAgICAuY2xzLTMgewogICAgICAgIGNsaXAtcGF0aDogdXJsKCNjbGlwcGF0aC0xKTsKICAgICAgfQoKICAgICAgLmNscy00IHsKICAgICAgICBjbGlwLXBhdGg6IHVybCgjY2xpcHBhdGgtMyk7CiAgICAgIH0KCiAgICAgIC5jbHMtNSB7CiAgICAgICAgY2xpcC1wYXRoOiB1cmwoI2NsaXBwYXRoLTIpOwogICAgICB9CgogICAgICAuY2xzLTYgewogICAgICAgIGlzb2xhdGlvbjogaXNvbGF0ZTsKICAgICAgfQogICAgPC9zdHlsZT4KICAgIDxjbGlwUGF0aCBpZD0iY2xpcHBhdGgiPgogICAgICA8cGF0aCBjbGFzcz0iY2xzLTIiIGQ9Ik05NS4zMSwxMDFoMGM3LjY0LTMwLjc2LTMuMjktNjMuMTUtMjgtODMsMTUuMzctOC43OCwzMy4zOS0xMS43MSw1MC43NC04LjI0bDItOEMxMTQuNjYuNjMsMTA5LjE3LjA0LDEwMy42NiwwYy0xNS40Ny0uMDMtMzAuNjIsNC4zNy00My42NiwxMi42OUM0Ni45OCw0LjM4LDMxLjg1LS4wMiwxNi40LDAsMTAuODksMCw1LjQuNTUsMCwxLjY2bDIsOGM0Ljc0LS45NCw5LjU3LTEuNDEsMTQuNC0xLjQsMTIuNzUtLjAzLDI1LjI4LDMuMzEsMzYuMzQsOS42Ni0zNS4wMiwyOC4xMi00MC42LDc5LjMxLTEyLjQ4LDExNC4zMiw0Ljk5LDYuMjEsMTAuODYsMTEuNjUsMTcuNDMsMTYuMTZsMi4zMSwxLjYsMi4zNC0xLjZjMTYuNDEtMTEuMjUsMjguMTQtMjguMSwzMi45Ny00Ny40Wk01NS45LDEzNi42Yy0yMi4wNS0xOS4wMS0zMC43LTQ5LjMxLTIyLTc3LjEsNC4xOCw2LjkzLDkuMzYsMTMuMjEsMTUuMzksMTguNjEsNC4yMiwzLjgsNi42Miw5LjIxLDYuNjEsMTQuODl2NDMuNlpNNjAsNzYuMjRjLTUuNTMtNC4xMy0xMC40NS05LjAyLTE0LjYxLTE0LjUzLTUuNDctNy4yMi01LjQ3LTE3LjIsMC0yNC40Miw0LjE1LTUuNTMsOS4wNy0xMC40MywxNC42MS0xNC41Nyw1LjY1LDQuMiwxMC42Niw5LjE4LDE0Ljg4LDE0LjgxLDUuMyw3LjEsNS4zLDE2Ljg0LDAsMjMuOTQtNC4yMyw1LjYxLTkuMjQsMTAuNTktMTQuODgsMTQuNzdaTTY0LjEzLDEzNi42di00My42YzAtNS42OCwyLjM5LTExLjA5LDYuNjEtMTQuODksNi4wMi01LjQxLDExLjIxLTExLjY4LDE1LjM4LTE4LjYxLDIuMjIsNy4wNiwzLjM0LDE0LjQyLDMuMzQsMjEuODIsMCwyMS4yMy05LjI0LDQxLjQtMjUuMyw1NS4yOGgtLjAzWiIvPgogICAgPC9jbGlwUGF0aD4KICAgIDxjbGlwUGF0aCBpZD0iY2xpcHBhdGgtMSI+CiAgICAgIDxyZWN0IGNsYXNzPSJjbHMtMiIgeT0iMCIgd2lkdGg9IjEyMC4wNiIgaGVpZ2h0PSIxNTAiLz4KICAgIDwvY2xpcFBhdGg+CiAgICA8Y2xpcFBhdGggaWQ9ImNsaXBwYXRoLTIiPgogICAgICA8cmVjdCBjbGFzcz0iY2xzLTIiIHg9Ii01IiB5PSItNSIgd2lkdGg9IjEzMC4wNiIgaGVpZ2h0PSIxNjAiLz4KICAgIDwvY2xpcFBhdGg+CiAgICA8Y2xpcFBhdGggaWQ9ImNsaXBwYXRoLTMiPgogICAgICA8cmVjdCBjbGFzcz0iY2xzLTIiIHg9Ii01IiB5PSItNSIgd2lkdGg9IjEzMC4wNiIgaGVpZ2h0PSIxNjAiLz4KICAgIDwvY2xpcFBhdGg+CiAgPC9kZWZzPgogIDxnIGlkPSJMYXllcl8xLTIiIGRhdGEtbmFtZT0iTGF5ZXIgMSI+CiAgICA8ZyBjbGFzcz0iY2xzLTEiPgogICAgICA8ZyBjbGFzcz0iY2xzLTMiPgogICAgICAgIDxnIGNsYXNzPSJjbHMtNSI+CiAgICAgICAgICA8ZyBjbGFzcz0iY2xzLTQiPgogICAgICAgICAgICA8aW1hZ2UgY2xhc3M9ImNscy02IiB3aWR0aD0iMjcyIiBoZWlnaHQ9IjMzNCIgdHJhbnNmb3JtPSJ0cmFuc2xhdGUoLTUuMTIgLTUuMjEpIHNjYWxlKC40OCkiIHhsaW5rOmhyZWY9ImRhdGE6aW1hZ2UvcG5nO2Jhc2U2NCxpVkJPUncwS0dnb0FBQUFOU1VoRVVnQUFBUkVBQUFGUENBWUFBQUJxRVBvSkFBQUFDWEJJV1hNQUFCY1JBQUFYRVFIS0p2TS9BQUFnQUVsRVFWUjRuT1M5emE0dFNZOGR0aGk1VDMyU0pvYW5CdndFSHRxUzBiRDdDUXg0Wm5qc2tkK2tBY0VOQVlZbkdnanczRU8vZ2lRYkVLRG5hZlYzejA1NlFDNXlNVEwzUHVkV1YzVlh3MWwxN3M2TWpCL0dEMWVRREVha3Vidmp4Zld2L3VXL3FmdS8vcXQvRFllOWl2cjdYMllBOHMrQ0RnZmd0dUQ1M21Gd0F3Q0RaVHh6aXlTa2ZUR1BCUWFaWlh5bXNYcFIrY1RqeWlnR3JBTXNpWG1heVcvUkcrbk1ESjc1bWRiSGtoYllER041Vy8zelRjWFJjdms4MnF2aXlidHI0OHJkaXpnK2Y0MDNyMGJQR0ZaZXorNHhpbnJZdWVUaEdjMWhEcmhrN243Q25DR1p4cjBKOENqRHQyZVdTUnBtdVIxbmordCs5clBuc3dQd3MyaHdMZlBNZEJtdWRMaVU1ZGpwT2lHanVlcGoyM05FY2xpV1VlMFBoNWxubXpsZ25uM0lkTkdXdkFjQU00Y3ZCcDJYL3ZxTHYvem4rTy8rOHI4RkFQeUx2L2l2OFJmLy9UL0hxOHZNek81QTVGLzl5MytEdi82cmYzMmI2QjhFU0V3WjVPZEJ4RmVHT0NhSXdBczBHa1F5ZlFMQlBZZ3NZQzJ3cEhjZ291a2hlUmJqMjlvWVhnQ01JS0tBeEgrMUxIbHpsNDgwNUtzR2xyc1g4V3ZNNmlDWDhQMXl6L1ROd0JHOGd3aGtBUDhhRU9HemxNRjBwd0RCRnlEaUxOUFBaTkttMGMvemR3UVJ3SkNNSEFSc0FKS2dZZEdXRTBTU0czOWpFTms3OVMvKzhsL2cvL3EvLzAvY1hXWmpxZ1ArL2IvN2ovZ3YvclAvNWlXQVJJVzkvdjVlTHJzTy9GdmhTV2NsRGJZWGRMTHhyZ1VLWTk2Ukl4MzBFNWVDMGsvRDhMZUt1NG4wQ2tCZWhMK3I5NGh1NCthR2xIZTB4TWdaWlZuZm1JMkEyK0ovdStzdXQ3c2VFakQ3QW9mdGkvRVRrVGdwM2NkN1djWk5vWmVXOW52T3RCZDhjRi9VRFAxLy8rMS93SC81bi85WCtOLy90M3RjS0JENTkvL3VQK0ovK2gvKzE1ZmszaGYxKzRMSmQ4WjAwSEc5SEE3T0dTWjRNV2FKWDAvWjN6SDl4dkt2TmNvUjU2cktLRDF2bVBvdS9NdkcvVzdqMi94N2RVa1Y3WExYSVpSRS9uNnViNVpqcTZWUXJPcUhJYTN1U1ZTaVJMZlAyeW5rQlRsM2FiN1dDT2I3TDJ0Nk82Ym05ZGQvOVgvZ2YvNGYvNWRMZUlISXp3TElMTzYzQnhPenZhRitQZVA2dWduN0xrSVZQVGVadkUveGsvRi85dEw4ZjZMdGh3U3dKTGltVXZ4YTJ0OWk0ZENNM2tsNjc4ci9lcUMvdTc1cXBhRSsvVlRPdkg1bHU3M002d282YituNjVoaFZWZWFWUFA3cStuLys3WCs0U0NRTG1BYlV2OHYxVzZrNlYvNysyYzd4eTB6djhDRW0ybGV6LzFlenh2dkV2ekxkZHUwMFhraStrVURlZ2VOUEF1ZHZlb1hoNnViRlZScjVmYTdmT2YrL3A3WWx0TnhkUG1JeGNNWWRVNDh0dFo1OFN4b0JRaUxSYTBYZ2F4dklyNzErTFppd0hxOW1xN3NjdnlyRlBHMFIvazI5SDk4WWNyZDJtZTlTcEFYOVhRZmZsN1YvSDI1VGgyL2o5Tzk3WGZ2MzEwNDhxaWU5MFBCdjdSVlVNUmJmVlBDdnZxU2NYNS9ITzdYMG5SU253TUZKZFBicnUrdDE2OStuVTJsay9WWlN5R3NTL1Bidk51NHR2VjgzeE84eDVtOHB2RFV0dE5ocHhaQjNSa01acXFVblU2NWMxTitZS1dxMXFEWHdud0NjMzhqZThUSjFMa25mMkdGK2F2WHVXeXRIZlBzcTd0ZVN6RmdJdXI3dFVHbjNieGxJWDFCS2VuOGVUSDVsdnd3UmU3MGRKN3RxNzZJQ2NaWHpPNWRLSXorcjZQOW0xenRnK1dvZ2Z0MjUzeGlRMmRnLzFkRTZBalB0OTIwcjM0aTM1M1VyL1RmejJrc20vR3FsWmw2OTZQa0ZqVVlBMmZPekVlUkROOTltK0ZzNnZtNGJXZHpGOTZXV1NkZFg4ZDZWL25hRlpuTUZ1R1E1SmdYSjlaVktmUU42WGt2bSt5V2NROEsydUpkKy9RbFJNK3pCZHZ2SGEvMGVxc3p2Y2Yyc2Y4cDNobVg4KzJ0TmFHL0VTaTdoQ2JQL2ZpckNyMlBLL1hyZnZsWlp2bHd5MzVua25aU2hLem1YZVBmcFhrc2l2TDZuNkg1TFNmNHBZZW9uMi9xYkt6UkR1UGhacWViVjBqRU1PS3NqTmNFc3ZwYWczeGREbGVZZlRCSzV2WFpIcHU4bXV3bjd1dEYvWHV5MGNsUzd4dDluSWhQR0d5VytjaklyaVhyZHpscVg4c2JyRzRaOGRmOE8vRzRMZWFNdTJGVVNlK1gvVWRMVEpmOXI5cjEwdXBlcmZiWDVsT3g5OG1MbS9qVlRSaXp2cmxxaDAyWGVGeW51eDlRdW5kU00vbzROdndCM1htNkFUVnVJVGd5dW9DRkFZcTc5eTNTeGVubCt3WWxjMmZsamdjaWI2NjQ2NTd2Mi9SVWxUQkQ0ZHFvYVV1OWkvYlIrTEF5aWZnYktrRzhTM3R5L3UvWmhvSkxDbnZkVnF2b2VrSHhGMHcxWTdVQllKUFZnZjNtWnBTZnhyNysrYXhmclYxUE1OL252cDY1dlJKKzVpblRoUmhRUUZUREgwUTJRWE8wZ2MydzVBa3p1L2tqQlB4b1FlWGY5WmxneWRGU0tkTmNtTWxyMFg4MmtZN2I4cGlyenExZHBYc3pFbDN4ZkFjOXV2M2pGb0Y5SlI3dll0YlpYRXdSdTVadHRwbjd4WXVRWjBWYS91MVVYdnQrMjMxSnZiYWZ2NXZtUzVoNU15clgrV2dDK292dnFxSDZWUUU1cDZYTjdOMmJoM1Zsd2R5TGNIUXJWSnZLV3lqL0U5WFduL21adWJoUXh0ODY3Ym9SN2tmWm5aaDJWS2tTVnlidXZrOTYvMmZMZXdpL1hHakcyV3VQZFlQWXlUTitVLyszbmtHS3Vkb1VKd2gwczRiOW02ckJkV3BnU0hpVyt1eFdhVUQxbUg5a05qWGNUeGdTbVdaK3AwbndOSElBVkVQaUlmd2ZMKzNqcWRBVW1wMDFWWndlVFd4Sm0rLzhqQUpINzY5d2IrNmJpWDZzWjM3MDJOeDdmNTQwdm1KNHgxQjd5UGdIL1NYMzhpNDY5TU5vck1ORUNEUHN3N0tkTkdobFpMSXhoazJYME9Md3IveDdRcnREeEl0NmxUbnRkOHJmYVRRWTVOMHJhakRzM0xzNDhYMDFLM3pIdU4vQzhXU0wreHFwZ3Ivd0Z3NXN5L2dzZ21aSUo0MDRndWNSSkd0MnQvcW9lTC81enN5SEUvUEZBNUp2RzFRV2ZPaDdUL1E3YkxyaVZmeC9JdDVMSDRHZjczbkxhdDFXWmZhbndqYlR4Y2lsMUFrNnRDdDRNYWxNUXVBR2tuOXFXTUd3c25SZjdxNWVPNzJpV2V3RXJNMjdtZTVPa3lyWXZNRldrRDBrRDd6NGUwa2ltc2NyZ2pwREpqTXlEN2FuRytLaExsM2xIcEtXdFk2K292N2huM0wxdmRjRTZOdjR1bURpUXVBZEkzUDA1d2hpcnZQZTRvZmIvdDVlOVdvcS9pM3ZCampucjNzNDBiamt6ZGx3OVorUjdCZDhSc3MzOHU3aGU1ZDhSMzBEeU15ckN5TUc0R0dJaXBSa3VjL3A0ZjNjdEdNNFhkZ25KN3phZnZUekxrRHVmR1NTRHZmTTZmcEh0aXlza2hhKzNVZ3k2eldyTXVlZDIvM3g5SW1iNFhIVEIzTmVqcG1YZTZSRUE3RkR2REZoKzlkelpPV1lVQW9uYk9TZG91VlQxNGYwZlR4SzVYQk5OS1NKK1I1MzU5dlhDREwvYlFrcDM5UkZybEQwa0lWY2J5KzVKK0k1ZTFuV2VKM0tSUXI0Uzh5OUlGMHVWWGNZMzZGSEpZVGV1U1Q1RzJYR2Y3bC9hWnlSY0J6cTZqKytKbVhtMTZyRG52U2Rsdkp0Tmg5cUdtWS9kMGc0QXE2UVJYWXF1ZnQ3YVIyMXNaWE41S1JMTmVycTMrcUQ5Ynk1eFdHL3BWNFkxdzl0RkdsSFdkKzhWRnlEQVJOV2JYZFdwZEplYy9tQlhWL28zMWs5ZUFRWm1SOVdidCtBazhaaldyc0J5T1R4bkE4UUJQT2tuTWdiN3V5YUlFZHdQTHdlb0FOTEZvN1NCK3U3dnBzRDBTbFVRUTdlZEFvOHl0dlBWRFgzRFg0RzNIWGJGeUhlZ2gycUhxWElLT0wvVmJmSk9wQ3FUZWhiSnVMK1BNdDRBQlEycTBsOTZtdDdkNVRiN2dxcVB1Ylcwd242bEFBTGxJNEtBNXIvWnQ0QUJKcVA4TitQaUR3c2lmWm44KzVNcEhTV1c5V3lRamtNUXh0ZUxoMHh0cFJxc0pRdFZQOTZwSWpxZ1pjYVlic1A3ekw1bllWM0cyelI3MkQ1WTl1Z3FsZXp4citYM24wQkRoZTJ1OE51djBMVnJqQVRlUzZsaUs5amZYZW81RExnN0NPbmtjRk0vc1c5VkdnSEJYelB1dXV3dVkwZ2pWYzRFa3FMSENBNlQ4ZDFhS2dGbzIyajdoQUpKcGJPclJPSzFFdlQ2ZWoyUlNEdG5XWDg4RVBrN3FDV3ZVdmJZL2pydlB0cHd6L21hOWw1SzZjRXhKWWlibVhhOFRZQVJKdGtIMkNYOVlKNzN0SGI4bXp5MmV1ekd3RXFURnJsdHJVcnVmUUxPYlZteVJPcGZNUGdXZm1udnY0c0txNlZZenVadjIwM3AvSVpLVS9UZWpKc3ZBRzNHczVUU2VreTBEWTFnWWppbHZibjhYaExGRGNoK0YweG1Pb2tyaHRnL3VHSDE1d1lKVWZQZXFPa3ZCdDJ1eWpCNDZ0azZJTFFEKzl6V0pyY010R1NhRWxYWE5sQzBYSnVHM2N1VWZjZjRkNENpejQ2aHhpaVI0MW9ZaUhlL3RodDUrSXdTRXBxUGR6TitWcVR1TTFXNmFQZjVueEtYOG5pYUU4MnNrdXA5Z2RabWI0M2ZxNFcwU1hpdklnZWowdUk1eTJtcVdJNDFDV0ljYldPMUREMm5BVFRlVkp3eXJrWlpYSFVDdk0xRjFFOXloTGZCbGZWWk9PMWtqOEFsSDQ2RWlzcXM2dkdBcHdodTByTjErYm94R3ZmekgwOFN3ZDF5b3pKeUcxVW5POTdQcHAzZXdHWEVnb0lYRnVqWE5vZzdTLzlOSGlyS3ltRGVkZlA3d25Pd0RxQ1FkQy9GKzd1OHZ3S1FYU2VPZkZSa3ZrZ3ExbTA3UlBNeTlrazlKRSs5L0taZWRrdm5PeWxGNGtsYlhYdzBDT1piNm51SjgvNHFTa2ZlczIwMXYxY1NpYXF4TlgySTVLWXE3bFNiQ2NLR2svdDNIRk1pY1U0WWhDbjFMeG0xcURUbFRPRkE3QWRhcmNLb1FYVmI5YXEvRFA1RGdzalBYSGRzTVhuOUc4dDR6S3NHbm5ZbU44UUo4MWpkQ1IzM1liMHRmaGNkNTZCbi9BS1EyOW9Wb2ZmdnFNL3ZJSENYMTYzSzFuUmYxWncycEhLTHhraDNCeHBtSWFWdDdUb055Uk13eUZ5emJHV3d1Tnk2VjdydUNrN3lKMjB5OXJac1RFdUFMT2JjMi9MU1hDdGZ2MkdqTmQvdGRic094UmY5S2dsT3U4a1RWSjhEU05nNlo5YkY3YnBxMDJEU2dFRXdVU0NmZngzR3lmd1ByczRBK3lCVnl6RUhrbGtzU2QySkVPLzA3YW5HMkNVTnhjeTdaVUdOZlVldkNUN3ZBOGN1blNLcDdYTFQ0SENwM3M1c0xQOExBTmxKMzZVSXZWNXNHWmVFS09KWTNyNkQxbktXOUJOYWtmS0xxQ0FQaHZRVEZObGZDWCtlT2tKL1JrTHB1RTgzTTRqeVFqMVNHZCszRlk5V2F3d2VVbFN0TWpuY0Y4ek9xZGFnc29wWlBNZHBxU3RnL2wzdVZDSzhHWjFOcW0zdUNIdUduN2xxQXl5cVNtbGMxV1k5TFdqbkJEQ2FndlJJOWsySTdRRWJMWEg5bzVWRTdvYjhTOFBZaVBSYVJPYjdlYi9GZFFHRk1iTnVjUTF5aHFYTWlIYzdaZ2xrRjdYRVpyYTNLazdHdmJXSkFCZjY3MlpzZ3BUTVRCMy9ha3VxR2V6RlV2RXRmZnA3cTZaZDQ5M0plbjNYZmluMnFrMzNjbDVKRmQrNUtNbHM5RnYycVkxdkR5VmRxdGFZMG5PaDlOTC94dmJZeGlQYklCZzVmWmFjaHRVMVZORVRWbVB6c3FYQjZMNGVLZ3pWcDNhRnA2cDdvSThNaXpDcnY3aitXSkxJSnNKZXc2OGRjSFhwM2RVS1lkTGRXQmtKSkd3TzJabHJxelZUcXBqMHhaM2FYbkFSeGNkZ1VTUHNIWUNNa3U2QTRoV2diSmR4OEhHUVpweDUrbzBJYzNlTTMrK2o3VkZBRXVGbjAzMVJLYTBTUmg5bFhQMytqeUd0a0t2Q1hhVU5JMU9vZHljbER4Sy91dkF5c3BLa29FTWxDbFp0U2lNUVowK0QyL2xXc29uMEM3Wk8rS241c1ZvMkpSSVl6TStVdERhVHBRR0dGUi9yU2lBWlhxdlpCbDYxV2RYZUtqMVpTa2FVL2RnTGJHYnRFdSttd0xLN3FoNFFXZVhDYzM4c0VQbm1kV0dSdXc2K0E0eWJuRGdEdklDdjJ4UWRTMmZTSFNqVW9Za3o1NDBNOUVVQTA5MlhaNUpFWmhuZ0J0aGVBTWdtTDljWEJHL2QwM3RBOStXZGxTOWdmSVNBYUNLQXVMdkZyNVZmcXN0bkFvaW9OVm55cHJaSTlpcVNFNGlrYkpONGUzMnEyQUVrbWE3eVhnRWtPUkh0SVBUR3lKYmtDSnJWdXhWZm43TUEwbkphTjhEUG9lUFZHQzNBcmFwWmdBMzdSWnFHendRd0hWVnNweG90TW5hSHVXRHJ5eVhrNndhOGY1UWdBdUNHYWE4SVdkZVlDTE5wcTlVMVM1bUo2dnU0elliWGRmeW1ZeTdyWG1KdGRJdmhUZ0ZCeTZia3NJbkFOSkxOL0UzeTZUSzA5QUVnZm8xM25hNXUycktBSmtDR3MycTlOazhnVWJzRWI2cUJydEpPTVppQXprNURTUjNOakRYRGsvRXZ0S3RFZzVRcVlzcG0wM2V4Q2dRTkpKSDRheUF4MmtkMmFZUVRWSnRXR2xpS1hMVVhqV0xIOEtIYUZxQndKbzdvWG5WS1dxUFJDbUQzNWR1dHhwWDNYbDdsWmxkZ0FmNkFJUExWemxBUlZtK3Y2MXhQeS92YWcrT0hBMkVSREc3QVJVR0gxdTc1VnRJb1EwK2pyVW1LbDdTUDA3cHRwcnBWZWQ0RENIVmNoU0Z5VUhzczJHalVpNi9BVFlOZkRxaDJHdTVrRnI4QUUxQTJJVmQxeG1aY2szQ1pLVldrTHlDeEVQMmpmVVNhMkQ3THhINDJ0TFFGUDFGK0p5a1pYUHdoQkcyQ3FVNHhydjRja0tnUFN5VEwwYnhKZlFTOEZoSVRzSTEwNURoaElXeWJnYVgzMHN3dWxUQm00KzhHUVpmUHpWNFAyUGpEZ2NqcmkralpEVE5QdzlDWU50SmMzeXFEb3p0Q21SR3pRZmY4OWhXZGR2M09aekpEWnhhcFdzN0VBSUVYZFdIZUUxclZBSFlGRUlxMy9ySWROZ0FaSUNoU3k3QVRiUFBkL2dIb0lVMUExQjYrVXdZRzJzQ3NvSkxKYVJkQjNtTXlDL1BIeGU1QXRtRjZUdjhKS2tsQWo2RHRXb1l3ZjZoYVF4VXQ2dmRxeGFpQSs1dEFFZzEwQWx3SlFqaDBtWE1jT2pidEtqL1lUVDRBekk1cUExVm5SamxRUlpBQXBEMUgrMVNDaVU0bUJ1ams2OTd4OVBlQjEwMzZqK2k2QVFhR3B4VFNjN09rSU5QdEtrTnRMbU9PQnF4MmNjNUluYzQ2VjNDV29DWDlKVGJraSt6OXEvT1Q1cU5oRzRBTW9GQlFHYW5xenNkVHh4c2s3dXJHZUN1ek1kOE9zR0IrcXRLZ1FHYmcwZkFpRmhDcWRNd3pHVm1OcVNQUDdxdkwxbmthUkh3RjA4SlJTRUM3QzZVUk5yRnZIQ0dHM2lHTklHMUhGN0srQmhLazZoY3FvSUN1RlF3S0lGNEhVRFVoaDU4M25aRk5OMzRuNzd3N3hOczRYaW55My8zd0xWdEZ1MmI4d0piOGozSGRjbDB6QUJucDNld3RNNTNDd1FBYVJ5N05OUk9iTUhUOVoxZXZ2MG5xWFNjcjNDdmdiSGxVZXRhdGJUR0F6ajUyL2ExN1hON04rZGFTakxscVZIZk9XZHc2dUdhNHVadVlCbFNkM1pxaE9tMlpPa3NtOTdTbEFMdW8zZlUrbWVVV3gxQVN3YklXS2tiOUptc1VrNDM2TkpBWVBURUZTQXBmc3QxVjZxMStTaW5KL0x4eXpIZFVHNkRHZ3VkeHgrVExzVkNXcUJJVW5FSmJTM3Zza3hpclp3SE83Ri9Qc2xScTgzRTNWcncwZjdOUlJ3Y0M5Q3BnZ0lnVytBOTAzUm55WG9SZktkMzN6Q1J3Mk5wbU83SlhEbnJPREdNMlhoVXZpaWU0S0IxWDZTVVlXQ1NRMjFsODBqdzlaUDFGM0FrT1Y1ZHg0QlpjWUJmVjRRNDh4aU5uN01wenh2ZEtscERqR0l4Mm1VMVRHQ25HR2pOdTA5V3FEUHRMaXE2NFpGRHE2TDJhRVJ6V29WZHdzUVFmdGI4SWtFZ2RkSlZuQUlsTnRhYUFSTzBqME1zdVFOS1MxVDdEcDhPWTVLSjFLSW1GdEd4cURkdk9xaDJ0N1M2Q2dXVUxDampJcG1BK3lnTmFpN2hPYVo5dXd3dUlhQUYvMzljcjV1bDdYVjNncktuU2lJMzdUTytOK3ZVZnBROVJSMHpRVzdkdEwyV2tRdnhKWTRQUXp0VENhR3p3Qy9BQWN3K0dna0hXVytqMHV6Z1htNG9DeVBmQVl4eU5OK284SlJscGlNNUdoWWs5Zm8zY3MyaHFTVVF1Z2dkdXlpdDFSQ1NTaGNqemxDSXFTM3FoQWxRWmdpVmRscE9MNFBKTzFmUksvMDhEeVZDNUpwREVvY2dOSkcwVUJteEZHeGdhSTdvNVpkd05pWVRqRzBWZmIxUTg1ZjBFeWp1N2pnaU1sOHNNbDFGcjZFbnFIOWl3ZXAzdDlwV1B3UUtXZnYrbTcvYnEzYy9vQWlOamRwc2VoamVuWGduUVRKVkc2Wm8rSVZybVZZcUIyRC91d0xOVEs0RGMxdlVDS2dvZ3UxaUxDM2cwR0tTb1AxOE9NRk82QnNQSmdQUnRzQWN0dWFvaEJrRGdTRFV0OVhmZEJjeEJieVlNTDZPNHlrN1ZoQ1RrUnVUaFdnNkFLQjY1Q3BBVWJ3bVFzRXJXYnVjc3VuYi8valNRRUJCeVQ0dGhBc25TdHJFaHJKZzA1cUNuMmdsWGlhUFNIS2srS1RkeGRTWXl1VHZPY1lDU2htdEhKeDJjMkc5QTVBYW1mdFByam5GMHNPWkE5SjRWNlpaN2wycEtKVXlkZ0dDb015dTZpTXhyVzhxMWRMeWlEY1JZcmdMT1ZycHV0V2JlMGJxbXZiblZiL3RWUUxTam40ZGExSnZDZkRBMjB3a0FERWEyWks1NzhJanJibVBnZnUzZ0JSUVN6MGxPQmlubXRGWWpuY0FGbU9lMkFEdUhQYVhLRzQ1b21RZWU2QXlTb1VVRHNteS8ya2VDcHJXQXhEYTZEUmcyaVIxSVNoWWh3S2xYN0ZkQWdvcHJ5M3BWZGdQTW9EZEJqaXRTcE5ndEpRdnI1c20ybW9jM1pXMkg3YVhiUTN1eXprL2hpdFBsTXV6TDVOMFArWThWaU55Qnh1OEJKRHVsZDg4WnRvbVdWWEd1WkhpTDlsY0E2YnpOcHpIMVZobGhZeWJ6dlhOUnIxUUtQdU9ZUU9za0k2M0pSSHNISkpTQ2VodDNkOUpxR3NjN1NKeWI5aVFvRHRtVmNYYXBvd0cxZ0hOa2RRMHJSaTFEQnlQdnZYYXJwS0NtM0tJVnpWZ0tQbVNxcWdkWGFqSWRORzBYVnUxTnJGTndFbG9MdU16ZUE0bGhVMjJrTnQ4QWtvdWhkZEhidEpkMnlkaTY1RXM2Z0lVNnpKbmdzZ1AxTnY0WTMyVUZhOE8xMnptamc0NTBzTnNqMkVqM1puWG10d0tTYjRDSFNiaElJSmFiZ3pneURaQVpLTktVZFZzWXhBQ1JJTkIyanMwT0FxRDhOc1lXY2N3NFNxdHR0RUpVa3hZejk3KzkvajM0bzNZS0JLdnpMT2F5Ti9uS3ZURGxWUzNFYUxmNU8ybTR6a3czdXo5OXVwcWJEa3J2cGxHeHZ0U2RYQ21KTGpvRFFDOEcxWjNwQmF6SytKdG5vby9wdlNwVVFOSXFndTVZM1NVU3p0UlgxUWFnR24xK0RTUUFnRE9yOHdwSXNzWHRBUHdzZGFya1h6NkxGTnhxaXh3MkpNQ0Rralc2bmFwSHRXMjBpWXcxNXBRdytWM3RhcU9EcTlweDg4WHF6TThDeVEyc3ZYeG4xL3NOUUw2WHIzcXhTaG9CamdzbEpzQlJhb01DRVNiVDJwNFB3L2Z0L3ZKa3VXSlVQRTVwSTZXU3NybklCN3hGaFhFRkR0UDMyaU1LSUZ6cFVFYlkwczlXeU9nMy9YQURudnN3Q09tazk3WTBZVEg2eDRsbEZOVzNZbUlwT0Q0VEFkdksySWRlQVluUVI5OFBncEMrbzVyekNraUdTakpwMm9Fa3F1SHdNOGZOWmZuWEttM1RheThra280YXdjeFQybGFBWkRhSFZWdjEwbTV2eFlzbU1wNExBS3BKV2cvdExnNlJUcStUaW0vRFFBRTlhT1lZL29aaDlUMEQvMXphTjg4SklBUVBWVkY2R1U2Wm9Tc2RjZnZ3NVdqY0hpVGM2WGhabGkyZ21jL2p3QnV4TWZSaExabEc3UmJLdEowZHNyVkgzbnVIWFNTUERUZ3VnRnZ2TnNtaUFNUWtmRyszSFRna3ZWM2pkRHRzU1R5ZGxQeWFHNUFHVlpHRTFmOENFTldCd0tOeTltNnpLQm8yUHhLMm1WVGhZbUVjQUtGQTBneDhqZWRTNXlUQ29UNW5LQWR5UXlNQzA2cHFrM1d1bGJVMGR2YTVyaXNOclZMWEJFeVZFVG9mdWkrRW53a1hocnZlVFh2WnA1SysvalpOTjVpNzM1L2puV3BRQjh4M2V2ME9TN3cvQVJ5OFJBS0p4d2FRVHROL2l0THFIMUpnSWN1bUJLWExFWFVsQ2FEakZBeTFDL3VRVWdnb3RGVVVUWDF1eUg2c1lVdERCWEZTdHRTTGtvZllXTlQyb3lBMWJVSThxelFIeFZCN3VveHUzNXNSczdWN3QzbkQ4QjVUWjhNMjVPWHIweHNZRWE3ZFpadkpNUjQ0a01xRksrQmJTaWhkemdBUy9oU1E5T3JHSUZXMUlRS1dacGxTUXZ4dm5lY0F1L0RQVUVCVGs4MlhRQUtBQndmUmg4TWxUdHMvbUZ1ZmtUcmFsZlNYT3FqZXN6d21vWDJpTkYwMmM3ZHROUVA3ckhuZXB6aDBvODV2S2x6bTk0M1ZtUnVtLzlaMUxlelY5VW9DbWVsbkhrc0h4bDBaQ1JZcWFZenpMSlhKdGNHa2tlWTdrVTQyTkIveENRaExwQlFkL0dMVWpKMmhYS0ZRcWNKcXRhalN5bk90MUdTNExnUHo3RTBGSGtwcSsvVjZzK05YZmI0WldzMWFYWEY1RnFsRDQ4NU5YVGFZak1YM1lUcU9WanMyWmg4U3lSWTIydncxa0VSWmZnc2tMZG50V2NySlkzZEF3c3lIVkpKTHZGbVBZTmg5TDR2dTk2RTd2aWVOQlBVRXptNXNVQklKTExhNWJhZ2t2U21wR0VITFVBMWpzTVpCN2JTYXdBZzI1STJYSUNJdC82dXVud0FQc3VVYkFQSHRJR0V5TmxZd1dvaUZxenJEMGpDcEFISjdLSytBUmgzc1FvWm5QdWkwa1JkZDBnVlVkSWEvbENQUHR0QTdoRk15S0REVDlySksxMkJnSXorcmRyQ1pUdWh5R09xVEFCd1lMOEJ2RDd2czBOMWo1TEtnZnU4a3dLS0JwS3FhREZPYitiSXRha0NtUk9EcWNTbUY4YmkvOWgzUEFwVHBkMVJnZXIvTGkzVzRBZ2t1Z09HU2h6Vkl1RUZYYm1qYTd3T1UzcWczSVRaaXFDb0tYSFdscWlqMDlQSTVsNFNueDIyMG83Uk5rajA4WE1WT29oSzN5eWE4OFU3YWovVFNmT3lyUUVSYis5WDFzNkR5UGZBQUVNNDJFcllEU0ora1pkSUpWak56QTBpckR2TzA3VWJlWWtCUmNmb0UrQmI1VllWcG0wY3lxTkZnT2lXSHRwMXNvRExDbE5GWkIxU1lxaktNNDN0YXFoRFZnQUk4dWVJQlVMSjVCU0N6ZjE2QnhxUWRvOHgyWWUrZHJiVVViQTBzc0kxSmxOSHpmYWtXdHNSZVVYaWJlYTRDcnhqMU5UQUVTT2cza29FS09tV2ZlQVVraHN2WnJpYmxsdzFJRGt6U2xadGlZZ1dQZVcvSUdYNzNQOGx4cXN1MzFkNmtiWXBhMVRiU3NLQnE3N0kxZ1AzVnphM0x2WjFYZkRhaVJvOGtueElzZC96cVpQbkZFdS9QWFUzYzlkck5iN3I2TXZlOVNHNCtRYVZGOVFYTzFOZDBXeTY3ZXNMQnBLcy9NdU0za0JzbUNFeG9ZYnFlR1JpdmYydndFeWk4OHlvR0g2Q0NrbENRQUtJU2greWVFTkJyQUlyc1Y3ZDFEckpkQW5rUEduYzJFOFp2U1NGWFBJdW1CaXNJalNnbUNIQndHVmJXQTZWbTFqUHZnN09kNTJ5SnV0SWI1NGhjbmdQakZMRUNtNFFpWHFiRXRlcVRwQ3ZuMkRwTnJXd0xUWE1iU0IydGVzelJQbFVqQVJJejRQUnFxd0lJUkE5MUh5VXdMWU9kbEY1MGRPM1NSNEtCclpTbXJoN0NvN2VOcXBUeXE0TTJtaVl3QWY0eVZIUS9XcnpjbG5oZlgxOHg2NHg3ZlFyRTNRZm83dU9STVVWSEF3U0FGRURRRER5WTJ0cEdZTmtnMTI5N0VFaTBIRkYvTkYrVDlNTDh3U0FjTEoyV1lFVEdocUw1eUNQcnN0azZ1czAyZ0t6QkVPbGlHS1FVZC9IL1FORXk5L1hvcitPZGtWWHphamFoWk5qTTBOSzRqL2pEbTdkV0I1UnBoUXltczNTSEwyL04zdmNpbzJpd1d5T1dTaUVnd2trQnVSeGNiZXNpMmFDeElxVWhTZ1psc3kwZ1lSR2xZK0NsVTlvT0pHbXpHRE4rOWIrTGxKWkcxN1hRcDltem92bXVWQ3VSTEl3MWE1RFYzbzgrT0xGeVF4OGt2RUNIRlQ3WVR2dWw2dElGUks3WHp3REhYZXErVW1XUW1hZkpzZTVucWlsUzlyU2I5RkM5VVBZRmdOUW5EekhqN2JhU3kwcU5ycFl3VFltWVNZa0JhdHlrWVhXdkI5V1RHQVRaRVlPT3BxZDRneXRSTmRnV0p0MFQzRGhzN3RTWEhhUm8xK0RCdm5yMUVCc3NXM2tFYmYyYTVSSGVsUG1xUnBRMG5DMmpFa1htWTRCNU16empCbm42MlRKdk5ZTUoyUzZlVG1pVVVNak1kY0NROUp0cUYxbXg1cWt2Z0tTZFBUWWdJVWl4dlNpeGtHNHllbS80S1RWS0xvYTFWclFCaWJaZnRhaExuVVJLTEpYcW1CMUhhYVpJYnpDOHZ4b0UyU3dYRVBtN0FjZGUramJUS2NONnF3YzlHOXd3bnN5eWQxOW11OWhCQkVDR3FzQ2NLdzh4cGhhNDVFeTdwdlF4d1lOMTNPMGlIVDdMbGZ0ZHVpbG0xM1RYNCtlbW16cEhsQUNQaEpzQTdSVkFPbWMxakFKNlBvalVad0Jzb1FYSURCZFIyZm14Skc4R0hST2VyaXIwdTJyU2ZBcWR2amZwZGVUVllDSXVuYjJaajZJMkhUb012ZlI1Vmg2SlBpVkoxdG1yY3A3SmF5Q0pja3ZWcVBxMGhMVUQ2TVhsbm0yZkJ6VEgwQklITTVHZ1ROS1ZBVmVGQVpablNXdU5zODREUUc0Zk9FWDY3aElhWklFK2N1QjY2Ykt3R284Zmc5QmZmZTNwWDRESEJXcXpRaWZYeXhzODRuVnZEcnVzVk53Q3lFNlZTbWRqbzdnQUFDQUFTVVJCVkNMMzc1WEoyd3R3VmRoZ1hGMmR1WkVFYkR6bi9RQWFHbHNaZGpXa1R1cTQ0ck9hQVpodVY0R3MyMjU4T1g3UWlXNWZUdjUxck9PMWZWcUpFRlZGQm1Hck1yT0lDeE1sdExVQmRTdXJnR21lSzBLM2F5dXh4NkZnWXBReUFNREQ4N1VsQkFXU2VJLzkwdyt5NzRlclFMWENvYllTcDlIVFc5dzNxOVdvTnZqaktwVmdBOCtTUWp6clJqOFNJYlU1UE1wa20rOWR0SU1KdTZEd3Z0TzVleTFpQU4ySFhucGNYTXZ1dnJ2TFZGSW5iK254Y1hjdzhkZlhxd1M3c1RLdjRiKzdENkNza0JPUko0QU1KaERtM1dmMHpqc1pVbXdaQXl3RW1KaW03Qmw1ZjdHQmVMOXJuRzVRcTJlN2dvK3pYYXAzcFNOM1NjTDNlc2hsMXJhUGl3UnlCWkN4YVU0OVlXZW0zUTF2QnNHMFNXV1lZSUZySUZDQTdXbUl0TVNIbnQyTDJ4QnloNlVMZmUrbmFmb01QR1Y5U3VIYjdBNFVrRVM3TzZ6c0d5eVM2b1BVbitLUzlSaThYOEZKcVlkMVVGVWt4MGY3dE54SUpjWHhWR2VBVWFGUzYyaHZlcTI2MjdMcWdONlZMR1ozSTNSbnlkYWROUldaRGgvSEtXNGwza0tLaklYSDNRd2tNZDlsSXhlWFRPOUwzc0YvNWk4Z0FRRGZVVi95K1E1QWJFMGdzR0x1SHR3RW1TWmpzNU1JYlRFejJIaFc1aVdZWFp0ZVFRVWhhdHMxdndrbWtQc0pFUFcxczVyMVpyc050VWhVUlRiK3JXT1pYVmR1Z0t2QXVMMnU4a05BNktHdW9MSWZLa3orcmRMY0pLdGVHVUcxVllCSjhEY055QVFUcjZyWnptbFpVSzBzSjNoNnJlSlF0ZG5xUnZXb2VYMERrdWhGejN0di8zY1U4NGtuZGFkTEZXUUVlemtUT2dHbmpMRms1cm54Z01vRVNSM0E0WjVmblVvSlRHU0p5aVBwckhHdnF6SFdKZXhlcmtyQjBEOWxRbjljY2U3dXVzYTVxQStqOFhSZzM4V2ZqQnEzd3BTVjFnYXh3NjRnK2N3VHluWUFVY0F3eEVlWUZZaW1XL3hrYnBZcTZZMTFzWTFtU2drTENvUWh0UjZWcnRXeWxpcktqckg3bm1CaGdJTkpHZkNiK0EwZ096ajA4bmpYb3dFdUxwOUpYbC95ZnB5Ym9WNmNFbDRxRENXVFlsQ2RsUk04d1BCMGF4SURhN2w1RzhaZzczdUQ1Y2E3QmdKVGNTQzVVTzBpa01qVlVCTkl6QUxnbkpMSWpaMkVqRi8xeVF6T1BOTzF4bldPTmRhZmIwdENPcVhQazl4eWg1ZWVGU0J4QXNocVYxVUNiWGZYSnJtTlNWU2FwcFovVWUzVzQ2TEhpMW1zSGdIZjJJRDNiWHZKYVVYTTk4QkRxWTlHMDBGZS9oS3ZBRVRVa09FOHBxcElnWVJJR1Z1ZURURHFvYXFudzFzTnFxWWpwWXBhaFpsQXhVRVVvS0h0SUdDVFpjWk1rdVc2MUgxdk44L1oyUnR3eWttcTZKU3hvYUJSb25HUEZ2WlJ4OW43YUpjKzdSSlNvZXozSEdHZTRFbkdLSnRJU1dLQTVWZmUrb05YZ0c2d3ExV05PaEVzNHp2elV2OE9UeWVxZEFnenBQY3JWejNPanV1Ty9rU0NqVElId0JRUUJSMnVCeUFsZ0F3cHA1cE9abk9ZbklmaXFJa0VMZ1hXRkpKWXRDKzdoOStIeWhac3czaU11bk0xc0NZSWVzNnlXdVNWYkxFV0d6dXQrcDlJaFM0SXNHOFJlWFFWcnBGZlhuZnlyaURiUFhEYzNPK2drZm40ZU5jQW9tVmR3a2pEZUplTm9LQlNIWmZ4SGJCMVNEeVozV1VaTnNmSEJkU0tnUWU0UmFmdG13cW5CTFZrcHM3Ym1qa3o3L3B0QUttQlYrSXdhZFgyRlpwMmNCaHRycTh1UTZWcDE2YS9JSW1PSDYvSzlMSTBRS0U5eG5iT3Zzc0VTSmozQmliZ1NvM2xyQ2o2RUFVR1ovV3oxVE5lR0VuUHlsZFBUL00wYUFaZENRNmxKbUpNMml3cnFrTWdtWkxJRUhhc0E0WXJQYVVaYlB6QnRNeWt1c3dRbjZZSUVOb1ZJaU5RY015WGFzUSswelM3TXN0Q1FvcVo1NWtRVExvNm84dDFQSlU2NDVQbHY3eEtGTHU1dGlYYUx2UTE2RFJnS0pJMkE3eVdRQlFnQUZ2elhBNmdBV01BaUFtQWFOeWtjOXBONkpLUG1RZW9nb2pOUmVyZ2Fyd3R5WWFnSW5ZU3lTOW1vRmFOV3BwWnFRS1EyVExPcHJhME1kTUtUTGRPR0cxK0J5Q1I5TnBYZDJlQjdCTlB1Ymx2UUdKREVzazR6RzkxWnVVK0R3RVRwTXJtbXdvalpaZkxlSWJXT3dmY2pueDNEaUJ4aTNkMnZnRVMwaW1GZGg5NkFzazVEYTdRZE5wK3ZKa1NCZk9maHpKeG1kZFJIOUptdUtaTlk2a2hBUTNLZDJuN2NEcnFDZkx5VHNaNzdiZGgxa0QxMFZpcElSaldNMEhrWnRDOHZOeXc1MG5TYklUby9WYUJ6ZDI5Skk4aVNrQ2lITCthNEdubkVQdEh2VzhtTFhWR21WWExLWFdIcGJ3R3NnRkl3cmdxeVhFUW1TTTNKMm1lQlIwUjVsM1BPNk1ucFE0T0R1ckpNYjdpOTJSWkZ4ZCtDaWJhdHQzbXF1UHFkYndhQzBNcXZMODQyTTdkYzNNd3Y1VmFBclJVQWhNd0hHRENQdWdWbDFJWG1JYVNEY3NoODlpVUJHeW9CV2x6V1N1QkJLaFBXakl6QWNNZXdwdFRXS2syR3dzWE1GTnQ2VWY2dHpqcG9XMkc0MjJzem1nY2d2a20vcFJUNCtpd0hHSWlqZXpPSlNOcVR5SmRzYmF2Vk5SOWZHVFdEK214S2JkZXk3cVFPdDl0N3plOWlkMDRFNnJxSXZFRlFCZzJKSkVkUUlxNjFlMUVEMVdWSG9TV3lHcXpsK3gvQTdqV0xEdWxoWDBGcVdhclVuMHNSM2hMSjV5ZFk1YmxMRVpBMnBaeHpZUTUrcFVhMzRBcHNQYitHMXplKzB4V1BMTnMvM3p6NjB0ekxoOVFNNXp1T0pMTWs2cFpsbjRhWGJqSlhET25saVR1d0VRWUhQVEtiUFdtbG8yVElTMHlhaWxGcEJCVlBRellnSVNoZWpoMEYwMEpJZTROd0lHRWNvREtoRzdpU3pxYUYxZ0hiK21xUUdpVHprRFFFQXpid1lsK0g4eHo2NlMydjZDUlhGdlo1bXBTamhETUQ1YjFKMHd2SU5TR1ZldkszbHl2bFowWDRUZml6dTFNYTFhRFR4bDJCNURkUGYwV1FFcEZhWWEvQU13QXRWWmZHc0FLZlRaZ3NpdDRWR2NxOW5LclAxV1BWR0ZjNnVBWkQ0WlFXY1NsZm9CQ3EwbzFHd3NvVWNyWWdkcHRNMDVMTjkxSkgxYlp6TkpMbVRBWkZqSStTbmJMQkFjNEtSbytNNStWREVkOXZ4bXZyeHJpeWZCU3V3dVl0RDJJbS9TeWQ4U3IxWlhaemNvaEt0SnZRSUxtYWU1UkNaQVRoeFlCN291ZlpOYkx2UjdBazhjbWs0cWNLc2lpMzcrTkJrM0FhOGpJL0h2bWFFbUVmakFDeUNxQlVySXhhWlNTS25yL1VKVmFkWXNkeWdyelNKVnd4dWQ0aXlpUE5xYWdJdlhES3dCNTgwN0NiMWRwQkZpYUxFMTdJNEVvY0FCdC94Z2cwaURRRE04eTUreHVJcW9WKzF6eWFsVm5QOUZkYTNYOUlwMkFXTG13cjV6TndyN1Jxb215amFUSFpuTVIyc241SlZsb2VnR1ZIcnA3KzNZcEs3TzdPTzlMa3NXQnZETVJ0dWNNK0FEd2xQRnpXa3NyWklqZFRBaERTU21pN0JSYnNTWUFwdkcxbG96YnBrR3BoRFlZanVjR0VwWjhUdFhHUWhxcFQ0UlN6UURhQjhNMW5NeExiK0tDTlFDTzNzVFg5cDI1S2lOZ1UyeXJBSklHVlVvTEpSR1k1T3U0OEdFOWVvNWh0V280RXZLM2RJeHhaQjZienFCU0pTY0VLZXpSTDJ6S2VSbisvbnIxM2k3L1hxUVNrU3hLdEdmMG9VWk1BQ25tdTJGNjVqZUJhUzdYRmhCdG5xdEQ0aHIxdHU3ZlBSeUdxYXJZQ0M5cFI2U0ljZVFoYkVnMURhS1cvU3pBcFIxWkt6TXRqcnExaExKTElDWjFLc2Q1dzVCQTdycmFScHQrNDhyeGMxZy9xb1AvV1FNUmVDcG9jSlVCeFNLYTZXc3d1UUJKZ3gzdEpUQlZOeHp4RFYzUGNSSE9iQVNZRmsyZTB0Nk9NcnJlWElPaDZNeFdQTW94UDl0elNEdDNLc0NOeEZaSFBXaWJORXVVNU5UaDFmRm9wYWJidjlPRnB6Q3loZW9GMFJjQVVsTFRTVXFIclJoV0RiaTAwemNHejIzalhobFNsMjExVUZhNFpYVk53U1B5VW1CWVN3eWxGMGxGeXBaQkNjWXQ2V0lESnBubHAvMWt5MyswdnRnL0tuMGJnc2RTYlZBT3cxRmllMGtqamw3TlNlQ2dqTlM3UWlrNTVTeHUvUHV5Y3diRlFNNURDU0M3V3JJL21GMC9YeFExdWI5T3MxYTkzVWUraEVkMTBPTGMvR1F2RFRCcHFhUkVkR0J6TW11bk1aZDNZSHd6dU93SWpyeFNBcWpsMGRYQUlxd1dTOHZlZmF0U2lLN2FVRFhSZXVuT3c4MVFHdmtmbUpMSjBBUEFsaGxqd1NRYytmbE4zWk16N0NNY2c5b1dHemRiU29iOFlCbzlXQUdSMXBvM0lWSmthM2tSZi9OWTNVYmxkMmVnRituMjVjY0xlR1RjNlhFcXFvNUtJWGZxeGgyQW1EQzdNTGhoQnhDeHU0aEwrc1hSelBiNHlRNlY3eEt3NE8vS00yRFYxckZKS3dyZVd4dmFKVTdUbzNhRlhxN01qaDZnS0ZLSTVtNmhvaHp5N0pKbVVqSUJvMC9CdjUwL1NhRklYcElXZ0M1OXBtY09uZ2lwNlBUWVF6TkZiNkhHd2lad1VtQW93MnZhald3N3dNZGJnbkJWZ1RJZk9sWVJUSXA5YWRkZ1BQbEk5Z1FRQVlyYVI1T2pWK3R2amo0bmxUWExzamxtMFhGWVo3c3dmbm5hWkJUNk5rMlFMVklScWd3dys2M0FkYWcwbTNwRDBDajY0b2ZnVXBRWVFlUXRVSHdCSXV3MC9kVXFiMkF5QnNTbzhVMDZCWWNYd0hFbmdkZ280eDVBTHFvUE8ydW9PQXB1M0JoR01NQWM0em1qWlV2bmVHT1pqRDlWbHhhWGFmOXdqUDB4MGdjdGRSQWNwaG9UUU5EdnRIK3FSblo5bm4wd3UyZ1B0KzNkNjVGaDE1Y3lHeXY2T0J3UG9JeXZ5MEx5T2MrV1NFTGlFR2J3R0d2azVRWVQ3dlpsem5HY1FFa0tBTnk1UzdkSHk4VkxrMGNHK05sU01WREx1QU9OUzMwaFlHM3RWcXBOQWtYeFl6NUxtNW5hWURnV2VDOTBCUDJVMDFCcDJxZEp3dW05TzhKdDVGZjFGMGtwb20wZ2ttVlgzNWtKaUd3OTdwcnNHNUtJbC9GeFNoNnNNcDhoejN0bktOanM1M2lNZlM2WTc1cEVBWmdxYndPYmthNkJwQnVVREkxT3Q0ZXJCR0p0SUUwdVRzblB5dUdMTEh2clVEZlNxaTdMZGlVVDlXcFFTUjJYYXpLdVd3eEJxaTVBU0IveDJ4VDBpZ2hHZW1uWThiaXdEYlM3eTdDelpReFFCVjFLQ3hYdXRmL2kwNEcxTEwrb2t1cVU1NUl4OG9IdGVRc21VWWpsOGlzY1VLZXp0cFU0ZEhwcmoxY3h2cFo2UlNEeS9MOVRoU3BCa1ZDV2htdEF0TVN5ZTFFVW1IZ3lmS2syVnU4dlM3UjFXaDZxdk80cStwV1FOZ0JGZjhZWUUvMEV4Z1lpSHFGd0U3ZUk3L2NYU2VSdWlGNzhPKzZ1VWdNa2x4MDhoZ1RRejBQdDJVQkNHL1FDSUdUd1YrZWwyb0tlNUY3eGh1b2krZkxkSnJWRTg2bFhxdEl2bnJJRkhoRSs4N0JyZXBrNXh3WSs4SFdyVUJOZ3NsMGttd2d2cUNzZ1gwbm5ta25nRXFaOW9VR2pTU3Z4MTJPQk9ESFdZRFNaZ01wWWozSEhZMnNhU2lsUGkvZWZBNTRFVEJJY21uODhiU2JBOThERWt4bytXeklUMDY1S093QnkxRzloMEFaUjdjenpqTlZkWVRPNkM0R0dWbnFiUnBPTEx5NkhHT2tjNDE2SmFYWEp4OHRwa0RYMW9CMWppU3M0SEpNcUpXbWRDU0t6cTE5ZVgxdm9kK0M0aHUzcXpaQWNWSHF4WnZnR2xreWhBRExlUVJnNFZaaWZBUkFwbzVkV21kY2FORVFHeHlnTENoeEQ2c0JOV05kaEFuVE9CTFlLZEZTVkdaSklEcEMyaCtnZUxNTmg2UkZnWWtCRkQzT2xSdWRBNmE2bTFKVFZ2NzUwOE5iWThoZHhqTHpqM1FkZUxGaEljakJScWpyUG5OWExwcEhNMmdkQ3A0K09FMHk4Wm00M3JqWlFTTEJtRktPZHhMY1RBM29scUVKdmphMnFabVMrREZwZStYRDBPOVVWRXhwWSswUkU3b3Nwek16VUNqd0RLSXJQOW9iUE5OWE1LcmxVbFBtYkQxUURxOTQxcGxNU2VXOHV1N3R1aHRNdU9aakpvTnpBdzlvd1U4QWhSRjBCWkFPZE82WTB2aWNJSFJlYVZDV2FJTkVnVUxQdXpmR0pMMEZsMEM5Z0FucWcybkQ5N3ZhYWVYQ2xwd0drM3haRGdmbGtmYXJaUkpKS0NwWVVWVlJhZjJDVWFnL3pvUVRPSzJ4SldaN1FzbDl6bUxKZmV3N2taQXlVWkM4MjFtU2ViQTgxTUZvbGFLWjRyUEJETVRoT0FnZHlFM25aNTZ5WnE4QUVtQ3FPZ1pKRjlZdDR3Y2JXQmZyM25QQTJwRVRMVUgzYUJJKzRrZThEZytrb2xTUU5MWWVBem5GdDFGRmc0cGh6aVFkcEoxV2hoSmI5WVZtdTZGU1BaRit4UksreEE2QytLYU05cktDbm5YNVpuWGtGSXh5Z2I5NE9hVVVCbzRCNFUyUFVDQm9ET1J0c2x6NnFJellBTVVDL0cwTkdYbHkyNHB1U2REYkdsM3ptNldNYktFazYwempNRzVtK1FJYjFKK2VneXhyNWRWNmVTOWZ2VkVlSDlaNDFpL3NhV0F6RHRIdU1FbFZsSWJEd3BsNFhhN2VkOXlhOXkwRGhld1dhOGtobVdxb1lkZ1dWTnZKMStTaEJtNk84L1VVT1FUczZ0cms3bmpWKzU3YXpDU1pVdTdrZnB6bWlUa1dqSkZKcVRUQzRPVGZjb1pkeFN4SlJCbFdITGtHWmtqVEVBYzIwN1V6U05KaVN3VnNLRmJTWGhoeEx1dEtVbCtNZWhMK2FRcTkzYWpjNVg4aWdzVjgwd3I5bEU3bGVXNW9iS2FSNVdJMlZXM3BwaUgyNy92VFczTk5zQUNKcDF5YUJqSDB4T3Z1L2xHUjJNTm5TRDhleSszakRlWTd2cXk4YWRGcnl3TXhUUUpiN1dxaWl4SUNqYWpNc01vQXFZcFpQalJHaitZZWtxSDNDdERiZjdmT0hXZnVRTEEza2Jhb2c1NGd2SmZsTTdFZ2VxMVBKckNibVdvNzBpbGxwam1LQnB2SDBYdUdaenJhWnp4bU1IUG0zenFKcURwSUdOMDRISjV3bnBaY0t3YU1HVVAwOURjOHU4Vmx6QlFqUCtsblRwdUpnQVVPbUwrbWdKWlo2ZjNGdjM1WnhMZXUzaVFraGtVcStoaEhqVGt2aGRoVXVxbnp6WkxONVRmdUlYY1AzWHkzajhvNmdvd0NTYVJxSkJBd1kwbDlIRDJaYldEYlZESk1HTkdIa05zaDJuamJzR1NoNmF1ditqU1RSMHMrK2ZOeEx0WHJxVmh0UDFXc1ZZZkFyVW9YT3BQVTBZZUtVUUNZZ2lXcWlOUklnb0pSUyswb3c4MUMxWllES1BqdzI4TEV0YWcyNVpQb2ltM1lOY0JBMkhYcW9qMHFkWExteEJKUlNIN3haUmNXZGxkTEtndUdaYWcvVHVQQm8yMXZWK0JvWnQyRlZEMGhLdFdaVE4zalhMdlRGMmxDd202N3lXVDkrVnJPQVE2VVRobVVGNmExYWZhRnhza1N1QUJGVXRUTXNKNTZxWC9jbGlsNU5MWDNJUHBOeFVBNkkvSXptYmpEdGs2MGxyUERvS3RwY2dHUGN2MzZuWlJpTm5GbEM3RDJKOUd2azFRTnNCeEFUQUJtT1lhckdsTVNpMG9Rc21XbFpkd0N5K1pIMDg1TDBVbmIxdlBUb2tHd0FsL0pKNXpMK0U4aHdVS3hXVUtnMndLQ2Z3S1B1N2laVkt5bXhtM0pVS1lCR1ZZRzRGSVN1dUxLSFZDcU5kSjFRMGNaZmsrZ3FhN1QvQzRiclF0bFdpTmZjWjVMQWN5U0lubFJ6Q0NCc2k5UXFRaGlKRXVtMEZqbUlWQUpDaHRXTWIxbEtPSk1KdWdGUTIwclZTRjNjRFJocXF4NlBrS3BLSGQ3RWc1RHN6a001NDRyRTAzNURLUEJvWlBjQ2s4NGh5aTZYQW1UL2kyOU85WlgybXd5cVIzZFgxc2NVTk9LYWpDelhPK0NRc09rZDJ2V25idGJTQndwQXBqMUN3VVBTeUZiK2kvOUgzVS9tbmR2L21YVUR3dDAzZHRzL1JHbTRXNG5adkZLVmpzcDNTVWV2WWxpNnBWRlNvTXBKZnduT003WmxYVTFaSUdNRElCaDlxaTVTRXhQR3lyZVhzU3BBcEFEbTFhNmRmOVhCR3h5NmIyVTg1MS90YlpPQ3grSFBtSkpLU1VLZW5xRlpRcS80Nld4L0R5YkVCck1BRUY4NTNpbVpKRlBXcHlQU3haeUdTUU1Od09KRUIzR2xyN25hR3lDU0h2Q3pGZ0tPZlpHaFdVOUtHamJhTUJ1SkRRS0t1OXlUMDIwNVFmaGNBOVl2NWZQZGMybGJ6alFLSkFVaXZXVEt6dExaZjBhK1hDS3Uxd0JVUDRmQmVETXZOWEpTK3VnWmlneXZ6TEVEU1B5T0ZaeWFRVHRzYk55N01ENmxGSlV3dXZ5U1lnWkFFbEFvUVhEbjQyc0FvZTU0RG5VcmR2b2VFbnNoZDJjU0hLeFhUMkNUeXRIc21hZDY2Z1F3Mk1objJrRW1ZRFNRekZhSWR4T0NDRGhYc09GdlEwZEpEOHpMT2lvWjBCTXN3MWJSaEhBSmM2em1lWGh2TEkvelN5d2ptdmV4QTRVU2JqalRTODBNT0R6eUN6QkpDZUl3TEtMWUlnMHBsUkJNRXFoR1c2bjZVMUtEK2dQSk4yOEF0QkZXbWI3dkp6UXpmTUdYU3Y4aVFjZ2t5SlBaK3BNaUVQQUkrc244YnBZYklidmNNNHRqbk9mTm1TWkF2eThnb1RxaktEbFhSVzR1QXNDb3R4Ukk0NUlBMHlWdEFnQm5UNzdxbWJEMWZ4cE9WekU3RzM2RjkrUlFEWlF0cG8xbExERVhNTWdlbVFFV0NoQ0dxY0wwcHpnbmdHekFJZm55MmZONEFKZDZIYkpWSUxDZ0pRUFdaM2lWanVhMEdpd0tORFhjZHBXRVpkc0lFbEwzK0prUHk2cDhCVDRNMVpmN3BaTGVudTNsYkE2aytGd0hCc2ZnTmNqTzM3eE9XMWdlQTMrbE9ITUtUK3lyUFZhdTZSWlNSNElKamJ4UDl6aU56UjByVjJZTlNLbEU2dUFJMnhWNHhDSjU0Y3grMEsyS3hvWkN5VndsV1FDMU1tVEtCVlFmc2g2cmFsRFpvV3NsQnRmTXh3NDlvYVRBd3dWUXpzekhzeDI5Nk5EMjdlZFQ2c2oxcmdra0JKRXhxbkFGa1RHVDJJeVhUS0RBSUxFbXdGVGExWU5lcmxCcEtIM29xZ3VCUjJ3ZVdkNDBqT2EvQ1E1MW5rYzFvb0NMc3VYZE5uNWpLYnNOeEtvTTdPOHV2L09lazlCUnZpalJCbGJGTlJEYTZyWlUxL1Vtc1VHNDdSUStRTDJXSjZYTHhuRTBNakExKytsNEpLZWRkYWRlTk5kWDRITjM3YW95ck8waVBlbjEzcG1hb0lWMkFzRHAwYTRySldqM3VTOTJJUWM3UGRaWFNpeG43dVYxeHlQTFBUMlpKdFVrNDZxTWU5aE02b2pDT055SThkb0xsdnpRZFJqMUVPT3FROUt3eldtTHFRR0J0bytvNUVBbk9TREhqSWMzcjZYRTZBSkVtWWJBY0ZvRGd5Tk9tK1BGOE9mcXNIb0hDN0FFTUk0a1VNT3F5NHlJbTA3dSt3MFVMa0F3NDNValdUL2RTRHB6RjYrcUlBMGdiTWFTTUJSQUtyMkN6NUljT24ra042aGxnL1FJSlVEczRNR2FhYmltZ1lSMUhuR3RZa1RQajJvdHBDNHZuVVdianZwOEhPZ0JSVkRrTENMUkJpQ05Qa0NuSlVsMVJwWG1DNUZhTkdPK3d3elRMeE5Vbkd1WFpudDV6MlpDTzkrZTZBbjdQSnNCSFJaU1JqYmhCWkRJN0JtZmtrUi8zaUdldVJScGx2WUtEeFZtY2NKTDBMSFRzYml4RHdZL2svMDhwSjJXa29IMVBGRmIrVE5mMGl5NmZVYm4yU1hhbWxwUDFIUHZsWXA4UEpQc1BoNWhZRDVyMG42dW9QbHB3R21PNVdLUk1WRkJzdXdHRk92dkpqTStiS2d5RVVhclhCcDRzMVAwcTNxUFZnTnd5K0Q3TllGQnd1OGtpd3hiZDRDelRXZTlhckFEME9xQnZwMVExdWxTR3FwOFZSb3g3S3NxZHNuaktvMTBuRmo1S1dsbUFCU0FHMkF4elFNQWJHRnhkMi9TVlVMREJpQXFtV2kva05rWWQyWFRWVGo2bjk0YmtmV1ZRWHRuNzlMYUZIRGtnNzd6MFkxVHl0RjZ1YUdPUXFqVm5qRlJkY3NWa0N3clNRbVk1NWdNRUVsZTVkZ3Foa203Q004bW9aM2tURWEwWjMrQWU3bUgxeXNuOTJWWXA2ZGtrMHp0QVNZcXRwZXdCMndTaXBkVVBxKzBWWWo5d2VBNEY3Y3licUNTSFZtU2hFZ2xCSmFuQVhXb05JRFBSZlhFRWtTeXpteEhVeFdtS1dNYzlVTTV6VEFsSjlaaDFpZXVvN3hhNjJTemFLcjNJREpVRnJPeEsvRm0vVVZtKzJhTVN6bktMTEFhSUQ3U3lWZnJLclhsLzFaTXJneWlBRUxwWm5pUklyZjNYODRZb1gyRWM2bEptRlc2NlV4R092UVpuVy9saDdxSEFFZ3hhd0ZGKzFLb25jVHpua09vd2FMcFJXZWRzeU9Fa3FIeGpOWndhMkRYMVFPcHpHaDNQdStTaXM0TlM4YkxQS0tncGNxRllBeERPNmRGL1N3WmpiYWhTSG9DUUFLT3QvNVRLZ2tnREpTSkFreFczeE1zUFBOMnp3T1YxRDdpdVpLVGJVSWJEYzlqQmVJOEgrWUZMeENZVkJqMDg1b25nY0kxUGdHSVFCZlBvWDUwLzV4R0dFcmFNajRCNFhPaEpDTzFYYWhhdzdEeUdrbkpoMldzd1dkK25YUUlqTjRhdzBOWitHNmJ1WUpEWlM4RFpZT01iVUJkR1cza1pKMVc3U3FMREcrR0FUeVZiS29xVXdYaWZSdEJkd0RSb3hITFJrS3dLVEFKWTFXL04zZ0JTSmUxYnVoaDNwNnFrWGFMcmZaQ2pWKzJsZlJEQXRKMFNlLzJkNDJmNFRVM2tWenRFcUhadGpBeXFPWlBjRkRBZWJWM3BvQWtINVMyQVloYm53L3d5aWVWUktydlNJZlV1K2h5S3p1QURVTHk2SUJBQ0FBR1B3SWMzT21EMFRhWVVHZGlkV2VkS2Myc2xtNUt5dkFna3NiS0VFSWM5Zlc5NmgyZ2EwVURhaHgyTklDRHRUWWVQL0RBbVFaVlRocW44Ujd5dS9MZG1iWU53K2ZpL3FGVHBCQ0N4aWx0R21wUDk1SDNlQWR3REpQSUZROEFoRk9uMU9BeGxtRlR1dGl2SzdiWTZ3amJrUUJqQU44QVRZVmJTeEw4K0hWTEtaYi9UenNDeVBCRE5USW9PeWlBVERicFBCUlFHbm1QZnBjQTBnRFZBM3hobHQyQVNEVXVxNEpFakdSWVJmMENDeVBOa0xvUHJKMDJEV1hXYWlPTVZOciswbmpYL0NTWWUzU21XblBuN0xSbFdmM1hSTmVSQTVCWmRVT1NtTnQ2WTZBdEs2ZkhNcFZJRSttTTZ0NzVFaGdNYVd4TkNjTkxZalpZcnJvRS8vZXNiWTQ4K014d2VFdEFYQ0ttSlBJOFE0MmhwQUlUc0VFd0pTK0NTcXNuK1dKUjlUamg5a2lnTUR5UEJNZzhFSWxBNHFaNXRTM2t0QU5QQStBblBoZUJiNDZYVS9nbUp1amNmNzROcnMwVU1wNEhleXRBVUoycGhjN0xpTVdyZ0F0bzdERmI5TDhIRGh1eDQ1MFZvL1l6Umo3S0VDcDVhSmwwaFo4TTMwQWtnQ0hxRGdBc1hiNHRYeEI1Sm1CTmlBSlZIb0tWU2xzRG5wWjRrU0lBWk5RZ2dZWnhodHBocUhBQ0VPTndkWWJhS2xXcUJvVE9mMGd3c3dkbXYwaTVwTkp2NHFQVXFLbnFRZ0NGZEVMeUdobGxQSjQ4WDE3c2krRk5UMjd5TDRBN0VDcEx1SGYwN01xakZ1R2V5N2xJaDdOb3NBQ0dxTkVpMEp4aEtIVUxvQ25EN0hJY3ArTXBtNERjSE0vbDVlbTZlQktiZ0E4QWNBbjNlY1N5ZE5zbWVEcFpqSm5ueWlWWDA2TUcybzV5aXNweW1wZWE0K2FsNHBqUm4yYnk4d050Z0Y1aXBHYllJYUN3bHNGdzRubmFCaUl1OXpKK1VteTV1TDN2MXhTekFXWHNmSkw3bndDUExkNjk5TEhOOXRZTU84dWJZRkhzVzV5NGdjNk5CTksvVnVWUUpYS2hSV3Z6RWtCcyttNk0raWRBVmw3TE5Nb0FpbndOK29JRXMwYTR6dkRrMmNQWWJwcWhBTXFRT3RtZXBDdnJKS0RBUzhHdlZJb2JJTkJ5bEZhVlluWWdZd1dDZFhyd25rS0w3Z1plWmdVS0FsR3h1cEx2SG5DY2FTZzlBVHdTUUE2RVg4aEtTUWVlRy9YZzRGZTNsanZPbGN1K0FOWjU0alBRQ3V0NXhuR09GdUQxbWFyR3c4NHd6QjZyN0lRMGRuSWxaT1VTS1EyOTBVNk81MnBqZEsrbUJJandlQVBtVTZvUUQxSktCODFpY2k1RHczTU1admd5ckFFRUFoeE1zemgrRGp5OHBiU0llMXpTdVFIeThTcG1mQThtdzhmL0p0NWdnUXR6TTA3L0MwSExJTVk2VEtRUEx2RXhuSWJUa2FzQ2lMV2tVQ1VPRDFVQmlmSkQ2ZmhyMkc4bVBWdXBYWmNDcDdaOERBbUUzSzNxaHpWemNMYTFCQXJZdElVSEE4bXNzQm80bFBuTCtDZ0FRc0RTNTJhNWlUT1I3QW9hRnp1STVyZVBBd1dxa2U2R0ppbUhWTGtlK3llMDdtQUd0RzlNemVicHFDWTlYOHdSL2lEa1BjZVJsWG9pWGEzU0hnSnZDWWJuQ0JrYzU1TmY3Z00rMXdGN09sYjZrSHljSHU3MHg0RS9lL2ZuZWJaMXhKTE9wN2VwNExrYU1BZ3FibDdEK1drSlpxbStSQk9jb2Q2WnR2VlpuLzQ0ektOOHAwV0Z0b3V6SmZJRUlQYjFRL2JzeEJ5ZTRFYXVMdzlpZG9xTVRyRWlQRGlMM29xcnVBTVh1OFliZ0NHcktucDNKNkZjN0Nkek51OTBhcXZZVEwzRjZMckswdUF5UVl0QUE3aW9KZzAyUjhWelRXZjloZCtXT0NpK2I0T1h4QzBDQnRVUHEwOU10b01ZaXNkVUFyRE9jSWF6VmtNU0lFQTE0M1Z0TzBERDFSK0ZBTklEdnZ1b3pXMlFla1JFTlowTmtFR25QMDNlWlI3cXc5QjJuQld6djc3enBxM3l5dnpQempDSzlPbERvLzFTVFZCRzFTWTFKSk9RTmp5bEVBTWxrQUNJMkdBSDBLY0VEOGVmenhCVFlqVW5WU1BMaFlFRUpFc0dQQityYVQ3RE1Rd0F6dVdwWC9Ec2tuQkNQQTN3RllCV2JidTZOby80dEdDQTVBcjNBOE9KWlk2REhyK3lhcVhxWXZ4T3FZV045T3l0NGRXQ0lZRDR6Y3dUL2ViNkdjMEJDdCtRTlBacm9ieDdqd0FBSUFCSlJFRlVlQnJVd0drdXVLZzJpbWdzcGFRUlppS3NxUUN5U1RJTkRPOEFoUG4wcXN5Q3hJUG11UVlqbFFOYnhsdG8wTjEzR0Z2ZFR0K0xCYzRtMTNXY2ZwSXFWMFlOR3NMem9sYk8wczNVQm1MNkk2QXNnUlV3QVkzcEhGMytlSUdaZkFDTGRaUWw0YXgzRzJ5dFFNQVJvdlJ6RURCQmcwQTZTSkJuZzdqSUYxTm5QRWYwbW91MUlST3RjRlZGN1g4UjNscG5lS1krWXowWmgrV2k4ek9QT0JRZ09jVFB4YlVjK2VXOVpSL1lBdnlNN3hlSC8wZElJNDh5TEhmcXRYb01yTlZnc0ZMZFdNc25ZdCtVVHArUTRkRktVRHM2c2U1ZXJ2YVVXV2c0SjJKemU3K1JNVjRDUjhTbklWVExuSUFSeGtJV3VndnJaRVROWUFLQmVxYnVnTlNTaTRFdTdvenI0ejFnZG1RSGJzdkRKWVdFVng0QlJHMGNDampUMjFYeTRaUEhLOXBLekRZVlpXSmMxckZiaEtvTmphVmt6RnE1MkZVVWs0SFpMWlVEVXB5SEdOZWFWclYxbUpGWlIyY093Nll5L2F3UXFndVplOWxPTmpxWjV6a0l0bEpyUjhYUWo3N1JNZXBxRnVwTXpjQTl0bml1U2JqRWwzZ0RoK0dUVEhtZXRWeUwwK01EYVFCd25uaWVZVmd0eG5QZ2VEaWVuMmQ2d0hZN3grcnNXU2ZVQjZDMDV5ek04TWk4N0lpMnBNVHhnR2U5UE52S0UrYzhqNFpRS1FJd3Fpa1pGaUF3Z1lQZkZ6NHpYdm1GVkJNMVdKaGFVbWNQZCtnbWZqTE5RNW55cTJ1WE9QcG5TeXZNZmc4Z2pQWTFnT3owV2QxdmRndUZSdTZmRVFDSjMvMlRtanlzT1paekhRMGdTMzZiUFZxRm9VUFU5cW1yTk5xbHpjbDZtVmYvVkxLbzZodmJiVGJKa2s2bmdaSTgxZ0RTN2E4K0gyTVRuZ0xJenRWa0pzeDhsaEtDQ1laRDh1akF5ZWo1V0dxR3RWcTZnRkpoRnRMNFNCcU5kb0xadHFmbVhYUmErZHdrYXdFSVFEeVgzS2MwOGlscG55azlIT2VDMTRvTjhHTXRIQjdIQXJnREg1L1BOb1ltS05oSFNDVG5HWWJhVDA3VzNzNW92dExiK2p6eHlPTXZQODhBbitNUkk0ajdmY2pRWnYzcmxLQUlZT2FqRGM2TWR5NjJDWE5xYmwvK3JObUMva2ZzWHdJVmREeVNscHZMK0k0cWNpWjYrL0dxS1pRRE9pTDdWb2RRK3hpV0RuK2p1a1N5elZaUmVjVmZTU2daM2xLUGRZcXhWNlk0ZEx4L3BaWk1DU1J2MFdYMlNrSFhqelAxWERybUFuQWo4N0UyY0V2R1ZudEg3SjlwZ0ZBZ0pCVzFkMGJvWUR1WjNEZVRDZzlyaTg5aXNuNnRXakRjUjhJTkVDQ2dOWUkxYytZei9VekszVHgvdWRxZ3o0Mk0vQlV3a1RydkZTS3pmaVpUSFRScW1vWHZCS1lMdlFGNFdueDU3NW41ZkI3QThRekhyZWRhT0IzNHJEMHh3SC82T0hxUGpDTUJCbGpQRXdlQzJSK2I5Nnp1U05iZDJzdmI5dlBNZ1hPY3NVUHd4eUx3aGNKc3VWSVU3NEduZVoxVlphbVdtS1Y5UlVBRVFHMllPKzBCK0JtR2xseE5ZdlAxeW96MDlUZXUyb3kzU3lLYXgxaVIwVUZGWVg0RGprcVhIRExWaHBuUkxsMVU2VEkxZndVZ0JJR2VTTVhETlo5YmFzQWJDYVRCWjNpMFlwNWNXdW5RK2ZQZlEySVVqWVA1VVNwSVNlUXVkU2R3ckdwZEFPblhNRUxRVEtQbGRSWUZLdFZ1TWpaS0NwSEVDZ1k2UzZtUXN0c2hGTVFHNkZpUG16MXV1YjlidHBxaFZaM3FoZ1paTXd0bXFyTHozVUk3bjRGcHVveW5XZTRsNlkxLzdXTVJLbFNvR21IZ2ZGcXJWVCtPTUhJK1U1STh6dDdjNy9CMGdVMi9FNDhWblQ5L2ZLVEtBRHpPUG0vMWg4Vkd0aU50Q2N2eitBQTRubmJneDJFVi94bThIZmM4V1lLdG41Nnc1K1BJbmNyME1abExzMnMxZUZUVGVFK09RSHh2dUMzVzdLVElpenQzVngwSnVTT0t3UEFDL0Z5cFhrWFFRenVrYnVYZm9RYU4yWnpsMmVYOVZwMThOMDJKcXZJME0rTWVRRGJRYWwyYjkvMStxY05hQVFicG9JckVNcms3V0VGbjJsYTZOVmlQU0xNa3h1S3NlZE4rN1U0dS8yWVZxeHdDU2Q1U3JPY3FEdDlWdmZMZWJRc0hHYnZSdzlBQXd1SlV6ZGs5RlFrby9LVGxhSXBxTnlHYmVjS0daT09nSkNYdExhQktkYWxXaXN6cVdFaTMvRzdPRXJXTWFWYWZhUUVJRUZpdmdpRjEvOU5pZHlzU1lKeS83bVBjY2xWRlZ6UnJGUVdod254K3hwZUN6NVJFZ0FBTFI2N2tKSDgremhNLzhyZzJNdjV6TGZ6cDh3emd5dUg4STkxTHpXTi9Tdk84bzVqV2F3Q0Fydk9XN2J5TXpuSFNRYnNmUjJVU2NXd2djQzZyQS9BRXIrZW1lY1FuUzJNM2srS0tHZUNIeWVxTU11MSttVzF2MUNoSHd0ZE5hZ1dXeVlSVDNkakJROUsrQVpDOEcrbW5TZ1BVVXJNN3FOYXMvV3pWblZZT1FkOXNMTFFXRklBMFU1SUhpb2FWUUtYTnVra0ZvMDRTYi9vL01GekJaK3NOQllxaUpRZWR4dnZpdnZmcWpPd0xBSFZKbGVsRytKNW5BdUJwZGdVMVNtVm9kUVdHL0FxY05VanlFQ2NGa3N6THNwMmVDVGluTitDY3kzRGFNWmFqbjlrbnB4bU9aWGcrdmZ4S3hnWXpTaW9DSVB3OVBwOVJUZ0xEV2xZZzhlZFArUlM1MFNiU1BmQUJ4OS9tekgydzhmeXNVVlJ5c0lleGRIRVYwaDN1WnhxM0taM3czSkdjYUE3Yit1SE1pV0F6ZUNIRFNrMUxRTEd5U21HT0RzYWhtNnNBRk5JK2xTejBrQ2x4dXk0aktqUG9jTjJscVdtbWEvWHNwRjM2c0JHL0hZNFVRQlE4cWh6SnF5VWtTaEVpTVkxVDBEcnRMRStBQ2RNMm9ZNXBjWVRDVkdPcTFvYllVcDZ0UlBwOENZQVkyd0RUQUpxemk4QUZic250MTMwdTZHeVY3V1lERlFVb0lWeno5KzBad295ZFowdEtsVTRCRXlvbFRRQlIveENJeEhHbXRGRm9aaE5VU1BWcE1lNStyRlJobHBWUENiZXlQNDZGSDBENlVSaFdpbHUvckpXQTVWWEdTb09uTWlKWFYycHNXOWhLbnVuZCtzc0NudWhJZjNvWW5pbkdMQXRqS1Qrd0ZTM2crTk5qNFhSUDc5azgxNFFsTE1ETTRTZjlXR2p3ekNNY3EvMUNFdUd4a0d0NTlvTnk0WkdVVGFrREZYckNlUlNjUlo0TkZMT2ZLN1ZQZ0RGYldIQllMZ3UvbGtRdW1UVkF2SUNJTGFiYzNVb2pDaUJ0SUxYQjhCTkFxTUpNTUZvU0Q2Q2lZVmxaeGovUW42WU11dFlBRnE3R3dMUlduUDBDUUVxT3lVRnJDR1ltUTVTUnZLMWZza3RYMjZNaEN6VUlyR2RsdEgya1BWUjdNSkZSK2RoR2JIR1hsL0QyUU4zVW1qRXdybllSWHB5blJwZEtQdHVyQ3oxRmUyWktkYTBLRXVCZzAvbGFzZEpsTGJId1pDNUh2SDhHSjZWdHhlS1VzZ1U4ajZQYzQ0KzF5dEJkMys5WlZzdXV6Tk9BUEQ4a0FTenRCYkdKTDR5b2RnTG4wZmFSNkVYSDQ3bnd5TG5mVnFnOGo4OVFQNG9CM2NvdlpOVXBaN25uNTJBN2gvOEo3WmErSE1zWFRqL1R3NXdyaGtKZlhzOXRPczhjc28wVklvL0lwY3JnOTNoWDF1WXVuNk1rR0owUk9YNGVkMHV2TzZoY1ZKSzdGQmRaZVBNc0pRRXkzeGFBU1A3dGF0MmdOUUJrNUtjQW9uNGJFMEJxeUJPa2JtQ3dHYWhCaHFYUmo4Zk01S1FybEdoZWVTZWo2RXlyT2VaWUtpQU9zUERLcDFVU1NXNHRxZkRTVlJodHkxcitsWEFWNjZPOHpsT2xIZm5wRlpIUkdpaXZVT1pWWUdpUUR4bzFjSFZscHZTekEwaEhvNlJnb3hJMHRCSVlZc2RxZ0V5QUErbGU4TWVDcmY0d0dEOXRVRXZHRmh2aUFCU0FQSE1wVmsvL0FzNDYwTmpoZVBwQ2VvREVjbTdhUE94MG5QOEV3RE0rZ3VVckRlS240M2ltUGNGalZlWFR3bjdpTk1LS1NtRnd1TmRVaGhNbnpBL2dqRDB5VkVLaW5qTHBVTDJTOGREN1pQaCs0YWt1N3U0cGhNZnlzQWthVFI2L0tzVzlRUkM5T3FQSnpPZ3BkMVZUZHZpWW9MRURTSThBem9pWGZBUkFPTWk5NHNRSTFObjdsUVN5eWxWZE51WHIyU0NVSHpZQVViZDIyOVNheUszZ3FEd3ZMd0N5TXM1cUduWEZvdUx5SVprT3pNY21FSEZCZ2xKSTFiK1laTnBXck9weDB4TTJ0ekxvUFNPV00xdmxKYlJyWmdSSXhpdXlVa3BnZTZqa1krS1loNjVYcjZhSWhMTENYa1dKbzhEZkxIYy9HdzdySmR0MUxOVHhrbXZCamxpeWhTM2dzZEluSXd5YTVxMHVuY3R3SGtmN2p5UW8xZG1qcVlwRWpmSWMxR3JmUEVJUnFEaHhXcXJqdzhLT2RwNGhNZmdaS3NMblo2N2tJQmgvSWVMNTg4Qkt5T0ttU1F2WWlIYm1jWTBFalpOaUErMHh2YVZnZVp3WjI1M1Zxb2Z1VGRvM1RYb2VzV2o3SUhsem1YbjBFK1ZTcWpQejVDS3FEaGRZeUV4dWh1c3VnVEFmeGhqbm8wSm1wUWtnQXl4QVpxR29kUWNndW1tdWowT3M5Sm5HSmVkYWhkRzhpa2tWUUZibFVia3BnSUF6dkpXQnJrQ0JBMTAyMXdHOUdzSGlpdEVFUUFwTUFLR3ArNks4Vm0vNlpVZ2hRcWkydWFiVjU5c2VGN0FhSnJvQ2xJNmdLZW4vRWJhTG5rQW9uWm1vZXFqOFRBQTF4WDV4ODRhbFk1clJ4cFR2Vm9CSjJFRkNvb0Fac0JZK2o0WHpXRHlqR1ovTDRBd3p3eFB0ck5YK096Nk1vckU0NHJEMGx6ZXoycFMyMG5iQkl3TE5EUWNNc1NxVGFzN0hBcDZ4bzdpa3l0bzlIQ095SmdqMk8vZmpuTjR1VEtkbHNXM2I0QjZhMHdFOG45SURxZ0hFUDF3Q1BzK29MNERpblRPMXN5SHAraFZOb2kvYjlnaEFqMGU4QTRHK1RIdHl2dENIK1FxQTdsTVpZMllzd1RLZlp2d2UvL01Zd3U4Q1NNR1hLWmcwS0hpVmgzNWY1Wk8rOWhScHBhYm4xVjVoVVB4aG1hU0ZYVzVsdjlXbVZDRFplTEptWkl4c3U0M3J6cHFJdlRkMjFVYkRoMnF6NTNsMzNRQks1VmZ0Y1RNRzluZ01JdzJHV3A3bHl0YktkM1MzUDFkSUtGeitQVll1YzJTZ0p6QTlhVU5KQUxHMTRNZUNMNHZQYVQ0T1BJK0YwMEpDNGF4OEd2bzBNVU1CeUhPZEFZVGNHK1BBMzN3QXg5Tmg1NG5EZzZuaWpOanNhWGQ4eEVZYjJKa1N3UVA0ZU1haFFZYXdwUnp1ZUxqWUdmZ1ppbFNUQW94eTJEcmd2dkRqR1RTdTViQ3RyWi9Ib3lXb2ZMV2tMbHhST281eGRseDJxQTlQWmFiVmRCMDJSd0ZOQ0MrT0F0QWhSdVo5QVNaN2lQcGw5R2pQd2JObS9Gc0FzU1pUWjdGSy8rc0FCRkR4RGpkeENSM3hKTEJUWUVmUnoxZVdYR29NaXFGcnhtRlFOcVdDQmV1K24xL1p0aFFCSFFKT3BpMXBwS1NQcWJZTSt3TzZYKytra0c1aXRVWE45Mm9rbGNwS0hXMEU4NGNMbU5YdTFxcEtnVnhLRmtDb0xaYmxFUnhLNmxtR2M2MEFrR05WM1gydFVuYytsdUhQanlOQVp4bCtaTHhucWk4QktBRWU2empBcjl0OWNvZ3N3SEdtMzBtZUdwYTJBM2ZIRXc0NzRxaUE1eE9oSmhuZ2x0OXVPWE9aOWx6NDVYbkMvU2dtWGg3THdZY0hLSDNtVi9ZZXViR1B2YnJ5M2p5OFUwUGRpSFIvT3M3RUhDdXZWbDRmOFB3TWhoaHNFU29TRUV6KzlGQUpvME1vYWJXRXdtdCtNY0p1YjR1ZnVNUjdLMm0wb251YjRRVkdkaU9xelpqQlFMM08wUG1xVzNyRTEwT0hJa2FLVGh5QVh3REl2dmVtamFnTVZ3QnBnR29BV1FOQVNGeXBNMFdIWkx2YU5sQzExcnhOYkFlczdIQjVMOGdiem1YYTR0b1Zjd205VjQxQzR2VUxrR2hrdFhleHpXZDNDU2dNK3E2cWxQcDhLS2hNMmh2NHZPclhiWFdneFdJQ1ZweWhFbTBVeTdZQkRBOGlLMVdiVkYzT3g4TDVPQklrVXVVNUZqNFRkUHh4Z0dyT3VSWmduaDZ4SVVXRTkyb1lRVCtYd3o3Q29PbDJ3aXgyNy81aXdPY1pPM00rUHNJdVlJakRpYmoveFR3WStXK2VxODRVT2N6eCtZejd4K2NKNURvaEFEeEVEUW12MXNqbjhEaDBHWmxuYktkWitIaEdLNTRXNTZLa2VUV2N5RDRXL0R6TFV6VStYblhBMHUvNU5OUytJZ0J4RkNSQmJxemVYSzl6clRndVFRREdZWVU0TC94RVp0ZzcwSmlwYkF4RSttamNwOTJZOFEyQWtDbHMwOG51QVNRaHdBQ0hTaUdiNmlJemFKK0JvbHYxU3htcFJpZzlQV2ZWVldFTmhPcWhTdENwTEt3WmIrUk5PamJnNStORmhUQzJENWw0QXlpYmFjcllleU0xZ1BsZHNYL0U3L215RTNhWk84QjAzWmtEbmJnV29zMjQzTW95R0VhdjFIWjF0M2JnbzhweXJIS0w5N1dBeHhFRzFzY3FvSGd1dzNtc0xvT3JPVWVDME9FNTI2ZjBRNkJhd0hFNEhnczRrMnVPTkNBZXp5ZmNUL3h5R0dBSDRDYytuOCtvbitjU0xRd1BkenlmanNNY3oyY0F3bU1CUHo0Vy9weUE4blJyb0VDcUZGUTd2SmVYNmRIdWJ2Zzh2STJyYVE5NWVQaHJQTTF3ZUM3R3U5VmVucENXVnJtMDF5Ny9Bemh5TGZucFI1MUZmYjJTM3VPNGhIRWN2dlFUNlFHVmxYc0JIQjFOOHJrRmp4cVpQZmZKYUg0UElBWjhFMEFvZlF3bUVnQXhBY2oyUHdVYVlDRFEwL1AwYnZ3amJVdktVZ2Y3dllyRlZCc3pGK09EZG93OWo2dDZzb1BQYUdNRmtKUTZkQlhHNzlKSmJnVklONEMySHpLa2FZbzJrL00vTWg4YVcxdllidnFvMGl6ZWs2bFhTaFFybk1Wc3JUalRNOE04SGNud1dIZ2VDODhqYkI2VVh2QTRzTmJDajFSdnNGWTRxQzNENTNMNFl3R0hOME1aWU9iNDA1RVRyTVU1SDU2Znlmd3d4L094ME1kdVJHMCsvVWlwdytGK3BCMGg5OTA4VDN3dXgzRTRqZ1VjSi9ETGVlTDVERVByOHd5SnhXRGhDeUx0Nk05d05EdkxhU1RPVzZsUFFxUnRKYjZ6RTE2cW54NXl0RG53WitjNGl0OFR1U0lrQStDWmptbEh1dDlmemFraGhmQmEyN2VTcjBjQmpNN2QxZ0Z1SU9yaUZ6SVlmVVRzOTZ6VW1PRk1aaXl5dWN4UW1OK2NBVkQ3YTE0QlNNZk5HVVp5SnVOTENBS2Fyb3hkVG1VcCtqSzk1YjNPNENiM1ZGWDJzemdZY1RCcUFSSUc4RElPWmhRQkFSdGg5NkFndE9VbFN1VlFiUmkzZ1VMVm5MMXRXdlZndVA3U09GcDJMU1lzb0xBSzU2dGxJQ2RIMnBVNlBBMnBsRTdTdHJHT01KdytVMTJ4STFTWDV6cGltZlU0UWxwTXdQbHhITEFGZkpxSDVITEU2c2s2d25ZUXVPWDQ1UWlWNUlCajRVeEh1MmZXS3dIQzJZS09EM1A4U0dRNWhVTS9BVHpYZ2NmaDhPY0pPeGFPODhSNXhsS3duM21JOHVtcGNEU3pucWZqMC9LME5Tdzh6eWpiTE5KU3RPUjVKb3VPays1RHRZcjlkdFluMWVjbW1kcG9od0JPRDBRWlY5aWM1cWppL3FDeW4rVzRlT0ZzeGtGa2U5Q3JoMkxXbGpKdXdFUENYd0VJTW93MjQva3hIWjZ6S2lDeUFVamxLMEIwQlJEbXNDNk1XNHhDc1U5RWN0bytiTnlqQVNhclZ3WkhxYkpyM3BqU3ptVUxmNUhmYmNXTTlpNW9xWUhlb1UwM0kxOE1vRnEycENkZGxVN29tcW02WHRwdUxyK1JXVXRlOU1iVlBPbC9vOUlkRGFXV3hsU3NsUkpKZ0FRbEVUdENiVG5MWUdwbGFEMGY2VDZ3VmtvakZ0K2RNV0E5RnZ3QnJNZUJkUUFmcTQ4Vk5EaittVGtNSjQ1YUhPNjl2RFJVTnZTZU9BejRXd2QrWk9VL3pQRWpwWWEvV2JueXd5TVNuWjdQS3lTTjU1a2dNR1dBNHp4anlkb0I4eE1mYkd4M2ZKNDA0a2FCWFBGaFovTHpGYWV2Y01FSGNsbTV5M0M2eG51QVNHUy8yVVVXelFGS0dMQ2U4VG1MY3EyQ3JNN1V3TGlBZ0w0TC9Rb2VIOXpSOEVyNUJYanduWXJWcWtxWUFJaHRJTkE3ZkZYRnNUY0FJalJZSHY1c0E0S0VVbVh5akVOT3R3MUFXQVRRWWVpNENoYXF4cWdxVXN4MWFUL21yWWdnVENyTkdQMkJJcWJVbGdIVUU3eUF5Y2o4UHE4YVBabmZCTkV1VHdGSWFSbjNkbjAzbk50bzA3QnVpNkw3V0FVazdFWmY0a0tmVWtqNUJWbEtScGtHeDFGZ2RLWWtzOWJDK1FEc1lWaXB5aHptT0E3Z0Z3QUhUdnhUZThZbXVHcVJNOEUrd09LZ29teHhhandRd3RNL1MrQTRBUHdBOEFNaFhmd0M0RDlKUTV4dy9MRG9sWE1aUGg5SG42cnVIbWVZZU81dGNjUGo2WENleUpjdC9qalBNSHVjVmtEaUZtcVhlUUNNNTJwUVRPenE4dWRqMStTaGptbHI0MmVSbU1zaitQUXlwbXAvYjUrTTJObHFtM3M0b0Y2Y2tYcE4yd05qTDJPNjBodG8rMmh2emNuU3V3cWpaVndCUkNzYTkxY0FLY1dwNGw0QkpOOHVLZFVJZExpQVNyZlBaTUtadnpDTmh1MXR6YkxRaVRxdWxMbTF5RjFycTdyVEo2UFpKUjdmQXplMkUydmo2Z2hrZkpIWXREeERHUDFZWnhwVTI5amJSdXFXU2hCcVM2bzFua0JBbFFYMC8xZ3BoZENEOVZoNFBBN2dXUGh4eEVGRmxFNXNBZlpMcUM5WWFieGRqZzk4NGdPT1g4enhTeDU0ek5vL0VHckVBYys5TVY1dEUyUkdtTUh4VHd6NVdRckRCL0kwTlhOOGVJREgwME1kNExzZjZRcHZkdUtIaHgzbmdiQlRHQUNjSi96QkprcmdjR0N0WERseUx5a2lEcUJlNlRVV1FLUThPWnpIMGxqcmZnNUpneDhDWDVBVDVQSXEzRm1NRzIzRUlmUzRHMzVYZThkOGYza2pETjloVjZEWjR4Y2JxWXFpNmdOa2JiczVOUjlqanFpOUdnSkdYUnBocE11NHdzZkc0SGQxY3k5UFM0TXdZdEdDQVpidHl6RnRIMXJHbmIxakFFT1dzWi9qTUpIbFV2VElNTkxQY3RVbWNya2NvMzUzUUhKek80cmVIZWhnK1pFcEFSZkdvZlN4SkR6b3p0NDcya3NWQlNhcnBCaWphbU1HNUVZNzJsdldXamdTU0hBc3JBK0xqVzRMWWF5MUUvL1VRbFg0QmJIYThURERuOEFOY3NpelBvTFZ3c2lhTTcrWWlFTnU5bGh0c2NqbkI1cngrdG54QTVhZmdvaDZmbFFjMUw2Y00wR0h5ekpQV0h3ZmVJVnJ2NTBuL0lrR1luaTR5Y09BRXpqUEhOdmlobXFJYitmR2xUdUVUcnBYcEtFV1lhYzVWeTV5dk9IM00zY0NNOFpERWV2MThNQTJValgyM2R6MzRsa0FvcVdVSzVpOFVqVGE3akE5T1hZVlJrR285TzRYRURJQnBBZHZKTFBNbGdDSDJvSmZUTDVtRzlDWU9oaFlWWndORkxvOUtERTA5Uzd2QzF6UXFvY2xUU1BEL3JuMFFKYzMzK2tPM2FzZGhPRVRkc2MrREpId3FxSWlEaGRObERUMjdQUGRTakNnRVJYSXpYSVpkcVM5dzFlRWU5cEZjQ1NBcFAxa3JaUitIZ3YyOFFpMi84aFBWNjd3M2ZnVHVPOGxRQ0xBSkZaa0hwYXFTNExKSTc4ell3a2tManJCUWpENlNrdkREKzhKNE5PanJNTU5QMUtkUGkxc21DdmYvY2pmSnhKQUxISi9CT3NIeTFzWWJaOEFmaXdNSjhjbnduQjZuakhScmZPTXJ3SXVQYnpweUc1cFA1S1RhOGRaRlZ0VWJPLzUzSWZKNUVpYkNOdm5Sb3JvbnIyN3ZnNS81U28vV0hlb0o5aWVkL1ZJQnJBY0xLUWIreTc3WVdDYkJITEo2WmEyMnE4d0FFU1lBTUtNSm93RE5RSjM4NVZHcE0xaFNzL1dKSmxHQVVRcDF0bWE2WWZFSUNDd24vbWg0VnJ1WXI3ZnZPNmlXdEZ2TXlZYlMvNXFseTdCRm5ZQk9QV2dEWmYzTUtMU0M5WFMySHFrTDRnZkN6amlmQkdlL3JNT0xnTmI3czB6TER6eFlhR2VIQWdwNlJlY0JTWkhxaStIZWFnek9KTm52ZHFKMnNGaDZRbUtBQWFlTWZMME1NRitHQ2U0a0JUT2pQUEQ0eDFnK0VqUStPR2VlM3FhT2RPbEJKL1pCZ3NvOXpIbnI4ZldnU05QZTNlUEQyejF5a0NhTU9HZ0NYZTNrL1JoMFM5NHczR3hZT2pZZlhNbzBadE05enRsbmhlQXBKcTdLV0FJaU5YS3lUYnIxWnhXQnk4cnpSTkErSDZOY3BoTEk3ZzJSRDB2MGpvWmd1THo1QWtUSmhBSlJJRkNRR2kyQllUWkovY005UWI2SUViSWFrZDlmWlZLcXQ4bDNsQnZPTWhFYXREZjBWanlYRzFtYlNlNXFENXlxZkc0TThsY1JDV2gya0tKcFRiYmlZVGhLNWsyNHg0SklINnMrSjV0cWp3ck45cUZTL3NDVnRnZ2psZzBUUkFKVDlSSDFpRWtrTWp6QUFFbDVOMWU3YVJVNHVXZ3hmcXRCSm9sc3NyVERYK3k4UGQ0V2h5VzlFdmFYWjVwaDNpQ1FCU1NSN3FNeGNGSzZJOXNBWEdjd0RQYjhmVDQ4aDBCQmZuN0ROZGU4QnNHWjg1Z05BYXpyMko5eEdLejN3WmNlbXpycXdtRzQrZmh3bWkzRVYrRTc0eDhtL3Q0WndVbWQ5SkhxeXE4bzhxaThaYWtVOTF0aDRXK253QnlWV01VZXByQkNXWVlQZzBqMTFSajZFZzFxczNucmF4eEtCREROTWxFa0dMVUNTQzNmSDE3UDJoNmtXWlBTK05hMGVXYlhhYm83anJXaXRCV3VlbFBRSkJ0L3hvc3RpOXlCU1dIOHVxNFBQR01HL1RPOUVRMStvODhGdXc0MGd2VjByaTY0TWVCejhlQkk0MnFzY0licDdQL0tWZGJQbkRpa2RhYVh5eVlQNEFqbG1xNVJjVU15WkRFdUtDVFVnUEI1TXdSR21xSzVUa2lubEtLUmZuRm5JWWZxU2J4azU0RURTRFVsdzh6L0FCZzduVkMvWUhJK3dUVm9CNUZaMG96Uy9JQXJDU3BSdzRHTGxTZkdmRzAvbjdPVWlQc1JvK09GMVc5eHk3ZXQ0QXhIbCtBeDNqWDczZUpvK09wQ25JRkVNMXZoem5hUVh4M0prTWJVcHVLZXdEUnQ3WHVYWU1XN1hhZGtRdlc3QW9nV2dldFk1VzFyL0RJZS9sSGZteEVWcitmQUNUaFZ3R3RYYldaYWFZYWN3bERML2t5L2UyU3JvMnM2K3IzVjFWcmwxVEc4ckJaSFNGS0lPR0tUZm5iOEdEZ1BDdkVWK3pvOVhKL3Q5ek5HLzMzTU9ESE10aGhzYUNUSStaUE9IR1EyVFBzd3d3TFowb2duajV2SVlFVWdCQmtFSDNoYWJkOE9yK2RtK1BLMmxEOFJCcGhTMktSejRpMkpTK1pmODcrUHl6dExOa0JQeEFiOGY0TVMxVW8xS1lQZUFIVTArTG9SSzRBcVVwSXNEa1JkWHg2MVArSmRnNWszL0U2cFFlUHJjZHpXUU9BbnZhT3VUUjN1U1R6TzlIMTNnNHltVGxDcm9BU0JrSUZrTHM4NDc2MzhtdmVtenF6cVRFN2dQVHZsWW1IaUMrZE1OdDVTZ1lBWWdueFJwSVlvQUVsZVd0cG0rbnF6bDdRZktuTC9mMEZXT1I2ZWFCUlBxam1YR2t3VlI5S0lHTVpXYlBaNnNRUFVwZGZCOXQ1c1c1WjU1UUFseUVQWWJZSkh0Ykx2N1lzTnR2WndwK09PQXJnODNIRXlsNFd2QUQ4Q1hHQWtNRkxuU0dZUE14eU9UZVlJa2dLUm00VHpnS2RzbUxXWmh0NkdrRlI3dU1zbXFCaENLTXNEeUh5Zk1mUFpSNUpCeEFTU2Fnd1lReitBZURENDJSN0I0ODBzWlJFZW9kWUcyNnQ5dFlneThuUHQ0RStxeXZkc1hkSE4yUzVsb2JueXp2VWNVMnR6c3pwNmoyUTlHQWt3OTBOd1lsb3UzVkRBYVFVbVRjU3lIc0EyWm5BU25SZUhmS0t3c2FPSE1pV2c3VTMybGxMNll3c0dwWGl3WVdoUmNxNGV6Y0ptVFNhSnRxdVFZYk44RmFON0JLbXozZFo3N1RTbTlHMzVacTdObVNlRjhES1NEbm1rN20wZnpwT2ZUNEM3TWRvbGppNmtFQ3pjRmdjMG55a3J3aFBMRnNyQUllSEQ5a0tZNk92TUp3ZW16SDFnUk9IaFROWis0T0V3Yk5PRzBzR3B5SzlHeVROWXNjdGJXS1VLQUpNWW9hbmluT21GQkFwV3hLbzVYU1B0amtSdmlYUkVON2ZwQWxSSnRRWjZZL1RMZFVoSzlYcEk2V1VZOFhxRDlCZTdaWmwwKzVDUXkzN0FBZzFLZEpjUndwQlY2ZVk4UVU4RlQzdkIxb3plQS9UYWxJWnZIWk5rK21ZaGh2c1hnT0lERk83MlJQVFVGYi9UV0RaNDJscW9hOHdpVGRrZ0JzQXNXYndVVlBCQlBXVHVBT05uYkVMYkpXQnNqMUhjMncxVXhvd25wR2Z5ZGg2eUdiYVhRclpPM3dIaWYzYXdRbWtHYk9OMWJZU3J0UnNjK3RUd2N6Q2ljeVFIcWZwWVpvN2I0MnFUTmJ4U0FQcjh3Z0RLbGRxZm53Y09JK2pWbkt3ckZaZEhrQ3BJZytFYndoQU93alNSZ0ljeHRtWkVrak15cDVNaXVxZnJoalBLTFZNeTMwckJCTlk3c1kxTXJPbDMwbllReXhWb2dBSmZtekxTL3BZOEZvU2ZzS2dMdXluTmZPZjNwSUZuZVRZT2ZSSmVicmh3ODd3UHhud1Azb3I4N3Uzait5ajRIR04xcDAvVm5WdXBRNE85aXQ0WERoQXdxNlN6QlZBYWpoZTBwdUFHTEtMRlRLWXVqZlZYV3NueklRK0hDaEFvR2ZCSnQ4YWFLekRHbUM2TG1ObFNTS1k1TTk4NTRIUm1PbUV4dkh1cGg2T3ZWNTI4ZmQ0cTZwV1BnSm9XZmlaOVM3bXgxeVJ1YU5yQnhoYnZkUGpRZ01OcWg0ZjByWmxsZkZhVWc4RGFtZHZkdEJodVdQM2lHTUF6TklHY25CWGNOdXh1UHJ5UVZYR3ZEYWlIZWI0b0MyazZwUTJFYkJQdVJuRDVTMzd0RDhCY2NwTXNSQ1NDZTBhRWQ4VFBMSTlZNWNjb0Fac1QvdEpnay9VUHlMUTc2bWxoNVk4VG5OOGdNdkVzYzd6Z2JQMjhrUnpwOVFpdlhlNnk1WVNWTmpUTHIxVktzNVRldk55UE9LOXR5cDdzYTk3OEpBNE45TEtnSWNiZGVVT1FOU2xmVW9CcUtab1E2cEN5MWF2dlFiSjRBTUVLcHRzOEpzTTdqQnRQRyszOWJ1OVU5dkNtcVNQRUxkeEFBQWdBRWxFUVZRT2ZiTVAwbTFrTWNsRDZmd0tKTHJzTjNGdC9BeXcyTUZxYndzam85UjlvNDZ0WHBYaGI1MURDN1RuYWQ1elUxMmRPMEsxcHBiYmU2UGVZWWEvTll1ZHNoYnU4SThqS0gvZ3hJRW5ma20vajRkeGgyNUlIWmFHMXBaQWR0QkFoVHRvSTdHeFl6Znd6Y096bEhCWlRCOUFFdEZ6K1RiQmdRY3poK3JUTm93ajJ4R2VOaUZBSkpHSS95d1lDeG9ZRCtpOENSUUJITW44U1NPTnBrOUVHKy8yajZlRkcvNjVoZk5hRUxmM2R5N3UxOEUydVdnWU02UDFiOTVoaEYybGxUWXl0QWhzdUc2cVkyWTlqQlZBbUhvdncrUnZVRllTaHcwQUdTZXE4NSsxdzlKVVZ5cTl0cDQxVUZZNkFxdmtQOEdtODNrRlBneTdsSVdiQlBYKzJzZGZBYzdvSlorZm5WVG5OTTI2YlI2ZHo3NDhyQVpaYzlTcUN0VlJTaG1jRTJqQXpqMXBKZTdYMFFCNXY5SSt3bk5WSGJuVjNzTDcwMnJRaDRSSzU2MkhXY1RERGg3MWFYcHBqVDc5blVEU2NnbkQwR216d3JSOVBKMHU4eXRYZUNMMVliRmpOOEFqd3M3VVNNTitRWWtzQU9LSE5iQVJlTUxHWVFCT2ZCamxrUFFiZ2FWUGlpZWdpS0dWNmFYZllnOVFYRStvQy8vc1J6NS9lVkR6c05EZnFDMDdnTndaTXB0RkdKZkZ4Ni9MaGprRkVBSkZaRytTUTQ2d2tqNTJPOGpPbkUycjRwQnRIRXBMKzhqUDhzbnFzYXRyV205Y1FXSXJQK3JhZE5Tc25TWGVIMHAwQlJUTmUrUXZhc3dJNTVHSk4xMXQyNE52Z1lQeDM0VHQrYlg2MHdTVkVaRzAwSmtNMWp0MnhlRXNITXhXSFVCRXlXUHh1ekxMd0tNQytOeE1IbVh4MEtpVk0ydXJNaEYyaUoyampmR2NuazU1bmlxQUphRGdCWkFRaGhvTmVXcDhlN2dHZURTUW5HTHdwR1J6Sk5ON3FqaHVZWU5ncVRTQ0l0TTk4L3pXbHFLQzdnK2pZMXRjWElIaE40ZjEzRExkZ0hkbkV1Z2VqdXM5aUpodGIxK3h4Z1NQam5FRmxnRWdHekI0bHRrcjZFM0hIWUMwUE5CMDNZSFlnSlZrL0lzYUE1VGowN29tTG1ZUEd3cEpzaVpQS0hFTmxESkhIR0YydTBhNzFBRkFxZ1ZidU5EM2tySHZwSk9iZTZwV1pJb2R4RVo1K2loMVUrMkx2NjBPOVliSld5QXFDU1Q3eEpKOUV4RksrbkhPNTVHSEorSHFuTVloWHI0ZEw4VHlJTEUrSmxJVTFXY2lCQmJkVnZtV3VOU0E1NWd5bExQNjZTSXhaTDI1bzVmK09CaEF3dm9SWkhqQUVNcFEyNDV0VzRlQXEwNWNibTRielFITDc5OHdyTUZuM1RRTDFldlQyd2Zrdk1UcnhZNUh0dGlMeHIwZGtwbmthMEM1U2g1eGJ6Vk5XYzlNRXJZRHlMUzVOSUJ3T0hJRzJVOG5Vemd4emM4SUhoeXNBTDlwMnFzaUJKY0dISjQvcFF5Z1lMUzNDTE5SbE5scU1nQnVaMjZYSjlzalNCb0hoTzR0aHJHTkoyMzd2UVB4SVNTcEx3ZnYzWkx4dEk5NERhaTlmaTJSeERYMjhtUmZOQ2lqajFGMHh1bjdNaXF1NkNzYUorbW9GV0o2T2xDaDNkbHo5MytwTUJ3NWg0eVpwc21ya2kxcFVQM3BqWGh3cXRKcFgwaGJnN1pMMlRhQXRsTll0TjdUUlNVd2RVbTNWTUd5bmZORWVxb1lvUmJ4WTVoNjVka2dGbU8wUGtMbkhOTnRqUDJVaExFVjRIS3dXWGNpYzdkMmlhKzBtZDlZNHAzcDE0dncxd0NpYktSTVcwT3VkN2NWUStvbkZTOWJrTjhBQ0o4YnJpYUFLTDExdjJ4VWw0eXAzM3RWLzRYeWxyU05xUm04QTBpUmVsVUIrN1ZWK2d1aEk5MnNDMGtaT1hYVFhlYlp1MTU5MVM1M2NSeHRqN2dtZmlXaDhyV05lOWMwbEJUTThrTlJYWWxLSnk3dy9XRXBDOVVsKytVMGk5M1dsbC9ITTdaUjI3UmFKYjdXbGxwVFR3d0MyVVpBU0tuelJzWnJzTEV5bHNaZ0loakV1KzVQTWFZaUpRSHZiUVpzYUFmcWc5MDB2SDRBK0V5QVVoVm5jdWpLNy83R2RaYVVyRklaWGZHNVE3anphRmY1cTZHZkFLYXFrMTV5c3BsNGltNi8xK3NLSHZzQW4rQWtBQUl4bU5ZTWQ2L0NURkNTNlFvcWgrajlIZTJUNGVLK1JoeEtNZ0YwYXVocW9BZGs3cy9MMmZWUzQ3Zk5kQUVRemRzRWlMYjhSdGlMRi90UzhYZnZXVGFBQWVpdGZraTg3VjdybzVuZmJkWmlHdDJUUTFkcnM3bkVmbWtiYnpEanNVQU9ubEhTVEZqZXI3U0xXTHUxNzFMSU1qV1pkczFhWUV3QUdiSmdoek1Od1laMkR0cVRtSXA3VjFyRkM5ZDFXTmcvRkVnTUxjOGRsTkxsL0pKSFNoVlBVWEhZVjAxa043N3VnV0diR1ZxdElxRFFBNVdTeUVkbU1iNnBsMkg2eVltdVAvRFl3Y05tTEh4MVhZYitEaDRBOUFOU0E1M3ptUUJTa0hMSm93SEU5cmc5bkcrSWl4NnR2UUhXNFkxSDJYM1dhbEh4aGpUSWt1ZHkxWWFBaWVhTksrUHRCUEt4VG5MYndrZHJ2VWpMOUVMMmpMUFJWdThKRmp2cDhrdUcyR1JES2JjQmxYbU80a3g5VllSR3FTeXh2Tk5ibk9xZWtrcTR0aWVET1VvQ0NkcERCZGFWSTUxVnFlSXh6S3QzQ1FRTmJXYW5WT1FpUXdqWXJPWmU2V0ZqQjBzZVlSeU5QQWdtUWZjRUVyTm1lRzZNczB4UFZZNU0zMFdHaWxPbHFZaWNSSVFOSlRMUXNYaGc2N084M2M5SnZaeWJtdGZuZGtvYU1QYk9ZR1BlSzd2ZnZWTnhWTi8yeHJoNG5yK283ZnMwVGRYN2J3REliZ081K0VsazYxdDJ3RnkydGNvV01pRFZtR29KRk0yWWt1YUxxOENoQU9vR1VDUkxaWDdiTXBtcTNjYVFFbllWdHJkckdEQTZsenNWYUl6SFYrODZtN2ZscWloOXAzSnhpbFIxc0JaQkRXMGtsZU1DdUpkbTVZb00weDZJbFFhQ2ZUQ2VRN2ZueDZxTUQ3QXhJQnpGck1FalFPYmRCNTIwMVZVTmlyeE15b2k2V0tvb0UwamNMRDc3UUJEMFdRTEhhMzJDNC8rajdsM1hKRWQxYmRFaEludWY5My9jdlNxTnpnL2RCVGdjV2RsenpVMS8xZW13dVJtamdXNElGajNMeE5CelpkakZUaXR6cFE4NHdQNkJERkI2MmlzdHJPZnIxLzAvWkNQRHdZbGsvVVl2Y2c4aFFZQ1IxOTZtazNWRzlhQ2V2WVMxYSsyRFowbjJCdGVWeitWdUF4QktaanVieFFsTmN1OTkyalF1aEZMRzB3WThDNlpNcmM1c1B0OUE1MjJLNlJ1MTNBR0t0N054aWU5dDl6cmkzYU56S3dtMWRqUFdrU2tNRTJpVDdZSmV3ZFRTQytLQUpkSEwyYXUxZjFQQnhRNnlNdk1vb0NJTWtsNUZ3VUZzRkJmTWhEc1NaMEdwYkN4dWVUVHlXNStCeEVzbW90MEJ5UmNEM3dva3dybUVCU2FjSGhLaEc2QzRXMzArOVFpNndjODI2ZFV2WTBwWlMzNkVERTZKL0kxN0dtbDBBQXVQZUZmWjVrZ0psOGNTQU5XOGRRcUdQRXpsWGdXZVRtTDZ0N1MvaWpGb0pjbzlzZ21VaU44bjdUTXh4aFIxb0FvZzN1WTdha2VhWnVsVkMwQXRsWjVlak5KNzdRdHN2eVZ0Y3JaTVR1QzV2N3U2TkpOeEZndm8yRmp4S2k1MThjN0VscEVDRWZsSmRYcE5yK1RhVG9DZHhXdG45OGFhRUxvUUF4Qnk3OVQ2RC9xczk5NGN0RXdRTXMxSWFCWGdrY1RHRFpERWIzUDNranNaU0FpRVNkT3RraThJRi9NQ3E5NGl2RW96OVJ0M1lSdjQwaWZ4TTRNNGlVdnh2TnJQUWkvMGRDblB3Rk1YSVQzZTlNelU3S2JST29HN1dHSWRqMGFGaUd5aXRuV3VBQWc4VHhXdmRrZE90UjRtR2MvbnlJNG9VNzBaQ0NqOWVJY05wemdpUHEzUzg2NzNPTjN6N3ZVZTBPWmQ4L1VHZy9xMFRoM2QzZFUrb1l5UDl5VUJhZ2JEM09hZHVPWkJpTFR1UEZZOXBPU09MSWNpMWdWYmJhTnRqOC9oYnZXa0FCTGl5aGRJTFRFWkVKUkxRUnd4NmZ5S3pWUGZ4YXY1MlFBbWo5Z2RrQURodEJidklrQWlMU05GT2JNQkUwQkJVZFo2RFRxWUx5MFg5d095TExoUUZpZFhQNC84R2Z1YzJBTkxIbmZuVFBIbThLcEg5M2VuM3FXOFdleFo4K3hLcWFLVnN0dTdBRWcyU1FXeDZNZHI5Vk1tQmwyaGF1VHpZT3NYS2sxRVU3aVNIY1czeEsycVRFeFdSeUgrRzZLdTliZjNheGs2eUlUcC9OalZwY3dLQnBYQWQvMlF0dURqQ3gyelhFL0djOU4zWktBZCt0Q0d3NzZkNlR4QUtycFE2TUZFNFNpRThpTFJKOGhlajlyWDNJT2hNMG9XelhMWXA5VmEzc3NBVHMrWUEwTjI5azU5b1ZIZTdxUWpBYkpUT3ZuOTZoZjcwbkNMZG1SV3R2d1kvQUh3ZlRnRzZCWm13TEw3amw1dDNMOHQ5VGRzNHRVSEhJbU5yS1V2cWFCUG1WUDJkcThRN2pyRkFXb2lENWRuU0M5bUdFOEk5MitkWXBxNzJtWE9mYk5IbTNkS1JGdVVxUTRVRzZEelY0MG41WGpNL0N5K1dCQ3o5M3ZUeGVQdmtNWkI2enRrYjljMm9tdDlTZVhEbXpIcGRlZDIzM1c0QUxYZTI0dEJwSjg3WHNiRzBNeUhMbzVveFV6Q3VsdndJZS8vaStSd0t3MFBRREFGTEZ5QkhvclQ1UXkzOGlxMjFUK1NjUVl6d0Fyc0VCRGdHSnhNbjhGQ25LS290VTEzWHJkN3RzTDdhWjZrTnVZdkJKall1MWwvNWJTOGNFdVBlOUoyRm1jaW41WG5PTXpiZXBRNGxlNUlkcGRleExxUlQyb3JidS9kS2VmMHE1clVhdmtDS2wxbmtzaXArbGtFSll1b3BIdHFXaCtZS0RUWVJNbmFkdWlwVHZKc0tTbjVsMGxrOStMQ1Y0V0dFMHRidTkvdDlkUGJGd0pjMis3NmhKcHZiUFFOYXo5V0FUTXJWWGRsdGltRFdQc2dXYS9sODFYSGxMSG5Qa2NESFFKOE4rKzJlV1pnc3BoNjlacW03RU1oWXRDUWc2M2wySVFwRy9HOGYrcjRUUVRvdGQyYjlneXhGaDllSHdDbEdTbnpRYmJQcDNkUG94SEFxUDN3cXV1ZWFBSnJ4UGpRR1pxWUlFUWZsR0VnQnM3QXFQb2lOZzBNWEp5aHBrc0pEbzFhSDZSL1h3bFE2cnZ2UitWRjJXTjFsM1ZoZmZRMjU5WE1YVzVxdVlXd3l6VFdSL1ZlQlorY2dnc0J4WTVES2dEU1h3LzJGVmFDM2hGUW12aDJzenVXbFdydXhBOXR1Nzl4YWV0VUxEMjJ6OXlmV3hmelJyM2xQVk5tbnpQdkV1MHEyWlF0TDNiNlprQkJHeUtmdk8rR2J0czhCWWZCTEFCaTV2dWVCdWpJZWRSOGVRRTAzb0xjVXBUVjlyV1p5Ty9pQnZyaWFaNmdzUkdQbkNzeHVzbG96QnFnS0dEZmxhckp5eldOaUZPZUFRY29kZ2c3VndKNExGZ2JtL29la2pLNFBMV1hHa2RUOTg3QWE2cVoyelNwTzFCM2dKRHJlVGRqZWo3Ym5HWGtsTlk0NVVBNlo3S3ZOaEdFWGJ1Z0RlY3FLQk9pdjV1M0tKTzN2VjhIQXdjcy9XRzlEcVZUTFZ2cmIvVjZOZW4rRGlEOTc4cHB2Qjl5eW44T2pWRE42MVhIS2xsSllOZEh5c095ZkxjVG1CQnVYbUV5Nkovd0lTWGxLV3oxZnBkcysvc04wK1A1cEJjaTBxVFk2WHEvS21jemtCZzQ1Q01qVFA5UlJ5ci9WbXFpMlA4VG0va3lrTVI4ZElZeWlVdmlQNUoxTFVsWGduVmNXZVduN0h4Mjhvd1o3Vm4vZmw5Y1o5U2htcGl1em5KcGJYMHFiNloyV2pWWDRIQ1NvUFdlQVVoNWt2cmJvM2UxUnIzTm9JdGNnRndoNk5hZXcxQVE5WXY2ZXpkRmNwYWxqNDNBL04yMjc5VTRpcmNvdXRZSEpFSitVSDdIZ2ZYbnVaMjhGcjhqYU85WDdzZmtvdDh2TGc1MnZpeUVTdWlyalRXdktzSE1qV1RRc0wweVJoRC9ZRTNELytVQjMzM2RIQUlnSC80VWVVVUo2L0NMZFhReXhBb3kyREdibFdPSnMyanl3aFptYWdBc2dhY3ZuVHZNbVN1TGZUSVozRHJBRTFPWmozMWNyUDhkYkw3V0loMEFJcG5WcEtlNjByd0hrT3d2UXNpeXN4cmxzcmlTYTczaGgvMTVCZ1liM3dRa3dMb2E5Y2p0R1lEODFrM2JydWlrV0syWGNkQjc5NFM5VU9jTmRweTVrSzBkN0tBM09pcFJTOWwrRVdXTWVFNEhIT1hxZDFuNlBRRVpMbzVhQmp4WCt1MVBWVTl5U3B6bW15azhMYTJiMkNJaVdEeVQxdTY0NE81eGtldkl6ODN2STN5bXFrTFdBQ0lEaVhjQmxTTkpENlEyNVZCa0Y3UHBUd0xZKzBhNzZGbHkwTHRiUEhTaTVMcXMvVkUvTVMzbFp2cVhKOUg2SWxGK2ViYmhNanl2UDlzNHRTbTRqS0RNOURCcTJ3R0lLMVA5TzFBQ0MzdWUrMSt2RjlVTyt1Z3NuVjN5RUdFenF1dTc1RHc3N3VvZDhXM3hMN1gvT0owVTFJZTJmZndKV3gzRlI4MHh0N0dMOFN6V2xnTm9qTW02WWMybSt0cmp5MkhocFFScSszZk5kNFdVcVBwWEk2emI4V0p0bG45MVZGeVA0V0JiQmFQeXNNODdpcnpScTdBOHZaREdQdWVpZURzN0s0ZElOeDhTdVRYSy9yTnlRLy9sTjE1Y0taeTI0cDhIaGU3VG5OTy9lTmNCMExxbk1RL0MrbGtyU0ZRWG5RNHVzYUlaRi9KV2dibEpOcUFHSUtERWhXd0ppbFlyaCtYTmlzTWRrbXhBWTgxVWYxSzdYakRzWFVxQWFML2ZBY3hkeWlaZkFwd0ljeDByN0I4YTY1KzBqWHVWSk5XY1MrU1J5bWdvZ1dmOVZXdW05TW1VckVERUk0R0N5ZlRKN2VMM0xoekE2VlVzWmJZOXdncEU3dmQxN29ERTZLRzNHbTlvUUlMeXhOb25CeVQ3RjBTdDgxYm5yMFVyZTZVMkhBd29uNmRqYjJhQUlpbnZ1dmErcExGMksyMEdqVVVUcmVDUkU3ZnIzVUJXTFVhVUtVQlNRQ0pMYjVKOHI4U0dDK2tRdGl5NUZLaWMrMkhYb1F1SkNkNC9lTzlQVDVadFllTTNlTFA3ZFFMZGZKblhpMTB0dlE5dkNYNlR1cEwxV0VkcXd6VWdPODZsS1AxcUk3YUJUa0JBSG9YSGFweDMyOTk1NmZOSVJLU2N5V3RPZk0ySkY0dkoxNUlwRUMxWjFQTnVFV1FtNTBCRWIxSkhJcE4wNkEyR3orblFpWlRST0x6SkRralM5Z3Q5TmpaQUV0RWpnOHUyc1RCbmZlY2lpQlJBS0k3TVFNUlhOVkRadldQMEtjYTZ2QUZWc0drdmx3WnYyZldYMFhqOTFJYWdUTkN0M0wxczFhc0lDSTJleFRzSmJJaTB0NWVMVXVUUDRzaWl6OVVmSndBd0x1UW9EdlNWY3NNMWhSaTFLZDZ1OTFNdkErNnA5SzZlcW9ndHExcnIweTJobHI5M2NIZ3F6d0lvV044UldNVzIzSjVkNXdBU3BleE9COEx3bUNIMmQ5TWpUQTdPSkMrQ2ZYRzBWSldJNjJLSFZGZnZWdG1pVDVHM2cwWHVJNDdQQWtpeU9CS2NoNDFYNVZUc1dYWjY3RzEyQytUNnZUS2dFRHFQTW1wUkswT0orbmJWMGpMb1dXdWNCeVBDd3VqOWc1Z1N3Wmw3MXp0ZnN4UmM2cEZ2WmpPSlVqWmFsS2l1aEYycjhnelVmbHZxVG14TUJwTHRKVGJsdDhUUmJuYnNLTlhTSWQvRHhEdmcyL1RoU2VWN2pEdjdoUkFRbXpodFN6dEQvK25oMHZvWGV1My9FTEk0SmU1bGFXT3o3UjJBSzFZOUZHWkw1bjRXM0VoNm4rajk5cDFBMEdNaklzVjdSYkdxUDlsSkFmR3N0MGZseWJsUEN5M1ozTlQ3Tzg0aUFNVW9yeTVPVWRhOGJMMHVuWjBGT1BxTHJDL1hjd1hDMFRJb2pIcmtRTVl3c2pmTWRWRU1Ua2JPN2JRZ0pCMUlJL0RXMGRJSzdkOHVsKytjemk1T2FlbDNmOWpiZmtLZ2QwQ3l1ZmVVVTd1ckwrZmZLMGczRXZ3eVByVHFXSjYweS9GOHQxZ0k0S3prTG1BaXlmVUNKaUxOYW9FTFh4SnBSVHhNWHVxZ1JaaW1KRTNXak41ekN5ZVkzODYwQ05hR0pkL3QybEJVSXJZREZoYXB2KzJ0aUh0TTV6eFptU3Y5c1JLMFBDdlBIUXF5d2puYnVlSWVBSXpnTm5LbjltaHJYYlAvTzZ0TUJJdGVGaDlMZjFOTXJrcVlDYndRN05ZZWxmZUovQmpHcmdnMmhLanZ0Wmh5RWV2TG5sb3JLQzFiOXZza09ZNVp6NWVxU00vekNwUGJ2YXVPT04rcWpuTGxIYWlQUnF1U1lpelJ3TUFWb2JYVFV1ZEJWRHIyTzdmRHo4aWtKSVlyVG9FVjlNYUJBN0dXNVBqSUZaS3kwMXFzMkpLM0tsajlSUkFMWmg0UXVacHFSTnBGWlpkNkZtRjhtZmYxVmU1R0ttaW81N1g5YUx0NmRrQ3lyWDBCbE5xYkpzNjg2MmlhOWw3cEdVSFRCbWpITWJadUhEaWV5cUpGV3ZVZlZWUHVrLzhOMGUxU25NeTJRUmxVZ2xpd3lYNi9vNFlickxySkhyOTVkL2Q1Mm5JWUgvYXBGVHM4ek4reVlRN2pLR1k4U2M1ZHFHTFZSQnk0bUpPNk1kblhOL01DblNBWE40eXIrSGJ1UU9keTRrWTZZQno3QldwQVV1ZHp1RW1zUko1NlhQNnVnTkRmOEYycWZRNGRTbGhob3AzZHZ6VXhXeDFJNHRIYnc2dk96eXdrdmYvVzNOV0pCdGk5dUQzdG1KYlVSZmM5eVdCQnRjNVN2eE1KNVdVMWlHZFIva2FWcDZudUxQOWhhSEl6NjBQQ3FlakNUZVYrZEJEcm1VNi83N1M3cC9yUy9kTlV6NXhVZWZaempLdktnM2RBTTgwWmpkeENNN1djYk1ZVEovZVo0cSthMDNxL0pneDg4OVFOYVBwMnJ2VzFPemttQ01kQ0NOT2hCSGRPNmY4b1YvRDh1L3RsS0c0L2FuNitDTnczWlpNelBVRjMrbHJveFdlQ3I4elJOZThHMmpab2xQVWw2VjlGNmhJeFFmL1VJNE44cTcvbXlacHhRbVluVGRhcXJ5UEFrRUVnQTRUOGpYaXE1SG1zamQwUWVac2IwYVVEVjFhMmIwV1pUVnVMb3JXMTMyL20rM2Y2amx6SjNmcjJvTGkwQlRnM1J5bkRUZ0c3bE40QTRGMWJBUlNoUEYyU2NTMFdLNWNaUEhIa1pJZ1pMelgxV3Y1czZqWE9ZNEx3emFJWHlTWmRhYktLTlZBaVkwREZHdE9mVU9Fc010ZXc3RGRiZnRmNmV0cmRxK1ZycEwrOUIrdHVDV2dMdGw2TzhtejNyOWVTbC9wS3AydWhyYUoxOTFLVkhZejdocmpSb1ppY0dURGczbkxSZzFnOUxNOXRIOTVRRDdmWHl2Y3BtdHVuKzBYaFFRV2JoclZJSitLblRSNGFPTjU5QWpKUDJyQ1ZyT0hIVW42M1R2NUcydlhSTERuV0lqRXc1cFIvU1h3YUVIMkk1ZXdyc0lnMWFTSGg1QzZQOCtZMG42dHBoYmw0RkREYUF3YWxaNUZQMzBMMUtWRm1EeGFSOXA3L0t4aGs3dCs0M3c0Ym8veWo3VCtwWWRqLzk0anpscVhjSWxiRWtVQjZhZjhBTk1EYk9LeFpIS290MUNZcDJINVY4ckNhY2UyMGVEUGxScGRpdVM2OHhVR2ZjNGVaSjZMZmRMT0E0ZC94K2Z2MjcrN3RucFhyQTVkMUhQdVV2NzhLdDc5V2VHa2pwUjVhOGtreWsyOXdKK3pjeW1BUmFVemhTc0RDalhRUWtITnBxMkpVd0NJNWtCVlpraHdNTGxBaldwbUxwa3pOOTIyUmxQcFA3OTJVdDZtT0RESjF6bFg0amlFOXQxRm5wY0lBcmYrQ01hai9jaHFKRTdrOVJuUHR4T2xlSlJZTGNndWdvR3pBUjBmR1d1Y0IxclpwcDJQdzF4a0JGSFp1aDdEdFZQTjIzdzNLTWkyVzhkbUdldHdzeVM0aW5QTHFqNm9QaWI2c0V1LzVVejFoaUpaNWYwaFcxeUxLRUJENzBEY2NSMW9KdXM1TW5vdStJWWZsTTRML05QRmswQWdkQUU4R2t3REppeWF1TWVvSmU3QzVxSm9BbHBuR3F1Q2FHQkI1U2UrcjhDMmdZT0FraWtrVHl5TXdrYjhKak9Sc0V4NVFkK1BhWnJ6dE95MGZpSmRycTJtVysxUTJMTjZuL2Z6S3lZNzZmSnJNUVA2bTBaWFFUd0NDOUFFQ0pmVkFJbjllU2hZQzJxMmVRdWRCYkxiSzJjRkd3WjNrVWxIM1lzVmVzMFgrRFdEY0Y5dFgxTjJxOTZseVE0WFQyUlRPMy9hdTdnVW9seGIzZFR6YXI3VG9ndTc3NGRkM2M4dzROM2NxQTRpbkw0RjFUdytYNlpvZDBheFZFMm1JdVd6TTYvMHlyaUxtcFgyUFptM1Jmd3k0OWNZbWxYRWZQYjlZWmJMZU1OY1Ziek01TEVicjhPKy9WbmVDa3orZldHNnM1TjMzUGkvbC9aMXZXbjRISG9ocm92b2JtVEJvQWFsQ0xDbGZabzBkS1kxVDhDYjBRNXRQcng5aTFHdk9qZTBKS3RlM0xYN3ptemZQZHhibTIzbXhkdVBtUHBVL3A2bzZ0N0lDUmJ0bjRONnIvNUE3dU9XUW9BRFZHcWY4d2UyU0srQVl1R0NHQ0RONEpnc05pa2pEczVJdGdLSmd0VnBkWWNvcW5rQ0k0dUs0M3Bsa0RVaENaeEUwc1FLUEFWS3ZZN1Q2Z2NuanZRYWh0SEgzdlBacmwwelA4MlFUb2Z1OExPS2FwTEZYbm5hZzJJRkhCWkRkZWswdGo2VzBadHl1akQzeDRicjhKdVVlUnZwNG00cjVmb3lGdTluNmpEeFlyVDlKbGFJUFhicHY4ekduOGFER2QvTzRUM1FubTgxOVh3aGFBN2ZySHdWSEpCYVpHZVhNNVZNN1lyOE5URmlCaGlZcko0TEYyN1U3amxYZkVWSWdDUkNwUGg3ckNKcWVaQjIzRGlRL25UY1ZmUHI5ck5NUmErajl3dCtWeERZR09hMmcwZW0zdnMvR092TnVxdTBISTF0akNQZXNVc2YzNHFHSi9SYmtUOUxPbHUwYjgwN3NnbkU4dVdNM0tXZFpSWS9zM2ZrRFlOaGo3eU1VT0kwYmJYNmM5czk4a25ZNnFhVXZSLytQazI2QXk2UE1jZVJuUEJsUXNjVkZtaGxrTlpLWWswV2FQeER1NDhKd2dMRGR2VVpRZnphY2hvRk1Gb2NXTHVVZUl0KzhlVGNoQjdldzgwYTlvOFhlangxNGNIbStVd3l2ZmQrOTN5YXkyYW1EbXp4TmpGbjJBOXlzMmlhK2pEYXdGc0x0NWRVSTBUTkNxMis2RUN0amZpSEZ1YXgxZVJkU29EdC8zUkhFL2dHOUNkRjREamUzMXYwQitXNUE2L2lvZG1mSnNSUExTa1dMSldkZjkxR0hZdzNyUHhvbytyRmRuZjZUVFlVWjFxTWU3OVBEQnFyU0ZsQ094S0xEZzRESkdBUDR6aXN5eVdselFuQ01ieDRBVGIybngwQXg5TWdHcVNjdmJnRWs4b3dRNW9Uc2NMbkN4djVsbVVOSjdDZmZlUnQzNmF5bzNaVmRZRnh2dUlGaDBSOTEwRnFYNk9heGVwcUNLL0gxWjg4QVpFWFRyUC9JSmFaTzNtekxqcUlyNVlZNDA2MGRVZTdUVmZiVFZLWUpRWWpBbE1DRlh0NXpJVzl1cFdlMEdaL2ZUVnR3UGJSVE9MUjB6NTMrNUREWm1qTlJwdmlnc1FPQS9iYkRXSGdZeHpIMVRCcDJVU2U3d1dNeUJrMlFXbWxzZFFjVWVGZ3NMV1kxZkNtUU1DYSt5Q0JqQWt3YWpSMjZtT1V2V0dldmhGVE1NNjg2VUtiOTNsNk9BSThLWDhlY1dnM3BHZmU4KzFRNWp3cS9TeDFaSDNDVHNvZXU1WTFvNzArNGtiZjZnSHNBZWNMb1pZNmpTQmRVUXg3bVE3SnplclR5STFqNXNscTJPcGJPM2QxclZDeU16OTBhZ1dOTWpWMTlqM0ZINStlbldIS3l6QnpyYVRnZzBlZ0FxS3Y1Z2hOeU1yZVBzK2hRa3BoanVvdlk2aHBpU3dJVmI1dE5uR0VGRnV1SEtGZFpEK2NaekhpeEJMbjZ3c1EzalFRbWRSVTNzK2xvTHpkWlRzQ0RtblNOZThrbjRFVWc1T0FpTlBCaStsWDRMNC9NYm0xWC9tTTM0S2NuZE1oRG03dW52TzlUamhtYjAwR2NlUW9lblFzNXBaVlV1aWlUVTEvNTltSUc2Y3FnN1NkYUMyWWxBQ2R6TzRzSXNoRno5djJpL1Nmb1hCSFZNZGxORHIvWEZ1UWZNeElya3ljL2FXM0gycnBMcmk4eFVDanRrTE1YdnA1UzNQNGsrVGRScm8yYmIwZlBQT1lVZ2pNT3c2MDJwRG9UQ2xCU1FESHZWQm9BdlpMWTBHSTdrQWt5R3RCNXFvaGozMnJDV0JLV00zVjlTT1JxK21od3VyTG5NWFBjcE51KytFNlhOM1UwcXNVcUFaelg4eVI5UHJ2dVJTbEpodzE0QnV0M3R1ZERPVm8xeUxsVyt5QURxejdDSmliYS9YclVRNnFOcWRCN2p1cStsZldmY0JUSFJPVlNISldlRlo4bG40TG9qOUdpcG8rQmh4cUEwUGF5RjlHMnFEYVlITSs4UHhUdmQ0Smlod2xsNDBXSHNaOVBwTG9DRDFSazQ2ZUtNMllHcmltQWxqa1dzOUxReEtBQlpzWnJBc0RFOTh0TXJHbEF5RGdKRVhIK1lYTzZvblQyTGp0Ulh4d2dRTWhSNU5OR1BudkhSZndQMGd5SU1oMWhIU2w0VFJsc0t1RGtQV2c5TFJzQUU0RHRQR2hIeTk5MVVMdDJ2bkJLRHdIa2lVdFZMOWRMVE9NYStuQWxiZ0tBcjRnTDRUUnVvemU1YTVYTHMxMyswdVZTN3ZhTm53ekhnV3M0Wm14enNFL0pkMjNucnhXaTd6T2xiMzVmQWo4MmNmdVJuU2F1dU5LT1FtOEJFZ0I1clVSUU9CTGxOQVF3V0x4R1hRd2lQeFhQdUkvQmpJdUJxU0JpQVl1dklRSU1UY2Fsb0dVaUNMRVFGWk1ReFFVUmdSaXFIVWxBY2lIMmxkZ29UczY4aDNLdEJCQXh2cm5IaWxkT0tOMnhORGRmaFJCZzdRR1BiSnp1eEJ4bVZROVVVNjN5VXRqQnp0NDFuemRYa2ZZZ2NnQ1FYdjFLc0U4NWwrQkN5c1E1UUpLOGZ5Z29UUzQxNTZYdHRFNHMrUEpaZHY0ZnU5czM5R0oxNWhWMzIvdlBNZlp6cnZPM3Vack1uWHppRjdORDJLZmxHUWtzZ2tPeFNaSkJKWnhUdVZoZ2VNQk53SFJONERYRWsxM2Q0WW1GUk1ZWW9oOFp3M1VaRjlUSGxJRkpJZ0w5QTdnajJndENpQUljbHgrcUhXZkk2T3VtLzR1ZVI1V25xT2Y1bGhQbDZvQ2xBYkVyZSsvR1ZUUUtZczc5aVB0NWMrQlBkQ0hSaitoUHJyK0N5QTRFa3VOUHIrNVpYS1I5WndqQ2dReGJvUlFUT0hXeWFzTGJIRlhSeFlBbDYwVXl4NUxMbi9weVNuMXF2R1ZCYUhlOXI3SGVQZVcvVDJjUEZCMjN6a1dsMzFXazJSS21uaXdBQUNBQVNVUkJWSE9JcmRKQ0xOc0dkbXdSeDFlZzFMREhUL1dQRzE4M050bVJ5QzA4cFBPK1AyWnF1SWFYeDJRTlRnU3V4QlZ3RUhQdEN3Q1A0ZmxEbkluK3ZrQzRWRXgrZ2ZGTkE2TkJoT1RRQTZaSXhKb1hCVmNEMUlPdlFqc1NsbzNzODhHSjkwQWhUa29Xa3p1T1lRV1lsWEpxZWtJWHUvekFaZzRBK0tvVElXWHZSMFVVTURtRHpUN0Zzd3dnblRNd1hja2RnTVFucWZmNzlTbXQ1OHpjbCtpZmI5UnZ0OG4veHJtc0FOOU5SYTBQL1N0UnZuOTY4WFMvR3plZXBpZmkyOWFrM3NvYWR3RUlMdEFMWXF4cG5jczZqZWxBa2pnUlU1NWFDd3puWXZoS2gwVyswaHhONWwrSmVDWkFNaVpqanJDczJMY3pjTGxZT0pBdkVqSGpoUnhuVmZvME5OOEF3R1R6TnBTNTluOFRaN2g5eWJ6Y3NiTERIVHo2ak9vaXpJU1lxNzBPVUlHbStrVnF1WittN0pTMmlqTnZOdkt3T3QvUWp6V0RDV3ZOMllzU3VPaVZiWTFaMnNkYmVrbHlmeUxVTk5HZGlHOTlXZHJ2UklBL0gvbzlnREN3bUVTN2RXZlhROXBtb09YU3hNRnRaeDcwTnpxNXljQlcvNkdDREY3SnpPdEFtRVdYb2JLL09ZNGxQWWVzek5LWUVTNG1nNU1UaHhwT2ZPUGR4QlI5d0JpaWV5SGdOYWM2bWszZ0JmeVBha3pDdkN2cFNxODBtZkdDV1dwZUNKVnM3TmE5QUZ4Ni9pMUJvc3JuaFhEMUliRzY2OWs3RlVDOEI4a1NsTVFVcngzTjRyTitrUE5Cb0g4ajNraDlqUk01ellTRlBIVUxOUVJNSG5JaFVrN21pdFVXQXh5Y2lEM3ppWmFzTXhZSHRWaGxnTmlwcS9QUWZqNG0rUnZzNkk5dWpKQzN0TFF6UVBpbjlmZXJkZndOWUozYmVzTjlOWitaM0w4bDNYQWkrOHJUbCtVMnNibjlUZGVzWEFsRFF3a2tTdzNVbEFzS0R4QVRLa3pQQVJvWWMrTFBhM2k3eEl5dktib1JJQk9sY01OVWxqUVJxd1lZTHhxWWZJRXdNR2lLNHRSWEFUTVRKNEltMGJkY2JBdVgrY1JJZTdLcWR6V3I1T1BFbGR0MTRsMWlTQTFNd0l1Y2NLZDhmWjlPWmVPN3BUUFduNGdvTmx0Q3BEbXo1U3Q0K0Zrakl3MVRYdUZUZTY0ZnNlK3c3UTkwRXNjdVhoN3JwSGVYWGxwN1dYdDcvdzZIUW40UkhGYmxlblpsQ3ZuY05GTkFKVjN2WFB1OURCeFBOd0Q0V1hvTFloa1BvSVJqcnVMR25yZDZ5RDZzK29WQUxUUVNFMFM4VERIRE5NdFQ1SjhJU2lTV0Z4NGluckJhZDRpRFhFeXZ3c3FaOEJRU0hDUjZGZVBIbVJTUXpQcVIzdnNGSTNmMnpYai9LTGN3bENOaFZwY3psdVBHcC9xUEVNY3hsY0k5eVRJNU9kdEZBZzdDNmFHVGZGN0Ewd0xMVmFNekU5K3pGMlhXSkF2NC9SZCtFbGo3YXh0SXVSRnIrZVZMUG1LVlNKbDJBV2JLNXJjb2xxQ0lTcFdGVU1hR0VBb1lWRXY0bG1GYlJJWDBMUGNkOTBSbUgrbVJGRlR1VWVEdklldjd5ajRzOGlFZS9xZ052QmZ2L0hsM1dnUEEvbUUwYWxsajFmTDhOYzVqbXFnRG1XdXVWTDBtZUF6Z3VtU2NWY3doOHdxZHdsblU4MmlFdzVnUTgyOG1LTEgrRFlsckF0bWc5NktKYjRYdkx6QUdCcWJEaTREQVlBTUNVbXZPOUdoaDFhdlZXdWw4aFExMVpjY2NwQ21QbXowMURjeE9vYnAzRnpOKzhKT1VYZkd6YnVkclBhSnhuUkpISXZYbGp2cU5YZTZhYU0zTkZKdWNPdERFc1pyVkpuUy8ycWJkdEwzZnAyNlI5VzNQOWR3QndUc1FldHVKaXMxN1FNUTZibmZwYVgrZWNFVmVKejM2d3FVUFI0eFZERENnb2NTYUd1ZGg5N2dBQ0ljSldCUUxZclVaS3E2WUI2dUpPU01zUEVTY09CVngwUi82Kzd1Z0RFRjRqZ21HZUpvU3hGdjFHd0lrc3BvUHZCeVM0bTN0bEwzSm9qTXhhTEJ0ZjhQblUvNmlCT0xyd0kzSUdFWFVOQ3R2RkZHNW1KTWU1SkdISzF2OU96eW85elp1Ny9mclpRV1V0Si9GdUJMcTA2dXRMc21jYUxielBDbk5KcjhsSUVMVmhSakJVMkE2cER1VktMYXZjbmcvMmp6eE90cTdwRHVkMjRrODNXVzgzVHZVMTd2WXVhK2xMR1BWRThFNHJmTzMvRW04MDE3L3B6V1lDb0JmS0RGQnlCeksyRWlMbE5EWmdjR2QxOVQ3VkF3QndtVk1OZUZDUlNUVHBkRmtWWDZ5aXRXaWVCMUZKQmk0ZEI0ejJkYjVUa2k2cEttL3lUZEpPYkhhS0QvQUF5L1Z6VnpRUFRlNGZIRTBSeldHeEMweGZpaFRBR3U1YlVpTDlqZnVzNDVZTGxXL1REZHFBNVhicStuNVYvMDZhOFhlTWQ5ME8wR1gybksyMUdTTnVDNERZUi9SYURlQWgwcDlPUnhBN3VxT2Mrckl2c1huRFhlMDV0cmZNWjFNeGs0bnRCMHdXVHZXNTRRMFB5TEtRKy9XN3U2QmNEc0Z1ajVEZnhpaCtXMk9mUy9oQ3BEZTNXYXFIWUhvckVkYllkWFB3NjB4VTVIUlRMT1lFdWhiNUlRQWxWU0grWWxNNHphbTZGaDg5V2Yxa2xVcmtjY29tUk5ERmF6ZnFCL01RUDhDbENjaGZNSFVLdXJyd2lyT21LTFYrR2xpTUEvUncraDdUUkFHVFFmKzhDdVI1K0xmc2lwSVRmSDc4cjA2NVdPbDZ3MzRNRnFONzNmRlBKMkhYOXZjUlVSNVU3MHB5TnB5S3hPdDFlRTZrVFVHZ2ttVjJkeGJ5Nnl0ZCtMc290R2pBTUNIMXpReENsUUpKcmRkOHdlSEZmMDRjd0FBc0RCdDJEUjBrNDVnMThGMHh5SFJpVW1qMHJrQTRaOXhMTHVxclcra1hxYXJMd2lVK0FDd2hENzBZRVNUd1pnZ3NnMTRBRWhYN3lHbVRsSEVLbDY5aGdZblNseU5jakVFa3BBVGM0cCtneWZvSmNIK3J0VFp6QjNibUgvN0hOWU5leWJhTU9HRjhJQmx4YnR2V1hGaG1wY3JpVzNHSlVWN2pCZForTVRWRWY1eUpheU41SDZndzVlajV0bUpPVnZ6ODZibStsenFPTGk5UHdRUUlDbkw0djcreU1ZQUVHQVRMMFF2akhncmtFakd6RlhzY08vazRiS3dmZ3V0SG9oOXcwSDR2ZHlKRCtqckVURStyTzhSRS9oRGNlVTNFcmRyMi81dmVnZzJxa3k2amhCZjRGeUk3Y3cxY1llUzkrcFd4SGtOa0hJamxvK25qUHlBUklKbjFabDhmVi9nMXhBL1ZCSSs0YVdjdy9XS3VLY2k2b2h5VmVLcnlvd1Q3b05oWjdFTURKQnJRQVNtSjZhN0xvaGh1TTVnKzVYMzNaaFN0dWFyM3JPdlJPYm1oMkxnNTV6UTVsdlVPazMwdWcyNHZLUzhlM2dESXVkSnQyNitTcHlGY2F5SStYQnI2clE2UWE0eE53STE1Ty9jaHcrMi95L2F6QmtkMkh5cHBZWGdxWldsZkhQYjJ3MmluRkpyWTF2bFRWMGRKTHZ5ZXl1Q0pNZTkzMHdMWU4vSmVwdlNJcmtjTEZvOTc5VEZSVGtTbHdCTVRNbGdBMVR3SVBFK0JUS0hFOGRyWW9yK3hBQ0pwd1FpbkVRQW1lV0d3T3BQSWx2OUdhd2VyVkNPQWh3Y0NJRjlUdzJrR2hDeGEwQU10TXhNekF5dDEyWTVVc1M5REJIc2dCQTZqdmphSnZwOGc3Uk9lWGFsV1hFQllBMm9sRVo0R1hNZ3p0QXBHcUlERzcvVG9TUVE0U0l6clNzOXRRblVPUktFSEF3czNxREZWYjFWbm9uQ3pXQVUreFBNQzNCQ1BrTHZodHY0eXg0R1lEZmhzMk5iVjlJU1ROVHE2YmNvOHdhZ0UvTDg3L0VPYTFyNmt0Z3dOcUYrczFMc3B1dWkvYjFMRStBWCs2b2t6cTdzd0NHV1YrTkN5RjNhellPVmRYY3ZFYXRQQ2R3eVkwQmc5OGZrRU8vVUZYNUFqdVNja09OT0xySnVpYzlxQkZHVXZKT25lSTV3N0xkNUFjcUQySHJQdUhSZmpuRXhRbGN5SzAya29NUlJrRHFxbVVmcW9IQzlKelVuR3doWWFTREVrMHVwUHZRdThVVXFWenpMekwrMHp1RlB0NThJUUhFMnErbkUvbFRxcTdPbmIvQUtVWUhYNTlUMUg4YjhoY1JtSXM5RUZWZjYvcGREVDEwdXp2ZVBMRnRsUlFybmNoYVZHc1h2aU9nQktMd1ZjUm9ZZmdJeUZYQmJ5ZVgzcHhXZThsU3dNRkdHRVFUUGlkc1doN0FJdnNNVEdDOVc0MHMrN0lvOFA4ZzRFNzF4QVdBSjJzeER5SHRpYU1Nei9FVW96UzJTOElkMkdLUTRLakplbVBoV1BkOGdGa0o4UlVRMHpqT1ZwODVINHprRVhBUkFiQzlZRHVuTUNrUzZUVDh0M1VGelJoR2NGS0l5WUVXUG5QTDFqN1FQTTVCYkNlcnNadW1jZW5EblhmcktPd3EzS2ZQTkIrN2lOS2M2QTlhVEtkK3IrQklzM0syYnVOWFIyLy9KTW40cXN3SEZkK0paVG5lSzMvOWs4bllYTzNEL2pXMEhGOHZhWDd5SWN5Um1xZEZCWmJQWVRMajM2aGoyRUhxZVROcGdsK1lVWTZvNE4wS3NZK1VxSU5hWlNWQ2xyQkQrSU1JY0NqTGFqUmViNkVLZ1FiNDdsd1lCMThTZllXMHFKd1BDQ3dNWGM4eFppQmdrKzNNTWFrakxlSnd5NTJRdXpaK2pueG13bURnVExkcTJrQjVJSU9BcDdzVUlyZTZZOFR0ekw4QWFYa0E0emgwZkVtWE9RWWxLL2ozcHRMVkdyOGk5bnUzWjlsUzUzdGRXdTAwWmR6S0RzWGFTUkxUWjlDSk5jQTkybEloNTB2c0FCdWVvWFBVZE1qKzI1citwb1JEbG5zVzRCU3RxN1ZvZExYUFpjNFEwbHI1dXJLS2gxL3BHOU9obTNtM2lZUERNL0d2QmYvd01HWTVtbVNsNVpDcXBUV0FNTlI4cjEwSHByM0V4REZZbHFtb2NCcW5qR2R3aU14RytGK0p6cElHSTFPTERyNEhYSmZ0cjNEOXRTbHhXUUE3Qk1yZDJobTI4Uy81SkVNSi9ZY0xERFJDQmVHSlFDTXJFWW4zNVp1RjBTSjNhQUZYOElvNmhGWVZuc0cxQkJ5SEcyTEdYSWZMSU8xL3h0YlR1bzN3QllDZnllSXR1aWRtZERIdytpM2VoakJXaFRza1dsYmRSc05KamZ3SFNEMFMxQzFuSHNtT3JncUFPZmRvU1RFaUN1Mkx5NlNJWTBwS25WWGIvdHFlT3Jhei9PZmQ2dDMrbU95YmhHUU5oZ0JUZ1hlbzQ4YlE2VWpzdXhmeElUS1R4QU5ZTWtEcWRtVmpDdW5vWU40SUJ6Q2ttWFhaUktJdEVwRXBUQWdhREpzUkhSQzAxckNLTlJabW42NUxUNW96TkhRU290V1pNeHZjQTZGTHVSWisvTG5Ga0c1Z0t6TUxabUQ3RXdFRjBkem5zb0tDY2dXUzR1S3M1R1dHR0ZlNkQzWHBqeDF1ODNNUnIzeVBRMnh6U3J2Wk5Mbzc4d1d0VVVGbmptc212dkRNNG5tUVFXMmZRMTNGVzdWYTF0eGtEazdlRXZuRjBZcVF6WnJnK004Yk5pWnlpekdJR1R1VjJ5UjJpVHM5VHZVQnRvL2ZKZ3g0bCtzOFdxYWVKdGNBUnBFcjZvT0ozYmY1UzJXZWdGUHRYU05sRTk5VlFqYm5FUzAwQVFYQUFRWHFHT2VGbndpU1ZueHdkUWY3Y3pMeHNISlVSOG1RVmExd2RDb3ROSW50elhpS0NFT01iNm5GS0dnNXhLaWtSY05FVVN4RFp6R1FYeDBQNHNONlppQ05IWDVtcE53REZkZ3lIK0RMOUh1TTd6VWh6d2lQbGFySTF4bmJ4NUMrVmw3YnVtNVg5VW5aVVVTT2puYWhNN24vVm16bGJ0dFI4T3ZYNmVucWViaGtNc2x0N0pyQmNxMzAyOXhCRkZYdjhydjR2aUR1b05BQ0RVZ0Y5UnJuWHROeXJqYVErSFY3eHhDazhJMERyVTN1My82YlVMVFJnbjdEbThKZGNRUFRkQTMyZHM3SDlMMlo1VVU2RVhxUzZEU3NTM0lYcVRIVVNXVCtrd2I0N1dKUU1NK0s1TWtkLzlMQXJubXFPSllHWUw1NzRob2hUNEF2WGF3aG5RZ1JtMGFsY0NpUlpOL2ZTdnRtd3lPcHU3dkhUQlJjN2dGTTBJaThQM3doQTh4b01HUGdJeEl6RWlRZ0l6ZVFjRjdTNmc0WkJxMzRqNnphRE05TFA0cyt5VDByOWRsOTdlS2hzd1MyRVpBSXN0elAwdEttZjVweVpiVTI1YXZYUUpyLzFpaW5RR3dnQXlYNG0xT3ZZcHIwb1VYcmMzNjhCbXlQOURTZFJJSlUyNzNkVFprbjA1bnM4VE9mMmplclRybHVXMWJtWTZlMS9oNHE0L0RVSlg0amNkQ05zVzEvSUppUkZnT2RMZEJSazNBTlVSOEZpK2pldWc3UDcrd1I0MlA2YjJJQVgzQ0lEMTVEdEwrREVlc3FXZnVGcU5BQ0FjaTgwcDRoY0dwN2d4U1FkSWNBVXVuYnVjN3lqRE1zZmhnVU13RXZGaVg4UVcvV0g2bHFHaGhGNDZhd21UUHhCQkZnVUphdjBPVUlpaGRlSGh4bElYN2JxUHhJdGM5VjZqQUlPa1lMTHNTL1lGTmtwUGZOWTVib1pxT3hoNFVPUjNRemJURGhCYXgwNkN1SmZWbkJDQVpwOHY0c2hkeUtGclF3NUhONnVlMDc0MnBIY2hyZHpicWErdzBIeGlRUit2WS9idW5jQWRIQXlXODRjM29pU3Y4SFY1SHBPdTBiemZEWi9vbTAyamdzbUV2Rmo2Tml6QUFnenE4K0lpVGlBY3ljRG9oOVJrWVlucWIrNnpDcWlnVG1VMDFFQThQREpOREZKQlk4aGhQd0ZBRHpFM012aUd2K3QzTkEvUkFBRy9neUV6d3BaYUlHc2M5QllKSXprK0tVbVhqWTRNSjhSRTAwc3FKTGtqZnRDM015c0d5ZlZoeVFOOEVnbHF1SzFmd3E1eXVFUFZvbWpmeVQxSG03cTE2WVR5U3hSZWtBaHdTMHBzZlhueG50WDhuV3dVcWQ4bVlDenN4Z1NuVHl4cUhqdmpDV3dzZ3hmZGFPKzFlbHNuZnNyT0lTVWV5cFRPbEh5L0NaaC8zWWRuQzhJSVQ2a1JHeVBxMGhUYXpGVzJEYnR3UmxDTnN1TmlSOVNxK3RLbk50UXNTVkVtYWlUS1hRdS9BSXdKK1pMd3dDd2JPQmp3Qld0WW1rUkFLSkxZb3N3RTZaRUVRZFlOdVlOQXY1Z2lzZzFSUWZ6eFl6Skw4ekJIaDFOeG1TNnFmanl0N0Qza3lTT2tTOTg0VW9jeE10Rkhna2FUUmg4cGNQR0piM0FIam5Oek1kRDQ4YktjTmJ2SWpxTkxzWVkzV1ZZMmMzNHRZdzlzZnlORTZuclNxOGdFOWl2VFBsa01ZbGR1ZXZreks5cW9rdUU0Ti8wUG5XdE0zVzNQVjVXODdpZEg1a1hiVStmQUpuM2JjZWRsVHBYVS9MQzRmd1hwTjNZTXBDaWhpbThKRG1JcHloQXdZa0xnVTVPTzQvR2dnMkJmUVVaQ1VETTNCczdmc1UySzBHVGh5ei9FcEFWUEFiR05jR0Q4RDFFOGVybVhlTmttQU9NTUpSenVrQ3ZGMTRYZzNqSUVaMVQvRU5lSUlGTGx0aXRZZ0pXZGtzWEsrbTk2VGdJWDRrTE1RMUozWmtyK2hQUmdMd3dDaFJsUFVXSU83SnoyUGlTNFVMUEt5aW5mWmswMWtEaFppSkZtZUQwSTU5dndDdEZOMEdhYlJ1M0U1UlBBdnQ3NGlIc3ZySHNMVjlXRnBKMXFvb3JXemI5MEdvUk4yZ1ZWWnhycVU0c0ZUWTU1ZHU5RWExNm5qdnZXU3R6ekhCSUhTeE9JbHFBeWM4Zy9kTXlQYUtWOVkwTFY1SzVqYmhkRkt5V0kwUnUyUForWnAyR3htMllzblRDZC94bXk0MXRQR08rVkZlaFRsMUV1SXcwaHV5SkVZQWhnRW1PNDFUdXdVekdVemYyK1VLbDd2RHpKZm9RMG5nbGNWQ1ZtSC9aenZkVlB4SVRuMlJQallDSjdoT1d6WHNFQ2Z3c3VhUXQvZjJsZ0dGZXJRQnc0YVdBTVJFdVp3RXNOaXZGdEd0VU1QQnlGL2tLRUMvOVBsZDV0czRFQ1ZDdDErV3JXeUJyMWo3N3M3MEwxdW9vdHMyMks0bDNUTHF0eE9hcjhjSXFScHdJeUo1bDhPamlqeDlMMGNQdXBlc09ORUM0MjF2WHp3Q3g3OWRhOFhPUkpiL1BUd0RvVjVPaWdOSENKLzJnVkhibnQ1WmprRGdyYWl3NjYyK0xYRGJpdHh6VUxXQXlJZWJqYVFTdU9oUURIRXdoUEZ1WlRhemhPWFZzTlQ0SjFDckRHdnRqRUpoSUlwZ041V1RFOHd4anlHN2VQME00a0RrSWRJbGU0SHVRTzM1OTI1RVZERnpxckRsMWNadWNWM1NKbkNZR3FkZ3o4d1VKeGFqOGt1WVdLQmxwenkyeHdlaEVwMkJUVmdNUkNjMzI4YXh1OFVFVjJTNGJ2aVUxVDA1bEE5NHF3YnlmTlpXVGVaZlRpQ0puREVMWmU4eXR5ZlZZV0Rma21lVW1FLzg3MEdpOVc3cVlSUzRwUmt1WjJNQ0hJL0E4VFlXRDJYVU9HODR1cDUrS09YL0p6aGpZMkJnWTB5cm52T2dxcnhZV2V5NGVwd0NyV2RUc29oN05QZGdkclRSSFFJTUlDV05nVG5VeGJaN3hjd3JCRWtFMjZNR0NGRmxmVlRUU1BUZXFkRkdSU0pxL2hvbzkraDZUQ0lNRy9xanlsOWk4WFlWem1vUHcrcjVFdExGSThwbjlVdnR2RURicHJ0dHdNdnNXMXphOEZBQytFdGxuYmxCVXNtSWlOczdGRHBoNHBYem1neUpXb2hCM0lvVjZ3UFUyS3lDVVh6V2V5SEhGMjVNZkwzbWVMTWxPWW5CR3lDYmZRNkxiS1R1UGVUZGN4QzcyeWRMTlZMNWlYWk1adGgycHRXK2dzbFQ2QmllaTRINEJrSElOTFArS2Fka1VYcWRSRW1tT3J2SFNFM0xkUW43VXl5am5ZVXBXQnhmSkozVWd4Q0VXTmx2M3pnbk5xNzZERkVoTVh5SXJQZ0U4d1ZNMUV3VDNjQ1VNVVpsbzN1bUtGcWx6S2lnTlRQRUhJWEgwbXVNbEIyV3g1Sm1EOEE4ei9veWhQaUs2Z28rQk1hL2dTcFF2aWkzdjh0ZUk4WDlVRndJdy9rQjhiQ2FaSTF0TWhHejdlU25ZQU5uUzhzSUxzNGtyd3EvODRTcnU1SFFCRVlWT1c4cHB0cm9zUGRzN2M1Zm9SRlg5OTU0U2JIVS8rNG5VZWpxaEdPZGJQVmlyRXZnZFlXMVg5UStvTVZiYzlIdFRoZVJUTnYrRTI3dlU4T3NwcC9GbSs4dW13Tk1PbGE3VjkwcHQxN295eXhMQ25XKzMxUDB2RVplR2dodFI3a2Fjd2tRY1lmWFBJQ0RNdkNOMEY2SkxKZEFRQUxId2hLeW5VdkVMb3RnZEE2UWNCQWlZRkRvT3dzUmwxaHc3ZEV2QmFpbzdUTHBEK05KTmV2aDZBV1NHSGJXN3NGaHZMQ1FqcTI0bXpxRUJURjZjUGk0cWRqRHdCN0lmNTZYUFhzbno5MXZGRlNpWFlja0FKZXNZWnpyREQ0QkdaS3NwTzVlOURqUnIrV3lQMHlaUTgvdDB2eHJkTDUxTTZpZDZhUExkUEQ0UmFDK2Y0NUJ3dnE5Ly9VeVExdVlKZEVLRU92ZU8wWHhKcks1UDJRT3FuRm9IelVkYzI0ZE5mb1FmT3lSdlhCVVVHSHl1VkViVWdkZUJ4QUNDUS94d3hpQnRySEpQMDFmc3hRSGdIQW1SV2loNEFLK3BFMkdDTHhLS2ZBM2YvK0lJUkdGQndUWEFReHFjREl4eDRlSWg1L2dPNkc1aUJ0VEUrMzgxK3RtOEdKTUc1cDhML0RYMEd6SjRpSFBjRjR1K2cyMlBoMjRJTk1ESkl3Y3lwU2ZwWGVGVy9rRDFoaHdEbVlNbkdnRGt5R25mTUVDd3VWKy84Z1Zzd1lJeDhOM3VkUTJKSlFFUjIrcTdNYTBlRTlVTGR5SjZBQU9uK0JuR2pRaDNHeFRVcFlpODRzZWFWc3NNM1BjQ0hJclhMbFpZTzZjM21hMyswdmE3OUl0bTJVOU55dHM2RG5lSmcxdXlmQ1pTeUE3U3RPTzI2YVU4TFVCajhvb3k1em9Xdm12WGFZamNqd09zazcrdEFteW1YNEtlVjhPaFVEWHdFZ3JSTThGMXNNd0tkSWw1bVM1VjhMNGs4b2U0bnV0Sk1rUWF0SG5LdTE2a1JFK1lNRzZGNFJLQ3VwVHpId0dyQVltbGVvRkFRNGoxbTBtVXY0a0RtY1pNK0lScUhBVVIvbURpcFRnNDAwSVg0a2Z3R0RtYVdaaUFSWEc3N29OUlRvZTRlYm51a3ZBMUE1ZkVzdFZlZkowbjllZlRrM1hacFpXWDdUa2hyL2N1eC82Ky9LWHpTMmVBT05XeklVQ3ZyUkhQSFhleWFNU3QzaHZMU3VGOGVwYi9JdDhQQU05bEltNEFUS21zdWJwcnZqTWJhZk1pYWhMampWWStGU3lHaVVCcXlEZXhCNllrVlNYck1BZFdFbEVEeGsxby8rd1FiQklDK3g5U2JsUHN0YmdzN3lEZ1cveEF4SElqYmN3aGgxdlJsT2p2dG8rR2lQR0hXYXhCbDNpWFhpd0tWekRqeFJJcGJXWVpQUG1WWkk0a3hnVWFYVTMzMlNSekwxQjFGTk04K0xRU0Uya3NPSFIyZFk4eWx2Y0pEL3RDWGpadWRDTHIxN1k5RHlWbE5qWGwyOWV6bzZnNlNXV1Z6M3NjVzUwR1ZOYWYxRW9YWmZyYjVDN0hEL25WQVdFLzE1OXZSZHh4L0V2YlBlOFRZbjJYNXk5QTZLa291VDZ3SGRKUmc4ZFF2aXNEcVBTZjIwaEFvcE9BWVo2dDZ1Y2hjb2I3aTNDV1VRMkNkS2V1aUVVaXNyZ25xMWxmcjZubEJ2aWErRktUTGlsUlQ2Mlg1cVhtVzlsNGQybXNrakhsWUFmZjJ3T1p5OTlFK0NMZzJ4U3l5ZzN3R0xnb2NXQ0RYSzYyTnJNejE1NXNlTHZWNUlMVzAzUVpCRDBDbzR4eTlWRUJ3cEtURDhZNmZqb3RaNkVYUDFhc3ZsZEMzaTQxNjdOVVg0Z21kc3JYQjBrQnhVeS9kbS9MY1VTR2RwODJkOWV5SFJ6RzVoM1E4a2pmYUh2L045TU92RDVxd3dzY1NuS01YM0J1N2IwT01pR3JTR09pRXR0TkpYRFNWVnlLaEg0RWNNbkd0KzZ6Z1ZDbU9lTWFvSE5vYXBSMUl2R012Y3paYk1obUhJZ1lFLzRvak5jay9FbWVyUkpZU1BlekVJdWlsV1dEM3FRQjVvblhDUDBCRStPTENIL0k5dDVHQUdsTTBVMUk2RVh5ZUs2QVdJam9KYUVnWFVkaVdtYlNSYzYvaTk3ajRFREljWUljMUFBa0N3d2xYWXFsOExDcTV0eDdqaVNXYm5PUXUwblBIY3RzN1Rqc3NWbUlOb2kxckVKa0VFSnVVYjFwMFArNmlFTmhwZW1lcjNmVkxQMUkxWGR1WjV1blhXZVpOU2R6c0hxWEhFeFR4YjhOTnJ2Mm5qL1E3KzNPTVNhMlFGZlRKTG9FSkxoVG9jK3JKTzQ0SnpOSjZ0QVBTYnBTbTFzN3FhZXBSWFEzeWNXL3pSUkNsamlyN09JTTV6TnVrQ3cyWmhhR0FCUURlRm1BWjNWN053STBXNUpFSzlNeHVOZzN4UDNSTWZqV2tBYkVNenh1QWVrb1JHZngzUmRlMW5PR0FmekJCa2lHcURYRnp5WjJKazlERHhQNzlDZEI4bng3Y2Q2QVNLU3BBRzdwR3dLQ3I2Wk1MZUdjOWVJV1JQSnA2ZXZEL2YxdE4rL2s2dmJzY2hRMDlxcUtHZWFZbFBHUXk3TWFoTUNlcjhSTC91enVIYnp1QTFlenFiSmZMbVdPUTlIeTNvWVgrQlJWOW02ano4U2ZOZy8yZ0ZQcld0NUZiMGcySVJwYk11Q0VSakFOcW9NRUEyRDdvcHBubXI4SGZKV21FWlkzRTJkWXdjSGxFdGJkc1VSZ010ZDRBUE1DMlNZOURUWWs3NnVIWkdrUUlPY2d6S1NyNzN1cGJ3bVJFUFc0SnE1QmVMRmFuQkNIWkUxOTF4ZkpQVnhpd2ZGSWJmNHVuTVlWNmpVTDRCS2R4MUNVQ2NkWURzeUdLWDlEWExrNDhPaVlDSTEyTGozSTNGVE5JY0prTHZEbmZpSlBKNkRKZjZpcmMzK2ZQRW45WUN2N0RkbDAxU09UN1hVZnd1WmFvTnlsMjNoR3FDc1JjSG9YZmRiZWY2WXkyMFc4NWUvajhYWTBiNENiMG9xM25Tc1BkU1hLVE54MjVsWnYzaW9ncUtMUCtXMXpLdE5ycTlBdE5WS0o3ZkIxRUNDekJoaWxXZjJ5bzlWaWs3RGZ6VnlHV25zMGdqenI1SnBLQ1R3RytIdXFLS1NlcWlUYzdEV21jeWlEUkxscCt0MUpxakNlbCtvM3hMUXIxaVR5czMxZnBFcFVIWkJ2YmRkaUJwdkRPak5BZzEzSGVnMDVYWStHZ2tHYVlKZkdJSmtzTzR5N2Qyay9OK2ExeWJQOWRvaHh0VWEvZGJ5WkEwenlJdlloaUR5YTZpWGZ2dHYzOVREaUdNTXVtOFZ1MzdXYWNEMFBybUVCSTN1TzZpdXk2MVVSUzlwSy9Jem9uNC9ZWHl0TWQ2QytINlp5NDlRL2g2TWJkaWhKSzlzMFc1ZXkxNm5kY1AxSTBoWldNUzQydklWZ0JOMUhJNzlOcFdzS1ZpYjI4M2NOU0FBQUYwUk1VckNSZnhSY0M3TjIyam8rTUJqNElzWWZDRWN5aU5RQlRZRnRwTTEzREpDNXVldkVzNVAxdmpEY3VleWxvZ25EQWpRRmh3Sjk5VzlCWHpYckN0UVkrRmttV1Z4bk9oTkh5Zy9xbStzZ2dhdTNVZHRyZXJWdjZmcVZZdkdCN3RtUjlOWEZsVldFZVU4Q083K1BIYnVkdXBDcXAwTzdBU1l2ck0vTStjdkFPYzlOcVc4Ly82MWZQVkRMcmE3aWpnTm85ZmI3eSsrYmZIK1Yvc0lxQTl6M3BYTUw1MHBNTkFtUjArcU53NzdEQzdWM080TkhSSHlQdUNUaXdRcXRZemlRMk5lM2FUNmdtK3lNd0MwZk01Z0dXRGZVbVFmcEpDRXcxdmdoSU5MVlpvSUdnUWJoYTA3OElRMGdydjRna3dDYWx6aTRhZGxwWVI1MWc1NnRWbnhwSUlDaGUyd2czRGNwbDJLUjFIQkJOLytwRzc0ZXBEWGRCTXplTnNEVjV3UUFLS0xBQjFHd0QrNytHSllBaEc5ZW4zdlZpY0lZZWU5TVE2ZkF3bnpudEJReG9CNm83OFN0VDVLYi9STGw2UUp5UDlFMXI3T3JldDhkMkZMdXFnVGVCQ0RhS0VlZElCQ2dZLzJrVk82OXFmaDlLcUMwNFlMZXB0Ym9xUjhHM3YyTGIvdVVjT1FvMG5DSUlvYmszQ3IzVUlzTTNUOHJBMmg2Z1NpZkZwZmtDd0lWQW9EaDRRK3p3NVlRWlhpRXF0ZVhXRnZzdER0MW5lZFVUa1FiMVh1QW5Cc2hLSzZRaUREaXp4WWdSSUNMU2F6RXpUeTh6R1VCbFFnZzFaZllIc0VYaHdPWXFUQ3pPNFBCcG5FRVpFQmlZem9aODVXK21PcHl6UHRUU2RUSjNNZXBKSzlzNFVRQTFhZG9QK3o1eFhCRjhJMnpXVTRaMnM1SlByRnlGcmZpekp0S05DbW8vM1hLeXRramFHalRwejB0ZVpmdWxuYmEvZjdlMmRucVUwL1RQTS9mdGZ2K3dUbVZJdjVqclNobVErSXFOaHdXSzBkQ0hDQmhsbzJzTitIS2d2ZzM4a0JESnNqWWJ1QUdKTEtwRGU0bkloWmJnc2NBSzIvUzZ3QUFJQUJKUkVGVUVTcFFtUjdLZldnZmhtcHZSMXA1YkFGSzZNQ0RYSWRCUkNDYUhyckNuazJJQ1B5dHZ2cXNkWTRwZW9VSmlKZnRaZFlWS1dlbjRVMWZtQmhRSzdRYmRrajBNYXo5TWt2S1pkQmFQcHorcy9xbUhCbnFhZXc1RW9EdzNXWnVDUm5BRWtJU05IVjN0SElpNGE2ZUNPaEkrUjN4N0Y1ZGxlVkhZMy9mTEtHMkFhcUlSbVJzVTF3UENBdFkyR0FYaVZMN3FibXNrSzE3VzR3NDk0VFNRV0VITEdqM2RXNFV3Q2tyL1E5QjhSbU1QMHczbGF6dlNVdkVkdVJybmF5WlM3TnZiNzg1VlJ5Y0JaQjFJbDZHYlI3cE13TVFCQ2RpZTNIY2o0UWh4MjNhaGpvWFh3UklPSEUyRjNFNGs2bWZCMmxNRUZPVVNyRG40RVJZKy8yQ2lDbG14TGtBUFZKQ3pNVmtjeFBTNW1UaGFxWUJsUUtpeldlQWl4RmhLQmN0aDRUTElKc2Z5UXNXQXdYNEhwbnZocHVTQjBzd2FIZCtNNDljMy9PaktmbW5iSlBXbjR0RVVYWi9QNXVWRW91MjFlZXJSMDVIamlVRUlBdnNVajIrVWhXb3JQbVM2VTV5Mmp6YXIvNDIrYUxMT1Y4UlNmd2VGVTVsbDlkK2l3aCtBQndEcTF6SkJ4YXMrRjJmUFdFcWZzQjR2Sy9MT0FrRTk3RGpQTUpkUGU0YmR5SGZ2QmJ3RUFFMlZUSkk2Zis1N2YrWG1CbmFEZ0d3S080TThDQlphWWxDU1dwaWpuWFJGaG8xc3hvM1lkWVVRTGdSQnppd2hCd2drakVnTy9tT1FUUlVUd2RjdXF1WEFBRWxwUjE2YVVnQ3NCNHNybHhMTXU4S0NNRWQxY2dBbWNRYWFWczd3dk5YUllwTTNTYkxNYnQrOEtVZzREdmt6Y1JkUHJMbDJjd2FCcnFhSTMrYjZSeWhjaUtucVZjeEpHSkQxRVQxN3h1Q2VldmIwS3B6eG93b0RnZWlZR1FKTlVMOHloSUVGeENnUXNVcHplVHhIUkVhTU9TcU84ajBadmMzN3RNT3VKRGEvbjhoNWZuQjdZVWNTTEljWjU2cTlzYkdWU1J4aDVtaUhNVDB5YnFyZDlvSGQ5WkZxekg5Z3haemhvTGdWaHUzNWlCeFNZRHZ1L0V3QXlRSzBXbnU5SUlFSUxwVUtVb1lMQWJhUDBOQWFaQWRyeWtjc0RtVlhTU0tHaUtTT0NOZ1AwV3ZucGtVUjJSWWU4cHUrSHd3TDlWckNOaGNsQTUwVUlDVk51WFd5ejVFU3hKcGJXTHhwRDQ0UlpyZnlTREM1REFZUHpUeHNpdU1QcU1RSTFPZEpIZGN5RnBzbXphWUt2ZnB3ZDZYeklxMzV6dmkzUkszUGFPMTdCRU1Ea21JNjc4VEtPcTdzSDlET25FamQ3VVlrU3JOOTRHWDFkMEFLSXMxQVFaSU1pSVhNVWpMRE5PaHdFMjlGazFza3ZhYnhHL1dWbnNER0Z1b0JOZ00zRWdWcnlvZUVWU1UwYzRQT1IxUDZpSTV1MWZkNGlWSktNVVgyWmxLMHprWDVnQWhBcXMvU0F6SjlEWUFUTVo0cWZuWndVWCtraFlhTm42MjMwY0g5cFhFSHZlNVNla0ZnQVp3K1phKzJMbmovbGE1Z0lwSDNkL2s2OFFkM01YTmlKNjl6L0t1bmt4RXpySWxIVWF3bXdBVUtHeVQzcGF6dHNtVzZyUVFBOGExbUxoTTZSbGdmaUhCT0RySFE4dTh0MjZtRnoyLzV5ZlE2OVg4SmJoOHFydzk5dEZ4SUFDaEgyUmwyYXJDeEM1VE9WL050VkM2ZE9VcXdWM01MUTl4T3BqSk9CZ3h2d1RBekRSdmpPRHlQTWd2bURnVEVUSElBY0l0UU80N01MM1BKcTR4a2UvRE1XYmhCY2FmSVNmTVNZUXdjcmQ1NVdQdzBvQktBT1NJQ1lxK0RhUjNWaHd5UUtEdkt6Z3BDRGZXaVpzaFBpemt3OGF5Y2RteWFHaks1Zk5lSGcvRUk4NWZKSDU1VjF1WWdlUTM0Z09aZENKOUFwM2M3SmVKVmlaS3A2eFRhbVkvdTV0a1k3a1JQeklITWpqWVk1TjdCeHU0eEd6Sm0vaTZoU1kzNC9QRlZqVDlQZW04Q2JDLzVsYk1hUURaWDdsNzV2NDBQUVdwWGI0N1R1anRHckd0TUhNVjlqMHp1S1E1WXJvQzV6SlNQbzg5a3VhRSs2RUlaelM5M3Z4WDgzTW9ZOHZPMkJRR2dJalVZUTAxUEFDUm0wSTVnODFNejZHYkJ0V3RuVXl4NmVmUEFJTXYyZUJuL2lsSkgvSmlpY0hLUk9MQ1ByVU5JdERGdnVCTkVzL1hBWnYzOGc0djdjcTNBUXRwU0dvRGVMTEE1NGtGU2V5TzRjcy95SUFVUU16YTN0ZGtDVmF0YVNUNXp5T2JWWVZXU2hRc2JFNlY3Vyt6YUlOU3B5bHUrMXk4M3JMVXgyUkowS3BzWjd4MERzZGc0b3h0M3BNNWtEYUR4Zis4Vi9ta01uOXlBTUkrUHZWVjk0UzRFNW1Xc3B0OFBiMEZpRSs1bGtLclNaZHhhbXZIVlNTeDVwTjlQSVRzQm0rNHNkL01seFd5Wmg1MkxvTUJnd21mUnlZQzJYMEhrTXpaSXNJdUlyZ1pUa29GMitDV1FjVlkyREFlU0c3ZmxxUGdPZFNKL1k4QmhIcVVzdXBMTG5DWWk3V040YjRyQWtvV0VNa0hETW1hUXNuMUhBWVU4c3NCMjBBUXVtbFppVDI3d3Z2aW1ya04xYS8wemF1djYzNDUrYnFkb1lkWnhmcDJTMnlSWTJIMi9MdVZiN1VPVVV3eVJEVzV0ZXphM2psb21aaTA3YnZYRTNQMU51VEEyNzAxRzI3alJJeHZSUXg3RDFxRndFODRqVHVpL2tTMHVxMURnV1FhdXVkdmxQYzVaUUF5ZnhIc2dDUytMeWNnOFZYUEljTVdDaTdjTW1jdzVPU2NoZENOTU9BT3FWYXovWnVtMDREcFFvU2dMeFp1WlFDWWwzRUx0c0VOS2dacGxETlNUMXBJWFpPR2NnY1E5M2psZnNhRWI2TGpNYnlmVFBCNHI3WmJXR0tVUU1Ram90UmpHLy9nZUdVK3luTnhqSlAzWEFObkpUQnhFclU2WTh6djByZkdpQVgrTWxCemxtOVBSRk9meG1yakgvdzBveHRIWUZIRFpZUHl6bGtzT0k2OXcxaVlJTW42YnZrdEw5R1d5T3llT1J1OUgySVVFTnk5WWdseDE5cTZLOWY3OU9QMEtUZXlLOWc0a2x3d2EvZ0RKQVJJZ3VuSWhLOXNOL2xEYmNJNzZRMFl0cGhJYXh5cWNSZ08xcmFmaFFSSVdMbUpYSWV5ckdGSzFUMDNETktnemxMUWpzazAwY2FDQy9IRm1HTkdLTVZMVUdvQXdCQ0JheXFISW5OWWRCY2lscWc0eTFYL1lGeUl4MFpSOEJoTzhUSFFUdlQraUQySGNUclVadXhNZEJVYXgrQndIbmw0WHRQM3ozeTFQcVVVMnUvdFU3MXZtNkJpMGJpYmhoejFZZzhnQVM1YkpDZ2l5RTQ4c00yYWZWNG53Y2JlcnBROGlob0p5N0lTdHI1UmF2OGR0L0V3UGFyRDJQdERPbkpFYitwLzl3VjducFBuNnJJN04zRWtuTXBPcEVGemZRZnl4eE9Pd2poYTJGNGFjUkN6eXFZK0MrNmpBaU9EVmZlaG5JeTFZVUE0Z0hGTmZGc0lSQVRuNGZ0TEZPU0tDRFRaUXlQeVNDUTc1VjFFNVdIZ29XNEp5aWtNTlpXK0tJamJYdHMzN3lxQWhrcURmU3h6dEhoNzVleDg2UTVuNmJPWXQ2c0FwL3ExcExnaFhDU0cvYWE5cnpRQU4rTE1pYXowNlFGNGpuWFpJTitXVHpVZFJKOWNWaWFCbm9RRy9TZ1VTbFF2VFRyMWJLNGFLTkJtUDQ0RG5MazRMMitTOHY0R1hMVDBnQ1A0UkFkeG0yNGFXUjZWRzhZVHJtV1c2N0xoQmo3NWJjYXpjak9DTFdGZDhWQUF6WjNhclVSMnhkQzRJM3BQSHh2bjc0NWErdUZ6VUdnaURRUmtvR0FCa2FiMDZhSWhEbWUyYWtFY3c2eVB6S1R1Nk1ZTnNWdDZKblRCRzhDY2pQa2lId3ZSalFCamtQdUpNRlFFQW53bjhBdXNSMjVLTkRTeFRCcG9LQmpaSXVMelZwYlhXSU5GcVVzSU9lN0ZObnFHb2JOOGJCbXFzNGd4V0pkdzN6dFRpNjdwT0pQWGxUem15bHBvUjZpbHVnUG5VZk5JTzBiNDcxYk0vTHdITjRxcHVISVl4VUV0M1hjbExLMTFXTmUzS3ovRk56bjExZkVvdVl6L0pOMDA0K21UdWpmWVVWc3FleVdvRG9abmpSV3p1RklYSlNwOFRzVzFWTVJXdjFuajB0eGpyWlRJdGtjb3NXWUFnYzVKWnQycFcxZm4zQ1hqWkw5SEJQWDU1cUhoWFEyRU9NUVppTTdFZlZaR21HWVpjYUtkQlJRWmhCQnAySFFuQWhDWHo3c1FnMnh5dk13ajFWWTJjejhIVzZYbGJRSmJHTUJWZEIzVXZvMGxTdDkyY1VCek1FKzJycG5GR1cvOGcrbVZzbmFPT3ZZNTFOcFBxVnBtVUNkcTBrYWFDZmdiUXZTMnMxZFpEYlhjUkRDaWhidEdyaTZ6MUhqRVZSUkh0MFRvdVozVEtCWm56ZDRXdlIvNUQ3L084L1NHRytsWkZqTzhQVFdPQWthd2V0dHdvcmpCUThVVExFREN0VXE0L2t6ckY4YUIzUG1La2dJMnpMNUs0TzZPSDZDU3VYTXo0OW83MlQ4UU1OS0JWc3l5OC9haTJHakh1Y3kwdWFlTEZlbU9YSUxzSGlZQzIycHZvZ1pGL0ZWem9ROHJrenExa2VsNGpHc2c1VTdJeGFGOERzMEV1eDVGQ3lpbkZkdFlLRDJUNzFrTXZBVkljdXhnQWJkaUxIY2EveXE1YnRNZGFzaUlkcjFETFpaWGxuY2sxenJldkdRTVBQSkw1cmdpTzkxRmU0T2w1Vk9acDhsWVpnTWs0NFFXSFlxSlNhZCtIYmlnWTFvKzlJZnBJVEx0bUpCZ1BPNTdXL1FoV3NnbUxBTWhneXRRekpSSGlGeWVXenpsSEt2RW5jOFlMdDlQQnkzcnJpRkY0a3hkM3BIbm1iMEhnTHloYlFEQWxFQkJsbStvZGVMU1dDQThMQ1NBaGxIVXVxYnBWTXpsM1RrSkZTOG0xSUlUb3BIVXAzNGhDaHJTT2VWR0ZFaTlQakF3MUtIUzlDaE56SmthRHJJeUxJa0xjYzVTL2dobjFHWVQxeDhtU2pXZHlOMWtTTTgycTNZNEJCMktuMnAxaWtvVEk2ZE5XOGEyc2w1L2tXV3ovdWw5SkhTUHhiRndTWXhFdVBhTnMwTFF5bXhFakx2Uk1vSU9qOFZISS9zamJ1T0hRMS9yTUJiL2hpTmIzbUhETlRxb09JZWcyWnlEcVFnVWpFUndNV2E2ZE1BaGVXN2ZBUnpFN0p3R2hMQW1zU3BjNjZMblpuOGwrT0JTVXVleWZzUTRFdFk5S0NUeFRPWHdLREhDVE9naFhvTjh2NXBaUXlhZ2dZb0VRQ2FnNFJOMURCUjBET3orWkNEUm96M2RXa2d4UG02NUlZQkdhSWpjdXhXY2xQczZGdmJxQ1JRS0g3Q1J0UzhBWDJudG5wMXhTSDMvbDhJanBoSnBlVFRPNGxUSEowUlFPQStPVUllNTlxNEhRWDd1ZVBnNTJlWTI3TnFyTWRrZnRUL3ZXam1OU3IrLzI3UHp1TElIMlc2RGMyUHpqUllXaGFJZUFDYjIyY0puM0lVdkdxbThjU1ZBN1A2VmNrRTRJZXdIZ2ZveEVnZ3hPbSt5c3lZbFJvZ0NpY2dwR2k5VmdNczY2aVpwVWs1Q09RTytwUFpoU2szQWZVTW0xR0pFSVRFWkxJWjdnaDRTRHJpYi9zempSN0lhbW9neFIzQ3RGbVhkZlVNMGZBR0RoQnRDV29MemVNSzJlZGlPNVB3aElzM2xrek8rR2FwODNpN3YvbDAySUhJL0E5OFJla3p5bFhNSk1ETk5jWFpOcHRUc0FXUnNjRWowSGlXN0xYZXA3TW5OL1VrcVFKRzVFSW9KdVNocUUvZHlxdXZUOU5OeWYxWDN3V3YzbUl6b0RDa1MyNXdqcDNWallkMlBrN2krSk40WTREaVlaTTRsNlVOc1pSaWNITkswTHVOTXJWenhVVEltQkVnaEpES2ZveWtGS3dKQ3BQNER5QUhqSU4zVHdyNm9zU3BUVFN5WVRCanFSbTYrSDE0ZlNVQ2dRWVR4MHJDT2VucGUrS2ZDQVFRWE85YzI5RVd5ZzEzODFYSURMVStkcGJZbkxVcm9hOU5jdzRKbzRtdFJyRm9GdGNDeVVpVzAzNmNiVG9NNjU5VFc5TGN1bmZzNkRYRjdVWk5uT3padG04a0QrTkE2MHJtYVhOVUN0a2xPUGpUOVdmclVndE9zUG9rK0R5elFBVWgyYkJncWU4eWJzZzRRK3FOekpJWUROVEN3WjlJc01rY1NVKzdmaTduUFRVNmNSNEJUaVI2bVpVbWQwcERua2s1SDQwaXNEdExmUmxjajVibG04TmxpOWczVHFRUTcwdGdoVS9wNkFhNElOUytOQzhEMWZhV0F6MGtVVzRaR2VaMWgzQmVVRTlMNzZST1ErWkhadTI4K0xjRjhVdXhGYlRUYjkzUVRyL3o4V2ptTCt2SDJuRWRlQlpEWTM4TzB6aE9zVXgybHg0bUxLVUJsQk1BeE1LeDE1YU1TanUzMzIyWHloNHYya1o3YS9kUGhWUDc4M0pOam5mMStwbE04ckcrWDd4RVgxRElGc2ZmSjAvNEM1WnZVT3MzNUtYMmZ4cEZrWWhERktMa01EeGp4eHJjSHd3TVdtNVZHK3FwRUE5c3pZKzlrV3kxRUhBaXZWZ1RueERGLzQ2eVhER2FBbi8zcjdlYjlKcXB3cGJvNGljN1RuTkJzVURXNDBRalhMbGFIclRnVVhQL09TL3hJRm03TEsvZDNOSTRGa1BHUmt6RzRlSjdHL09EMWU0MzB4Mnp3QTBoeEJlQVI4L085bzA1a04ybE9xYkZQZTBDaW01bG01ZDl4Ti92a3E1dHAydk96SklzemdKZXpqeEhnU0pBN3VXR3ZIYXR0MFg1SU9xRjJ6dWZ3OCtibUlUM04rd1BXcHVORkFaSTNGZDU5M3NLVjVNVWc5ZFBCcEhOTFB0aEpSTWtnazBRYU1YZ1lvWkZPdXl6YXhMV3RvbVpHdFgwOG5GREFYZWpCUmNGbEovVmxoZVd3L0ltck1MQTBiTGdRQ2wvR3dJdFpYT2w1NmdGYWNDYzFNeU1QV1NWOUxIMFRYdjRteGhHTnNJejYwUkZwdHJwaWRLeGZrOXhaTlFVM21tSFFsVU8yVnM5VlkxcnVBelhmelIwYitIcXpGWXBCS0t3VjF4SzlyZVhlaG5JOVJnaFNQQkRVbmIyV2RqRVJ0Z3JYSkhKa2dPb0EwYnZ6bEdaenZ6cEQ5cVN1ZCtSODI0K21yMWttMHM5cWpWd0tKTy9XSFhabkpTcWQ0ZlRjK21uZkk0NUFNQzlWVzNVUjg5REFoUnRua3JtUDBzY0FrK0s1cXM4WXRyaHBuMjN2akQ0aFJIakdMNHRaZXNGREhrNG8xMkYwbi80T3Z2QU5ra092bkQ1aUY0dVYrU1lFc1NqSFljclkwTStxMDlrRkRFcldId0M0N0UzU1Y1d0FMVURDcTJJMXlhZmxnTzlNa3orenpxVDBZTUk4WWFhN0hMdkx2L05tTFlRT1cyMmlYMDdzbEFHSEtpdmFldnFXQUxRdnE2dDhaUnUzWE1tbXJ1VzczWWxKZEYvWDJ3WWZwTDBhWkE4N3UvNTNwMVhQbHkwcWZqOXFXRGdUNzA4eVlXWTJzSUZKY1lKS1N2eHMvSlZ2cDREQzhMTitqVFB4eUdiV1p3SkthRklOK0N5WENsd09EQkxqMWJnUzQwUVlHcjRRNUlwTENZdEl2ajFqNW5nZ2JPZmFxRDRFNFlsTHRoblB1WTFZd0Uxa21oVGNBb09YNHg4Y0RKaEFvODMyOU8zTkY4VlNOdTk2UDJiMkUwSCs2SjJNQmpxdnNVL3Z3YUxuMWk2bGUvdnl0cHJVM0d1TEY5cjVNbzBpZkQ3UXZyZTk3bVVqWS92dGRkQnR0b1hEK2RmVHFhRUhGcGUwc0crKytscHhBV3VyZzFHK1Z3Ym5ITDFNeWxjeDlpUTVHUkRGM3BwNElucVREazU2clJ5SzVTTUZEU0xBZ2dBWmdsbFFLM3NwZDEwM3dpS0JvdW5FYkdNbGVoZHh6cEx1ZjVHWms2VnNiTGZYUDBQR0lnZjk1Z0Y4TWN1aDRMaGNBU3lBUXlsYXU3QWhQTm5qam9SaldmMXFMa1g2RGVzQWF6akhQTWJkR3pYVmt5dWRpWTh6RU9ITmRNbnBKN3FLM1BTNzJtTTFtczB6dGExVUNmV0JhblZoYUtTbnRGcWJ6ZjYrVlN5V21pd21GUk54S25RSENBc25ranJ4aEVONWxEWXZSdmx2SStUbzNNb1I3T281NmFsM0hzbTc3MXVBSkJZL3J5TVRmVGFuQXVSdGMvcm4zQTNIYkp5SlFzTFNJZzBaMStHaVVVTEY2cHRxdWcwQkV3SThZcG0xblVGeSt2OWliTXlhUVVUNE11c1A2N1lNMHJOaTlKVjk3cXJ1eHJ4VEo1RkhjTGZneXVhZ05rM00wWk83THdERFBjc0NRSndIVWRBaHNKOW1SMkE5Ukt0K0wzRnUyeXdYaCtNMlI1ODNtbTExZXk4anQzUFhlcHFrb3UyeG5Ia0pmOERvN0pSMlQxWjJYM3gzUlBNQmQ3RFIyejRvcEgxNDJNYlJaUHVUdHQrbEIyQWkrVklmMHMxSEMwdnZNK2NwRmhZQ2JvQmlSZVVmbGU0YW9BQkljVmMxYjdKaVpBZkhjQzRMamxEMnVyQ3hpUUprcWlWMWlOSHFwbjlJTHFLYWM3WWNiUWdXc1lvNjZoVnJ6RmZGVGhHclNhdzMrYjd2NU5YNVlIdGxMZzV4S3J1ajJ4bkNzVmdsL1FaUjBXZDA3cHBKek14WjdObDVwbHI2VHA0K0JIS3VaQTBGUU8zdlRkcXhQUGZ6ZmExMFovSzFDMFlhZU5yVVRlSFJsMW0zQlRGYjNYdGlqYnV1b00yRXJTdEpnZFJscFg3T1lWUnU2TVA5TW9mMGNkbFB3S1J3YkhtbDZhdlArNnJzb3JyQkM2QVVEbUM3U3BMUGhZeXZEaXJaK29QZ1BESVJENFQzNnVUSXpCek9YYm1mV3BIOGowTi9rcFdSMFlaeEY4SGwybXU4eUxnTjhpb0h5NkZZazFRa2QwNWJBTWFDSnBzSUJJaG81ZUxPZGJuaTJheERJd0dLS1hvQnNSQU5ienVKUXhCZ2w4UERuNjVZM0lJUy9XSzZNOWVlTERRL1NSMVZqeHZvc2pLUFk0VnhiSG96OS92ajVmZUdnL2hiTUZqU29iSzdkdDY5VjYySThSUkl1Tnd3VWNMR29Bdkh0VTV1ZjUxNExhczVZU0FSWlJLZ0JDU3ltM3V0azVRSThuM3JHeU9KTjVCNkE3QzZjci9pNi9RR1VDeFFOb2ZONVowQjkvY2dabnhCWXEzYTBLclRxSVBqQlJMQ2hZUmlIQkNSdzBDT3lPWjVqSEVjUTZwMVpnOWNqVmRvVWRpZzcyWVczQmZ5TGwvanVPSWJYV0FQbnZRazJSZzlBSkdUU0ZObjZSWTRLSjd5UmlhcEU3SXNkWFV5VXIwSUFCQTJqZ2hMY05uZE1KQnpQYWxsYW5ucjQwZDAyQUhzWFptaWVFd2dkQXRXZ0JQNlk0RGFETVN4YkhNSTh6YmYxR2NQOW10Q2lBemVSbnVIMmJMbWxUL0RsSldaSEk1ZlhrNi9hMTlFV2R1MGJmcEFFaDAwTThGRWt3QVlheTBERHdEM1lIV2dTNEFTdWswRkZIMW1FZG92SzI4Z1FVTFVIa01rM1FjaVJJaHpNZEV0NzEyNDRMTkhoTDhJb0xucU5CaXN6L3E3c2kvSXhvVmNXQzA3Z0FEYzd2N2p3NnMrU28wUS96TGJSODNlRXRqVDFabndmbVcyS2plTGVEUnpKdnAzc0p6TC9CVlhVOWp4Si9sWFFpL2Y2RkNmQVlEcmovS0N3SnlJTFNOMGpWbWIveFk5aU4xRElvQzJ5cGgxd2ZPYkpjZGVDVWloRk1QVDFlWk0rQmdsY1NLMTRCcys5ZDJ5eUdNN2Q5M1h3NEhOZ0VNSWRYQUd6YlNTd0NvR3ZrampsckI2bi9wamdnVnd2a2lERkJFbGhXcGFCSlp2SFlNMXlVN25TOStYMWtVK3Z6czVCTGY5VDkxUEpPSXlQaU5ya1F0VHc3M2p0NU0ySzVQdXljWlF1bk1wdGpMMzV5VXJvU3FkSGhCUzV5cmNtN1dMTFZibG9kMmM3Rm5PYzZleTd0M3NkSHZDd2x1TS9BUk1iblFsVHZTTVF1QzVxRnRrRGwydzZ3SXUydDZPKytFMGVCa29zdWRxNGRvb1RpS3c2K3BTSDg1bm1SQW5JYzd3M1h6UWNLbXZXeVd5cWRmMVBCWnJ0UTJsVlJuaXVISkVVM1FsTDZRQTF5bGVxeml2Nmk1aVg1d2lES0pkRE1pSmVyc2tlcUI2SkVUbWJFNnBDeEVPNUtaWTdkdSs5OXZBTi9kdTNOaFRKbXpDRkQwbzFhNGJvUnBoR3hGNkpEUDlrSlhZTndCaWxwbmRhdHFTckFnL3QxSGxmcjhiaHp1UStDc0FzVG8rL2hBcm0zVzN4RkQ3Y0xMNjEyZThlUkV2MXI2ZE8xUVpZRFZ3NnZvdGY5WTRIN09VWkREM09jQm0ybzArWjdlTE9LWWg5MWtpbFJFb2dNMFZydTJkdEhNWG9DRVdxdytKNVdXSXFNQnBzUDRQUzRBaklLdzBTcjRlSGQ0VXJGOHdIeE9IeUNVTkNOZndUZmtqMUxTR0JFQVpZTnQ3azcvMUIrSk1oeVBHQ1VpMm9SR3pncE5TejdLSmIrbjVNNjRvOTlCWXlvK0lQZ0ZOZGdCcTNkdTArUXdnaS9jTHZTZjI2RmR1NjM4cE5TRFpmWld6a2p6QzhzMlVOK3RCTnBjSkpFSkVZZFFwWnlFWWw3NGduTWF5L29OU1BYSXJlYk0yeXh3QTEybVFuZTliK2krL3JzUWgrZEVVME9HYUFSWlpCTWd1L0g3UGlUOGVTYldNTCtWR0xvUlBCMFBtMURkWmFFVXAvLzhwa0l3cCsyejZtYm1XNzhzVnpPdW8rM0VXZnJjT01wUDZ1U0NrbGg5YloycjhqdzhJNDRNV09ndVN1WkJzRnQyV2ZOaWh1MVUvZnU4cnk1NG1kNXhBQnJXUHJDYWxuL2VBOWZ2amIrMEMySWk0ZWRVdGVkdFZ2Wll2RnY5SDhTTmFTbkV0bVRNdHlsWHZsLzBtNVVpaWJnY1J6U08wYnR2c1VoOElhWWR2YmxOeXloazJrc2xCQXdFUXNwcnJlNDdoMXBhY04xbVdIWHpNYWxQR3c3ZndjMlRXMjErTmMrRlU5dUtkcXdQWENQR3dSYmVPZkE1MVVFdFhMdVFENjh5L2xYNUlUUzBKNTNDZngvUWNCbndHUmdDd0Y5L2VwTFFhVXIzbDlRUGs3SGFJVzVIM0UxZDRkOVgrTDBxVVYvSXFLQnpUc3ZJcFY5S1ZyYlk3MSsvbmE4UUM0cHQyWVlEZzFYcGJSRXIwMGVxaXN6Rll5eHhTTnUweUlqQ1FjekFjc1VsbVFqUFdmcG5ZZ2psZENaczVaZWRhSExHQVpUSVJuQk5rRU94NFQ1dGpjYlJvWlAraTRDZ0FkZzlZQUhneDQwL21Nc2hjOXV0M3k4R05yamJ2c3U3a3ZaOUlMdHpZbVYwNlQ1Kyt5aXk5aXU3MWxZMjJHVzhUN1pvNEZIV05lMmt6RkdaUEF6ZlRjcUYxcGNtNDIxMzhuMHovSGdoeCs3cytXamlNN1NvbnliZnZaMytQNHQ4QnVBZnFwdFhnT05LUzMrN1puaDFHbk84cjNDc2xIWW9VenlEbllwSzNwY0N6Y1QyL0V0RUxoMkRkVVZEUXlTZXJlK0s0OC9nUVhKOHovVDB5Y3NaQ1pjWGRKeVhWTVJKaGZSblhaQURWa2gzNzZmbTFuWjBqdkhuSDM1L0ZhNG1zKy85TEpOQW93RnlDanc1bW1semtlVU85L2JIOUZvV3FUankrSjhTZkFzU3VYUUJKZC9KZnhvTGN2V2htL3h0ZExIVm9XaG51eEZYWWI2MDNlNklDaVRDdDdkSjA4RDE1YjZ2OTMxbnphVUdLQWt5RTR3Z3pMdGZpVVNmYkhETlFRbkFQQml3Y2J1d3p0V1BsL1pCeUd3K1NNaGJiMUV6RVdWUUpjSTVKNldDU2xLL2VWeC9rb09CWENWZ1VlVWNaMXdRK2dKdVdlL29sY2VicFJOL2xTelB1UFVmOHVOVVVSMFlIK09jSFFnRUpRQTZyNkhyNzUzekhIVmo5TjBCSzM3b1BJRlpvYnIrQkl2L24xQzAxR1F3eUZySHBOdG9Fem5xVnNGeGtiZ1J1L3ZXU0JoWkVEaURaZW1NVlpCRExxN0Q5Tm9LMVl5VVl5blZNZXkvcDJ3WGpQZ0lnZkR1LzlzVkZwelIrbWZEREJNMzZDaHZPSUhjK2Y1Zjg4dW4rbURHV3l4VXQyUUVBcjlicW8yanZ0L1JjV2pESTFGOExGWndvcjRzNmtuVTVhQ2NwMzk3MmI5UFVzdk9Vd2dwelI1U25aNTFENkNMUmZlblAydnEzeTc1TmQ1UGc1dG45V21DZ2tGWjZ0dnQxOVMrZW9LMFc5d1hKR1hGWWk3U3RhV1prempvUTJ3Y1Q4c29Mb2c4UXdLQlNpWnRwRFd4TVpHSEpXOCs4TVk1RWZ4dVlwRDR6RXZtb0tDWDR3aWxXU0hvcDV6cUNrSmtoUjFka3Z5ak5zMWgwU1JTdnZZL2FmQjA4NDByTXd6ZFBOanNCNy9uQy96ZFR0YTh0SDVadUprYXJFVEQzNFpvSUtBUDV6bHVGRWFFRnZJZHBOVG1WS1gyaGZlN2doTnJDelhIL2YxdGZrdE9ObjluSEtTSzlCN0hFR1BPeXd0c2k2aXR2ZXNhQW00Y0wwRkNhODdxV1pXL1hBS05ZeGJOelduaWVHb2NTN1pndWE0SkRKR2FFQjZ4MjJEa3RVczlaTmUzNFpqeXRYdm9oTDJKRUR5aEJHeEJsb25XbGFwUVBNS0x5VitxeGp6ZnhJdkxmcjVFT0U5WisxcFJBaitwdkd5LzROOVN6ZzNtbThJai8wZVRDMXY2cGp0YVJvR2g3cVlUYXVBMG9NTFNLMm5ER3ZRWUNTLzN0OTMyNkE1LzlRZUgvZHVxaTdLNzluR2NqK3NhenpYWC9XeXJpbXQ5QkFhZ21Rd1NCM2k4N1JuVWg1cFN0OTBoK0tXakFrMWJ1Ykw3TnlMUHpZd2szVnVtdkJTQzZFR0pVRGs4QWJjdjBHbmwzTEd1Znd0UktXejBFZ0tMM21lMDN0emFkd3lFMVhXdmU3K3ZTNXlhbjlLK1VBYWErTXM4MUw5RUVjM0JjZXhCWnZ0cmYrR3V1RlhMaXIzWkJibTVGRElydCtIWmcxZnZXSTJQUmxTQSt6b3ZoQnhMbHowMjVoMVQrM0RSK2ZndC9jejREU2ZSdGplOTYxK3dudU5RNWpqdlErRFJSSmxScnI3ZHY5empBbzNBWUNJTEtlZU1ER2RIWHZ3RkU3S0tGY1FreDlnRVlUcHl0YjBCOGc5N09WU2VKNzRteDdvWE9STGtham04ektYRmF0aHVaYXVEbkdkM0RZbzNocUNmL0xRQnBNc25pQTZLbTN6SlI2dk00TytrR2FPd3QvVUR2WDFrUkgxTHpwaHg3cCt6bHd4MzVwQWNSemlQczgxLzJnZ2RPcFJQMTM3N3lCUG0yN2dWWWx0WnFzbWRIVHVUQlVMNEQyay9TVDhGalY4eGN5RE1vNUx3ZEpEWnNTNzFpVzIwamgra2lBakxTSHBZMDlhVjhxcStZakZWRVFialdsNjRrL1V2bVVDOEFOR1dINjZWM1g1amlSbTdNRWZUN0F1QXA1OHdVSlcwallQTVZtVWIwQmlBWmJOd0NFODV0eG9ITXRDQmJ1ZXkzNytEbDc4Qm9SMXVqNmtTcVRtYy9UK3VYZlNETy9DMFg4aTZ0dlh5NkNSQm9idXFKUXQ5eU5PMjNLTHhXb0xtdHh3RGtCeFQ5aEJOWk9yUC8rUjlObVRodGhjOXU1ZW54VXE3ODQ4cHBPR09SL3RraVhIYm9wdldDVTZZdGM0SmF3V29IQUFBZ0FFbEVRVlNta0ZSVXloekRZdkdoeGhXbDU4UHprK3QwM0FFczlSY0lzQ09FenU2YjlHeGZxOUNkMVdySG84K2s3WEtBcHlwZVozSWJMVHhDNDFBNkp3SUFmTVc3NWtRRUNVSzlxVyszYlBnR3ZITHpLVFcwMXYrM0p2VVNkYjJuamR4eGF6NXRtdTJmdmRlUkhUcW0zK0FzL3ZWdjBHVDYzZjNzeDVIL0Zpc0tWbkd5UmlJTDRvdGRzL0FYTEJ3T0s5SFBhTXRiaXNYWVk0dEduNkxlREJxNWt5dDNFcGM1OEJGRG5MbDJPaGlrK1RTMUw5bjEvU29OU1A1OEVMa0ZPTXFjakkyM3I1dk9LWVdUbnR5blc2N1d3U1daWXhoNzBCbUhXc3lKYmRuRlc5UG0yVzF3b1g2SDJ0OW43ZVN0M0U5NmxiR0NOczl6eGgwSGNPYTFXc1k4bWR2VC9zYWU1NDZsMk4zOURVVDU3ZFJBd05MdS9DQnUxNVFJTGxkSG16SytFOVlKSXA1bjEzVlo4ZG5yTWdLMVp4MHNqTEJqWTFzbTlPaEk5aXlPamtad292SU82WHFtZjFaM0JHY1Axc21QMUdDQVZZakpkVmpGVXptV21LeUpRN0g4SGtITlJzczRhZnRXY2tidkNncHMzZEYyNjhLNXB3VU96bWN6QmpmaXpMODlrKy9yWDhCTnhRM3lvanVBQzJSdXB2TDFSemFOUVJTcm9KaElTOXR2ZXI4REw5SSt2UnZLbmlXZzEyYmR2bzcvTlBmUmlhZ0RRVTlsSVRpdkNhVWRaODhUZ0NDM1phTFRobnVSUGtaQjYvTk05V1p3UW42dVB5YkZQUWVQVnFCN3NzYW1QM1V1YS9YbTk1a2c5VWpOb0luVVJqckUyd3FTdERIMDl5VEN1T1RhdmJLSkV4ZGlwKzF0UjF2N0VhdHBOdUgybE1kemZhaWN5TG1aUTFvNmx2ak1CK2szWGVlemVUQ3c0Y0FxYkZMaGsvUkhCNUJDM0dRcnlNUCt2ZS9DWDZkL3JmN0dmWlN2eTNYc2RseUlsK2YxV2VaUXZHNWVQVU1wWFhNQ2o4eEpNTGVOYytrK2tBaVpheDlDUVJ2OXpENG1mcTlONnlMSzZGeTcwanZhM3puTml6WnYxQXRnZFpPdi9zM3pib0xMYnpzVGh3eEFWSDZUSGJsMnJYcWFRTnNOaU5nM05RdG5CWSs1Q1dhMG0xOFRMTEZkZndZaU4xTzJjQTYwamo1K0YwQjJYVnE2Z0hCMHVrdDl2NGZmMTV1N2s5MTIzTWxUMEhpYkx5K3hEMnI5OVZGOUF5RDljaGxocmxUVk9aaWhoTmpGQzN0T1dzZU9jL0M4Q2FRS2tDbGFkRURpVkNiMnZWU2dzTU8xUkRSSlk1RGVJODU1SWJDYU9NME4zWjVuVDlnY0tNc1dxMWlJT0ludXhrWG9HYnh0VklYYklMeDRPamk4RWdpS2ZwQWRjQWF6Um9YUG94RFhVeGZndkRsdkpsWEZtWXZoOEZEVnZ3Y1ErWVZwNmVaYXEydXp4aHR4THNTN2IvK2RJblduN3pBZmtYZjdVZXh3b1oxWVV2TnkrdittcmZ4N3lXUTNmc2FqL091Y3pZMzQ0amRQUmUxeDg4SGc5c3pldm51akFvbnpVQ0RvejJJYmZtdXZjd3U1dTVuajRhaHJPbmhFVzRCWVBiZ2pWMnRHOURCQlNabWpBVWdqc09zQlZxa2FFNVdHYUNKQVdsWkt5VDBCdC9xbFRUeVJmbEt4d0JpVjVkNGFNUGlwZmkyWnpxVUFDRW1vQVBZNjFuTDJ2cFp1Z2hMbHp1L3ZoNDdsbnR6ZXBvMW85STViNlo2bEJpeTMrbUg5SnFSdGxKZ0k1K3dmM2N2dzRCNndYQitlUUtEZmo5WFlSS2c5UVA1cStnQkErbnRMLzFMNU4yQ0REZmRnSHFmWmV6VTQ4MUMwR2dCUnUyZDl5S0RsZW9WMHYvcVoxSExkT1RORUlmYThWajYvKzFTUndwVENZeWIzZVYwN21aUnpNTlpKQ2Q5aWVZaHJuRVV0YVp4SUlmYllST0Q4RENGWndBaURMNmVqb216VkNXUzZsZXpYTzlJN25kSTFLdkNFVGlSTnpETUIxL3M5VUhWdTNRTGkvbllxL0F6dEQrYnVydTlMWVUzTElWUjJmNkhVM1l1K1R3dzhHSU8yMnR3MDB5TnYvVnZpaTlYL0RrQ2t5T3JUYURkczJqcW4wUEprUXZkOUxJbjdRSHBtUHpLZ21FZ3pHMEJFbmZuVjJydWxaK2F3WlJhZTRwQ1czb05hSHR0OU85UDdXbjJXYkpmc1NISVZKMHFOQlRvVGN3eVFoMHhzNHNpTDFjbVM0QjZqeFdYZXB5ekhtQkxWUmMzN094MzRMSFhudFp5Kyt0Ymh6b2s4QlpCanltRGtIY2wvODhqUlRkNEhTYi9zcnNRN3J1UVRBdnc4NzJja3p1MDl0cVUzTjM4VlNCcVJyYy9YZG51VWRnUE5USUI1OVY2YWFnREVYT3NzMWcrdStmMytaQmRMckl5MVc0RFErOVhDQ1NSUUtHMXlKZW1JN2NGeGx1M0NhaVd3MDA3SDZRQVpSWUlMSmw5cGxqZlRyQ2xmYm9lZ25JN1dUakgyQXd6d0xMNG0zbTRla1BTM21PbUpWTTlpQUpaQnNTSklEbUQ5SmRWOURpQTdHOFVUVWNUTHJ3dlViZHFCdzd1Z1JFL1kvMCtKTVRUYjVXYVlsdC9VLzJsb0p5UGNjVlB5eDZEeTBLSm15UUJrYVpmYjkrdjZCUDlmWFhkZEZHbjVRd3pjaUNlS0dnNFduZWlSQ0ZucmQ3T3JhbEd0THRzY1p3ZGZ5ejEyZ1BFNlhSZkRWUTd6L25wSW9uSlByMkRFTHlLWUVxOFAyb3hzYVFDK1ppTDNqVU1POGRRZ1FmcXVXdjlVNWF0bnB4MXRWY0NTTzdRQVNrSGlsbDVtemdMd2RaN1NQM1YzMzNFaEs4RDRxV1VsN3o3WkNXTC83RmJsRFp2MmFVL2xCMVY1TDdmeHRvZVptOWdiZ1prakh1eGRmZS9hV2tHcHN0MC9UZHorTGxIRUFBZUhRaDRwMzlZaGkrTlpiOHkzNTBmVktReGdyU05NdkZ6cnNPZlFzWGRnV2prUXEzT3lPWWVKU0dKSzFyTEhaVE1YaUt2SWdqUW1VYkxQYVBreXduZ3czSG5NT1kyWlpUV3dUNURJU3h5VWt0TVhYMkFhVVRkWVFVWGVMUnQ1bG9qeW1ldkJXci9CSW1tdnRzbGpyRzczaEwrZmpwc2hmbHVtNXZ0c3lyK3djaGFVTHJabjRpNlVSYzR4OUVlRVhSMHJhWnArNHFkUjIyc3RhWUwxdnVkN0RGOXBjaDg3Z0h6VXBZMFkwd0hFNnVTV1AxZGh6OXE2MnlxdUcvUGtWblVhSytmQ2xNYTU5TUVCZ25uaGFDcXdjTnFkRzhHYTJZQURBaDdmbXZleWVuUlJpckhmTDNPK2JZOXp0SGhHejB6dWswOGdDcEFyWEVpaUprSUdGV3JQZDFSM1lYZU80RXN2UXl3YTdtdVNReHgyNjAyRVlHVGRhTG9Ca1BpSUFIYldtVWVIVXIxUEIrejZoWFFuWW1XZ1dTT2ttN2pCcUh4V3p0YWZsYVNUK3lOeGhQY2hDMjVCNDVQNjhmc0FVcDd0UkpOOGU2UGZrTDlyT1ZPRTdnQWxsMmN3YUxZOUtLazhVdmtneURDWGR1Q3pQTzRmQWlHUUsxbGxyQWNqNzhNSHg0UktmU1J3RVUza25pbzVmZktrWnl5V0Y3ZVkrTjdkSzRrTitac1ljUFdadWZuYis3bTBMV2JyZ2FuK0ovbDVpRU5BNkZjczlmT1NsdVFna2kwU2Z3MGdYWFI1STZiOFpXczVWUjhTQVpBVEdKd3NNNVJZdnBHcGNxSFF1SkZYVDVYUU5ybTZPMzJzdXYzY2xlT0l0WWU3dkprYk9LWWJQY2dhcTJhL1g0UTdoOUF5N0N3aVd4TndxejhUVTFaNnVxakJuSzRyWjJHZ2NhV3FZckVQcnNTdExFaHU0bENITDg2ZjNGYndJRm95Zll4TEhKbnpxS3R6ZlNsVzRFbTBRZDlhdisyNGtjNjY4NWxhZ0x3V0FwQjBIUXZIWFBRZ0ZWRGM1NlA0SE9RY2t0OWk2dXpTb3VCRmZKOHYvWVZPV3JteU8wZHZXbVdHWGE3bHpzWXA3MWpQYUVENzFuR3MzY2dUWTNFSW95aTM5QkU0NDhnaDM3dmNQeEdEY3BIZE1hRmxRZ05iMFU0ZVZPTE9mKzg0RUU0WDlpeVBjZWdoMnVSczk0MmJzSHNGb0Z4NUdmVWIwV2V6cmdWdE52MUg4UmxwTDBiTStFYTBOelZleDRSWVdvenBFS3RHQXcyTDRaRkJaTWxUZ1VPY3dqb0E1TC9tdWNwMWpDbjhURno4Y1lna2JUdDloRzZHTFdCVzUwTE9GMVZVZ25MUEV3b25Nd2xrYmR6VldNQXh0K0YrSXQwRUhBMjg0U1lhUmQ4ck9hbVVpVVBFNDluVEpLdStEUEE0RVUzdUovYStJVFdQUU9sWTdnYW5BQmJ4aE91amN6OXB6V2EvR2NxaHRJOTZxbGJBQXRYaGJpTTJVS3Z2WGNyV2tGNVhBWkFzU3VTeUc2NGs2MHVDODJaL0JzUVllRmhERXk4MFQzRW9TN0tLeGZLNFVyczhBNFRzWHJpb2gyWEg4cHBvRzB6RVNpVDJMMVlTVnQrS05FQ2s3MFVDUkpRNWhvNXE2VnJHZlBvRWNhL1VQTjRVQkE1dytHZk5lby9TMkFEWUNCUUdmdm5PU3FqNXZPU3NSS1pySmpybmxGK3V2K1JWOXVUMTJTRUxCZ3k5NjN0ZzZVZjVNaG5xMWpaZlhBL1RXU3Q2ZHBKY2lLcE5PVm1BWFg2WTZGRzRoblF0encrdCttMjUyTkh6MEhuWngyb25uaXhTVk9wWE9ZK0ZhbDRUejV3enUrRkMwSjdic3pzQUdlMmU5Y3VKdWt6c1BhZVErNVc5UmltSkxkbVVhOEJRNm1wNkR3TWVNMjVNanJaTUdadWR4TUxVR3NtSnh4NU1meXNBd1cyd2xhVjBmME9jSWV2YXN3dkVRNVNlamZOWmdVWi9aYkJJNHhZbjZHWGl4akp4dW5XMDUrSGwyUXFxNjJzcGlQd09nTlNTUzJQZW9lQThTdkNWbTZic1JMclhEamtiNGRpUFhURG1wZDVEYjdlaEFIcTdiNTYvYnlYTlBRcHV4RG16SnpYbStidXJHL0NWZlZlK0FIelhZYlFDQzFnZ3l1VGlBVHB4dzlTUE9ZL0h3RWgxeDdQYzV3QVlxNWE4M3ZSKzJvQ0pTMGhna2QzY1dUUExLcHZFaXBSQ1pNbmVuRWJzK28rc00reGxYTWhpRTd4V2tVTUF5MFNYcXl5a0xqb1VEOUpBdStBKzRPS1NoVUtzNXRyK01XeE9TZC96b3BOZnZpeEF2YjFUS2pxUm5BaWdqeFdzT3lJNTFVRzNvSEdxdllvKzFvaXRDT0Y0OW5hcmZtTUxHRTJKbWk1am5LczJwWEl2VVNDSXQ5Ym5ZbktxTThTUjlLTnhTYjJmSjVHSWtNWWw5V1dYQ25oWTU5b3p2OTRBQ0FBLzlDblBNeDhuanNKR3VGNVh5Vk1keVp5TGNJTG5RdlJUMjUwVVJ5RndlZ2NDTUdlcTA0REY4alA4ekprOGgyUzRrc0l5allmb0o2S1hQaEtKVTNHZEJReElwdWVNY3hyekNJYW9VemZjb1Q3UEwzZ3JuakR5Ti9TcFZFU2gzTDdvTjJRQnEwQ1g1OFlwS0ZnSnNoNStJcjE3OTBTKzd1Nmp3L1hmcHk2bzBMc21kSmxqckJuZEhxSXIwQ0pqNk9QdWkvR3VmM2xQeTdsN0FVSWRESUE5YTltWjI3WndwRDRzVFcyQnhJR3hnM0RLNTNQU1ZxdU51T0xFM3dIRWlSNlYrK0NVS3hYTTFoSURGcUYxMXNWYUNSOXduY2szNEhxTkdhams5K3hXZUpoQzlSK2lTQ1drbFRxOXY5M1A0a2hzYzJNYmpEVFkwOHU3V09vY2dRSUtFZUFnWktPVTY0YzdYSGJPTGxMV3JhQk9pb1hJdVV5R1hkUTVZR2JrMVFma3RGQWxvTDJXdE00ZktWRmpyTDRsSFZwTVFIdDJ1Z0tMRE80cUpOd1QzcWtINzh2SVRrYkE5QVp5cHE2VjJ5UHNybSsrYWgwSUhLaFlWRGdGcnR6SFZqbXl5Ui95OXY0ZE96ZXkzRzlJZFZSME4xR256TWxadVlURkRaMnRpU3hPeUVWK0RnT0ZST0Zlcjk0emZ4QW9lSEJxSnhQTGxWRWlEMERxVTlhaGdCVk11SHFmR3VSYi95M3pTQVFlQXhJQUVsSFVPUStJMURsWi9FTTRRQVNJRUo5Wjk1RTVoQ3pXaFdnVS9jdHRSUDZVcUQ0UHZFcWxhWDMzbWpobHQxbDBNclAwb2xMd3dRYTgzc0FuejYzYkZla1dGM2hFSEk4QWx2YXlKNnJTRkNEeE9UOFVvbER0ZVcyeS9USk1TT0N3RVBXeWdnUkFvRDNLS1pzSlpiR3F2VGlWc3d4OWJ1MVh1VlFmVzdiVkw2VHJUUEtTWHgrdEFCSkVIL25OMzhQb3lqaUthY1NUS3NtQmZtQjB5T0ZqNC90ZjdKNCt0NmpzRjBlUTVzS0ZxQ2syT0FBZHRNSTZLU0F3d25Scno3VXhUbmtwNGhscXViVXVXQi9UMXdoT3BuNmhNa3Z5K0ZQNnc3a1BmVWJzMjl0WldhUjBQNVB3U1RKeEJqOEhrUGRjeVByczlIUnU5cTZRY3pFVmhONUpOSXgzRy9QV0dud1MwcjcrREZKTGZjWVdiL1F0eTZkVk9YM0xtVENTK1RDdlFHMmdONkNTKzFaeWIyVGJOcS9xNnArZVpUWTdMbU01ekFEaUVnZ3JoMklna3dGblpsZDM5ajByT1ErS3ZzWEVuRmd0dS91Njl6WHBVTXdINUU5Yno4MDhHd1NmaU4zYUl3QzJud1VHVHRaaUFnWlcvVWRXaEpySVUzUVpHVXp5cy94OXJZOFpkT3JZN3poTHpubTluVDFJV0IxYlhVY1c2NGoyZVhiSk9KRnR0UGRTeHpQdHdNK2Z4N1A1QzdGSWRweklPd2N2MnhpWFYrUjhZbG9YaVh4bE14eElON0xYcXpkcGkxY0RDQU9VV0NtRFpiVVZhdHR0SS9pbXUza0hIcldPK0xQYkhOZTVENXV3dmlEYmI4dXFSQmdnc1FJSUZEQktIV2tzTEsvSENVbjVzMWVxazB2U2hWaitDNHovaWU2RW5rSkZrbUhIMFhrSFdvdzFyeEQ2UGUzK1RHUE5ZSFVRcTg1aDhIZk1JMFJ6QXhEdE90L2R4V2hma0I4SDhhVHBNcmhQa09ZSjI1TnNDSHpJeTJzMXQ4ZG9jdXJCbGpDWE9xc3U1S2JkSFU3Sy9ZUzQ3eUtRNVRvdnhLWWpRNDJUdmJ4N0pZcUpWWFB3Zmd6RmJ5UWVldFZjbWp6Mmo4cEZBb3FiZCtzeXNJdDhDWXp5aHFtN0ZLdGQ2MXNEaXh5eUwrakNDRDN5RzYxeCtwSEJnd0hWNDJXbForVXNnaTRDUUhaYi9ZdUpkOVlYTVpQd1pNWTN4NFk2R2F0UWdBcVE1RTdYUUVEaXlzNVJzU2xGblZPeExmc1VkWm5qbVlKU0FRcjl1T0Z6a2tkZmxEK1ppd24rSWRlVHhpZW52anJsKzZhL1NWbjhFUmhwYzA5TDkvTm56Ujc1ajRHYXVabG9lbDlQdjk2Mi9aRGJtTW5wYXh2dmtlSzA4bU5FczE5S0owSjNwelRMWjhCSFZjZHpDNEFKMFBLRVd6eUJVUmVqTHRBY3YwOEhqeXllbFBxNFBzNHJxZ05zQTVCR3lBWWd2cWducEhCUU1KQ1liVzJlQVJyZ0ZISE14QlJPb1FEMHQxM2FVWlhmcmh1cDNFWDFNaldMaXZZMkVhUTdDemdoK292RXRmV0JFQ2pQUURYVEZuVDFnYXB4UVZMQUVFdVV2c01LOTNIWlZSZDVOY09xQjhuWmZIZlJJa3FmS1h4SlBrK1RUbVROYzErSm1ZRlA4VGRxV2pVWm9VaFNmTDRCZ1lGNFgyTUU3blFXZjVQeTNvSE1ramozYXpMbFNhZGlmMzNTMUczN0FMYmVwZEljbDd4bGo4eE5lN3NGWlB0VlR0K0tWKzZENjBYb1FyeHZKbDZ3WjV0V2g5TzRYR1JUTEtzU0pCL3NaS0tOQVJCUEN5S2srVHgwV1lnM01IQkJCQkw2VnNVZ1pZSm5kb3F5UFY0TVZ1L294T2w2bWZBTGlVUEpiV3UrMlpEeTJHUlF6UU5ZNE5GVFY3YkdaanhySzlWVFJLNVQ0a1JMTi9scDgrUFF4L085VFZJd1cwRGtIWUIwSFFhd0MrYno3M0lHMGdML1ZUUHZ1QmViR0ZsbllSZlpsNFRSVDAzRDIzNlZlQldrZFhEb1hmSzhJTzJIZzlRVEg1YUgrcENzQUEybW84YThLTXcxcDc0NVY1THJpSlVaU0taVkI0c0FLNTdwbm5lYjVhQnM1VVJ3elFReUNUdzRnZ29aNlVNQmhKSnZCcXZZVUUyeVVnL3BCeFdyaXh1WmtRSVF5S3B0b3BZVHRYeHh0MlFVWWsvamJ2MXdKWG9kZk03M0ZDakpRS0ZWMVc4VUx1WGtXYnAxRktGMi96UlBibVpZNFdRYkovSWVQSGFWcTA3Qkp2WkRBSlBWbUlvdTRtbHk1YVpQaERjOXpXT21uYlFKYjBHZm5XQVQ4ZXhpZ05SM0NQUHI0aXFmd0FZSmJMTDRVNERKM3EwZ1I2a3EzZUR6czFOcVkxekFRNS9uRlRZV3hBb2cwbnoySzhpZ2tUa1JWaVpnQXg2S0FEVzhZYlFyQU1MaHN3N05qNXJmKzhjV204TWExNysybnlYbGxXK2hnT0diNUNhcU01bUFTbXk5ZDNSTjQzOVZHYmFCQ0dWUVlobnhOTUxwdXlRRmF0NjB0MHpzOWdFN2FHMG4reTQvQUxKanErNEk3L0JzdVMwTmk0bjNSd0J5eVBJR0ZQNy85cjRmNlpZZnR3NjhNM3VZY3VBVk9KaXFHVTA0TGp0ejVBVTR0TXVseEpGSzhlUU9uR2tOQ3J3Q1JWS2d6Skc5Q3BYMzhMdUVBK0lBQnlEWTNmZjczcE1WREYrOTczYnpmNVBFSVFDQ0pBQUVEc1pyajRpaGlmUmtuOHlkdTdyMXJwNHo0Z0JoejlDQk1KQmc4bHRSallzb3RpUlZYNUw4TVlFMThUaHU1enA5Q0gzTTloMGdlZzl3WE5pSng1ZGRoZUtSU0JKQW9pbmNWMXRjenhrZ05nbE1scDRrYkVoV21acnk4WTE2c29BQWI5NjZHclVmYnIzTUt5bFdobERZdE5VVy8vNHAwUUpFc0JydlEwdVk0UHNCZDZqUld1RkprMzdWa1dnaDZjNWFOQTlLejNzRmpRd1VGTFlpOENpYlc5enJ3dlp5STVaeElqOENRTkwzWFc3dlArZWp0ekVPNlhncFZvSzRkUlNybEljRmRNR2JLYndEQk40MXpSNUpISEZheklaajNCMHQ5Z0pJQ0ZDNmVyTC94dFYxZzZXQUJvM2pxR2NaOEE0V1NLOEJqZ3dndkRRcklxRThMYUlMY3gvaWFaU0FaUG05TFkwTUVYMHpVUWNSYVRYbFhzanVjWndyR0t1TWdNaFF2cXFveUp6ZXFIdWU5SnZSSVAwNjV6RkVWTjhScTh3Q1NublZNMWpjalRJcUVtZ0pwYlAydmxxbFM1eElWMTRGb0U3RHlNQm00U1pLM1Z5amVjblFYNmJNQnlnL3o0ZDM5NzUwL3h6ZFV1U3crNXFWMGdrQThteVErNFgzMjZBTzZOY0ZYaXVjYlV5cTJMTGVBeDBZQ0RpdWc4MDhJQWgvcTJJMmFxUGx5QVFlY0pQRFN2RGtjeXVjZGtNbjRUdHVqZGl4VWM2QnBBQkRFa1dJZzFsRUhPOFQxMU5hL1hTQzhNRjNxT2xEZ0dBQkxLNUlGWWc2cTRKak5pS0s4RE56RVpyODk5L2g4WE1qRzVnd3dYc2Mzak5NdnllNW5QTW0vY25WNWR1dFMvb1k2eWZpa2pvM0pITkNBUFdJa0cxanZuaU41alB1aFcwKytoalhPMWx3U2RDdFV6c3FmdytRejNtYlV4bnJUMVZzTWtpa2dWTTJOZmt2QU9sUU14ZUxOQS9DQ2hJbi9kdXVtK0xacTB1MGd3ZnZvUFZnRWtId0FGSERWenJoTCtMb05JdVJHVGIyUVRSQkhCVURtd2xpQmxscStYWDRJRUFnd2xLMElzaFdaZWs0SUxaRVE2UlMvRXlRZEF0ditXM2FFZzArdFN6aFdpNWVIdGhLRWVZaVpLdzdmYnV4a0M2dVNuaDFvb21EMmZxVy9HQlFCcUNxeXRwTlJaTXpiQ3hXOC90dUFqdU9iL2ZtODUrNWRGRHNWaktGTlNzV0xjQ25qdC9kUnBpMVVBSUFQdnNqQVFuTkdDcFlvaTRjakUwT2RhbjM5SDJiZnpPRyttRVZoS1dhL1RybGZROGU2dzhzUm9kekZSRm4wcUFLT3c3NEsyeXJJZzM5ZXBrRm1BSTIxdFBpUHV3NWNSOEVKRjVoOVZRaXhKRzR2bVFJRTF5SUw3TVJMNExnMDRvS3dxYWxwN1p5b1BXbVZKc3dpZmdBTWd3MnRSZTNUc1YzSmpiZlAvdGFiMWJ6dmhoRTNRNzN6cGwvNFVUMlljd2c4MVdUOUc1ZnpDY094TWUxZzBLVkZaK0lLeUxwTkxTNko3Yk9NZFdDMWVzdG5aNUJDQWtDSExDSnJ3S1BEeWdZazQwYW1NR3JVNmpXcGp0eEpUNkFrVmxLbDhFanoweWsrQ01SWlJHT1BjNHc5bktjNEJtSlJCWndIUTRnNEN4TVpNRXlMUzZUa2hrSEtVOEhBSEdldzU5VlBUelpkS0F1VHN3UkJzN0ZXMGRGbGlYSzhCS0V5aFEyVkV1TXNLTmthZEhaK0JHd2VKdFNyM0dlSFV1WlJBZnlkekVLUzlkNXZQWVVkZ01lbzRPZVdNRmhtNmM5NnhYMjY2YW1SM2NsbnR5WnZIOTZ5ZFNWY2U2VlRtVG5xNkxRcTVVY0VIc0hTRnZlUk4wS2dHVHhoV2VGSk12S3NSTlQzclZlbkpVR1VMak9oaUpXMFNxQWN2MDZpU0FPRFdqV2U3QkJtR2dHajVXUnBuQmZ1Wm1VMzV1QVpkSktpeG1kdWE1RVFPNG9lNGtkVXdOUzhEczhKb2dYbkFhQmlLY2hjVWFFQUdKS1FESG5zYmM4ZHUrcTl6ZEhzYlFzSXZtRVJLMnZuTFowY0RxeHJPWU56K2hMTmtJODZUVHlaNVE2TitFNzU4SHRXT3ZEenUxRVBoZEJVS2tuTmg0cXIwdk54NlZDY011TGdOdEE2ZXFjVWd5cmJRblhRV0Q5cG53TGtHeW03VUpBd1d5QzZycldNR2U5Q0xEVU5ZY1hZdSsrdGFieHV1WTBDUlBvVzUxVUd1RHdkQ2Z3c0djdHorNlh3RU9UZ2xSRlpVN1p3S09lVndJdTQrMXpPMFFXMW5sWU9TSWkrZzUvMnVQQ2lzczR3Mk9GKzNVUEVrREM4VVB0R1g2d1QwRjdxRWpzN2VIWS9vZGFmeEx4SmtBNDZTMzJ6bU53U25Yd2Q4UUxYeGJzbys3U2d3V1h6ZTZCTGxJaGo4dnQ2c3lWV3djVWZTS21mRWVzcWVJTW5Dc2hLMnRYdk9LZ0dNcUlaM0wyMWwyQmluZ1ZwSmk0L1E0WnpjdTU2T05CZFdLQVVLbjM1alFGQ3dHbThpY2Myak9ObngwNHl0aVVvY1NBazNqVDZURkVKQjBneEZ5SUg1MW80Wk9KejdnUDdJdkJzcXVLR0lDRTJ0U1ZvdUFuWE9FNUF6aVNEZ01Wc3UzNU13U2gwSVV3QU1Uem9EQnVsRTBKcTV3SE5iTEhLSGxneHpEaXB1ZzVibjg1Rk9kZHd6VEhjOUZhVHlsdWdTSHEyVTFaZTdUTHZUUDNqbWJxUWloN3pNOUJnM1VIdjBoVWtqOXIwM0hKRGpSRHdJVTAvQTRSSmhOMHQ2UmNRVVl2bmgxQXZKSWpJMW9EK2o2K2lDdmFkU0c2alVQT0xuSEs3V0F0K2hKVVQ1MDBmVFlFOElRTmljYkdPUVZOa2hKMnlrSU5FYmZ0bUc4cVJHU0pOaU1teFFrQUdWRGFadjBFaUQ4MElpSnBOVVkxeDljQWxRd3NXb3k2TEw0M01MWExDSkNNdHNoRW1WWlc5c2JlbjBsY3VicEZMb0ZTN2VSeEFBVkt6MER5WGNjMk5KZnhMUGdibkVoMkFKUHpwZGltclBuME95OVlxM1lqbTdRYW1UMWJ4TlZjSjdVTW1CdHhwV2xiQ2RsRklpN0hadHFSQmd2VndUd0cyWVBzTWVPMWxsRUJUSVFCUFFOSEJaSEZSUkE0ZVYwUnpBcFEwSldKTGdpak0wenhybk1hVFJzRWpOWEdMdUlJbEtSV0JrQkRndXZBOHpKS3kwdTViRFRtd0FEUnhlTDZTVjBldDdScEhWTWhyMFdyRlc2RTgxRk9CK2VkUVRvRm52VzQrRVAvY3FSODdZWjBBenRuQWZzYUhzOVh3UEpGcWFDbS93S0luQ3UxQ09iSlJWTG5YTHBUbFRqdXhpbG9ORnBIN05CcHBIcWxqaDk3aFFoSXNLcVNWcW5FeDF0d0FGVTNkZnErUGFwVktWSlVZOFU4K0xJSXRYNTVzSVV5YzY5UURHUmVvZ1hYZ2Vja3lvaUVEb1Azc3BnQldOenJvb3ZibUpwMUhqYTdCeXlFNk9LS1QvTVZBb2VKZFJoZWFVRjQ1MmYrS3ZndTdBVVdGeXNBemtsM1F0d2krL1AzSit2UVk4K2lyc1VWSGNvaFpRa3RiNnBwa3RuS2JFVDUvdVZjL2hhL1dzQTJZUFJGY2FZbi9YVFFqMXlEQk1lNURGZVZYOTJ3WjZkdDljZy82MFZzbHBYYTRIbHZqQk9ucFIwMGtMQ2psKytJQ1pGRmZLRFc3Ky8wSHpVcy9Xclg5eGN0NXRpZ1B1RmhiOGRJNGY3UnptSDRxazNoU05DQXdCNFlpbUIvaTRwSk1DYnFySTExSzk4cEFSN0lpdGRaRm8rZ1ZqK1ZxVk9tRXowNGxCQnZOcUFBMTJFTjVVQkVYSWQyN3lLVVZ6ekhoTitBaU1vaEQvSHgwQ2xMR1JZU3VGeVpwNWQweWJlTDc5Vmwya1BkUkQ1ZEV1WHY2amkxelp6Q29ud0FJdGNjeUpNMGZNWENwKzRFVEh5ejNYYnFtTG04UXFNcHpyQzBLdXNTOE1uK09WdmhNejlHRHBCbDhyNzhoeDE2MHJVS3hLT3ViNE1CeVlIYktkL2sxbmVvbGIxOG1BUDN0aUJBMk1BRC92Z1dwWGZiVFJzN2JlbWNEOHRMYmN0K3N2VlFYdjhJQUZsUEFJQXBNM2lPOVU5ZDRQRTJoNWl5bm0xUFN0b3BHOENRUWNlKzJ3NUE4V1hhUkpUcUltd0ZpNnl2UUwycDdRRkNXNDk0emltZnpuVUFFMklNaDczVyswTXI3bDBVb3FuTHo1QzlTWHNidHA1L3pVUjlWb0orWDJIVDVYMENodSs0bW1maVJwb0NtY2c0a04vNEFPZXhwYkYzQmpFajBtR2lrRjlucWVyNUlOMWU0WGprTWs3Zkd2a3dTR1FSSncxOUE0bHFzczdnSWJwMEpRUHhUSXhSeTl0RkZuQWhZcnBWY0I4SkxCaENJRzRzTUhnTHhCYWx0Z1NSdituYmcvc0k4U1krTmhNcnI5Z1l4SzVLVzNicWRVd0VuNW81ZzB3b1hKc3hQRHhGYm1OL3JGektnWnRYelpXbzZmU2RsV2cxN1YwSjJCRFlLZTcyMTR1YTdvbU85ODU4WlZXbDV2K2pBWUpkdTMwL3l5OUhoN3B4SFhtL2loODRwTUU1dkZSY2RISndsOUw0RFF1NGZqUUFTS3ZvVXNCTDlZUXBXUkdiaXVGWkIyVlNIWGhzSXI2QlJRQVBRRVBjcG1LSytuNFd0enhGT2dJUEZRQUkyNXNhUVFzREF4WndwOW1DdkFOdUNvaGtRQWpSWmRlSFNJUWh2Y2NaSWdyckV3YTFISi9kSnRJSXRSV3pwOXRzUk1DUnpPRHBuZnlPVGcraWk0akk2N1dEQzl3WUV0eVpCT2V4eGUvTU9QdnlsdTlKV1JEdWZNWnFrZGEvQ3lxbi9JOEVlWENoS00yNmkvaHQ3bXJCZUNvRnNnVXJpenk4RXpjTkYrVjNLbGpFN1VOVzBmMjNSWjNMektFZ1dtb0hLb3VITzc1WjhZbHNJWXBDS3l2TFpUSzRtREVWbjk4aEN2Q1FBQXR3SGtkeTl5b0FBQjltU1VSQlZNVklMTWc3WnZmcDRMRXluVVM0d1l0TW1XWXdCb0xIZmJiK0xVb2xJRXlaYTlFU3orcWJUaW1MeHZSOUx3cXRDemRIYXFoa1dacDFHd1dzYWRDQ3kvRThoT3BWMHd1TmtSWXdEcFF3MzdzZjhwbVZjOW5CWnVsSkRpQjBjZ2Z1eHdvUmtROTBJa2N1SmVWZmxKWnl0MC8zbWF0V3AzZTh4dXBQSTdyQ1ZXenhLTDlVam1yY3dVR0hEcmsyWDRibldjVU8zbnduSXR0ZEhoa2N1TTY1Zm56RFhKb0V2Y3dZaENvR1JqdCtCT3J3TTRrdjZ5UXgwelpndVFXV3A3OHNic0NQT3FYNlFud1IwbXRBNU5EMGJ4SGoxR1d1cnRCeElBMUVJRDdwaTBFRGNUU1djTkhIOFJ3QXc1eEc1QVl1S2gvYUhLS0s3TW93cEV4OWR4cjdJc3E2QmpkMjY1d0tuK1BUTHlQRDBhaHRsYkkxZmgzVlhEK09lemR0SCtRZTk0cDlOVjg2VCtTWjJQT2M3RThYNXFocXFpQnl3ZlVRUERNelYxTHpxUHRoQmoyd0ZhbHl1SWgzbHVxeWNvbGpJRmNpdHdFcFgxajMySEU4bnArRzlzWnRyTS9nZE9sT0doQUYxNVB5RU4wUEZ4SVJ2LzlsNWEzQUM2RzEybGh0K1NVclRCZEV4S0RQTmg3VDN5R3dpQU1EeU9sdEt5L01oUUIwTTVmaFgrUUtWUkpUbUdzaDBNamdVWWxmU3BuVXdBaDFNQm5sKzNJN2h0OU9pTjB1NE96VmdVVUhEalcrN0hWSmFkcHA2Y0k5REZlUjdWSTJ6ZTMxYldPemoyN015dFZMRGlieENEdEpibC9WczNUNERQQ1plajVURlNMRzBvdFFoMEowa2VCZ2F0NVFvckxJNVlyVk1xRzUzcVFBUllvYS9acmJYRGxPdkxob1o3VEZ5cnVaV0FzVndjbGhieU9TcVg0RlpRSVBKa1JSRVgxTDVqdnNqZlFmem8wWUp6SVM0VFBZaU1TNW95R3E5TFlpaFJ0eFVTUFh6N2ZnbzM3ZVBqdHgxcFVIM2VMaHdmaXhWdittMG9zb09TOVAzd0hWeWJWZ3RMOTNkRkpqRDM1bzhDeWxWYzVSdHdKK21NWHF5YkdSbDhZY0s3emNXNDI1cnZQYmxhTHB2V2xCVlpWWHMyMGFtb3NCMFlReTZ0cTE2N2J1ZEhibVRGWE1yZ1R0a01acGlDdnhrOEVoYTZZaUhUREFPU1BPdzhERGQ0d2JRYmhpZE9LWEZJQnM2MEZaSmdBeEVIQ2lIMElnQWNDWXZubHVBVVdzcGd4VHBETEhvc3hwT0FINnhod0J0K1JpRW5NYS91MzVJS0dkb3hFSlpXY0dEQ1pnTDhzWkEvVThVL043WXpkaDdOY1JKUDMxV2VYRFNmaFluNjJjcXpSd2xWanVVOVRYSHdJaVR5K2t1cklUOGFQMkRzNnRVYW1UbWJqdklHZ2FrQ1JpOS9RRUpKVGhBZ0FHTG9zM2N0bUl0NmgrRFlxNjh4Y0RKVVFscnJPSk8wcDlRK1g2WWM5U3hwdUJBdXRrd0ZreFFEa29UUEdOY0VMTHRDaHYrcXBNZ01jS21wYlBPd09Hei9odldWYXBsTktKR0dLT2lCaUl6Q1I2RUpnNGlHaFVuTUNsSGk2MGlUTFVEbUdzUm0yVmtWWXl6SERhUEJLeitMTDdzbDk5dlNlTDVzck1LNWVyc1B0ZHVhZGdsWUJ3bTVHM0NseUF5RmNGQjVFcWpOVGR1MWM1Vis0S0cvQmF4ckhvTzhKZjBvcE5WMllBUXdZSVA4QklDL0hLOEhoeEJFS0FrcE93QVVqWFhTRDR6TGxFR2NGZHdHOEVBUXFsSmM0RG5JT0xMVmFidHdLVWdDS3lPSThpdHVDZ29QaUdBQkNWYVhWOW16LzJzc1RTTEVEQzdUMElDQUpBVkJScENnQXdlQ0I5QUdEb1RKWS93RWtDY0xickdiSm9JL1lkM0FjQUp2Zkx2ZVRsdTBXcWR3Q25LYW1POUhrQ3BCdDNJcEt1b09yMUNiaW95bms1dUpRM3lzaU9nNXEvQXhhZXUreGZraDJBNUs0MHB3MnhLelE1RDhHZHVQS2xhclBwZWhCei9GMERVR1R3OFYvRU9sUnVZSXpkcG1OckNjVEZnTVpSQVNNNEJBY0d5aitBVGR6Zk9ROFNaNEoyZVBldzFkYWxsWFdlaHI0aHNpQXZUV0lMcXJ2K0w5RWxhVGFTT0FPdVJFUTMwU1dXWlZWak9WY0VGckljVnNGRHZXN2U2aEJaZUdNYjdEOVN2R2p3VVA2eGpjYWFWWHk1bCtKeHIwM0t3N25KUThlNlZ3eUtYRjV4ck92aU90MHFNVDV3bWVQaXNnOUU4M1RadDNKMzVuNjZUZ1J1ellvUDlSNFBsTFZNZk1NR0JnREN4Ujc4RGlKdzRsNHdTRlRFcVo4Vm9ValB3STV4V3ZVdkZmd2liajZ4UHZiaVVENEFMd0c0YSt3dUx1TUJGMEloRDFhZVRnM2xyTWdDR3I5QTZtMTZCajRVdWRTYkQvSlIva2YzdTB4Zm5zMkgrdmpTYStJc0ZvQ3cwdFMrMXNHRDlTRXUzZ0FFQVJ4Vi9Pa0FRa1NDWXdudzJRYTkrUTNOWjU0aTNzWmhuQis2V0Mxd1JOMXFLdlBEcWY3c05sQ2g5SGY2dzFNZFVONWRXWHZDUHE1TnRqOEJSUFoxRmR5REt2SU1JT0JBa0s0UEViR0RrSEs4YXVWWm5ZczNJa2E0TEtiazJSL0Q4bGVhTUNJT0ZsbzVXdGpZSnlBQ3JTVGE2THA4TVp5SlRocDlycExQOEpCU0IzeUxLQTFLamJnaEJ1Q2RabWhZbk5KRjJqQmRjdkJ3dllZSWN4Y1pRRlIyemlPSTI4VWJWNkxHYWczcTUxeUlVM0RsUU1EMVdIcWxSaUlRWVpCSVJLcElUeDJDT0E3Nk5SMGhjM1JSTzFIdzI3Nk1MRGVHWkR0UTFYS0hENkJUL1BOZStVZmlrdVpySThZbjUzUG8vdndEUU9RemNQQnpPcGdMNk9LQjdYeGFBOXF5WHdGaWN6Uzc4MGtBUGdpa0EzUHU1T1dtckc5NHJaZDk5N1JpaVRqT0Z3OHdNaUFDd2xDZEFBSkpNY3NQQUFuUnRLOWx6UEMzQ2pwaDZveEI3YVR2aWhTeTYzQkF3RHZkOGdLeHBKd3NGbWwwV1lJby9HTkZCY3BaV3NEMXBkNEFpdDFnYkgwT3JjcW9rejg5V3h5RXA4NER5QkhZY2lkU0hzeVo1RDZCdG13bjVvakRnTGFMRGh1SDQ0TUVQbFhOZnVHT0EvdTU0ektVQnh1WGNYTE1rc3NQNTBST3h5dUhnMDdncW83SWhiY2tjN3VCRzBrV3NSWVBGcVlWU0xZK2syaUh4RVJBVjBIeGh1WklXcjZVYVp0MUdOV2YxVkpKckJHS1NIV2F0YzRBWGxVSGlFVkxScEJ6Z1JOTzVYTkFtckhocmdPUElNc3B1UFNKVHdCUkVsOEdMRTZ4bE92N1plS2RsYStUUkJrVkVWNWhtUVFZektsa1FBQkhRaTFOSEplUEpEWmMyMENFR2w0SUxKSzRxS2t6Y3JlbzhDRXYwYTluWFVJalNFVmVoeFFPUElXZ1hWK1d0bkxjQU0ybmJsZjZYTVJkUHc4dFZ1OWNRVnZjUG5iQkh1V2wzQjVTazk2aWpVRnhHVWhLK2dva0lwbTR0NW5lWGhheGh5L3JTWkFYak5BMjNjZndxNlZsYU9oRFdIbjZRbjZldGxSRUk4OEVoSFF0SlM2Qk1zck15N3EwWW9NelRXY2drWE1QSkZ3SW9ETUE0MDNjQ0RpRE1GbWZMS2JJQXBHZCsyQVFDS0FRUUJLQUxXUTBFbWVvUGdWQXhOTklpUi9oOUxXYmYvVHRjS1JtRUVrNVdMdE5GZ0hjb3JYdXdjbnA2azF6SHNkbnRGMXNxWWNQSlhHSlJaNkRYcVFEbkdmdTY2RDBFM1FpNCtQcklhNXpDOGRna1BRa011UWxtc0pyMnM2bEJqZVU4TXVwTVBtaytJM1lVOHRVM2VvcEVtZTk0a0JrcE1XNGVDbVZLemFnbEg4WElBeUFoWVc1eGFuQ2ZIMG1PZ0tlUTZ3SzBRWHdNQVVIQXdIK0hBaU02eGdnOGlLZXFJR0o1OVdBQ0N0SFJTRHVCSUJrOEFBbkVjQ1RPQXhNR1FWSVdQelorcmdGRWRuaVpzTXlnRVYwVXRyeXIxVDJ3YUVsai9WQnZaMzV1TmFqYlBsZzdCN0VJTld6M3VSSE9aVDNnMEdFUG9qcWZ3VW9uNXJOYzB6c29ZRi9JbXJWYldkdEl2RENHZzZwWEZJUU5JaWVYUWR1c3drYkFBV2E1VmpaS3FKRmRESFd2WW83T0MzZDBxbkZpL3R3Tld5OFJHalFVeG1pQWtUaTNiWll1bDA2bmd3ZzRoeEh0ZzJaYVUvTFcxUkdwRk53RUZra2d2Z0NMZ1FmQ0FCSTFxc2VIdm9RM2ZxSHVJZkRHR3BCeEorcjZCUHR4ZE9EQTBjdEc4OE5yUVljZGJvVWlrZkw4SERqbFFIbEtWZXg2VjFFTjVENUZGaTZOTkdtNitkYlYwWThyc2czT1pPYWxHZjYwL1B5S0tqUDRlMnN0WE1hekVsVTRFanNKZGZUQmh6MkFuRysyVndIc3hxNEJJdXJJSFRMMjY5YnNFNUZQSnpwb1NMRHdtU1UvSVVXYkRXdVI0ZzlLMHRaT3Z3ZG9rd0FoaS9wa3EwR25sblVXYnVCMys0UHJtZXQxZ1E0Yk9ickpPb0V3Y0l2K2k3MzJXclZ0RVRjZFdnQmlyelNZKy9NWmNDZlR4QURVQjJHZkIxS1hCdUk5dG9Rd0dtUCt6cjhtbTdDb1RxMkJOM0s1ZnMzV1FZZk8rVTJwSUdPMnYrejJZbDg2bFJqQisrOXVyWkpMNzI0d1FEQjd6WGRxVTRJUDEwdDhkSVZMeWxSQlJ2NGFJNVRURzJaRTNsTkxlQ0JyZm9ZVUJLaWpJYUhTdjRXSjI2SlFZQVZtSFhHNlpUQnhtTUdFbENzQnFDWURzVFluR21Bd2N1NXpxbE1zM1AxWldHMXk3cnRHQUFYV3pLUWhMaGpiY3orMXJoWjNJQTNzVjdVUjZrVkhLdWlMU0JHN0tETCtleEswOHJsblp6Nm4zM1dac2NHNzl0TzJacWc1aWM3b053NUZnV1BjUzVXUlgzTXVjZFBBWkVyRXZ3eE9iL0tlL2Nzc2dQSFZyUGl3VG9RUElQWU96Q3FaYmt1ZytKN1BZelRnRjhjcWhTY2lGQzZ0ZHFDUENLT3o4YTJDaE5Ld1NnUCtXTE5CQjAvYlVVRm9JQ3dZVUF3Y0FZN0FjaVV0MENud1lwU1ZxSXVCZTh2VHVSK2lUZUJDZnhjSE5LM2N3U1REdE9wb2t3U1Z5U2VjL3pUdStYanhFZmxEQU9yaTZHNlFRaU5qWTZMVGZVNHBMMkttN25haXdROFpnbWd3TG1jMHE2Rmh1dDZwd3B4L2swY0R2djFWN2J5ZndvVWxaTkxSS2FoY1VnaWdZU3gyUk0zWmNpdnRwbEtrb204YzRXRU9FTmlabUdPN1hTSWtCTTZmWXYzSllDQThnRUo4RVkrWVpEUUtFdk5nMDhhRzJJYjR6QjRvZnV3bkZFR0ZLZjhEeXBQeEZnQThqYmRoL25wK29XZVlzaVVxYjg0WUFpSkxBNUVHbUlPZENPczgzQWRDT3M5V044eExjOFovajRERnBIR1A4SEJoZnFsQlpFTUhFNmtRek1IUi8wa0k0c1dKMnFBRWpucEdXNjVrczhuMXVqYkMyVkxkWTFJRmtIUGJLNDYvYzBKU0JLSWlNaVh6Z1Q1VWU2VHM4L3U0dkdLVGVvNnplOU9yTkh5QkJBTkoxRXE0WUFDc1psbkVSdW9zS29kUXZFTHlJQzl3SXpKSjdBamJ4eVlERnVReW4wdzFMMFRiSUNnTUtjR0J6SnNOKzZ3c09uK3ZLcnlkb0tlckNSVmNCTHhQbDNzS2Vic2lZUFJwUzlCSG5NV1BRbWFoQ2s4NjBJQ1pFalZYZlFiZkxSaFFNaFlBSGhCazlyNHBYaStDcFgzb205a1U4cVlna083dXp6UFN0TXJycUhWaXpUUncwemllN1NkMGpkWi9ZdlJpUXo2VlFrT2hJSGh1MjVuOTI1bUNhVzZOS2lpR3FzakRnRDJqRitVTUNpL0JVUWEzSWFCQk5KQVI4SkdaTkFuMEZaQmllNE56aU5tNytuMkxxSG5DQ095NFhvT015SlQ4Qm9MUU5nT0pDL1hRc1JaNVUvb1JpYkVJWWc4QUFjQ0Q5ZVpCRGNpemgwVURxTWUySU84a0E4cFBzTUtObVpUcTUyMS9EdEFpcm8zVDJDV1J4SVg2bkNJZXFySUFSbm9jYWovZGhzelRpczNHS2Mxck5xZGJIVzdxTXVkdTExRjZxRlZSQWhFZmhZMzhwVlZtU3ZtcjVJOVJCNmVHZXB4aDFra09yRnJOc01vcFFOQmozTDJxajNDYUF3bnhUc1FNcWRSU3ZTVkY0MDA0QU9jTTBGK05rUEgwdk1RbU5vemdMejlLYmlQSWJIblpWVWpMRTBCSUd6YWpqVWNpQ2d6Y1I2dzlWZ3JPZE1CQkRxUGQ0NkhiOGUxRDh5VitLOTZXd1p4MHBmTnNBOVpYSlJwZWhLblV2dHd4dVhpUW1ObGhIS2JIU3Mydld5WFJWR1g5WGNVLyt4MkVsUC9jMDdXaTBLcjFoMkFmTUpSZkJyL2llUDhlREwrQ1p6SXplenVsV2duOXo1SEE0YVBWMmt1dUxBNkp2cTV3dUxZYkZsUGpZV09CVjljeTJCeENpNzBpSmhGeGE5b0VGMEQrMlg3WEJidEI2c1BBSUdXQXZYR0wwUVVMTmNLQVFuZUJ5M1hRc1NCc1ZsbktDWUFEK2N3UktweVZXWUFpT29VbkVwZVFVTjFHZ2Npem1Yd1VtK1dEVlRXWWNZenZoSEFRb3F0R05qY0pwSTBIRGdVbVNjWGtRRDdkY2lVWnBCUndMYlZ0Y292byt2eGNDM1VKRkRaU3R3SHBlZjFraDhOQ0RuL3p3R0hSYWg4Nzh4amJ1UVpVSHpIM1pWUXVSRVdlZkFKcndRTnkvTmQ0a0tKMmEvMHFNZnhNQnM3SHMvbGRaUzVpSWQvd1ZSamxoUndPTWpVbTF4RjVycXJkb2k0aFdxR2lSMDRWbnVvQTRTRGhpL2hpdnZsWExRQkVCQTVGS2ZxQUJJclBqbXViL2MzN2dOTHRuTXlOeFBpVElnbVJ1d1FDYjFPYUU4TndCbFd0Z05JRURtcFQ1dW45WlowVVJ4Q3dMS1BOeEIxaG90QmVoY0hLRTUyMG45MG5wN0VlWjZ0b2pyMjVlWXIwUWExRzE4d2p1aHdqTG43SEhuVmErTkVuZ0hKTXdDNUUyVStXZWZXclpleng0bVQ0RzMvY0s1N2VGQzh4amp5OTIzbFprM1BpVHZCd0g0TDYwZ1VPSkdCQXk5R09FUEZUMlh2T0E3OE1vQ0F1TmxvVE9oOVJRMURNSVZZUXpZZ3JzZUFyb1JXWjhJK0pIUWxrMFdZR2JvUzV6aFVseG0rNEJlZ1lTYjJHdXRKNFBaVzNxdlJYSElaNEVCQU9JQmxKaUhxS0VVdlJ6UFgzNW9NaWZNK05DWnlvVUZRTThqamNJV1BOT2l5Sm03WWVBbk9DYm0wTjBWMjRIY0FqeHpuZlR2SU55Nm5jR3Z3aXlZWlc1WmZGR2QrUGlmeXJKVFNnUmZ0eWdqcndKNW1lVWxjQThJMk5TenBSQncwcWg4QkY4OUFVMHRlaGlnZUJ6cVI3Y2hDQmc0R2t2VjNpRXBpMzBWbEFRZ0dOZlFVTUdkZmZuRW1LcmdMRzg2bVFJWEZxSU9NNjFKWVBQbkZ3MkxwMWo1S0dVQ01VeUx4QmVVNVl6L3MyK253bGlseFlSTURBU1lnaGhKd0xkYmFNdVZWK3E4WklNeGFDb3M0NXhuZTR5WjVkZERmSEMvSzdzUXFwUy9wM2JiQjFGT2M2bGpZaUdQR0lWTFZ6MTBpWHNQSmpMMFZXeEQ1Lzdua2UrV1lyUW9ia2t6cUdIS25GWjJyeTdFcjB4amN4NTRpemdvUm1oY2wvaElJVGEzbTdzWWh6ZUJxcHNMYU5lSUFNdkNjQVFRRUdBQXhqUC9CMGkxYmRQcGRMd0FKQ2E1ajBYSUdEeVd1Z3ExVUUwZGlKdTRBQ1JkcFpxU0hQZ1ZuaHpqSWF2NUNISXdFc1NYSXpwYWh5Y3l3OGlDUzNnT0FYdklXYlRkeHZTZ05vMGVKMjJrK1J4RTlxQVpVZXE4VHdXOHE2b3JPZW1DNU00cDRmS2QyR3RkRkQ2TWl6czIxemZEeUpGL2tSQjRvVG44UUJ0VnNUdklaZHZKeXVqdE9aZ0ZScFBITmNuaHYwbXlJclZuZUhpTHlwcm80a0ZEOW5lTUFvZUNVZFEyUmh3ZjRXOUpjYThRWVhBUkVFUkdWb1crWnhvWEVJVDl2d2RKdWlDMnh4QnBXcWF6YmNQNUVjTG4ySks0aUFRZ0FCeHpJeEdYY0FCZ3lYK2Z2d0t4R0t6VXU0T2hNM01Ed05haG9aeWZpUVgzZ2JLYjE1QmhpVmlMVXAvbHlhOCszY2lFRldIemxibUNzNVBHV09Zb2FSanZFUFFEbFhDdHA4MWQveDlVeWRoRHN5dXk0RkJ3eUpYSjFGKyszdUpFNzhzM0VlaG12aWJSWDYxemVGU0NJN09CemNyV0VVQXhLRWxuZXJ2Q0tjaFh4RFNSOEQ0NksrTmtmbUpreGtMMWNMVU12dHV4REtRbU9SR1hHWmtjL1haMEJoRGdHQnhRU2NZUzRFNnpnUUtUaFpWNEFqNEdGVG9RUnlMaXhHdG1EMU5QUzFaL1FTaXVmb1RaUmhJRVhWQXlaVytSK20zYTc2WWJ5cXg5NnBzTDdWVVVjakRKcG9RNnZkRlhJMGhQRTBZTFFtZXhqYzVRUllVQ3A4WFlXT1JyLzhZbXk5QTUwT2xEWlJ2cWVhL0g2U2NabUoyYnV4K2hSd0JidnZoY2xQQ2orYXJkeGdJU2tnWllHSWVKVzdib2pReGlVWWFZWFQyOGc0N2xDelloMzVrN1lnQXcybEFFVTA0Q0c0NGRmNUFQUkJNdTR2SGVGZCtsNlduQU0wK3hJU0l3WkUrSVBiRU9tZjRIS1RFVG5SbGo0QzA1cUtQWFIyNTk1eHVlOVUzd2cxZUpxb2oyOUw4elFxM1k5bExXTFkzdVpMQmtLTTlXUkFVVWJpOWR0d0l4Vm5wNkpjQ2x1N3ppQWk0R3E4eHdtVWtDR2dldVFYUXNZVnh4TFRpMXlBeUkvVmpmeU9ZQjhSU1I2QkNTRjZuMEg3Y2pOQitMbmdXc0JiYllxZExnUVJWMmNSckFZWGtmSEZxVnZEV0tZbkk5QWVWbzVFUFdZdk5WZnpOK2hTQ1ZBUTZDZkNNTFB4bUt4bUF0Unc0M1B3SFdRR0ZNQkpCMFJZQ3RFYnNFN0FINGoybmlVZWh2S3VwTFQ5aEFvS1JPRy8xbHVzTEtMOVlxR0N6bzBEYWpCSE1CNFIxbUdZMVVQTUliS204R29BZ2d0Ni9RbTdpUHkzTkp0c1J1L2gvUlRRV2FFNXJETDZhUzB2YTlQdUM5d0lzOCtacC9WZnhRbmNvOHNWekY4a0RUTlZkZnlPeTRqMG9mYjRpQ3Rpcy9ld1JSZzVpVWdJYUFoMDZxWXFVVXlnRGlKQTBCV3ZtNW5ZZm9JY0NvZVYxbHNpV1ZjUHBOak9zZGh0aDR5QXh6bUx5VGlyTzlZQUVMY2h5bEh1VHdBb0M5enkxaWN6MGluWnNnWVpNc3k1Z1lHNEdhOGIzVEUwZitxRy9HaUR5QUs1WUhSZ0k3MkkxUnRTemMyZU00eFhQUlpSZE5nSnpCSkhOQjJSMUhvYzVvQ3UxcnM0L1VKUFRHb05JRGlRZjc3K2N4OUN5S2ZjeVAzSC9ZRVNyNnJtSTNWbTkxZGdvenM5VU5lYUFaVFc2NFY5b0l3djhoYUdWTFhkV1NETWY2dTZZTVd3QklsRjM2R3VBdFVoR2VjRUduV21TQnNDSzhPRXA0emdRRHlnYzFINkd3TVVPWjdnZHNNc0VvMklQcDI1YW1LeWh5eHNVL0hKRjBQMVVqVk9BZFNucm94MlF3RE1PWWNvRk9TUmNRRGFaVENreU1pcjZhcUtaWUVlS2lLR3FIeFpXTlFmQ01mY0p2VDN2bXlzMjR4S0JjTFRxcFU1Z3Npems3d0I4cHk3eGd6R1lBZXIrZTA3a05PNUR1Y3hJL1JoOEE5d1pqVFNnNWNCUnJOZjdhOE9HU3dIeWxabVN2eExmMlVNSEVna3ZVNG9iRVlWQTdMK3BwbWxrRmNTTkkvaURnWDR0d0lyRmsxT0EySU1Ya1g3bnFleEhINEJkMlRkVEFTUk96Nk9PTk1JTElvYTEraUlkYkVicHdHVmxaRVhlUkEvaSs2STRmN2FBTU1MQituSXk5ek9PNVFybTQ3cG9MYVY4Y3J3RVR5WktybGFsaVJOYm1rNWQraHE4eDVBb2dHY2FvTGUzdTVwNkVEcURSRlp0aWdPak5iMXVTWWkxdWhQM1VYNzBsUmllYkltNnFmdWlCVEJvR2o2Z2RBQWt1eTRxNDRGdFExbHlwZTYwUS9uTWJOMVRYcUlPcW1xanNOOUlyaXpKZHdvc3lGVUVaTkxraHIzQTVab2JwMUtuUWpFdjdnSGhaWWhvSFptRys3K0h2cFJWaUVXUmU1QVRrQmlKTjBOVllqQU1VUS8vVnYwWDdIYmJ6WHRuaEpITm1JUE9FWVZIVGpGRGpPYUxnVkJvU1J5aDFiNVJoVUpoMzRQVlJrZGdOc1ZpR1k2ME1qS3RtclBPUThvbFo3L0lzc3h0YTJwMXh6dVk5QTVMbEk4eGtnbkFDa0FzOFZvZCtCZ01qT2tRVGZJRDY0cjlSSlE5YXFUTDFLSW5NUjlrelhVMlpnaVJKYlRzZitNcm5Ka1F1SjJCNUdBOENYY0VzNTZtbkFtV2lBZzdJdVJtMzJqcnRreEV6Ykp3REJPUTIxZ1I0N2g1ZWZPSmd3TnlWajJvcUpySm1hUUd2VnZRZEM5OTNDdzdRNzdGdHlxMjUzM2FyOThRdXFoK0JnSkorOG9mTVpyNWkxVlUya1lWNVJSQVVjaTRyS0szRW82NDZrNVorK2lhOXhiRG1WVHptUEo3U0h5VzhqaG85SXR3by8vMkxPRS9uRWZXWFhZV2MxK3FNZEsyWkhIczBjS3lJL3laTklmNDFmYUdSb1lIc2M0eEFRbmppSm1kcU1pZFUxSUFxZGl0cGhTQUNiVWlOYTRWaytVelRuVHZXaXVzbzBGajhESHRxaUFuVEh0ZVYyUVZud3pUQWZETTZCSUhYMlFNSlJ5bXJIcHVjd1l6WjAwQ0F1Y2NyTDlMNHFJbS9Yb2F3TUNGVFlmdVpLNXJvOFJpM3l1TnVUbGdId3JDKzZkeXZSVHdlUks1R21jNitpWUlTN0ZUdjBHa3pyL2IxWHJ1NU5VRzBPWnU2K3FmVzhlWjh6ZFQ4QW9YNUwxWWVnRWo3TFk4VW5Ld1BDZXBVcm5UaWFadWJYdVN4ZkovUWlzYVRyVjE5T3NrOHhNU0lVdFkyNGRkaHZBTDRNSE16OWhNaXRWYlBNZHFsNW9na2dnYkN4ZEZPVHVJc1hjVVY5VFRhSVVucm50aDlEWHE2ZkVoZDNYQm03bWMyWlFqZlptWlI2dEczWUNjSjdReC9QWHUwQXBYVm5BbnI5dC8vMGg0dUUvL3h1WG1sQ3lYV044Z1hPN0poSGRTY1RIMWVxWG9MVUxsYjBzYzRoZTB3dHYxelhtZnhTcUU2cDJwWnJCNDZFTjlDcGxXTEx5R01abXkzT29SQ3VhalRPQzg4Qlp0MjVxV3Z5QmllRmNPWjh0cS9xb0xEOUZrN3A0OFU1b1FwK1BjajIrWmU0dnZvVlpReFZlWlgvSG40Y1lYZFZ1YXNuWXFtY3h4ZnF3SnM1Ni8vZC9lbFAvMWxFSHA3ejh6UDBJU0lYM1hQSUtyTzdmV3I0N3ByenI5ZEhKQy94bnB5TE1nL2E2eXBHM0ZyWGtVa0FoNys3Z3BTT0p0YmFRaGxtVkRSdjVjZXFqU3dSUWlYay9ycEVIS0pSNUxPWEVXblV5c3pWMGZQN3FmMUc4KzJsbE01dHdFYS9LaEpMd0lkOHRQeTdpdHNVWGo1QjAvam9nR1NNVXNhZy85ZUZsZjlkakNlMC9CazF2Lzd0Yi8vVkI5Ry81azZnOEJYRGxpZU5NT1U1UnhQNTN1Y0p0L1ZseDJVK3lETnJEZFF6WDh4MVZxS2VjdmowdmVwQ09yZk5pbldEMkJqT0NZNjdnYzNoSXhNaHJ3S1poekFSbklrWDlhL3ZVTlJPdVNJbXJhQkg1ZCtWaVhLdWFsRmROODd2SnBwanV6S2czQUxMNGZ1L0NTUkRSUDdpZHlIQnZQNzQyOS9JYi8vTnZ6NW45RkFCK0hWMzZLZ0xES2lxdnM1VlR1U3ErTXVZbDMyMGR3ZlR3NGR6MXEzckJxOGVCZ3BjNmorRllSblZTRXMrRERMZ1RqVE9WVVdjWlJSRzYwMjBoWCt2YThPRkpQM0IrWnZqVzVIbTlKK3lZLzJsZithcGpYWWd5YUhYTTN2SzQvaCtQV0UrRm10TzdoWlV2Z2NrOWIrSXlELzgvZDk0bkplSXlOLzlqLy93VVowL2RWKzFQcjFxZUxUNzBTcjFRWmtWSURNWGVZMHVPM2p3REIvRUpTSmhtUzdzZ2RsV1pHTEZ3clh3ZVltMm44bDJVQW1sYkdiNy9SQWhpdXZISGlZQ0RpVXRHbkRRYzVZS2huM0pvTFJVdWZWbDhkV3ZibWJuR2pHaFZwM1A3cTZ4WFFNQU44Nm5QR3Y5c09hOXErL3RBTHNIa3BOWTQrRkR0LytQM0E4R2t1cisrcS8rTXIwN0RmN1AvLzRmbTdwOGYvNTg5dDJuZ1pLbnFIZDVWMzJtNjJqemJyNk5iMzdmTm1CcGZlODZwZVROTS80SytHZytZOTlJT2ZPN3MrQnlIUHg1MzgxUlJYeFJNNDZsTmlWZERkVHBieXFhbDhZcUYzTGpPa1dmYm42Yy9UVUJ0VUNDZW5GYzFMUFc5NUMrRjdlK1JqOVgzTWhqUVBsSlFQTDczLzNCRmFwd0RpSi8vTzF2V2lENWp2dnUvcGZ2dXROWXhRSEsxV0dmUnp4M2lmZFh0emdsNXVWaDhyWnVKMWZYVkk2eStVVWVjYVZDY0IycXdabkVRVVpRdWk3Z2dURlY5dzBKY0YzdlVaY3d6MXhJZnRmeS96clY4bVRPNCtUNnR1cVY5VTArRjBEQy9OTmVhdUUrSG5JalErYXRhUE1SZDBJMXVxdmp5ZjMrZDM5SVlneGNvcVUvL3ZZMzhrOS85MTh2ZFNSUDNTZmY5cFRqcWRka2lseFBhcFZUNVdIWmRjOXRZN3IrWUhkTG1kdm5sNGRtckg2b2lKdUFJLysxenBKVExxS2VXNzU5clVNUWdFM0h5bm9lMnFxYnpaK3c2M3N1OFVMbmpkNXdJVWNnOUExNUxPcWRPWkRQYTNrQ0VuNnFNMGF0MzNWZVgzRlZyQUdZZkZsWDByclBnZVN2Lytvdld3Qlp4UnhHeXovK24vOHIvL2kvLzBsRVJQN21iLzhYb2w4VVF6c1cyaHhQSnU0MnlKc1RtMm8rQ1VRMGN2eFZZMG4rSzl0NUdSYWtWQU1yTTkwall6UDBvUGVSd2lRTklyLzVqcTYxOUhSRlNSY0hROXZBeEtxRStRMTZ6cnFCMlBBbStpWmRCKzZQQWVlUU45bmhOanVjNWg0MzErRktUSno2enN1N2IvZVhhWnZ0NWk5Rzk4dmszZE9JY1NsK3MxeWNRN0sreXc0NGV0bjlNdzVva3I4OW9KVDBGeFkyeXJEVytEbnRKSEcvdXd0eTAxaGtZNjdSeG1qakR5cG43Q24ySElvb1h0K3A3S3VWeFo2SEZta3RYVVV1MnVKY0J1cjYrOS85aGZ6N2YvYzcrZE9mL3NzNTd1a2UwRCs3UDdzL3V6KzdwKzcvQVdoWmZRNWpScFViQUFBQUFFbEZUa1N1UW1DQyIvPgogICAgICAgICAgPC9nPgogICAgICAgIDwvZz4KICAgICAgPC9nPgogICAgPC9nPgogIDwvZz4KPC9zdmc+", } # RedisCloud color: #0D6EFD @@ -63,6 +85,23 @@ def getPatternShape(i): DB.PgVector.value: "#4C779A", DB.Redis.value: "#0D6EFD", DB.AWSOpenSearch.value: "#0DCAF0", + DB.OSSOpenSearch.value: "#0DCAF0", DB.TiDB.value: "#0D6EFD", DB.Vespa.value: "#61d790", + DB.Doris.value: "#52CAA3", } + +COLORS_10 = [ + "#a6cee3", + "#1f78b4", + "#b2df8a", + "#33a02c", + "#fb9a99", + "#e31a1c", + "#fdbf6f", + "#ff7f00", + "#cab2d6", + "#6a3d9a", +] + +COLORS_2 = ["#1f78b4", "#d8b365"] diff --git a/vectordb_bench/frontend/pages/concurrent.py b/vectordb_bench/frontend/pages/concurrent.py index 045239494..20b273044 100644 --- a/vectordb_bench/frontend/pages/concurrent.py +++ b/vectordb_bench/frontend/pages/concurrent.py @@ -4,6 +4,7 @@ from vectordb_bench.frontend.components.check_results.nav import ( NavToResults, NavToRunTest, + NavToPages, ) from vectordb_bench.frontend.components.check_results.filters import getshownData from vectordb_bench.frontend.components.concurrent.charts import drawChartsByCase @@ -25,6 +26,9 @@ def main(): # header drawHeaderIcon(st) + # navigate + NavToPages(st) + allResults = benchmark_runner.get_results() def check_conc_data(res: TestResult): @@ -42,7 +46,7 @@ def check_conc_data(res: TestResult): # results selector resultSelectorContainer = st.sidebar.container() - shownData, _, showCaseNames = getshownData(checkedResults, resultSelectorContainer) + shownData, _, showCaseNames = getshownData(resultSelectorContainer, checkedResults) resultSelectorContainer.divider() @@ -56,7 +60,7 @@ def check_conc_data(res: TestResult): getResults(resultesContainer, "vectordb_bench_concurrent") # main - latency_type = st.radio("Latency Type", options=["latency_p99", "latency_avg"]) + latency_type = st.radio("Latency Type", options=["latency_p99", "latency_p95", "latency_avg"]) drawChartsByCase(shownData, showCaseNames, st.container(), latency_type=latency_type) # footer diff --git a/vectordb_bench/frontend/pages/custom.py b/vectordb_bench/frontend/pages/custom.py index 4f6beed91..6f1235fb0 100644 --- a/vectordb_bench/frontend/pages/custom.py +++ b/vectordb_bench/frontend/pages/custom.py @@ -1,15 +1,23 @@ from functools import partial import streamlit as st from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon +from vectordb_bench.frontend.components.check_results.nav import NavToPages from vectordb_bench.frontend.components.custom.displayCustomCase import ( displayCustomCase, ) +from vectordb_bench.frontend.components.custom.displayCustomStreamingCase import ( + displayCustomStreamingCase, +) from vectordb_bench.frontend.components.custom.displaypPrams import displayParams from vectordb_bench.frontend.components.custom.getCustomConfig import ( CustomCaseConfig, + CustomStreamingCaseConfig, generate_custom_case, + generate_custom_streaming_case, get_custom_configs, + get_custom_streaming_configs, save_custom_configs, + save_all_custom_configs, ) from vectordb_bench.frontend.components.custom.initStyle import initStyle from vectordb_bench.frontend.config.styles import FAVICON, PAGE_TITLE @@ -32,7 +40,33 @@ def deleteCase(self, idx: int): self.save() def save(self): - save_custom_configs(self.customCaseItems) + # Save performance configs along with existing streaming configs + streaming_configs = get_custom_streaming_configs() + save_all_custom_configs(self.customCaseItems, streaming_configs) + + +class StreamingCaseManager: + streamingCaseItems: list[CustomStreamingCaseConfig] + + def __init__(self): + self.streamingCaseItems = get_custom_streaming_configs() + + def addCase(self): + new_streaming_case = generate_custom_streaming_case() + new_streaming_case.dataset_config.name = ( + f"{new_streaming_case.dataset_config.name} {len(self.streamingCaseItems)}" + ) + self.streamingCaseItems += [new_streaming_case] + self.save() + + def deleteCase(self, idx: int): + self.streamingCaseItems.pop(idx) + self.save() + + def save(self): + # Save streaming configs along with existing performance configs + performance_configs = get_custom_configs() + save_all_custom_configs(performance_configs, self.streamingCaseItems) def main(): @@ -49,8 +83,16 @@ def main(): # init style initStyle(st) + # navigate + NavToPages(st) + st.title("Custom Dataset") displayParams(st) + + # Performance Test Datasets Section + st.subheader("Performance Test Datasets") + st.markdown("These datasets are used for search performance tests.") + customCaseManager = CustomCaseManager() for idx, customCase in enumerate(customCaseManager.customCaseItems): @@ -74,12 +116,46 @@ def main(): ) st.button( - "\+ New Dataset", + "+ New Dataset", key="add_custom_configs", type="primary", on_click=lambda: customCaseManager.addCase(), ) + st.divider() + + # Streaming Test Datasets Section + st.subheader("Streaming Test Datasets") + st.markdown("These datasets are used for streaming performance tests (insertion + search).") + + streamingCaseManager = StreamingCaseManager() + + for idx, streamingCase in enumerate(streamingCaseManager.streamingCaseItems): + expander = st.expander(streamingCase.dataset_config.name, expanded=True) + key = f"streaming_case_{idx}" + displayCustomStreamingCase(streamingCase, expander, key=key) + + columns = expander.columns(8) + columns[0].button( + "Save", + key=f"{key}_save", + type="secondary", + on_click=lambda: streamingCaseManager.save(), + ) + columns[1].button( + ":red[Delete]", + key=f"{key}_delete", + type="secondary", + on_click=partial(lambda idx: streamingCaseManager.deleteCase(idx), idx=idx), + ) + + st.button( + "+ New Streaming Dataset", + key="add_streaming_config", + type="primary", + on_click=lambda: streamingCaseManager.addCase(), + ) + if __name__ == "__main__": main() diff --git a/vectordb_bench/frontend/pages/int_filter.py b/vectordb_bench/frontend/pages/int_filter.py new file mode 100644 index 000000000..0408fd310 --- /dev/null +++ b/vectordb_bench/frontend/pages/int_filter.py @@ -0,0 +1,56 @@ +import streamlit as st +from vectordb_bench.backend.filter import FilterOp +from vectordb_bench.frontend.components.check_results.footer import footer +from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon +from vectordb_bench.frontend.components.check_results.nav import ( + NavToQuriesPerDollar, + NavToRunTest, + NavToPages, +) +from vectordb_bench.frontend.components.int_filter.charts import drawCharts +from vectordb_bench.frontend.components.check_results.filters import getshownData +from vectordb_bench.frontend.config.styles import FAVICON +from vectordb_bench.interface import benchmark_runner + + +def main(): + # set page config + st.set_page_config( + page_title="Int Filter", + page_icon=FAVICON, + layout="wide", + # initial_sidebar_state="collapsed", + ) + + # header + drawHeaderIcon(st) + + # navigate + NavToPages(st) + + allResults = benchmark_runner.get_results() + + st.title("Vector Database Benchmark (Int Filter)") + + # results selector and filter + resultSelectorContainer = st.sidebar.container() + shownData, failedTasks, showCaseNames = getshownData( + resultSelectorContainer, allResults, filter_type=FilterOp.NumGE + ) + + resultSelectorContainer.divider() + + # nav + navContainer = st.sidebar.container() + NavToRunTest(navContainer) + NavToQuriesPerDollar(navContainer) + + # charts + drawCharts(st, shownData) + + # footer + footer(st.container()) + + +if __name__ == "__main__": + main() diff --git a/vectordb_bench/frontend/pages/label_filter.py b/vectordb_bench/frontend/pages/label_filter.py new file mode 100644 index 000000000..deab8b73a --- /dev/null +++ b/vectordb_bench/frontend/pages/label_filter.py @@ -0,0 +1,56 @@ +import streamlit as st +from vectordb_bench.backend.filter import FilterOp +from vectordb_bench.frontend.components.check_results.footer import footer +from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon +from vectordb_bench.frontend.components.check_results.nav import ( + NavToQuriesPerDollar, + NavToRunTest, + NavToPages, +) +from vectordb_bench.frontend.components.label_filter.charts import drawCharts +from vectordb_bench.frontend.components.check_results.filters import getshownData +from vectordb_bench.frontend.config.styles import FAVICON +from vectordb_bench.interface import benchmark_runner + + +def main(): + # set page config + st.set_page_config( + page_title="Label Filter", + page_icon=FAVICON, + layout="wide", + # initial_sidebar_state="collapsed", + ) + + # header + drawHeaderIcon(st) + + # navigate + NavToPages(st) + + allResults = benchmark_runner.get_results() + + st.title("Vector Database Benchmark (Label Filter)") + + # results selector and filter + resultSelectorContainer = st.sidebar.container() + shownData, failedTasks, showCaseNames = getshownData( + resultSelectorContainer, allResults, filter_type=FilterOp.StrEqual + ) + + resultSelectorContainer.divider() + + # nav + navContainer = st.sidebar.container() + NavToRunTest(navContainer) + NavToQuriesPerDollar(navContainer) + + # charts + drawCharts(st, shownData) + + # footer + footer(st.container()) + + +if __name__ == "__main__": + main() diff --git a/vectordb_bench/frontend/pages/qps_recall.py b/vectordb_bench/frontend/pages/qps_recall.py new file mode 100644 index 000000000..27f9c4691 --- /dev/null +++ b/vectordb_bench/frontend/pages/qps_recall.py @@ -0,0 +1,73 @@ +import streamlit as st +from vectordb_bench.backend.cases import CaseLabel +from vectordb_bench.backend.filter import FilterOp +from vectordb_bench.frontend.components.check_results.footer import footer +from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon +from vectordb_bench.frontend.components.check_results.nav import ( + NavToQuriesPerDollar, + NavToRunTest, + NavToPages, +) +from vectordb_bench.frontend.components.qps_recall.charts import drawCharts +from vectordb_bench.frontend.components.qps_recall.data import getshownData +from vectordb_bench.frontend.components.get_results.saveAsImage import getResults + +from vectordb_bench.frontend.config.styles import FAVICON +from vectordb_bench.interface import benchmark_runner +from vectordb_bench.models import CaseResult + + +def main(): + # set page config + st.set_page_config( + page_title="Label Filter", + page_icon=FAVICON, + layout="wide", + # initial_sidebar_state="collapsed", + ) + + # header + drawHeaderIcon(st) + + # navigate + NavToPages(st) + + allResults = benchmark_runner.get_results() + + st.title("Vector Database Benchmark (Qps & Recall)") + + # results selector and filter + resultSelectorContainer = st.sidebar.container() + + def case_results_filter(case_result: CaseResult) -> bool: + case = case_result.task_config.case_config.case + return case.label == CaseLabel.Performance and case.filters.type == FilterOp.NonFilter + + default_selected_task_labels = ["standard_2025"] + shownData, failedTasks, showCaseNames = getshownData( + resultSelectorContainer, + allResults, + case_results_filter=case_results_filter, + default_selected_task_labels=default_selected_task_labels, + ) + + resultSelectorContainer.divider() + + # nav + navContainer = st.sidebar.container() + NavToRunTest(navContainer) + NavToQuriesPerDollar(navContainer) + + # save or share + resultesContainer = st.sidebar.container() + getResults(resultesContainer, "vectordb_bench") + + # charts + drawCharts(st, shownData, showCaseNames) + + # footer + footer(st.container()) + + +if __name__ == "__main__": + main() diff --git a/vectordb_bench/frontend/pages/quries_per_dollar.py b/vectordb_bench/frontend/pages/quries_per_dollar.py index 2822f2864..078d8490d 100644 --- a/vectordb_bench/frontend/pages/quries_per_dollar.py +++ b/vectordb_bench/frontend/pages/quries_per_dollar.py @@ -9,6 +9,7 @@ ) from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon from vectordb_bench.frontend.components.check_results.nav import ( + NavToPages, NavToResults, NavToRunTest, ) @@ -27,13 +28,16 @@ def main(): # header drawHeaderIcon(st) + # navigate + NavToPages(st) + allResults = benchmark_runner.get_results() st.title("Vector DB Benchmark (QP$)") # results selector resultSelectorContainer = st.sidebar.container() - shownData, _, showCaseNames = getshownData(allResults, resultSelectorContainer) + shownData, _, showCaseNames = getshownData(resultSelectorContainer, allResults) resultSelectorContainer.divider() diff --git a/vectordb_bench/frontend/vdb_benchmark.py b/vectordb_bench/frontend/pages/results.py similarity index 70% rename from vectordb_bench/frontend/vdb_benchmark.py rename to vectordb_bench/frontend/pages/results.py index cf261bf1d..a146f2fdc 100644 --- a/vectordb_bench/frontend/vdb_benchmark.py +++ b/vectordb_bench/frontend/pages/results.py @@ -7,6 +7,7 @@ from vectordb_bench.frontend.components.check_results.nav import ( NavToQuriesPerDollar, NavToRunTest, + NavToPages, ) from vectordb_bench.frontend.components.check_results.charts import drawCharts from vectordb_bench.frontend.components.check_results.filters import getshownData @@ -22,17 +23,22 @@ def main(): # header drawHeaderIcon(st) + # navigate + NavToPages(st) + allResults = benchmark_runner.get_results() st.title("Vector Database Benchmark") st.caption( - "Except for zillizcloud-v2024.1, which was tested in _January 2024_, all other tests were completed before _August 2023_." + "Choose your desired test results to display from the sidebar. " + "For your reference, we've included two standard benchmarks tested by our team. " + "Note that `standard_2025` was tested in 2025; the others in 2023. " + "Unless explicitly labeled as distributed multi-node, test with single-node mode by default." ) - st.caption("All tested milvus are in _standalone_ mode.") - + st.caption("We welcome community contributions for better results, parameter configurations, and optimizations.") # results selector and filter resultSelectorContainer = st.sidebar.container() - shownData, failedTasks, showCaseNames = getshownData(allResults, resultSelectorContainer) + shownData, failedTasks, showCaseNames = getshownData(resultSelectorContainer, allResults) resultSelectorContainer.divider() diff --git a/vectordb_bench/frontend/pages/run_test.py b/vectordb_bench/frontend/pages/run_test.py index 3da8ea2c0..e4472e767 100644 --- a/vectordb_bench/frontend/pages/run_test.py +++ b/vectordb_bench/frontend/pages/run_test.py @@ -7,7 +7,7 @@ from vectordb_bench.frontend.components.run_test.hideSidebar import hideSidebar from vectordb_bench.frontend.components.run_test.initStyle import initStyle from vectordb_bench.frontend.components.run_test.submitTask import submitTask -from vectordb_bench.frontend.components.check_results.nav import NavToResults +from vectordb_bench.frontend.components.check_results.nav import NavToResults, NavToPages from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon from vectordb_bench.frontend.components.check_results.stPageConfig import initRunTestPageConfig @@ -25,8 +25,8 @@ def main(): # hide sidebar hideSidebar(st) - # nav to results - NavToResults(st) + # navigate + NavToPages(st) # header st.title("Run Your Test") diff --git a/vectordb_bench/frontend/pages/streaming.py b/vectordb_bench/frontend/pages/streaming.py new file mode 100644 index 000000000..811c4d497 --- /dev/null +++ b/vectordb_bench/frontend/pages/streaming.py @@ -0,0 +1,156 @@ +import logging +import streamlit as st +from vectordb_bench.backend.cases import CaseLabel +from vectordb_bench.frontend.components.check_results.footer import footer +from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon +from vectordb_bench.frontend.components.check_results.nav import ( + NavToResults, + NavToRunTest, + NavToPages, +) +from vectordb_bench.frontend.components.check_results.filters import getshownData +from vectordb_bench.frontend.components.streaming.charts import drawChartsByCase +from vectordb_bench.frontend.components.get_results.saveAsImage import getResults +from vectordb_bench.frontend.components.streaming.data import DisplayedMetric +from vectordb_bench.frontend.config.styles import FAVICON +from vectordb_bench.interface import benchmark_runner +from vectordb_bench.models import CaseResult, TestResult + +log = logging.getLogger("vectordb_bench") + + +def main(): + # set page config + st.set_page_config( + page_title="VDBB Streaming Perf", + page_icon=FAVICON, + layout="wide", + # initial_sidebar_state="collapsed", + ) + + # header + drawHeaderIcon(st) + + # navigate + NavToPages(st) + + allResults = benchmark_runner.get_results() + + def check_streaming_data(res: TestResult): + case_results = res.results + flag = False + for case_result in case_results: + if case_result.task_config.case_config.case.label == CaseLabel.Streaming: + flag = True + + return flag + + checkedResults = [res for res in allResults if check_streaming_data(res)] + + st.title("VDBBench - Streaming Performance") + + # results selector + resultSelectorContainer = st.sidebar.container() + + def case_results_filter(case_result: CaseResult) -> bool: + return len(case_result.metrics.st_search_stage_list) > 0 + + shownData, _, showCaseNames = getshownData( + resultSelectorContainer, checkedResults, case_results_filter=case_results_filter + ) + + resultSelectorContainer.divider() + + # nav + navContainer = st.sidebar.container() + NavToRunTest(navContainer) + NavToResults(navContainer) + + # save or share + resultesContainer = st.sidebar.container() + getResults(resultesContainer, "vectordb_bench_streaming") + + # # main + st.markdown("Tests search performance with a **stable** and **fixed** insertion rate.") + control_panel = st.columns(3) + compared_with_optimized = control_panel[0].toggle( + "Compare with **optimezed** performance.", + value=True, + help="VectorDB is allowed to do **optimizations** after all insertions done and then test search performance.", + ) + x_use_actual_time = control_panel[0].toggle( + "Use **actual time** as X-axis instead of search stage.", + value=False, + help="Since vdbbench inserts may be faster than vetordb can process them, the time it actually reaches search_stage may have different delays.", + ) + + # Latency type selection + latency_type = control_panel[2].radio( + "Latency Type", + options=["latency_p99", "latency_p95"], + index=0, + help="Choose between P99 (slowest 1%) or P95 (slowest 5%) latency metrics.", + ) + + accuracy_metric = DisplayedMetric.recall + show_ndcg = control_panel[1].toggle( + "Show **NDCG** instead of Recall.", + value=False, + help="A more appropriate indicator to measure ANN search accuracy than Recall.", + ) + need_adjust = control_panel[1].toggle( + "Adjust the NDCG/Recall value based on the search stage.", + value=True, + help="NDCG/Recall is calculated using the ground truth file of the **entire** database, **divided by the search stage** to simulate the actual value.", + ) + if show_ndcg: + if need_adjust: + accuracy_metric = DisplayedMetric.adjusted_ndcg + else: + accuracy_metric = DisplayedMetric.ndcg + else: + if need_adjust: + accuracy_metric = DisplayedMetric.adjusted_recall + + # Determine which latency metric to display + latency_metric = DisplayedMetric.latency_p99 if latency_type == "latency_p99" else DisplayedMetric.latency_p95 + latency_desc = "serial lantency (p99)" if latency_type == "latency_p99" else "serial lantency (p95)" + + line_chart_displayed_y_metrics: list[tuple[DisplayedMetric, str]] = [ + ( + DisplayedMetric.qps, + "max-qps of increasing **concurrency search** tests in each search stage.", + ), + (accuracy_metric, "calculated in each search_stage."), + ( + latency_metric, + f"{latency_desc} of **serial search** tests in each search stage.", + ), + ] + line_chart_displayed_x_metric = DisplayedMetric.search_stage + if x_use_actual_time: + line_chart_displayed_x_metric = DisplayedMetric.search_time + + drawChartsByCase( + st.container(), + shownData, + showCaseNames, + with_last_optimized_data=compared_with_optimized, + line_chart_displayed_x_metric=line_chart_displayed_x_metric, + line_chart_displayed_y_metrics=line_chart_displayed_y_metrics, + ) + + # Concurrent performance detail section + from vectordb_bench.frontend.components.streaming.concurrent_detail import drawConcurrentPerformanceSection + + for case_name in showCaseNames: + case_data_list = [d for d in shownData if d["case_name"] == case_name] + if case_data_list: + drawConcurrentPerformanceSection(st.container(), case_data_list[0], case_name) + + # footer + footer(st.container()) + + +if __name__ == "__main__": + main() diff --git a/vectordb_bench/frontend/pages/tables.py b/vectordb_bench/frontend/pages/tables.py index c088dc930..7240e2375 100644 --- a/vectordb_bench/frontend/pages/tables.py +++ b/vectordb_bench/frontend/pages/tables.py @@ -1,5 +1,6 @@ import streamlit as st from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon +from vectordb_bench.frontend.components.check_results.nav import NavToPages from vectordb_bench.frontend.components.tables.data import getNewResults from vectordb_bench.frontend.config.styles import FAVICON @@ -16,6 +17,9 @@ def main(): # header drawHeaderIcon(st) + # navigate + NavToPages(st) + df = getNewResults() st.dataframe(df, height=800) diff --git a/vectordb_bench/frontend/utils.py b/vectordb_bench/frontend/utils.py index 407dd497d..dead61a6c 100644 --- a/vectordb_bench/frontend/utils.py +++ b/vectordb_bench/frontend/utils.py @@ -1,7 +1,6 @@ import random import string - passwordKeys = ["password", "api_key"] diff --git a/vectordb_bench/frontend/vdbbench.py b/vectordb_bench/frontend/vdbbench.py new file mode 100644 index 000000000..860467734 --- /dev/null +++ b/vectordb_bench/frontend/vdbbench.py @@ -0,0 +1,31 @@ +import streamlit as st +from vectordb_bench.frontend.components.check_results.headerIcon import drawHeaderIcon +from vectordb_bench.frontend.components.custom.initStyle import initStyle +from vectordb_bench.frontend.components.welcome.explainPrams import explainPrams +from vectordb_bench.frontend.components.welcome.welcomePrams import welcomePrams +from vectordb_bench.frontend.config.styles import FAVICON, PAGE_TITLE + + +def main(): + st.set_page_config( + page_title=PAGE_TITLE, + page_icon=FAVICON, + layout="wide", + initial_sidebar_state="collapsed", + ) + + # header + drawHeaderIcon(st) + + # init style + initStyle(st) + + # page + welcomePrams(st) + + # description + explainPrams(st) + + +if __name__ == "__main__": + main() diff --git a/vectordb_bench/interface.py b/vectordb_bench/interface.py index 2e573fdc7..42dc876b0 100644 --- a/vectordb_bench/interface.py +++ b/vectordb_bench/interface.py @@ -12,7 +12,7 @@ import psutil from . import config -from .backend.assembler import Assembler +from .backend.assembler import Assembler, FilterNotSupportedError from .backend.data_source import DatasetSource from .backend.result_collector import ResultCollector from .backend.task_runner import TaskRunner @@ -43,7 +43,11 @@ def __init__(self): self.running_task: TaskRunner | None = None self.latest_error: str | None = None self.drop_old: bool = True - self.dataset_source: DatasetSource = DatasetSource.S3 + # set default data source by ENV + if config.DATASET_SOURCE.upper() == "ALIYUNOSS": + self.dataset_source: DatasetSource = DatasetSource.AliyunOSS + else: + self.dataset_source: DatasetSource = DatasetSource.S3 def set_drop_old(self, drop_old: bool): self.drop_old = drop_old @@ -88,16 +92,21 @@ def run(self, tasks: list[TaskConfig], task_label: str | None = None) -> bool: log.warning(msg) self.latest_error = msg return True + except FilterNotSupportedError as e: + log.warning(e.args[0]) + self.latest_error = e.args[0] + return True return self._run_async(send_conn) - def get_results(self, result_dir: pathlib.Path | None = None) -> list[TestResult]: + @staticmethod + def get_results(result_dir: pathlib.Path | None = None) -> list[TestResult]: """results of all runs, each TestResult represents one run.""" target_dir = result_dir if result_dir else config.RESULTS_LOCAL_DIR return ResultCollector.collect(target_dir) def _try_get_signal(self): - if self.receive_conn and self.receive_conn.poll(): + while self.receive_conn and self.receive_conn.poll(): sig, received = self.receive_conn.recv() log.debug(f"Sigal received to process: {sig}, {received}") if sig == SIGNAL.ERROR: @@ -270,7 +279,7 @@ def kill_proc_tree( p.send_signal(sig) except psutil.NoSuchProcess: pass - gone, alive = psutil.wait_procs(children, timeout=timeout, callback=on_terminate) + _, alive = psutil.wait_procs(children, timeout=timeout, callback=on_terminate) for p in alive: log.warning(f"force killing child process: {p}") diff --git a/vectordb_bench/log_util.py b/vectordb_bench/log_util.py index 6ca6ccabf..69dae293d 100644 --- a/vectordb_bench/log_util.py +++ b/vectordb_bench/log_util.py @@ -3,10 +3,9 @@ from pathlib import Path -def init(log_level: str): +def init(log_level: str, log_file: Path): # Create logs directory if it doesn't exist - log_dir = Path("logs") - log_dir.mkdir(exist_ok=True) + log_file.parent.mkdir(exist_ok=True, parents=True) log_config = { "version": 1, @@ -32,7 +31,7 @@ def init(log_level: str): "file": { "class": "logging.handlers.RotatingFileHandler", "formatter": "default", - "filename": "logs/vectordb_bench.log", + "filename": log_file, "maxBytes": 10485760, # 10MB "backupCount": 5, "encoding": "utf8", diff --git a/vectordb_bench/metric.py b/vectordb_bench/metric.py index e0b6cff0e..3634b2114 100644 --- a/vectordb_bench/metric.py +++ b/vectordb_bench/metric.py @@ -13,21 +13,46 @@ class Metric: # for load cases max_load_count: int = 0 + # for both performace and streaming cases + insert_duration: float = 0.0 + optimize_duration: float = 0.0 + load_duration: float = 0.0 # insert + optimize + # for performance cases - load_duration: float = 0.0 # duration to load all dataset into DB qps: float = 0.0 serial_latency_p99: float = 0.0 + serial_latency_p95: float = 0.0 recall: float = 0.0 ndcg: float = 0.0 conc_num_list: list[int] = field(default_factory=list) conc_qps_list: list[float] = field(default_factory=list) conc_latency_p99_list: list[float] = field(default_factory=list) + conc_latency_p95_list: list[float] = field(default_factory=list) conc_latency_avg_list: list[float] = field(default_factory=list) + # for streaming cases + st_ideal_insert_duration: int = 0 + st_search_stage_list: list[int] = field(default_factory=list) + st_search_time_list: list[float] = field(default_factory=list) + st_max_qps_list_list: list[float] = field(default_factory=list) + st_recall_list: list[float] = field(default_factory=list) + st_ndcg_list: list[float] = field(default_factory=list) + st_serial_latency_p99_list: list[float] = field(default_factory=list) + st_serial_latency_p95_list: list[float] = field(default_factory=list) + st_conc_failed_rate_list: list[float] = field(default_factory=list) + + # for streaming cases - concurrent latency data per stage + st_conc_num_list_list: list[list[int]] = field(default_factory=list) + st_conc_qps_list_list: list[list[float]] = field(default_factory=list) + st_conc_latency_p99_list_list: list[list[float]] = field(default_factory=list) + st_conc_latency_p95_list_list: list[list[float]] = field(default_factory=list) + st_conc_latency_avg_list_list: list[list[float]] = field(default_factory=list) + QURIES_PER_DOLLAR_METRIC = "QP$ (Quries per Dollar)" LOAD_DURATION_METRIC = "load_duration" SERIAL_LATENCY_P99_METRIC = "serial_latency_p99" +SERIAL_LATENCY_P95_METRIC = "serial_latency_p95" MAX_LOAD_COUNT_METRIC = "max_load_count" QPS_METRIC = "qps" RECALL_METRIC = "recall" @@ -35,6 +60,7 @@ class Metric: metric_unit_map = { LOAD_DURATION_METRIC: "s", SERIAL_LATENCY_P99_METRIC: "ms", + SERIAL_LATENCY_P95_METRIC: "ms", MAX_LOAD_COUNT_METRIC: "K", QURIES_PER_DOLLAR_METRIC: "K", } @@ -42,6 +68,7 @@ class Metric: lower_is_better_metrics = [ LOAD_DURATION_METRIC, SERIAL_LATENCY_P99_METRIC, + SERIAL_LATENCY_P95_METRIC, ] metric_order = [ @@ -49,6 +76,7 @@ class Metric: RECALL_METRIC, LOAD_DURATION_METRIC, SERIAL_LATENCY_P99_METRIC, + SERIAL_LATENCY_P95_METRIC, MAX_LOAD_COUNT_METRIC, ] diff --git a/vectordb_bench/models.py b/vectordb_bench/models.py index b28521096..44aff6a79 100644 --- a/vectordb_bench/models.py +++ b/vectordb_bench/models.py @@ -1,17 +1,21 @@ import logging import pathlib from datetime import date, datetime -from enum import Enum, StrEnum, auto +from enum import Enum, StrEnum from typing import Self import ujson +from vectordb_bench.backend.cases import type2case +from vectordb_bench.backend.dataset import DatasetWithSizeMap + from . import config -from .backend.cases import CaseType +from .backend.cases import Case, CaseType from .backend.clients import ( DB, DBCaseConfig, DBConfig, + EmptyDBCaseConfig, ) from .base import BaseModel from .metric import Metric @@ -29,12 +33,18 @@ def __init__(self): super().__init__("Performance case optimize timeout") +class ConcurrencySlotTimeoutError(TimeoutError): + def __init__(self): + super().__init__("Timeout while waiting for a concurrency slot to become available") + + class CaseConfigParamType(Enum): """ Value will be the key of CaseConfig.params and displayed in UI """ IndexType = "IndexType" + index = "index" M = "M" EFConstruction = "efConstruction" ef_construction = "ef_construction" @@ -55,6 +65,7 @@ class CaseConfigParamType(Enum): quantizedFetchLimit = "quantized_fetch_limit" m = "m" nbits = "nbits" + nrq = "nrq" intermediate_graph_degree = "intermediate_graph_degree" graph_degree = "graph_degree" itopk_size = "itopk_size" @@ -65,12 +76,22 @@ class CaseConfigParamType(Enum): build_algo = "build_algo" cache_dataset_on_device = "cache_dataset_on_device" refine_ratio = "refine_ratio" + refine = "refine" + refine_type = "refine_type" + refine_k = "refine_k" + rbq_bits_query = "rbq_bits_query" + sq_type = "sq_type" + with_raw_data = "with_raw_data" + reorder_k = "reorder_k" level = "level" maintenance_work_mem = "maintenance_work_mem" max_parallel_workers = "max_parallel_workers" storage_layout = "storage_layout" num_neighbors = "num_neighbors" max_neighbors = "max_neighbors" + quantized_fetch_limit = "quantized_fetch_limit" + pq_param_num_chunks = "pq_param_num_chunks" + reranking_metric = "reranking_metric" l_value_ib = "l_value_ib" l_value_is = "l_value_is" search_list_size = "search_list_size" @@ -90,10 +111,49 @@ class CaseConfigParamType(Enum): maxNumPrefetchDatasets = "max_num_prefetch_datasets" storage_engine = "storage_engine" max_cache_size = "max_cache_size" - - # mongodb params + num_partitions = "num_partitions" + num_sub_vectors = "num_sub_vectors" + sample_rate = "sample_rate" + index_thread_qty_during_force_merge = "index_thread_qty_during_force_merge" + number_of_indexing_clients = "number_of_indexing_clients" + number_of_shards = "number_of_shards" + number_of_replicas = "number_of_replicas" + index_thread_qty = "index_thread_qty" + engine_name = "engine_name" + metric_type_name = "metric_type_name" mongodb_quantization_type = "quantization" mongodb_num_candidates_ratio = "num_candidates_ratio" + use_partition_key = "use_partition_key" + refresh_interval = "refresh_interval" + use_rescore = "use_rescore" + oversample_ratio = "oversample_ratio" + use_routing = "use_routing" + replication_type = "replication_type" + knn_derived_source_enabled = "knn_derived_source_enabled" + memory_optimized_search = "memory_optimized_search" + on_disk = "on_disk" + compression_level = "compression_level" + oversample_factor = "oversample_factor" + confidence_interval = "confidence_interval" + clip = "clip" + + # OceanBase IVF parameters + sample_per_nlist = "sample_per_nlist" + ivf_nprobes = "ivf_nprobes" + + # CockroachDB parameters + min_partition_size = "min_partition_size" + max_partition_size = "max_partition_size" + build_beam_size = "build_beam_size" + vector_search_beam_size = "vector_search_beam_size" + + dataset_with_size_type = "dataset_with_size_type" + filter_rate = "filter_rate" + insert_rate = "insert_rate" + search_stages = "search_stages" + concurrencies = "concurrencies" + optimize_after_write = "optimize_after_write" + read_dur_after_write = "read_dur_after_write" class CustomizedCase(BaseModel): @@ -103,6 +163,7 @@ class CustomizedCase(BaseModel): class ConcurrencySearchConfig(BaseModel): num_concurrency: list[int] = config.NUM_CONCURRENCY concurrency_duration: int = config.CONCURRENCY_DURATION + concurrency_timeout: int = config.CONCURRENCY_TIMEOUT class CaseConfig(BaseModel): @@ -128,14 +189,22 @@ def k(self, value): def __hash__(self) -> int: return hash(self.json()) + @property + def case(self) -> Case: + return self.case_id.case_cls(self.custom_case) + + @property + def case_name(self) -> str: + return self.case.name + class TaskStage(StrEnum): """Enumerations of various stages of the task""" - DROP_OLD = auto() - LOAD = auto() - SEARCH_SERIAL = auto() - SEARCH_CONCURRENT = auto() + DROP_OLD = "drop_old" + LOAD = "load" + SEARCH_SERIAL = "search_serial" + SEARCH_CONCURRENT = "search_concurrent" def __repr__(self) -> str: return str.__repr__(self.value) @@ -229,6 +298,21 @@ def write_db_file(self, result_dir: pathlib.Path, partial: Self, db: str): b = partial.json(exclude={"db_config": {"password", "api_key"}}) f.write(b) + def get_case_config(case_config: CaseConfig) -> dict[CaseConfig]: + if case_config["case_id"] in {6, 7, 8, 9, 12, 13, 14, 15}: + case_instance = type2case[CaseType(case_config["case_id"])]() + custom_case = case_config["custom_case"] + if custom_case is None: + custom_case = {} + custom_case["filter_rate"] = case_instance.filter_rate + for dataset, size_type in DatasetWithSizeMap.items(): + if case_instance.dataset == size_type: + custom_case["dataset_with_size_type"] = dataset + break + case_config["case_id"] = CaseType.NewIntFilterPerformanceCase + case_config["custom_case"] = custom_case + return case_config + @classmethod def read_file(cls, full_path: pathlib.Path, trans_unit: bool = False) -> Self: if not full_path.exists(): @@ -239,16 +323,23 @@ def read_file(cls, full_path: pathlib.Path, trans_unit: bool = False) -> Self: test_result = ujson.loads(f.read()) if "task_label" not in test_result: test_result["task_label"] = test_result["run_id"] - for case_result in test_result["results"]: task_config = case_result.get("task_config") + case_config = task_config.get("case_config") db = DB(task_config.get("db")) task_config["db_config"] = db.config_cls(**task_config["db_config"]) - task_config["db_case_config"] = db.case_config_cls( - index_type=task_config["db_case_config"].get("index", None), - )(**task_config["db_case_config"]) + # Safely instantiate DBCaseConfig (fallback to EmptyDBCaseConfig on None) + raw_case_cfg = task_config.get("db_case_config") or {} + index_value = raw_case_cfg.get("index", None) + try: + task_config["db_case_config"] = db.case_config_cls(index_type=index_value)(**raw_case_cfg) + except Exception: + log.exception(f"Couldn't get class for index '{index_value}' ({full_path})") + task_config["db_case_config"] = EmptyDBCaseConfig(**raw_case_cfg) + + task_config["case_config"] = cls.get_case_config(case_config=case_config) case_result["task_config"] = task_config if trans_unit: @@ -261,6 +352,16 @@ def read_file(cls, full_path: pathlib.Path, trans_unit: bool = False) -> Self: case_result["metrics"]["serial_latency_p99"] = ( cur_latency * 1000 if cur_latency > 0 else cur_latency ) + + # Handle P95 latency for backward compatibility with existing result files + if "serial_latency_p95" in case_result["metrics"]: + cur_latency_p95 = case_result["metrics"]["serial_latency_p95"] + case_result["metrics"]["serial_latency_p95"] = ( + cur_latency_p95 * 1000 if cur_latency_p95 > 0 else cur_latency_p95 + ) + else: + # Default to 0 for older result files that don't have P95 data + case_result["metrics"]["serial_latency_p95"] = 0.0 return TestResult.validate(test_result) def display(self, dbs: list[DB] | None = None): @@ -270,12 +371,14 @@ def display(self, dbs: list[DB] | None = None): key=lambda x: ( x.task_config.db.name, x.task_config.db_config.db_label, - x.task_config.case_config.case_id.name, + x.task_config.case_config.case_name, ), reverse=True, ) filtered_results = [r for r in sorted_results if not filter_list or r.task_config.db not in filter_list] + if len(filtered_results) == 0: + return def append_return(x: any, y: any): x.append(y) @@ -283,7 +386,7 @@ def append_return(x: any, y: any): max_db = max(map(len, [f.task_config.db.name for f in filtered_results])) max_db_labels = max(map(len, [f.task_config.db_config.db_label for f in filtered_results])) + 3 - max_case = max(map(len, [f.task_config.case_config.case_id.name for f in filtered_results])) + max_case = max(map(len, [f.task_config.case_config.case_name for f in filtered_results])) max_load_dur = max(map(len, [str(f.metrics.load_duration) for f in filtered_results])) + 3 max_qps = max(map(len, [str(f.metrics.qps) for f in filtered_results])) + 3 max_recall = max(map(len, [str(f.metrics.recall) for f in filtered_results])) + 3 @@ -301,6 +404,7 @@ def append_return(x: any, y: any): max_load_dur, max_qps, 15, + 15, max_recall, 14, 5, @@ -308,7 +412,7 @@ def append_return(x: any, y: any): DATA_FORMAT = ( # noqa: N806 f"%-{max_db}s | %-{max_db_labels}s %-{max_case}s %-{len(self.task_label)}s" - f" | %-{max_load_dur}s %-{max_qps}s %-15s %-{max_recall}s %-14s" + f" | %-{max_load_dur}s %-{max_qps}s %-15s %-15s %-{max_recall}s %-14s" f" | %-5s" ) @@ -320,6 +424,7 @@ def append_return(x: any, y: any): "load_dur", "qps", "latency(p99)", + "latency(p95)", "recall", "max_load_count", "label", @@ -337,11 +442,12 @@ def append_return(x: any, y: any): % ( f.task_config.db.name, f.task_config.db_config.db_label, - f.task_config.case_config.case_id.name, + f.task_config.case_config.case_name, self.task_label, f.metrics.load_duration, f.metrics.qps, f.metrics.serial_latency_p99, + f.metrics.serial_latency_p95, f.metrics.recall, f.metrics.max_load_count, f.label.value, diff --git a/vectordb_bench/restful/__init__.py b/vectordb_bench/restful/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/vectordb_bench/restful/app.py b/vectordb_bench/restful/app.py new file mode 100644 index 000000000..ad0336501 --- /dev/null +++ b/vectordb_bench/restful/app.py @@ -0,0 +1,101 @@ +from flask import Flask, jsonify, request + +from vectordb_bench.backend.clients import DB +from vectordb_bench.interface import benchmark_runner +from vectordb_bench.models import ALL_TASK_STAGES, CaseConfig, TaskConfig, TaskStage +from vectordb_bench.restful.format_res import format_results + +app = Flask(__name__) + + +def res_wrapper(code: int = 0, message: str = "", data: any = None): # noqa: RUF013 + return jsonify({"code": code, "message": message, "data": data}), 200 + + +def success_res(data: any = None, message: str = "success"): # noqa: RUF013 + return res_wrapper(code=0, message=message, data=data) + + +def failed_res(data: any = None, message: str = "failed"): # noqa: RUF013 + return res_wrapper(code=1, message=message, data=data) + + +@app.route("/get_res", methods=["GET"]) +def get_res(): + """task label -> res""" + task_label = request.args.get("task_label", "standard") + all_results = benchmark_runner.get_results() + res = format_results(all_results, task_label=task_label) + + return success_res(res) + + +@app.route("/get_status", methods=["GET"]) +def get_status(): + "running 5/18, not running" + is_running = benchmark_runner.has_running() + tasks_count = benchmark_runner.get_tasks_count() + if is_running: + tasks_count = benchmark_runner.get_tasks_count() + cur_task_idx = benchmark_runner.get_current_task_id() + return success_res( + data={ + "is_running": is_running, + "tasks_count": tasks_count, + "cur_task_idx": cur_task_idx, + } + ) + return success_res(data={"is_running": is_running}) + + +@app.route("/stop", methods=["GET"]) +def stop(): + benchmark_runner.stop_running() + return success_res(message="stopped") + + +@app.route("/run", methods=["post"]) +def run(): + if benchmark_runner.has_running(): + return failed_res(message="There are already running tasks.") + data = request.get_json() + task_label = data.get("task_label", "test") + use_aliyun = data.get("use_aliyun", False) + task_configs: list[TaskConfig] = [] + try: + tasks = data.get("tasks", []) + if len(tasks) == 0: + return failed_res(message="empty tasks") + for task in tasks: + db = DB(task["db"]) + db_config = db.config_cls(**task["db_config"]) + case_config = CaseConfig(**task["case_config"]) + print(case_config) # noqa: T201 + db_case_config = db.case_config_cls(index_type=task["db_case_config"].get("index", None))( + **task["db_case_config"] + ) + stages = [TaskStage(stage) for stage in task.get("stages", ALL_TASK_STAGES)] + print(stages) # noqa: T201 + task_config = TaskConfig( + db=db, + db_config=db_config, + case_config=case_config, + db_case_config=db_case_config, + stages=stages, + ) + task_configs.append(task_config) + except Exception as e: + return failed_res(message=f"invalid tasks: {e}") + + benchmark_runner.set_download_address(use_aliyun) + benchmark_runner.run(task_configs, task_label) + + return success_res(message="start") + + +def main(): + app.run(host="0.0.0.0", port=5000, debug=False) # noqa: S104 + + +if __name__ == "__main__": + main() diff --git a/vectordb_bench/restful/format_res.py b/vectordb_bench/restful/format_res.py new file mode 100644 index 000000000..2e289ec3b --- /dev/null +++ b/vectordb_bench/restful/format_res.py @@ -0,0 +1,76 @@ +from dataclasses import asdict + +from pydantic import BaseModel + +from vectordb_bench.backend.cases import CaseLabel +from vectordb_bench.models import TestResult + + +class FormatResult(BaseModel): + # db_config + task_label: str = "" + timestamp: int = 0 + db: str = "" + db_label: str = "" # perf-x86 + version: str = "" + note: str = "" + + # params + params: dict = {} + + # case_config + case_name: str = "" + dataset: str = "" + dim: int = 0 + filter_type: str = "" # FilterType(Enum).value + filter_rate: float = 0 + k: int = 100 + + # metrics + max_load_count: int = 0 + load_duration: int = 0 + qps: float = 0 + serial_latency_p99: float = 0 + recall: float = 0 + ndcg: float = 0 + conc_num_list: list[int] = [] + conc_qps_list: list[float] = [] + conc_latency_p99_list: list[float] = [] + conc_latency_avg_list: list[float] = [] + + +def format_results(test_results: list[TestResult], task_label: str) -> list[dict]: + results = [] + for test_result in test_results: + if test_result.task_label == task_label: + for case_result in test_result.results: + task_config = case_result.task_config + case_config = task_config.case_config + case = case_config.case + if case.label == CaseLabel.Load: + continue + dataset = case.dataset.data + filter_ = case.filters + metrics = asdict(case_result.metrics) + for k, v in metrics.items(): + if isinstance(v, list) and len(v) > 0: + metrics[k] = [round(d, 6) if isinstance(d, float) else d for d in v] + results.append( + FormatResult( + task_label=test_result.task_label, + timestamp=int(test_result.timestamp), + db=task_config.db.value, + db_label=task_config.db_config.db_label, + version=task_config.db_config.version, + note=task_config.db_config.note, + params=task_config.db_case_config.dict(), + case_name=case.name, + dataset=dataset.full_name, + dim=dataset.dim, + filter_type=filter_.type.name, + filter_rate=filter_.filter_rate, + k=task_config.case_config.k, + **metrics, + ).dict() + ) + return results diff --git a/vectordb_bench/results/ElasticCloud/result_20250318_standard_elasticcloud.json b/vectordb_bench/results/ElasticCloud/result_20250318_standard_elasticcloud.json new file mode 100644 index 000000000..083b27e2d --- /dev/null +++ b/vectordb_bench/results/ElasticCloud/result_20250318_standard_elasticcloud.json @@ -0,0 +1,5890 @@ +{ + "run_id": "d60af9965dec47739a5924d55bd3ca8d", + "task_label": "standard_2025", + "results": [ + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4462.3855, + "optimize_duration": 0.2241, + "load_duration": 4462.6096, + "qps": 597.3641, + "serial_latency_p99": 0.0121, + "recall": 0.9221, + "ndcg": 0.926, + "conc_num_list": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 395.269, + 512.5172, + 577.6407, + 590.9046, + 596.1805, + 597.3641, + 597.3275 + ], + "conc_latency_p99_list": [ + 0.02428084506420418, + 0.040611707251810014, + 0.068308472202625, + 0.09077592421497684, + 0.13307022775406954, + 0.19506115851225447, + 0.2525385084026492 + ], + "conc_latency_avg_list": [ + 0.01262955671240114, + 0.01946360412380657, + 0.03441716100827359, + 0.050292129541154446, + 0.06609827694557764, + 0.09819079190909193, + 0.12972003787247258 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": false, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 100 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4462.3855, + "optimize_duration": 0.2241, + "load_duration": 4462.6096, + "qps": 551.1757, + "serial_latency_p99": 0.0107, + "recall": 0.9327, + "ndcg": 0.9357, + "conc_num_list": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 357.1187, + 470.551, + 532.14, + 541.514, + 548.109, + 548.8482, + 551.1757 + ], + "conc_latency_p99_list": [ + 0.026570463064126647, + 0.042799862385436434, + 0.0729518641019239, + 0.10374750539020205, + 0.13363405620853877, + 0.22282758549437925, + 0.2940616585547104 + ], + "conc_latency_avg_list": [ + 0.013980312773958717, + 0.02120342718605561, + 0.03736902676982878, + 0.05488599078711187, + 0.07197649905903823, + 0.1068562171161369, + 0.14065833669884753 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": false, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 120 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4462.3855, + "optimize_duration": 0.2241, + "load_duration": 4462.6096, + "qps": 492.9765, + "serial_latency_p99": 0.0134, + "recall": 0.9443, + "ndcg": 0.9464, + "conc_num_list": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 338.453, + 430.3471, + 477.9321, + 491.2854, + 488.7062, + 490.916, + 492.9765 + ], + "conc_latency_p99_list": [ + 0.029508496299968053, + 0.049514623872528314, + 0.07994905100058532, + 0.11117909936583599, + 0.15021729007246903, + 0.22374880456714866, + 0.29201414171140644 + ], + "conc_latency_avg_list": [ + 0.014750132206160912, + 0.023181328265314896, + 0.04163701848682529, + 0.06046374529986848, + 0.08071813144222399, + 0.11958033338938644, + 0.15741490795172944 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": false, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 150 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4462.3855, + "optimize_duration": 0.2241, + "load_duration": 4462.6096, + "qps": 431.0155, + "serial_latency_p99": 0.0152, + "recall": 0.954, + "ndcg": 0.9554, + "conc_num_list": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 302.2866, + 384.9532, + 420.1644, + 427.2244, + 425.4381, + 429.2175, + 431.0155 + ], + "conc_latency_p99_list": [ + 0.0338288504688535, + 0.05324577716120983, + 0.09183125322684636, + 0.1189696359599474, + 0.18243243085336866, + 0.2524550164569517, + 0.33026724230294346 + ], + "conc_latency_avg_list": [ + 0.016513007043925075, + 0.02591919342966166, + 0.04731025857372484, + 0.06952097599413332, + 0.09280017317350533, + 0.13685943087178828, + 0.1803111859785834 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": false, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 200 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4462.3855, + "optimize_duration": 0.2241, + "load_duration": 4462.6096, + "qps": 377.3634, + "serial_latency_p99": 0.0162, + "recall": 0.9615, + "ndcg": 0.9623, + "conc_num_list": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 286.9011, + 338.4546, + 372.4043, + 374.5822, + 376.4039, + 374.0685, + 377.3634 + ], + "conc_latency_p99_list": [ + 0.03640121627395273, + 0.05944062268477835, + 0.09974031736142927, + 0.1534240699876684, + 0.18773965532658624, + 0.3118939118360868, + 0.37713164429995233 + ], + "conc_latency_avg_list": [ + 0.017398873269744687, + 0.02948108780951262, + 0.053426811048435024, + 0.07936868500502132, + 0.10483299783856122, + 0.15706712722044575, + 0.20577504630973428 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": false, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 250 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4462.3855, + "optimize_duration": 0.2241, + "load_duration": 4462.6096, + "qps": 337.5819, + "serial_latency_p99": 0.015, + "recall": 0.9666, + "ndcg": 0.9671, + "conc_num_list": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 264.7308, + 314.5167, + 333.0128, + 334.1814, + 337.5819, + 309.7245, + 337.35 + ], + "conc_latency_p99_list": [ + 0.041220994928153225, + 0.06530364287202248, + 0.10899166988325305, + 0.16976174306386396, + 0.20064824797940672, + 0.3487667758911266, + 0.41615317933290497 + ], + "conc_latency_avg_list": [ + 0.01884961538471635, + 0.03172069117770432, + 0.05973395985988017, + 0.08898184886548736, + 0.1169871897767767, + 0.1898733944512457, + 0.23047925780921352 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": false, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 300 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4462.3855, + "optimize_duration": 0.2241, + "load_duration": 4462.6096, + "qps": 279.7257, + "serial_latency_p99": 0.0181, + "recall": 0.9729, + "ndcg": 0.9732, + "conc_num_list": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 232.2991, + 265.5341, + 277.1642, + 278.572, + 277.375, + 276.6423, + 279.7257 + ], + "conc_latency_p99_list": [ + 0.047881220409180934, + 0.07799559644976398, + 0.13447143511555623, + 0.18796787327068165, + 0.24402899272768927, + 0.37727205709583356, + 0.5082543775314116 + ], + "conc_latency_avg_list": [ + 0.021488945142678767, + 0.0375805999000639, + 0.07173718960972499, + 0.10671031748955834, + 0.14241200100107454, + 0.21247307196733553, + 0.2782147101519429 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": false, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 400 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3220.2828, + "optimize_duration": 1200.4104, + "load_duration": 4420.6932, + "qps": 3033.786, + "serial_latency_p99": 0.0087, + "recall": 0.9934, + "ndcg": 0.9946, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 171.3173, + 988.9131, + 1849.131, + 2731.703, + 3033.786, + 3012.0359, + 3012.0566, + 2961.3975 + ], + "conc_latency_p99_list": [ + 0.00986613344026404, + 0.009091488920530551, + 0.010154825200297635, + 0.014488676959917934, + 0.01911291930031447, + 0.025788147120183562, + 0.03412763896067189, + 0.04556362671923119 + ], + "conc_latency_avg_list": [ + 0.005830613218117401, + 0.005046585332449434, + 0.005393133037928759, + 0.007271472929665846, + 0.009784678825768013, + 0.013052646548874476, + 0.01933719507254384, + 0.02576283494911338 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-routing-64shard", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 64, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": true, + "metric_type": "COSINE", + "efConstruction": 128, + "M": 20, + "num_candidates": 100 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.001 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3220.2828, + "optimize_duration": 1200.4104, + "load_duration": 4420.6932, + "qps": 3019.2416, + "serial_latency_p99": 0.0095, + "recall": 0.9765, + "ndcg": 0.9801, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 211.9455, + 896.2262, + 1738.3795, + 2479.4668, + 2873.766, + 3019.2416, + 2962.7701, + 2946.3363 + ], + "conc_latency_p99_list": [ + 0.00753889083032845, + 0.008928428989274826, + 0.011828452839872628, + 0.01626177466001536, + 0.021502937200493764, + 0.027220318640102055, + 0.03823795130052531, + 0.04880324912010102 + ], + "conc_latency_avg_list": [ + 0.004712842634654995, + 0.005568882235627175, + 0.00573637476181858, + 0.008010954455716361, + 0.010327488345820037, + 0.01303257293467593, + 0.01966842981308053, + 0.025918906719302838 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-routing-64shard", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 64, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": true, + "metric_type": "COSINE", + "efConstruction": 128, + "M": 20, + "num_candidates": 100 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.002 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3220.2828, + "optimize_duration": 1200.4104, + "load_duration": 4420.6932, + "qps": 2890.9523, + "serial_latency_p99": 0.0094, + "recall": 0.9625, + "ndcg": 0.9676, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 195.5042, + 844.8123, + 1611.9428, + 2349.462, + 2700.064, + 2857.4914, + 2890.9523, + 2873.7518 + ], + "conc_latency_p99_list": [ + 0.007314486349605448, + 0.010615806440982848, + 0.011671199639749823, + 0.016326509860155054, + 0.022739153360307678, + 0.02945896186000017, + 0.04726895104049631, + 0.056495660799555425 + ], + "conc_latency_avg_list": [ + 0.005109193440372416, + 0.005907495424502432, + 0.0061870027121100814, + 0.008453858351398382, + 0.01098470285137785, + 0.013785079167210192, + 0.020165582354075742, + 0.026528787757583095 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-routing-64shard", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 64, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": true, + "metric_type": "COSINE", + "efConstruction": 128, + "M": 20, + "num_candidates": 100 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.005 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3220.2828, + "optimize_duration": 1200.4104, + "load_duration": 4420.6932, + "qps": 2789.7212, + "serial_latency_p99": 0.0082, + "recall": 0.9538, + "ndcg": 0.9599, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 187.7147, + 790.0392, + 1568.2311, + 2160.227, + 2517.4255, + 2711.2084, + 2789.1106, + 2789.7212 + ], + "conc_latency_p99_list": [ + 0.010081769549469752, + 0.0109710518003703, + 0.012124352669907239, + 0.01771552109958064, + 0.024714085860414323, + 0.03186119547061019, + 0.048207919230080734, + 0.0643082677908205 + ], + "conc_latency_avg_list": [ + 0.005321289296137064, + 0.0063164580307041515, + 0.0063585442041394526, + 0.009198757858614844, + 0.011790848298209502, + 0.014516948020300353, + 0.020905207977552253, + 0.02743217300181711 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-routing-64shard", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 64, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": true, + "metric_type": "COSINE", + "efConstruction": 128, + "M": 20, + "num_candidates": 100 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.01 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3220.2828, + "optimize_duration": 1200.4104, + "load_duration": 4420.6932, + "qps": 2457.2628, + "serial_latency_p99": 0.009, + "recall": 0.9378, + "ndcg": 0.9461, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 176.2278, + 833.7751, + 1436.7779, + 1933.9114, + 2306.8957, + 2397.0843, + 2450.2949, + 2457.2628 + ], + "conc_latency_p99_list": [ + 0.009253813101167912, + 0.010676891319999413, + 0.01286152699894953, + 0.02038855383014378, + 0.027638759799992838, + 0.03862647846068284, + 0.059232252310594084, + 0.07537427417042651 + ], + "conc_latency_avg_list": [ + 0.005668332196364645, + 0.005985699734939198, + 0.0069413793897639825, + 0.010279926368116274, + 0.012856083654536108, + 0.016422975422817794, + 0.023800908507308042, + 0.03117156902103535 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-routing-64shard", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 64, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": true, + "metric_type": "COSINE", + "efConstruction": 128, + "M": 20, + "num_candidates": 100 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.02 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3220.2828, + "optimize_duration": 1200.4104, + "load_duration": 4420.6932, + "qps": 2209.4973, + "serial_latency_p99": 0.0137, + "recall": 0.9228, + "ndcg": 0.9323, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 166.097, + 767.0308, + 1355.9803, + 1858.1247, + 2069.839, + 2159.0699, + 2198.4431, + 2209.4973 + ], + "conc_latency_p99_list": [ + 0.00994646317969455, + 0.011774108719619108, + 0.013473388360289395, + 0.020621689120907827, + 0.0317548710792471, + 0.041773796350480515, + 0.059154976981153594, + 0.07612881149940229 + ], + "conc_latency_avg_list": [ + 0.006014063604761952, + 0.0065066427797729216, + 0.007354484436362972, + 0.010693596719007341, + 0.01435514220911703, + 0.018246548108566008, + 0.026522891900358805, + 0.03464110841638243 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-routing-64shard", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 64, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": true, + "metric_type": "COSINE", + "efConstruction": 128, + "M": 20, + "num_candidates": 100 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.05 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3220.2828, + "optimize_duration": 1200.4104, + "load_duration": 4420.6932, + "qps": 1960.388, + "serial_latency_p99": 0.011, + "recall": 0.9076, + "ndcg": 0.9183, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 154.5374, + 680.8459, + 1240.8903, + 1712.335, + 1875.7902, + 1789.3469, + 1951.5912, + 1960.388 + ], + "conc_latency_p99_list": [ + 0.010991374070417802, + 0.01303631832006431, + 0.014634679119899413, + 0.024568987360617026, + 0.03446696100836562, + 0.050123761999566335, + 0.0632476110205789, + 0.07854344711013231 + ], + "conc_latency_avg_list": [ + 0.00646397726874715, + 0.007330470952054709, + 0.008035348803205463, + 0.01160235952016014, + 0.01582212666513276, + 0.022027896125587452, + 0.02991914343378913, + 0.0391320737039498 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-routing-64shard", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 64, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": true, + "metric_type": "COSINE", + "efConstruction": 128, + "M": 20, + "num_candidates": 100 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.1 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3220.2828, + "optimize_duration": 1200.4104, + "load_duration": 4420.6932, + "qps": 1725.092, + "serial_latency_p99": 0.0117, + "recall": 0.8969, + "ndcg": 0.9087, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 122.0479, + 665.2295, + 1138.0947, + 1571.6197, + 1660.1707, + 1692.2344, + 1725.092, + 1723.3345 + ], + "conc_latency_p99_list": [ + 0.014851733879404526, + 0.01298943935076754, + 0.016143094779908974, + 0.026223052008772355, + 0.03797812369930397, + 0.04643950719064378, + 0.06557580025884817, + 0.0805934695012184 + ], + "conc_latency_avg_list": [ + 0.0081853636652019, + 0.007503171781177628, + 0.0087615113050103, + 0.012645740355108013, + 0.01786915490195627, + 0.023279210251350654, + 0.03386742286518899, + 0.04450301105616658 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-routing-64shard", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 64, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": true, + "metric_type": "COSINE", + "efConstruction": 128, + "M": 20, + "num_candidates": 100 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.2 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3220.2828, + "optimize_duration": 1200.4104, + "load_duration": 4420.6932, + "qps": 1307.419, + "serial_latency_p99": 0.0123, + "recall": 0.8925, + "ndcg": 0.9057, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 114.6057, + 563.8925, + 943.7315, + 1229.6602, + 1269.0178, + 1285.5229, + 1301.9796, + 1307.419 + ], + "conc_latency_p99_list": [ + 0.01444264779966032, + 0.015197520039782817, + 0.01938553451960618, + 0.032749490199785486, + 0.045934247429449925, + 0.06017261420063734, + 0.08376838080881499, + 0.11117431335980656 + ], + "conc_latency_avg_list": [ + 0.008717005708497339, + 0.008851620666425078, + 0.010567571521281948, + 0.01616415017790857, + 0.02339584929798066, + 0.030654568339697305, + 0.044892512198437276, + 0.05891358510479617 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-routing-64shard", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 64, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": true, + "metric_type": "COSINE", + "efConstruction": 128, + "M": 20, + "num_candidates": 100 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.5 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 39596.6657, + "optimize_duration": 24063.7134, + "load_duration": 63660.3791, + "qps": 1520.4145, + "serial_latency_p99": 0.0125, + "recall": 0.9028, + "ndcg": 0.9125, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 24.221, + 579.3695, + 1096.7277, + 1394.6228, + 1470.6245, + 1493.4561, + 1517.1403, + 1520.4145 + ], + "conc_latency_p99_list": [ + 0.06766113003759533, + 0.04018300544179503, + 0.016403487479255994, + 0.029981104246689868, + 0.042006730000139214, + 0.05324200500181176, + 0.07257076007546856, + 0.09801365754741706 + ], + "conc_latency_avg_list": [ + 0.041253548960002676, + 0.008615638096160802, + 0.009095006995122178, + 0.014248329475609056, + 0.02017964017911973, + 0.026367178042071962, + 0.03846423123262499, + 0.050548138766329874 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 4, + "num_force_merge_threads": 4, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 150 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 39596.6657, + "optimize_duration": 24063.7134, + "load_duration": 63660.3791, + "qps": 1273.3452, + "serial_latency_p99": 0.0123, + "recall": 0.9242, + "ndcg": 0.9314, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 21.9399, + 467.6594, + 947.4937, + 1195.4049, + 1238.1862, + 1254.5401, + 1270.8528, + 1273.3452 + ], + "conc_latency_p99_list": [ + 0.07273197013535536, + 0.05036091234127525, + 0.019338091995450668, + 0.03592271460511248, + 0.04962454480701152, + 0.06060341739794241, + 0.08387701554311225, + 0.10805174599983713 + ], + "conc_latency_avg_list": [ + 0.045541895148732095, + 0.010673792463065985, + 0.010526112467391208, + 0.01662998542759522, + 0.023989798058590632, + 0.031371624616794096, + 0.04601654642139651, + 0.06047165020405831 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 4, + "num_force_merge_threads": 4, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 200 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 39596.6657, + "optimize_duration": 24063.7134, + "load_duration": 63660.3791, + "qps": 1011.7943, + "serial_latency_p99": 0.0152, + "recall": 0.945, + "ndcg": 0.95, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 26.6098, + 431.2744, + 776.8254, + 929.1711, + 951.0091, + 928.9986, + 967.024, + 1011.7943 + ], + "conc_latency_p99_list": [ + 0.05648490337829571, + 0.0386239069236035, + 0.023575824353028907, + 0.04496162842129706, + 0.060359924563672396, + 0.08103636549552896, + 0.11248240799977804, + 0.1423560941571486 + ], + "conc_latency_avg_list": [ + 0.03755005148797251, + 0.011572865380706697, + 0.012838429590534642, + 0.02140012998001872, + 0.031221435913642806, + 0.042421910820666466, + 0.06056575095810523, + 0.0763648814238968 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 4, + "num_force_merge_threads": 4, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 300 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 39596.6657, + "optimize_duration": 24063.7134, + "load_duration": 63660.3791, + "qps": 824.5097, + "serial_latency_p99": 0.0155, + "recall": 0.9558, + "ndcg": 0.9593, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 81.1041, + 399.9531, + 679.9957, + 805.1294, + 814.4951, + 811.9792, + 822.8943, + 824.5097 + ], + "conc_latency_p99_list": [ + 0.016786643958184813, + 0.019362325391412014, + 0.02615840801983722, + 0.04972246614532196, + 0.06908724288106895, + 0.09227640260171031, + 0.12893694854894414, + 0.17315666322989273 + ], + "conc_latency_avg_list": [ + 0.012318815084033716, + 0.01248130858127074, + 0.014670442694464094, + 0.024685397580044423, + 0.03646099999006706, + 0.04855725435301298, + 0.07124351446032214, + 0.0936678083820624 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 4, + "num_force_merge_threads": 4, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 400 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 39596.6657, + "optimize_duration": 24063.7134, + "load_duration": 63660.3791, + "qps": 350.0132, + "serial_latency_p99": 0.0297, + "recall": 1.0, + "ndcg": 1.0, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 30.189, + 202.0183, + 329.1136, + 348.3959, + 350.0132, + 324.9324, + 317.9269, + 322.8882 + ], + "conc_latency_p99_list": [ + 0.14354584991495356, + 0.03533772260561818, + 0.05166047208404052, + 0.10170137415159816, + 0.14068774665880487, + 0.2119204730365892, + 0.3369732468592699, + 0.4408465734421043 + ], + "conc_latency_avg_list": [ + 0.03309652915874472, + 0.024709168786110305, + 0.03030780355606144, + 0.05706132287546322, + 0.08490060004975826, + 0.12148673139386354, + 0.18495983740738778, + 0.24065524627838059 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "metric_type": "COSINE", + "efConstruction": 128, + "M": 20, + "num_candidates": 100 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.001 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 39596.6657, + "optimize_duration": 24063.7134, + "load_duration": 63660.3791, + "qps": 179.5204, + "serial_latency_p99": 0.0514, + "recall": 1.0, + "ndcg": 1.0, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 21.7554, + 106.0393, + 155.691, + 174.45, + 163.971, + 173.1731, + 179.1318, + 179.5204 + ], + "conc_latency_p99_list": [ + 0.054327325366612056, + 0.06324659004691055, + 0.11160609391925388, + 0.20149715809209792, + 0.3140436876972681, + 0.4114896462386239, + 0.5784214028120914, + 0.7762917730069603 + ], + "conc_latency_avg_list": [ + 0.04592851437159226, + 0.0470701281315362, + 0.06408596296098705, + 0.11399034227531618, + 0.18136467783575563, + 0.22803818083394028, + 0.3288339756413207, + 0.43407606787507386 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "metric_type": "COSINE", + "efConstruction": 128, + "M": 20, + "num_candidates": 100 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.002 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 39596.6657, + "optimize_duration": 24063.7134, + "load_duration": 63660.3791, + "qps": 72.99, + "serial_latency_p99": 0.1114, + "recall": 1.0, + "ndcg": 1.0, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 9.7426, + 46.8834, + 72.3067, + 72.8674, + 72.99, + 72.6925, + 72.8358, + 72.3283 + ], + "conc_latency_p99_list": [ + 0.11130997327854855, + 0.1259609358898888, + 0.2224418902012985, + 0.43908577279071326, + 0.578639255479793, + 0.8689724769996248, + 1.3653563368118196, + 2.0325072374472812 + ], + "conc_latency_avg_list": [ + 0.10256189537841912, + 0.1064656898095914, + 0.1377911506870457, + 0.272800471373339, + 0.40706825332169033, + 0.5434245949670153, + 0.808838379367314, + 1.0816969225682052 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "metric_type": "COSINE", + "efConstruction": 128, + "M": 20, + "num_candidates": 100 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.005 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 39596.6657, + "optimize_duration": 24063.7134, + "load_duration": 63660.3791, + "qps": 42.9877, + "serial_latency_p99": 0.2019, + "recall": 0.9912, + "ndcg": 0.9929, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 5.727, + 27.8637, + 42.5288, + 41.9223, + 42.9877, + 42.1593, + 42.4545, + 42.3419 + ], + "conc_latency_p99_list": [ + 0.20021002789610065, + 0.22682934462005505, + 0.3925052056678396, + 0.8056472404801751, + 0.9897030728071694, + 1.4796203714807044, + 2.3374601107032507, + 3.306067794993578 + ], + "conc_latency_avg_list": [ + 0.1744833367613251, + 0.17901221965706401, + 0.23384167156214541, + 0.4733009735020448, + 0.6905679366842797, + 0.9330381558967351, + 1.3896590621471547, + 1.849662769520519 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "metric_type": "COSINE", + "efConstruction": 128, + "M": 20, + "num_candidates": 100 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.01 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 39596.6657, + "optimize_duration": 24063.7134, + "load_duration": 63660.3791, + "qps": 96.4987, + "serial_latency_p99": 0.1131, + "recall": 0.9296, + "ndcg": 0.9411, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 12.9513, + 60.216, + 96.4987, + 95.1649, + 92.1386, + 94.3561, + 95.8224, + 95.2701 + ], + "conc_latency_p99_list": [ + 0.10948647427780091, + 0.12042870999779551, + 0.18358716510309012, + 0.36187229419941946, + 0.5856245036846651, + 0.7280805009612231, + 1.0667982940314693, + 1.4735178799458661 + ], + "conc_latency_avg_list": [ + 0.07715283264290058, + 0.08286472822213026, + 0.10323851633939674, + 0.20893523034262207, + 0.32252879094005354, + 0.41906054915255675, + 0.6158218542638204, + 0.8201011425161081 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "metric_type": "COSINE", + "efConstruction": 128, + "M": 20, + "num_candidates": 100 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.02 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 39596.6657, + "optimize_duration": 24063.7134, + "load_duration": 63660.3791, + "qps": 189.3789, + "serial_latency_p99": 0.0588, + "recall": 0.9149, + "ndcg": 0.928, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 23.6783, + 114.139, + 182.9405, + 187.8722, + 189.3789, + 187.7961, + 186.6971, + 187.7485 + ], + "conc_latency_p99_list": [ + 0.059495461299957225, + 0.06393014971690715, + 0.09572662997758018, + 0.1895439447218084, + 0.255334898908768, + 0.35292769323656104, + 0.553561170498142, + 0.7738216283213114 + ], + "conc_latency_avg_list": [ + 0.04219837012496805, + 0.043731544458172296, + 0.054534455034600735, + 0.10581119944443726, + 0.156956135013416, + 0.21021060112421777, + 0.315328764000099, + 0.41469871270199665 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "metric_type": "COSINE", + "efConstruction": 128, + "M": 20, + "num_candidates": 100 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.05 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 39596.6657, + "optimize_duration": 24063.7134, + "load_duration": 63660.3791, + "qps": 246.7071, + "serial_latency_p99": 0.0451, + "recall": 0.9018, + "ndcg": 0.916, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 30.1938, + 144.3978, + 236.5727, + 245.8587, + 242.124, + 246.7071, + 244.58, + 244.9365 + ], + "conc_latency_p99_list": [ + 0.04237762329081305, + 0.048242829417577036, + 0.07318876069213723, + 0.14206332232395655, + 0.20044887658208604, + 0.2696536789105448, + 0.42686230165723826, + 0.5910197727425839 + ], + "conc_latency_avg_list": [ + 0.03309223717948473, + 0.03455971971190328, + 0.04217053192847151, + 0.08087146556707489, + 0.12268943649874192, + 0.1600527504710669, + 0.24059453467157757, + 0.31774949439749 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "metric_type": "COSINE", + "efConstruction": 128, + "M": 20, + "num_candidates": 100 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.1 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 39596.6657, + "optimize_duration": 24063.7134, + "load_duration": 63660.3791, + "qps": 229.0379, + "serial_latency_p99": 0.043, + "recall": 0.8908, + "ndcg": 0.9063, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 26.675, + 131.8044, + 220.7888, + 227.9404, + 228.4212, + 227.452, + 229.0379, + 228.7601 + ], + "conc_latency_p99_list": [ + 0.04286034200049471, + 0.05564767431715154, + 0.07601444866158999, + 0.1459674067923334, + 0.1985925487536586, + 0.2894491201985509, + 0.4477402601003996, + 0.6190519509909791 + ], + "conc_latency_avg_list": [ + 0.03745701761558604, + 0.037853602510613815, + 0.045172565537501964, + 0.08725491285592099, + 0.13003450933279023, + 0.17345787048645864, + 0.2567663866901463, + 0.3402430916891983 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "metric_type": "COSINE", + "efConstruction": 128, + "M": 20, + "num_candidates": 100 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.2 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 39596.6657, + "optimize_duration": 24063.7134, + "load_duration": 63660.3791, + "qps": 125.6164, + "serial_latency_p99": 0.0698, + "recall": 0.8746, + "ndcg": 0.891, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 15.3422, + 76.2538, + 122.6611, + 125.0317, + 125.4911, + 120.5753, + 125.6164, + 125.1983 + ], + "conc_latency_p99_list": [ + 0.07054545239952857, + 0.07998410858854184, + 0.13297973903187085, + 0.26306906722413265, + 0.37342901344614776, + 0.5632732941280119, + 0.8050925397532404, + 1.152696885180194 + ], + "conc_latency_avg_list": [ + 0.0651288382010345, + 0.065486741059813, + 0.08131069783742807, + 0.15910322454786585, + 0.2370319710536986, + 0.3280755620784844, + 0.4687025319161917, + 0.624200094859833 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "metric_type": "COSINE", + "efConstruction": 128, + "M": 20, + "num_candidates": 100 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.5 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 40991.3527, + "optimize_duration": 0.6919, + "load_duration": 40992.0446, + "qps": 376.3752, + "serial_latency_p99": 0.0145, + "recall": 0.9039, + "ndcg": 0.9043, + "conc_num_list": [ + 5, + 10, + 20, + 30, + 40 + ], + "conc_qps_list": [ + 284.7149, + 343.6169, + 339.2284, + 373.073, + 376.3752 + ], + "conc_latency_p99_list": [ + 0.038023115200921845, + 0.061496057020704036, + 0.12280098611256109, + 0.15119025315914775, + 0.18649004211867568 + ], + "conc_latency_avg_list": [ + 0.017535867670884744, + 0.029040141628230817, + 0.058604699480998744, + 0.07965158962214107, + 0.1048074060237018 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": false, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 100 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10, + 20, + 30, + 40 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 40991.3527, + "optimize_duration": 0.6919, + "load_duration": 40992.0446, + "qps": 341.2325, + "serial_latency_p99": 0.0134, + "recall": 0.9136, + "ndcg": 0.9131, + "conc_num_list": [ + 5, + 10, + 20, + 30, + 40 + ], + "conc_qps_list": [ + 260.3704, + 313.343, + 335.3463, + 324.026, + 341.2325 + ], + "conc_latency_p99_list": [ + 0.04311569791840149, + 0.06562681823968888, + 0.11681612865460919, + 0.1848230326386692, + 0.20782419364404622 + ], + "conc_latency_avg_list": [ + 0.019173563722295974, + 0.03183562085990805, + 0.05933402927199527, + 0.09171436688962788, + 0.11567525811923458 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": false, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 120 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10, + 20, + 30, + 40 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 40991.3527, + "optimize_duration": 0.6919, + "load_duration": 40992.0446, + "qps": 300.7678, + "serial_latency_p99": 0.0128, + "recall": 0.922, + "ndcg": 0.9212, + "conc_num_list": [ + 5, + 10, + 20, + 30, + 40 + ], + "conc_qps_list": [ + 237.2867, + 280.1085, + 296.3958, + 300.7678, + 300.721 + ], + "conc_latency_p99_list": [ + 0.047234738606493916, + 0.07721925374789862, + 0.13282600400387273, + 0.17545931982953344, + 0.27934584984643124 + ], + "conc_latency_avg_list": [ + 0.021042162094585457, + 0.03561130033846358, + 0.06709285259658587, + 0.09887049252019542, + 0.1312858844206962 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": false, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 150 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10, + 20, + 30, + 40 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 40991.3527, + "optimize_duration": 0.6919, + "load_duration": 40992.0446, + "qps": 257.0398, + "serial_latency_p99": 0.0154, + "recall": 0.9303, + "ndcg": 0.9291, + "conc_num_list": [ + 5, + 10, + 20, + 30, + 40 + ], + "conc_qps_list": [ + 210.7914, + 241.8564, + 256.4376, + 255.6649, + 257.0398 + ], + "conc_latency_p99_list": [ + 0.05430803515686421, + 0.09554423218738524, + 0.14379516940141904, + 0.22543662490206773, + 0.2677164784834895 + ], + "conc_latency_avg_list": [ + 0.023683769543997573, + 0.04125546566194685, + 0.07753743665646405, + 0.11630100177019857, + 0.1536159756672878 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": false, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 200 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10, + 20, + 30, + 40 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 40991.3527, + "optimize_duration": 0.6919, + "load_duration": 40992.0446, + "qps": 228.7734, + "serial_latency_p99": 0.0162, + "recall": 0.9374, + "ndcg": 0.9359, + "conc_num_list": [ + 5, + 10, + 20, + 30, + 40 + ], + "conc_qps_list": [ + 191.2988, + 216.2727, + 228.7734, + 225.7422, + 227.4209 + ], + "conc_latency_p99_list": [ + 0.061723858116893, + 0.10097386688430558, + 0.15939864667074288, + 0.25189011551876317, + 0.31115387232406594 + ], + "conc_latency_avg_list": [ + 0.026094311957258916, + 0.04613517383636675, + 0.08694830267011693, + 0.13161782983642376, + 0.1737659302521534 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": false, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 250 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10, + 20, + 30, + 40 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 40991.3527, + "optimize_duration": 0.6919, + "load_duration": 40992.0446, + "qps": 204.3654, + "serial_latency_p99": 0.0182, + "recall": 0.9424, + "ndcg": 0.9409, + "conc_num_list": [ + 5, + 10, + 20, + 30, + 40 + ], + "conc_qps_list": [ + 174.8742, + 195.2021, + 201.5426, + 201.9721, + 204.3654 + ], + "conc_latency_p99_list": [ + 0.06966985324397679, + 0.10897553668095504, + 0.18663148074236244, + 0.2712056294218928, + 0.3379151822178391 + ], + "conc_latency_avg_list": [ + 0.02854508534763364, + 0.051106142788872794, + 0.09866751262359505, + 0.14723437007271403, + 0.19326494307025388 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": false, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 300 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10, + 20, + 30, + 40 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 40991.3527, + "optimize_duration": 0.6919, + "load_duration": 40992.0446, + "qps": 167.5075, + "serial_latency_p99": 0.018, + "recall": 0.9501, + "ndcg": 0.9486, + "conc_num_list": [ + 5, + 10, + 20, + 30, + 40 + ], + "conc_qps_list": [ + 153.5101, + 164.3028, + 166.4256, + 163.8313, + 167.5075 + ], + "conc_latency_p99_list": [ + 0.0804899533648857, + 0.13407693340559512, + 0.21588489400528488, + 0.3063497315940913, + 0.4205354855087352 + ], + "conc_latency_avg_list": [ + 0.03252350957454059, + 0.06069995024000615, + 0.11951330028537865, + 0.18153271715626226, + 0.23586995845371916 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": false, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 400 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10, + 20, + 30, + 40 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 40991.3527, + "optimize_duration": 0.6919, + "load_duration": 40992.0446, + "qps": 146.339, + "serial_latency_p99": 0.0209, + "recall": 0.9557, + "ndcg": 0.9541, + "conc_num_list": [ + 5, + 10, + 20, + 30, + 40 + ], + "conc_qps_list": [ + 135.1134, + 143.8297, + 146.339, + 144.9193, + 145.9755 + ], + "conc_latency_p99_list": [ + 0.09162835260329304, + 0.14955960479128394, + 0.23480458039557578, + 0.33011506061244306, + 0.4504245700011961 + ], + "conc_latency_avg_list": [ + 0.03693265747256734, + 0.06934756789631864, + 0.13590242936463795, + 0.20529936556457673, + 0.270686836796947 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": false, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 500 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10, + 20, + 30, + 40 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 40991.3527, + "optimize_duration": 0.6919, + "load_duration": 40992.0446, + "qps": 129.0705, + "serial_latency_p99": 0.0243, + "recall": 0.96, + "ndcg": 0.9582, + "conc_num_list": [ + 5, + 10, + 20, + 30, + 40 + ], + "conc_qps_list": [ + 121.2135, + 127.3458, + 127.6954, + 127.4566, + 129.0705 + ], + "conc_latency_p99_list": [ + 0.10567279188137035, + 0.17365819440252378, + 0.28041247279630604, + 0.38204556540411444, + 0.4839373561236425 + ], + "conc_latency_avg_list": [ + 0.04116497685931799, + 0.07825621391509528, + 0.15570164347808868, + 0.23321040680301122, + 0.30594657744881415 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "use_force_merge": false, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 600 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10, + 20, + 30, + 40 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4530.1908, + "optimize_duration": 1676.9701, + "load_duration": 6207.160, + "qps": 2095.7067, + "serial_latency_p99": 0.0124, + "recall": 0.8961, + "ndcg": 0.9092, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 131.3991, + 774.8783, + 1317.9307, + 1797.7081, + 1926.6955, + 1990.0554, + 2095.7067, + 2090.8084 + ], + "conc_latency_p99_list": [ + 0.015690910800549327, + 0.011446378519867725, + 0.013741492258886873, + 0.022739400640421084, + 0.033914373079824124, + 0.04397136338036944, + 0.05835181880047455, + 0.07804022703876416 + ], + "conc_latency_avg_list": [ + 0.007601659338827488, + 0.006441365978870764, + 0.007567220661367456, + 0.011042008665171953, + 0.015401763195187974, + 0.019776933597657306, + 0.027812056277810044, + 0.03656947128988507 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 4, + "num_force_merge_threads": 4, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 100 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4530.1908, + "optimize_duration": 1676.9701, + "load_duration": 6207.160, + "qps": 1925.3019, + "serial_latency_p99": 0.0113, + "recall": 0.9141, + "ndcg": 0.9245, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 158.3642, + 732.6737, + 1239.4122, + 1685.6764, + 1835.8759, + 1877.2875, + 1925.3019, + 1920.011 + ], + "conc_latency_p99_list": [ + 0.01005217809979513, + 0.011581572799623246, + 0.013986513500349287, + 0.023246563820775917, + 0.03567562469997948, + 0.0441909884493543, + 0.061103092700177455, + 0.08129918775874083 + ], + "conc_latency_avg_list": [ + 0.006307905002310119, + 0.006811968801108101, + 0.00804568539214758, + 0.011779868775216024, + 0.016166472543425802, + 0.020969326137863648, + 0.030296366749477357, + 0.03994612813430818 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 4, + "num_force_merge_threads": 4, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 120 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4530.1908, + "optimize_duration": 1676.9701, + "load_duration": 6207.160, + "qps": 1707.8841, + "serial_latency_p99": 0.01, + "recall": 0.9314, + "ndcg": 0.9395, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 127.3704, + 674.8269, + 1152.545, + 1546.3691, + 1642.2792, + 1676.5245, + 1707.8841, + 1705.0103 + ], + "conc_latency_p99_list": [ + 0.010540789518781814, + 0.012187652799402728, + 0.015178287000344428, + 0.025788482340394693, + 0.03828581649981066, + 0.04652964145940132, + 0.06480870366072838, + 0.0838760693897712 + ], + "conc_latency_avg_list": [ + 0.007843436171781779, + 0.007396548627419337, + 0.008652023494678583, + 0.01285044286194289, + 0.018089506248153496, + 0.02346478465381785, + 0.034210515247513125, + 0.04507128129201286 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 4, + "num_force_merge_threads": 4, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 150 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4530.1908, + "optimize_duration": 1676.9701, + "load_duration": 6207.160, + "qps": 1442.0638, + "serial_latency_p99": 0.0101, + "recall": 0.9482, + "ndcg": 0.9539, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 125.7684, + 580.2593, + 997.5756, + 1345.8706, + 1404.3805, + 1419.9672, + 1437.6303, + 1442.0638 + ], + "conc_latency_p99_list": [ + 0.011577769249470293, + 0.013634080159881708, + 0.017814238680221018, + 0.030242646999795397, + 0.04235087486078556, + 0.05374349544936194, + 0.0768482715600839, + 0.09357972234020034 + ], + "conc_latency_avg_list": [ + 0.007943463779131396, + 0.008601688507340145, + 0.00999767148572112, + 0.014761524109006004, + 0.021138602808305183, + 0.027740650142166475, + 0.04063036255752577, + 0.05334783920530767 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 4, + "num_force_merge_threads": 4, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 200 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4530.1908, + "optimize_duration": 1676.9701, + "load_duration": 6207.160, + "qps": 1115.106, + "serial_latency_p99": 0.0131, + "recall": 0.9662, + "ndcg": 0.9697, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 108.1492, + 507.5024, + 812.0506, + 1009.3652, + 1086.3536, + 1097.7277, + 1101.8633, + 1115.106 + ], + "conc_latency_p99_list": [ + 0.012068989860163123, + 0.015582748999804608, + 0.02093161099946882, + 0.04279054374956104, + 0.05375390150067684, + 0.06864247275007074, + 0.10126859904001331, + 0.12600917334939357 + ], + "conc_latency_avg_list": [ + 0.009237458367411733, + 0.009834906161432578, + 0.012284435064239468, + 0.019690290592407986, + 0.027342236453092243, + 0.035909869885359914, + 0.053056361605666805, + 0.06917595765467553 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 4, + "num_force_merge_threads": 4, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 300 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4530.1908, + "optimize_duration": 1676.9701, + "load_duration": 6207.160, + "qps": 910.4322, + "serial_latency_p99": 0.0142, + "recall": 0.9748, + "ndcg": 0.9773, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 92.2302, + 432.7789, + 729.5481, + 882.0311, + 894.8931, + 888.6049, + 910.4322, + 876.0556 + ], + "conc_latency_p99_list": [ + 0.015010915440580034, + 0.01750927904877245, + 0.0241673191609334, + 0.04669977979909164, + 0.06316791614968684, + 0.08509454171988183, + 0.11886005545020445, + 0.16528269460095546 + ], + "conc_latency_avg_list": [ + 0.010832640377777925, + 0.011533445673372963, + 0.013673801420844147, + 0.022528944359111267, + 0.03317176811982702, + 0.044350525969073594, + 0.06433605796228238, + 0.08834964573655582 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 4, + "num_force_merge_threads": 4, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 400 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4530.1908, + "optimize_duration": 1676.9701, + "load_duration": 6207.160, + "qps": 2175.2694, + "serial_latency_p99": 0.0098, + "recall": 1.0, + "ndcg": 1.0, + "conc_num_list": [ + 10, + 20, + 40 + ], + "conc_qps_list": [ + 1238.4298, + 1885.244, + 2175.2694 + ], + "conc_latency_p99_list": [ + 0.016785892559419136, + 0.020560235699304017, + 0.04369625502964481 + ], + "conc_latency_avg_list": [ + 0.008053811192478286, + 0.010537932286414798, + 0.01811391299750922 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "metric_type": "COSINE", + "efConstruction": 128, + "M": 20, + "num_candidates": 100 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.001 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10, + 20, + 40 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4530.1908, + "optimize_duration": 1676.9701, + "load_duration": 6207.160, + "qps": 1430.0244, + "serial_latency_p99": 0.0126, + "recall": 1.0, + "ndcg": 1.0, + "conc_num_list": [ + 10, + 20, + 40 + ], + "conc_qps_list": [ + 1011.9198, + 1384.3633, + 1430.0244 + ], + "conc_latency_p99_list": [ + 0.01842798869944089, + 0.029953522890955344, + 0.053862998309923485 + ], + "conc_latency_avg_list": [ + 0.00985820518877243, + 0.014352770589781459, + 0.027523691707991408 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "metric_type": "COSINE", + "efConstruction": 128, + "M": 20, + "num_candidates": 100 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.002 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10, + 20, + 40 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4530.1908, + "optimize_duration": 1676.9701, + "load_duration": 6207.160, + "qps": 692.5751, + "serial_latency_p99": 0.0187, + "recall": 1.0, + "ndcg": 1.0, + "conc_num_list": [ + 10, + 20, + 40 + ], + "conc_qps_list": [ + 596.3292, + 684.5562, + 692.5751 + ], + "conc_latency_p99_list": [ + 0.029882075879540934, + 0.05715290480060497, + 0.10387547983918921 + ], + "conc_latency_avg_list": [ + 0.016728474928452138, + 0.029027120243393233, + 0.05694341078923998 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "metric_type": "COSINE", + "efConstruction": 128, + "M": 20, + "num_candidates": 100 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.005 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10, + 20, + 40 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4530.1908, + "optimize_duration": 1676.9701, + "load_duration": 6207.160, + "qps": 364.3516, + "serial_latency_p99": 0.0264, + "recall": 1.0, + "ndcg": 1.0, + "conc_num_list": [ + 10, + 20, + 40 + ], + "conc_qps_list": [ + 328.9804, + 358.9538, + 364.3516 + ], + "conc_latency_p99_list": [ + 0.05235626121851967, + 0.09535526725047022, + 0.1882201465800245 + ], + "conc_latency_avg_list": [ + 0.030319677868984995, + 0.055379793703873824, + 0.10829728571829991 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "metric_type": "COSINE", + "efConstruction": 128, + "M": 20, + "num_candidates": 100 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.01 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10, + 20, + 40 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4530.1908, + "optimize_duration": 1676.9701, + "load_duration": 6207.160, + "qps": 190.3777, + "serial_latency_p99": 0.0479, + "recall": 1.0, + "ndcg": 1.0, + "conc_num_list": [ + 10, + 20, + 40 + ], + "conc_qps_list": [ + 184.1532, + 189.4159, + 190.3777 + ], + "conc_latency_p99_list": [ + 0.08904289054002218, + 0.17340957510110436, + 0.3400676797504821 + ], + "conc_latency_avg_list": [ + 0.054153508416118205, + 0.10495251031708744, + 0.2076730151012285 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "metric_type": "COSINE", + "efConstruction": 128, + "M": 20, + "num_candidates": 100 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.02 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10, + 20, + 40 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4530.1908, + "optimize_duration": 1676.9701, + "load_duration": 6207.160, + "qps": 249.3519, + "serial_latency_p99": 0.0447, + "recall": 0.9446, + "ndcg": 0.954, + "conc_num_list": [ + 10, + 20, + 40 + ], + "conc_qps_list": [ + 242.0696, + 249.3519, + 246.0743 + ], + "conc_latency_p99_list": [ + 0.0733962430487736, + 0.141016656800366, + 0.2714228479002486 + ], + "conc_latency_avg_list": [ + 0.04119079519215202, + 0.07966342473896967, + 0.1605093370005195 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "metric_type": "COSINE", + "efConstruction": 128, + "M": 20, + "num_candidates": 100 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.05 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10, + 20, + 40 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4530.1908, + "optimize_duration": 1676.9701, + "load_duration": 6207.160, + "qps": 437.8735, + "serial_latency_p99": 0.0271, + "recall": 0.9364, + "ndcg": 0.9468, + "conc_num_list": [ + 10, + 20, + 40 + ], + "conc_qps_list": [ + 389.1903, + 415.7841, + 437.8735 + ], + "conc_latency_p99_list": [ + 0.04511592213035328, + 0.08920857917089962, + 0.15596479624928178 + ], + "conc_latency_avg_list": [ + 0.02562190307889434, + 0.04779609405297262, + 0.09009148640155527 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "metric_type": "COSINE", + "efConstruction": 128, + "M": 20, + "num_candidates": 100 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.1 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10, + 20, + 40 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4530.1908, + "optimize_duration": 1676.9701, + "load_duration": 6207.160, + "qps": 669.9441, + "serial_latency_p99": 0.0191, + "recall": 0.9227, + "ndcg": 0.9351, + "conc_num_list": [ + 10, + 20, + 40 + ], + "conc_qps_list": [ + 588.1177, + 664.6064, + 669.9441 + ], + "conc_latency_p99_list": [ + 0.029614504389592188, + 0.05973446130046791, + 0.10905227464991184 + ], + "conc_latency_avg_list": [ + 0.016960461349472145, + 0.029901492867268054, + 0.0588588060561928 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "metric_type": "COSINE", + "efConstruction": 128, + "M": 20, + "num_candidates": 100 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.2 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10, + 20, + 40 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4530.1908, + "optimize_duration": 1676.9701, + "load_duration": 6207.160, + "qps": 899.3114, + "serial_latency_p99": 0.0149, + "recall": 0.9072, + "ndcg": 0.9205, + "conc_num_list": [ + 10, + 20, + 40 + ], + "conc_qps_list": [ + 696.9797, + 849.2138, + 899.3114 + ], + "conc_latency_p99_list": [ + 0.025854850579053165, + 0.05033877524005214, + 0.08175756704062223 + ], + "conc_latency_avg_list": [ + 0.01429894712810464, + 0.02340364411279915, + 0.04380456948198429 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g-force_merge", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 8, + "use_rescore": false, + "oversample_ratio": 2.0, + "metric_type": "COSINE", + "efConstruction": 128, + "M": 20, + "num_candidates": 100 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.5 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 10, + 20, + 40 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 20389.08444745702, + "optimize_duration": 26339.51041048577, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [], + "conc_qps_list": [], + "conc_latency_p99_list": [], + "conc_latency_avg_list": [], + "st_ideal_insert_duration": 20000, + "st_search_stage_list": [ + 10, + 20, + 30, + 40, + 50, + 60, + 70, + 80, + 90, + 100, + 110 + ], + "st_search_time_list": [ + 2001.3354, + 4001.3802, + 6008.1472, + 8007.5765, + 10055.0974, + 12094.3767, + 14096.0948, + 16279.1147, + 18277.522, + 20390.5944, + 46839.5735 + ], + "st_max_qps_list_list": [ + 100.4082, + 68.6664, + 82.445, + 69.027, + 70.8964, + 76.7213, + 56.8557, + 71.158, + 61.6708, + 67.3289, + 1279.8773 + ], + "st_recall_list": [ + 0.0944, + 0.188, + 0.2778, + 0.3644, + 0.4513, + 0.5357, + 0.6213, + 0.7071, + 0.7939, + 0.8812, + 0.9033 + ], + "st_ndcg_list": [ + 0.0944, + 0.1877, + 0.2779, + 0.3641, + 0.4509, + 0.5352, + 0.6203, + 0.7056, + 0.7913, + 0.8778, + 0.9104 + ], + "st_serial_latency_p99_list": [ + 0.0343, + 0.0607, + 0.0762, + 0.0727, + 0.0803, + 0.0618, + 0.0662, + 0.1353, + 0.0794, + 0.0633, + 0.0194 + ], + "st_conc_failed_rate_list": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 4, + "num_force_merge_threads": 4, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 160 + }, + "case_config": { + "case_id": 200, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "search_stages": [ + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "concurrencies": [ + 5, + 10, + 15 + ], + "insert_rate": 500, + "optimize_after_write": true + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 14596.751037248003, + "optimize_duration": 28095.967201844129, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [], + "conc_qps_list": [], + "conc_latency_p99_list": [], + "conc_latency_avg_list": [], + "st_ideal_insert_duration": 10000, + "st_search_stage_list": [ + 10, + 20, + 30, + 40, + 50, + 60, + 70, + 80, + 90, + 100, + 110 + ], + "st_search_time_list": [ + 1001.5323, + 2154.4995, + 3523.1943, + 5093.1255, + 6435.0815, + 8087.0502, + 9725.4128, + 11205.4954, + 13018.0616, + 14598.1749, + 42802.8495 + ], + "st_max_qps_list_list": [ + 81.3147, + 64.8161, + 67.308, + 65.0485, + 68.9195, + 63.9133, + 58.2076, + 54.1085, + 61.8172, + 91.7721, + 1293.7108 + ], + "st_recall_list": [ + 0.0943, + 0.1857, + 0.2748, + 0.3602, + 0.4444, + 0.5291, + 0.6179, + 0.7053, + 0.7927, + 0.8677, + 0.9051 + ], + "st_ndcg_list": [ + 0.0943, + 0.1853, + 0.2749, + 0.3597, + 0.4437, + 0.5279, + 0.6169, + 0.7035, + 0.7908, + 0.8645, + 0.9092 + ], + "st_serial_latency_p99_list": [ + 0.0376, + 0.0594, + 0.0776, + 0.1418, + 0.0805, + 0.0773, + 0.3247, + 0.0735, + 0.2223, + 0.2501, + 0.0189 + ], + "st_conc_failed_rate_list": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "task_config": { + "db": "ElasticCloud", + "db_config": { + "db_label": "8c60g", + "version": "8.17", + "note": "", + "cloud_id": "**********", + "password": "**********" + }, + "db_case_config": { + "element_type": "float", + "index": "hnsw", + "number_of_shards": 1, + "number_of_replicas": 0, + "refresh_interval": "30s", + "merge_max_thread_count": 4, + "num_force_merge_threads": 4, + "use_rescore": false, + "oversample_ratio": 2.0, + "use_routing": false, + "metric_type": "COSINE", + "efConstruction": 256, + "M": 16, + "num_candidates": 160 + }, + "case_config": { + "case_id": 200, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "search_stages": [ + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "concurrencies": [ + 5, + 10, + 15 + ], + "insert_rate": 1000, + "optimize_after_write": true + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + } + ], + "file_fmt": "result_{}_{}_{}.json", + "timestamp": 1742256000.0 +} \ No newline at end of file diff --git a/vectordb_bench/results/Milvus/result_20250509_standard_milvus.json b/vectordb_bench/results/Milvus/result_20250509_standard_milvus.json new file mode 100644 index 000000000..284538cda --- /dev/null +++ b/vectordb_bench/results/Milvus/result_20250509_standard_milvus.json @@ -0,0 +1,6138 @@ +{ + "run_id": "d60af9965dec47739a5924d55bd3ca8d", + "task_label": "standard_20250519", + "results": [ + { + "metrics": { + "max_load_count": 0, + "insert_duration": 307.2743, + "optimize_duration": 609.0158, + "load_duration": 916.29, + "qps": 3465.1696, + "serial_latency_p99": 0.0022, + "recall": 0.9528, + "ndcg": 0.957, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 532.2409, + 2009.0973, + 2583.1463, + 2970.1054, + 3143.8239, + 3240.6279, + 3101.2146, + 3465.1696 + ], + "conc_latency_p99_list": [ + 0.002124513557355385, + 0.0031260556002962398, + 0.0055929991888115085, + 0.012370029596932008, + 0.01691194544022436, + 0.021083038998767744, + 0.029270698031832583, + 0.035461206219843 + ], + "conc_latency_avg_list": [ + 0.001876429236465402, + 0.002484698704309982, + 0.003864614006847556, + 0.006717536835843297, + 0.009495690287660783, + 0.012239128866584396, + 0.0172460200800993, + 0.022011112315464 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 307.2743, + "optimize_duration": 609.0158, + "load_duration": 916.29, + "qps": 3102.1518, + "serial_latency_p99": 0.0023, + "recall": 0.9608, + "ndcg": 0.9644, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 503.4466, + 1873.1947, + 2388.7285, + 2444.6672, + 2813.8453, + 2921.6966, + 2828.853, + 3102.1518 + ], + "conc_latency_p99_list": [ + 0.0022688782594195798, + 0.003395149860662058, + 0.006240734964376302, + 0.013228418048165626, + 0.018191038916993422, + 0.022251176000281703, + 0.030577128853110494, + 0.039724538203154226 + ], + "conc_latency_avg_list": [ + 0.0019839456720220036, + 0.002665196167049644, + 0.004178073522664654, + 0.0074283677540322975, + 0.010590303179083615, + 0.013558271773570291, + 0.01913941447519525, + 0.02466243088985968 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 120, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 307.2743, + "optimize_duration": 609.0158, + "load_duration": 916.29, + "qps": 2681.2848, + "serial_latency_p99": 0.0025, + "recall": 0.9681, + "ndcg": 0.9709, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 467.9514, + 1726.7454, + 2149.9845, + 2362.4893, + 2471.1186, + 2538.8461, + 2404.394, + 2681.2848 + ], + "conc_latency_p99_list": [ + 0.0024857501595397474, + 0.0037884765173657807, + 0.007135819507966513, + 0.014403291748749325, + 0.01957427468092646, + 0.024791473661753115, + 0.03465020745054073, + 0.04322717387694864 + ], + "conc_latency_avg_list": [ + 0.002134563151304929, + 0.0028918128837812076, + 0.004643519319465046, + 0.008440618380640339, + 0.012072748818469706, + 0.015590693785188008, + 0.02244991383736933, + 0.028421631748089905 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 150, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 307.2743, + "optimize_duration": 609.0158, + "load_duration": 916.29, + "qps": 2232.9105, + "serial_latency_p99": 0.0029, + "recall": 0.9757, + "ndcg": 0.9781, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 418.7892, + 1514.8484, + 1864.9131, + 1974.1517, + 2046.7492, + 2097.2906, + 2005.1328, + 2232.9105 + ], + "conc_latency_p99_list": [ + 0.002839671221008757, + 0.004669282099348493, + 0.008661248351709215, + 0.015958899196120906, + 0.02241251588340674, + 0.028817034818930446, + 0.03897370800114003, + 0.04830567583878292 + ], + "conc_latency_avg_list": [ + 0.002385222759447107, + 0.003296297278936589, + 0.0053535212933248175, + 0.010104320763196046, + 0.014580544512525781, + 0.018897057048997214, + 0.026836630779984168, + 0.03405441290847284 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 200, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 307.2743, + "optimize_duration": 609.0158, + "load_duration": 916.29, + "qps": 1913.17, + "serial_latency_p99": 0.0032, + "recall": 0.9797, + "ndcg": 0.9819, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 381.7997, + 1361.7034, + 1645.8947, + 1700.692, + 1607.9872, + 1809.9524, + 1707.9465, + 1913.17 + ], + "conc_latency_p99_list": [ + 0.003156380761647597, + 0.005428295295860154, + 0.010003156159364155, + 0.01791602954006521, + 0.024907285197114108, + 0.0321867404325894, + 0.04480843118042684, + 0.05596002343867439 + ], + "conc_latency_avg_list": [ + 0.0026165285302883045, + 0.0036676513390399546, + 0.006067466006458217, + 0.01173631721322714, + 0.01690247802722395, + 0.021831370660184004, + 0.031227139229540585, + 0.03982062627491065 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 250, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 307.2743, + "optimize_duration": 609.0158, + "load_duration": 916.29, + "qps": 1575.8137, + "serial_latency_p99": 0.0035, + "recall": 0.9822, + "ndcg": 0.9843, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 352.4872, + 1221.2298, + 1452.6216, + 1504.2088, + 1412.034, + 1575.8137, + 1524.2361, + 1531.2333 + ], + "conc_latency_p99_list": [ + 0.0034940582773560873, + 0.00622464356274577, + 0.011316185915929956, + 0.01939814951343581, + 0.02765800280154508, + 0.035504716126015405, + 0.04907120082971232, + 0.06320833449455673 + ], + "conc_latency_avg_list": [ + 0.002834249962840069, + 0.004089734635197713, + 0.006875491850782743, + 0.013269717270589515, + 0.019246825423301958, + 0.02516177274076156, + 0.03547240681425613, + 0.04553424434448767 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 300, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 307.2743, + "optimize_duration": 609.0158, + "load_duration": 916.29, + "qps": 1344.5652, + "serial_latency_p99": 0.0042, + "recall": 0.9849, + "ndcg": 0.987, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 305.9443, + 1030.8126, + 1086.1837, + 1222.3353, + 1253.234, + 1286.3153, + 1325.8026, + 1344.5652 + ], + "conc_latency_p99_list": [ + 0.0041222776586073455, + 0.007499401999666588, + 0.0131768821500009, + 0.023312807287657052, + 0.032819842278549914, + 0.04173283703785273, + 0.05881287370066274, + 0.07744551679570585 + ], + "conc_latency_avg_list": [ + 0.003265679799643682, + 0.004845199029043003, + 0.008363285670635592, + 0.016323376673238937, + 0.0238552610209504, + 0.03076185268014987, + 0.044068707980292054, + 0.05614307274786666 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 400, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 307.2743, + "optimize_duration": 609.0158, + "load_duration": 916.29, + "qps": 1148.7813, + "serial_latency_p99": 0.0048, + "recall": 0.9861, + "ndcg": 0.9882, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 269.5022, + 890.3388, + 1013.4633, + 975.7435, + 1057.1967, + 1082.9939, + 1119.6985, + 1148.7813 + ], + "conc_latency_p99_list": [ + 0.004903405122749973, + 0.008865842801606051, + 0.01472508188075153, + 0.027190196317096703, + 0.03865172098761832, + 0.04917521619936451, + 0.0698939428481389, + 0.08927739279897659 + ], + "conc_latency_avg_list": [ + 0.0037075314767917073, + 0.005610315808414844, + 0.009856416789595928, + 0.019187529354542345, + 0.028235225465375567, + 0.03668463694606078, + 0.05210596655562254, + 0.06646545865826 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 500, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 307.2743, + "optimize_duration": 609.0158, + "load_duration": 916.29, + "qps": 3680.6045, + "serial_latency_p99": 0.0022, + "recall": 0.9954, + "ndcg": 0.9967, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 577.9631, + 2048.7797, + 2645.3396, + 3121.0494, + 3322.2571, + 3449.3207, + 3599.8683, + 3680.6045 + ], + "conc_latency_p99_list": [ + 0.0021069889620412142, + 0.00300036284403177, + 0.005232453710777884, + 0.011433810488888412, + 0.0157309884406277, + 0.019307836279476753, + 0.02617319536191643, + 0.033005707949632715 + ], + "conc_latency_avg_list": [ + 0.001727951389557224, + 0.002436855342246593, + 0.003772879611576482, + 0.006388213077436368, + 0.00898044016129782, + 0.011497054844835273, + 0.016255841102205074, + 0.02076874549244545 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.001 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 307.2743, + "optimize_duration": 609.0158, + "load_duration": 916.29, + "qps": 3407.9972, + "serial_latency_p99": 0.0022, + "recall": 0.994, + "ndcg": 0.9957, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 555.4232, + 1971.6292, + 2526.6117, + 2759.2613, + 3152.6046, + 3276.1762, + 3407.9972, + 3321.8121 + ], + "conc_latency_p99_list": [ + 0.002169656099431449, + 0.003117794104036874, + 0.005509768904448726, + 0.01204809755668976, + 0.01608947631873889, + 0.020046225998448793, + 0.027793191453019967, + 0.03398696479853241 + ], + "conc_latency_avg_list": [ + 0.0017980005317151555, + 0.0025319322762065253, + 0.003950555567705964, + 0.00678597387501831, + 0.009456637215385566, + 0.012109263845443934, + 0.01715762185443751, + 0.02182660488502949 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.002 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 307.2743, + "optimize_duration": 609.0158, + "load_duration": 916.29, + "qps": 3062.6755, + "serial_latency_p99": 0.0024, + "recall": 0.9932, + "ndcg": 0.9951, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 514.5573, + 1824.7996, + 2326.6444, + 2616.9497, + 2533.6291, + 2874.7723, + 2991.7453, + 3062.6755 + ], + "conc_latency_p99_list": [ + 0.002340779922014917, + 0.003460889162088279, + 0.006418144360795848, + 0.013004017297498643, + 0.017724068614625128, + 0.021886619948418232, + 0.0303224491421861, + 0.038395417397259754 + ], + "conc_latency_avg_list": [ + 0.0019410542857132853, + 0.0027361909536010197, + 0.004290979883876869, + 0.007619526891310287, + 0.010750764794947291, + 0.01378983240240322, + 0.01962656767091678, + 0.024893208148054725 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.005 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 307.2743, + "optimize_duration": 609.0158, + "load_duration": 916.29, + "qps": 2568.3371, + "serial_latency_p99": 0.0027, + "recall": 0.9927, + "ndcg": 0.9947, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 464.6824, + 1641.1197, + 2053.1323, + 2227.9202, + 2352.6976, + 2436.4817, + 2514.5488, + 2568.3371 + ], + "conc_latency_p99_list": [ + 0.0026186897985462565, + 0.0038988129297649666, + 0.007515603159845343, + 0.014763638905787958, + 0.020218795160762942, + 0.024756647483736738, + 0.034098059750249377, + 0.04383702200138949 + ], + "conc_latency_avg_list": [ + 0.0021494436331336627, + 0.00304260607906732, + 0.004862697743532062, + 0.008954207500006837, + 0.012681185279542212, + 0.01626313683824525, + 0.02326411987256159, + 0.02972949649635349 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.01 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 307.2743, + "optimize_duration": 609.0158, + "load_duration": 916.29, + "qps": 1886.2891, + "serial_latency_p99": 0.0033, + "recall": 0.992, + "ndcg": 0.9943, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 371.9331, + 1305.7923, + 1600.471, + 1686.055, + 1769.2388, + 1658.0011, + 1886.2891, + 1838.3459 + ], + "conc_latency_p99_list": [ + 0.0032482112433353904, + 0.005436603297857802, + 0.010106810779834623, + 0.01757893640307884, + 0.024348094486849773, + 0.030540541783120722, + 0.042648282299778656, + 0.05530555399891455 + ], + "conc_latency_avg_list": [ + 0.0026859828722719903, + 0.0038242266499175217, + 0.0062400044490047496, + 0.011835305188237886, + 0.016870283847420238, + 0.0217500563576271, + 0.031009552852119225, + 0.03949698731460025 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.02 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 307.2743, + "optimize_duration": 609.0158, + "load_duration": 916.29, + "qps": 923.8685, + "serial_latency_p99": 0.0067, + "recall": 0.9919, + "ndcg": 0.9941, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 181.3414, + 622.2492, + 671.0542, + 764.5985, + 744.4012, + 826.6565, + 885.3278, + 923.8685 + ], + "conc_latency_p99_list": [ + 0.00670429674355546, + 0.011902232438005741, + 0.018455940921121523, + 0.03629805936237972, + 0.049964750518556686, + 0.06229904215855641, + 0.0865070858814579, + 0.11306153929159345 + ], + "conc_latency_avg_list": [ + 0.005510754523208972, + 0.008028060402110022, + 0.013536888050997085, + 0.02609999302298258, + 0.03774375663290419, + 0.047889240376042114, + 0.0659747554287514, + 0.08290130450380996 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.05 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 307.2743, + "optimize_duration": 609.0158, + "load_duration": 916.29, + "qps": 1442.1076, + "serial_latency_p99": 0.0041, + "recall": 0.9517, + "ndcg": 0.9579, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 306.5094, + 1054.0645, + 1199.1765, + 1154.7538, + 1314.0769, + 1347.5656, + 1309.3901, + 1442.1076 + ], + "conc_latency_p99_list": [ + 0.004009728922028444, + 0.0070912296183814755, + 0.013089907130633946, + 0.0226678342372179, + 0.03182988779939478, + 0.04078589414930321, + 0.05722206579812337, + 0.07408286570134803 + ], + "conc_latency_avg_list": [ + 0.003259605582834015, + 0.004738441926062677, + 0.008141236161905065, + 0.01573439538238641, + 0.02269659797200437, + 0.029391336884733124, + 0.041621421631708046, + 0.053085492786877124 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.1 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 307.2743, + "optimize_duration": 609.0158, + "load_duration": 916.29, + "qps": 2002.9865, + "serial_latency_p99": 0.0032, + "recall": 0.9467, + "ndcg": 0.9525, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 382.339, + 1350.0184, + 1627.9899, + 1716.4553, + 1794.9255, + 1856.2618, + 1942.215, + 2002.9865 + ], + "conc_latency_p99_list": [ + 0.0030824300047243014, + 0.00519196090026526, + 0.009860773039690678, + 0.017780575102078724, + 0.02459146206834703, + 0.031362373668744115, + 0.042387562902149505, + 0.05431748189846983 + ], + "conc_latency_avg_list": [ + 0.0026128414437139973, + 0.0036989371497274906, + 0.006132647172510258, + 0.01162158751776679, + 0.01661823194035456, + 0.02136927870453069, + 0.030009667759459884, + 0.03807211310263073 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.2 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 307.2743, + "optimize_duration": 609.0158, + "load_duration": 916.29, + "qps": 3201.9438, + "serial_latency_p99": 0.0023, + "recall": 0.922, + "ndcg": 0.9279, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 504.3541, + 1859.4944, + 2141.8149, + 2645.7303, + 2852.2906, + 2751.3916, + 3149.4563, + 3201.9438 + ], + "conc_latency_p99_list": [ + 0.0022312257001613034, + 0.0033456607614061793, + 0.006167679148347818, + 0.013803666250169034, + 0.01820054187250207, + 0.022561932749522385, + 0.031100147141551127, + 0.03799860495331811 + ], + "conc_latency_avg_list": [ + 0.0019803474683102794, + 0.0026848738368728843, + 0.004292210274856248, + 0.007535752628887223, + 0.010452347427673423, + 0.013201119502799242, + 0.01860147262180211, + 0.023259287570251475 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.5 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 276.5846, + "optimize_duration": 559.7462, + "load_duration": 836.3308, + "qps": 11763.5538, + "serial_latency_p99": 0.0015, + "recall": 1.0, + "ndcg": 1.0, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 762.1964, + 3379.2835, + 5940.0997, + 9366.8841, + 9460.8742, + 11155.4888, + 11763.5538, + 10997.8789 + ], + "conc_latency_p99_list": [ + 0.0014307849970646204, + 0.001786595129960915, + 0.0021783469044021303, + 0.003745632997379289, + 0.0055022907689271875, + 0.007276052479282951, + 0.010882212848300692, + 0.01410220092133385 + ], + "conc_latency_avg_list": [ + 0.0013098269680292818, + 0.001476372690548297, + 0.0016787072945413815, + 0.002126154402009081, + 0.0028671046236934863, + 0.003549566181546185, + 0.0049695624206545705, + 0.0069093157715497495 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-partition_key", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.001 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 276.5846, + "optimize_duration": 559.7462, + "load_duration": 836.3308, + "qps": 11803.1944, + "serial_latency_p99": 0.0015, + "recall": 0.9778, + "ndcg": 0.9815, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 777.5611, + 3450.5162, + 6020.6949, + 9602.4047, + 10817.646, + 11475.4689, + 11803.1944, + 11085.8018 + ], + "conc_latency_p99_list": [ + 0.0014560520982195156, + 0.00187936223926954, + 0.0022623318403202494, + 0.003631137506090454, + 0.005380373901716666, + 0.0071127725030237, + 0.011069638402113939, + 0.01384772803954548 + ], + "conc_latency_avg_list": [ + 0.0012838423712838217, + 0.0014460008427277394, + 0.0016562709412299716, + 0.002074207873264524, + 0.0027524051579152034, + 0.003441895338736161, + 0.004914141845207628, + 0.0068891246590032824 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-partition_key", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.002 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 276.5846, + "optimize_duration": 559.7462, + "load_duration": 836.3308, + "qps": 11520.9234, + "serial_latency_p99": 0.0015, + "recall": 0.9634, + "ndcg": 0.9693, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 756.6066, + 3297.2973, + 5854.1128, + 9285.7633, + 10412.0673, + 10793.5215, + 11520.9234, + 11058.1724 + ], + "conc_latency_p99_list": [ + 0.0014800821972312402, + 0.002019465318880977, + 0.0023897097984445282, + 0.0037559883544599866, + 0.00559702909828047, + 0.00745401686319383, + 0.011251274780661324, + 0.014582253761473112 + ], + "conc_latency_avg_list": [ + 0.0013195688511677636, + 0.0015132651259121511, + 0.0017034843385337118, + 0.0021448258112339415, + 0.002859668627997472, + 0.003593739693680354, + 0.00509109536139674, + 0.006882433065064554 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-partition_key", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.005 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 276.5846, + "optimize_duration": 559.7462, + "load_duration": 836.3308, + "qps": 11280.0849, + "serial_latency_p99": 0.0016, + "recall": 0.9507, + "ndcg": 0.9583, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 737.4034, + 3246.2474, + 5725.8606, + 9031.7319, + 10157.2626, + 10750.8006, + 11280.0849, + 10378.4128 + ], + "conc_latency_p99_list": [ + 0.0015326132146583399, + 0.0020956344805017585, + 0.0024661214616207875, + 0.0038491962752596046, + 0.005670193499099696, + 0.007590787559674955, + 0.01134285949956394, + 0.014728240660333543 + ], + "conc_latency_avg_list": [ + 0.0013539057569469733, + 0.0015367961496261372, + 0.001741748759880887, + 0.0022048930230442573, + 0.0029289298298160044, + 0.003668308206122339, + 0.0051780173664949984, + 0.007128565346592537 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-partition_key", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.01 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 276.5846, + "optimize_duration": 559.7462, + "load_duration": 836.3308, + "qps": 10671.8925, + "serial_latency_p99": 0.0017, + "recall": 0.9339, + "ndcg": 0.9436, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 721.2688, + 3165.9145, + 5584.3949, + 8871.8046, + 9884.2635, + 10412.1356, + 10671.8925, + 9689.7362 + ], + "conc_latency_p99_list": [ + 0.0015749457225319931, + 0.002170479700725992, + 0.0025455483744008236, + 0.003923563041171292, + 0.005833359650205241, + 0.007832636859675411, + 0.01177176035489539, + 0.015113587800078668 + ], + "conc_latency_avg_list": [ + 0.0013842298229888238, + 0.0015760963956703178, + 0.0017861258011547523, + 0.0022448004943659283, + 0.0030135993328317917, + 0.0037990046673942036, + 0.005514933604356482, + 0.007567272727381095 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-partition_key", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.02 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 276.5846, + "optimize_duration": 559.7462, + "load_duration": 836.3308, + "qps": 10258.2661, + "serial_latency_p99": 0.0017, + "recall": 0.9139, + "ndcg": 0.9261, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 692.9036, + 3032.6032, + 5334.3498, + 8374.5013, + 8889.6848, + 9832.7117, + 10258.2661, + 10250.8369 + ], + "conc_latency_p99_list": [ + 0.00164179176237667, + 0.0022601902986934884, + 0.002627119809985742, + 0.004133462904428594, + 0.0060811046016169745, + 0.008235614609147887, + 0.012345839151385025, + 0.01612723067606567 + ], + "conc_latency_avg_list": [ + 0.0014409592926640255, + 0.0016453740886882727, + 0.0018695080565112826, + 0.0023778161610121785, + 0.00319070961610905, + 0.004022284165798754, + 0.005687069021673367, + 0.007439906504224085 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-partition_key", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.05 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 276.5846, + "optimize_duration": 559.7462, + "load_duration": 836.3308, + "qps": 9681.5656, + "serial_latency_p99": 0.0019, + "recall": 0.9008, + "ndcg": 0.9149, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 659.9458, + 2858.8452, + 5017.6719, + 7828.4077, + 8636.7679, + 9080.9449, + 8800.8328, + 9681.5656 + ], + "conc_latency_p99_list": [ + 0.001720651980067487, + 0.0023408066833508208, + 0.002704984121082817, + 0.004247271218846437, + 0.006543758550178608, + 0.008892759013178878, + 0.012987471480446407, + 0.01702890696295072 + ], + "conc_latency_avg_list": [ + 0.0015130825783296348, + 0.001745432671085798, + 0.0019878988638465387, + 0.0025455178726995703, + 0.003450542124169974, + 0.004350189947917705, + 0.006071857900552507, + 0.00787036244150588 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-partition_key", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.1 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 276.5846, + "optimize_duration": 559.7462, + "load_duration": 836.3308, + "qps": 8945.4041, + "serial_latency_p99": 0.0019, + "recall": 0.8894, + "ndcg": 0.9052, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 615.8685, + 2686.5178, + 4774.0322, + 7269.8613, + 7968.4918, + 8364.6577, + 8780.9006, + 8945.4041 + ], + "conc_latency_p99_list": [ + 0.001835183917719405, + 0.002413980961136984, + 0.002795249518603668, + 0.0045262172740331156, + 0.006903012750626658, + 0.009463024759606923, + 0.013903731200116455, + 0.01837892190087589 + ], + "conc_latency_avg_list": [ + 0.001621413161838361, + 0.0018577839383309936, + 0.0020894208451563744, + 0.0027383987680678437, + 0.0037405956508111534, + 0.004721064007758078, + 0.006656528248941548, + 0.008531987887327818 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-partition_key", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.2 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 276.5846, + "optimize_duration": 559.7462, + "load_duration": 836.3308, + "qps": 5436.8907, + "serial_latency_p99": 0.002, + "recall": 0.929, + "ndcg": 0.9377, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 566.7372, + 2357.7612, + 3650.5283, + 4549.3919, + 4870.722, + 5109.3415, + 5436.8907, + 5343.8822 + ], + "conc_latency_p99_list": [ + 0.0020163483193755384, + 0.0026659807385294686, + 0.003647062997333703, + 0.007679334198473952, + 0.011922982141404631, + 0.015414087230019498, + 0.021256146834348316, + 0.026203914303187043 + ], + "conc_latency_avg_list": [ + 0.001762171234314572, + 0.0021172856582630344, + 0.002733276211205968, + 0.0043790915723091834, + 0.0061216021197250964, + 0.007756161809275313, + 0.010732330419348917, + 0.013606554491115131 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-partition_key", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.5 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3928.1671, + "optimize_duration": 4636.2807, + "load_duration": 8564.4477, + "qps": 437.1695, + "serial_latency_p99": 0.0051, + "recall": 0.951, + "ndcg": 0.9524, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 224.2715, + 374.6163, + 388.5875, + 385.2169, + 407.8345, + 424.7925, + 411.1939, + 437.1695 + ], + "conc_latency_p99_list": [ + 0.00506570509969606, + 0.01886953882029047, + 0.042465579029667376, + 0.07574911420051647, + 0.10906004564931213, + 0.138387782878417, + 0.19396055204073492, + 0.2544852299337808 + ], + "conc_latency_avg_list": [ + 0.004455487624114796, + 0.013337285918295708, + 0.02570904686197033, + 0.05181718132345756, + 0.07319053975940637, + 0.09303699148838844, + 0.13037410829773755, + 0.16583684812911237 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3928.1671, + "optimize_duration": 4636.2807, + "load_duration": 8564.4477, + "qps": 400.0053, + "serial_latency_p99": 0.0056, + "recall": 0.9558, + "ndcg": 0.9573, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 202.8688, + 339.742, + 348.901, + 338.993, + 358.962, + 377.7443, + 392.5102, + 400.0053 + ], + "conc_latency_p99_list": [ + 0.0056473868231842065, + 0.022245068539559727, + 0.04864812623971373, + 0.08637130765055188, + 0.12404834494023817, + 0.15563446857988308, + 0.2324545464208495, + 0.2949201859999448 + ], + "conc_latency_avg_list": [ + 0.004925194662773988, + 0.014707193316122963, + 0.02863698796667435, + 0.05889247244746554, + 0.0832601042506366, + 0.10520734495196365, + 0.14993878136320568, + 0.1912677135864971 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 120, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3928.1671, + "optimize_duration": 4636.2807, + "load_duration": 8564.4477, + "qps": 343.938, + "serial_latency_p99": 0.0065, + "recall": 0.9605, + "ndcg": 0.9622, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 163.9314, + 297.2904, + 299.1411, + 293.0262, + 283.4141, + 320.4574, + 335.7743, + 343.938 + ], + "conc_latency_p99_list": [ + 0.006456664010183886, + 0.026814749158656923, + 0.05690493760121172, + 0.1050943159227608, + 0.14454019517099367, + 0.18068842753942604, + 0.2621750693718058, + 0.3419199272806873 + ], + "conc_latency_avg_list": [ + 0.005542080578212975, + 0.01680649020155906, + 0.03339920996821813, + 0.0681361154032586, + 0.0960985414766123, + 0.12375873650313164, + 0.17414150543058843, + 0.2233449605688373 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 150, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3928.1671, + "optimize_duration": 4636.2807, + "load_duration": 8564.4477, + "qps": 275.2271, + "serial_latency_p99": 0.0077, + "recall": 0.9649, + "ndcg": 0.9671, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 152.7522, + 244.0961, + 247.2248, + 240.8924, + 253.9762, + 264.8579, + 275.2271, + 260.2012 + ], + "conc_latency_p99_list": [ + 0.007601265637495089, + 0.03272976869993726, + 0.070731162323209, + 0.12965803624065303, + 0.17791054399785944, + 0.22711048271856252, + 0.31685792019852654, + 0.43727977679991453 + ], + "conc_latency_avg_list": [ + 0.006542474580601463, + 0.020469475202850432, + 0.04041643057812953, + 0.08287717597441084, + 0.11761116652479021, + 0.14978475074210712, + 0.2127345127065771, + 0.27807623642544166 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 200, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3928.1671, + "optimize_duration": 4636.2807, + "load_duration": 8564.4477, + "qps": 234.9937, + "serial_latency_p99": 0.0087, + "recall": 0.9677, + "ndcg": 0.97, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 134.8979, + 211.0951, + 211.0551, + 205.1835, + 219.1624, + 226.4716, + 231.0525, + 234.9937 + ], + "conc_latency_p99_list": [ + 0.00878637835936388, + 0.03746770535020915, + 0.08182770514889849, + 0.15646389696892585, + 0.20737264125906196, + 0.26277444176055725, + 0.38411660082834714, + 0.8854517003270992 + ], + "conc_latency_avg_list": [ + 0.007408434917209078, + 0.02366922159438629, + 0.047340457128173745, + 0.09729917928061678, + 0.13604490300440156, + 0.17458267754305823, + 0.2536266195027217, + 0.3264117141404211 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 250, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3928.1671, + "optimize_duration": 4636.2807, + "load_duration": 8564.4477, + "qps": 202.6975, + "serial_latency_p99": 0.0099, + "recall": 0.9696, + "ndcg": 0.9722, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 121.404, + 183.4631, + 184.5319, + 181.2122, + 188.8141, + 180.4191, + 202.6975, + 186.5406 + ], + "conc_latency_p99_list": [ + 0.009961063989321707, + 0.04387273639906198, + 0.09406828430073802, + 0.17931525031330242, + 0.24440658472274662, + 0.31735501231421953, + 0.45487315600432343, + 1.14656314789607 + ], + "conc_latency_avg_list": [ + 0.008232223599586709, + 0.027232883572123886, + 0.0541335331442152, + 0.11017310219528904, + 0.1583824419471285, + 0.20013688256091194, + 0.28877972055990925, + 0.3754691504282929 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 300, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3928.1671, + "optimize_duration": 4636.2807, + "load_duration": 8564.4477, + "qps": 164.6073, + "serial_latency_p99": 0.0121, + "recall": 0.972, + "ndcg": 0.9749, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 100.583, + 148.979, + 149.5835, + 133.9422, + 151.9675, + 159.1746, + 151.8253, + 164.6073 + ], + "conc_latency_p99_list": [ + 0.01218523735748022, + 0.05383459220916846, + 0.11772526517612279, + 0.22714489553276507, + 0.33377324595901875, + 0.38293391765902934, + 0.8939676048349087, + 1.3331147295603176 + ], + "conc_latency_avg_list": [ + 0.009936652570779761, + 0.03353812887299468, + 0.06679692365964107, + 0.13552989226155768, + 0.19672164763885555, + 0.25014387604381855, + 0.3591584290077747, + 0.46665355697584576 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 400, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3928.1671, + "optimize_duration": 4636.2807, + "load_duration": 8564.4477, + "qps": 135.9624, + "serial_latency_p99": 0.0139, + "recall": 0.9733, + "ndcg": 0.9765, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 87.0587, + 127.1129, + 126.126, + 124.2353, + 130.8372, + 133.5469, + 135.9624, + 133.2934 + ], + "conc_latency_p99_list": [ + 0.01406715408258606, + 0.062342452342709305, + 0.14069045348172948, + 0.2659736835179501, + 0.3743512690176432, + 0.5165272444396396, + 1.2337270625311016, + 1.4841110607952546 + ], + "conc_latency_avg_list": [ + 0.011480531560723007, + 0.03930321285991955, + 0.07919771343164675, + 0.16068093155144925, + 0.22844357366393134, + 0.29758923403564963, + 0.43373544498444927, + 0.5611058557884795 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 500, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3928.1671, + "optimize_duration": 4636.2807, + "load_duration": 8564.4477, + "qps": 432.8374, + "serial_latency_p99": 0.005, + "recall": 0.9865, + "ndcg": 0.9902, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 222.5387, + 360.9123, + 369.9704, + 366.8423, + 389.6087, + 401.3948, + 391.2497, + 432.8374 + ], + "conc_latency_p99_list": [ + 0.00491988352034241, + 0.020307911051713742, + 0.04510842189847608, + 0.08157805455979543, + 0.11306029689985732, + 0.1425366488834697, + 0.2152457902976312, + 0.2844128835469746 + ], + "conc_latency_avg_list": [ + 0.0044901991521070815, + 0.013844215543195341, + 0.027006460472747157, + 0.054408617549983614, + 0.07670073422091665, + 0.09864999180283393, + 0.13788800434236897, + 0.1770138288660264 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.001 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3928.1671, + "optimize_duration": 4636.2807, + "load_duration": 8564.4477, + "qps": 368.6042, + "serial_latency_p99": 0.0054, + "recall": 0.9843, + "ndcg": 0.9887, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 203.6095, + 332.539, + 341.3625, + 330.7569, + 357.66, + 340.9937, + 361.8337, + 368.6042 + ], + "conc_latency_p99_list": [ + 0.005436191904300358, + 0.022288193698841476, + 0.04983115282113431, + 0.09119614049632219, + 0.12533250444132135, + 0.16243703891770567, + 0.22909809296368616, + 0.2922112326935166 + ], + "conc_latency_avg_list": [ + 0.004907877380590727, + 0.015024147714186293, + 0.02927040202857776, + 0.06036156999999262, + 0.08333768625692393, + 0.10627057199283074, + 0.1492781564711874, + 0.19132545039541268 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.002 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3928.1671, + "optimize_duration": 4636.2807, + "load_duration": 8564.4477, + "qps": 318.8159, + "serial_latency_p99": 0.0071, + "recall": 0.9836, + "ndcg": 0.9882, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 141.2928, + 254.7206, + 256.6041, + 250.5448, + 275.0399, + 290.3701, + 309.7315, + 318.8159 + ], + "conc_latency_p99_list": [ + 0.0071132864788523865, + 0.03133945180248701, + 0.06669663324973953, + 0.12438690307921206, + 0.16231345123611401, + 0.19979521224508065, + 0.2778699773298295, + 0.37893655615858735 + ], + "conc_latency_avg_list": [ + 0.006430271306714754, + 0.019614416078760834, + 0.038940404658880906, + 0.07965924000938335, + 0.1085803136693593, + 0.13665701334247005, + 0.1902190428589035, + 0.23992846668770065 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.005 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3928.1671, + "optimize_duration": 4636.2807, + "load_duration": 8564.4477, + "qps": 217.0963, + "serial_latency_p99": 0.0091, + "recall": 0.9822, + "ndcg": 0.9871, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 120.3061, + 186.8829, + 190.2293, + 186.7842, + 203.8845, + 215.6092, + 217.0963, + 213.6487 + ], + "conc_latency_p99_list": [ + 0.009213091398123653, + 0.042447957301919796, + 0.08981985074933618, + 0.17112119033030468, + 0.22217497309851753, + 0.29305693579954095, + 0.40971125315933027, + 0.9038828990300181 + ], + "conc_latency_avg_list": [ + 0.008307261453424173, + 0.026738918145085888, + 0.05252079213350113, + 0.10687940907637063, + 0.1466296506934748, + 0.18391824885850308, + 0.262884111228773, + 0.33612484889385125 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.01 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3928.1671, + "optimize_duration": 4636.2807, + "load_duration": 8564.4477, + "qps": 150.9989, + "serial_latency_p99": 0.0125, + "recall": 0.9814, + "ndcg": 0.9866, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 87.0558, + 130.205, + 130.6092, + 128.847, + 138.1481, + 142.6731, + 149.4681, + 150.9989 + ], + "conc_latency_p99_list": [ + 0.01281819700525375, + 0.06043463322777824, + 0.13403809684758017, + 0.2525081645636237, + 0.32649048999475777, + 0.4110537648105674, + 1.0990009665273828, + 1.3628681470390425 + ], + "conc_latency_avg_list": [ + 0.011480799489817662, + 0.038364531792930576, + 0.07650150266510701, + 0.15490539538917095, + 0.216438267826439, + 0.26994097922984056, + 0.3917819305367073, + 0.5095931625685134 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.02 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3928.1671, + "optimize_duration": 4636.2807, + "load_duration": 8564.4477, + "qps": 76.0341, + "serial_latency_p99": 0.0231, + "recall": 0.9797, + "ndcg": 0.9853, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 48.7694, + 68.0017, + 67.66, + 66.8855, + 70.912, + 73.0532, + 76.0341, + 74.3696 + ], + "conc_latency_p99_list": [ + 0.023012421059320318, + 0.11701748956249501, + 0.26127517399610933, + 0.4879960888674395, + 0.7670954282392596, + 1.124323272672772, + 1.8801539649192995, + 2.094726295059445 + ], + "conc_latency_avg_list": [ + 0.02049505297681272, + 0.07345891982469353, + 0.14762136780671775, + 0.2981996634100875, + 0.42065000500752725, + 0.5442770043987755, + 0.7822122534560405, + 1.0395536049691676 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.05 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3928.1671, + "optimize_duration": 4636.2807, + "load_duration": 8564.4477, + "qps": 166.5611, + "serial_latency_p99": 0.0115, + "recall": 0.9675, + "ndcg": 0.9709, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 103.6011, + 154.0558, + 155.798, + 152.984, + 161.7451, + 159.3009, + 166.3349, + 166.5611 + ], + "conc_latency_p99_list": [ + 0.011761713554005834, + 0.05226627752213971, + 0.11276366079691796, + 0.21525146570267673, + 0.2920573918185256, + 0.36234682153015474, + 0.9376167270618877, + 1.2500303659016943 + ], + "conc_latency_avg_list": [ + 0.009647129346883554, + 0.03243046778194412, + 0.06411922614315148, + 0.13048754619744168, + 0.18497559175039416, + 0.2354866400600271, + 0.33842179445562187, + 0.4328670918448006 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.1 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3928.1671, + "optimize_duration": 4636.2807, + "load_duration": 8564.4477, + "qps": 248.9625, + "serial_latency_p99": 0.0083, + "recall": 0.9608, + "ndcg": 0.9627, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 135.2763, + 212.4065, + 214.1029, + 208.5995, + 222.8129, + 233.6588, + 242.4265, + 248.9625 + ], + "conc_latency_p99_list": [ + 0.008451240030044573, + 0.0376529621588998, + 0.08214534900107534, + 0.15387262199510587, + 0.2009427697598585, + 0.25251875603658847, + 0.36905777663974737, + 0.7857984768023 + ], + "conc_latency_avg_list": [ + 0.007387821282659186, + 0.023521519055140303, + 0.04666653643300269, + 0.09571002196384615, + 0.134209675959919, + 0.16961017413058038, + 0.2421030474436561, + 0.3134035979713554 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.2 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3928.1671, + "optimize_duration": 4636.2807, + "load_duration": 8564.4477, + "qps": 435.3358, + "serial_latency_p99": 0.0082, + "recall": 0.9417, + "ndcg": 0.943, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 204.3421, + 338.2876, + 349.2291, + 342.644, + 338.5345, + 390.8324, + 417.1332, + 435.3358 + ], + "conc_latency_p99_list": [ + 0.005513706598139834, + 0.019982896999863442, + 0.046979727581929205, + 0.08727635818657294, + 0.11981394489077503, + 0.15066112712403992, + 0.20701479665644, + 0.2573620026832215 + ], + "conc_latency_avg_list": [ + 0.004890332739593665, + 0.014770007230334713, + 0.028609076336329513, + 0.058247510957779094, + 0.08051730406522835, + 0.10147225046488784, + 0.14075887308364796, + 0.17617929290077974 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.5 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4332.235, + "optimize_duration": 4593.8837, + "load_duration": 8926.1187, + "qps": 11397.7043, + "serial_latency_p99": 0.0016, + "recall": 0.9597, + "ndcg": 0.967, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 744.7271, + 3297.6908, + 5786.8456, + 9187.8979, + 10336.6864, + 9948.7845, + 11397.7043, + 10572.2249 + ], + "conc_latency_p99_list": [ + 0.0015157912735594436, + 0.0020211415598168974, + 0.0024168739427113902, + 0.0037648637557867876, + 0.005586733183881725, + 0.007430122999357989, + 0.011225006281165396, + 0.013903739756497082 + ], + "conc_latency_avg_list": [ + 0.0013406155175477007, + 0.0015129828403533522, + 0.0017232399154656348, + 0.0021679149323633085, + 0.0028835628391416473, + 0.0036326526202308924, + 0.005153060407992853, + 0.007305941831577214 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-partition_key", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.001 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4332.235, + "optimize_duration": 4593.8837, + "load_duration": 8926.1187, + "qps": 10891.7531, + "serial_latency_p99": 0.0017, + "recall": 0.9408, + "ndcg": 0.9508, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 724.0472, + 3203.5917, + 5669.0527, + 8908.6013, + 9969.7995, + 10563.36, + 10891.7531, + 9480.5635 + ], + "conc_latency_p99_list": [ + 0.0015763271538889961, + 0.0021275701503327585, + 0.0024897494004108003, + 0.003868878800421927, + 0.005726297509681899, + 0.007634905402665003, + 0.011535528134118098, + 0.014994531754200605 + ], + "conc_latency_avg_list": [ + 0.001378910087136501, + 0.0015575836747456066, + 0.0017593341522658564, + 0.0022352685038325715, + 0.0029909941851255116, + 0.0037423492383691598, + 0.005342349153920851, + 0.007381548306882306 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-partition_key", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.002 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4332.235, + "optimize_duration": 4593.8837, + "load_duration": 8926.1187, + "qps": 10276.7451, + "serial_latency_p99": 0.0017, + "recall": 0.9159, + "ndcg": 0.9285, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 692.1855, + 3046.3982, + 5367.075, + 8037.3288, + 9390.7259, + 9834.4662, + 9975.0539, + 10276.7451 + ], + "conc_latency_p99_list": [ + 0.001648877248953795, + 0.002235071731847711, + 0.0026013776997569917, + 0.004009247573121684, + 0.006049733492109272, + 0.008144711529021154, + 0.012462468992453068, + 0.015755305530765325 + ], + "conc_latency_avg_list": [ + 0.001442468872899054, + 0.001637813455235224, + 0.0018581155308037942, + 0.0023621886172303604, + 0.003172589629128751, + 0.004021616184078096, + 0.00571248271234091, + 0.007422909765290445 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-partition_key", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.005 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4332.235, + "optimize_duration": 4593.8837, + "load_duration": 8926.1187, + "qps": 9664.2855, + "serial_latency_p99": 0.0018, + "recall": 0.899, + "ndcg": 0.9137, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 656.3459, + 2852.8753, + 5061.0364, + 7830.3728, + 8626.4028, + 8338.1922, + 9570.2667, + 9664.2855 + ], + "conc_latency_p99_list": [ + 0.0017280330721405335, + 0.002328505715704523, + 0.0026939198205945987, + 0.00426532451296226, + 0.006485446713340934, + 0.008702867588872351, + 0.013051993001135978, + 0.017212612797447933 + ], + "conc_latency_avg_list": [ + 0.0015213526332631842, + 0.001749187461728483, + 0.0019709007094981384, + 0.0025449839188546536, + 0.0034507837036955977, + 0.004323049303739524, + 0.0060676739416377955, + 0.007875197581292966 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-partition_key", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.01 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4332.235, + "optimize_duration": 4593.8837, + "load_duration": 8926.1187, + "qps": 8936.7962, + "serial_latency_p99": 0.002, + "recall": 0.8835, + "ndcg": 0.8986, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 617.0435, + 2682.8898, + 4745.2171, + 7195.1942, + 7934.9733, + 8288.4294, + 8748.3038, + 8936.7962 + ], + "conc_latency_p99_list": [ + 0.0018323541997233405, + 0.0024193258199375132, + 0.0028239865193609155, + 0.004531520398450091, + 0.006998135330795772, + 0.009559728798922147, + 0.014029081004991891, + 0.018367440584988803 + ], + "conc_latency_avg_list": [ + 0.0016182575558253492, + 0.0018604371185201418, + 0.0021023141227611035, + 0.002767034698350024, + 0.0037559381095488686, + 0.004760505241094998, + 0.006698510988361889, + 0.00849557210562161 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-partition_key", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.02 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4332.235, + "optimize_duration": 4593.8837, + "load_duration": 8926.1187, + "qps": 5671.2562, + "serial_latency_p99": 0.0021, + "recall": 0.903, + "ndcg": 0.9148, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 561.4932, + 2301.2329, + 3653.0713, + 4576.7837, + 4900.393, + 5180.0113, + 5505.2242, + 5671.2562 + ], + "conc_latency_p99_list": [ + 0.0020966800436144693, + 0.0027066844335058705, + 0.0036274588777450837, + 0.0076524819050973695, + 0.012055617608857564, + 0.015303766338183778, + 0.02117642219600381, + 0.026647148802294386 + ], + "conc_latency_avg_list": [ + 0.001778655378162086, + 0.0021693953743239895, + 0.0027315414728786816, + 0.004357371089986433, + 0.006088828948072214, + 0.007644099210448053, + 0.010647006590783227, + 0.01341964285511407 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-partition_key", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.05 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4332.235, + "optimize_duration": 4593.8837, + "load_duration": 8926.1187, + "qps": 3157.707, + "serial_latency_p99": 0.0023, + "recall": 0.9347, + "ndcg": 0.9407, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 500.7257, + 1851.3759, + 2312.752, + 2656.4202, + 2859.5382, + 2985.0059, + 3157.707, + 3015.7188 + ], + "conc_latency_p99_list": [ + 0.0022447816882049665, + 0.003354803599067962, + 0.0061705887413700065, + 0.013564400935429143, + 0.018209473774768413, + 0.02274693135055718, + 0.030506293503713117, + 0.03878846518709906 + ], + "conc_latency_avg_list": [ + 0.0019947245923933907, + 0.002697035754796493, + 0.004316841985501248, + 0.007508466295392368, + 0.010435763094294353, + 0.013287670410652703, + 0.018521718246965444, + 0.02325211081411987 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-partition_key", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.1 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4332.235, + "optimize_duration": 4593.8837, + "load_duration": 8926.1187, + "qps": 1985.8124, + "serial_latency_p99": 0.0026, + "recall": 0.9407, + "ndcg": 0.945, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 451.1186, + 1307.0711, + 1611.0158, + 1718.6457, + 1687.7489, + 1956.928, + 1940.0614, + 1985.8124 + ], + "conc_latency_p99_list": [ + 0.0024890708172461023, + 0.004719519282807596, + 0.009819361164845756, + 0.01870765387066057, + 0.025398347603040793, + 0.03117593849368859, + 0.043236402257753076, + 0.05431298743918891 + ], + "conc_latency_avg_list": [ + 0.002214184319437533, + 0.00382052748074971, + 0.00619948464586717, + 0.011602779421806592, + 0.016093278142367197, + 0.020284899032372627, + 0.027680786225420435, + 0.03541031140956185 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-partition_key", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.2 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4332.235, + "optimize_duration": 4593.8837, + "load_duration": 8926.1187, + "qps": 920.9627, + "serial_latency_p99": 0.0034, + "recall": 0.9488, + "ndcg": 0.9514, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 348.71, + 678.3473, + 745.7578, + 750.1194, + 813.4585, + 784.0479, + 920.9627, + 892.3854 + ], + "conc_latency_p99_list": [ + 0.003354209200915648, + 0.010283075643237681, + 0.021658108461415387, + 0.0422369472630089, + 0.055763817621918864, + 0.06853605159558358, + 0.09335638748438212, + 0.1146255450003082 + ], + "conc_latency_avg_list": [ + 0.00286492744098909, + 0.007364507617572439, + 0.013394994010937956, + 0.02660646795922508, + 0.03667321400834777, + 0.04620695736896999, + 0.06338143891022638, + 0.07989895167245711 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8-partition_key", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.5 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 20000.127256961016, + "optimize_duration": 2662.8117079710064, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [], + "conc_qps_list": [], + "conc_latency_p99_list": [], + "conc_latency_avg_list": [], + "st_ideal_insert_duration": 20000, + "st_search_stage_list": [ + 10, + 20, + 30, + 40, + 50, + 60, + 70, + 80, + 90, + 100, + 110 + ], + "st_search_time_list": [ + 2000.999, + 4000.9948, + 6000.9942, + 8000.992, + 10001.0007, + 12001.0007, + 14001.0004, + 16000.9947, + 18001.0076, + 20001.4226, + 22791.0781 + ], + "st_max_qps_list_list": [ + 1071.0615, + 751.8463, + 630.9971, + 543.1725, + 456.6263, + 414.5926, + 407.1462, + 381.8069, + 305.9971, + 358.6118, + 443.1588 + ], + "st_recall_list": [ + 0.097, + 0.1944, + 0.2892, + 0.3827, + 0.4777, + 0.5724, + 0.6688, + 0.7669, + 0.8648, + 0.9496, + 0.949 + ], + "st_ndcg_list": [ + 0.097, + 0.1943, + 0.2897, + 0.3831, + 0.4782, + 0.5724, + 0.6683, + 0.7662, + 0.8636, + 0.9507, + 0.9499 + ], + "st_serial_latency_p99_list": [ + 0.0026, + 0.0055, + 0.0068, + 0.0105, + 0.007, + 0.0081, + 0.0082, + 0.0087, + 0.005, + 0.0077, + 0.0048 + ], + "st_conc_failed_rate_list": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 200, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "insert_rate": 500, + "search_stages": [ + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "concurrencies": [ + 5, + 10, + 15 + ], + "optimize_after_write": true + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 10011.807067317975, + "optimize_duration": 3591.1319504730239, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [], + "conc_qps_list": [], + "conc_latency_p99_list": [], + "conc_latency_avg_list": [], + "st_ideal_insert_duration": 10000, + "st_search_stage_list": [ + 10, + 20, + 30, + 40, + 50, + 60, + 70, + 80, + 90, + 100, + 110 + ], + "st_search_time_list": [ + 1001.0736, + 2003.1287, + 3004.4569, + 4006.8058, + 5005.2177, + 6003.5969, + 7001.8987, + 8003.4794, + 9002.1033, + 10013.1107, + 13802.9591 + ], + "st_max_qps_list_list": [ + 603.3553, + 454.3238, + 384.3155, + 271.5658, + 289.6276, + 246.9424, + 193.982, + 179.9923, + 155.9613, + 261.0217, + 429.1983 + ], + "st_recall_list": [ + 0.0977, + 0.1956, + 0.2912, + 0.3845, + 0.4799, + 0.5756, + 0.6708, + 0.7702, + 0.8683, + 0.9523, + 0.9504 + ], + "st_ndcg_list": [ + 0.0977, + 0.1955, + 0.2916, + 0.3848, + 0.4805, + 0.5758, + 0.6705, + 0.7695, + 0.8674, + 0.9535, + 0.9519 + ], + "st_serial_latency_p99_list": [ + 0.007, + 0.0076, + 0.0095, + 0.0089, + 0.0086, + 0.0398, + 0.0129, + 0.0251, + 0.0203, + 0.047, + 0.0053 + ], + "st_conc_failed_rate_list": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "task_config": { + "db": "Milvus", + "db_config": { + "db_label": "16c64g-sq8", + "version": "2.5.11", + "note": "", + "uri": "**********", + "user": null, + "password": null + }, + "db_case_config": { + "index": "HNSW_SQ", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "M": 16, + "efConstruction": 300, + "ef": 100, + "sq_type": "SQ8", + "refine": false, + "refine_type": "FP32", + "refine_k": 1 + }, + "case_config": { + "case_id": 200, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "insert_rate": 1000, + "search_stages": [ + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "concurrencies": [ + 5, + 10, + 15 + ], + "optimize_after_write": true + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + } + ], + "file_fmt": "result_{}_{}_{}.json", + "timestamp": 1746748800.0 +} \ No newline at end of file diff --git a/vectordb_bench/results/OpenSearch/result_20250224_standard_opensearch.json b/vectordb_bench/results/OpenSearch/result_20250224_standard_opensearch.json new file mode 100644 index 000000000..309142526 --- /dev/null +++ b/vectordb_bench/results/OpenSearch/result_20250224_standard_opensearch.json @@ -0,0 +1,7319 @@ +{ + "run_id": "d60af9965dec47739a5924d55bd3ca8d", + "task_label": "standard_2025", + "results": [ + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3569.1383, + "optimize_duration": 2.7599, + "load_duration": 3571.8982, + "qps": 950.6332, + "serial_latency_p99": 0.0132, + "recall": 0.914, + "ndcg": 0.9195, + "conc_num_list": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 408.3592, + 735.2307, + 895.8551, + 928.0196, + 922.2802, + 950.6332, + 947.3421 + ], + "conc_latency_p99_list": [ + 0.013164653459170948, + 0.020184824560128618, + 0.04238946050823869, + 0.06796553365118597, + 0.09233781352937519, + 0.12036370635032653, + 0.15179237727999864 + ], + "conc_latency_avg_list": [ + 0.012240874334956922, + 0.013594033096746037, + 0.02230051837176076, + 0.03219372699347206, + 0.04309711778366417, + 0.062436932770777505, + 0.08312596033111856 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 40, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": false, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3569.1383, + "optimize_duration": 2.7599, + "load_duration": 3571.8982, + "qps": 823.2224, + "serial_latency_p99": 0.0135, + "recall": 0.9434, + "ndcg": 0.9467, + "conc_num_list": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 373.7061, + 663.3808, + 784.3118, + 818.9952, + 823.2224, + 802.0242, + 819.3185 + ], + "conc_latency_p99_list": [ + 0.014557021219952731, + 0.02230411163982355, + 0.04965524877996358, + 0.07211998842096352, + 0.09609327161102554, + 0.1435420441808675, + 0.16229878603975445 + ], + "conc_latency_avg_list": [ + 0.013374717280510946, + 0.015067398479432127, + 0.025472923999326238, + 0.036507759303348375, + 0.0483145931457863, + 0.07398122038375213, + 0.09618061819464098 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 60, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": false, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3569.1383, + "optimize_duration": 2.7599, + "load_duration": 3571.8982, + "qps": 743.9815, + "serial_latency_p99": 0.0148, + "recall": 0.9583, + "ndcg": 0.9603, + "conc_num_list": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 346.5528, + 608.2365, + 714.0153, + 730.438, + 743.9815, + 734.9942, + 738.5282 + ], + "conc_latency_p99_list": [ + 0.01614496053844051, + 0.024739955421355268, + 0.05110240299909489, + 0.0786753849209345, + 0.09768270315951662, + 0.15077884534020997, + 0.1736048336992099 + ], + "conc_latency_avg_list": [ + 0.014424590280885987, + 0.01643356644040233, + 0.027992797970724112, + 0.040903460491745536, + 0.05346867178443852, + 0.08058048623295616, + 0.10672953581089406 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 80, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": false, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3569.1383, + "optimize_duration": 2.7599, + "load_duration": 3571.8982, + "qps": 683.1873, + "serial_latency_p99": 0.0157, + "recall": 0.9677, + "ndcg": 0.9689, + "conc_num_list": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 326.8488, + 564.7963, + 641.473, + 674.6726, + 683.1873, + 677.9066, + 682.8606 + ], + "conc_latency_p99_list": [ + 0.0169064632581285, + 0.026572566999529947, + 0.05759345456033773, + 0.08145290232030634, + 0.1020857048698962, + 0.14635401369909232, + 0.19430979738284807 + ], + "conc_latency_avg_list": [ + 0.015294472532152563, + 0.01769905725158247, + 0.031132205127249707, + 0.04430302725702861, + 0.05819433083505743, + 0.08705286904889194, + 0.11546231433985402 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 100, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": false, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3569.1383, + "optimize_duration": 2.7599, + "load_duration": 3571.8982, + "qps": 619.7468, + "serial_latency_p99": 0.0172, + "recall": 0.9738, + "ndcg": 0.9747, + "conc_num_list": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 306.0542, + 523.1539, + 606.2821, + 610.9287, + 613.4695, + 619.7468, + 615.7953 + ], + "conc_latency_p99_list": [ + 0.01825076190089021, + 0.02971042854878764, + 0.05864030350967375, + 0.08953102107872843, + 0.11617971984131148, + 0.1470403557085226, + 0.19660749083879614 + ], + "conc_latency_avg_list": [ + 0.01633371011266979, + 0.019105616017707928, + 0.032966326695242756, + 0.048906327538473106, + 0.06479810916895794, + 0.09501697349163322, + 0.12711305966720626 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 120, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": false, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3569.1383, + "optimize_duration": 2.7599, + "load_duration": 3571.8982, + "qps": 537.4082, + "serial_latency_p99": 0.0188, + "recall": 0.9809, + "ndcg": 0.9813, + "conc_num_list": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 272.7286, + 451.6863, + 524.4, + 535.7328, + 527.345, + 537.4082, + 521.6465 + ], + "conc_latency_p99_list": [ + 0.02035768547844782, + 0.037040259399909695, + 0.0670176460990115, + 0.09549709711893223, + 0.13596346661812286, + 0.16673180414833041, + 0.21747396636044863 + ], + "conc_latency_avg_list": [ + 0.01832976075732554, + 0.022128600360551726, + 0.03809860787700106, + 0.05583361108221295, + 0.07542607721766255, + 0.11064273113585636, + 0.14678293346900315 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 160, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": false, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3569.1383, + "optimize_duration": 2.7599, + "load_duration": 3571.8982, + "qps": 474.9941, + "serial_latency_p99": 0.0209, + "recall": 0.9848, + "ndcg": 0.9848, + "conc_num_list": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 246.0094, + 412.2446, + 464.62, + 469.4471, + 470.8386, + 474.9941, + 469.4248 + ], + "conc_latency_p99_list": [ + 0.024411036561359584, + 0.039366942282140355, + 0.07809855500090633, + 0.11717723666028164, + 0.13866828196201814, + 0.18897367645949997, + 0.25369487300122273 + ], + "conc_latency_avg_list": [ + 0.020314580877730252, + 0.024244077928585295, + 0.04299313152468402, + 0.06369494327492183, + 0.08410996069385063, + 0.12512816705061675, + 0.16810011745284692 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 200, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": false, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 36572.2691, + "optimize_duration": 90.6008, + "load_duration": 36662.87, + "qps": 505.7458, + "serial_latency_p99": 0.0207, + "recall": 0.9068, + "ndcg": 0.9048, + "conc_num_list": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 254.7958, + 432.8248, + 494.0042, + 494.7877, + 502.7045, + 505.7458, + 503.0119 + ], + "conc_latency_p99_list": [ + 0.02140844455512706, + 0.04155713716638272, + 0.0729441204090836, + 0.1043288180342643, + 0.13288631950126728, + 0.18626255444993156, + 0.23226736870099582 + ], + "conc_latency_avg_list": [ + 0.019613098884841272, + 0.023092215123744025, + 0.04045413027243247, + 0.06054573188457158, + 0.07914080866692273, + 0.11744035323027442, + 0.15688514898966457 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 40, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": false, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 36572.2691, + "optimize_duration": 90.6008, + "load_duration": 36662.87, + "qps": 433.9034, + "serial_latency_p99": 0.0231, + "recall": 0.931, + "ndcg": 0.9285, + "conc_num_list": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 222.3512, + 383.5741, + 419.0419, + 433.7678, + 433.9034, + 432.9478, + 429.4115 + ], + "conc_latency_p99_list": [ + 0.024808250494970707, + 0.04179750306342611, + 0.08707014881001661, + 0.10919756919684014, + 0.14193167329794973, + 0.20521861547604203, + 0.2640936708569643 + ], + "conc_latency_avg_list": [ + 0.022476263830368923, + 0.02605607603018165, + 0.047695931415144006, + 0.06893192599975166, + 0.09163463729098002, + 0.13727825028412483, + 0.18385189699439497 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 60, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": false, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 36572.2691, + "optimize_duration": 90.6008, + "load_duration": 36662.87, + "qps": 381.7737, + "serial_latency_p99": 0.0257, + "recall": 0.9431, + "ndcg": 0.9406, + "conc_num_list": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 197.7924, + 340.6889, + 370.6487, + 378.9375, + 381.7737, + 380.4162, + 379.6756 + ], + "conc_latency_p99_list": [ + 0.035081012492883056, + 0.047591839662636595, + 0.09336233659269055, + 0.1284681179496695, + 0.1587851186006447, + 0.2299024180864217, + 0.29563142965693257 + ], + "conc_latency_avg_list": [ + 0.025273467646480426, + 0.029338515264655005, + 0.053922846726825124, + 0.0789646390194819, + 0.10419213217550603, + 0.15633249287543544, + 0.20801867901052612 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 80, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": false, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 36572.2691, + "optimize_duration": 90.6008, + "load_duration": 36662.87, + "qps": 342.1123, + "serial_latency_p99": 0.029, + "recall": 0.951, + "ndcg": 0.9489, + "conc_num_list": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 179.2589, + 307.7787, + 334.2708, + 339.101, + 338.8609, + 342.1123, + 341.5674 + ], + "conc_latency_p99_list": [ + 0.04371840045190758, + 0.054143276517570484, + 0.09777786302249301, + 0.1403756496869025, + 0.18019018244231114, + 0.24242567380308178, + 0.312930864148948 + ], + "conc_latency_avg_list": [ + 0.027881605935334314, + 0.032467751267399515, + 0.059737810290962125, + 0.08828310663116992, + 0.11733927103427558, + 0.17382575876342551, + 0.23132881972781874 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 100, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": false, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 36572.2691, + "optimize_duration": 90.6008, + "load_duration": 36662.87, + "qps": 308.2216, + "serial_latency_p99": 0.0313, + "recall": 0.9561, + "ndcg": 0.9543, + "conc_num_list": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 165.1113, + 278.672, + 302.7952, + 306.5519, + 307.9669, + 306.2498, + 308.2216 + ], + "conc_latency_p99_list": [ + 0.03405818410246866, + 0.06025193863664753, + 0.10646312752578535, + 0.1523254965059459, + 0.19731556479993737, + 0.2722447675326839, + 0.35651373491913546 + ], + "conc_latency_avg_list": [ + 0.030274068938214806, + 0.035865593418728074, + 0.0659967396254144, + 0.09764338286413679, + 0.12915447181238707, + 0.1941272743441711, + 0.25641679649838883 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 120, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": false, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 36572.2691, + "optimize_duration": 90.6008, + "load_duration": 36662.87, + "qps": 257.7928, + "serial_latency_p99": 0.0364, + "recall": 0.9626, + "ndcg": 0.9608, + "conc_num_list": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 141.2009, + 237.6209, + 253.6438, + 255.9563, + 256.9318, + 257.7928, + 254.8414 + ], + "conc_latency_p99_list": [ + 0.03998605055588996, + 0.06612689679837787, + 0.1263161951844813, + 0.18321261116361715, + 0.2266083036735654, + 0.3211539388803067, + 0.44623331853770654 + ], + "conc_latency_avg_list": [ + 0.035401401102899396, + 0.042061662548614105, + 0.0787759000626649, + 0.1168115360718647, + 0.15489123629833007, + 0.23087606622437556, + 0.31013903427396206 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 160, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": false, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 36572.2691, + "optimize_duration": 90.6008, + "load_duration": 36662.87, + "qps": 223.8166, + "serial_latency_p99": 0.0421, + "recall": 0.9666, + "ndcg": 0.9648, + "conc_num_list": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 123.528, + 208.4727, + 221.582, + 222.5288, + 222.3231, + 220.0288, + 223.8166 + ], + "conc_latency_p99_list": [ + 0.04607281966731534, + 0.0719662698241882, + 0.13897963380150027, + 0.200974844537559, + 0.26876690683711785, + 0.44886413440544853, + 0.4661812005330285 + ], + "conc_latency_avg_list": [ + 0.040460764962844425, + 0.04793753643764161, + 0.09016441214054693, + 0.13454640692510333, + 0.17906950363342047, + 0.27056642289621663, + 0.3534901822091123 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 200, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": false, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3588.445, + "optimize_duration": 483.5584, + "load_duration": 4072.0033, + "qps": 3055.0123, + "serial_latency_p99": 0.0072, + "recall": 0.9066, + "ndcg": 0.9203, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 163.2092, + 768.106, + 1415.0254, + 2395.3309, + 2629.4831, + 2781.704, + 3055.0123, + 3049.557 + ], + "conc_latency_p99_list": [ + 0.0066042609803844245, + 0.0075984859868185595, + 0.008239348817733116, + 0.015747258555202256, + 0.024836629910278154, + 0.03219453388766851, + 0.044554478969075655, + 0.05271297475672325 + ], + "conc_latency_avg_list": [ + 0.006125045602801523, + 0.006506492539732507, + 0.007061801430665638, + 0.008341504582019071, + 0.011361368916648613, + 0.014265056406982746, + 0.019358650994712963, + 0.02555385850054925 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 120, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": true, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3588.445, + "optimize_duration": 483.5584, + "load_duration": 4072.0033, + "qps": 3013.4439, + "serial_latency_p99": 0.0069, + "recall": 0.9268, + "ndcg": 0.9374, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 158.9802, + 754.8615, + 1404.9938, + 2320.2259, + 2559.2791, + 2794.6087, + 2971.6271, + 3013.4439 + ], + "conc_latency_p99_list": [ + 0.006915464034827896, + 0.0077156836647191114, + 0.007996030399226587, + 0.016514454447315134, + 0.02521299222542439, + 0.03257178844534795, + 0.04605080244073179, + 0.05599282969807972 + ], + "conc_latency_avg_list": [ + 0.006287995105696182, + 0.006620287455668505, + 0.007112943989739012, + 0.008611291108323475, + 0.01167839231543837, + 0.014199657722321086, + 0.019886441917191695, + 0.025911668374613735 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 150, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": true, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3588.445, + "optimize_duration": 483.5584, + "load_duration": 4072.0033, + "qps": 2801.7241, + "serial_latency_p99": 0.0074, + "recall": 0.9476, + "ndcg": 0.9547, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 150.3637, + 723.4761, + 1353.4646, + 2199.469, + 2459.2478, + 2661.0816, + 2782.1851, + 2801.7241 + ], + "conc_latency_p99_list": [ + 0.007429877024551388, + 0.007951207885635084, + 0.008615271116723293, + 0.01740019528806437, + 0.025537098896165865, + 0.03309714750677801, + 0.04944766081636781, + 0.06722774161782573 + ], + "conc_latency_avg_list": [ + 0.006648452549404496, + 0.006907931990973957, + 0.007381756507573101, + 0.00907555062424815, + 0.012154000445018477, + 0.014924604728711878, + 0.02124353583550704, + 0.02780025977934295 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 200, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": true, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3588.445, + "optimize_duration": 483.5584, + "load_duration": 4072.0033, + "qps": 2590.3809, + "serial_latency_p99": 0.0086, + "recall": 0.9679, + "ndcg": 0.972, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 137.8652, + 656.3986, + 1251.0291, + 2000.2881, + 2255.0399, + 2401.4937, + 2547.9913, + 2590.3809 + ], + "conc_latency_p99_list": [ + 0.007918751771794632, + 0.00972480603377335, + 0.009297928751038853, + 0.01875622761435807, + 0.027135657450999155, + 0.036018074577441425, + 0.053395299218827894, + 0.067309333984158 + ], + "conc_latency_avg_list": [ + 0.007251337084407966, + 0.007614036386816826, + 0.007989510304307909, + 0.009989053024811402, + 0.013242737272806476, + 0.01653379115487861, + 0.02319072952935361, + 0.0301506326462797 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 300, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": true, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3588.445, + "optimize_duration": 483.5584, + "load_duration": 4072.0033, + "qps": 2291.2159, + "serial_latency_p99": 0.0089, + "recall": 0.9764, + "ndcg": 0.9791, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 125.2547, + 609.9485, + 1154.4166, + 1829.2675, + 2053.688, + 2185.0917, + 2291.2159, + 1958.8103 + ], + "conc_latency_p99_list": [ + 0.008834127507288939, + 0.0094147697329754, + 0.009852253164863213, + 0.01990537133126054, + 0.0295318710131687, + 0.037631657014717386, + 0.05613542024948421, + 0.06533492525340992 + ], + "conc_latency_avg_list": [ + 0.007981524242708626, + 0.008194479599281004, + 0.008658134923769042, + 0.010921130424987624, + 0.014555789546540908, + 0.01818163764880708, + 0.025823034352508798, + 0.03988797952868245 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 400, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": true, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3588.445, + "optimize_duration": 483.5584, + "load_duration": 4072.0033, + "qps": 3099.4124, + "serial_latency_p99": 0.0062, + "recall": 1.0, + "ndcg": 1.0, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 171.8092, + 784.252, + 1412.3687, + 2599.9085, + 2806.959, + 3022.4173, + 3099.4124, + 2734.1758 + ], + "conc_latency_p99_list": [ + 0.006316660190932453, + 0.007400143945123998, + 0.007663118318887427, + 0.014384424227173462, + 0.023472172073088583, + 0.02935590967477764, + 0.037410706279333676, + 0.051547722959658204 + ], + "conc_latency_avg_list": [ + 0.005818297643807042, + 0.0063723405151156105, + 0.007074497835730965, + 0.0076785677160999795, + 0.010637162754967549, + 0.01313621851691824, + 0.019061258279396632, + 0.028444820936257383 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 100, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.001 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3588.445, + "optimize_duration": 483.5584, + "load_duration": 4072.0033, + "qps": 3014.2483, + "serial_latency_p99": 0.007, + "recall": 1.0, + "ndcg": 1.0, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 155.7362, + 721.2526, + 1273.8551, + 2414.5988, + 2671.5813, + 2887.4052, + 2804.1587, + 3014.2483 + ], + "conc_latency_p99_list": [ + 0.007549113626591859, + 0.00784715255285846, + 0.008555541039677334, + 0.01580227653001202, + 0.02434933522948997, + 0.031728020490845665, + 0.04500318020029225, + 0.05078681191487711 + ], + "conc_latency_avg_list": [ + 0.006419092907810258, + 0.006928135313320276, + 0.0078455503292644, + 0.008267252144362823, + 0.011190491416124795, + 0.01375242258266705, + 0.021104762239632032, + 0.02588368512811002 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 100, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.002 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3588.445, + "optimize_duration": 483.5584, + "load_duration": 4072.0033, + "qps": 2073.2153, + "serial_latency_p99": 0.011, + "recall": 1.0, + "ndcg": 1.0, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 104.6063, + 491.307, + 901.6966, + 1603.2264, + 1813.9461, + 1886.4281, + 2028.0559, + 2073.2153 + ], + "conc_latency_p99_list": [ + 0.010429918854497369, + 0.010766250875312834, + 0.014169930647476575, + 0.02164671814913162, + 0.031141857685288408, + 0.04649328580882867, + 0.06406753115355973, + 0.09019827687880023 + ], + "conc_latency_avg_list": [ + 0.009557379905084967, + 0.01017368296873642, + 0.011085640643842048, + 0.012457270307054184, + 0.01649312201199831, + 0.02105301696896077, + 0.029175159319272993, + 0.037729860105790214 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 100, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.005 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3588.445, + "optimize_duration": 483.5584, + "load_duration": 4072.0033, + "qps": 1507.6899, + "serial_latency_p99": 0.0128, + "recall": 1.0, + "ndcg": 1.0, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 83.6182, + 391.2633, + 723.4543, + 1250.109, + 1376.3293, + 1456.6011, + 1464.3895, + 1507.6899 + ], + "conc_latency_p99_list": [ + 0.012811561551643537, + 0.01336541799246334, + 0.016278088598046452, + 0.026901620205899222, + 0.0438782859675121, + 0.054718950600363234, + 0.08420395661029033, + 0.1035638961393852 + ], + "conc_latency_avg_list": [ + 0.011956779498877612, + 0.012775699308254267, + 0.01381764708989311, + 0.015979814802968912, + 0.02171518606405887, + 0.0272911971786362, + 0.040444921689700354, + 0.052059012002956095 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 100, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.01 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3588.445, + "optimize_duration": 483.5584, + "load_duration": 4072.0033, + "qps": 942.2296, + "serial_latency_p99": 0.0182, + "recall": 1.0, + "ndcg": 1.0, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 57.5102, + 275.0919, + 521.9154, + 841.7804, + 919.0783, + 942.2296, + 940.4008, + 937.7659 + ], + "conc_latency_p99_list": [ + 0.01904684674082091, + 0.018614977796096353, + 0.021934586943825708, + 0.044934772892156645, + 0.060170259049627924, + 0.07775546840042806, + 0.11126087368698785, + 0.1392669416032732 + ], + "conc_latency_avg_list": [ + 0.017385657720739505, + 0.018172097744988812, + 0.019152677609988004, + 0.023736293038846563, + 0.03249417960942893, + 0.04218772347479584, + 0.06307773679841584, + 0.08400089795798343 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 100, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.02 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3588.445, + "optimize_duration": 483.5584, + "load_duration": 4072.0033, + "qps": 677.1414, + "serial_latency_p99": 0.0335, + "recall": 0.7655, + "ndcg": 0.7921, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 42.5888, + 211.6872, + 407.1376, + 623.1022, + 670.4218, + 677.1414, + 669.0331, + 601.8884 + ], + "conc_latency_p99_list": [ + 0.03397621757321758, + 0.03439795919839526, + 0.039301968923828076, + 0.06968730082735422, + 0.09789937798632309, + 0.12366854983760263, + 0.16776830756920377, + 0.21250570197706012 + ], + "conc_latency_avg_list": [ + 0.023477284650023914, + 0.023606476280852905, + 0.024546608424613432, + 0.03206853948469295, + 0.04457944931544234, + 0.058732158994757, + 0.08878038772811638, + 0.1309730745888651 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 100, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.05 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3588.445, + "optimize_duration": 483.5584, + "load_duration": 4072.0033, + "qps": 2685.6654, + "serial_latency_p99": 0.0076, + "recall": 0.4914, + "ndcg": 0.5629, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 141.6944, + 668.9311, + 1257.089, + 2031.1627, + 2356.4461, + 2371.7364, + 2637.2292, + 2685.6654 + ], + "conc_latency_p99_list": [ + 0.01269043767475519, + 0.00822650338814128, + 0.012178605884546105, + 0.025561591938603717, + 0.03320253159909043, + 0.05208354838046928, + 0.06661171691492222, + 0.08517426395555969 + ], + "conc_latency_avg_list": [ + 0.007055281598246543, + 0.007471853039319951, + 0.007947702280261822, + 0.009831211199667554, + 0.01268791915069861, + 0.016745610185608897, + 0.02238822883995875, + 0.029078296411875544 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 100, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.1 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3588.445, + "optimize_duration": 483.5584, + "load_duration": 4072.0033, + "qps": 2604.4444, + "serial_latency_p99": 0.0078, + "recall": 0.63, + "ndcg": 0.6874, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 140.1715, + 655.2148, + 1187.3286, + 2066.6191, + 2282.514, + 2471.9677, + 2604.4444, + 2443.4421 + ], + "conc_latency_p99_list": [ + 0.007675001978350337, + 0.0085531802411424, + 0.00926328763773199, + 0.01804770500602898, + 0.027568208608427045, + 0.03512031817226671, + 0.050422279936028644, + 0.06621705831552398 + ], + "conc_latency_avg_list": [ + 0.007131988809897826, + 0.007627937425679999, + 0.008417805785843524, + 0.00966865429092301, + 0.013085665442797694, + 0.01606077057077693, + 0.02268640906685616, + 0.03189756304636577 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 100, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.2 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3588.445, + "optimize_duration": 483.5584, + "load_duration": 4072.0033, + "qps": 2159.051, + "serial_latency_p99": 0.0094, + "recall": 0.801, + "ndcg": 0.8332, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 113.2134, + 539.3392, + 994.8052, + 1691.1574, + 1905.0322, + 2038.4478, + 2159.051, + 2144.7377 + ], + "conc_latency_p99_list": [ + 0.01006929111201316, + 0.012256881904613683, + 0.01087272682110779, + 0.02134280821483112, + 0.03109253071714193, + 0.039616082641296145, + 0.06374024719698355, + 0.08666856052295772 + ], + "conc_latency_avg_list": [ + 0.00883063611796952, + 0.00926644040313672, + 0.010046632364018785, + 0.011815918077522513, + 0.015672935429414725, + 0.019486485648803818, + 0.027420063671646417, + 0.0363942886873006 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 100, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.5 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3383.1906, + "optimize_duration": 422.1602, + "load_duration": 3805.3508, + "qps": 2251.1274, + "serial_latency_p99": 0.0087, + "recall": 0.8848, + "ndcg": 0.9011, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 121.6795, + 569.4114, + 1042.4735, + 1774.5911, + 2002.5872, + 2159.1206, + 2251.1274, + 2152.421 + ], + "conc_latency_p99_list": [ + 0.00928823251160793, + 0.009900132236070931, + 0.010385960509884173, + 0.020927903205156376, + 0.029365679626353083, + 0.03857057009998245, + 0.06097544160089447, + 0.08084093646728435 + ], + "conc_latency_avg_list": [ + 0.008216018987978599, + 0.008777535692041216, + 0.009587645351542377, + 0.011258805059329672, + 0.014918621646763143, + 0.018391872591513005, + 0.026281002446710826, + 0.036310092587277305 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-routing-64shard-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 100, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 64, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": true, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.5 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3383.1906, + "optimize_duration": 422.1602, + "load_duration": 3805.3508, + "qps": 3103.0539, + "serial_latency_p99": 0.0056, + "recall": 1.0, + "ndcg": 1.0, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 185.1193, + 881.9045, + 1617.4851, + 2657.272, + 2949.3946, + 3044.2812, + 3103.0539, + 3028.8089 + ], + "conc_latency_p99_list": [ + 0.006990675125853159, + 0.006732461913488806, + 0.00707246400997974, + 0.016263234553043727, + 0.022491881826426863, + 0.028577061778632926, + 0.0368180122470949, + 0.04633565897762302 + ], + "conc_latency_avg_list": [ + 0.0053999179745601745, + 0.005666654303260133, + 0.00617772421529738, + 0.00751451870044384, + 0.010121368966753455, + 0.013055493901889984, + 0.019021594995459013, + 0.02569502201479951 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-routing-64shard-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 100, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 64, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": true, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.001 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3383.1906, + "optimize_duration": 422.1602, + "load_duration": 3805.3508, + "qps": 3086.1957, + "serial_latency_p99": 0.0067, + "recall": 1.0, + "ndcg": 1.0, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 178.9918, + 848.9912, + 1540.6091, + 2623.0268, + 2813.5476, + 3049.9907, + 3086.1957, + 3053.3069 + ], + "conc_latency_p99_list": [ + 0.006047973496606574, + 0.006976987943053245, + 0.007519919599872081, + 0.014478880249080249, + 0.023875563326291743, + 0.029755084948847075, + 0.03666247386252507, + 0.04414218366146096 + ], + "conc_latency_avg_list": [ + 0.005584841876544247, + 0.005886361231245138, + 0.0064864551689040656, + 0.007616064642805336, + 0.010614586504837481, + 0.013013207187041272, + 0.019113645751718217, + 0.025535433929771763 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-routing-64shard-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 100, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 64, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": true, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.002 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3383.1906, + "optimize_duration": 422.1602, + "load_duration": 3805.3508, + "qps": 3090.0478, + "serial_latency_p99": 0.0064, + "recall": 0.9628, + "ndcg": 0.9683, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 176.2682, + 818.6475, + 1451.0023, + 2479.2743, + 2733.7058, + 3003.7768, + 3090.0478, + 3035.5952 + ], + "conc_latency_p99_list": [ + 0.0066193450265564025, + 0.007285079330904409, + 0.007923564807861112, + 0.01650290552061051, + 0.024602460751775704, + 0.030100287265959192, + 0.03883700771955774, + 0.0469774965575197 + ], + "conc_latency_avg_list": [ + 0.005671220657214157, + 0.006104722327129857, + 0.006885890449604546, + 0.00805760929883198, + 0.010920613097492803, + 0.013213106838382808, + 0.019108201888594578, + 0.02566828746904539 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-routing-64shard-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 100, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 64, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": true, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.005 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3383.1906, + "optimize_duration": 422.1602, + "load_duration": 3805.3508, + "qps": 3064.6288, + "serial_latency_p99": 0.0065, + "recall": 0.9507, + "ndcg": 0.9585, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 162.5604, + 797.6132, + 1424.7756, + 2444.0806, + 2723.5924, + 2892.6262, + 3064.6288, + 3051.5842 + ], + "conc_latency_p99_list": [ + 0.007069946802221237, + 0.0073999632225604725, + 0.008083791472017765, + 0.016095753709087165, + 0.024376636135275468, + 0.030631057044956816, + 0.040737123828148473, + 0.04727083401521672 + ], + "conc_latency_avg_list": [ + 0.006149272652936782, + 0.006265699607445621, + 0.007011809638998383, + 0.008175384402960487, + 0.01097475854248672, + 0.013727037689627335, + 0.019277000032261057, + 0.025535321168339897 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-routing-64shard-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 100, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 64, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": true, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.01 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3383.1906, + "optimize_duration": 422.1602, + "load_duration": 3805.3508, + "qps": 3065.6134, + "serial_latency_p99": 0.0062, + "recall": 0.9328, + "ndcg": 0.9429, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 172.9649, + 778.9863, + 1389.969, + 2462.8121, + 2677.7871, + 2913.1302, + 3065.6134, + 3047.5862 + ], + "conc_latency_p99_list": [ + 0.006182093108654952, + 0.007406606914009899, + 0.008093432011082771, + 0.015239890594966722, + 0.024869270838680648, + 0.032014878002228214, + 0.041195261786342585, + 0.04755387699697167 + ], + "conc_latency_avg_list": [ + 0.005779474808902682, + 0.0064158858712963385, + 0.007189517322705905, + 0.008113123178075, + 0.011162125426314914, + 0.013630981777540626, + 0.01926476095971519, + 0.025575679181467997 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-routing-64shard-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 100, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 64, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": true, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.02 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3383.1906, + "optimize_duration": 422.1602, + "load_duration": 3805.3508, + "qps": 3028.858, + "serial_latency_p99": 0.0067, + "recall": 0.9133, + "ndcg": 0.9258, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 159.0925, + 758.6065, + 1345.1913, + 2329.8545, + 2607.2489, + 2794.8683, + 3028.858, + 2462.6937 + ], + "conc_latency_p99_list": [ + 0.007272385801188647, + 0.00757142439018935, + 0.008144815475679933, + 0.017634137393906697, + 0.025243892944417846, + 0.03292968007212041, + 0.04465119283879176, + 0.05406008135527367 + ], + "conc_latency_avg_list": [ + 0.006283604892761422, + 0.00658817211884869, + 0.0074293878190202795, + 0.008571725427461435, + 0.011454154745049655, + 0.014210889507190997, + 0.019513813342326797, + 0.03167162396994278 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-routing-64shard-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 100, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 64, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": true, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.05 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3383.1906, + "optimize_duration": 422.1602, + "load_duration": 3805.3508, + "qps": 2935.9403, + "serial_latency_p99": 0.0068, + "recall": 0.8992, + "ndcg": 0.9134, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 158.4178, + 716.2477, + 1310.2376, + 2240.6542, + 2524.6522, + 2706.6489, + 2825.4365, + 2935.9403 + ], + "conc_latency_p99_list": [ + 0.007314871074631807, + 0.007610194396693258, + 0.010022854272392578, + 0.01712287428032136, + 0.0252519301319262, + 0.03431479361141101, + 0.04928987917955965, + 0.06064554240147118 + ], + "conc_latency_avg_list": [ + 0.006310439965010313, + 0.0069778828940568385, + 0.00762750852255128, + 0.008913865097414757, + 0.01182858275069365, + 0.014658637716515127, + 0.020985419003976442, + 0.0265203123740125 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-routing-64shard-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 100, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 64, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": true, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.1 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 3383.1906, + "optimize_duration": 422.1602, + "load_duration": 3805.3508, + "qps": 2771.2009, + "serial_latency_p99": 0.0076, + "recall": 0.889, + "ndcg": 0.9045, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 141.8445, + 681.268, + 1248.9943, + 2109.1046, + 2369.8076, + 2557.821, + 2509.6505, + 2771.2009 + ], + "conc_latency_p99_list": [ + 0.007627456617774441, + 0.008933711226563893, + 0.00924032126087695, + 0.0178996564168483, + 0.026518509005545636, + 0.03481165589531884, + 0.04889344602474013, + 0.06475056778755965 + ], + "conc_latency_avg_list": [ + 0.00704781344132045, + 0.007335110199767871, + 0.008002086008581868, + 0.009473561560555328, + 0.012602565485701262, + 0.015532794837116134, + 0.023558604105094996, + 0.028250790234232666 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-routing-64shard-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 100, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 64, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": true, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.2 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 36925.0192, + "optimize_duration": 6711.6160, + "load_duration": 43636.6352, + "qps": 1610.9496, + "serial_latency_p99": 0.0108, + "recall": 0.9, + "ndcg": 0.9119, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 53.8846, + 469.7145, + 874.9726, + 1378.32, + 1532.355, + 1587.5972, + 1567.3349, + 1610.9496 + ], + "conc_latency_p99_list": [ + 0.01170312531408853, + 0.011496570790186523, + 0.013123990170424807, + 0.02510298976325429, + 0.03705705945845694, + 0.05179626225697579, + 0.07615553563897265, + 0.10091682218830102 + ], + "conc_latency_avg_list": [ + 0.01855551537217232, + 0.010641614042458138, + 0.011423641079512197, + 0.014488117554519083, + 0.019502822394563853, + 0.025028242111180318, + 0.0378238977507149, + 0.04872019506240195 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 150, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 36925.0192, + "optimize_duration": 6711.6160, + "load_duration": 43636.6352, + "qps": 1557.3623, + "serial_latency_p99": 0.0108, + "recall": 0.9244, + "ndcg": 0.9332, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 34.3253, + 456.0127, + 858.5571, + 1364.0158, + 1398.8797, + 1497.5824, + 1544.3447, + 1557.3623 + ], + "conc_latency_p99_list": [ + 0.04185159296292114, + 0.011857569686835632, + 0.01320805333962198, + 0.024817747705383236, + 0.04194092803401872, + 0.05550435414188537, + 0.07973901570367166, + 0.10457535300520249 + ], + "conc_latency_avg_list": [ + 0.02912987234558033, + 0.010961446217923507, + 0.011639589056232403, + 0.014629203394371645, + 0.021352729735536417, + 0.02655041915673649, + 0.038351728854688544, + 0.05033554529140941 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 200, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 36925.0192, + "optimize_duration": 6711.6160, + "load_duration": 43636.6352, + "qps": 1473.9256, + "serial_latency_p99": 0.0117, + "recall": 0.9484, + "ndcg": 0.9539, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 23.7127, + 363.211, + 813.8705, + 1238.1709, + 1346.8023, + 1441.1112, + 1473.9256, + 1449.8131 + ], + "conc_latency_p99_list": [ + 0.06029965095687657, + 0.04964189273392549, + 0.013696876754693221, + 0.027977724110824054, + 0.04364553554943996, + 0.055556264364859095, + 0.08215764560736719, + 0.11175056304695312 + ], + "conc_latency_avg_list": [ + 0.042167731182662156, + 0.013762879326449358, + 0.012281659847473723, + 0.01613552393124177, + 0.022222220223888697, + 0.02755962000032243, + 0.040180453415489675, + 0.0541367507144873 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 300, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 36925.0192, + "optimize_duration": 6711.6160, + "load_duration": 43636.6352, + "qps": 1388.5547, + "serial_latency_p99": 0.0125, + "recall": 0.9597, + "ndcg": 0.9636, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 30.4562, + 405.1718, + 774.9717, + 1187.3118, + 1323.3969, + 1384.3901, + 1369.4037, + 1388.5547 + ], + "conc_latency_p99_list": [ + 0.04589326520566829, + 0.013528318731114267, + 0.014656805314007215, + 0.029597530398750658, + 0.04340258422103934, + 0.057421390032686766, + 0.08636225329595618, + 0.10786245728726504 + ], + "conc_latency_avg_list": [ + 0.03283067534690294, + 0.012337288544504669, + 0.01289899602183251, + 0.016830618253860664, + 0.022605185395416345, + 0.028691693463946226, + 0.04330866531350486, + 0.05658665176983458 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 400, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 36925.0192, + "optimize_duration": 6711.6160, + "load_duration": 43636.6352, + "qps": 1022.2696, + "serial_latency_p99": 0.0179, + "recall": 0.936, + "ndcg": 0.936, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 60.0765, + 289.2498, + 556.5826, + 885.7546, + 982.2101, + 1007.5661, + 1022.2696, + 1019.9772 + ], + "conc_latency_p99_list": [ + 0.018322616752120668, + 0.019986357510206332, + 0.023461824607802523, + 0.04082249249040615, + 0.06024408715136813, + 0.07426108589977952, + 0.106976693357574, + 0.1296220435993745 + ], + "conc_latency_avg_list": [ + 0.01664275044720417, + 0.017281532152101594, + 0.017955597924931448, + 0.022538618777411427, + 0.03041328722691892, + 0.03945461804065251, + 0.058044523883584005, + 0.07712270373460872 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 160, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.001 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 36925.0192, + "optimize_duration": 6711.6160, + "load_duration": 43636.6352, + "qps": 696.9777, + "serial_latency_p99": 0.0246, + "recall": 0.997, + "ndcg": 0.997, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 42.6831, + 201.2408, + 385.006, + 633.902, + 681.6608, + 696.9777, + 683.368, + 669.7159 + ], + "conc_latency_p99_list": [ + 0.02564948840299621, + 0.025750786846328994, + 0.0383817074005492, + 0.05452750360127533, + 0.07794399080303273, + 0.09952751863980648, + 0.13890661563229514, + 0.1944875482906355 + ], + "conc_latency_avg_list": [ + 0.023425532061792764, + 0.024841867621552167, + 0.0259583638898489, + 0.03152283461639994, + 0.04385263389947924, + 0.05705725334200993, + 0.08685234079467409, + 0.117749565339635 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 160, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.002 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 36925.0192, + "optimize_duration": 6711.6160, + "load_duration": 43636.6352, + "qps": 353.7862, + "serial_latency_p99": 0.0452, + "recall": 1.0, + "ndcg": 1.0, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 22.921, + 112.2557, + 215.7583, + 338.8218, + 352.1511, + 353.7862, + 346.631, + 350.7445 + ], + "conc_latency_p99_list": [ + 0.046015396596922074, + 0.04539609341387404, + 0.04967247349122772, + 0.09652035661623802, + 0.1349584854079874, + 0.17367849684931574, + 0.29407529990945475, + 0.31688454625837037 + ], + "conc_latency_avg_list": [ + 0.043624330757750636, + 0.044535936394865965, + 0.04634016870794412, + 0.0589488654298943, + 0.08495648071789637, + 0.11235583889329129, + 0.1716003423765812, + 0.2251483244341916 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 160, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.005 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 36925.0192, + "optimize_duration": 6711.6160, + "load_duration": 43636.6352, + "qps": 210.3227, + "serial_latency_p99": 0.0714, + "recall": 1.0, + "ndcg": 1.0, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 14.401, + 70.5432, + 137.7491, + 205.0468, + 206.3997, + 208.7499, + 210.3227, + 208.7295 + ], + "conc_latency_p99_list": [ + 0.07131039915489965, + 0.08340260779368695, + 0.07616257827117806, + 0.15413040314801038, + 0.25033438460552115, + 0.28559843958937564, + 0.3847291337477509, + 0.5490922974195565 + ], + "conc_latency_avg_list": [ + 0.06943479371325685, + 0.07080461257013589, + 0.07258478194845791, + 0.09740673729728754, + 0.14484712146586623, + 0.1904356387610746, + 0.2826340142452903, + 0.3788729431179725 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 160, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.01 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 36925.0192, + "optimize_duration": 6711.6160, + "load_duration": 43636.6352, + "qps": 114.8061, + "serial_latency_p99": 0.1266, + "recall": 0.9985, + "ndcg": 0.9986, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 8.1931, + 40.4253, + 78.5689, + 113.6462, + 113.1772, + 114.8061, + 111.6615, + 114.2256 + ], + "conc_latency_p99_list": [ + 0.12705827128957028, + 0.1270009018640849, + 0.1321824944659602, + 0.2687498642553692, + 0.38293197758612224, + 0.4930506916076407, + 0.7402389765577391, + 0.9452139304328012 + ], + "conc_latency_avg_list": [ + 0.12204628932652749, + 0.12367472266882223, + 0.12708447563124034, + 0.1756430902825239, + 0.2641998496810496, + 0.3462401548822255, + 0.5327965690087065, + 0.692894005049949 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 160, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.02 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 36925.0192, + "optimize_duration": 6711.6160, + "load_duration": 43636.6352, + "qps": 504.9179, + "serial_latency_p99": 0.2726, + "recall": 0.4664, + "ndcg": 0.5349, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 34.6307, + 162.9753, + 310.0105, + 489.6641, + 498.0772, + 494.8286, + 503.8285, + 504.9179 + ], + "conc_latency_p99_list": [ + 0.272263118980336, + 0.27918726618227085, + 0.29008815855777353, + 0.3707974057542742, + 0.6390006234429894, + 0.6944013835574149, + 0.7231798599578906, + 0.7704306148213801 + ], + "conc_latency_avg_list": [ + 0.028873127393703055, + 0.030547719571582768, + 0.03208098224331353, + 0.040613748396232, + 0.059785429035266714, + 0.07987173318031592, + 0.11724478767697981, + 0.1553268011026498 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 160, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.05 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 36925.0192, + "optimize_duration": 6711.6160, + "load_duration": 43636.6352, + "qps": 1053.1495, + "serial_latency_p99": 0.0177, + "recall": 0.5673, + "ndcg": 0.6299, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 63.0311, + 296.0404, + 556.1192, + 925.1802, + 997.5453, + 1015.5067, + 1053.1495, + 1047.7951 + ], + "conc_latency_p99_list": [ + 0.016600224517169408, + 0.017769198848691305, + 0.022467443438363415, + 0.03778083413519198, + 0.0572584696713602, + 0.0763117267540656, + 0.09910864746081642, + 0.11949356114491826 + ], + "conc_latency_avg_list": [ + 0.015862627148795105, + 0.01688611445815503, + 0.017974171339078256, + 0.021593720323230682, + 0.029951745150563012, + 0.03915111400897159, + 0.056326803647380366, + 0.07507795252838707 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 160, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.1 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 36925.0192, + "optimize_duration": 6711.6160, + "load_duration": 43636.6352, + "qps": 808.3294, + "serial_latency_p99": 0.0222, + "recall": 0.7016, + "ndcg": 0.7471, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 48.0066, + 227.5717, + 434.5289, + 732.8684, + 766.9106, + 797.0652, + 808.3294, + 804.1809 + ], + "conc_latency_p99_list": [ + 0.021631912409793584, + 0.023000416591530667, + 0.03537725956179198, + 0.047619079137803055, + 0.07339557849918492, + 0.08872271321422887, + 0.11852589981979683, + 0.1494658819923643 + ], + "conc_latency_avg_list": [ + 0.020827651459856687, + 0.021967089303425943, + 0.02300247060659166, + 0.027269348111090115, + 0.03896613053807968, + 0.04985159310989647, + 0.0734307873677574, + 0.09797491626179661 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 160, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.2 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 36925.0192, + "optimize_duration": 6711.6160, + "load_duration": 43636.6352, + "qps": 8.0584, + "serial_latency_p99": 1.7579, + "recall": 1.0, + "ndcg": 1.0, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 0.5879, + 3.1567, + 6.1815, + 8.0584, + 7.9634, + 7.8623, + 8.0098, + 8.0318 + ], + "conc_latency_p99_list": [ + 1.749007154185092, + 1.6867517305520596, + 1.7522149846766841, + 3.1832436079136093, + 5.523627587031222, + 7.994091255792592, + 9.914578136393102, + 15.065710022186394 + ], + "conc_latency_avg_list": [ + 1.700821360663718, + 1.5838630229914805, + 1.5959614569047087, + 2.4160799000905744, + 3.6336039522476, + 4.847980393715206, + 7.092948105416856, + 9.450866754826531 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 160, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.5 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 34413.869, + "optimize_duration": 5668.6058, + "load_duration": 40082.4748, + "qps": 3033.5491, + "serial_latency_p99": 0.0064, + "recall": 0.9844, + "ndcg": 0.9867, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 168.9916, + 778.6111, + 1407.6305, + 2408.4799, + 2688.691, + 2908.3108, + 3033.5491, + 3011.2316 + ], + "conc_latency_p99_list": [ + 0.006865841214312243, + 0.007612921928521246, + 0.00899710349040106, + 0.016064837931771717, + 0.024311259835958502, + 0.031335968609200786, + 0.04329377864836715, + 0.0512553511629812 + ], + "conc_latency_avg_list": [ + 0.005915417774878218, + 0.006418791217535814, + 0.00709928154840627, + 0.008294629719645745, + 0.011126658693027796, + 0.013653376659851864, + 0.01948274043745226, + 0.025902595333750545 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-routing-64shard-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 160, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 64, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": true, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.001 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 34413.869, + "optimize_duration": 5668.6058, + "load_duration": 40082.4748, + "qps": 2988.4205, + "serial_latency_p99": 0.0076, + "recall": 0.9741, + "ndcg": 0.9782, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 158.2245, + 753.136, + 1327.8273, + 2332.4539, + 2545.6963, + 2698.8538, + 2967.4722, + 2988.4205 + ], + "conc_latency_p99_list": [ + 0.006750254008802585, + 0.007702123774215579, + 0.008405687906779348, + 0.016145123434253037, + 0.025915445153368636, + 0.03413967659987973, + 0.046689527585404074, + 0.05781707321293652 + ], + "conc_latency_avg_list": [ + 0.006318081589874171, + 0.00663559809690917, + 0.007526238257817682, + 0.008559678216712016, + 0.011745542134694423, + 0.014742117589583496, + 0.0199089358691619, + 0.026080047949598645 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-routing-64shard-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 160, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 64, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": true, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.002 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 34413.869, + "optimize_duration": 5668.6058, + "load_duration": 40082.4748, + "qps": 2950.717, + "serial_latency_p99": 0.0069, + "recall": 0.9558, + "ndcg": 0.9616, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 159.2964, + 728.5607, + 1291.2884, + 2226.059, + 2462.2164, + 2688.4524, + 2887.211, + 2950.717 + ], + "conc_latency_p99_list": [ + 0.006686870129778981, + 0.008058294614893383, + 0.008435492953285575, + 0.017880187905393537, + 0.026330558414920192, + 0.03369499690539674, + 0.047588404216803666, + 0.05904510042164477 + ], + "conc_latency_avg_list": [ + 0.006275652251564352, + 0.006859945017807636, + 0.007737485310123428, + 0.008976250375263266, + 0.012143334537503391, + 0.014762697609008776, + 0.020460857541943458, + 0.026473027478124463 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-routing-64shard-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 160, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 64, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": true, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.005 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 34413.869, + "optimize_duration": 5668.6058, + "load_duration": 40082.4748, + "qps": 2782.0274, + "serial_latency_p99": 0.0074, + "recall": 0.9466, + "ndcg": 0.9538, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 148.7094, + 715.9396, + 1266.473, + 2162.5105, + 2441.679, + 2642.8566, + 2782.0274, + 2781.3658 + ], + "conc_latency_p99_list": [ + 0.007837161368224774, + 0.007938623183872551, + 0.008681609311606733, + 0.01798132527852431, + 0.02592220397898927, + 0.033147562923841134, + 0.05105302116135136, + 0.06536141846445404 + ], + "conc_latency_avg_list": [ + 0.006722579950256255, + 0.006980893145180981, + 0.007891725514940605, + 0.009239785749608735, + 0.012236909761512997, + 0.015020811732499572, + 0.021232629696676646, + 0.027994364598216644 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-routing-64shard-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 160, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 64, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": true, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.01 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 34413.869, + "optimize_duration": 5668.6058, + "load_duration": 40082.4748, + "qps": 2708.6752, + "serial_latency_p99": 0.0084, + "recall": 0.9337, + "ndcg": 0.9421, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 144.2243, + 665.5193, + 1198.5898, + 2076.3524, + 2341.4521, + 2498.9153, + 2644.0053, + 2708.6752 + ], + "conc_latency_p99_list": [ + 0.007411315756617113, + 0.008029354491154665, + 0.009307191087282259, + 0.017992621505982243, + 0.026243340777000385, + 0.03575368172023438, + 0.051434570233686784, + 0.06524198179831725 + ], + "conc_latency_avg_list": [ + 0.006931432060696536, + 0.0075101415226275624, + 0.008335764845439459, + 0.009618119684023764, + 0.012771751563880107, + 0.015891051460812697, + 0.02236659972908588, + 0.02882319588918729 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-routing-64shard-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 160, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 64, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": true, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.02 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 34413.869, + "optimize_duration": 5668.6058, + "load_duration": 40082.4748, + "qps": 2275.2854, + "serial_latency_p99": 0.0091, + "recall": 0.917, + "ndcg": 0.9267, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 119.4811, + 577.5631, + 1075.4255, + 1695.0003, + 1994.0162, + 2139.935, + 2275.2854, + 2023.8564 + ], + "conc_latency_p99_list": [ + 0.008942532020155339, + 0.009683048777515068, + 0.010252964289975353, + 0.020505544719053432, + 0.030148459173506104, + 0.039047053221729536, + 0.06028119866969064, + 0.08413551170320714 + ], + "conc_latency_avg_list": [ + 0.008367336399134279, + 0.008652874658949426, + 0.009294509635623824, + 0.011784097668778842, + 0.014984572026232037, + 0.01856857040379459, + 0.02600065209846948, + 0.03899481642999583 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-routing-64shard-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 160, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 64, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": true, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.05 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 34413.869, + "optimize_duration": 5668.6058, + "load_duration": 40082.4748, + "qps": 1844.8918, + "serial_latency_p99": 0.0106, + "recall": 0.9085, + "ndcg": 0.9185, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 99.688, + 474.7716, + 879.1999, + 1493.8525, + 1668.8237, + 1767.742, + 1774.0898, + 1844.8918 + ], + "conc_latency_p99_list": [ + 0.011324520478956398, + 0.011186246748547998, + 0.01208984132390469, + 0.02323826192645357, + 0.0341993449802976, + 0.0452169142634375, + 0.07676567180315033, + 0.10037542200414462 + ], + "conc_latency_avg_list": [ + 0.010028773400036103, + 0.010528362089527458, + 0.011367270002728599, + 0.013378370834629099, + 0.01791746213567045, + 0.02247262808143572, + 0.03336734442166666, + 0.042408404063917 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-routing-64shard-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 160, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 64, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": true, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.1 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 34413.869, + "optimize_duration": 5668.6058, + "load_duration": 40082.4748, + "qps": 1301.4102, + "serial_latency_p99": 0.0147, + "recall": 0.9011, + "ndcg": 0.9123, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 71.6666, + 355.1443, + 669.3072, + 1119.3356, + 1252.8855, + 1270.9173, + 1301.4102, + 1283.7962 + ], + "conc_latency_p99_list": [ + 0.015162544004851952, + 0.014791368388105184, + 0.016686693376395854, + 0.03165059352177193, + 0.04592963926261291, + 0.0642598616215398, + 0.0869210927630775, + 0.11044685186352565 + ], + "conc_latency_avg_list": [ + 0.013951069921286936, + 0.014075315946713003, + 0.014930788658576543, + 0.017853541387592296, + 0.023848382964513345, + 0.031257849409901665, + 0.04551228368447557, + 0.06103752377796862 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-routing-64shard-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 160, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 64, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": true, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.2 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 34413.869, + "optimize_duration": 5668.6058, + "load_duration": 40082.4748, + "qps": 13.0379, + "serial_latency_p99": 1.0635, + "recall": 1.0, + "ndcg": 1.0, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 1.0187, + 5.1278, + 9.7957, + 13.0379, + 12.9486, + 12.9361, + 12.9199, + 12.9226 + ], + "conc_latency_p99_list": [ + 1.0420157980057412, + 1.0607401755137835, + 1.1162800249876454, + 2.1852373406756667, + 3.350393191084731, + 4.317272136004176, + 6.257540527207311, + 9.138370136335725 + ], + "conc_latency_avg_list": [ + 0.9815727771962092, + 0.9604360580388691, + 1.0173465720603345, + 1.506102410791395, + 2.2523986929141033, + 3.00999245447352, + 4.483657312179465, + 5.949204422428237 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g-routing-64shard-force_merge", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 160, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 64, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": true, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.5 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 10585.000578920008, + "optimize_duration": 6258.889497272961, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [], + "conc_qps_list": [], + "conc_latency_p99_list": [], + "conc_latency_avg_list": [], + "st_ideal_insert_duration": 10000, + "st_search_stage_list": [ + 10, + 20, + 30, + 40, + 50, + 60, + 70, + 80, + 90, + 100, + 110 + ], + "st_search_time_list": [ + 1005.4936, + 2019.661, + 3027.4571, + 4131.5908, + 5177.6757, + 6248.6686, + 7348.6694, + 8408.3537, + 9511.2749, + 10586.6527, + 17464.0694 + ], + "st_max_qps_list_list": [ + 235.8332, + 228.4368, + 204.5192, + 184.9172, + 182.9547, + 154.5351, + 153.4711, + 136.2799, + 149.7168, + 253.7409, + 1243.9365 + ], + "st_recall_list": [ + 0.0984, + 0.198, + 0.2939, + 0.3862, + 0.4831, + 0.5793, + 0.6779, + 0.7766, + 0.8772, + 0.9732, + 0.9036 + ], + "st_ndcg_list": [ + 0.0984, + 0.1977, + 0.2941, + 0.3864, + 0.4835, + 0.5796, + 0.6778, + 0.7762, + 0.8761, + 0.9713, + 0.9158 + ], + "st_serial_latency_p99_list": [ + 0.0277, + 0.0628, + 0.0826, + 0.0634, + 0.0689, + 0.0677, + 0.0883, + 0.0685, + 0.098, + 0.0555, + 0.0139 + ], + "st_conc_failed_rate_list": [ + 0.19461695282763172, + 0.0, + 0.3678012137673471, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.20009038133972956, + 0.0, + 0.0 + ] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 160, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 200, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "search_stages": [ + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "concurrencies": [ + 5, + 10, + 15 + ], + "insert_rate": 1000, + "read_dur_after_write": 180 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 20000.11579315999, + "optimize_duration": 6260.046561305993, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [], + "conc_qps_list": [], + "conc_latency_p99_list": [], + "conc_latency_avg_list": [], + "st_ideal_insert_duration": 20000, + "st_search_stage_list": [ + 10, + 20, + 30, + 40, + 50, + 60, + 70, + 80, + 90, + 100, + 110 + ], + "st_search_time_list": [ + 2001.1743, + 4001.1558, + 6001.1575, + 8002.0661, + 10001.182, + 12001.1748, + 14001.1779, + 16008.1445, + 18006.6904, + 20005.8712, + 26881.4204 + ], + "st_max_qps_list_list": [ + 305.8192, + 268.4947, + 246.2895, + 212.1724, + 211.6411, + 188.0931, + 182.9401, + 173.6893, + 161.6694, + 193.3462, + 1199.9692 + ], + "st_recall_list": [ + 0.0981, + 0.1975, + 0.2935, + 0.3875, + 0.4838, + 0.5807, + 0.6775, + 0.7774, + 0.8772, + 0.9734, + 0.9069 + ], + "st_ndcg_list": [ + 0.0982, + 0.1975, + 0.2939, + 0.388, + 0.4845, + 0.5812, + 0.6774, + 0.7773, + 0.876, + 0.9716, + 0.9191 + ], + "st_serial_latency_p99_list": [ + 0.0228, + 0.0469, + 0.0436, + 0.0521, + 0.0404, + 0.071, + 0.0485, + 0.0509, + 0.052, + 0.0544, + 0.0137 + ], + "st_conc_failed_rate_list": [ + 0.03352009696780883, + 0.03585615257084998, + 0.0, + 0.18768886814239735, + 0.0, + 0.0, + 0.0, + 0.24673846153846155, + 0.22096203409601425, + 0.0, + 0.0 + ] + }, + "task_config": { + "db": "OpenSearch", + "db_config": { + "db_label": "16c128g", + "version": "2.17", + "note": "", + "host": "***", + "port": 443, + "user": "***", + "password": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "engine": "faiss", + "efConstruction": 256, + "efSearch": 160, + "M": 16, + "index_thread_qty": 4, + "number_of_shards": 1, + "number_of_replicas": 0, + "number_of_segments": 1, + "refresh_interval": "30s", + "force_merge_enabled": true, + "flush_threshold_size": "5120mb", + "number_of_indexing_clients": 1, + "index_thread_qty_during_force_merge": 8, + "cb_threshold": "50%", + "use_routing": false, + "use_quant": false, + "oversample_factor": 1.0 + }, + "case_config": { + "case_id": 200, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "search_stages": [ + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "concurrencies": [ + 5, + 10, + 15 + ], + "insert_rate": 500, + "read_dur_after_write": 180 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + } + ], + "file_fmt": "result_{}_{}_{}.json", + "timestamp": 1740355200.0 +} \ No newline at end of file diff --git a/vectordb_bench/results/Pinecone/result_20250124_standard_pinecone.json b/vectordb_bench/results/Pinecone/result_20250124_standard_pinecone.json new file mode 100644 index 000000000..dfee51193 --- /dev/null +++ b/vectordb_bench/results/Pinecone/result_20250124_standard_pinecone.json @@ -0,0 +1,2365 @@ +{ + "run_id": "d60af9965dec47739a5924d55bd3ca8d", + "task_label": "standard_2025", + "results": [ + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 2126.2513, + "qps": 1146.5286, + "serial_latency_p99": 0.0137, + "recall": 0.9262, + "ndcg": 0.9375, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 89.7103, + 451.0448, + 800.7429, + 1146.5286, + 1127.2749, + 1100.9547, + 1090.3778, + 1093.5092 + ], + "conc_latency_p99_list": [ + 0.013791601013508625, + 0.015605529843596738, + 0.016484409999102355, + 0.031263737997505814, + 0.057778209032840054, + 0.06871087484061714, + 0.09453640032414101, + 0.12561580335852338 + ], + "conc_latency_avg_list": [ + 0.011108749377182344, + 0.011024819903126834, + 0.012355533754605541, + 0.01724225393612859, + 0.026232689714905187, + 0.0333120584177395, + 0.05166847654314772, + 0.07077118145617282 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x8-1node", + "version": "v2025.1", + "note": "", + "api_key": "**********", + "index_name": "***" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 2126.2513, + "qps": 1148.1735, + "serial_latency_p99": 0.0089, + "recall": 0.9801, + "ndcg": 0.9856, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 118.4761, + 629.4662, + 1003.6994, + 1148.1735, + 1134.485, + 1106.5084, + 1057.0022, + 1095.6968 + ], + "conc_latency_p99_list": [ + 0.010198568325722595, + 0.011580736243049615, + 0.012750407813873606, + 0.035005421918758634, + 0.0488859723976929, + 0.06385278060872217, + 0.09591716366281611, + 0.12398869160097085 + ], + "conc_latency_avg_list": [ + 0.008404164337577874, + 0.007886417234427203, + 0.009873810345217133, + 0.017262488735400622, + 0.026062020521209463, + 0.03509230240574552, + 0.05255895479978243, + 0.07044940903706345 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x8-1node", + "version": "v2025.1", + "note": "", + "api_key": "**********", + "index_name": "***" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.001 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 2126.2513, + "qps": 1149.1219, + "serial_latency_p99": 0.0103, + "recall": 0.9764, + "ndcg": 0.9829, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 122.1141, + 599.9108, + 992.8201, + 1149.1219, + 1127.7954, + 1127.6137, + 1109.5524, + 1094.059 + ], + "conc_latency_p99_list": [ + 0.010228544765122935, + 0.012254990946530599, + 0.013002205097582181, + 0.03493648969888454, + 0.051992585002153695, + 0.06399628202518218, + 0.09365851940048733, + 0.1258584584266646 + ], + "conc_latency_avg_list": [ + 0.0081461877431091, + 0.008292232734199538, + 0.009986036937053603, + 0.017203606093717096, + 0.026305949480317403, + 0.03492611695088149, + 0.052883698617605966, + 0.07113856674401632 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x8-1node", + "version": "v2025.1", + "note": "", + "api_key": "**********", + "index_name": "***" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.002 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 2126.2513, + "qps": 1140.4099, + "serial_latency_p99": 0.0135, + "recall": 0.9716, + "ndcg": 0.9795, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 91.0535, + 511.9188, + 895.68, + 1140.4099, + 1137.9673, + 1111.8374, + 1104.0823, + 1094.6803 + ], + "conc_latency_p99_list": [ + 0.012947398360993246, + 0.014624544030375547, + 0.015493546993820922, + 0.03511927481769818, + 0.0549006280042522, + 0.06538223753042983, + 0.09603407779533879, + 0.12705528265360044 + ], + "conc_latency_avg_list": [ + 0.010938345796112522, + 0.009713978906251959, + 0.011060514624400991, + 0.017303961415158265, + 0.02605609118725089, + 0.035318296011843484, + 0.05268246433266521, + 0.07052755559466646 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x8-1node", + "version": "v2025.1", + "note": "", + "api_key": "**********", + "index_name": "***" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.005 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 2126.2513, + "qps": 1123.5147, + "serial_latency_p99": 0.0185, + "recall": 0.9688, + "ndcg": 0.9774, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 65.4827, + 335.3104, + 574.1917, + 1094.9839, + 1076.2269, + 1123.5147, + 1103.555, + 1096.3059 + ], + "conc_latency_p99_list": [ + 0.016914594155969097, + 0.019950488850008698, + 0.021503399780340264, + 0.03213390766904922, + 0.05303824923801584, + 0.06920307081076317, + 0.10063651869277243, + 0.12733303321176206 + ], + "conc_latency_avg_list": [ + 0.015205357505339415, + 0.014773703509475542, + 0.017270813409320684, + 0.018030726756816676, + 0.02745925030449534, + 0.034960502325556767, + 0.05308033917166618, + 0.07048289867591259 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x8-1node", + "version": "v2025.1", + "note": "", + "api_key": "**********", + "index_name": "***" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.01 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 2126.2513, + "qps": 487.8343, + "serial_latency_p99": 0.0254, + "recall": 0.9668, + "ndcg": 0.9759, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 40.5157, + 207.5031, + 386.2551, + 475.1791, + 426.7192, + 433.6369, + 437.2957, + 487.8343 + ], + "conc_latency_p99_list": [ + 0.02630955680506304, + 0.027984925381169888, + 0.03076737125229556, + 0.054454336302878804, + 0.15612186648242682, + 0.7413580868815188, + 0.7662904710927975, + 0.2712144359975355 + ], + "conc_latency_avg_list": [ + 0.024577685126248767, + 0.02396539893233684, + 0.02569801655532319, + 0.04173864294326265, + 0.06936145049048321, + 0.09078064205038794, + 0.13272548683380164, + 0.15918668983699333 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x8-1node", + "version": "v2025.1", + "note": "", + "api_key": "**********", + "index_name": "***" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.02 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 2126.2513, + "qps": 264.9324, + "serial_latency_p99": 0.0496, + "recall": 0.936, + "ndcg": 0.9483, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 29.6317, + 148.7753, + 264.9324, + 249.6521, + 241.4427, + 241.4529, + 240.9326, + 252.3715 + ], + "conc_latency_p99_list": [ + 0.047371959277661524, + 0.050403122927527834, + 0.05848660764226224, + 0.11839474710359522, + 0.7284840404131682, + 0.7582129785086726, + 0.8299701955541969, + 0.8885020964744035 + ], + "conc_latency_avg_list": [ + 0.03361354643990148, + 0.03342152088152943, + 0.037538623392592466, + 0.0793943308500512, + 0.12274700045002969, + 0.16313217267037464, + 0.2431635238924095, + 0.30585054528823524 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x8-1node", + "version": "v2025.1", + "note": "", + "api_key": "**********", + "index_name": "***" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.05 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 2126.2513, + "qps": 492.4887, + "serial_latency_p99": 0.0296, + "recall": 0.9269, + "ndcg": 0.9406, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 45.6283, + 228.8224, + 429.0672, + 492.362, + 492.4887, + 492.1122, + 483.3098, + 479.2334 + ], + "conc_latency_p99_list": [ + 0.029842858490883372, + 0.03140155753877477, + 0.03398649209295399, + 0.05796141434111629, + 0.07744239267485681, + 0.09997811100183752, + 0.14843028450690327, + 0.19112906069698504 + ], + "conc_latency_avg_list": [ + 0.02182855403709056, + 0.021738771298434958, + 0.02315883672415145, + 0.0401627410417922, + 0.060190110414398854, + 0.0800248239793686, + 0.12079962916444917, + 0.1626050597608812 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x8-1node", + "version": "v2025.1", + "note": "", + "api_key": "**********", + "index_name": "***" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.1 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 2126.2513, + "qps": 823.1775, + "serial_latency_p99": 0.0205, + "recall": 0.9148, + "ndcg": 0.9295, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 63.9655, + 325.1899, + 596.2744, + 822.2527, + 823.1775, + 815.7257, + 815.4912, + 810.8009 + ], + "conc_latency_p99_list": [ + 0.020400863321847283, + 0.02129151475324759, + 0.023600401997100562, + 0.03618957087208394, + 0.04801011800780543, + 0.061494000008678995, + 0.08658009324790328, + 0.11109914914180989 + ], + "conc_latency_avg_list": [ + 0.015562828481866986, + 0.015247509371216874, + 0.01666730220395721, + 0.02408525876172286, + 0.03596944002898247, + 0.048311288232140044, + 0.07183615920169807, + 0.09593956600052911 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x8-1node", + "version": "v2025.1", + "note": "", + "api_key": "**********", + "index_name": "***" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.2 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 2126.2513, + "qps": 1147.1977, + "serial_latency_p99": 0.0133, + "recall": 0.8999, + "ndcg": 0.9157, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 93.1754, + 440.7722, + 796.7488, + 1147.1977, + 1132.5094, + 1130.2844, + 1110.4012, + 1096.8985 + ], + "conc_latency_p99_list": [ + 0.013215548178413883, + 0.01585558185703121, + 0.01664793356670997, + 0.030344165831920688, + 0.054945265403948715, + 0.06816595857744691, + 0.09562398723370269, + 0.12422144682612266 + ], + "conc_latency_avg_list": [ + 0.010685030243638288, + 0.01128223451835917, + 0.01246204521816795, + 0.017187018085831767, + 0.0261614998043631, + 0.03484493213140737, + 0.05278776547064244, + 0.07062121728866878 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x8-1node", + "version": "v2025.1", + "note": "", + "api_key": "**********", + "index_name": "***" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.5 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 28509.4338, + "qps": 1131.3087, + "serial_latency_p99": 0.0141, + "recall": 0.9024, + "ndcg": 0.9154, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 90.849, + 407.02, + 735.5371, + 1118.1641, + 1131.3087, + 1120.8586, + 1099.0279, + 1057.3003 + ], + "conc_latency_p99_list": [ + 0.013515190919133602, + 0.017234839714365093, + 0.018275173522415568, + 0.02860271738085433, + 0.056761728207493305, + 0.06929460785097043, + 0.097589283852576, + 0.12905132810046782 + ], + "conc_latency_avg_list": [ + 0.01095240891166025, + 0.012221095620562733, + 0.01347222318774823, + 0.01769807080872599, + 0.025982393898555493, + 0.034840590310236876, + 0.05306375182203569, + 0.07060993108546462 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x8-1node", + "version": "v2025.1", + "note": "", + "api_key": "**********", + "index_name": "***" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 28509.4338, + "qps": 1114.952, + "serial_latency_p99": 0.0127, + "recall": 0.97, + "ndcg": 0.9783, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 90.5627, + 437.4478, + 760.7818, + 1106.9575, + 1113.5451, + 1114.952, + 1090.4887, + 1092.3931 + ], + "conc_latency_p99_list": [ + 0.012854363380465656, + 0.014500056750948715, + 0.015446821160840049, + 0.030688926820985216, + 0.05206975059918472, + 0.06555691049230505, + 0.09492760553170229, + 0.12494878920042535 + ], + "conc_latency_avg_list": [ + 0.010506901054991424, + 0.01073385374424058, + 0.012292819722916409, + 0.016879903060319217, + 0.02529760059472378, + 0.03279151763150447, + 0.051204498251696685, + 0.06913170831099856 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x8-1node", + "version": "v2025.1", + "note": "", + "api_key": "**********", + "index_name": "***" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.001 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 28509.4338, + "qps": 583.5009, + "serial_latency_p99": 0.023, + "recall": 0.9668, + "ndcg": 0.9759, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 67.3878, + 263.1014, + 430.7184, + 571.6975, + 583.5009, + 568.4436, + 568.1708, + 572.4906 + ], + "conc_latency_p99_list": [ + 0.021462515089478985, + 0.023866306961608644, + 0.027115660702111206, + 0.04820888617956375, + 0.06530862996038196, + 0.08745708624104737, + 0.12193633456117824, + 0.1545460256014484 + ], + "conc_latency_avg_list": [ + 0.014776227643867547, + 0.017753697074181677, + 0.02213448188138263, + 0.0329851098273663, + 0.05030243790175194, + 0.06737997281617648, + 0.09680808542082442, + 0.13057080247166022 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x8-1node", + "version": "v2025.1", + "note": "", + "api_key": "**********", + "index_name": "***" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.002 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 28509.4338, + "qps": 31.4779, + "serial_latency_p99": 0.351, + "recall": 0.9414, + "ndcg": 0.9541, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 4.3198, + 19.1413, + 30.4599, + 30.4017, + 30.6316, + 30.2655, + 31.4779, + 29.6477 + ], + "conc_latency_p99_list": [ + 0.3619136803019499, + 0.3523363888596213, + 0.4751970390998031, + 0.7914956652001276, + 1.107237294701481, + 1.4272101175198257, + 2.0577846891398073, + 2.7552646648690278 + ], + "conc_latency_avg_list": [ + 0.22943486568687052, + 0.24802064679067656, + 0.3134136860633699, + 0.6188877826036492, + 0.9182841400914843, + 1.2256008265330056, + 1.7779670730124897, + 2.4486907511976583 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x8-1node", + "version": "v2025.1", + "note": "", + "api_key": "**********", + "index_name": "***" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.005 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 28509.4338, + "qps": 57.8988, + "serial_latency_p99": 0.2001, + "recall": 0.9332, + "ndcg": 0.9468, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 7.0749, + 34.8679, + 55.2805, + 56.1467, + 56.665, + 57.8988, + 57.1717, + 56.2749 + ], + "conc_latency_p99_list": [ + 0.21104647180181932, + 0.2108999284995661, + 0.27348768418189134, + 0.4394125210181663, + 0.6318989706414868, + 0.7912053661224491, + 1.1383832841957338, + 1.5200451774567771 + ], + "conc_latency_avg_list": [ + 0.13628724267859302, + 0.13768700175984208, + 0.17218041142733337, + 0.3409596207650808, + 0.4953826850098498, + 0.6516730440882924, + 0.9772891496444811, + 1.3051331751763418 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x8-1node", + "version": "v2025.1", + "note": "", + "api_key": "**********", + "index_name": "***" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.01 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 28509.4338, + "qps": 101.1774, + "serial_latency_p99": 0.1161, + "recall": 0.9241, + "ndcg": 0.9388, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 12.648, + 60.967, + 99.3555, + 97.0975, + 100.2324, + 99.6057, + 99.3196, + 101.1774 + ], + "conc_latency_p99_list": [ + 0.112565766798798, + 0.11971512002324741, + 0.1487955656446866, + 0.2592605054263549, + 0.36047108154816676, + 0.46077777919563234, + 0.6784228897774299, + 0.8538449384335399 + ], + "conc_latency_avg_list": [ + 0.07408682850995668, + 0.07850101477429011, + 0.09643367660443229, + 0.19343726049070478, + 0.27824098990330887, + 0.37553753237614473, + 0.5652041354910056, + 0.7495476608112455 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x8-1node", + "version": "v2025.1", + "note": "", + "api_key": "**********", + "index_name": "***" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.02 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 28509.4338, + "qps": 212.7466, + "serial_latency_p99": 0.0587, + "recall": 0.9099, + "ndcg": 0.9257, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 25.439, + 119.731, + 196.4422, + 212.7466, + 211.5371, + 209.1208, + 209.4669, + 211.1247 + ], + "conc_latency_p99_list": [ + 0.05828394692100118, + 0.0634566752925457, + 0.07846361254138173, + 0.12517219290144568, + 0.17513354650014662, + 0.22177759199665162, + 0.3242466379706457, + 0.4192473373207031 + ], + "conc_latency_avg_list": [ + 0.03905844482848436, + 0.04145640161689432, + 0.048258142588734565, + 0.09264635079011461, + 0.13337490028865584, + 0.1819488868174765, + 0.2705504307427727, + 0.3517622626939955 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x8-1node", + "version": "v2025.1", + "note": "", + "api_key": "**********", + "index_name": "***" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.05 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 28509.4338, + "qps": 372.2462, + "serial_latency_p99": 0.0359, + "recall": 0.8977, + "ndcg": 0.9144, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 40.3583, + 156.8383, + 325.9717, + 356.9097, + 368.8438, + 372.2462, + 367.3872, + 356.9887 + ], + "conc_latency_p99_list": [ + 0.035145813025228546, + 0.03881652459967883, + 0.04533428478971472, + 0.07258239452348784, + 0.10110301843073102, + 0.12868865168478805, + 0.1909981646368396, + 0.24811056845937857 + ], + "conc_latency_avg_list": [ + 0.02412668327262766, + 0.025689656896213994, + 0.028737803124804118, + 0.05169956989733857, + 0.07589492486475118, + 0.1009001980597061, + 0.15286646664873463, + 0.19985151834115583 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x8-1node", + "version": "v2025.1", + "note": "", + "api_key": "**********", + "index_name": "***" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.1 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 28509.4338, + "qps": 617.0881, + "serial_latency_p99": 0.0224, + "recall": 0.8844, + "ndcg": 0.9025, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 55.7853, + 266.7827, + 488.8253, + 607.1061, + 609.3167, + 617.0881, + 614.9136, + 603.6542 + ], + "conc_latency_p99_list": [ + 0.023146135569768377, + 0.025465623447453237, + 0.02846403649033163, + 0.04483356902128431, + 0.06086368759803008, + 0.0785337177220208, + 0.11133191424924009, + 0.1459527112510841 + ], + "conc_latency_avg_list": [ + 0.017103368856267256, + 0.017628148835740175, + 0.019325287352050427, + 0.030972281445919656, + 0.04654549774614595, + 0.0609126179573557, + 0.09111022406320932, + 0.12400019281335911 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x8-1node", + "version": "v2025.1", + "note": "", + "api_key": "**********", + "index_name": "***" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.2 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 28509.4338, + "qps": 1094.5967, + "serial_latency_p99": 0.0143, + "recall": 0.8659, + "ndcg": 0.8854, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 87.2893, + 391.005, + 703.4222, + 1079.1062, + 1088.7975, + 1094.5967, + 1077.4104, + 1065.4778 + ], + "conc_latency_p99_list": [ + 0.013830654038756615, + 0.01712545307993423, + 0.01850793520308798, + 0.025446242557372878, + 0.05512046027812172, + 0.07343901599670064, + 0.10232758972008019, + 0.13014544384568572 + ], + "conc_latency_avg_list": [ + 0.010906762150558113, + 0.012191352217792134, + 0.013591502213806766, + 0.01745915249624072, + 0.025780819081531257, + 0.03341044864251269, + 0.05180792240455562, + 0.06978683498213636 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x8-1node", + "version": "v2025.1", + "note": "", + "api_key": "**********", + "index_name": "***" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.5 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 22825.014884851, + "optimize_duration": 0.03767456900095567, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [], + "conc_qps_list": [], + "conc_latency_p99_list": [], + "conc_latency_avg_list": [], + "st_ideal_insert_duration": 20000, + "st_search_stage_list": [ + 10, + 20, + 30, + 40, + 50, + 60, + 70, + 80, + 90, + 100, + 110 + ], + "st_search_time_list": [ + 2000.7527, + 4164.1309, + 6360.6054, + 8632.6723, + 10934.6608, + 13275.7304, + 15664.5283, + 18059.2308, + 20477.9703, + 22826.2556, + 23005.368 + ], + "st_max_qps_list_list": [ + 349.7492, + 373.5938, + 392.9454, + 379.7178, + 396.0267, + 394.7847, + 381.3392, + 368.2059, + 367.4299, + 1133.86, + 1157.1545 + ], + "st_recall_list": [ + 0.0986, + 0.1962, + 0.2897, + 0.38, + 0.4727, + 0.5663, + 0.6597, + 0.7526, + 0.845, + 0.9023, + 0.9023 + ], + "st_ndcg_list": [ + 0.0987, + 0.1965, + 0.291, + 0.3814, + 0.4753, + 0.5695, + 0.6638, + 0.7583, + 0.8533, + 0.9152, + 0.9152 + ], + "st_serial_latency_p99_list": [ + 0.1627, + 0.186, + 0.1995, + 0.2034, + 0.8869, + 1.299, + 1.5292, + 1.5506, + 1.8286, + 0.2083, + 0.0146 + ], + "st_conc_failed_rate_list": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 2.8953616306676705e-05, + 0.0 + ] + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x8-1node", + "version": "v2025.1", + "note": "", + "api_key": "**********", + "index_name": "***" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 200, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "insert_rate": 500, + "search_stages": [ + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "concurrencies": [ + 10, + 15, + 20 + ] + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30, + 35, + 40, + 45, + 50, + 55, + 60, + 65, + 70, + 75, + 80, + 85, + 90, + 95, + 100 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 18419.824247966004, + "optimize_duration": 0.03693918000499252, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [], + "conc_qps_list": [], + "conc_latency_p99_list": [], + "conc_latency_avg_list": [], + "st_ideal_insert_duration": 10000, + "st_search_stage_list": [ + 10, + 20, + 30, + 40, + 50, + 60, + 70, + 80, + 90, + 100, + 110 + ], + "st_search_time_list": [ + 1635.6835, + 3462.1376, + 5305.9346, + 7176.6576, + 9000.4662, + 10845.2049, + 12698.407, + 14569.2157, + 16470.5286, + 18421.0961, + 18749.8044 + ], + "st_max_qps_list_list": [ + 331.1683, + 359.9183, + 391.4605, + 396.0265, + 392.1749, + 399.3687, + 398.9593, + 398.7458, + 369.6771, + 1092.3705, + 1125.0873 + ], + "st_recall_list": [ + 0.1007, + 0.2042, + 0.3004, + 0.3915, + 0.4826, + 0.5735, + 0.6647, + 0.7572, + 0.8451, + 0.9025, + 0.9028 + ], + "st_ndcg_list": [ + 0.1007, + 0.2045, + 0.3016, + 0.3931, + 0.4851, + 0.5768, + 0.6688, + 0.763, + 0.8532, + 0.9154, + 0.9157 + ], + "st_serial_latency_p99_list": [ + 0.315, + 1.1674, + 5.0472, + 1.4148, + 2.3238, + 3.68, + 4.1368, + 5.7254, + 5.992, + 6.9972, + 0.014 + ], + "st_conc_failed_rate_list": [ + 0.0, + 0.0, + 0.0, + 1.642764444006374e-05, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "task_config": { + "db": "Pinecone", + "db_config": { + "db_label": "p2.x8-1node", + "version": "v2025.1", + "note": "", + "api_key": "**********", + "index_name": "***" + }, + "db_case_config": { + "null": null + }, + "case_config": { + "case_id": 200, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "insert_rate": 1000, + "search_stages": [ + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "concurrencies": [ + 10, + 15, + 20 + ] + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + } + ], + "file_fmt": "result_{}_{}_{}.json", + "timestamp": 1737676800.0 +} \ No newline at end of file diff --git a/vectordb_bench/results/QdrantCloud/result_20250602_standard_qdrantcloud.json b/vectordb_bench/results/QdrantCloud/result_20250602_standard_qdrantcloud.json new file mode 100644 index 000000000..affb2eb97 --- /dev/null +++ b/vectordb_bench/results/QdrantCloud/result_20250602_standard_qdrantcloud.json @@ -0,0 +1,3556 @@ +{ + "run_id": "d60af9965dec47739a5924d55bd3ca8d", + "task_label": "standard_2025", + "results": [ + { + "metrics": { + "max_load_count": 0, + "insert_duration": 1319.08, + "optimize_duration": 20.1262, + "load_duration": 1339.2062, + "qps": 4318.9697, + "serial_latency_p99": 0.0043, + "recall": 1.0, + "ndcg": 1.0, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 293.2625, + 1418.2237, + 2503.3143, + 4040.6711, + 4318.9697, + 4231.1635, + 3968.657, + 3873.9566 + ], + "conc_latency_p99_list": [ + 0.004108009400451913, + 0.00469942291992993, + 0.005965515149819113, + 0.009690882539725878, + 0.015310119479809219, + 0.019062151899652286, + 0.02478109099949506, + 0.030745067499719874 + ], + "conc_latency_avg_list": [ + 0.0033968326940370785, + 0.003504317851431097, + 0.003958530295749986, + 0.004864683437812893, + 0.006765798381240783, + 0.009086413973290202, + 0.014139072129676184, + 0.018795127060711247 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "16c64g-payload-index", + "version": "1.14.1", + "note": "", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "m": 16, + "payload_m": 16, + "create_payload_int_index": false, + "create_payload_keyword_index": true, + "is_tenant": false, + "use_scalar_quant": false, + "sq_quantile": 0.99, + "default_segment_number": 0, + "use_rescore": false, + "oversampling": 1.0, + "indexed_only": false, + "hnsw_ef": 100, + "exact": false, + "with_payload": false + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.001 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 1319.08, + "optimize_duration": 20.1262, + "load_duration": 1339.2062, + "qps": 4250.2894, + "serial_latency_p99": 0.0046, + "recall": 1.0, + "ndcg": 1.0, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 277.4786, + 1318.99, + 2393.7592, + 3833.7677, + 4250.2894, + 4184.1166, + 4000.5926, + 3862.6429 + ], + "conc_latency_p99_list": [ + 0.004508941039603082, + 0.004984925999924594, + 0.006595623899847849, + 0.010582116120258365, + 0.01574212464009181, + 0.018370582070228906, + 0.02465197935991454, + 0.030694995440426295 + ], + "conc_latency_avg_list": [ + 0.0035900876433641237, + 0.003769701729715891, + 0.004140400060134759, + 0.005126895276394362, + 0.006872501639850592, + 0.00918103882152727, + 0.014044806699768219, + 0.01873456016794685 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "16c64g-payload-index", + "version": "1.14.1", + "note": "", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "m": 16, + "payload_m": 16, + "create_payload_int_index": false, + "create_payload_keyword_index": true, + "is_tenant": false, + "use_scalar_quant": false, + "sq_quantile": 0.99, + "default_segment_number": 0, + "use_rescore": false, + "oversampling": 1.0, + "indexed_only": false, + "hnsw_ef": 100, + "exact": false, + "with_payload": false + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.002 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 1319.08, + "optimize_duration": 20.1262, + "load_duration": 1339.2062, + "qps": 2997.4391, + "serial_latency_p99": 0.0061, + "recall": 1.0, + "ndcg": 1.0, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 259.2959, + 1178.8468, + 2018.6883, + 2490.6234, + 2893.8137, + 2996.8477, + 2988.859, + 2997.4391 + ], + "conc_latency_p99_list": [ + 0.004596146079784376, + 0.005777928030111063, + 0.00855043300963189, + 0.015129041220061491, + 0.019837499320237838, + 0.02368464209943336, + 0.03564275155032192, + 0.046053308929758716 + ], + "conc_latency_avg_list": [ + 0.003842165242025026, + 0.004217991263362464, + 0.00491051189097537, + 0.007205748604411966, + 0.010093602282638538, + 0.01287320282645513, + 0.018831706458738242, + 0.024423919263035544 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "16c64g-payload-index", + "version": "1.14.1", + "note": "", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "m": 16, + "payload_m": 16, + "create_payload_int_index": false, + "create_payload_keyword_index": true, + "is_tenant": false, + "use_scalar_quant": false, + "sq_quantile": 0.99, + "default_segment_number": 0, + "use_rescore": false, + "oversampling": 1.0, + "indexed_only": false, + "hnsw_ef": 100, + "exact": false, + "with_payload": false + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.005 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 1319.08, + "optimize_duration": 20.1262, + "load_duration": 1339.2062, + "qps": 1494.5334, + "serial_latency_p99": 0.007, + "recall": 1.0, + "ndcg": 1.0, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 203.6708, + 969.6341, + 1428.3563, + 1486.8859, + 1494.5334, + 1483.2116, + 1474.5178, + 1368.0349 + ], + "conc_latency_p99_list": [ + 0.005929753999680542, + 0.007170458190130376, + 0.013735425040104018, + 0.021000384040271457, + 0.0289086314002634, + 0.0352984104501047, + 0.04928370412053478, + 0.062043446079951534 + ], + "conc_latency_avg_list": [ + 0.004891689839344045, + 0.005125268532403679, + 0.006941333195860831, + 0.01323872120834617, + 0.019574734836527944, + 0.026080113358016896, + 0.03856528288604421, + 0.05103612621286848 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "16c64g-payload-index", + "version": "1.14.1", + "note": "", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "m": 16, + "payload_m": 16, + "create_payload_int_index": false, + "create_payload_keyword_index": true, + "is_tenant": false, + "use_scalar_quant": false, + "sq_quantile": 0.99, + "default_segment_number": 0, + "use_rescore": false, + "oversampling": 1.0, + "indexed_only": false, + "hnsw_ef": 100, + "exact": false, + "with_payload": false + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.01 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 1319.08, + "optimize_duration": 20.1262, + "load_duration": 1339.2062, + "qps": 1108.6473, + "serial_latency_p99": 0.0074, + "recall": 0.995, + "ndcg": 0.9953, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 155.0156, + 759.7266, + 1098.4745, + 1108.6473, + 1107.189, + 1010.1489, + 1100.3861, + 1007.2433 + ], + "conc_latency_p99_list": [ + 0.00785500895030964, + 0.008967568600201054, + 0.015656784659768146, + 0.028236765000110544, + 0.03794425167947339, + 0.04669991659957308, + 0.06410694274995876, + 0.08198375469952227 + ], + "conc_latency_avg_list": [ + 0.006427684545643792, + 0.006545558126009102, + 0.009028796154110398, + 0.017762212212337446, + 0.0264548593750186, + 0.03507294947193464, + 0.051781498749312135, + 0.06840506317045031 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "16c64g-payload-index", + "version": "1.14.1", + "note": "", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "m": 16, + "payload_m": 16, + "create_payload_int_index": false, + "create_payload_keyword_index": true, + "is_tenant": false, + "use_scalar_quant": false, + "sq_quantile": 0.99, + "default_segment_number": 0, + "use_rescore": false, + "oversampling": 1.0, + "indexed_only": false, + "hnsw_ef": 100, + "exact": false, + "with_payload": false + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.02 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 1319.08, + "optimize_duration": 20.1262, + "load_duration": 1339.2062, + "qps": 1289.5164, + "serial_latency_p99": 0.0064, + "recall": 0.9906, + "ndcg": 0.9914, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 192.663, + 894.383, + 1260.0815, + 1289.5164, + 1287.7963, + 1175.2373, + 1168.3505, + 1162.7818 + ], + "conc_latency_p99_list": [ + 0.006316989139695581, + 0.008128716850251365, + 0.015395553400521745, + 0.026328772799861355, + 0.03552735675981239, + 0.04340343913015204, + 0.05868663019987247, + 0.07328483299988875 + ], + "conc_latency_avg_list": [ + 0.005170983138963148, + 0.0055568424552364356, + 0.00787051162470654, + 0.015257475040559534, + 0.022746841193685464, + 0.030291182935339646, + 0.04500688047272738, + 0.058709776506761735 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "16c64g-payload-index", + "version": "1.14.1", + "note": "", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "m": 16, + "payload_m": 16, + "create_payload_int_index": false, + "create_payload_keyword_index": true, + "is_tenant": false, + "use_scalar_quant": false, + "sq_quantile": 0.99, + "default_segment_number": 0, + "use_rescore": false, + "oversampling": 1.0, + "indexed_only": false, + "hnsw_ef": 100, + "exact": false, + "with_payload": false + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.05 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 1319.08, + "optimize_duration": 20.1262, + "load_duration": 1339.2062, + "qps": 1059.3394, + "serial_latency_p99": 0.0078, + "recall": 0.9856, + "ndcg": 0.9865, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 182.0687, + 817.4502, + 1050.4022, + 1059.3394, + 1057.7988, + 1027.603, + 963.131, + 959.5137 + ], + "conc_latency_p99_list": [ + 0.006470868200085524, + 0.008868805800193518, + 0.017504915279787377, + 0.029769426550319604, + 0.03973150839992743, + 0.04859697681950819, + 0.06808674165962657, + 0.08627920324966 + ], + "conc_latency_avg_list": [ + 0.005472145389340478, + 0.006080082797610136, + 0.009440536055800997, + 0.018597708752576308, + 0.027687777141859785, + 0.036729990492687103, + 0.05458301345296143, + 0.07152907761295225 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "16c64g-payload-index", + "version": "1.14.1", + "note": "", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "m": 16, + "payload_m": 16, + "create_payload_int_index": false, + "create_payload_keyword_index": true, + "is_tenant": false, + "use_scalar_quant": false, + "sq_quantile": 0.99, + "default_segment_number": 0, + "use_rescore": false, + "oversampling": 1.0, + "indexed_only": false, + "hnsw_ef": 100, + "exact": false, + "with_payload": false + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.1 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 1319.08, + "optimize_duration": 20.1262, + "load_duration": 1339.2062, + "qps": 987.0795, + "serial_latency_p99": 0.0071, + "recall": 0.9804, + "ndcg": 0.9814, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 170.2351, + 799.1863, + 976.5353, + 987.0795, + 982.7753, + 896.5894, + 971.9598, + 883.3496 + ], + "conc_latency_p99_list": [ + 0.007034681559271121, + 0.00902814224973554, + 0.018402551080162078, + 0.031121550620100604, + 0.04181672295933825, + 0.051991278479690664, + 0.07187199053005315, + 0.09237229285055035 + ], + "conc_latency_avg_list": [ + 0.005852643577169901, + 0.006221069343522768, + 0.010157681946698735, + 0.019942006644400066, + 0.0298103925036599, + 0.039504880680964735, + 0.058654024613022694, + 0.077525677224489 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "16c64g-payload-index", + "version": "1.14.1", + "note": "", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "m": 16, + "payload_m": 16, + "create_payload_int_index": false, + "create_payload_keyword_index": true, + "is_tenant": false, + "use_scalar_quant": false, + "sq_quantile": 0.99, + "default_segment_number": 0, + "use_rescore": false, + "oversampling": 1.0, + "indexed_only": false, + "hnsw_ef": 100, + "exact": false, + "with_payload": false + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.2 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 1319.08, + "optimize_duration": 20.1262, + "load_duration": 1339.2062, + "qps": 1591.7055, + "serial_latency_p99": 0.0078, + "recall": 0.8506, + "ndcg": 0.8538, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 204.7748, + 985.4012, + 1468.5406, + 1591.7055, + 1457.7184, + 1590.3698, + 1489.2644, + 1454.2574 + ], + "conc_latency_p99_list": [ + 0.006079022370122401, + 0.007035757980193004, + 0.01645851233961361, + 0.02284473285999411, + 0.03139393492060659, + 0.03826638910008112, + 0.05060729249998985, + 0.06300347028023684 + ], + "conc_latency_avg_list": [ + 0.004865269940793477, + 0.005044995872658895, + 0.0067500134283456316, + 0.012369487155991286, + 0.01833301637215765, + 0.024331188459427163, + 0.0360246762552596, + 0.047445003783414805 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "16c64g-payload-index", + "version": "1.14.1", + "note": "", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "m": 16, + "payload_m": 16, + "create_payload_int_index": false, + "create_payload_keyword_index": true, + "is_tenant": false, + "use_scalar_quant": false, + "sq_quantile": 0.99, + "default_segment_number": 0, + "use_rescore": false, + "oversampling": 1.0, + "indexed_only": false, + "hnsw_ef": 100, + "exact": false, + "with_payload": false + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.5 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 14293.8033, + "optimize_duration": 15.235, + "load_duration": 14309.0383, + "qps": 1202.8677, + "serial_latency_p99": 0.007, + "recall": 1.0, + "ndcg": 1.0, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 206.967, + 911.4194, + 1148.5274, + 1100.4431, + 1119.6533, + 1110.5234, + 1201.3445, + 1202.8677 + ], + "conc_latency_p99_list": [ + 0.005655336518393597, + 0.008416454441903623, + 0.021868546077021246, + 0.028908773838775235, + 0.03925096478807966, + 0.04731271557880973, + 0.06442837212060115, + 0.08021722926117947 + ], + "conc_latency_avg_list": [ + 0.004814049540212592, + 0.005456099164853924, + 0.008637005004549426, + 0.0163061362063892, + 0.024269634272066843, + 0.03209051234716432, + 0.047439863532524634, + 0.06192339823549479 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "16c64g-payload-index", + "version": "1.14.1", + "note": "", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "m": 16, + "payload_m": 16, + "create_payload_int_index": false, + "create_payload_keyword_index": true, + "is_tenant": false, + "use_scalar_quant": false, + "sq_quantile": 0.99, + "default_segment_number": 0, + "use_rescore": false, + "oversampling": 1.0, + "indexed_only": false, + "hnsw_ef": 100, + "exact": false, + "with_payload": false + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.001 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 14293.8033, + "optimize_duration": 15.235, + "load_duration": 14309.0383, + "qps": 639.3991, + "serial_latency_p99": 0.0073, + "recall": 1.0, + "ndcg": 1.0, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 167.8168, + 629.6706, + 636.3062, + 639.3991, + 635.9869, + 585.4089, + 633.1348, + 631.6696 + ], + "conc_latency_p99_list": [ + 0.006691822270004194, + 0.014931110736870323, + 0.025915584146059694, + 0.041898286758514605, + 0.05745977048325586, + 0.07294807943864724, + 0.10194685441900218, + 0.1351531705029629 + ], + "conc_latency_avg_list": [ + 0.0059370943387183715, + 0.007896892117463701, + 0.015588208636262504, + 0.03081194091131072, + 0.046093567302827605, + 0.06091807750021593, + 0.09030891374826104, + 0.11862098875833996 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "16c64g-payload-index", + "version": "1.14.1", + "note": "", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "m": 16, + "payload_m": 16, + "create_payload_int_index": false, + "create_payload_keyword_index": true, + "is_tenant": false, + "use_scalar_quant": false, + "sq_quantile": 0.99, + "default_segment_number": 0, + "use_rescore": false, + "oversampling": 1.0, + "indexed_only": false, + "hnsw_ef": 100, + "exact": false, + "with_payload": false + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.002 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 14293.8033, + "optimize_duration": 15.235, + "load_duration": 14309.0383, + "qps": 274.8559, + "serial_latency_p99": 0.0099, + "recall": 1.0, + "ndcg": 1.0, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 111.8443, + 274.8559, + 273.919, + 273.2946, + 248.1112, + 272.0351, + 248.232, + 251.5627 + ], + "conc_latency_p99_list": [ + 0.010098347063612897, + 0.02121279940183743, + 0.03981989816180431, + 0.07721344975107058, + 0.11303051400172993, + 0.15008354073906957, + 0.22340935355983674, + 0.29631453135276387 + ], + "conc_latency_avg_list": [ + 0.008908439349240458, + 0.01809537173827183, + 0.036229195362535445, + 0.07212283733843941, + 0.10802966199286358, + 0.14274027652628404, + 0.2124717999302204, + 0.279949356427757 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "16c64g-payload-index", + "version": "1.14.1", + "note": "", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "m": 16, + "payload_m": 16, + "create_payload_int_index": false, + "create_payload_keyword_index": true, + "is_tenant": false, + "use_scalar_quant": false, + "sq_quantile": 0.99, + "default_segment_number": 0, + "use_rescore": false, + "oversampling": 1.0, + "indexed_only": false, + "hnsw_ef": 100, + "exact": false, + "with_payload": false + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.005 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 14293.8033, + "optimize_duration": 15.235, + "load_duration": 14309.0383, + "qps": 441.4152, + "serial_latency_p99": 0.0083, + "recall": 0.997, + "ndcg": 0.9971, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 143.4669, + 436.254, + 441.4152, + 438.6738, + 399.6966, + 432.7183, + 429.2645, + 394.5522 + ], + "conc_latency_p99_list": [ + 0.008762944238405885, + 0.01626307624981564, + 0.028729068860702648, + 0.05270900674076984, + 0.07277014915714972, + 0.09666912357643015, + 0.1437478300393559, + 0.18816683990880848 + ], + "conc_latency_avg_list": [ + 0.006945026401799453, + 0.011399085068879888, + 0.022478999864959583, + 0.04491813725015799, + 0.06726079640580303, + 0.08962663279288086, + 0.1333593012912701, + 0.1745350267776829 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "16c64g-payload-index", + "version": "1.14.1", + "note": "", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "m": 16, + "payload_m": 16, + "create_payload_int_index": false, + "create_payload_keyword_index": true, + "is_tenant": false, + "use_scalar_quant": false, + "sq_quantile": 0.99, + "default_segment_number": 0, + "use_rescore": false, + "oversampling": 1.0, + "indexed_only": false, + "hnsw_ef": 100, + "exact": false, + "with_payload": false + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.01 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 14293.8033, + "optimize_duration": 15.235, + "load_duration": 14309.0383, + "qps": 358.8949, + "serial_latency_p99": 0.0095, + "recall": 0.995, + "ndcg": 0.9951, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 116.5584, + 356.5275, + 358.8949, + 324.8888, + 355.3511, + 351.7406, + 326.7391, + 349.4246 + ], + "conc_latency_p99_list": [ + 0.010250616880657617, + 0.01863548950059339, + 0.0321760157468816, + 0.06016570479914661, + 0.0885519881368236, + 0.11717015530601202, + 0.17205338899948402, + 0.23131503604490716 + ], + "conc_latency_avg_list": [ + 0.008548762044159492, + 0.01394755542425568, + 0.027644554680582297, + 0.05522321678649889, + 0.08252783976879721, + 0.1103059612746877, + 0.16326103678644524, + 0.2149607624005845 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "16c64g-payload-index", + "version": "1.14.1", + "note": "", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "m": 16, + "payload_m": 16, + "create_payload_int_index": false, + "create_payload_keyword_index": true, + "is_tenant": false, + "use_scalar_quant": false, + "sq_quantile": 0.99, + "default_segment_number": 0, + "use_rescore": false, + "oversampling": 1.0, + "indexed_only": false, + "hnsw_ef": 100, + "exact": false, + "with_payload": false + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.02 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 14293.8033, + "optimize_duration": 15.235, + "load_duration": 14309.0383, + "qps": 325.2245, + "serial_latency_p99": 0.0103, + "recall": 0.9909, + "ndcg": 0.9909, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 123.0427, + 325.1471, + 325.2245, + 295.1217, + 292.9007, + 322.4289, + 319.6973, + 320.7764 + ], + "conc_latency_p99_list": [ + 0.00931106200099748, + 0.020588611690036486, + 0.035066349752305555, + 0.06571827707251941, + 0.09726342149588163, + 0.12768566494181868, + 0.1897035929947742, + 0.25184306000272044 + ], + "conc_latency_avg_list": [ + 0.008098515189237565, + 0.01529155639642761, + 0.030518992861499915, + 0.06095409537227186, + 0.0914411181375523, + 0.12030570511547133, + 0.17938279938524543, + 0.23454913445770756 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "16c64g-payload-index", + "version": "1.14.1", + "note": "", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "m": 16, + "payload_m": 16, + "create_payload_int_index": false, + "create_payload_keyword_index": true, + "is_tenant": false, + "use_scalar_quant": false, + "sq_quantile": 0.99, + "default_segment_number": 0, + "use_rescore": false, + "oversampling": 1.0, + "indexed_only": false, + "hnsw_ef": 100, + "exact": false, + "with_payload": false + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.05 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 14293.8033, + "optimize_duration": 15.235, + "load_duration": 14309.0383, + "qps": 273.4174, + "serial_latency_p99": 0.0133, + "recall": 0.9789, + "ndcg": 0.9787, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 110.8047, + 272.4043, + 273.4174, + 271.6536, + 247.8823, + 270.479, + 244.699, + 246.9894 + ], + "conc_latency_p99_list": [ + 0.010369178601467872, + 0.023040348696231366, + 0.04003402912028832, + 0.07837427703983849, + 0.11477441169990926, + 0.15143905847704442, + 0.22774383698764722, + 0.3014751199996681 + ], + "conc_latency_avg_list": [ + 0.00899149234024069, + 0.018262957926889967, + 0.03629607029711298, + 0.07254300809577825, + 0.10818437217162749, + 0.14350989930487565, + 0.21475331164430142, + 0.28076783016090423 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "16c64g-payload-index", + "version": "1.14.1", + "note": "", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "m": 16, + "payload_m": 16, + "create_payload_int_index": false, + "create_payload_keyword_index": true, + "is_tenant": false, + "use_scalar_quant": false, + "sq_quantile": 0.99, + "default_segment_number": 0, + "use_rescore": false, + "oversampling": 1.0, + "indexed_only": false, + "hnsw_ef": 100, + "exact": false, + "with_payload": false + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.1 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 14293.8033, + "optimize_duration": 15.235, + "load_duration": 14309.0383, + "qps": 262.8314, + "serial_latency_p99": 0.0113, + "recall": 0.9808, + "ndcg": 0.9803, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 108.7174, + 262.1674, + 255.4049, + 262.8314, + 250.187, + 259.8525, + 260.7161, + 244.5148 + ], + "conc_latency_p99_list": [ + 0.01143218348093795, + 0.024378688477445384, + 0.04247373699261517, + 0.08093938276135304, + 0.11894121329678455, + 0.15802012887754244, + 0.23411726899939822, + 0.31060514779637743 + ], + "conc_latency_avg_list": [ + 0.009165488431407798, + 0.018973467987600433, + 0.03771409502210758, + 0.07494889929330883, + 0.1121941421376435, + 0.14947845852134778, + 0.21990355361726233, + 0.2905831009492956 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "16c64g-payload-index", + "version": "1.14.1", + "note": "", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "m": 16, + "payload_m": 16, + "create_payload_int_index": false, + "create_payload_keyword_index": true, + "is_tenant": false, + "use_scalar_quant": false, + "sq_quantile": 0.99, + "default_segment_number": 0, + "use_rescore": false, + "oversampling": 1.0, + "indexed_only": false, + "hnsw_ef": 100, + "exact": false, + "with_payload": false + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.2 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 14293.8033, + "optimize_duration": 15.235, + "load_duration": 14309.0383, + "qps": 434.5481, + "serial_latency_p99": 0.0085, + "recall": 0.7237, + "ndcg": 0.7219, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 146.5507, + 432.7754, + 434.5481, + 432.2032, + 427.358, + 429.1023, + 397.9899, + 420.7208 + ], + "conc_latency_p99_list": [ + 0.008506639694678568, + 0.016457689051821954, + 0.03029727175802691, + 0.05180674914037809, + 0.07415859206215827, + 0.09747324070434843, + 0.144179983921349, + 0.18923682705222744 + ], + "conc_latency_avg_list": [ + 0.006799216841598663, + 0.011487961056909447, + 0.022828979750202868, + 0.04558859892952997, + 0.06864003222227749, + 0.09041309401874563, + 0.13320785231271542, + 0.17517457189081279 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "16c64g-payload-index", + "version": "1.14.1", + "note": "", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "m": 16, + "payload_m": 16, + "create_payload_int_index": false, + "create_payload_keyword_index": true, + "is_tenant": false, + "use_scalar_quant": false, + "sq_quantile": 0.99, + "default_segment_number": 0, + "use_rescore": false, + "oversampling": 1.0, + "indexed_only": false, + "hnsw_ef": 100, + "exact": false, + "with_payload": false + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.5 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 15697.2116, + "optimize_duration": 25.248, + "load_duration": 15722.4597, + "qps": 446.9116, + "serial_latency_p99": 0.0092, + "recall": 0.9357, + "ndcg": 0.9335, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 142.7501, + 443.3815, + 446.9116, + 443.2777, + 441.5765, + 401.0699, + 402.4964, + 398.8342 + ], + "conc_latency_p99_list": [ + 0.008392416880233212, + 0.01584005208598683, + 0.028442546800943082, + 0.05118929252625094, + 0.07514514699869324, + 0.09705145264073507, + 0.14375627939181868, + 0.19555685989616903 + ], + "conc_latency_avg_list": [ + 0.006980073732057059, + 0.01121524413576223, + 0.022193516598336436, + 0.04443628341357066, + 0.06647024051023769, + 0.08843647627449298, + 0.13041777449940556, + 0.1724508536656257 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "16c64g", + "version": "1.14.1", + "note": "", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "m": 16, + "payload_m": 16, + "create_payload_int_index": false, + "create_payload_keyword_index": true, + "is_tenant": false, + "use_scalar_quant": false, + "sq_quantile": 0.99, + "default_segment_number": 0, + "use_rescore": false, + "oversampling": 1.0, + "indexed_only": false, + "hnsw_ef": 100, + "exact": false, + "with_payload": false + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 15697.2116, + "optimize_duration": 25.248, + "load_duration": 15722.4597, + "qps": 388.3028, + "serial_latency_p99": 0.0096, + "recall": 0.9431, + "ndcg": 0.9408, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 131.5849, + 386.79, + 388.3028, + 384.3793, + 350.2126, + 382.9576, + 357.9247, + 345.2787 + ], + "conc_latency_p99_list": [ + 0.009005484194494788, + 0.017353906916105187, + 0.03040585625058156, + 0.057311439691548, + 0.08304099037777633, + 0.10989078817598055, + 0.17235435267328283, + 0.2187008540763054 + ], + "conc_latency_avg_list": [ + 0.007572031373672018, + 0.012854121107101395, + 0.025544531089812063, + 0.05126603037090653, + 0.07649834288550998, + 0.10133032444037796, + 0.15425695780864176, + 0.20059700252203141 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "16c64g", + "version": "1.14.1", + "note": "", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "m": 16, + "payload_m": 16, + "create_payload_int_index": false, + "create_payload_keyword_index": true, + "is_tenant": false, + "use_scalar_quant": false, + "sq_quantile": 0.99, + "default_segment_number": 0, + "use_rescore": false, + "oversampling": 1.0, + "indexed_only": false, + "hnsw_ef": 120, + "exact": false, + "with_payload": false + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 15697.2116, + "optimize_duration": 25.248, + "load_duration": 15722.4597, + "qps": 323.3964, + "serial_latency_p99": 0.0098, + "recall": 0.9507, + "ndcg": 0.9483, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 124.2697, + 318.5581, + 322.1527, + 323.3964, + 322.1696, + 320.3444, + 294.1645, + 299.0843 + ], + "conc_latency_p99_list": [ + 0.010349974400014606, + 0.020900160419550954, + 0.0361085274803918, + 0.06776922240387649, + 0.1010101246323029, + 0.1304961043482763, + 0.19832630217249972, + 0.2581200256987359 + ], + "conc_latency_avg_list": [ + 0.008017921087583696, + 0.015609509724329028, + 0.030780220204673667, + 0.060905707038577506, + 0.09105612227316368, + 0.12116364518938445, + 0.18088966608844861, + 0.23796571711282502 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "16c64g", + "version": "1.14.1", + "note": "", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "m": 16, + "payload_m": 16, + "create_payload_int_index": false, + "create_payload_keyword_index": true, + "is_tenant": false, + "use_scalar_quant": false, + "sq_quantile": 0.99, + "default_segment_number": 0, + "use_rescore": false, + "oversampling": 1.0, + "indexed_only": false, + "hnsw_ef": 150, + "exact": false, + "with_payload": false + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 15697.2116, + "optimize_duration": 25.248, + "load_duration": 15722.4597, + "qps": 256.4668, + "serial_latency_p99": 0.0113, + "recall": 0.9588, + "ndcg": 0.9565, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 109.5611, + 256.4668, + 256.1527, + 254.6602, + 254.0357, + 230.7082, + 231.7745, + 231.2401 + ], + "conc_latency_p99_list": [ + 0.011419588899880154, + 0.023989050570235114, + 0.04468748835133739, + 0.08459992891002913, + 0.12495911995429194, + 0.1651804892499058, + 0.24398564357354185, + 0.3289348165634146 + ], + "conc_latency_avg_list": [ + 0.009094595299014826, + 0.01939382785093295, + 0.03874371727302175, + 0.07740749485565217, + 0.11551361239043029, + 0.15364336360536066, + 0.22721854368843947, + 0.30011830827724206 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "16c64g", + "version": "1.14.1", + "note": "", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "m": 16, + "payload_m": 16, + "create_payload_int_index": false, + "create_payload_keyword_index": true, + "is_tenant": false, + "use_scalar_quant": false, + "sq_quantile": 0.99, + "default_segment_number": 0, + "use_rescore": false, + "oversampling": 1.0, + "indexed_only": false, + "hnsw_ef": 200, + "exact": false, + "with_payload": false + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 15697.2116, + "optimize_duration": 25.248, + "load_duration": 15722.4597, + "qps": 145.5316, + "serial_latency_p99": 0.0184, + "recall": 0.9726, + "ndcg": 0.9707, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 74.3446, + 145.5316, + 145.1482, + 145.102, + 143.753, + 142.6857, + 143.4072, + 131.2489 + ], + "conc_latency_p99_list": [ + 0.0165358613866556, + 0.04113009970023996, + 0.077625898544502, + 0.14813462275924394, + 0.22295340399432462, + 0.294517316972051, + 0.4269371400595992, + 0.57246596743993 + ], + "conc_latency_avg_list": [ + 0.013403153333653934, + 0.03417354834242769, + 0.06837094118196142, + 0.13579538898822568, + 0.20402374799944983, + 0.2720148916482139, + 0.40060691868951254, + 0.5273792696663513 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "16c64g", + "version": "1.14.1", + "note": "", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "m": 16, + "payload_m": 16, + "create_payload_int_index": false, + "create_payload_keyword_index": true, + "is_tenant": false, + "use_scalar_quant": false, + "sq_quantile": 0.99, + "default_segment_number": 0, + "use_rescore": false, + "oversampling": 1.0, + "indexed_only": false, + "hnsw_ef": 400, + "exact": false, + "with_payload": false + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 1494.757, + "optimize_duration": 5.1229, + "load_duration": 1499.8799, + "qps": 1242.428, + "serial_latency_p99": 0.0064, + "recall": 0.9474, + "ndcg": 0.9486, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 207.8059, + 902.797, + 1219.065, + 1240.8819, + 1242.428, + 1237.4131, + 1124.9325, + 1229.1616 + ], + "conc_latency_p99_list": [ + 0.0060893742035841554, + 0.008183636299363537, + 0.01749036556459033, + 0.027307031642994833, + 0.03601452884089667, + 0.04419223924487593, + 0.060375755269196814, + 0.07551046840235356 + ], + "conc_latency_avg_list": [ + 0.004791590131146785, + 0.005505182189747005, + 0.008129613384402488, + 0.015862831474249578, + 0.02355766877181347, + 0.03122876961641923, + 0.04634096147879825, + 0.06034169457423981 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "16c64g", + "version": "1.14.1", + "note": "", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "m": 16, + "payload_m": 16, + "create_payload_int_index": false, + "create_payload_keyword_index": true, + "is_tenant": false, + "use_scalar_quant": false, + "sq_quantile": 0.99, + "default_segment_number": 0, + "use_rescore": false, + "oversampling": 1.0, + "indexed_only": false, + "hnsw_ef": 100, + "exact": false, + "with_payload": false + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 1494.757, + "optimize_duration": 5.1229, + "load_duration": 1499.8799, + "qps": 1111.3633, + "serial_latency_p99": 0.007, + "recall": 0.955, + "ndcg": 0.956, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 199.8141, + 785.9127, + 1091.3633, + 1111.3633, + 1107.3346, + 1019.6958, + 1093.839, + 1001.0577 + ], + "conc_latency_p99_list": [ + 0.006530326353822624, + 0.008225571255024985, + 0.017999547327926833, + 0.029327484158857265, + 0.03849634404294192, + 0.047883413638337514, + 0.06567232204484762, + 0.08299460219568575 + ], + "conc_latency_avg_list": [ + 0.00498352885090105, + 0.005752180062709554, + 0.009082087941841361, + 0.017690200270742782, + 0.026413517018950422, + 0.03529832812277323, + 0.052006293456525306, + 0.06830146688681735 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "16c64g", + "version": "1.14.1", + "note": "", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "m": 16, + "payload_m": 16, + "create_payload_int_index": false, + "create_payload_keyword_index": true, + "is_tenant": false, + "use_scalar_quant": false, + "sq_quantile": 0.99, + "default_segment_number": 0, + "use_rescore": false, + "oversampling": 1.0, + "indexed_only": false, + "hnsw_ef": 120, + "exact": false, + "with_payload": false + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 1494.757, + "optimize_duration": 5.1229, + "load_duration": 1499.8799, + "qps": 955.4701, + "serial_latency_p99": 0.0072, + "recall": 0.9629, + "ndcg": 0.9632, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 185.6622, + 793.3415, + 946.8505, + 874.562, + 955.4701, + 885.0771, + 864.3945, + 878.4182 + ], + "conc_latency_p99_list": [ + 0.0064235684403683956, + 0.009337397581984989, + 0.018315727375738763, + 0.031765532111166984, + 0.042694028073310616, + 0.05360910270901515, + 0.07373746164739715, + 0.09410388803691604 + ], + "conc_latency_avg_list": [ + 0.005364062557172294, + 0.006263937348206006, + 0.01046897073615085, + 0.02055041627559809, + 0.03061415695899272, + 0.040724336093685407, + 0.06015554927031619, + 0.0788253454650213 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "16c64g", + "version": "1.14.1", + "note": "", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "m": 16, + "payload_m": 16, + "create_payload_int_index": false, + "create_payload_keyword_index": true, + "is_tenant": false, + "use_scalar_quant": false, + "sq_quantile": 0.99, + "default_segment_number": 0, + "use_rescore": false, + "oversampling": 1.0, + "indexed_only": false, + "hnsw_ef": 150, + "exact": false, + "with_payload": false + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 1494.757, + "optimize_duration": 5.1229, + "load_duration": 1499.8799, + "qps": 783.5207, + "serial_latency_p99": 0.0077, + "recall": 0.971, + "ndcg": 0.9707, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 166.4231, + 708.7004, + 782.3072, + 749.9331, + 783.5207, + 715.0206, + 772.7293, + 766.1657 + ], + "conc_latency_p99_list": [ + 0.007752005998918312, + 0.011055799241439674, + 0.01983165598663617, + 0.03580881697766016, + 0.04827545183929032, + 0.06106030683557037, + 0.0860168857887038, + 0.11303307615977244 + ], + "conc_latency_avg_list": [ + 0.005984532021070551, + 0.007016301564799948, + 0.012666817424782246, + 0.025068033731047566, + 0.03733852664949933, + 0.04976950311346699, + 0.07373640385407187, + 0.09734759330709476 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "16c64g", + "version": "1.14.1", + "note": "", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "m": 16, + "payload_m": 16, + "create_payload_int_index": false, + "create_payload_keyword_index": true, + "is_tenant": false, + "use_scalar_quant": false, + "sq_quantile": 0.99, + "default_segment_number": 0, + "use_rescore": false, + "oversampling": 1.0, + "indexed_only": false, + "hnsw_ef": 200, + "exact": false, + "with_payload": false + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 1494.757, + "optimize_duration": 5.1229, + "load_duration": 1499.8799, + "qps": 470.8546, + "serial_latency_p99": 0.0095, + "recall": 0.9835, + "ndcg": 0.983, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "conc_qps_list": [ + 128.7549, + 463.3066, + 470.8546, + 449.3394, + 425.8643, + 467.8202, + 424.3519, + 461.6996 + ], + "conc_latency_p99_list": [ + 0.009610975488612894, + 0.015854567807982656, + 0.02673192903806921, + 0.04810779881445335, + 0.0697591826786811, + 0.08984742730972357, + 0.133888148713595, + 0.17759855342650552 + ], + "conc_latency_avg_list": [ + 0.00773696570017495, + 0.010733455494322586, + 0.021064175327174202, + 0.0418164996085016, + 0.06287635106079846, + 0.08275500306312698, + 0.1233307168734776, + 0.16212584284777362 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "16c64g", + "version": "1.14.1", + "note": "", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "m": 16, + "payload_m": 16, + "create_payload_int_index": false, + "create_payload_keyword_index": true, + "is_tenant": false, + "use_scalar_quant": false, + "sq_quantile": 0.99, + "default_segment_number": 0, + "use_rescore": false, + "oversampling": 1.0, + "indexed_only": false, + "hnsw_ef": 400, + "exact": false, + "with_payload": false + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 20000.296593046995, + "optimize_duration": 5.1060515040007886, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [], + "conc_qps_list": [], + "conc_latency_p99_list": [], + "conc_latency_avg_list": [], + "st_ideal_insert_duration": 20000, + "st_search_stage_list": [ + 10, + 20, + 30, + 40, + 50, + 60, + 70, + 80, + 90, + 100, + 110 + ], + "st_search_time_list": [ + 2001.8252, + 4001.8289, + 6001.8338, + 8001.8531, + 10001.845, + 12001.8527, + 14001.8583, + 16001.8614, + 18001.8645, + 20001.823, + 20138.2828 + ], + "st_max_qps_list_list": [ + 917.7631, + 798.2215, + 783.596, + 689.1114, + 602.6186, + 523.3032, + 471.7532, + 423.088, + 393.753, + 432.3835, + 439.1116 + ], + "st_recall_list": [ + 0.0964, + 0.1889, + 0.2779, + 0.3667, + 0.4587, + 0.5518, + 0.6444, + 0.7394, + 0.835, + 0.9279, + 0.9278 + ], + "st_ndcg_list": [ + 0.0962, + 0.1884, + 0.2781, + 0.367, + 0.4592, + 0.5519, + 0.644, + 0.7386, + 0.8337, + 0.9261, + 0.926 + ], + "st_serial_latency_p99_list": [ + 0.006, + 0.0088, + 0.0094, + 0.009, + 0.0091, + 0.0095, + 0.0098, + 0.0093, + 0.0162, + 0.0091, + 0.0082 + ], + "st_conc_failed_rate_list": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "16c64g", + "version": "1.14.1", + "note": "", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "m": 16, + "payload_m": 16, + "create_payload_int_index": false, + "create_payload_keyword_index": false, + "is_tenant": false, + "use_scalar_quant": false, + "sq_quantile": 0.99, + "default_segment_number": 0, + "use_rescore": false, + "oversampling": 1.0, + "indexed_only": false, + "hnsw_ef": 100, + "exact": false, + "with_payload": false + }, + "case_config": { + "case_id": 200, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "insert_rate": 500, + "search_stages": [ + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "concurrencies": [ + 5, + 10, + 20 + ] + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 11178.049474759988, + "optimize_duration": 5.107795905001694, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [], + "conc_qps_list": [], + "conc_latency_p99_list": [], + "conc_latency_avg_list": [], + "st_ideal_insert_duration": 10000, + "st_search_stage_list": [ + 10, + 20, + 30, + 40, + 50, + 60, + 70, + 80, + 90, + 100, + 110 + ], + "st_search_time_list": [ + 1052.3021, + 2196.7439, + 3334.1675, + 4464.7879, + 5593.3489, + 6716.0947, + 7834.9031, + 8949.4354, + 10068.6546, + 11179.6041, + 11317.8056 + ], + "st_max_qps_list_list": [ + 735.1562, + 563.6946, + 623.9262, + 584.5904, + 517.4969, + 450.3825, + 385.8495, + 351.788, + 347.5774, + 436.6905, + 434.8848 + ], + "st_recall_list": [ + 0.0972, + 0.1923, + 0.2838, + 0.3735, + 0.4657, + 0.5586, + 0.6521, + 0.7481, + 0.8439, + 0.9364, + 0.9363 + ], + "st_ndcg_list": [ + 0.097, + 0.1919, + 0.2839, + 0.3733, + 0.4657, + 0.5581, + 0.6513, + 0.7468, + 0.842, + 0.9339, + 0.9338 + ], + "st_serial_latency_p99_list": [ + 0.0094, + 0.0123, + 0.0098, + 0.0132, + 0.0118, + 0.0111, + 0.0131, + 0.0116, + 0.0118, + 0.011, + 0.0088 + ], + "st_conc_failed_rate_list": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "task_config": { + "db": "QdrantCloud", + "db_config": { + "db_label": "16c64g", + "version": "1.14.1", + "note": "", + "url": "**********", + "api_key": "**********" + }, + "db_case_config": { + "metric_type": "COSINE", + "m": 16, + "payload_m": 16, + "create_payload_int_index": false, + "create_payload_keyword_index": false, + "is_tenant": false, + "use_scalar_quant": false, + "sq_quantile": 0.99, + "default_segment_number": 0, + "use_rescore": false, + "oversampling": 1.0, + "indexed_only": false, + "hnsw_ef": 100, + "exact": false, + "with_payload": false + }, + "case_config": { + "case_id": 200, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "insert_rate": 1000, + "search_stages": [ + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "concurrencies": [ + 5, + 10, + 20 + ] + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + } + ], + "file_fmt": "result_{}_{}_{}.json", + "timestamp": 1748822400.0 +} \ No newline at end of file diff --git a/vectordb_bench/results/S3Vectors/result_20250722_standard_s3vectors.json b/vectordb_bench/results/S3Vectors/result_20250722_standard_s3vectors.json new file mode 100644 index 000000000..9171d4d9b --- /dev/null +++ b/vectordb_bench/results/S3Vectors/result_20250722_standard_s3vectors.json @@ -0,0 +1,2509 @@ +{ + "run_id": "d60af9965dec47739a5924d55bd3ca8d", + "task_label": "standard_2025", + "results": [ + { + "metrics": { + "max_load_count": 0, + "insert_duration": 2970.4726, + "optimize_duration": 0.0324, + "load_duration": 2970.505, + "qps": 199.4972, + "serial_latency_p99": 0.3371, + "recall": 0.8717, + "ndcg": 0.8838, + "conc_num_list": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 70, + 80, + 90, + 100 + ], + "conc_qps_list": [ + 5.5921, + 32.448, + 69.6075, + 106.5674, + 145.5834, + 186.4408, + 199.4972, + 148.2356, + 138.4338, + 136.3931, + 141.4886, + 140.7321, + 143.3282, + 158.5491 + ], + "conc_latency_p99_list": [ + 0.3057268663402646, + 0.29286937245284206, + 0.2689738546160515, + 0.2666801520221633, + 0.2591465408721705, + 0.2562654330645455, + 0.9055340573028662, + 2.6205989362485758, + 4.214876335293486, + 5.548211481111834, + 5.860073060470753, + 7.030681503431443, + 7.150496072808162, + 7.454363987408566 + ], + "conc_latency_avg_list": [ + 0.17877813466726789, + 0.1539926040293229, + 0.14355259011723448, + 0.14057451599100018, + 0.13709053315632305, + 0.13375001383421867, + 0.14955758877293948, + 0.2574387009127791, + 0.31749838548130993, + 0.3639443052363144, + 0.3930872011390246, + 0.4370199322238293, + 0.44931074305892155, + 0.44128390770244413 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "S3Vectors", + "db_config": { + "db_label": "", + "version": "", + "note": "", + "region_name": "us-east-1", + "access_key_id": "**********", + "secret_access_key": "**********", + "bucket_name": "s3vectors-east-1", + "index_name": "vdbbench-index" + }, + "db_case_config": { + "metric_type": "COSINE", + "data_type": "float32" + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 30, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 70, + 80, + 90, + 100 + ], + "concurrency_duration": 180, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 192.1164, + "serial_latency_p99": 0.3456, + "recall": 0.4276, + "ndcg": 0.4798, + "conc_num_list": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30 + ], + "conc_qps_list": [ + 4.8758, + 30.4836, + 72.5956, + 107.1847, + 149.9406, + 192.1164, + 170.8968 + ], + "conc_latency_p99_list": [ + 0.32926209951518104, + 0.2970751485106298, + 0.25644857229141094, + 0.2727285704837415, + 0.254873042370891, + 0.2459075423690955, + 1.0701303746562918 + ], + "conc_latency_avg_list": [ + 0.20494631408819447, + 0.16344266400725194, + 0.1374262750580602, + 0.13936750741124818, + 0.13279457812828163, + 0.12940155565726036, + 0.16526157093735963 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "S3Vectors", + "db_config": { + "db_label": "", + "version": "", + "note": "", + "region_name": "us-east-1", + "access_key_id": "**********", + "secret_access_key": "**********", + "bucket_name": "s3vectors-east-1", + "index_name": "vdbbench-index" + }, + "db_case_config": { + "metric_type": "COSINE", + "data_type": "float32" + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.001 + }, + "k": 30, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30 + ], + "concurrency_duration": 60, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 197.4455, + "serial_latency_p99": 0.3493, + "recall": 0.5314, + "ndcg": 0.5755, + "conc_num_list": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30 + ], + "conc_qps_list": [ + 5.1084, + 33.5471, + 68.2957, + 105.8788, + 144.8791, + 185.4856, + 197.4455 + ], + "conc_latency_p99_list": [ + 0.7816582751926034, + 0.29464222808601315, + 0.30184637255442787, + 0.27352764647221156, + 0.2619378386589227, + 0.2627684409084031, + 0.8492211898852832 + ], + "conc_latency_avg_list": [ + 0.19561624129629673, + 0.14872712125238183, + 0.1460010165238937, + 0.14107773315843697, + 0.1373406008466785, + 0.1340945516297796, + 0.14899160872176243 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "S3Vectors", + "db_config": { + "db_label": "", + "version": "", + "note": "", + "region_name": "us-east-1", + "access_key_id": "**********", + "secret_access_key": "**********", + "bucket_name": "s3vectors-east-1", + "index_name": "vdbbench-index" + }, + "db_case_config": { + "metric_type": "COSINE", + "data_type": "float32" + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.002 + }, + "k": 30, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30 + ], + "concurrency_duration": 60, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 196.9391, + "serial_latency_p99": 0.2634, + "recall": 0.6549, + "ndcg": 0.6898, + "conc_num_list": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30 + ], + "conc_qps_list": [ + 5.675, + 31.8811, + 67.4951, + 102.5763, + 141.6207, + 179.814, + 196.9391 + ], + "conc_latency_p99_list": [ + 0.7444819935830348, + 0.3069360829598733, + 0.29420441156718896, + 0.26985632870404475, + 0.2657526725350176, + 0.2610060953185895, + 0.815150408020133 + ], + "conc_latency_avg_list": [ + 0.17608577868139202, + 0.15649657269173345, + 0.147809565744648, + 0.14573910107307306, + 0.14054977110591665, + 0.1380776505557241, + 0.1512662943119532 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "S3Vectors", + "db_config": { + "db_label": "", + "version": "", + "note": "", + "region_name": "us-east-1", + "access_key_id": "**********", + "secret_access_key": "**********", + "bucket_name": "s3vectors-east-1", + "index_name": "vdbbench-index" + }, + "db_case_config": { + "metric_type": "COSINE", + "data_type": "float32" + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.005 + }, + "k": 30, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30 + ], + "concurrency_duration": 60, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 201.5401, + "serial_latency_p99": 0.2824, + "recall": 0.7086, + "ndcg": 0.7368, + "conc_num_list": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30 + ], + "conc_qps_list": [ + 6.0604, + 31.3102, + 66.8635, + 102.4307, + 137.0544, + 174.9476, + 201.5401 + ], + "conc_latency_p99_list": [ + 0.2943547100026627, + 0.288298530371394, + 0.27279032936668957, + 0.26144427472900134, + 0.2709142048709327, + 0.25804895919049164, + 0.37156653454701877 + ], + "conc_latency_avg_list": [ + 0.16488580476664708, + 0.1593260142217583, + 0.14920043242154704, + 0.14590159505882977, + 0.14526904837368781, + 0.14220184481398934, + 0.14764247648479953 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "S3Vectors", + "db_config": { + "db_label": "", + "version": "", + "note": "", + "region_name": "us-east-1", + "access_key_id": "**********", + "secret_access_key": "**********", + "bucket_name": "s3vectors-east-1", + "index_name": "vdbbench-index" + }, + "db_case_config": { + "metric_type": "COSINE", + "data_type": "float32" + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.01 + }, + "k": 30, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30 + ], + "concurrency_duration": 60, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 202.2424, + "serial_latency_p99": 0.3017, + "recall": 0.7592, + "ndcg": 0.7812, + "conc_num_list": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30 + ], + "conc_qps_list": [ + 5.5099, + 31.0969, + 64.0501, + 96.6608, + 132.3824, + 169.7848, + 202.2424 + ], + "conc_latency_p99_list": [ + 0.28144256640225646, + 0.28266542118974025, + 0.28745006633573184, + 0.2887180608802009, + 0.2737992306554224, + 0.266253309850581, + 0.3075238355837064 + ], + "conc_latency_avg_list": [ + 0.1813592702564305, + 0.16037075466013828, + 0.15561127622732157, + 0.15461495328035568, + 0.1499613637952411, + 0.14630668502042168, + 0.14729734342766834 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "S3Vectors", + "db_config": { + "db_label": "", + "version": "", + "note": "", + "region_name": "us-east-1", + "access_key_id": "**********", + "secret_access_key": "**********", + "bucket_name": "s3vectors-east-1", + "index_name": "vdbbench-index" + }, + "db_case_config": { + "metric_type": "COSINE", + "data_type": "float32" + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.02 + }, + "k": 30, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30 + ], + "concurrency_duration": 60, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 198.599, + "serial_latency_p99": 0.3588, + "recall": 0.8085, + "ndcg": 0.824, + "conc_num_list": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30 + ], + "conc_qps_list": [ + 5.4389, + 32.3909, + 63.6649, + 99.8077, + 135.1746, + 170.4683, + 198.599 + ], + "conc_latency_p99_list": [ + 0.3162948441389017, + 0.2929340754216537, + 0.29956613731395926, + 0.27734781362582034, + 0.2848844401980749, + 0.27146368539193677, + 0.3210610778466799 + ], + "conc_latency_avg_list": [ + 0.1837291808513033, + 0.15405556739487042, + 0.15673738707459153, + 0.14965542208934704, + 0.14737036746554527, + 0.14562790749368398, + 0.14859330926523254 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "S3Vectors", + "db_config": { + "db_label": "", + "version": "", + "note": "", + "region_name": "us-east-1", + "access_key_id": "**********", + "secret_access_key": "**********", + "bucket_name": "s3vectors-east-1", + "index_name": "vdbbench-index" + }, + "db_case_config": { + "metric_type": "COSINE", + "data_type": "float32" + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.05 + }, + "k": 30, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30 + ], + "concurrency_duration": 60, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 199.0349, + "serial_latency_p99": 0.2753, + "recall": 0.8325, + "ndcg": 0.8442, + "conc_num_list": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30 + ], + "conc_qps_list": [ + 4.8696, + 31.3333, + 66.5694, + 104.2362, + 139.0719, + 172.2119, + 199.0349 + ], + "conc_latency_p99_list": [ + 0.9459384199604335, + 0.4258276104211084, + 0.2784014346660115, + 0.26199020874337287, + 0.2646811329695631, + 0.27463690769043747, + 0.37999025870813036 + ], + "conc_latency_avg_list": [ + 0.20520533806105612, + 0.159053406815181, + 0.14975306031034372, + 0.14334582615591107, + 0.1429548468611905, + 0.14433168370307262, + 0.14961306354543724 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "S3Vectors", + "db_config": { + "db_label": "", + "version": "", + "note": "", + "region_name": "us-east-1", + "access_key_id": "**********", + "secret_access_key": "**********", + "bucket_name": "s3vectors-east-1", + "index_name": "vdbbench-index" + }, + "db_case_config": { + "metric_type": "COSINE", + "data_type": "float32" + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.1 + }, + "k": 30, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30 + ], + "concurrency_duration": 60, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 202.1405, + "serial_latency_p99": 0.2826, + "recall": 0.8492, + "ndcg": 0.8593, + "conc_num_list": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30 + ], + "conc_qps_list": [ + 6.2425, + 33.4941, + 67.3777, + 104.4543, + 138.9241, + 146.6979, + 202.1405 + ], + "conc_latency_p99_list": [ + 0.2561170053563546, + 0.2629233490268233, + 0.2786194805294506, + 0.2620396741002323, + 0.2721249624039043, + 0.275872534196824, + 0.5125790551502745 + ], + "conc_latency_avg_list": [ + 0.16007621229657282, + 0.14897182012048557, + 0.14795937857751476, + 0.14309594409935683, + 0.14326260496059487, + 0.15233913391160195, + 0.14743800728245085 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "S3Vectors", + "db_config": { + "db_label": "", + "version": "", + "note": "", + "region_name": "us-east-1", + "access_key_id": "**********", + "secret_access_key": "**********", + "bucket_name": "s3vectors-east-1", + "index_name": "vdbbench-index" + }, + "db_case_config": { + "metric_type": "COSINE", + "data_type": "float32" + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.2 + }, + "k": 30, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30 + ], + "concurrency_duration": 60, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 201.1282, + "serial_latency_p99": 0.2692, + "recall": 0.8637, + "ndcg": 0.8749, + "conc_num_list": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30 + ], + "conc_qps_list": [ + 5.8624, + 33.5361, + 69.1636, + 104.8417, + 140.9376, + 176.9636, + 201.1282 + ], + "conc_latency_p99_list": [ + 0.3044514204916778, + 0.26760380705818526, + 0.26628886790014816, + 0.26692828512517686, + 0.26154959074920053, + 0.26205542255192993, + 0.4432655484159461 + ], + "conc_latency_avg_list": [ + 0.17045623565768023, + 0.14877978406152068, + 0.14426341717878755, + 0.1425052129256506, + 0.14110002266788405, + 0.1404733362146569, + 0.1478939573660762 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "S3Vectors", + "db_config": { + "db_label": "", + "version": "", + "note": "", + "region_name": "us-east-1", + "access_key_id": "**********", + "secret_access_key": "**********", + "bucket_name": "s3vectors-east-1", + "index_name": "vdbbench-index" + }, + "db_case_config": { + "metric_type": "COSINE", + "data_type": "float32" + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.5 + }, + "k": 30, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30 + ], + "concurrency_duration": 60, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 18283.7299, + "optimize_duration": 0.0431, + "load_duration": 18283.773, + "qps": 194.8021, + "serial_latency_p99": 0.5598, + "recall": 0.86, + "ndcg": 0.8711, + "conc_num_list": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 70, + 80, + 90, + 100 + ], + "conc_qps_list": [ + 3.0143, + 19.7214, + 41.2286, + 68.42, + 91.9925, + 119.2562, + 145.7568, + 194.8021, + 175.3113, + 154.094, + 139.0243, + 137.0152, + 136.9753, + 148.2486 + ], + "conc_latency_p99_list": [ + 0.5265677035803672, + 0.4156031924713169, + 0.4006210068403743, + 0.36790279168810214, + 0.3758108571916819, + 0.3622747381753288, + 0.35375411363318543, + 0.35835539473453565, + 1.80868591776699, + 3.120272358419594, + 5.120826761483369, + 6.037255773596679, + 6.555112633776209, + 7.530691414061005 + ], + "conc_latency_avg_list": [ + 0.33162897625148235, + 0.2531889479229043, + 0.2419514681069688, + 0.21862107713167664, + 0.21690321055106906, + 0.208987360773395, + 0.20500410319548723, + 0.20425706607350047, + 0.2735218531669894, + 0.3597729722675132, + 0.42812774128895226, + 0.4839677828505012, + 0.5255217312511777, + 0.5344663876457051 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "S3Vectors", + "db_config": { + "db_label": "", + "version": "", + "note": "", + "region_name": "us-east-1", + "access_key_id": "**********", + "secret_access_key": "**********", + "bucket_name": "s3vectors-east-1", + "index_name": "vdbbench-index" + }, + "db_case_config": { + "metric_type": "COSINE", + "data_type": "float32" + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 30, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50, + 60, + 70, + 80, + 90, + 100 + ], + "concurrency_duration": 120, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 187.4268, + "serial_latency_p99": 0.4537, + "recall": 0.4692, + "ndcg": 0.5189, + "conc_num_list": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50 + ], + "conc_qps_list": [ + 2.6726, + 15.2372, + 32.3581, + 51.8251, + 68.8006, + 89.7959, + 114.1564, + 145.0215, + 187.4268 + ], + "conc_latency_p99_list": [ + 0.558423354185652, + 0.4868009101017379, + 0.49411141405231285, + 0.46562842677813004, + 0.46276188636024046, + 0.44683693168219185, + 0.422478049620986, + 0.45179960425244653, + 0.43213706685928627 + ], + "conc_latency_avg_list": [ + 0.37403186548314843, + 0.32768382831440807, + 0.30850593052599434, + 0.2886127291405558, + 0.2898631480925624, + 0.2774442439592566, + 0.2618160285977717, + 0.27425664379211806, + 0.26500430787944396 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "S3Vectors", + "db_config": { + "db_label": "", + "version": "", + "note": "", + "region_name": "us-east-1", + "access_key_id": "**********", + "secret_access_key": "**********", + "bucket_name": "s3vectors-east-1", + "index_name": "vdbbench-index" + }, + "db_case_config": { + "metric_type": "COSINE", + "data_type": "float32" + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.001 + }, + "k": 30, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50 + ], + "concurrency_duration": 120, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 198.397, + "serial_latency_p99": 0.5069, + "recall": 0.5409, + "ndcg": 0.583, + "conc_num_list": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50 + ], + "conc_qps_list": [ + 2.7458, + 16.2434, + 34.5524, + 52.7311, + 72.7504, + 96.1006, + 116.6115, + 158.4903, + 198.397 + ], + "conc_latency_p99_list": [ + 0.5412072664377042, + 0.4398492124193581, + 0.4616037914011393, + 0.4520418062619862, + 0.43112011939403594, + 0.4201923584972978, + 0.41751666231662987, + 0.41244613734539565, + 0.8685830348095661 + ], + "conc_latency_avg_list": [ + 0.364050299453789, + 0.3075824910749804, + 0.2885927725157661, + 0.28370028398216446, + 0.2741255327259186, + 0.2591086112359427, + 0.256095442670376, + 0.2511196609211916, + 0.2495782575736313 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "S3Vectors", + "db_config": { + "db_label": "", + "version": "", + "note": "", + "region_name": "us-east-1", + "access_key_id": "**********", + "secret_access_key": "**********", + "bucket_name": "s3vectors-east-1", + "index_name": "vdbbench-index" + }, + "db_case_config": { + "metric_type": "COSINE", + "data_type": "float32" + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.002 + }, + "k": 30, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50 + ], + "concurrency_duration": 120, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 174.3549, + "serial_latency_p99": 0.4969, + "recall": 0.6279, + "ndcg": 0.6601, + "conc_num_list": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50 + ], + "conc_qps_list": [ + 2.8553, + 16.5337, + 36.7964, + 56.7117, + 75.5556, + 82.9818, + 99.0002, + 132.4292, + 174.3549 + ], + "conc_latency_p99_list": [ + 0.45376017893664516, + 0.44813311758218316, + 0.42432168450904995, + 0.43569353796076044, + 0.440997409076081, + 0.484473014894174, + 0.5039040255069267, + 0.49455512693559317, + 0.46598410551319797 + ], + "conc_latency_avg_list": [ + 0.35009942597841287, + 0.3019104116590019, + 0.2712048981974938, + 0.26384581109458205, + 0.2638924889765985, + 0.30024374225892075, + 0.3016029519634407, + 0.3005380444163379, + 0.2840002673315207 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "S3Vectors", + "db_config": { + "db_label": "", + "version": "", + "note": "", + "region_name": "us-east-1", + "access_key_id": "**********", + "secret_access_key": "**********", + "bucket_name": "s3vectors-east-1", + "index_name": "vdbbench-index" + }, + "db_case_config": { + "metric_type": "COSINE", + "data_type": "float32" + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.005 + }, + "k": 30, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50 + ], + "concurrency_duration": 120, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 172.95, + "serial_latency_p99": 0.5156, + "recall": 0.7004, + "ndcg": 0.7253, + "conc_num_list": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50 + ], + "conc_qps_list": [ + 2.6274, + 14.6154, + 31.2299, + 50.9982, + 67.2038, + 85.2195, + 106.2608, + 133.4809, + 172.95 + ], + "conc_latency_p99_list": [ + 0.547288335094345, + 0.5017968477343676, + 0.49248093454807523, + 0.464813823304139, + 0.47070235666353255, + 0.4728234008484287, + 0.4532600020460086, + 0.4910402821341996, + 0.46973878620890896 + ], + "conc_latency_avg_list": [ + 0.3804705326892642, + 0.3414292937254368, + 0.31917288861747456, + 0.293509111682151, + 0.296753375912928, + 0.2921559299730053, + 0.2811883913532294, + 0.2978465235595297, + 0.28710858363851804 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "S3Vectors", + "db_config": { + "db_label": "", + "version": "", + "note": "", + "region_name": "us-east-1", + "access_key_id": "**********", + "secret_access_key": "**********", + "bucket_name": "s3vectors-east-1", + "index_name": "vdbbench-index" + }, + "db_case_config": { + "metric_type": "COSINE", + "data_type": "float32" + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.01 + }, + "k": 30, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50 + ], + "concurrency_duration": 120, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 190.9747, + "serial_latency_p99": 0.5174, + "recall": 0.7398, + "ndcg": 0.7609, + "conc_num_list": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50 + ], + "conc_qps_list": [ + 2.5905, + 15.1315, + 32.5661, + 51.3531, + 70.859, + 92.1273, + 106.2614, + 146.6415, + 190.9747 + ], + "conc_latency_p99_list": [ + 0.5359640973678322, + 0.5038611659430894, + 0.4727835458039771, + 0.4578898529458093, + 0.44936108179041145, + 0.4493892845755921, + 0.44525163223734127, + 0.4456483786657919, + 0.42966096448653823 + ], + "conc_latency_avg_list": [ + 0.3858861763917817, + 0.3300191992241019, + 0.3063704047322927, + 0.2914914843424382, + 0.2812321226193738, + 0.2703860804524464, + 0.28123297384133383, + 0.27112162543941165, + 0.2602231424690716 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "S3Vectors", + "db_config": { + "db_label": "", + "version": "", + "note": "", + "region_name": "us-east-1", + "access_key_id": "**********", + "secret_access_key": "**********", + "bucket_name": "s3vectors-east-1", + "index_name": "vdbbench-index" + }, + "db_case_config": { + "metric_type": "COSINE", + "data_type": "float32" + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.02 + }, + "k": 30, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50 + ], + "concurrency_duration": 120, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 186.0237, + "serial_latency_p99": 0.474, + "recall": 0.7847, + "ndcg": 0.803, + "conc_num_list": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50 + ], + "conc_qps_list": [ + 2.5318, + 14.5521, + 31.2245, + 50.2005, + 65.5451, + 85.5296, + 104.9942, + 143.4204, + 186.0237 + ], + "conc_latency_p99_list": [ + 0.5324654311995247, + 0.5404962703760248, + 0.4962297047779427, + 0.4716085807973288, + 0.5047204344696364, + 0.4746078232186847, + 0.47090065246447926, + 0.452439703999554, + 0.4452963033004198 + ], + "conc_latency_avg_list": [ + 0.39482895703737375, + 0.3431497387012991, + 0.3196624369807591, + 0.29812974119250274, + 0.30423598276791675, + 0.2911854029193207, + 0.28455497776356575, + 0.27746096306112, + 0.2668085950808965 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "S3Vectors", + "db_config": { + "db_label": "", + "version": "", + "note": "", + "region_name": "us-east-1", + "access_key_id": "**********", + "secret_access_key": "**********", + "bucket_name": "s3vectors-east-1", + "index_name": "vdbbench-index" + }, + "db_case_config": { + "metric_type": "COSINE", + "data_type": "float32" + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.05 + }, + "k": 30, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50 + ], + "concurrency_duration": 120, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 192.1458, + "serial_latency_p99": 0.4805, + "recall": 0.8103, + "ndcg": 0.8229, + "conc_num_list": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50 + ], + "conc_qps_list": [ + 2.7398, + 15.4167, + 33.9829, + 55.0042, + 70.9437, + 94.0349, + 114.3355, + 148.4342, + 192.1458 + ], + "conc_latency_p99_list": [ + 0.5046815221232828, + 0.484930241847178, + 0.4836470659670888, + 0.43167894504615095, + 0.45311440610093995, + 0.4434100269706679, + 0.426801529576187, + 0.44005047504382694, + 0.4274993393034665 + ], + "conc_latency_avg_list": [ + 0.36485076367777464, + 0.32397690761592385, + 0.29381537385921624, + 0.27217157280678167, + 0.2811709847599061, + 0.26485245462376733, + 0.2612452901977557, + 0.267842898266305, + 0.2585550665767442 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "S3Vectors", + "db_config": { + "db_label": "", + "version": "", + "note": "", + "region_name": "us-east-1", + "access_key_id": "**********", + "secret_access_key": "**********", + "bucket_name": "s3vectors-east-1", + "index_name": "vdbbench-index" + }, + "db_case_config": { + "metric_type": "COSINE", + "data_type": "float32" + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.1 + }, + "k": 30, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50 + ], + "concurrency_duration": 120, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 179.4203, + "serial_latency_p99": 0.4975, + "recall": 0.8273, + "ndcg": 0.841, + "conc_num_list": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50 + ], + "conc_qps_list": [ + 2.8308, + 16.0583, + 30.5308, + 47.3615, + 63.1972, + 81.2939, + 101.5995, + 141.9673, + 179.4203 + ], + "conc_latency_p99_list": [ + 0.47774682923685763, + 0.4583682000241243, + 0.5285271699976875, + 0.49221092734835037, + 0.5155015790706966, + 0.51426090602763, + 0.4712264502665494, + 0.4578040716639959, + 0.46363746090210045 + ], + "conc_latency_avg_list": [ + 0.3531220891750897, + 0.3106564776484316, + 0.32694680486576516, + 0.31603691522206145, + 0.31527973038386675, + 0.3064528471658255, + 0.29378804824334526, + 0.2803020946615617, + 0.27547515726924465 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "S3Vectors", + "db_config": { + "db_label": "", + "version": "", + "note": "", + "region_name": "us-east-1", + "access_key_id": "**********", + "secret_access_key": "**********", + "bucket_name": "s3vectors-east-1", + "index_name": "vdbbench-index" + }, + "db_case_config": { + "metric_type": "COSINE", + "data_type": "float32" + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.2 + }, + "k": 30, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50 + ], + "concurrency_duration": 120, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 0.0, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 199.5444, + "serial_latency_p99": 0.4639, + "recall": 0.8478, + "ndcg": 0.8577, + "conc_num_list": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50 + ], + "conc_qps_list": [ + 2.829, + 16.3022, + 34.3169, + 55.8885, + 72.7821, + 93.6099, + 116.6718, + 160.6381, + 199.5444 + ], + "conc_latency_p99_list": [ + 0.4829293444426732, + 0.46692242537683337, + 0.454294371768483, + 0.43225170329562346, + 0.4534143943234808, + 0.4341565519862345, + 0.41950699321751017, + 0.4045942402590298, + 0.4522178557817817 + ], + "conc_latency_avg_list": [ + 0.35335366733505746, + 0.3060283094405901, + 0.2909273705048065, + 0.2676834368088299, + 0.273801436712963, + 0.266098154148968, + 0.2560115152344485, + 0.24769191672002275, + 0.24878019274411803 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "S3Vectors", + "db_config": { + "db_label": "", + "version": "", + "note": "", + "region_name": "us-east-1", + "access_key_id": "**********", + "secret_access_key": "**********", + "bucket_name": "s3vectors-east-1", + "index_name": "vdbbench-index" + }, + "db_case_config": { + "metric_type": "COSINE", + "data_type": "float32" + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.5 + }, + "k": 30, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 15, + 20, + 25, + 30, + 40, + 50 + ], + "concurrency_duration": 120, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 20000.23065447202, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [], + "conc_qps_list": [], + "conc_latency_p99_list": [], + "conc_latency_avg_list": [], + "st_ideal_insert_duration": 20000, + "st_search_stage_list": [ + 10, + 20, + 30, + 40, + 50, + 60, + 70, + 80, + 90, + 100 + ], + "st_search_time_list": [ + 2001.1755, + 4032.922, + 6056.7118, + 8087.6446, + 10117.7889, + 12147.7899, + 14178.1805, + 16209.3112, + 18239.0474, + 20259.618 + ], + "st_max_qps_list_list": [ + 194.5102, + 197.6167, + 197.9959, + 193.94, + 197.5605, + 188.6792, + 190.5382, + 190.4678, + 180.9549, + 173.8912 + ], + "st_recall_list": [ + 0.0985, + 0.1886, + 0.283, + 0.368, + 0.4588, + 0.542, + 0.6331, + 0.721, + 0.7969, + 0.8498 + ], + "st_ndcg_list": [ + 0.0997, + 0.1912, + 0.286, + 0.3725, + 0.4629, + 0.5474, + 0.6394, + 0.7288, + 0.8064, + 0.8619 + ], + "st_serial_latency_p99_list": [ + 0.3944, + 0.6105, + 0.8203, + 0.3878, + 0.4302, + 0.397, + 0.4123, + 0.4301, + 0.4204, + 0.4575 + ], + "st_conc_failed_rate_list": [ + 1.776151612301626e-05, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "task_config": { + "db": "S3Vectors", + "db_config": { + "db_label": "", + "version": "", + "note": "", + "region_name": "us-east-1", + "access_key_id": "**********", + "secret_access_key": "**********", + "bucket_name": "s3vectors-east-1", + "index_name": "vdbbench-index" + }, + "db_case_config": { + "metric_type": "COSINE", + "data_type": "float32" + }, + "case_config": { + "case_id": 200, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "insert_rate": 500, + "search_stages": [ + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "concurrencies": [ + 20, + 30, + 40 + ], + "optimize_after_write": false + }, + "k": 30, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + },{ + "metrics": { + "max_load_count": 0, + "insert_duration": 13068.03754887695, + "optimize_duration": 0.0, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [], + "conc_qps_list": [], + "conc_latency_p99_list": [], + "conc_latency_avg_list": [], + "st_ideal_insert_duration": 10000, + "st_search_stage_list": [ + 10, + 20, + 30, + 40, + 50, + 60, + 70, + 80, + 90, + 100 + ], + "st_search_time_list": [ + 1267.0537, + 2591.9491, + 3904.5826, + 5208.0713, + 6507.959, + 7817.1373, + 9135.001, + 10448.1642, + 11759.1251, + 13069.5158 + ], + "st_max_qps_list_list": [ + 195.9212, + 191.7485, + 191.769, + 193.3097, + 193.2091, + 184.6033, + 188.7796, + 189.7405, + 167.2689, + 154.8537 + ], + "st_recall_list": [ + 0.0999, + 0.1935, + 0.2839, + 0.3678, + 0.4552, + 0.5427, + 0.6333, + 0.7139, + 0.7946, + 0.8532 + ], + "st_ndcg_list": [ + 0.1014, + 0.1954, + 0.2869, + 0.3709, + 0.4593, + 0.5477, + 0.6385, + 0.7219, + 0.8042, + 0.8647 + ], + "st_serial_latency_p99_list": [ + 0.3862, + 0.4236, + 0.4858, + 0.4288, + 0.4312, + 0.437, + 0.4461, + 0.4943, + 0.5048, + 0.5632 + ], + "st_conc_failed_rate_list": [ + 0.0, + 4.4161808867691223e-05, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "task_config": { + "db": "S3Vectors", + "db_config": { + "db_label": "", + "version": "", + "note": "", + "region_name": "us-east-1", + "access_key_id": "**********", + "secret_access_key": "**********", + "bucket_name": "s3vectors-east-1", + "index_name": "vdbbench-index" + }, + "db_case_config": { + "metric_type": "COSINE", + "data_type": "float32" + }, + "case_config": { + "case_id": 200, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "insert_rate": 1000, + "search_stages": [ + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "concurrencies": [ + 20, + 30, + 40 + ], + "optimize_after_write": false + }, + "k": 30, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80 + ], + "concurrency_duration": 30, + "concurrency_timeout": 3600 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + } + ], + "file_fmt": "result_{}_{}_{}.json", + "timestamp": 1753142400.0 +} \ No newline at end of file diff --git a/vectordb_bench/results/ZillizCloud/result_20230727_standard_zillizcloud.json b/vectordb_bench/results/ZillizCloud/result_20230727_standard_zillizcloud.json deleted file mode 100644 index e20a2eab6..000000000 --- a/vectordb_bench/results/ZillizCloud/result_20230727_standard_zillizcloud.json +++ /dev/null @@ -1,791 +0,0 @@ -{ - "run_id": "5c1e8bd468224ffda1b39b08cdc342c3", - "task_label": "standard", - "results": [ - { - "metrics": { - "max_load_count": 6700000, - "load_duration": 0, - "qps": 0, - "serial_latency_p99": 0, - "recall": 0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-cap", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "L2" - }, - "case_config": { - "case_id": 2, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 12700000, - "load_duration": 0, - "qps": 0, - "serial_latency_p99": 0, - "recall": 0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-cap", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "L2" - }, - "case_config": { - "case_id": 1, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 2412.0463, - "qps": 330.0144, - "serial_latency_p99": 0.009, - "recall": 0.9507 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-cap", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 5, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 2412.0463, - "qps": 271.6585, - "serial_latency_p99": 0.0101, - "recall": 0.9678 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-cap", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 7, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 2412.0463, - "qps": 216.5226, - "serial_latency_p99": 0.0129, - "recall": 1 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-cap", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 9, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0, - "qps": 0, - "serial_latency_p99": 0, - "recall": 0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-cap", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 4, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0, - "qps": 0, - "serial_latency_p99": 0, - "recall": 0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-cap", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 6, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0, - "qps": 0, - "serial_latency_p99": 0, - "recall": 0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-cap", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 8, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 14521.4008, - "qps": 123.9553, - "serial_latency_p99": 0.023, - "recall": 0.971 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "2cu-cap", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 4, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 14521.4008, - "qps": 59.1479, - "serial_latency_p99": 0.0445, - "recall": 0.9906 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "2cu-cap", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 6, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 14521.4008, - "qps": 40.999, - "serial_latency_p99": 0.0553, - "recall": 1 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "2cu-cap", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 8, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 2340.5671, - "qps": 579.9416, - "serial_latency_p99": 0.0094, - "recall": 0.9213 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "2cu-cap", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 5, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 2340.5671, - "qps": 425.2529, - "serial_latency_p99": 0.0113, - "recall": 0.9686 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "2cu-cap", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 7, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 2340.5671, - "qps": 397.0539, - "serial_latency_p99": 0.0138, - "recall": 1 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "2cu-cap", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 9, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 5000000, - "load_duration": 0, - "qps": 0, - "serial_latency_p99": 0, - "recall": 0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-perf", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 1, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 1200000, - "load_duration": 0, - "qps": 0, - "serial_latency_p99": 0, - "recall": 0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-perf", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 2, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1100.6681, - "qps": 516.27, - "serial_latency_p99": 0.007, - "recall": 0.9463 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-perf", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 5, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1100.6681, - "qps": 354.8416, - "serial_latency_p99": 0.01, - "recall": 0.9802 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-perf", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 7, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1100.6681, - "qps": 427.5229, - "serial_latency_p99": 0.0087, - "recall": 1 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-perf", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 9, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0, - "qps": 0, - "serial_latency_p99": 0, - "recall": 0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-perf", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 4, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0, - "qps": 0, - "serial_latency_p99": 0, - "recall": 0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-perf", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 6, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0, - "qps": 0, - "serial_latency_p99": 0, - "recall": 0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-perf", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 8, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1575.253, - "qps": 2884.689, - "serial_latency_p99": 0.0053, - "recall": 0.8801 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 5, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1853.989, - "qps": 1689.5799, - "serial_latency_p99": 0.0066, - "recall": 0.9493 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 7, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1575.253, - "qps": 1517.6792, - "serial_latency_p99": 0.01, - "recall": 1 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 9, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 7635.0576, - "qps": 822.5318, - "serial_latency_p99": 0.0056, - "recall": 0.9294 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 4, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 7635.0576, - "qps": 378.9146, - "serial_latency_p99": 0.0103, - "recall": 0.9758 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 6, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 7635.0576, - "qps": 218.6854, - "serial_latency_p99": 0.0162, - "recall": 1 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 8, - "custom_case": {} - } - }, - "label": ":)" - } - ], - "file_fmt": "result_{}_{}_{}.json" -} \ No newline at end of file diff --git a/vectordb_bench/results/ZillizCloud/result_20230808_standard_zillizcloud.json b/vectordb_bench/results/ZillizCloud/result_20230808_standard_zillizcloud.json deleted file mode 100644 index e48158a02..000000000 --- a/vectordb_bench/results/ZillizCloud/result_20230808_standard_zillizcloud.json +++ /dev/null @@ -1,679 +0,0 @@ -{ - "run_id": "5c1e8bd468224ffda1b39b08cdc342c3", - "task_label": "standard", - "results": [ - { - "metrics": { - "max_load_count": 0, - "load_duration": 1079.0, - "qps": 297.5, - "serial_latency_p99": 0.0072, - "recall": 0.974 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-perf", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 10, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-perf", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 11, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1060.0, - "qps": 228.3, - "serial_latency_p99": 0.0106, - "recall": 0.994 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-perf", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 12, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-perf", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 13, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1067.0, - "qps": 584.0, - "serial_latency_p99": 0.0046, - "recall": 1.0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-perf", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 14, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-perf", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 15, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1786.0, - "qps": 1871.0, - "serial_latency_p99": 0.007, - "recall": 0.9602 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 10, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 6848.0, - "qps": 556.7, - "serial_latency_p99": 0.0067, - "recall": 0.9723 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 11, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1786.0, - "qps": 1583.0, - "serial_latency_p99": 0.0068, - "recall": 0.9836 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 12, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 6976.0, - "qps": 294.3, - "serial_latency_p99": 0.0109, - "recall": 0.9939 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 13, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1786.0, - "qps": 2345.0, - "serial_latency_p99": 0.0089, - "recall": 1.0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 14, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 6976.0, - "qps": 295.6, - "serial_latency_p99": 0.0123, - "recall": 1.0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 15, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 2408.0, - "qps": 143.0, - "serial_latency_p99": 0.0335, - "recall": 0.9818 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-cap", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 10, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-cap", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 11, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 2408.0, - "qps": 106.0, - "serial_latency_p99": 0.0207, - "recall": 0.9887 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-cap", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 12, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-cap", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 13, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 2408.0, - "qps": 189.0, - "serial_latency_p99": 0.0116, - "recall": 1.0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-cap", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 14, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-cap", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 15, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 2586.0, - "qps": 379.9721, - "serial_latency_p99": 0.0124, - "recall": 0.982 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "2cu-cap", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 10, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 15050.0, - "qps": 71.74, - "serial_latency_p99": 0.0508, - "recall": 0.9883 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "2cu-cap", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 11, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 2586.0, - "qps": 287.0, - "serial_latency_p99": 0.0149, - "recall": 0.9865 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "2cu-cap", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 12, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 13793.0, - "qps": 34.6654, - "serial_latency_p99": 0.0647, - "recall": 0.9961 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "2cu-cap", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 13, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 2586.0, - "qps": 412.0, - "serial_latency_p99": 0.0103, - "recall": 1.0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "2cu-cap", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 14, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 13793.0, - "qps": 42.169, - "serial_latency_p99": 0.0468, - "recall": 1.0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "2cu-cap", - "version": "v2023.6", - "uri": "**********", - "user": "root", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 15, - "custom_case": {} - } - }, - "label": ":)" - } - ], - "file_fmt": "result_{}_{}_{}.json" -} \ No newline at end of file diff --git a/vectordb_bench/results/ZillizCloud/result_20240105_standard_202401_zillizcloud.json b/vectordb_bench/results/ZillizCloud/result_20240105_standard_202401_zillizcloud.json deleted file mode 100644 index 5d9907754..000000000 --- a/vectordb_bench/results/ZillizCloud/result_20240105_standard_202401_zillizcloud.json +++ /dev/null @@ -1,1352 +0,0 @@ -{ - "run_id": "0ae10e14e34c4c3c9a7116e7a9591d01", - "task_label": "standard_202401", - "results": [ - { - "metrics": { - "max_load_count": 0, - "load_duration": 861.8295, - "qps": 5115.5303, - "serial_latency_p99": 0.0087, - "recall": 0.9469 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 10, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 861.8295, - "qps": 3685.0767, - "serial_latency_p99": 0.0091, - "recall": 0.9736 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 12, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 861.8295, - "qps": 4742.1617, - "serial_latency_p99": 0.0091, - "recall": 0.9936 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 14, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 888.3719, - "qps": 6054.4428, - "serial_latency_p99": 0.0081, - "recall": 0.9155 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 5, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 888.3719, - "qps": 4104.2598, - "serial_latency_p99": 0.0106, - "recall": 0.9506 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 7, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 888.3719, - "qps": 4252.1267, - "serial_latency_p99": 0.0092, - "recall": 0.9964 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 9, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 7920.7947, - "qps": 1685.3091, - "serial_latency_p99": 0.0133, - "recall": 0.9718 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 11, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 7920.7947, - "qps": 769.8991, - "serial_latency_p99": 0.0107, - "recall": 0.9884 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 13, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 7920.7947, - "qps": 945.4061, - "serial_latency_p99": 0.0109, - "recall": 0.9941 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 15, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 9149.0776, - "qps": 2214.9028, - "serial_latency_p99": 0.0084, - "recall": 0.9249 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 4, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 9149.0776, - "qps": 827.975, - "serial_latency_p99": 0.012, - "recall": 0.9692 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 6, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 9149.0776, - "qps": 776.9454, - "serial_latency_p99": 0.0114, - "recall": 0.9966 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "8cu-perf", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 8, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 2689.1534, - "qps": 269.5464, - "serial_latency_p99": 0.0098, - "recall": 0.9776 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-cap", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 10, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 2689.1534, - "qps": 240.0363, - "serial_latency_p99": 0.0106, - "recall": 0.9822 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-cap", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 12, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 2689.1534, - "qps": 218.0627, - "serial_latency_p99": 0.0114, - "recall": 0.9936 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-cap", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 14, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 2431.0279, - "qps": 392.8825, - "serial_latency_p99": 0.0069, - "recall": 0.9581 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-cap", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 5, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 2431.0279, - "qps": 343.8204, - "serial_latency_p99": 0.0084, - "recall": 0.968 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-cap", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 7, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 2431.0279, - "qps": 216.6773, - "serial_latency_p99": 0.0101, - "recall": 0.9968 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-cap", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 9, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1730.3904, - "qps": 503.2284, - "serial_latency_p99": 0.009, - "recall": 0.9677 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "2cu-cap", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 10, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1730.3904, - "qps": 413.3232, - "serial_latency_p99": 0.0096, - "recall": 0.981 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "2cu-cap", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 12, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1730.3904, - "qps": 425.5492, - "serial_latency_p99": 0.0102, - "recall": 0.9936 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "2cu-cap", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 14, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1779.991, - "qps": 789.1229, - "serial_latency_p99": 0.0056, - "recall": 0.9396 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "2cu-cap", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 5, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1779.991, - "qps": 571.4257, - "serial_latency_p99": 0.0077, - "recall": 0.9668 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "2cu-cap", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 7, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 1779.991, - "qps": 411.7653, - "serial_latency_p99": 0.0091, - "recall": 0.9968 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "2cu-cap", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 9, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 12608.007, - "qps": 98.0448, - "serial_latency_p99": 0.0161, - "recall": 0.9803 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "2cu-cap", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 11, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 12608.007, - "qps": 58.3152, - "serial_latency_p99": 0.028, - "recall": 0.9891 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "2cu-cap", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 13, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 12608.007, - "qps": 46.8304, - "serial_latency_p99": 0.0286, - "recall": 0.9941 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "2cu-cap", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 15, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 14895.0657, - "qps": 170.5693, - "serial_latency_p99": 0.0089, - "recall": 0.9605 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "2cu-cap", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 4, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 14895.0657, - "qps": 94.7766, - "serial_latency_p99": 0.0196, - "recall": 0.9843 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "2cu-cap", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 6, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 14895.0657, - "qps": 44.8695, - "serial_latency_p99": 0.0309, - "recall": 0.9966 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "2cu-cap", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 8, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 949.3081, - "qps": 722.0315, - "serial_latency_p99": 0.0077, - "recall": 0.976 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-perf", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 10, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 949.3081, - "qps": 467.5795, - "serial_latency_p99": 0.0088, - "recall": 0.9898 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-perf", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 12, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 949.3081, - "qps": 975.2503, - "serial_latency_p99": 0.0082, - "recall": 0.9936 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-perf", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 14, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 963.5924, - "qps": 873.3712, - "serial_latency_p99": 0.0067, - "recall": 0.9477 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-perf", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 5, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 960.7296, - "qps": 544.6203, - "serial_latency_p99": 0.0084, - "recall": 0.977 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-perf", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 7, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 960.7296, - "qps": 930.9164, - "serial_latency_p99": 0.0096, - "recall": 0.9968 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-perf", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 9, - "custom_case": {} - } - }, - "label": ":)" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-perf", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 4, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-perf", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 6, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-perf", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 8, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-perf", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 11, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-perf", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 13, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-perf", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 15, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-cap", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 4, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-cap", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 6, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-cap", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 8, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-cap", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 11, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-cap", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 13, - "custom_case": {} - } - }, - "label": "x" - }, - { - "metrics": { - "max_load_count": 0, - "load_duration": 0.0, - "qps": 0.0, - "serial_latency_p99": 0.0, - "recall": 0.0 - }, - "task_config": { - "db": "ZillizCloud", - "db_config": { - "db_label": "1cu-cap", - "version": "v2024.1", - "uri": "**********", - "user": "db_admin", - "password": "**********" - }, - "db_case_config": { - "index": "AUTOINDEX", - "metric_type": "COSINE" - }, - "case_config": { - "case_id": 15, - "custom_case": {} - } - }, - "label": "x" - } - ], - "file_fmt": "result_{}_{}_{}.json", - "timestamp": 1704038400.0 -} \ No newline at end of file diff --git a/vectordb_bench/results/ZillizCloud/result_20250613_standard_zillizcloud.json b/vectordb_bench/results/ZillizCloud/result_20250613_standard_zillizcloud.json new file mode 100644 index 000000000..cd67ebb3b --- /dev/null +++ b/vectordb_bench/results/ZillizCloud/result_20250613_standard_zillizcloud.json @@ -0,0 +1,6290 @@ +{ + "run_id": "d60af9965dec47739a5924d55bd3ca8d", + "task_label": "standard_2025", + "results": [ + { + "metrics": { + "max_load_count": 0, + "insert_duration": 261.5267, + "optimize_duration": 753.9993, + "load_duration": 1015.526, + "qps": 9704.4214, + "serial_latency_p99": 0.0025, + "recall": 0.9169, + "ndcg": 0.9298, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 316.48, + 1931.772, + 3947.764, + 6717.7591, + 8344.1689, + 8912.2377, + 9318.1808, + 9704.4214, + 9658.7394 + ], + "conc_latency_p99_list": [ + 0.003489400758699048, + 0.003630623890203424, + 0.0036421223498109595, + 0.00520220138212608, + 0.007379636998957706, + 0.009708452988052167, + 0.014022858690041173, + 0.01782999514798576, + 0.021013210280070787 + ], + "conc_latency_avg_list": [ + 0.00315554470250659, + 0.0025837060953860405, + 0.002527189503321937, + 0.0029674111191947098, + 0.0035745033505707093, + 0.004457622381314819, + 0.0060737177257919805, + 0.008029072931477838, + 0.010034096320605774 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 1 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 261.5267, + "optimize_duration": 753.9993, + "load_duration": 1015.526, + "qps": 9463.4991, + "serial_latency_p99": 0.0026, + "recall": 0.9393, + "ndcg": 0.9489, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 302.2486, + 1767.3028, + 3007.8, + 6319.0177, + 7654.6954, + 8534.4853, + 9307.3272, + 9463.4991, + 9127.6806 + ], + "conc_latency_p99_list": [ + 0.003683128499687882, + 0.0037075912410364244, + 0.004188143842584398, + 0.005374252397450612, + 0.00770531116053462, + 0.010048962999462674, + 0.014806666160438903, + 0.01868447945955267, + 0.02176790560042719 + ], + "conc_latency_avg_list": [ + 0.0033039773710451547, + 0.002824024851822295, + 0.003159446049851744, + 0.0031551992673815948, + 0.0038985052193128726, + 0.004654319779715259, + 0.006307529062491997, + 0.008151252751261526, + 0.010547634149878646 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 2 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 261.5267, + "optimize_duration": 753.9993, + "load_duration": 1015.526, + "qps": 9194.9922, + "serial_latency_p99": 0.0027, + "recall": 0.9543, + "ndcg": 0.9616, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 418.515, + 1599.5435, + 3598.5261, + 6101.7088, + 7578.8921, + 8217.8541, + 9000.7685, + 9079.6692, + 9194.9922 + ], + "conc_latency_p99_list": [ + 0.002743048418778926, + 0.003811952280811966, + 0.00370975422010816, + 0.005627912879717769, + 0.008256554249783221, + 0.010774479839310496, + 0.01543271399987134, + 0.01932160449985534, + 0.02324684659924968 + ], + "conc_latency_avg_list": [ + 0.002385618288162752, + 0.0031208209393127078, + 0.0026413052232689776, + 0.003267708551824355, + 0.0039406149349344366, + 0.0048382244156899325, + 0.0065780353923480675, + 0.008475722667087458, + 0.010474621336238818 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 3 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 261.5267, + "optimize_duration": 753.9993, + "load_duration": 1015.526, + "qps": 8779.8779, + "serial_latency_p99": 0.0029, + "recall": 0.9685, + "ndcg": 0.9733, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 401.1508, + 1631.8403, + 3287.7465, + 5316.4377, + 6564.2223, + 7647.0406, + 8412.7975, + 8681.9304, + 8779.8779 + ], + "conc_latency_p99_list": [ + 0.0028018492820774564, + 0.003937064278397883, + 0.004262597860506501, + 0.0060385300012058, + 0.008731867500864608, + 0.011426367560488872, + 0.016466031518066335, + 0.021389961049862877, + 0.02568652452926481 + ], + "conc_latency_avg_list": [ + 0.0024894055354616616, + 0.0030588078886540863, + 0.0030344079928008674, + 0.0037512742121145032, + 0.0043425659686912215, + 0.005197448221281798, + 0.007019927240459036, + 0.009002809509075622, + 0.010975876492656012 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 4 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 261.5267, + "optimize_duration": 753.9993, + "load_duration": 1015.526, + "qps": 8153.4648, + "serial_latency_p99": 0.003, + "recall": 0.9757, + "ndcg": 0.9797, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 276.3471, + 1280.8332, + 2682.2578, + 5465.0991, + 6657.9098, + 7147.4999, + 7473.44, + 7976.4881, + 8153.4648 + ], + "conc_latency_p99_list": [ + 0.004023005141134493, + 0.004402717000812119, + 0.00487524802058033, + 0.006413850348508275, + 0.009425436149831509, + 0.012294427808919864, + 0.018093567679752596, + 0.023304151630472916, + 0.028990432242644595 + ], + "conc_latency_avg_list": [ + 0.0036141020840187747, + 0.00373359608368641, + 0.0037204073492682656, + 0.003648467936702471, + 0.004486077656159847, + 0.005557460767548427, + 0.007598120302398434, + 0.009639271254455939, + 0.01183764093195763 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 5 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 261.5267, + "optimize_duration": 753.9993, + "load_duration": 1015.526, + "qps": 6848.5254, + "serial_latency_p99": 0.0044, + "recall": 0.9835, + "ndcg": 0.9864, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 249.6732, + 1474.7716, + 2868.2148, + 4660.7825, + 5927.0176, + 6350.0819, + 6702.6421, + 6837.0113, + 6848.5254 + ], + "conc_latency_p99_list": [ + 0.00458599035962834, + 0.004444857600101386, + 0.004651562902072328, + 0.0071047105491743815, + 0.010493072840290555, + 0.014094214100259699, + 0.0209603712799435, + 0.028015291610827244, + 0.033395276517840106 + ], + "conc_latency_avg_list": [ + 0.0040003504651356814, + 0.0033849920410009547, + 0.00331449651860339, + 0.004279508661249433, + 0.005043042758622036, + 0.006261079695256975, + 0.008832638727502035, + 0.011413603662962802, + 0.013832915035513486 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 6 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 261.5267, + "optimize_duration": 753.9993, + "load_duration": 1015.526, + "qps": 6124.4431, + "serial_latency_p99": 0.0038, + "recall": 0.9873, + "ndcg": 0.9896, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 237.2, + 1149.4245, + 2370.2814, + 4418.7369, + 5239.2066, + 5272.0813, + 5549.6495, + 5924.7585, + 6124.4431 + ], + "conc_latency_p99_list": [ + 0.004785326481287484, + 0.005062950489082138, + 0.005544700718601233, + 0.007941588501125807, + 0.011658581997835427, + 0.015745819050425765, + 0.024691753346633042, + 0.03169662963860899, + 0.03684433942835311 + ], + "conc_latency_avg_list": [ + 0.004210859751671394, + 0.004343390144320536, + 0.004210967258468222, + 0.0045139876311441115, + 0.005701361721208284, + 0.007199706154519862, + 0.010225284894052862, + 0.013188149333010084, + 0.015736682704950457 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 7 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 261.5267, + "optimize_duration": 753.9993, + "load_duration": 1015.526, + "qps": 5186.8135, + "serial_latency_p99": 0.0053, + "recall": 0.9893, + "ndcg": 0.9913, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 280.6548, + 1285.5777, + 2482.4969, + 3957.9832, + 4810.5503, + 4996.1533, + 5176.9866, + 5171.7067, + 5186.8135 + ], + "conc_latency_p99_list": [ + 0.004152107281261123, + 0.005346049155923539, + 0.005538785279495641, + 0.008655988810860434, + 0.013338722899788966, + 0.01787870473926887, + 0.027657456735978504, + 0.03382840061043682, + 0.040950015698035705 + ], + "conc_latency_avg_list": [ + 0.0035586266123749442, + 0.003883513250844417, + 0.004020028535260059, + 0.005040609378459062, + 0.006213752961897755, + 0.007966579462086532, + 0.011444027615365912, + 0.014498529099389625, + 0.017861587605206234 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 8 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 261.5267, + "optimize_duration": 753.9993, + "load_duration": 1015.526, + "qps": 4898.2048, + "serial_latency_p99": 0.0056, + "recall": 0.9904, + "ndcg": 0.9922, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 202.4908, + 1155.2978, + 2300.4417, + 3419.5225, + 4330.2531, + 4479.5867, + 4667.4844, + 4580.7709, + 4898.2048 + ], + "conc_latency_p99_list": [ + 0.005706084268604172, + 0.005607872537366346, + 0.005976956876766051, + 0.009363154400489291, + 0.014783053322462367, + 0.020197440302581526, + 0.03032778007189335, + 0.036684535879758194, + 0.04259529887989629 + ], + "conc_latency_avg_list": [ + 0.004932904398891112, + 0.004321622453280767, + 0.004339001804574998, + 0.0055604633668235245, + 0.006904792945112498, + 0.008886108179666534, + 0.012703799580071921, + 0.016318549483188796, + 0.01974911081811399 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 9 + }, + "case_config": { + "case_id": 5, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 261.5267, + "optimize_duration": 753.9993, + "load_duration": 1015.526, + "qps": 9773.6593, + "serial_latency_p99": 0.0037, + "recall": 0.9955, + "ndcg": 0.9968, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 440.4414, + 1647.3494, + 3535.169, + 5819.4072, + 7270.1952, + 8469.8435, + 9524.3411, + 9773.6593, + 9554.3768 + ], + "conc_latency_p99_list": [ + 0.002577342949443846, + 0.0037217354716267415, + 0.003979771226004231, + 0.00574488876503893, + 0.0078669457471551, + 0.010159293771503144, + 0.014228192100199506, + 0.01829144147231998, + 0.02131603067740799 + ], + "conc_latency_avg_list": [ + 0.002267110172388146, + 0.003029865512010611, + 0.0026888036101079607, + 0.0034269701292704755, + 0.004109452647780382, + 0.004692168318693944, + 0.006221462706886718, + 0.007976906880586459, + 0.010088835722077921 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.001 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 261.5267, + "optimize_duration": 753.9993, + "load_duration": 1015.526, + "qps": 9081.1518, + "serial_latency_p99": 0.003, + "recall": 0.9943, + "ndcg": 0.9959, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 298.5712, + 1893.054, + 3413.6164, + 5510.6157, + 7304.7634, + 7940.9139, + 8614.6067, + 9081.1518, + 9072.5845 + ], + "conc_latency_p99_list": [ + 0.003737475997331785, + 0.0037780946371640312, + 0.004258470999047859, + 0.005984946849275729, + 0.00818996892019641, + 0.01053562169727229, + 0.014943244759051599, + 0.018613143279435414, + 0.022502940341219073 + ], + "conc_latency_avg_list": [ + 0.0033445511501306786, + 0.002636599590212312, + 0.002922664661032498, + 0.003618360623746242, + 0.004090629128962825, + 0.005001241244541393, + 0.006556129735225091, + 0.008300849949631699, + 0.010638661540229005 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.002 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 261.5267, + "optimize_duration": 753.9993, + "load_duration": 1015.526, + "qps": 8455.2896, + "serial_latency_p99": 0.004, + "recall": 0.9921, + "ndcg": 0.9942, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 274.545, + 1646.0797, + 2856.5165, + 5451.2937, + 6766.6553, + 7238.9078, + 8096.9975, + 8355.2759, + 8455.2896 + ], + "conc_latency_p99_list": [ + 0.00413616330421064, + 0.00401528277812758, + 0.004876473260519559, + 0.006280867997702444, + 0.009070846654794856, + 0.011840806598775091, + 0.016399328601255533, + 0.0208301552519697, + 0.024770659177156613 + ], + "conc_latency_avg_list": [ + 0.0036375716057345567, + 0.00303208297520401, + 0.003493588406256312, + 0.003658151022337062, + 0.00441579553585374, + 0.005488094637473112, + 0.007312952589141246, + 0.009343698734734313, + 0.01139474206128049 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.005 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 261.5267, + "optimize_duration": 753.9993, + "load_duration": 1015.526, + "qps": 7610.0519, + "serial_latency_p99": 0.0033, + "recall": 0.9903, + "ndcg": 0.9929, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 360.7535, + 1364.1724, + 2435.0753, + 4605.8456, + 5731.3633, + 6418.2553, + 7083.1357, + 7145.7958, + 7610.0519 + ], + "conc_latency_p99_list": [ + 0.003292209589053527, + 0.00475087727085338, + 0.005676962278375878, + 0.007410976898972874, + 0.010239582348731355, + 0.013297085160738803, + 0.018856629678630263, + 0.024606706481717994, + 0.030090755063138203 + ], + "conc_latency_avg_list": [ + 0.002768336817606511, + 0.0036593464882454436, + 0.004098402006661435, + 0.004330032120034815, + 0.0052144765270180264, + 0.006193831578188605, + 0.00837906453848952, + 0.010540270624565667, + 0.012658784934118594 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.01 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 261.5267, + "optimize_duration": 753.9993, + "load_duration": 1015.526, + "qps": 7589.664, + "serial_latency_p99": 0.0038, + "recall": 0.9235, + "ndcg": 0.934, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 251.5156, + 1331.9035, + 2567.5304, + 4863.0438, + 5577.6931, + 6384.9956, + 6887.335, + 7386.6078, + 7589.664 + ], + "conc_latency_p99_list": [ + 0.004447761932606226, + 0.004485363039129886, + 0.005287119843997056, + 0.00706968445010716, + 0.010473609903419836, + 0.013495494357412088, + 0.019466659059980906, + 0.024769910154500346, + 0.02960712759639139 + ], + "conc_latency_avg_list": [ + 0.003971096920542138, + 0.0037478134115411944, + 0.0038870361245953323, + 0.004101396772225186, + 0.005111316110886633, + 0.006223668344951101, + 0.008618922627847558, + 0.010567275195568061, + 0.012665978711043848 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.02 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 261.5267, + "optimize_duration": 753.9993, + "load_duration": 1015.526, + "qps": 6750.2495, + "serial_latency_p99": 0.0044, + "recall": 0.9105, + "ndcg": 0.9205, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 247.9798, + 1486.1133, + 2408.9981, + 4502.4851, + 5065.465, + 5536.7126, + 6390.8299, + 6586.8377, + 6750.2495 + ], + "conc_latency_p99_list": [ + 0.004678286461276003, + 0.004330151519097852, + 0.005487894870602752, + 0.009123132515160249, + 0.013280873120529576, + 0.015730040038615722, + 0.0212207965014386, + 0.028174651395238472, + 0.03464033055242904 + ], + "conc_latency_avg_list": [ + 0.0040274200437701345, + 0.003358401179275766, + 0.004143247878625875, + 0.00442874367776759, + 0.005900722488960889, + 0.006865572944251661, + 0.009233971089391507, + 0.011827073056792859, + 0.014318418615219736 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.05 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 261.5267, + "optimize_duration": 753.9993, + "load_duration": 1015.526, + "qps": 5506.1808, + "serial_latency_p99": 0.0055, + "recall": 0.9193, + "ndcg": 0.9338, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 220.6454, + 1343.5687, + 2190.6576, + 4029.2698, + 4653.8945, + 5148.5776, + 5390.1753, + 5465.2618, + 5506.1808 + ], + "conc_latency_p99_list": [ + 0.005227974839799572, + 0.004963128318195223, + 0.006750358079880241, + 0.0096191686293605, + 0.013617134921369145, + 0.01675103867404687, + 0.025566332196467603, + 0.03428883460088401, + 0.03934171804110515 + ], + "conc_latency_avg_list": [ + 0.004526929335629695, + 0.003715728696336781, + 0.004556541547689115, + 0.004950812551162553, + 0.006417888532504203, + 0.007725162994769883, + 0.010977864777391455, + 0.014264379140743473, + 0.016838589634100988 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.1 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 261.5267, + "optimize_duration": 753.9993, + "load_duration": 1015.526, + "qps": 6860.8577, + "serial_latency_p99": 0.0047, + "recall": 0.9226, + "ndcg": 0.9359, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 316.9605, + 1343.2966, + 2856.1091, + 4313.3256, + 5356.4958, + 5906.671, + 6548.9194, + 6468.0578, + 6860.8577 + ], + "conc_latency_p99_list": [ + 0.0037907035576063212, + 0.00478001015399059, + 0.0051615560023492435, + 0.008314900161130947, + 0.012217005001730286, + 0.015586223299760624, + 0.02143115329417921, + 0.028367364548321335, + 0.033541636278751 + ], + "conc_latency_avg_list": [ + 0.0031509553373910504, + 0.003716141051949268, + 0.0034942002592514955, + 0.004623906599701513, + 0.005578422171699753, + 0.006734368484163413, + 0.009108987542364097, + 0.0116603705633854, + 0.01389971736496045 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.2 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 261.5267, + "optimize_duration": 753.9993, + "load_duration": 1015.526, + "qps": 8468.4611, + "serial_latency_p99": 0.0031, + "recall": 0.8925, + "ndcg": 0.9086, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 279.7382, + 1512.6872, + 3215.0514, + 5129.3436, + 6392.1705, + 7140.4469, + 7818.959, + 8428.9938, + 8468.4611 + ], + "conc_latency_p99_list": [ + 0.004143827338266419, + 0.004607606401259542, + 0.005189361999509848, + 0.007334411201009058, + 0.00981477741341224, + 0.013026459062966756, + 0.018644948683504487, + 0.02245290092207141, + 0.026631316151542694 + ], + "conc_latency_avg_list": [ + 0.003570228337524447, + 0.0033001565118401903, + 0.00309206333116793, + 0.0038869957114734664, + 0.004675767061988868, + 0.005569727385843004, + 0.007561000310880382, + 0.009269846590713178, + 0.011397850596689489 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.5 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 560.466, + "optimize_duration": 673.4421, + "load_duration": 1233.9081, + "qps": 10089.4308, + "serial_latency_p99": 0.0026, + "recall": 0.9934, + "ndcg": 0.995, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 301.5589, + 1753.8963, + 3351.7182, + 6719.434, + 8302.5958, + 9050.243, + 10089.4308, + 9930.3333, + 9130.8746 + ], + "conc_latency_p99_list": [ + 0.004120293159503487, + 0.004035422801243836, + 0.004452557814729515, + 0.005472210197185626, + 0.00745348393713357, + 0.00984966567848459, + 0.013906377736857375, + 0.017616427392858902, + 0.020758999521785823 + ], + "conc_latency_avg_list": [ + 0.003311671678382902, + 0.002845363922995857, + 0.0029770398082771357, + 0.0029671067631446457, + 0.0035978994086276134, + 0.004180498197656481, + 0.0058625419564395966, + 0.007855047846364198, + 0.01053906301690295 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-partition_key", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.001 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 560.466, + "optimize_duration": 673.4421, + "load_duration": 1233.9081, + "qps": 10557.4373, + "serial_latency_p99": 0.0027, + "recall": 0.9393, + "ndcg": 0.9476, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 491.049, + 1813.3515, + 3326.2948, + 6683.6168, + 8781.8335, + 9447.4194, + 10557.4373, + 9546.1924, + 9704.7689 + ], + "conc_latency_p99_list": [ + 0.002425455652337406, + 0.0035593600005086046, + 0.004291364103846715, + 0.005441655600589005, + 0.007239447070605819, + 0.009250738600894691, + 0.01304742102976889, + 0.01707642012013821, + 0.02041322323333589 + ], + "conc_latency_avg_list": [ + 0.0020333511619985128, + 0.0027525795044520285, + 0.002999578324194531, + 0.002982309298426352, + 0.0034013866425970556, + 0.00402064299256204, + 0.005607672730511889, + 0.007844332068070566, + 0.009950249902790956 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-partition_key", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.002 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 560.466, + "optimize_duration": 673.4421, + "load_duration": 1233.9081, + "qps": 9805.0401, + "serial_latency_p99": 0.0026, + "recall": 0.9257, + "ndcg": 0.9357, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 493.2162, + 2293.0486, + 3387.5778, + 6383.1914, + 8495.6686, + 9131.4068, + 9805.0401, + 9767.2307, + 9413.7332 + ], + "conc_latency_p99_list": [ + 0.002379914499397273, + 0.0029166609832464068, + 0.004339208432211307, + 0.00531044274903252, + 0.007306510360212999, + 0.009634101699484742, + 0.014160893399093795, + 0.017475060481810974, + 0.020493118877566302 + ], + "conc_latency_avg_list": [ + 0.002024350873747714, + 0.0021764625585862034, + 0.0029453131926756317, + 0.003123570885141119, + 0.003514571036189243, + 0.004147324937122531, + 0.006038083005781032, + 0.0079963295266158, + 0.010232430884772355 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-partition_key", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.005 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 560.466, + "optimize_duration": 673.4421, + "load_duration": 1233.9081, + "qps": 10020.5299, + "serial_latency_p99": 0.0026, + "recall": 0.9788, + "ndcg": 0.9833, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 464.1265, + 2138.4017, + 3300.8104, + 6549.2365, + 8391.6217, + 8658.8351, + 9971.3013, + 10020.5299, + 9410.396 + ], + "conc_latency_p99_list": [ + 0.002622232757566962, + 0.0034374746595131, + 0.004287367561482835, + 0.005566385267666186, + 0.007679281996388454, + 0.009854602040722969, + 0.013919163036334793, + 0.01767971348046561, + 0.02079326605962707 + ], + "conc_latency_avg_list": [ + 0.0021513615237860645, + 0.002333536207257488, + 0.0030227332134616608, + 0.0030430583998758763, + 0.003559356143224123, + 0.004380378985449284, + 0.005941445579344076, + 0.007771052196938918, + 0.009946875281662451 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-partition_key", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.01 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 560.466, + "optimize_duration": 673.4421, + "load_duration": 1233.9081, + "qps": 10041.0338, + "serial_latency_p99": 0.0027, + "recall": 0.9693, + "ndcg": 0.9754, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 473.5785, + 1549.4934, + 3615.2443, + 6252.6074, + 8192.3404, + 9114.8321, + 9949.8602, + 10041.0338, + 9701.8908 + ], + "conc_latency_p99_list": [ + 0.0025054446909052795, + 0.003964638639299665, + 0.0040947790514473965, + 0.005634508699586149, + 0.007593685001484113, + 0.009971025249251394, + 0.01404055800048809, + 0.017582572136307133, + 0.020558674518251767 + ], + "conc_latency_avg_list": [ + 0.0021082491010956956, + 0.003221269634504649, + 0.002759589821420869, + 0.0031884359372492737, + 0.0036466010131465006, + 0.0043578029723181115, + 0.005904113241399662, + 0.00784949408724515, + 0.00992050082064174 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-partition_key", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.02 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 560.466, + "optimize_duration": 673.4421, + "load_duration": 1233.9081, + "qps": 9861.9686, + "serial_latency_p99": 0.0026, + "recall": 0.955, + "ndcg": 0.963, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 462.1929, + 2053.1676, + 3942.786, + 6499.4328, + 7741.1506, + 8594.2197, + 9835.0931, + 9861.9686, + 9432.4305 + ], + "conc_latency_p99_list": [ + 0.0025393456940946634, + 0.003409789892903064, + 0.0039090861921431495, + 0.0057103434971941015, + 0.007808439196378462, + 0.010239055119454863, + 0.01475203969734139, + 0.018468949822563523, + 0.021990215039695716 + ], + "conc_latency_avg_list": [ + 0.0021603535911756317, + 0.0024308501911775786, + 0.0025298371173651684, + 0.0030675433477564064, + 0.003859721605316449, + 0.00441378960438928, + 0.006028296160009628, + 0.007910052819394948, + 0.010219173139614757 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-partition_key", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.05 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 560.466, + "optimize_duration": 673.4421, + "load_duration": 1233.9081, + "qps": 9507.9991, + "serial_latency_p99": 0.0028, + "recall": 0.9453, + "ndcg": 0.9544, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 444.6826, + 1784.7887, + 3988.5699, + 6144.6093, + 7755.7951, + 8179.7066, + 9035.7377, + 9507.9991, + 9439.0852 + ], + "conc_latency_p99_list": [ + 0.0026217483921209344, + 0.0037886830490606372, + 0.003935028371488441, + 0.005680175043817143, + 0.00867969578182966, + 0.010488722523441539, + 0.015315433149080472, + 0.019454779771403953, + 0.023576263547511165 + ], + "conc_latency_avg_list": [ + 0.002245551751864168, + 0.002796474662229801, + 0.0025006867551888847, + 0.0032450206323157166, + 0.0038513666085506592, + 0.004630909529887228, + 0.006261018992728062, + 0.008205529087603759, + 0.010221179236688853 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-partition_key", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.1 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 560.466, + "optimize_duration": 673.4421, + "load_duration": 1233.9081, + "qps": 9428.4531, + "serial_latency_p99": 0.0026, + "recall": 0.9331, + "ndcg": 0.9436, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 440.2609, + 2069.6484, + 3732.4623, + 6094.7111, + 7584.8865, + 8383.8398, + 9119.7702, + 9428.4531, + 9163.3089 + ], + "conc_latency_p99_list": [ + 0.0026952438845182767, + 0.003149576502619315, + 0.004148402102146061, + 0.00644882453911126, + 0.008800019872433042, + 0.011569655568164298, + 0.01625659936165895, + 0.020326728798681935, + 0.023228999639250104 + ], + "conc_latency_avg_list": [ + 0.002268019151370056, + 0.002411547829939476, + 0.0026730088413622903, + 0.003271375270327803, + 0.00394000703776353, + 0.0047409327488048625, + 0.006469711061754865, + 0.008293673618728644, + 0.010281069419779815 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-partition_key", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.2 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 560.466, + "optimize_duration": 673.4421, + "load_duration": 1233.9081, + "qps": 9048.6431, + "serial_latency_p99": 0.0039, + "recall": 0.9216, + "ndcg": 0.9337, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 431.969, + 2029.6657, + 2864.6543, + 5885.9537, + 7258.7919, + 8162.6339, + 8979.9728, + 8952.3402, + 9048.6431 + ], + "conc_latency_p99_list": [ + 0.002593809844984207, + 0.003006730374472681, + 0.004567161117665818, + 0.005789556708114107, + 0.00828339132131077, + 0.01067148012458347, + 0.015291324068384718, + 0.019616145856853097, + 0.022960674363421214 + ], + "conc_latency_avg_list": [ + 0.0023116565276770666, + 0.0024589861683885195, + 0.003372779172483356, + 0.003387803512752006, + 0.004117207520943822, + 0.004872478646684868, + 0.0065960338327797985, + 0.00873581126963286, + 0.010644804433399125 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-partition_key", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Medium Cohere (768dim, 1M)", + "label_percentage": 0.5 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 5962.2419, + "optimize_duration": 1620.5805, + "load_duration": 7582.8223, + "qps": 8695.2765, + "serial_latency_p99": 0.0043, + "recall": 0.9603, + "ndcg": 0.9677, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 254.2235, + 1295.0507, + 2652.2639, + 5733.7893, + 6694.72, + 7549.668, + 8507.4085, + 8614.0069, + 8695.2765 + ], + "conc_latency_p99_list": [ + 0.004290025417503784, + 0.0049749610095386745, + 0.004958201188310344, + 0.005849076520607921, + 0.008438526201643976, + 0.011174128679995185, + 0.015520126641495146, + 0.019544252949526712, + 0.0229224258998147 + ], + "conc_latency_avg_list": [ + 0.003928518528911093, + 0.003854824949152844, + 0.00376250414764781, + 0.003477680304998273, + 0.004461620547091561, + 0.0052611100214170715, + 0.006969254247723128, + 0.008856325486621321, + 0.010983367431046413 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-partition_key", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.001 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 5962.2419, + "optimize_duration": 1620.5805, + "load_duration": 7582.8223, + "qps": 9244.1135, + "serial_latency_p99": 0.0042, + "recall": 0.9724, + "ndcg": 0.9783, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 389.1008, + 1901.297, + 3007.2954, + 6136.2887, + 8025.3223, + 8866.5356, + 9244.1135, + 8908.8585, + 8870.0303 + ], + "conc_latency_p99_list": [ + 0.00295199312036857, + 0.003666217737700205, + 0.00458198255946627, + 0.005548763198748933, + 0.007472231998690404, + 0.009877307381902931, + 0.01428791658130649, + 0.017352266439556844, + 0.02069346915002825 + ], + "conc_latency_avg_list": [ + 0.002566253764066292, + 0.0026250990307494686, + 0.0033182243811591514, + 0.00324918656994336, + 0.0037228168206677762, + 0.004482402903478176, + 0.006361563939334886, + 0.00865349010436204, + 0.010852228995638166 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-partition_key", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.002 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 5962.2419, + "optimize_duration": 1620.5805, + "load_duration": 7582.8223, + "qps": 9289.0118, + "serial_latency_p99": 0.0042, + "recall": 0.9574, + "ndcg": 0.9652, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 382.0834, + 1545.1625, + 3538.8594, + 6200.867, + 8009.1489, + 8609.5784, + 9289.0118, + 9152.2345, + 8849.9155 + ], + "conc_latency_p99_list": [ + 0.002987769600076717, + 0.0039817531191511085, + 0.004174642502111952, + 0.005411452799307872, + 0.007863285581188392, + 0.010465657060958622, + 0.014893052439583708, + 0.018844572730959037, + 0.02155722900017282 + ], + "conc_latency_avg_list": [ + 0.002613600987705965, + 0.00323046351990638, + 0.0028191352111886342, + 0.00321501948076307, + 0.0037283721200453575, + 0.004616157113372974, + 0.006369690217651501, + 0.008212230538910303, + 0.010550628563324073 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-partition_key", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.005 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 5962.2419, + "optimize_duration": 1620.5805, + "load_duration": 7582.8223, + "qps": 9374.8941, + "serial_latency_p99": 0.0042, + "recall": 0.9425, + "ndcg": 0.9526, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 270.5027, + 1874.3581, + 2904.1686, + 6184.6787, + 7414.8405, + 8425.7821, + 8805.2815, + 9374.8941, + 9235.7763 + ], + "conc_latency_p99_list": [ + 0.0040909598392318, + 0.0037201197618560392, + 0.004421225659789342, + 0.0054953261202899725, + 0.008041230752496631, + 0.010834711761563079, + 0.0150421849371196, + 0.019125744718512572, + 0.0224911583600624 + ], + "conc_latency_avg_list": [ + 0.003691858484983191, + 0.002662771093895926, + 0.003271968083063315, + 0.003223785878111255, + 0.004028478482942186, + 0.004713706157825403, + 0.006437188570256514, + 0.008328668563341475, + 0.010421247793241953 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-partition_key", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.01 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 5962.2419, + "optimize_duration": 1620.5805, + "load_duration": 7582.8223, + "qps": 9368.1325, + "serial_latency_p99": 0.0038, + "recall": 0.9292, + "ndcg": 0.9403, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 267.0278, + 1509.0396, + 2678.9755, + 5633.0687, + 7454.5283, + 8303.9699, + 9170.4649, + 9368.1325, + 9057.7337 + ], + "conc_latency_p99_list": [ + 0.004173219557778793, + 0.00415495203149476, + 0.004762636760315218, + 0.005755300149176037, + 0.008333134719687222, + 0.010706265271728622, + 0.015146266648662221, + 0.019109411801473433, + 0.02208015628071732 + ], + "conc_latency_avg_list": [ + 0.0037402337795340405, + 0.003150754717439926, + 0.003548245308171402, + 0.0035404174358017525, + 0.004003914269697601, + 0.004780430188018965, + 0.006442099404446145, + 0.008327493980002442, + 0.01047571381059081 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-partition_key", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.02 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 5962.2419, + "optimize_duration": 1620.5805, + "load_duration": 7582.8223, + "qps": 9220.3627, + "serial_latency_p99": 0.0038, + "recall": 0.9081, + "ndcg": 0.9212, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 411.4225, + 1949.8517, + 3673.0154, + 5637.8487, + 7312.3613, + 8144.3308, + 8892.9346, + 9220.3627, + 9127.1475 + ], + "conc_latency_p99_list": [ + 0.0028859092797210903, + 0.003063338641150039, + 0.0037194755187738193, + 0.005742244120046961, + 0.008229559029241497, + 0.011033196801145116, + 0.015391908401616087, + 0.019350335319322776, + 0.022807500500675815 + ], + "conc_latency_avg_list": [ + 0.0024271865216396902, + 0.002559789999950984, + 0.002716231018391776, + 0.003537029399905597, + 0.004086578609656591, + 0.0048809539202887805, + 0.0065705925121270275, + 0.00843916194988846, + 0.010548014534820094 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-partition_key", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.05 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 5962.2419, + "optimize_duration": 1620.5805, + "load_duration": 7582.8223, + "qps": 8633.8949, + "serial_latency_p99": 0.0041, + "recall": 0.8928, + "ndcg": 0.9074, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 288.5279, + 1719.4431, + 2811.3552, + 5757.7258, + 6949.5839, + 7729.6314, + 8280.1594, + 8562.1228, + 8633.8949 + ], + "conc_latency_p99_list": [ + 0.003802594057851821, + 0.0038520399028493546, + 0.004540082837993396, + 0.005908404397669056, + 0.008629464758923865, + 0.01158967702147493, + 0.015847183551486524, + 0.02003877184979503, + 0.024059910449141103 + ], + "conc_latency_avg_list": [ + 0.0034614087711447054, + 0.002902729754846898, + 0.003549688506525309, + 0.0034628667900877875, + 0.004299848632104843, + 0.005138771821287127, + 0.006954305481515403, + 0.008797347563458124, + 0.010874462157333639 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-partition_key", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.1 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 5962.2419, + "optimize_duration": 1620.5805, + "load_duration": 7582.8223, + "qps": 6820.6863, + "serial_latency_p99": 0.0032, + "recall": 0.9159, + "ndcg": 0.9248, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 351.0262, + 1525.3613, + 2583.5712, + 4108.3594, + 4831.4652, + 5195.779, + 6150.7098, + 6363.2968, + 6820.6863 + ], + "conc_latency_p99_list": [ + 0.003345775970374234, + 0.004217423321133538, + 0.005247439032100374, + 0.008924198697786783, + 0.013110358761587126, + 0.016151695159860528, + 0.021389665400056386, + 0.027015561380758325, + 0.03171861507857102 + ], + "conc_latency_avg_list": [ + 0.0028450376323789313, + 0.0032727600031104307, + 0.0038627056569949594, + 0.004855685931753599, + 0.00618667905719038, + 0.007317792780211714, + 0.009609665135663047, + 0.01190367844122976, + 0.013717531945212465 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-partition_key", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.2 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 5962.2419, + "optimize_duration": 1620.5805, + "load_duration": 7582.8223, + "qps": 3938.6004, + "serial_latency_p99": 0.0037, + "recall": 0.9196, + "ndcg": 0.9253, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 254.4328, + 1055.2951, + 1951.7332, + 2319.1739, + 2682.3906, + 3004.8382, + 3425.4241, + 3721.6149, + 3938.6004 + ], + "conc_latency_p99_list": [ + 0.004519681342644616, + 0.005808136349696736, + 0.008862463751029276, + 0.016211836919828786, + 0.022598761981062122, + 0.027642400040713246, + 0.03532528061201447, + 0.041608102380414444, + 0.048943750238540806 + ], + "conc_latency_avg_list": [ + 0.003925567922234654, + 0.004730470212539204, + 0.005113768719160474, + 0.008603492797795405, + 0.01115052077127321, + 0.013231647022558083, + 0.017275761965922374, + 0.020954334251646084, + 0.024499308782253915 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf-partition_key", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": true, + "num_partitions": 64, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.5 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4225.7562, + "optimize_duration": 1456.2294, + "load_duration": 5681.9856, + "qps": 3957.0757, + "serial_latency_p99": 0.0027, + "recall": 0.932, + "ndcg": 0.9356, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 419.361, + 1334.4745, + 1550.3904, + 2552.6813, + 3108.2917, + 3350.8492, + 3710.0422, + 3893.5411, + 3957.0757 + ], + "conc_latency_p99_list": [ + 0.0026540532501030624, + 0.004947375369906694, + 0.009614872399990872, + 0.013492345480258348, + 0.01677392020003026, + 0.02012457929017728, + 0.02668832603002101, + 0.032485457839848096, + 0.03849125690010169 + ], + "conc_latency_avg_list": [ + 0.002380754735726192, + 0.003740705991218102, + 0.006438204827892, + 0.007815330602281673, + 0.009617387987051201, + 0.011853545061625354, + 0.01595140442078024, + 0.019987424973373576, + 0.023781362345861194 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 1 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4225.7562, + "optimize_duration": 1456.2294, + "load_duration": 5681.9856, + "qps": 3539.2869, + "serial_latency_p99": 0.0043, + "recall": 0.9471, + "ndcg": 0.9497, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 404.7334, + 1437.162, + 1853.3169, + 2578.1789, + 2714.5391, + 3048.0088, + 3347.0855, + 3512.491, + 3539.2869 + ], + "conc_latency_p99_list": [ + 0.0027532103200246633, + 0.00505832540011397, + 0.007440276479837842, + 0.012550284719982307, + 0.018015394429912693, + 0.022034093910201592, + 0.029166633749923676, + 0.03591149632016823, + 0.042902185519978965 + ], + "conc_latency_avg_list": [ + 0.0024670617076742, + 0.003473248301772258, + 0.005385566896131411, + 0.007739138474192453, + 0.010507342591807012, + 0.013049724848704739, + 0.017725571980672004, + 0.02236461887738811, + 0.02664312777608926 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 2 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4225.7562, + "optimize_duration": 1456.2294, + "load_duration": 5681.9856, + "qps": 3154.6501, + "serial_latency_p99": 0.0039, + "recall": 0.9565, + "ndcg": 0.9586, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 267.1678, + 1441.7534, + 1673.2947, + 2397.942, + 2611.7936, + 2771.2549, + 2892.2094, + 3154.6501, + 3130.8121 + ], + "conc_latency_p99_list": [ + 0.004102769720102514, + 0.005119719020085541, + 0.007863903290117371, + 0.013464445300069195, + 0.019435442580011116, + 0.023709990960178395, + 0.03178576359996452, + 0.04005444816008095, + 0.04736919472008593 + ], + "conc_latency_avg_list": [ + 0.0037378313708950413, + 0.0034626399079898283, + 0.005963948479417637, + 0.008320695160894996, + 0.011451192510276107, + 0.01434331964258494, + 0.01958688870456675, + 0.024763059605503582, + 0.029772320040206055 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 3 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4225.7562, + "optimize_duration": 1456.2294, + "load_duration": 5681.9856, + "qps": 2743.4561, + "serial_latency_p99": 0.0043, + "recall": 0.9681, + "ndcg": 0.9699, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 260.0634, + 1293.7439, + 1626.5034, + 2097.621, + 2256.213, + 2288.9749, + 2571.8533, + 2594.0163, + 2743.4561 + ], + "conc_latency_p99_list": [ + 0.004216014330263533, + 0.005679349320321309, + 0.008387185859946836, + 0.015441323660070345, + 0.022420493819736268, + 0.027373204860086837, + 0.037205323399575746, + 0.04600767780029856, + 0.05655593189980206 + ], + "conc_latency_avg_list": [ + 0.0038402235035849826, + 0.003858581647738822, + 0.0061368915464091595, + 0.009511813520157399, + 0.01324650532721522, + 0.016605450056513444, + 0.023049488433792675, + 0.02910308621458791, + 0.03521191166211166 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 4 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4225.7562, + "optimize_duration": 1456.2294, + "load_duration": 5681.9856, + "qps": 2318.7835, + "serial_latency_p99": 0.0031, + "recall": 0.9763, + "ndcg": 0.9778, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 257.7769, + 1106.363, + 1293.0982, + 1816.0614, + 1947.7112, + 2050.9051, + 2190.9245, + 2206.8154, + 2318.7835 + ], + "conc_latency_p99_list": [ + 0.0042796858399015035, + 0.006116602120509922, + 0.009587991479711495, + 0.017683874139620456, + 0.02530405881941988, + 0.030504761879819844, + 0.04133546371998813, + 0.05358025119985541, + 0.06611231412975029 + ], + "conc_latency_avg_list": [ + 0.003873807191691351, + 0.004512439933178535, + 0.0077193695922919865, + 0.010988201040691089, + 0.015355391651436227, + 0.019409236427411304, + 0.027033452918125186, + 0.034322067918940845, + 0.04166767152714446 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 5 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4225.7562, + "optimize_duration": 1456.2294, + "load_duration": 5681.9856, + "qps": 1763.2054, + "serial_latency_p99": 0.005, + "recall": 0.9829, + "ndcg": 0.9843, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 328.6225, + 1083.4141, + 1333.0783, + 1462.9709, + 1484.5262, + 1628.7684, + 1645.8457, + 1705.7129, + 1763.2054 + ], + "conc_latency_p99_list": [ + 0.0034503032897009677, + 0.00671145100038302, + 0.01077077561076294, + 0.02162529785998231, + 0.02985431128035997, + 0.03729181721986607, + 0.05066972967016221, + 0.06722324037978977, + 0.08080457414031117 + ], + "conc_latency_avg_list": [ + 0.0030390202770354444, + 0.004607594465646954, + 0.007488635225480695, + 0.013642742161389991, + 0.019201134558262598, + 0.024422614169841837, + 0.03462971986224883, + 0.044267585750082994, + 0.05380442364539205 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 6 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4225.7562, + "optimize_duration": 1456.2294, + "load_duration": 5681.9856, + "qps": 1454.0462, + "serial_latency_p99": 0.004, + "recall": 0.9863, + "ndcg": 0.9877, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 306.4455, + 994.9869, + 1184.5526, + 1229.4767, + 1300.6206, + 1354.628, + 1421.9283, + 1454.0462, + 1444.6878 + ], + "conc_latency_p99_list": [ + 0.0037572138701034465, + 0.007317733329573454, + 0.012860286079794595, + 0.02518515336018021, + 0.0343727684004989, + 0.042978007880265075, + 0.05939922308024507, + 0.07749912459994453, + 0.09353050171957877 + ], + "conc_latency_avg_list": [ + 0.0032591253101849848, + 0.005017814298027458, + 0.008428596422596572, + 0.016221735250427825, + 0.02298165027600406, + 0.029348964854714985, + 0.04163192568630364, + 0.05382970078348951, + 0.06556926920274672 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 7 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4225.7562, + "optimize_duration": 1456.2294, + "load_duration": 5681.9856, + "qps": 1251.1255, + "serial_latency_p99": 0.0045, + "recall": 0.9884, + "ndcg": 0.9899, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 284.051, + 830.727, + 1046.2839, + 1070.4274, + 1124.6887, + 1165.778, + 1219.897, + 1242.4193, + 1251.1255 + ], + "conc_latency_p99_list": [ + 0.004278983039876038, + 0.008369231599317569, + 0.014842360080583603, + 0.028295795600161, + 0.03896912298985621, + 0.04953960974980872, + 0.06840401532964278, + 0.08881004050044797, + 0.11131876889940032 + ], + "conc_latency_avg_list": [ + 0.003516096326726783, + 0.0060103281173671174, + 0.009543299392051677, + 0.018646097366441957, + 0.026586509196342763, + 0.034196492235303265, + 0.04860977776688707, + 0.06282924016342287, + 0.0772341596758667 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 8 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4225.7562, + "optimize_duration": 1456.2294, + "load_duration": 5681.9856, + "qps": 1076.8329, + "serial_latency_p99": 0.0044, + "recall": 0.9897, + "ndcg": 0.9912, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 270.0504, + 855.1551, + 925.8737, + 946.3515, + 990.4556, + 1025.1124, + 1066.9244, + 1076.8329, + 1062.3547 + ], + "conc_latency_p99_list": [ + 0.004644059500105869, + 0.008460529000585668, + 0.01685488100043584, + 0.03152545959979763, + 0.04340440099986154, + 0.053203343229452, + 0.07726965996972467, + 0.09899732374969972, + 0.12431354600016675 + ], + "conc_latency_avg_list": [ + 0.0036985777419201588, + 0.005839061167411178, + 0.01078313605931198, + 0.021093019064264754, + 0.030203261023638767, + 0.03881098436511102, + 0.05565585728463111, + 0.07316505370040625, + 0.08823364428794842 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 9 + }, + "case_config": { + "case_id": 4, + "custom_case": null, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4225.7562, + "optimize_duration": 1456.2294, + "load_duration": 5681.9856, + "qps": 3411.0934, + "serial_latency_p99": 0.0033, + "recall": 0.995, + "ndcg": 0.9964, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 245.687, + 1067.1402, + 1487.0695, + 1856.0482, + 2176.5721, + 2438.4352, + 2731.0941, + 3056.4087, + 3411.0934 + ], + "conc_latency_p99_list": [ + 0.0044948810799905915, + 0.006187233940254374, + 0.008442803400284903, + 0.017126414620706777, + 0.02328724572026659, + 0.027354488999153527, + 0.03372818464013107, + 0.039414782819476385, + 0.04541570423059966 + ], + "conc_latency_avg_list": [ + 0.004064854448662219, + 0.0046774412373427315, + 0.006713065607661614, + 0.010751148516832404, + 0.01373562094358378, + 0.016322753585403707, + 0.020738331318422523, + 0.024429235049587546, + 0.028256371585000866 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.001 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4225.7562, + "optimize_duration": 1456.2294, + "load_duration": 5681.9856, + "qps": 2838.356, + "serial_latency_p99": 0.0038, + "recall": 0.9946, + "ndcg": 0.9961, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 328.8025, + 938.8486, + 1231.3288, + 1608.1098, + 1764.7573, + 2061.541, + 2403.6229, + 2637.8489, + 2838.356 + ], + "conc_latency_p99_list": [ + 0.0035165255493666333, + 0.006748664779879617, + 0.010227488239979748, + 0.01985492300009356, + 0.026509912758629055, + 0.03085665979009717, + 0.03765116640002817, + 0.046621859720471545, + 0.05403615004071981 + ], + "conc_latency_avg_list": [ + 0.0030370165532281924, + 0.005317917733134641, + 0.008107646787904127, + 0.012411419052371975, + 0.01616273265099634, + 0.019308168752756875, + 0.02466435519694083, + 0.029635552635364177, + 0.03408291782387855 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.002 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4225.7562, + "optimize_duration": 1456.2294, + "load_duration": 5681.9856, + "qps": 1826.0672, + "serial_latency_p99": 0.0053, + "recall": 0.9938, + "ndcg": 0.9956, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 217.67, + 879.0977, + 1073.5946, + 1157.3887, + 1254.3995, + 1377.1643, + 1648.9128, + 1765.1382, + 1826.0672 + ], + "conc_latency_p99_list": [ + 0.005198055219007067, + 0.0077035760004037, + 0.01387928455998917, + 0.026423963790130078, + 0.03396317152037228, + 0.040147247219138094, + 0.05136751736994479, + 0.06271904889927098, + 0.07345743855883481 + ], + "conc_latency_avg_list": [ + 0.004587923294517728, + 0.005679552467604719, + 0.009297075853948126, + 0.017245183992083385, + 0.02273378009488704, + 0.02755503922123505, + 0.03597796431436095, + 0.04339362147383606, + 0.05098362275221649 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.005 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4225.7562, + "optimize_duration": 1456.2294, + "load_duration": 5681.9856, + "qps": 1234.6534, + "serial_latency_p99": 0.0064, + "recall": 0.9942, + "ndcg": 0.9958, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 244.5849, + 709.3926, + 820.4206, + 833.5546, + 889.034, + 966.0459, + 1133.6158, + 1167.4271, + 1234.6534 + ], + "conc_latency_p99_list": [ + 0.005158514179183839, + 0.00998943412065273, + 0.018852321950089398, + 0.034644748879345565, + 0.045072588540097054, + 0.05358704311984185, + 0.07152618910062303, + 0.087975756078813, + 0.10822929728990857 + ], + "conc_latency_avg_list": [ + 0.004083635771251962, + 0.007039348723248797, + 0.012170117468190241, + 0.02394658229346141, + 0.03207684731646994, + 0.0393136712790348, + 0.05242812419404623, + 0.06472823170883037, + 0.07835855649334567 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.01 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4225.7562, + "optimize_duration": 1456.2294, + "load_duration": 5681.9856, + "qps": 1773.0919, + "serial_latency_p99": 0.0053, + "recall": 0.9699, + "ndcg": 0.9732, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 215.7282, + 883.9949, + 1057.8609, + 1201.0525, + 1271.4168, + 1391.1771, + 1623.0016, + 1674.0716, + 1773.0919 + ], + "conc_latency_p99_list": [ + 0.00524479119969327, + 0.007758614408721767, + 0.01242970660015999, + 0.026019128758998703, + 0.03468340343972159, + 0.04013932462947196, + 0.05336325084052078, + 0.0663368300805814, + 0.08064054260068589 + ], + "conc_latency_avg_list": [ + 0.004629876446319398, + 0.005648426846303219, + 0.00943738634640287, + 0.016617235453334735, + 0.02242435111765466, + 0.027334921019090727, + 0.036571726660526216, + 0.04543262834437737, + 0.054561891365580734 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.02 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4225.7562, + "optimize_duration": 1456.2294, + "load_duration": 5681.9856, + "qps": 1454.8382, + "serial_latency_p99": 0.0046, + "recall": 0.9659, + "ndcg": 0.9693, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 271.5569, + 813.2165, + 1004.0631, + 1041.7756, + 1149.8004, + 1184.9023, + 1309.1239, + 1425.4837, + 1454.8382 + ], + "conc_latency_p99_list": [ + 0.004502863998641261, + 0.008554997688697765, + 0.015295410599355823, + 0.02936530375976873, + 0.03860934948032081, + 0.04588267205970624, + 0.06241054024001639, + 0.07802726791967869, + 0.09400767996015931 + ], + "conc_latency_avg_list": [ + 0.0036778618663705345, + 0.006139530501506722, + 0.009944205485936301, + 0.019159285106803495, + 0.02602184281172056, + 0.03199948111605633, + 0.043396886714123784, + 0.05495553724055066, + 0.0666385034385921 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.05 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4225.7562, + "optimize_duration": 1456.2294, + "load_duration": 5681.9856, + "qps": 1373.0307, + "serial_latency_p99": 0.0057, + "recall": 0.9716, + "ndcg": 0.9755, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 200.4523, + 800.5756, + 984.4517, + 1013.3461, + 1110.0928, + 1135.0817, + 1239.1683, + 1334.8923, + 1373.0307 + ], + "conc_latency_p99_list": [ + 0.005704272559632959, + 0.008851215520990083, + 0.0158753866009647, + 0.02981916602109777, + 0.038871410919418835, + 0.047623363521343, + 0.06430617572048501, + 0.082922218260992, + 0.09843974902072661 + ], + "conc_latency_avg_list": [ + 0.004982340564404542, + 0.006236372731416691, + 0.010141251801990998, + 0.019696774029040326, + 0.026930790722259932, + 0.03347179553969378, + 0.046014799496427825, + 0.05820214271981885, + 0.07039597087894633 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.1 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4225.7562, + "optimize_duration": 1456.2294, + "load_duration": 5681.9856, + "qps": 2039.8673, + "serial_latency_p99": 0.0038, + "recall": 0.9559, + "ndcg": 0.9595, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 307.4868, + 965.5895, + 1184.4994, + 1390.7565, + 1551.6457, + 1681.6887, + 1862.98, + 1892.0572, + 2039.8673 + ], + "conc_latency_p99_list": [ + 0.003734841309396869, + 0.006497100140131806, + 0.01149535769909564, + 0.022612805601020226, + 0.030690668819952416, + 0.036714122139928806, + 0.048022087151093726, + 0.06010928088013308, + 0.07253854585906087 + ], + "conc_latency_avg_list": [ + 0.0032480978268018545, + 0.005170359720588587, + 0.008027595955155146, + 0.014350579991004676, + 0.019275705662134213, + 0.02366482070197517, + 0.03191472855261898, + 0.03966066432197402, + 0.04738732344730661 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.2 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 4225.7562, + "optimize_duration": 1456.2294, + "load_duration": 5681.9856, + "qps": 2950.8165, + "serial_latency_p99": 0.0033, + "recall": 0.9147, + "ndcg": 0.9194, + "conc_num_list": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "conc_qps_list": [ + 246.0386, + 884.9676, + 1492.2899, + 1784.7679, + 1906.178, + 2263.1899, + 2541.4456, + 2802.6333, + 2950.8165 + ], + "conc_latency_p99_list": [ + 0.004505093500029034, + 0.006600831510913864, + 0.008759755469782247, + 0.01855034028056251, + 0.0255201581403162, + 0.02945490450047145, + 0.03722894426045968, + 0.04554011100932261, + 0.052435637919843237 + ], + "conc_latency_avg_list": [ + 0.004059204455391714, + 0.005641385329001144, + 0.00668983886545188, + 0.01118067402602937, + 0.014943676916729164, + 0.017576634366334127, + 0.02292559520909751, + 0.027847151714525134, + 0.0327589624620861 + ], + "st_ideal_insert_duration": 0, + "st_search_stage_list": [], + "st_search_time_list": [], + "st_max_qps_list_list": [], + "st_recall_list": [], + "st_ndcg_list": [], + "st_serial_latency_p99_list": [], + "st_conc_failed_rate_list": [] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 1 + }, + "case_config": { + "case_id": 300, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "label_percentage": 0.5 + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 20000.13523218499, + "optimize_duration": 1026.3851764489955, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [], + "conc_qps_list": [], + "conc_latency_p99_list": [], + "conc_latency_avg_list": [], + "st_ideal_insert_duration": 20000, + "st_search_stage_list": [ + 10, + 20, + 30, + 40, + 50, + 60, + 70, + 80, + 90, + 100, + 110 + ], + "st_search_time_list": [ + 2001.0247, + 4006.2222, + 6010.203, + 8015.2667, + 10019.0173, + 12023.1918, + 14027.5442, + 16032.5467, + 18036.472, + 20041.0121, + 21169.1575 + ], + "st_max_qps_list_list": [ + 3042.8841, + 3412.9639, + 2984.6192, + 2900.4534, + 2566.0219, + 2560.7518, + 2396.8799, + 2252.0948, + 2118.7516, + 2128.6853, + 3241.731 + ], + "st_recall_list": [ + 0.0971, + 0.1907, + 0.2824, + 0.3747, + 0.4658, + 0.5575, + 0.6512, + 0.7451, + 0.8368, + 0.9262, + 0.9303 + ], + "st_ndcg_list": [ + 0.0972, + 0.191, + 0.2841, + 0.3765, + 0.4688, + 0.5599, + 0.6543, + 0.7484, + 0.8405, + 0.9302, + 0.9342 + ], + "st_serial_latency_p99_list": [ + 0.0031, + 0.0046, + 0.0033, + 0.0044, + 0.0049, + 0.006, + 0.0039, + 0.0068, + 0.0068, + 0.0046, + 0.0033 + ], + "st_conc_failed_rate_list": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 1 + }, + "case_config": { + "case_id": 200, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "insert_rate": 500, + "search_stages": [ + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "concurrencies": [ + 10, + 40 + ], + "optimize_after_write": true + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + }, + { + "metrics": { + "max_load_count": 0, + "insert_duration": 10000.142119308002, + "optimize_duration": 934.8234132769867, + "load_duration": 0.0, + "qps": 0.0, + "serial_latency_p99": 0.0, + "recall": 0.0, + "ndcg": 0.0, + "conc_num_list": [], + "conc_qps_list": [], + "conc_latency_p99_list": [], + "conc_latency_avg_list": [], + "st_ideal_insert_duration": 10000, + "st_search_stage_list": [ + 10, + 20, + 30, + 40, + 50, + 60, + 70, + 80, + 90, + 100, + 110 + ], + "st_search_time_list": [ + 1001.0859, + 2006.0204, + 3009.9719, + 4014.9464, + 5021.2584, + 6025.1563, + 7029.4325, + 8034.7597, + 9038.6783, + 10043.2069, + 11079.0357 + ], + "st_max_qps_list_list": [ + 2771.6843, + 2524.0971, + 2279.3015, + 2357.9659, + 2069.9849, + 1978.3208, + 1852.5178, + 1799.9004, + 1860.2575, + 2041.5146, + 3253.6315 + ], + "st_recall_list": [ + 0.0981, + 0.1951, + 0.2881, + 0.3795, + 0.4726, + 0.5659, + 0.6552, + 0.7534, + 0.8483, + 0.9357, + 0.9355 + ], + "st_ndcg_list": [ + 0.0982, + 0.1956, + 0.2895, + 0.3811, + 0.4751, + 0.5684, + 0.6581, + 0.7564, + 0.8517, + 0.939, + 0.9382 + ], + "st_serial_latency_p99_list": [ + 0.0091, + 0.0286, + 0.0074, + 0.0312, + 0.0067, + 0.006, + 0.005, + 0.0072, + 0.0101, + 0.0043, + 0.0043 + ], + "st_conc_failed_rate_list": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + }, + "task_config": { + "db": "ZillizCloud", + "db_config": { + "db_label": "8cu-perf", + "version": "v2025.6", + "note": "", + "uri": "**********", + "user": "db_admin", + "password": "**********" + }, + "db_case_config": { + "index": "AUTOINDEX", + "metric_type": "COSINE", + "use_partition_key": false, + "num_partitions": 1, + "level": 1 + }, + "case_config": { + "case_id": 200, + "custom_case": { + "dataset_with_size_type": "Large Cohere (768dim, 10M)", + "insert_rate": 1000, + "search_stages": [ + 0.1, + 0.2, + 0.3, + 0.4, + 0.5, + 0.6, + 0.7, + 0.8, + 0.9 + ], + "concurrencies": [ + 10, + 40 + ], + "optimize_after_write": true + }, + "k": 100, + "concurrency_search_config": { + "num_concurrency": [ + 1, + 5, + 10, + 20, + 30, + 40, + 60, + 80, + 100 + ], + "concurrency_duration": 60 + } + }, + "stages": [ + "drop_old", + "load", + "search_serial", + "search_concurrent" + ] + }, + "label": ":)" + } + ], + "file_fmt": "result_{}_{}_{}.json", + "timestamp": 1749772800.0 +} \ No newline at end of file diff --git a/vectordb_bench/results/dbPrices.json b/vectordb_bench/results/dbPrices.json index c20b16aa2..b9f1b51a8 100644 --- a/vectordb_bench/results/dbPrices.json +++ b/vectordb_bench/results/dbPrices.json @@ -11,13 +11,16 @@ "bus_crit": 32.6 }, "ElasticCloud": { - "upTo2.5c8g": 0.4793 + "upTo2.5c8g": 0.4793, + "8c60g": 1.26, + "8c60g-force_merge": 1.26 }, "QdrantCloud": { "0.5c4g-1node": 0.052, "2c8g-1node": 0.166, "4c16g-1node": 0.2852, - "4c16g-5node": 1.426 + "4c16g-5node": 1.426, + "16c64g": 1.14 }, "Pinecone": { "s1.x1": 0.0973, @@ -26,7 +29,12 @@ "p2.x1": 0.146, "p2.x1-8node": 1.168, "p1.x1-8node": 0.779, - "s1.x1-2node": 0.195 + "s1.x1-2node": 0.195, + "p2.x8-1node": 1.31 }, - "PgVector": {} + "PgVector": {}, + "OpenSearch": { + "16c128g": 1.418, + "16c128g-force_merge": 1.418 + } } \ No newline at end of file diff --git a/vectordb_bench/results/getLeaderboardDataV2.py b/vectordb_bench/results/getLeaderboardDataV2.py new file mode 100644 index 000000000..62440886f --- /dev/null +++ b/vectordb_bench/results/getLeaderboardDataV2.py @@ -0,0 +1,80 @@ +import json +import logging + + +from vectordb_bench.backend.cases import CaseType, StreamingPerformanceCase +from vectordb_bench.backend.clients import DB +from vectordb_bench.models import CaseResult +from vectordb_bench import config +import numpy as np + +logging.basicConfig(level=logging.DEBUG, format="%(asctime)s - %(levelname)s - %(message)s") + +from vectordb_bench.interface import BenchMarkRunner + + +def get_standard_2025_results() -> list[CaseResult]: + all_results = BenchMarkRunner.get_results() + standard_2025_case_results = [] + for result in all_results: + if result.task_label == "standard_2025": + standard_2025_case_results += result.results + return standard_2025_case_results + + +def save_to_json(data: list[dict], file_name: str): + with open(file_name, "w") as f: + json.dump(data, f, indent=4) + + +def main(): + standard_2025_case_results = get_standard_2025_results() + data = [] + streaming_data = [] + for case_result in standard_2025_case_results: + db = case_result.task_config.db + label = case_result.task_config.db_config.db_label + db_name = f"{db.value}{f'-{label}' if label else ''}" + metrics = case_result.metrics + qps = metrics.qps + latency = metrics.serial_latency_p99 + recall = metrics.recall + case = case_result.task_config.case_config.case + filter_ratio = case.filters.filter_rate + dataset = case.dataset.data.full_name + if case.case_id != CaseType.StreamingPerformanceCase: + data.append( + { + "dataset": dataset, + "db": db.value, + "label": label, + "db_name": db_name, + "qps": round(qps, 4), + "latency": round(latency, 4), + "recall": round(recall, 4), + "filter_ratio": round(filter_ratio, 3), + } + ) + else: + case: StreamingPerformanceCase = case + # use 90p search stage results to represent streaming performance + qps_90p = metrics.st_max_qps_list_list[metrics.st_search_stage_list.index(90)] + latency_90p = metrics.st_serial_latency_p99_list[metrics.st_search_stage_list.index(90)] + insert_rate = case.insert_rate + streaming_data.append( + { + "dataset": dataset, + "db": db.value, + "label": label, + "db_name": db_name, + "insert_rate": insert_rate, + "streaming_qps": round(qps_90p, 4), + "streaming_latency": round(latency_90p, 4), + } + ) + save_to_json(data, config.RESULTS_LOCAL_DIR / "leaderboard_v2.json") + save_to_json(streaming_data, config.RESULTS_LOCAL_DIR / "leaderboard_v2_streaming.json") + + +if __name__ == "__main__": + main() diff --git a/vectordb_bench/results/leaderboard_v2.json b/vectordb_bench/results/leaderboard_v2.json new file mode 100644 index 000000000..e4f8dece5 --- /dev/null +++ b/vectordb_bench/results/leaderboard_v2.json @@ -0,0 +1,2862 @@ +[ + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g", + "db_name": "ElasticCloud-8c60g", + "qps": 597.3641, + "latency": 12.1, + "recall": 0.9221, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g", + "db_name": "ElasticCloud-8c60g", + "qps": 551.1757, + "latency": 10.7, + "recall": 0.9327, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g", + "db_name": "ElasticCloud-8c60g", + "qps": 492.9765, + "latency": 13.4, + "recall": 0.9443, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g", + "db_name": "ElasticCloud-8c60g", + "qps": 431.0155, + "latency": 15.2, + "recall": 0.954, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g", + "db_name": "ElasticCloud-8c60g", + "qps": 377.3634, + "latency": 16.2, + "recall": 0.9615, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g", + "db_name": "ElasticCloud-8c60g", + "qps": 337.5819, + "latency": 15.0, + "recall": 0.9666, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g", + "db_name": "ElasticCloud-8c60g", + "qps": 279.7257, + "latency": 18.1, + "recall": 0.9729, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 3033.786, + "latency": 8.7, + "recall": 0.9934, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 3019.2416, + "latency": 9.5, + "recall": 0.9765, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 2890.9523, + "latency": 9.4, + "recall": 0.9625, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 2789.7212, + "latency": 8.2, + "recall": 0.9538, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 2457.2628, + "latency": 9.0, + "recall": 0.9378, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 2209.4973, + "latency": 13.7, + "recall": 0.9228, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 1960.388, + "latency": 11.0, + "recall": 0.9076, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 1725.092, + "latency": 11.7, + "recall": 0.8969, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-routing-64shard", + "db_name": "ElasticCloud-8c60g-routing-64shard", + "qps": 1307.419, + "latency": 12.3, + "recall": 0.8925, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1520.4145, + "latency": 12.5, + "recall": 0.9028, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1273.3452, + "latency": 12.3, + "recall": 0.9242, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1011.7943, + "latency": 15.2, + "recall": 0.945, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 824.5097, + "latency": 15.5, + "recall": 0.9558, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 350.0132, + "latency": 29.7, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 179.5204, + "latency": 51.4, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 72.99, + "latency": 111.4, + "recall": 1.0, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 42.9877, + "latency": 201.9, + "recall": 0.9912, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 96.4987, + "latency": 113.1, + "recall": 0.9296, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 189.3789, + "latency": 58.8, + "recall": 0.9149, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 246.7071, + "latency": 45.1, + "recall": 0.9018, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 229.0379, + "latency": 43.0, + "recall": 0.8908, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 125.6164, + "latency": 69.8, + "recall": 0.8746, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g", + "db_name": "ElasticCloud-8c60g", + "qps": 376.3752, + "latency": 14.5, + "recall": 0.9039, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g", + "db_name": "ElasticCloud-8c60g", + "qps": 341.2325, + "latency": 13.4, + "recall": 0.9136, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g", + "db_name": "ElasticCloud-8c60g", + "qps": 300.7678, + "latency": 12.8, + "recall": 0.922, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g", + "db_name": "ElasticCloud-8c60g", + "qps": 257.0398, + "latency": 15.4, + "recall": 0.9303, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g", + "db_name": "ElasticCloud-8c60g", + "qps": 228.7734, + "latency": 16.2, + "recall": 0.9374, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g", + "db_name": "ElasticCloud-8c60g", + "qps": 204.3654, + "latency": 18.2, + "recall": 0.9424, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g", + "db_name": "ElasticCloud-8c60g", + "qps": 167.5075, + "latency": 18.0, + "recall": 0.9501, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g", + "db_name": "ElasticCloud-8c60g", + "qps": 146.339, + "latency": 20.9, + "recall": 0.9557, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g", + "db_name": "ElasticCloud-8c60g", + "qps": 129.0705, + "latency": 24.3, + "recall": 0.96, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 2095.7067, + "latency": 12.4, + "recall": 0.8961, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1925.3019, + "latency": 11.3, + "recall": 0.9141, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1707.8841, + "latency": 10.0, + "recall": 0.9314, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1442.0638, + "latency": 10.1, + "recall": 0.9482, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1115.106, + "latency": 13.1, + "recall": 0.9662, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 910.4322, + "latency": 14.2, + "recall": 0.9748, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 2175.2694, + "latency": 9.8, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 1430.0244, + "latency": 12.6, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 692.5751, + "latency": 18.7, + "recall": 1.0, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 364.3516, + "latency": 26.4, + "recall": 1.0, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 190.3777, + "latency": 47.9, + "recall": 1.0, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 249.3519, + "latency": 44.7, + "recall": 0.9446, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 437.8735, + "latency": 27.1, + "recall": 0.9364, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 669.9441, + "latency": 19.1, + "recall": 0.9227, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "ElasticCloud", + "label": "8c60g-force_merge", + "db_name": "ElasticCloud-8c60g-force_merge", + "qps": 899.3114, + "latency": 14.9, + "recall": 0.9072, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "qps": 3465.1696, + "latency": 2.2, + "recall": 0.9528, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "qps": 3102.1518, + "latency": 2.3, + "recall": 0.9608, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "qps": 2681.2848, + "latency": 2.5, + "recall": 0.9681, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "qps": 2232.9105, + "latency": 2.9, + "recall": 0.9757, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "qps": 1913.17, + "latency": 3.2, + "recall": 0.9797, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "qps": 1575.8137, + "latency": 3.5, + "recall": 0.9822, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "qps": 1344.5652, + "latency": 4.2, + "recall": 0.9849, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "qps": 1148.7813, + "latency": 4.8, + "recall": 0.9861, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "qps": 3680.6045, + "latency": 2.2, + "recall": 0.9954, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "qps": 3407.9972, + "latency": 2.2, + "recall": 0.994, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "qps": 3062.6755, + "latency": 2.4, + "recall": 0.9932, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "qps": 2568.3371, + "latency": 2.7, + "recall": 0.9927, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "qps": 1886.2891, + "latency": 3.3, + "recall": 0.992, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "qps": 923.8685, + "latency": 6.7, + "recall": 0.9919, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "qps": 1442.1076, + "latency": 4.1, + "recall": 0.9517, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "qps": 2002.9865, + "latency": 3.2, + "recall": 0.9467, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "qps": 3201.9438, + "latency": 2.3, + "recall": 0.922, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 11763.5538, + "latency": 1.5, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 11803.1944, + "latency": 1.5, + "recall": 0.9778, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 11520.9234, + "latency": 1.5, + "recall": 0.9634, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 11280.0849, + "latency": 1.6, + "recall": 0.9507, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 10671.8925, + "latency": 1.7, + "recall": 0.9339, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 10258.2661, + "latency": 1.7, + "recall": 0.9139, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 9681.5656, + "latency": 1.9, + "recall": 0.9008, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 8945.4041, + "latency": 1.9, + "recall": 0.8894, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 5436.8907, + "latency": 2.0, + "recall": 0.929, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "qps": 437.1695, + "latency": 5.1, + "recall": 0.951, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "qps": 400.0053, + "latency": 5.6, + "recall": 0.9558, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "qps": 343.938, + "latency": 6.5, + "recall": 0.9605, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "qps": 275.2271, + "latency": 7.7, + "recall": 0.9649, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "qps": 234.9937, + "latency": 8.7, + "recall": 0.9677, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "qps": 202.6975, + "latency": 9.9, + "recall": 0.9696, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "qps": 164.6073, + "latency": 12.1, + "recall": 0.972, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "qps": 135.9624, + "latency": 13.9, + "recall": 0.9733, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "qps": 432.8374, + "latency": 5.0, + "recall": 0.9865, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "qps": 368.6042, + "latency": 5.4, + "recall": 0.9843, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "qps": 318.8159, + "latency": 7.1, + "recall": 0.9836, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "qps": 217.0963, + "latency": 9.1, + "recall": 0.9822, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "qps": 150.9989, + "latency": 12.5, + "recall": 0.9814, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "qps": 76.0341, + "latency": 23.1, + "recall": 0.9797, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "qps": 166.5611, + "latency": 11.5, + "recall": 0.9675, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "qps": 248.9625, + "latency": 8.3, + "recall": 0.9608, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "qps": 435.3358, + "latency": 8.2, + "recall": 0.9417, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 11397.7043, + "latency": 1.6, + "recall": 0.9597, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 10891.7531, + "latency": 1.7, + "recall": 0.9408, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 10276.7451, + "latency": 1.7, + "recall": 0.9159, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 9664.2855, + "latency": 1.8, + "recall": 0.899, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 8936.7962, + "latency": 2.0, + "recall": 0.8835, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 5671.2562, + "latency": 2.1, + "recall": 0.903, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 3157.707, + "latency": 2.3, + "recall": 0.9347, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 1985.8124, + "latency": 2.6, + "recall": 0.9407, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8-partition_key", + "db_name": "Milvus-16c64g-sq8-partition_key", + "qps": 920.9627, + "latency": 3.4, + "recall": 0.9488, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1146.5286, + "latency": 13.7, + "recall": 0.9262, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1148.1735, + "latency": 8.9, + "recall": 0.9801, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1149.1219, + "latency": 10.3, + "recall": 0.9764, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1140.4099, + "latency": 13.5, + "recall": 0.9716, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1123.5147, + "latency": 18.5, + "recall": 0.9688, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 487.8343, + "latency": 25.4, + "recall": 0.9668, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 264.9324, + "latency": 49.6, + "recall": 0.936, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 492.4887, + "latency": 29.6, + "recall": 0.9269, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 823.1775, + "latency": 20.5, + "recall": 0.9148, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1147.1977, + "latency": 13.3, + "recall": 0.8999, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1131.3087, + "latency": 14.1, + "recall": 0.9024, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1114.952, + "latency": 12.7, + "recall": 0.97, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 583.5009, + "latency": 23.0, + "recall": 0.9668, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 31.4779, + "latency": 351.0, + "recall": 0.9414, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 57.8988, + "latency": 200.1, + "recall": 0.9332, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 101.1774, + "latency": 116.1, + "recall": 0.9241, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 212.7466, + "latency": 58.7, + "recall": 0.9099, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 372.2462, + "latency": 35.9, + "recall": 0.8977, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 617.0881, + "latency": 22.4, + "recall": 0.8844, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "qps": 1094.5967, + "latency": 14.3, + "recall": 0.8659, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 4318.9697, + "latency": 4.3, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 4250.2894, + "latency": 4.6, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 2997.4391, + "latency": 6.1, + "recall": 1.0, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 1494.5334, + "latency": 7.0, + "recall": 1.0, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 1108.6473, + "latency": 7.4, + "recall": 0.995, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 1289.5164, + "latency": 6.4, + "recall": 0.9906, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 1059.3394, + "latency": 7.8, + "recall": 0.9856, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 987.0795, + "latency": 7.1, + "recall": 0.9804, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 1591.7055, + "latency": 7.8, + "recall": 0.8506, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 1202.8677, + "latency": 7.0, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 639.3991, + "latency": 7.3, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 274.8559, + "latency": 9.9, + "recall": 1.0, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 441.4152, + "latency": 8.3, + "recall": 0.997, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 358.8949, + "latency": 9.5, + "recall": 0.995, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 325.2245, + "latency": 10.3, + "recall": 0.9909, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 273.4174, + "latency": 13.3, + "recall": 0.9789, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 262.8314, + "latency": 11.3, + "recall": 0.9808, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g-payload-index", + "db_name": "QdrantCloud-16c64g-payload-index", + "qps": 434.5481, + "latency": 8.5, + "recall": 0.7237, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 446.9116, + "latency": 9.2, + "recall": 0.9357, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 388.3028, + "latency": 9.6, + "recall": 0.9431, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 323.3964, + "latency": 9.8, + "recall": 0.9507, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 256.4668, + "latency": 11.3, + "recall": 0.9588, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 145.5316, + "latency": 18.4, + "recall": 0.9726, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 1242.428, + "latency": 6.4, + "recall": 0.9474, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 1111.3633, + "latency": 7.0, + "recall": 0.955, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 955.4701, + "latency": 7.2, + "recall": 0.9629, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 783.5207, + "latency": 7.7, + "recall": 0.971, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "qps": 470.8546, + "latency": 9.5, + "recall": 0.9835, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 950.6332, + "latency": 13.2, + "recall": 0.914, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 823.2224, + "latency": 13.5, + "recall": 0.9434, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 743.9815, + "latency": 14.8, + "recall": 0.9583, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 683.1873, + "latency": 15.7, + "recall": 0.9677, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 619.7468, + "latency": 17.2, + "recall": 0.9738, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 537.4082, + "latency": 18.8, + "recall": 0.9809, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 474.9941, + "latency": 20.9, + "recall": 0.9848, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 505.7458, + "latency": 20.7, + "recall": 0.9068, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 433.9034, + "latency": 23.1, + "recall": 0.931, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 381.7737, + "latency": 25.7, + "recall": 0.9431, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 342.1123, + "latency": 29.0, + "recall": 0.951, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 308.2216, + "latency": 31.3, + "recall": 0.9561, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 257.7928, + "latency": 36.4, + "recall": 0.9626, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "qps": 223.8166, + "latency": 42.1, + "recall": 0.9666, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 3055.0123, + "latency": 7.2, + "recall": 0.9066, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 3013.4439, + "latency": 6.9, + "recall": 0.9268, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2801.7241, + "latency": 7.4, + "recall": 0.9476, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2590.3809, + "latency": 8.6, + "recall": 0.9679, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2291.2159, + "latency": 8.9, + "recall": 0.9764, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 3099.4124, + "latency": 6.2, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 3014.2483, + "latency": 7.0, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2073.2153, + "latency": 11.0, + "recall": 1.0, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1507.6899, + "latency": 12.8, + "recall": 1.0, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 942.2296, + "latency": 18.2, + "recall": 1.0, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 677.1414, + "latency": 33.5, + "recall": 0.7655, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2685.6654, + "latency": 7.6, + "recall": 0.4914, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2604.4444, + "latency": 7.8, + "recall": 0.63, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 2159.051, + "latency": 9.4, + "recall": 0.801, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2251.1274, + "latency": 8.7, + "recall": 0.8848, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3103.0539, + "latency": 5.6, + "recall": 1.0, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3086.1957, + "latency": 6.7, + "recall": 1.0, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3090.0478, + "latency": 6.4, + "recall": 0.9628, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3064.6288, + "latency": 6.5, + "recall": 0.9507, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3065.6134, + "latency": 6.2, + "recall": 0.9328, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3028.858, + "latency": 6.7, + "recall": 0.9133, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2935.9403, + "latency": 6.8, + "recall": 0.8992, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2771.2009, + "latency": 7.6, + "recall": 0.889, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1610.9496, + "latency": 10.8, + "recall": 0.9, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1557.3623, + "latency": 10.8, + "recall": 0.9244, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1473.9256, + "latency": 11.7, + "recall": 0.9484, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1388.5547, + "latency": 12.5, + "recall": 0.9597, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1022.2696, + "latency": 17.9, + "recall": 0.936, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 696.9777, + "latency": 24.6, + "recall": 0.997, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 353.7862, + "latency": 45.2, + "recall": 1.0, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 210.3227, + "latency": 71.4, + "recall": 1.0, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 114.8061, + "latency": 126.6, + "recall": 0.9985, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 504.9179, + "latency": 272.6, + "recall": 0.4664, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 1053.1495, + "latency": 17.7, + "recall": 0.5673, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 808.3294, + "latency": 22.2, + "recall": 0.7016, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-force_merge", + "db_name": "OpenSearch-16c128g-force_merge", + "qps": 8.0584, + "latency": 1757.9, + "recall": 1.0, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 3033.5491, + "latency": 6.4, + "recall": 0.9844, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2988.4205, + "latency": 7.6, + "recall": 0.9741, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2950.717, + "latency": 6.9, + "recall": 0.9558, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2782.0274, + "latency": 7.4, + "recall": 0.9466, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2708.6752, + "latency": 8.4, + "recall": 0.9337, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 2275.2854, + "latency": 9.1, + "recall": 0.917, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 1844.8918, + "latency": 10.6, + "recall": 0.9085, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 1301.4102, + "latency": 14.7, + "recall": 0.9011, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g-routing-64shard-force_merge", + "db_name": "OpenSearch-16c128g-routing-64shard-force_merge", + "qps": 13.0379, + "latency": 1063.5, + "recall": 1.0, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 9704.4214, + "latency": 2.5, + "recall": 0.9169, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 9463.4991, + "latency": 2.6, + "recall": 0.9393, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 9194.9922, + "latency": 2.7, + "recall": 0.9543, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 8779.8779, + "latency": 2.9, + "recall": 0.9685, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 8153.4648, + "latency": 3.0, + "recall": 0.9757, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 6848.5254, + "latency": 4.4, + "recall": 0.9835, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 6124.4431, + "latency": 3.8, + "recall": 0.9873, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 5186.8135, + "latency": 5.3, + "recall": 0.9893, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 4898.2048, + "latency": 5.6, + "recall": 0.9904, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 9773.6593, + "latency": 3.7, + "recall": 0.9955, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 9081.1518, + "latency": 3.0, + "recall": 0.9943, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 8455.2896, + "latency": 4.0, + "recall": 0.9921, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 7610.0519, + "latency": 3.3, + "recall": 0.9903, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 7589.664, + "latency": 3.8, + "recall": 0.9235, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 6750.2495, + "latency": 4.4, + "recall": 0.9105, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 5506.1808, + "latency": 5.5, + "recall": 0.9193, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 6860.8577, + "latency": 4.7, + "recall": 0.9226, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 8468.4611, + "latency": 3.1, + "recall": 0.8925, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 10089.4308, + "latency": 2.6, + "recall": 0.9934, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 10557.4373, + "latency": 2.7, + "recall": 0.9393, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9805.0401, + "latency": 2.6, + "recall": 0.9257, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 10020.5299, + "latency": 2.6, + "recall": 0.9788, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 10041.0338, + "latency": 2.7, + "recall": 0.9693, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9861.9686, + "latency": 2.6, + "recall": 0.955, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9507.9991, + "latency": 2.8, + "recall": 0.9453, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9428.4531, + "latency": 2.6, + "recall": 0.9331, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9048.6431, + "latency": 3.9, + "recall": 0.9216, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 8695.2765, + "latency": 4.3, + "recall": 0.9603, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9244.1135, + "latency": 4.2, + "recall": 0.9724, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9289.0118, + "latency": 4.2, + "recall": 0.9574, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9374.8941, + "latency": 4.2, + "recall": 0.9425, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9368.1325, + "latency": 3.8, + "recall": 0.9292, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 9220.3627, + "latency": 3.8, + "recall": 0.9081, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 8633.8949, + "latency": 4.1, + "recall": 0.8928, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 6820.6863, + "latency": 3.2, + "recall": 0.9159, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf-partition_key", + "db_name": "ZillizCloud-8cu-perf-partition_key", + "qps": 3938.6004, + "latency": 3.7, + "recall": 0.9196, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 3957.0757, + "latency": 2.7, + "recall": 0.932, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 3539.2869, + "latency": 4.3, + "recall": 0.9471, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 3154.6501, + "latency": 3.9, + "recall": 0.9565, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 2743.4561, + "latency": 4.3, + "recall": 0.9681, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 2318.7835, + "latency": 3.1, + "recall": 0.9763, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1763.2054, + "latency": 5.0, + "recall": 0.9829, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1454.0462, + "latency": 4.0, + "recall": 0.9863, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1251.1255, + "latency": 4.5, + "recall": 0.9884, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1076.8329, + "latency": 4.4, + "recall": 0.9897, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 3411.0934, + "latency": 3.3, + "recall": 0.995, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 2838.356, + "latency": 3.8, + "recall": 0.9946, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1826.0672, + "latency": 5.3, + "recall": 0.9938, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1234.6534, + "latency": 6.4, + "recall": 0.9942, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1773.0919, + "latency": 5.3, + "recall": 0.9699, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1454.8382, + "latency": 4.6, + "recall": 0.9659, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 1373.0307, + "latency": 5.7, + "recall": 0.9716, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 2039.8673, + "latency": 3.8, + "recall": 0.9559, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "qps": 2950.8165, + "latency": 3.3, + "recall": 0.9147, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 199.4972, + "latency": 337.1, + "recall": 0.8717, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 192.1164, + "latency": 345.6, + "recall": 0.4276, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 197.4455, + "latency": 349.3, + "recall": 0.5314, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 196.9391, + "latency": 263.4, + "recall": 0.6549, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 201.5401, + "latency": 282.4, + "recall": 0.7086, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 202.2424, + "latency": 301.7, + "recall": 0.7592, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 198.599, + "latency": 358.8, + "recall": 0.8085, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 199.0349, + "latency": 275.3, + "recall": 0.8325, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 202.1405, + "latency": 282.6, + "recall": 0.8492, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Medium)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 201.1282, + "latency": 269.2, + "recall": 0.8637, + "filter_ratio": 0.5 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 194.8021, + "latency": 559.8, + "recall": 0.86, + "filter_ratio": 0.0 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 187.4268, + "latency": 453.7, + "recall": 0.4692, + "filter_ratio": 0.999 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 198.397, + "latency": 506.9, + "recall": 0.5409, + "filter_ratio": 0.998 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 174.3549, + "latency": 496.9, + "recall": 0.6279, + "filter_ratio": 0.995 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 172.95, + "latency": 515.6, + "recall": 0.7004, + "filter_ratio": 0.99 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 190.9747, + "latency": 517.4, + "recall": 0.7398, + "filter_ratio": 0.98 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 186.0237, + "latency": 474.0, + "recall": 0.7847, + "filter_ratio": 0.95 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 192.1458, + "latency": 480.5, + "recall": 0.8103, + "filter_ratio": 0.9 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 179.4203, + "latency": 497.5, + "recall": 0.8273, + "filter_ratio": 0.8 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "qps": 199.5444, + "latency": 463.9, + "recall": 0.8478, + "filter_ratio": 0.5 + } +] \ No newline at end of file diff --git a/vectordb_bench/results/leaderboard_v2_streaming.json b/vectordb_bench/results/leaderboard_v2_streaming.json new file mode 100644 index 000000000..224c78d12 --- /dev/null +++ b/vectordb_bench/results/leaderboard_v2_streaming.json @@ -0,0 +1,128 @@ +[ + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g", + "db_name": "ElasticCloud-8c60g", + "insert_rate": 500, + "streaming_qps": 61.6708, + "streaming_latency": 0.0794 + }, + { + "dataset": "Cohere (Large)", + "db": "ElasticCloud", + "label": "8c60g", + "db_name": "ElasticCloud-8c60g", + "insert_rate": 1000, + "streaming_qps": 61.8172, + "streaming_latency": 0.2223 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "insert_rate": 500, + "streaming_qps": 305.9971, + "streaming_latency": 0.005 + }, + { + "dataset": "Cohere (Large)", + "db": "Milvus", + "label": "16c64g-sq8", + "db_name": "Milvus-16c64g-sq8", + "insert_rate": 1000, + "streaming_qps": 155.9613, + "streaming_latency": 0.0203 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "insert_rate": 500, + "streaming_qps": 367.4299, + "streaming_latency": 1.8286 + }, + { + "dataset": "Cohere (Large)", + "db": "Pinecone", + "label": "p2.x8-1node", + "db_name": "Pinecone-p2.x8-1node", + "insert_rate": 1000, + "streaming_qps": 369.6771, + "streaming_latency": 5.992 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "insert_rate": 500, + "streaming_qps": 393.753, + "streaming_latency": 0.0162 + }, + { + "dataset": "Cohere (Large)", + "db": "QdrantCloud", + "label": "16c64g", + "db_name": "QdrantCloud-16c64g", + "insert_rate": 1000, + "streaming_qps": 347.5774, + "streaming_latency": 0.0118 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "insert_rate": 1000, + "streaming_qps": 149.7168, + "streaming_latency": 0.098 + }, + { + "dataset": "Cohere (Large)", + "db": "OpenSearch", + "label": "16c128g", + "db_name": "OpenSearch-16c128g", + "insert_rate": 500, + "streaming_qps": 161.6694, + "streaming_latency": 0.052 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "insert_rate": 500, + "streaming_qps": 2118.7516, + "streaming_latency": 0.0068 + }, + { + "dataset": "Cohere (Large)", + "db": "ZillizCloud", + "label": "8cu-perf", + "db_name": "ZillizCloud-8cu-perf", + "insert_rate": 1000, + "streaming_qps": 1860.2575, + "streaming_latency": 0.0101 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "insert_rate": 500, + "streaming_qps": 180.9549, + "streaming_latency": 0.4204 + }, + { + "dataset": "Cohere (Large)", + "db": "S3Vectors", + "label": "", + "db_name": "S3Vectors", + "insert_rate": 1000, + "streaming_qps": 167.2689, + "streaming_latency": 0.5048 + } +] \ No newline at end of file