From 9b24c69edb17b79b6b145886fadb91d18a57834a Mon Sep 17 00:00:00 2001 From: kennytm Date: Sun, 8 Dec 2019 00:28:39 +0800 Subject: [PATCH 1/9] tidb-lightning: update to 2019 December --- dev/faq/tidb-lightning.md | 4 +- dev/reference/tools/tidb-lightning/config.md | 3 + .../tools/tidb-lightning/glossary.md | 187 ++++++++++++++++++ v3.0/faq/tidb-lightning.md | 4 +- .../tools/tidb-lightning/deployment.md | 3 + v3.1/faq/tidb-lightning.md | 4 +- .../tools/tidb-lightning/deployment.md | 3 + 7 files changed, 205 insertions(+), 3 deletions(-) create mode 100644 dev/reference/tools/tidb-lightning/glossary.md diff --git a/dev/faq/tidb-lightning.md b/dev/faq/tidb-lightning.md index 129973325338f..2fe8e6baa4ec6 100644 --- a/dev/faq/tidb-lightning.md +++ b/dev/faq/tidb-lightning.md @@ -28,11 +28,13 @@ TiDB Lightning requires the following privileges: * CREATE * DROP -If the target database is used to store checkpoints, it additionally requires these privileges: +If the ["TiDB" back end](/dev/reference/tools/tidb-lightning/tidb-backend.md) is chosen, or the target database is used to store checkpoints, it additionally requires these privileges: * INSERT * DELETE +The "Importer" back end does not require these two privileges because data are ingested into TiKV directly, which bypasses the entire TiDB privilege system. This is secure as long as the ports of TiKV, TiKV Importer and TiDB Lightning are not reachable outside the cluster. + If the `checksum` configuration item of TiDB Lightning is set to `true`, then the admin user privileges in the downstream TiDB need to be granted to TiDB Lightning. ## TiDB Lightning encountered an error when importing one table. Will it affect other tables? Will the process be terminated? diff --git a/dev/reference/tools/tidb-lightning/config.md b/dev/reference/tools/tidb-lightning/config.md index 802810ddb4579..44426d7467f19 100644 --- a/dev/reference/tools/tidb-lightning/config.md +++ b/dev/reference/tools/tidb-lightning/config.md @@ -184,6 +184,9 @@ checksum-table-concurrency = 16 # The default SQL mode used to parse and execute the SQL statements. sql-mode = "STRICT_TRANS_TABLES,NO_ENGINE_SUBSTITUTION" +# Sets maximum packet size allowed for SQL connections. +# Set this to 0 to automatically fetch the `max_allowed_packet` variable from server on every connection. +max-allowed-packet = 67_108_864 # When data importing is complete, tidb-lightning can automatically perform # the Checksum, Compact and Analyze operations. It is recommended to leave diff --git a/dev/reference/tools/tidb-lightning/glossary.md b/dev/reference/tools/tidb-lightning/glossary.md new file mode 100644 index 0000000000000..aef0199aa3d0f --- /dev/null +++ b/dev/reference/tools/tidb-lightning/glossary.md @@ -0,0 +1,187 @@ +--- +title: TiDB Lightning Glossary +summary: List of special terms used in TiDB Lightning. +category: glossary +--- + +# TiDB Lightning Glossary + +This page explains the special terms used in TiDB Lightning's logs, monitoring, configurations and documentations. + + + +## A + +### Analyze + +An operation to rebuild the [statistics](/dev/reference/performance/statistics.md) information of a TiDB table, i.e. running the [`ANALYZE TABLE`](/dev/reference/sql/statements/analyze-table.md) statement. + +Because TiDB Lightning imports data without going through TiDB, the statistics information is not automatically updated. Therefore, TiDB Lightning would explicitly analyze every table after importing. This step can be omitted by setting the configuration `post-restore.analyze` to `false`. + +### `AUTO_INCREMENT_ID` + +Every table has an associated `AUTO_INCREMENT_ID` counter to provide the default value of an auto-incrementing column. In TiDB, this counter is additionally used to assign row IDs. + +Because TiDB Lightning imports data without going through TiDB, the `AUTO_INCREMENT_ID` counter is not automatically updated. Therefore, TiDB Lightning will explicitly alter the `AUTO_INCREMENT_ID` to a valid value. This step is always performed, even if the table has no `AUTO_INCREMENT` columns. + + + +## B + +### Back end + +The back end is the destination where TiDB Lightning sends the parsed result to. Sometimes it is also spelled as "backend". + +See [TiDB Lightning "TiDB" Back End](/dev/reference/tools/tidb-lightning/tidb-backend.md) for details. + +### Black-white list + +A configuration list which specifies which tables should be imported, and which should be excluded. + +See [TiDB Lightning Table Filter](/dev/reference/tools/tidb-lightning/table-filter.md) for details. + + + +## C + +### Checkpoint + +TiDB Lightning continuously saves its progress into a local file or remote database while importing. This allows TiDB Lightning to resume from middle if it spuriously crashed. See the [Checkpoints](/dev/reference/tools/tidb-lightning/checkpoints.md) section for details. + +### Checksum + +In TiDB Lightning, the checksum of a table is a set of 3 numbers calculated from the content of each KV pair in that table. These are respectively: + +* the number of KV pairs, +* total length of all KV pairs, and +* the bitwise-XOR of [CRC-64-ECMA](https://en.wikipedia.org/wiki/Cyclic_redundancy_check) value each pair. + +TiDB Lightning [validates the imported data](/dev/faq/tidb-lightning.md#how-to-ensure-the-integrity-of-the-imported-data) by comparing the [local](/dev/reference/tools/tidb-lightning/glossary.md#local-checksum) and [remote checksums](/dev/reference/tools/tidb-lightning/glossary.md#remote-checksum) of every table. The program would stop if any pair is not equal. This check can be skipped by setting the configuration `post-restore.checksum` to `false`. + +See also the [Troubleshooting guide](/dev/how-to/troubleshoot/tidb-lightning.md#checksum-failed-checksum-mismatched-remote-vs-local) for how to properly handle checksum mismatch. + +### Chunk + +Equivalent to a single file in the data source. + +### Compaction + +An operation which merges multiple small SST files into a large SST file, and cleans up deleted entries. + +TiKV automatically compacts data in background while TiDB Lightning is importing. Due to legacy reasons, you can configure TiDB Lightning to explicitly trigger a compaction every time a table is imported. This is now discouraged and the corresponding settings are disabled by default. + +See [RocksDB's wiki page on Compaction](https://github.com/facebook/rocksdb/wiki/Compaction) for its technical details. + + + +## D + +### Data engine + +An [engine](/dev/reference/tools/tidb-lightning/glossary.md#engine) for sorting actual row data. + +When a table is very large, its data is placed into multiple data engines to improve task pipelining and save space of TiKV Importer. By default a new data engine is opened every 100 GB of SQL data, which can be configured through the `mydumper.batch-size` setting. + +TiDB Lightning processes multiple data engines concurrently. This is controlled by the `lightning.table-concurrency` setting. + + + +## E + +### Engine + +In TiKV Importer, an engine is a RocksDB instance for sorting KV pairs. + +TiDB Lightning transfers data to TiKV Importer through engines. It first opens an engine, sends lots of KV pairs to it (with no particular order), and finally closes the engine. The engine will sort the received KV pairs after it is closed. These closed engines can then be further uploaded to the TiKV stores for ingestion. + +Engines use TiKV Importer's `import-dir` as temporary storage, which are sometimes referred as "engine files". + +See also [data engine](/dev/reference/tools/tidb-lightning/glossary.md#data-engine) and [index engine](/dev/reference/tools/tidb-lightning/glossary.md#index-engine). + + + +## I + +### Import mode + +A configuration which optimizes TiKV for writing but reduces read speed and space utility. + +TiDB Lightning automatically switches into and out of import mode while running. However, if TiKV is somehow stuck in import mode, you can use `tidb-lightning-ctl` to [force them revert](/dev/faq/tidb-lightning.md#why-my-tidb-cluster-is-using-lots-of-cpu-resources-and-running-very-slowly-after-using-tidb-lightning) to [normal mode](/dev/reference/tools/tidb-lightning/glossary.md#normal-mode). + +### Index engine + +An [engine](/dev/reference/tools/tidb-lightning/glossary.md#engine) for sorting indices. + +Regardless of number of indices, every table is associated with exactly one index engine. + +TiDB Lightning processes multiple index engines concurrently. This is controlled by the `lightning.index-concurrency` setting. Since every table has exactly one index engine, this also configures the maximum number of tables to process in at the same time. + +### Ingestion + +An operation which inserts the entire content of an [SST file](/dev/reference/tools/tidb-lightning/glossary.md#sst-file) into the RocksDB (TiKV) store. + +Ingestion is a very fast operation compared with inserting each KV pair one-by-one. This operation is the basis of speed of TiDB Lightning. + +See [RocksDB's wiki page on Creating and Ingesting SST files](https://github.com/facebook/rocksdb/wiki/Creating-and-Ingesting-SST-files) for its technical details. + + + +## K + +### KV pair + +Abbreviation of "key-value pair". + +### KV encoder + +A routine which parses SQL or CSV rows to KV pairs. Multiple KV encoders run in parallel to speed up processing. + + + +## L + +### Local checksum + +The [checksum](/dev/reference/tools/tidb-lightning/glossary.md#checksum) of a table calculated by TiDB Lightning itself before sending the KV pairs to TiKV Importer. + + + +## N + +### Normal mode + +The mode where [import mode](/dev/reference/tools/tidb-lightning/glossary.md#import-mode) is disabled. + + + +## P + +### Post-processing + +The period of time after the entire data source is parsed and sent to TiKV Importer. TiDB Lightning is waiting for TiKV Importer to upload and [ingest](/dev/reference/tools/tidb-lightning/glossary.md#ingest) the [SST files](/dev/reference/tools/tidb-lightning/glossary.md#sst-file). + + + +## R + +### Remote checksum + +The [checksum](/dev/reference/tools/tidb-lightning/glossary.md#checksum) of a table calculated by TiDB after it has been imported. + + + +## S + +### Scattering + +An operation which randomly reassigns the leader and members of a [region](/dev/glossary.md#regionpeerraft-group). Scattering ensures the imported data are distributed evenly among all TiKV stores, and thus reduces PD's work. + +### Splitting + +An engine is typically very large (around 100 GB), which is not friendly to TiKV if treated as a single [region](/dev/glossary.md#regionpeerraft-group). TiKV Importer splits an engine into multiple small [SST files](/dev/reference/tools/tidb-lightning/glossary.md#sst-file) (configurable by TiKV Importer's `import.region-split-size` setting) before uploading. + +### SST file + +SST is the abbreviation of "sorted string table". An SST file is RocksDB's (and thus TiKV's) native storage format of a collection of KV pairs. + +TiKV Importer produces SST files from a closed [engine](/dev/reference/tools/tidb-lightning/glossary.md#engine). These SST files are uploaded and then [ingested](/dev/reference/tools/tidb-lightning/glossary.md#ingestion) into TiKV stores. diff --git a/v3.0/faq/tidb-lightning.md b/v3.0/faq/tidb-lightning.md index ef503ca988f1c..6271075e37c41 100644 --- a/v3.0/faq/tidb-lightning.md +++ b/v3.0/faq/tidb-lightning.md @@ -25,11 +25,13 @@ TiDB Lightning requires the following privileges: * CREATE * DROP -If the target database is used to store checkpoints, it additionally requires these privileges: +If the ["TiDB" back end](/v3.0/reference/tools/tidb-lightning/tidb-backend.md) is chosen, or the target database is used to store checkpoints, it additionally requires these privileges: * INSERT * DELETE +The "Importer" back end does not require these two privileges because data are ingested into TiKV directly, which bypasses the entire TiDB privilege system. This is secure as long as the ports of TiKV, TiKV Importer and TiDB Lightning are not reachable outside the cluster. + If the `checksum` configuration item of TiDB Lightning is set to `true`, then the admin user privileges in the downstream TiDB need to be granted to TiDB Lightning. ## TiDB Lightning encountered an error when importing one table. Will it affect other tables? Will the process be terminated? diff --git a/v3.0/reference/tools/tidb-lightning/deployment.md b/v3.0/reference/tools/tidb-lightning/deployment.md index 4ccbe7defb0af..044b88ea64c9d 100644 --- a/v3.0/reference/tools/tidb-lightning/deployment.md +++ b/v3.0/reference/tools/tidb-lightning/deployment.md @@ -435,6 +435,9 @@ Follow the link to download the TiDB Lightning package (choose the same version log-level = "error" # MySQL SQL Mode configuration # sql-mode = "" + # Sets maximum packet size allowed for SQL connections. + # Set this to 0 to automatically fetch the `max_allowed_packet` variable from server on every connection. + # max-allowed-packet = 67_108_864 # Sets the TiDB session variable to speed up the Checksum and Analyze operations. # See https://pingcap.com/docs/dev/reference/performance/statistics/#control-analyze-concurrency diff --git a/v3.1/faq/tidb-lightning.md b/v3.1/faq/tidb-lightning.md index d065aed55ded6..b20a65e12601d 100644 --- a/v3.1/faq/tidb-lightning.md +++ b/v3.1/faq/tidb-lightning.md @@ -24,11 +24,13 @@ TiDB Lightning requires the following privileges: * CREATE * DROP -If the target database is used to store checkpoints, it additionally requires these privileges: +If the ["TiDB" back end](/v3.1/reference/tools/tidb-lightning/tidb-backend.md) is chosen, or the target database is used to store checkpoints, it additionally requires these privileges: * INSERT * DELETE +The "Importer" back end does not require these two privileges because data are ingested into TiKV directly, which bypasses the entire TiDB privilege system. This is secure as long as the ports of TiKV, TiKV Importer and TiDB Lightning are not reachable outside the cluster. + If the `checksum` configuration item of TiDB Lightning is set to `true`, then the admin user privileges in the downstream TiDB need to be granted to TiDB Lightning. ## TiDB Lightning encountered an error when importing one table. Will it affect other tables? Will the process be terminated? diff --git a/v3.1/reference/tools/tidb-lightning/deployment.md b/v3.1/reference/tools/tidb-lightning/deployment.md index ca3b45af7b328..0afc5f0d87aea 100644 --- a/v3.1/reference/tools/tidb-lightning/deployment.md +++ b/v3.1/reference/tools/tidb-lightning/deployment.md @@ -435,6 +435,9 @@ Follow the link to download the TiDB Lightning package (choose the same version log-level = "error" # MySQL SQL Mode configuration # sql-mode = "" + # Sets maximum packet size allowed for SQL connections. + # Set this to 0 to automatically fetch the `max_allowed_packet` variable from server on every connection. + # max-allowed-packet = 67_108_864 # Sets the TiDB session variable to speed up the Checksum and Analyze operations. # See https://pingcap.com/docs/dev/reference/performance/statistics/#control-analyze-concurrency From 305864ca4c00341fe1b76cda2c6f0353be9581f0 Mon Sep 17 00:00:00 2001 From: kennytm Date: Fri, 13 Dec 2019 09:27:32 +0800 Subject: [PATCH 2/9] tidb-lightning: TOC, add hw req for tidb backend, add upgrade "guide" --- dev/TOC.md | 2 ++ dev/reference/tools/tidb-lightning/deployment.md | 6 ++++++ dev/reference/tools/tidb-lightning/tidb-backend.md | 8 ++++++++ v3.0/TOC.md | 1 + v3.0/reference/tools/tidb-lightning/deployment.md | 6 ++++++ v3.0/reference/tools/tidb-lightning/tidb-backend.md | 8 ++++++++ v3.1/TOC.md | 1 + v3.1/reference/tools/tidb-lightning/deployment.md | 6 ++++++ v3.1/reference/tools/tidb-lightning/tidb-backend.md | 8 ++++++++ 9 files changed, 46 insertions(+) diff --git a/dev/TOC.md b/dev/TOC.md index 28fc818457bbd..d822e20b559d7 100644 --- a/dev/TOC.md +++ b/dev/TOC.md @@ -350,10 +350,12 @@ - [Checkpoints](/dev/reference/tools/tidb-lightning/checkpoints.md) - [Table Filter](/dev/reference/tools/tidb-lightning/table-filter.md) - [CSV Support](/dev/reference/tools/tidb-lightning/csv.md) + - ["TiDB" Back End](/dev/reference/tools/tidb-lightning/tidb-backend.md) - [Web Interface](/dev/reference/tools/tidb-lightning/web.md) - [Monitor](/dev/reference/tools/tidb-lightning/monitor.md) - [Troubleshoot](/dev/how-to/troubleshoot/tidb-lightning.md) - [FAQ](/dev/faq/tidb-lightning.md) + - [Glossary](/dev/reference/tools/tidb-lightning/glossary.md) - [sync-diff-inspector](/dev/reference/tools/sync-diff-inspector/overview.md) - [PD Control](/dev/reference/tools/pd-control.md) - [PD Recover](/dev/reference/tools/pd-recover.md) diff --git a/dev/reference/tools/tidb-lightning/deployment.md b/dev/reference/tools/tidb-lightning/deployment.md index f6c86a56bb832..94fdea8c2b86b 100644 --- a/dev/reference/tools/tidb-lightning/deployment.md +++ b/dev/reference/tools/tidb-lightning/deployment.md @@ -275,3 +275,9 @@ Follow the link to download the TiDB Lightning package (choose the same version ```sh nohup ./tidb-lightning -config tidb-lightning.toml > nohup.out & ``` + +## Upgrading TiDB Lightning + +As of v3.1, TiDB Lightning can be upgraded by replacing the binaries alone. No reconfiguration should be needed. See [the FAQ](/dev/faq/tidb-lightning.md#how-to-properly-restart-tidb-lightning) for the precise steps of restarting TiDB Lightning. + +If an import task is running, we recommend you to wait for its finish before upgrading TiDB Lightning, or be prepared to reimport from scratch, as there is no guaratee that checkpoints work across versions. diff --git a/dev/reference/tools/tidb-lightning/tidb-backend.md b/dev/reference/tools/tidb-lightning/tidb-backend.md index a7c8e3dbf6ba4..3843760c385a2 100644 --- a/dev/reference/tools/tidb-lightning/tidb-backend.md +++ b/dev/reference/tools/tidb-lightning/tidb-backend.md @@ -26,6 +26,14 @@ When using the "TiDB" back end, you no longer need `tikv-importer`. Compared wit * Steps involving `tikv-importer` can all be skipped. * The configuration must be changed to indicate the "TiDB" back end is used. +### Hardware requirements + +The speed of TiDB Lightning using "TiDB" back end is limited by the SQL processing speed of TiDB. Therefore, even a lower-end machine may max out the possible performance. The recommended hardware configuration is: + +* 16 logical cores CPU +* An SSD large enough to store the entire data source, preferring higher read speed +* 1 Gigabit network card + ### Ansible deployment 1. The `[importer_server]` section in `inventory.ini` can be left blank. diff --git a/v3.0/TOC.md b/v3.0/TOC.md index b509d95a9fcd7..78d82eca26a52 100644 --- a/v3.0/TOC.md +++ b/v3.0/TOC.md @@ -345,6 +345,7 @@ - [Checkpoints](/v3.0/reference/tools/tidb-lightning/checkpoints.md) - [Table Filter](/v3.0/reference/tools/tidb-lightning/table-filter.md) - [CSV Support](/v3.0/reference/tools/tidb-lightning/csv.md) + - ["TiDB" Back End](/v3.0/reference/tools/tidb-lightning/tidb-backend.md) - [Monitor](/v3.0/reference/tools/tidb-lightning/monitor.md) - [Troubleshoot](/v3.0/how-to/troubleshoot/tidb-lightning.md) - [FAQ](/v3.0/faq/tidb-lightning.md) diff --git a/v3.0/reference/tools/tidb-lightning/deployment.md b/v3.0/reference/tools/tidb-lightning/deployment.md index 044b88ea64c9d..abcdd009bdcdc 100644 --- a/v3.0/reference/tools/tidb-lightning/deployment.md +++ b/v3.0/reference/tools/tidb-lightning/deployment.md @@ -484,3 +484,9 @@ Follow the link to download the TiDB Lightning package (choose the same version ```sh nohup ./tidb-lightning -config tidb-lightning.toml > nohup.out & ``` + +## Upgrading TiDB Lightning + +As of v3.0, TiDB Lightning can be upgraded by replacing the binaries alone. No reconfiguration should be needed. See [the FAQ](/v3.0/faq/tidb-lightning.md#how-to-properly-restart-tidb-lightning) for the precise steps of restarting TiDB Lightning. + +If an import task is running, we recommend you to wait for its finish before upgrading TiDB Lightning, or be prepared to reimport from scratch, as there is no guaratee that checkpoints work across versions. diff --git a/v3.0/reference/tools/tidb-lightning/tidb-backend.md b/v3.0/reference/tools/tidb-lightning/tidb-backend.md index 9242ec6653c7a..78975ed3b8ed3 100644 --- a/v3.0/reference/tools/tidb-lightning/tidb-backend.md +++ b/v3.0/reference/tools/tidb-lightning/tidb-backend.md @@ -26,6 +26,14 @@ When using the "TiDB" back end, you no longer need `tikv-importer`. Compared wit * Steps involving `tikv-importer` can all be skipped. * The configuration must be changed to indicate the "TiDB" back end is used. +### Hardware requirements + +The speed of TiDB Lightning using "TiDB" back end is limited by the SQL processing speed of TiDB. Therefore, even a lower-end machine may max out the possible performance. The recommended hardware configuration is: + +* 16 logical cores CPU +* An SSD large enough to store the entire data source, preferring higher read speed +* 1 Gigabit network card + ### Ansible deployment 1. The `[importer_server]` section in `inventory.ini` can be left blank. diff --git a/v3.1/TOC.md b/v3.1/TOC.md index b6a4400bbde1f..d638a23d37d08 100644 --- a/v3.1/TOC.md +++ b/v3.1/TOC.md @@ -348,6 +348,7 @@ - [Checkpoints](/v3.1/reference/tools/tidb-lightning/checkpoints.md) - [Table Filter](/v3.1/reference/tools/tidb-lightning/table-filter.md) - [CSV Support](/v3.1/reference/tools/tidb-lightning/csv.md) + - ["TiDB" Back End](/v3.1/reference/tools/tidb-lightning/tidb-backend.md) - [Monitor](/v3.1/reference/tools/tidb-lightning/monitor.md) - [Troubleshoot](/v3.1/how-to/troubleshoot/tidb-lightning.md) - [FAQ](/v3.1/faq/tidb-lightning.md) diff --git a/v3.1/reference/tools/tidb-lightning/deployment.md b/v3.1/reference/tools/tidb-lightning/deployment.md index 0afc5f0d87aea..b7a6178c029e7 100644 --- a/v3.1/reference/tools/tidb-lightning/deployment.md +++ b/v3.1/reference/tools/tidb-lightning/deployment.md @@ -484,3 +484,9 @@ Follow the link to download the TiDB Lightning package (choose the same version ```sh nohup ./tidb-lightning -config tidb-lightning.toml > nohup.out & ``` + +## Upgrading TiDB Lightning + +As of v3.1, TiDB Lightning can be upgraded by replacing the binaries alone. No reconfiguration should be needed. See [the FAQ](/v3.1/faq/tidb-lightning.md#how-to-properly-restart-tidb-lightning) for the precise steps of restarting TiDB Lightning. + +If an import task is running, we recommend you to wait for its finish before upgrading TiDB Lightning, or be prepared to reimport from scratch, as there is no guaratee that checkpoints work across versions. diff --git a/v3.1/reference/tools/tidb-lightning/tidb-backend.md b/v3.1/reference/tools/tidb-lightning/tidb-backend.md index c3a998404a8ca..958f37dfa4a04 100644 --- a/v3.1/reference/tools/tidb-lightning/tidb-backend.md +++ b/v3.1/reference/tools/tidb-lightning/tidb-backend.md @@ -26,6 +26,14 @@ When using the "TiDB" back end, you no longer need `tikv-importer`. Compared wit * Steps involving `tikv-importer` can all be skipped. * The configuration must be changed to indicate the "TiDB" back end is used. +### Hardware requirements + +The speed of TiDB Lightning using "TiDB" back end is limited by the SQL processing speed of TiDB. Therefore, even a lower-end machine may max out the possible performance. The recommended hardware configuration is: + +* 16 logical cores CPU +* An SSD large enough to store the entire data source, preferring higher read speed +* 1 Gigabit network card + ### Ansible deployment 1. The `[importer_server]` section in `inventory.ini` can be left blank. From 34215f8e5edfa7b40e7a0fb0716aee745b9961f2 Mon Sep 17 00:00:00 2001 From: kennytm Date: Fri, 27 Dec 2019 13:04:47 +0800 Subject: [PATCH 3/9] Apply suggestions from code review Co-Authored-By: Calvin Weng --- dev/faq/tidb-lightning.md | 4 +- .../tools/tidb-lightning/deployment.md | 4 +- .../tools/tidb-lightning/glossary.md | 40 ++++++++++--------- 3 files changed, 25 insertions(+), 23 deletions(-) diff --git a/dev/faq/tidb-lightning.md b/dev/faq/tidb-lightning.md index 2fe8e6baa4ec6..97388cd808edd 100644 --- a/dev/faq/tidb-lightning.md +++ b/dev/faq/tidb-lightning.md @@ -33,9 +33,9 @@ If the ["TiDB" back end](/dev/reference/tools/tidb-lightning/tidb-backend.md) is * INSERT * DELETE -The "Importer" back end does not require these two privileges because data are ingested into TiKV directly, which bypasses the entire TiDB privilege system. This is secure as long as the ports of TiKV, TiKV Importer and TiDB Lightning are not reachable outside the cluster. +The "Importer" back end does not require these two privileges because data is ingested into TiKV directly, which bypasses the entire TiDB privilege system. This is secure as long as the ports of TiKV, TiKV Importer and TiDB Lightning are not reachable outside the cluster. -If the `checksum` configuration item of TiDB Lightning is set to `true`, then the admin user privileges in the downstream TiDB need to be granted to TiDB Lightning. +If the `checksum` configuration of TiDB Lightning is set to `true`, then the admin user privileges in the downstream TiDB need to be granted to TiDB Lightning. ## TiDB Lightning encountered an error when importing one table. Will it affect other tables? Will the process be terminated? diff --git a/dev/reference/tools/tidb-lightning/deployment.md b/dev/reference/tools/tidb-lightning/deployment.md index 94fdea8c2b86b..780b40e769dbe 100644 --- a/dev/reference/tools/tidb-lightning/deployment.md +++ b/dev/reference/tools/tidb-lightning/deployment.md @@ -278,6 +278,6 @@ Follow the link to download the TiDB Lightning package (choose the same version ## Upgrading TiDB Lightning -As of v3.1, TiDB Lightning can be upgraded by replacing the binaries alone. No reconfiguration should be needed. See [the FAQ](/dev/faq/tidb-lightning.md#how-to-properly-restart-tidb-lightning) for the precise steps of restarting TiDB Lightning. +You can upgrade TiDB Lightning by replacing the binaries alone. No further configuration is needed. See [FAQ](/dev/faq/tidb-lightning.md#how-to-properly-restart-tidb-lightning) for the detailed instructions of restarting TiDB Lightning. -If an import task is running, we recommend you to wait for its finish before upgrading TiDB Lightning, or be prepared to reimport from scratch, as there is no guaratee that checkpoints work across versions. +If an import task is running, we recommend you to wait until it finishes before upgrading TiDB Lightning. Otherwise, there might be chances that you need to reimport from scratch, because there is no guarantee that checkpoints work across versions. diff --git a/dev/reference/tools/tidb-lightning/glossary.md b/dev/reference/tools/tidb-lightning/glossary.md index aef0199aa3d0f..d21542d2c1b54 100644 --- a/dev/reference/tools/tidb-lightning/glossary.md +++ b/dev/reference/tools/tidb-lightning/glossary.md @@ -6,7 +6,7 @@ category: glossary # TiDB Lightning Glossary -This page explains the special terms used in TiDB Lightning's logs, monitoring, configurations and documentations. +This page explains the special terms used in TiDB Lightning's logs, monitoring, configurations, and documentation. @@ -16,13 +16,13 @@ This page explains the special terms used in TiDB Lightning's logs, monitoring, An operation to rebuild the [statistics](/dev/reference/performance/statistics.md) information of a TiDB table, i.e. running the [`ANALYZE TABLE`](/dev/reference/sql/statements/analyze-table.md) statement. -Because TiDB Lightning imports data without going through TiDB, the statistics information is not automatically updated. Therefore, TiDB Lightning would explicitly analyze every table after importing. This step can be omitted by setting the configuration `post-restore.analyze` to `false`. +Because TiDB Lightning imports data without going through TiDB, the statistics information is not automatically updated. Therefore, TiDB Lightning explicitly analyzes every table after importing. This step can be omitted by setting the `post-restore.analyze` configuration to `false`. ### `AUTO_INCREMENT_ID` Every table has an associated `AUTO_INCREMENT_ID` counter to provide the default value of an auto-incrementing column. In TiDB, this counter is additionally used to assign row IDs. -Because TiDB Lightning imports data without going through TiDB, the `AUTO_INCREMENT_ID` counter is not automatically updated. Therefore, TiDB Lightning will explicitly alter the `AUTO_INCREMENT_ID` to a valid value. This step is always performed, even if the table has no `AUTO_INCREMENT` columns. +Because TiDB Lightning imports data without going through TiDB, the `AUTO_INCREMENT_ID` counter is not automatically updated. Therefore, TiDB Lightning explicitly alters `AUTO_INCREMENT_ID` to a valid value. This step is always performed, even if the table has no `AUTO_INCREMENT` columns. @@ -30,13 +30,13 @@ Because TiDB Lightning imports data without going through TiDB, the `AUTO_INCREM ### Back end -The back end is the destination where TiDB Lightning sends the parsed result to. Sometimes it is also spelled as "backend". +Back end is the destination where TiDB Lightning sends the parsed result. Also spelled as "backend". See [TiDB Lightning "TiDB" Back End](/dev/reference/tools/tidb-lightning/tidb-backend.md) for details. ### Black-white list -A configuration list which specifies which tables should be imported, and which should be excluded. +A configuration list that specifies which tables to be imported and which should be excluded. See [TiDB Lightning Table Filter](/dev/reference/tools/tidb-lightning/table-filter.md) for details. @@ -46,17 +46,17 @@ See [TiDB Lightning Table Filter](/dev/reference/tools/tidb-lightning/table-filt ### Checkpoint -TiDB Lightning continuously saves its progress into a local file or remote database while importing. This allows TiDB Lightning to resume from middle if it spuriously crashed. See the [Checkpoints](/dev/reference/tools/tidb-lightning/checkpoints.md) section for details. +TiDB Lightning continuously saves its progress into a local file or a remote database while importing. This allows it to resume from an intermediate state should it crashes in the process. See the [Checkpoints](/dev/reference/tools/tidb-lightning/checkpoints.md) section for details. ### Checksum -In TiDB Lightning, the checksum of a table is a set of 3 numbers calculated from the content of each KV pair in that table. These are respectively: +In TiDB Lightning, the checksum of a table is a set of 3 numbers calculated from the content of each KV pair in that table. These numbers are respectively: * the number of KV pairs, * total length of all KV pairs, and * the bitwise-XOR of [CRC-64-ECMA](https://en.wikipedia.org/wiki/Cyclic_redundancy_check) value each pair. -TiDB Lightning [validates the imported data](/dev/faq/tidb-lightning.md#how-to-ensure-the-integrity-of-the-imported-data) by comparing the [local](/dev/reference/tools/tidb-lightning/glossary.md#local-checksum) and [remote checksums](/dev/reference/tools/tidb-lightning/glossary.md#remote-checksum) of every table. The program would stop if any pair is not equal. This check can be skipped by setting the configuration `post-restore.checksum` to `false`. +TiDB Lightning [validates the imported data](/dev/faq/tidb-lightning.md#how-to-ensure-the-integrity-of-the-imported-data) by comparing the [local](/dev/reference/tools/tidb-lightning/glossary.md#local-checksum) and [remote checksums](/dev/reference/tools/tidb-lightning/glossary.md#remote-checksum) of every table. The program would stop if any pair does not match. You can skip this check by setting the `post-restore.checksum` configuration to `false`. See also the [Troubleshooting guide](/dev/how-to/troubleshoot/tidb-lightning.md#checksum-failed-checksum-mismatched-remote-vs-local) for how to properly handle checksum mismatch. @@ -66,9 +66,11 @@ Equivalent to a single file in the data source. ### Compaction -An operation which merges multiple small SST files into a large SST file, and cleans up deleted entries. - -TiKV automatically compacts data in background while TiDB Lightning is importing. Due to legacy reasons, you can configure TiDB Lightning to explicitly trigger a compaction every time a table is imported. This is now discouraged and the corresponding settings are disabled by default. +An operation that merges multiple small SST files into one large SST file, and cleans up the deleted entries. +TiKV automatically compacts data in background while TiDB Lightning is importing. +> **Note:** +> +> For legacy reasons, you can still configure TiDB Lightning to explicitly trigger a compaction every time a table is imported. However, this is not recommended and the corresponding settings are disabled by default. See [RocksDB's wiki page on Compaction](https://github.com/facebook/rocksdb/wiki/Compaction) for its technical details. @@ -80,7 +82,7 @@ See [RocksDB's wiki page on Compaction](https://github.com/facebook/rocksdb/wiki An [engine](/dev/reference/tools/tidb-lightning/glossary.md#engine) for sorting actual row data. -When a table is very large, its data is placed into multiple data engines to improve task pipelining and save space of TiKV Importer. By default a new data engine is opened every 100 GB of SQL data, which can be configured through the `mydumper.batch-size` setting. +When a table is very large, its data is placed into multiple data engines to improve task pipelining and save space of TiKV Importer. By default, a new data engine is opened for every 100 GB of SQL data, which can be configured through the `mydumper.batch-size` setting. TiDB Lightning processes multiple data engines concurrently. This is controlled by the `lightning.table-concurrency` setting. @@ -92,9 +94,9 @@ TiDB Lightning processes multiple data engines concurrently. This is controlled In TiKV Importer, an engine is a RocksDB instance for sorting KV pairs. -TiDB Lightning transfers data to TiKV Importer through engines. It first opens an engine, sends lots of KV pairs to it (with no particular order), and finally closes the engine. The engine will sort the received KV pairs after it is closed. These closed engines can then be further uploaded to the TiKV stores for ingestion. +TiDB Lightning transfers data to TiKV Importer through engines. It first opens an engine, sends KV pairs to it (with no particular order), and finally closes the engine. The engine sorts the received KV pairs after it is closed. These closed engines can then be further uploaded to the TiKV stores for ingestion. -Engines use TiKV Importer's `import-dir` as temporary storage, which are sometimes referred as "engine files". +Engines use TiKV Importer's `import-dir` as temporary storage, which are sometimes referred to as "engine files". See also [data engine](/dev/reference/tools/tidb-lightning/glossary.md#data-engine) and [index engine](/dev/reference/tools/tidb-lightning/glossary.md#index-engine). @@ -104,9 +106,9 @@ See also [data engine](/dev/reference/tools/tidb-lightning/glossary.md#data-engi ### Import mode -A configuration which optimizes TiKV for writing but reduces read speed and space utility. +A configuration that optimizes TiKV for writing at the cost of degraded read speed and space usage. -TiDB Lightning automatically switches into and out of import mode while running. However, if TiKV is somehow stuck in import mode, you can use `tidb-lightning-ctl` to [force them revert](/dev/faq/tidb-lightning.md#why-my-tidb-cluster-is-using-lots-of-cpu-resources-and-running-very-slowly-after-using-tidb-lightning) to [normal mode](/dev/reference/tools/tidb-lightning/glossary.md#normal-mode). +TiDB Lightning automatically switches to and off the import mode while running. However, if TiKV gets stuck in import mode, you can use `tidb-lightning-ctl` to [force revert](/dev/faq/tidb-lightning.md#why-my-tidb-cluster-is-using-lots-of-cpu-resources-and-running-very-slowly-after-using-tidb-lightning) to [normal mode](/dev/reference/tools/tidb-lightning/glossary.md#normal-mode). ### Index engine @@ -114,13 +116,13 @@ An [engine](/dev/reference/tools/tidb-lightning/glossary.md#engine) for sorting Regardless of number of indices, every table is associated with exactly one index engine. -TiDB Lightning processes multiple index engines concurrently. This is controlled by the `lightning.index-concurrency` setting. Since every table has exactly one index engine, this also configures the maximum number of tables to process in at the same time. +TiDB Lightning processes multiple index engines concurrently. This is controlled by the `lightning.index-concurrency` setting. Since every table has exactly one index engine, this also configures the maximum number of tables to process at the same time. ### Ingestion An operation which inserts the entire content of an [SST file](/dev/reference/tools/tidb-lightning/glossary.md#sst-file) into the RocksDB (TiKV) store. -Ingestion is a very fast operation compared with inserting each KV pair one-by-one. This operation is the basis of speed of TiDB Lightning. +Ingestion is a very fast operation compared with inserting KV pairs one by one. This operation is the determinant factor for the performance of TiDB Lightning. See [RocksDB's wiki page on Creating and Ingesting SST files](https://github.com/facebook/rocksdb/wiki/Creating-and-Ingesting-SST-files) for its technical details. @@ -174,7 +176,7 @@ The [checksum](/dev/reference/tools/tidb-lightning/glossary.md#checksum) of a ta ### Scattering -An operation which randomly reassigns the leader and members of a [region](/dev/glossary.md#regionpeerraft-group). Scattering ensures the imported data are distributed evenly among all TiKV stores, and thus reduces PD's work. +An operation which randomly reassigns the leader and the peers of a [region](/dev/glossary.md#regionpeerraft-group). Scattering ensures that the imported data are distributed evenly among TiKV stores. This reduces stress on PD. ### Splitting From 61dcb712e4a00c1e42148a3fdcb190595c5c9461 Mon Sep 17 00:00:00 2001 From: kennytm Date: Fri, 27 Dec 2019 13:13:08 +0800 Subject: [PATCH 4/9] tools/tidb-lightning: apply comments on v3.0 and v3.1 --- v3.0/faq/tidb-lightning.md | 4 ++-- v3.0/reference/tools/tidb-lightning/deployment.md | 4 ++-- v3.1/faq/tidb-lightning.md | 4 ++-- v3.1/reference/tools/tidb-lightning/deployment.md | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/v3.0/faq/tidb-lightning.md b/v3.0/faq/tidb-lightning.md index 6271075e37c41..be733a859a402 100644 --- a/v3.0/faq/tidb-lightning.md +++ b/v3.0/faq/tidb-lightning.md @@ -30,9 +30,9 @@ If the ["TiDB" back end](/v3.0/reference/tools/tidb-lightning/tidb-backend.md) i * INSERT * DELETE -The "Importer" back end does not require these two privileges because data are ingested into TiKV directly, which bypasses the entire TiDB privilege system. This is secure as long as the ports of TiKV, TiKV Importer and TiDB Lightning are not reachable outside the cluster. +The "Importer" back end does not require these two privileges because data is ingested into TiKV directly, which bypasses the entire TiDB privilege system. This is secure as long as the ports of TiKV, TiKV Importer and TiDB Lightning are not reachable outside the cluster. -If the `checksum` configuration item of TiDB Lightning is set to `true`, then the admin user privileges in the downstream TiDB need to be granted to TiDB Lightning. +If the `checksum` configuration of TiDB Lightning is set to `true`, then the admin user privileges in the downstream TiDB need to be granted to TiDB Lightning. ## TiDB Lightning encountered an error when importing one table. Will it affect other tables? Will the process be terminated? diff --git a/v3.0/reference/tools/tidb-lightning/deployment.md b/v3.0/reference/tools/tidb-lightning/deployment.md index abcdd009bdcdc..37129a2fd63ec 100644 --- a/v3.0/reference/tools/tidb-lightning/deployment.md +++ b/v3.0/reference/tools/tidb-lightning/deployment.md @@ -487,6 +487,6 @@ Follow the link to download the TiDB Lightning package (choose the same version ## Upgrading TiDB Lightning -As of v3.0, TiDB Lightning can be upgraded by replacing the binaries alone. No reconfiguration should be needed. See [the FAQ](/v3.0/faq/tidb-lightning.md#how-to-properly-restart-tidb-lightning) for the precise steps of restarting TiDB Lightning. +You can upgrade TiDB Lightning by replacing the binaries alone. No further configuration is needed. See [FAQ](/v3.0/faq/tidb-lightning.md#how-to-properly-restart-tidb-lightning) for the detailed instructions of restarting TiDB Lightning. -If an import task is running, we recommend you to wait for its finish before upgrading TiDB Lightning, or be prepared to reimport from scratch, as there is no guaratee that checkpoints work across versions. +If an import task is running, we recommend you to wait until it finishes before upgrading TiDB Lightning. Otherwise, there might be chances that you need to reimport from scratch, because there is no guarantee that checkpoints work across versions. diff --git a/v3.1/faq/tidb-lightning.md b/v3.1/faq/tidb-lightning.md index b20a65e12601d..47888b3cea726 100644 --- a/v3.1/faq/tidb-lightning.md +++ b/v3.1/faq/tidb-lightning.md @@ -29,9 +29,9 @@ If the ["TiDB" back end](/v3.1/reference/tools/tidb-lightning/tidb-backend.md) i * INSERT * DELETE -The "Importer" back end does not require these two privileges because data are ingested into TiKV directly, which bypasses the entire TiDB privilege system. This is secure as long as the ports of TiKV, TiKV Importer and TiDB Lightning are not reachable outside the cluster. +The "Importer" back end does not require these two privileges because data is ingested into TiKV directly, which bypasses the entire TiDB privilege system. This is secure as long as the ports of TiKV, TiKV Importer and TiDB Lightning are not reachable outside the cluster. -If the `checksum` configuration item of TiDB Lightning is set to `true`, then the admin user privileges in the downstream TiDB need to be granted to TiDB Lightning. +If the `checksum` configuration of TiDB Lightning is set to `true`, then the admin user privileges in the downstream TiDB need to be granted to TiDB Lightning. ## TiDB Lightning encountered an error when importing one table. Will it affect other tables? Will the process be terminated? diff --git a/v3.1/reference/tools/tidb-lightning/deployment.md b/v3.1/reference/tools/tidb-lightning/deployment.md index b7a6178c029e7..ca9db311816bd 100644 --- a/v3.1/reference/tools/tidb-lightning/deployment.md +++ b/v3.1/reference/tools/tidb-lightning/deployment.md @@ -487,6 +487,6 @@ Follow the link to download the TiDB Lightning package (choose the same version ## Upgrading TiDB Lightning -As of v3.1, TiDB Lightning can be upgraded by replacing the binaries alone. No reconfiguration should be needed. See [the FAQ](/v3.1/faq/tidb-lightning.md#how-to-properly-restart-tidb-lightning) for the precise steps of restarting TiDB Lightning. +You can upgrade TiDB Lightning by replacing the binaries alone. No further configuration is needed. See [FAQ](/v3.1/faq/tidb-lightning.md#how-to-properly-restart-tidb-lightning) for the detailed instructions of restarting TiDB Lightning. -If an import task is running, we recommend you to wait for its finish before upgrading TiDB Lightning, or be prepared to reimport from scratch, as there is no guaratee that checkpoints work across versions. +If an import task is running, we recommend you to wait until it finishes before upgrading TiDB Lightning. Otherwise, there might be chances that you need to reimport from scratch, because there is no guarantee that checkpoints work across versions. From 88c273a218080928da9a4c1401a1ccf319a4fb5a Mon Sep 17 00:00:00 2001 From: kennytm Date: Fri, 27 Dec 2019 13:16:59 +0800 Subject: [PATCH 5/9] tools/tidb-lightning/glossary: fix lint --- dev/reference/tools/tidb-lightning/glossary.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/reference/tools/tidb-lightning/glossary.md b/dev/reference/tools/tidb-lightning/glossary.md index d21542d2c1b54..7f3b6365d7a38 100644 --- a/dev/reference/tools/tidb-lightning/glossary.md +++ b/dev/reference/tools/tidb-lightning/glossary.md @@ -66,11 +66,11 @@ Equivalent to a single file in the data source. ### Compaction -An operation that merges multiple small SST files into one large SST file, and cleans up the deleted entries. -TiKV automatically compacts data in background while TiDB Lightning is importing. +An operation that merges multiple small SST files into one large SST file, and cleans up the deleted entries. TiKV automatically compacts data in background while TiDB Lightning is importing. + > **Note:** > -> For legacy reasons, you can still configure TiDB Lightning to explicitly trigger a compaction every time a table is imported. However, this is not recommended and the corresponding settings are disabled by default. +> For legacy reasons, you can still configure TiDB Lightning to explicitly trigger a compaction every time a table is imported. However, this is not recommended and the corresponding settings are disabled by default. See [RocksDB's wiki page on Compaction](https://github.com/facebook/rocksdb/wiki/Compaction) for its technical details. From c3c7e2afd4ec9df46426d94dfbcbd51f47ca548a Mon Sep 17 00:00:00 2001 From: kennytm Date: Sat, 28 Dec 2019 14:53:37 +0800 Subject: [PATCH 6/9] tidb-lightning: replace all `"XXX" back end` by `XXX-back-end` --- dev/TOC.md | 2 +- dev/faq/tidb-lightning.md | 4 +-- .../tools/tidb-lightning/deployment.md | 4 +-- .../tools/tidb-lightning/glossary.md | 2 +- .../tools/tidb-lightning/overview.md | 2 +- .../tools/tidb-lightning/tidb-backend.md | 28 +++++++++---------- v3.0/TOC.md | 2 +- v3.0/faq/tidb-lightning.md | 4 +-- .../tools/tidb-lightning/deployment.md | 4 +-- .../tools/tidb-lightning/overview.md | 2 +- .../tools/tidb-lightning/tidb-backend.md | 28 +++++++++---------- v3.1/TOC.md | 2 +- v3.1/faq/tidb-lightning.md | 4 +-- .../tools/tidb-lightning/deployment.md | 4 +-- .../tools/tidb-lightning/overview.md | 2 +- .../tools/tidb-lightning/tidb-backend.md | 28 +++++++++---------- 16 files changed, 61 insertions(+), 61 deletions(-) diff --git a/dev/TOC.md b/dev/TOC.md index e51cf411605ef..d244ac0880027 100644 --- a/dev/TOC.md +++ b/dev/TOC.md @@ -354,7 +354,7 @@ - [Checkpoints](/dev/reference/tools/tidb-lightning/checkpoints.md) - [Table Filter](/dev/reference/tools/tidb-lightning/table-filter.md) - [CSV Support](/dev/reference/tools/tidb-lightning/csv.md) - - ["TiDB" Back End](/dev/reference/tools/tidb-lightning/tidb-backend.md) + - [TiDB-Back-End](/dev/reference/tools/tidb-lightning/tidb-backend.md) - [Web Interface](/dev/reference/tools/tidb-lightning/web.md) - [Monitor](/dev/reference/tools/tidb-lightning/monitor.md) - [Troubleshoot](/dev/how-to/troubleshoot/tidb-lightning.md) diff --git a/dev/faq/tidb-lightning.md b/dev/faq/tidb-lightning.md index 97388cd808edd..11e20be9dc724 100644 --- a/dev/faq/tidb-lightning.md +++ b/dev/faq/tidb-lightning.md @@ -28,12 +28,12 @@ TiDB Lightning requires the following privileges: * CREATE * DROP -If the ["TiDB" back end](/dev/reference/tools/tidb-lightning/tidb-backend.md) is chosen, or the target database is used to store checkpoints, it additionally requires these privileges: +If the [TiDB-back-end](/dev/reference/tools/tidb-lightning/tidb-backend.md) is chosen, or the target database is used to store checkpoints, it additionally requires these privileges: * INSERT * DELETE -The "Importer" back end does not require these two privileges because data is ingested into TiKV directly, which bypasses the entire TiDB privilege system. This is secure as long as the ports of TiKV, TiKV Importer and TiDB Lightning are not reachable outside the cluster. +The Importer-back-end does not require these two privileges because data is ingested into TiKV directly, which bypasses the entire TiDB privilege system. This is secure as long as the ports of TiKV, TiKV Importer and TiDB Lightning are not reachable outside the cluster. If the `checksum` configuration of TiDB Lightning is set to `true`, then the admin user privileges in the downstream TiDB need to be granted to TiDB Lightning. diff --git a/dev/reference/tools/tidb-lightning/deployment.md b/dev/reference/tools/tidb-lightning/deployment.md index e7d01e8170c3e..f80b752fbf0dc 100644 --- a/dev/reference/tools/tidb-lightning/deployment.md +++ b/dev/reference/tools/tidb-lightning/deployment.md @@ -6,9 +6,9 @@ category: reference # TiDB Lightning Deployment -This document describes the hardware requirements of TiDB Lightning using the default "Importer" back end, and how to deploy it using Ansible or manually. +This document describes the hardware requirements of TiDB Lightning using the default Importer-back-end, and how to deploy it using Ansible or manually. -If you wish to use the "TiDB" back end, also read [TiDB Lightning "TiDB" Back End](/dev/reference/tools/tidb-lightning/tidb-backend.md) for the changes to the deployment steps. +If you wish to use the TiDB-back-end, also read [TiDB Lightning TiDB-Back-End](/dev/reference/tools/tidb-lightning/tidb-backend.md) for the changes to the deployment steps. ## Notes diff --git a/dev/reference/tools/tidb-lightning/glossary.md b/dev/reference/tools/tidb-lightning/glossary.md index 7f3b6365d7a38..8d173d1289e1b 100644 --- a/dev/reference/tools/tidb-lightning/glossary.md +++ b/dev/reference/tools/tidb-lightning/glossary.md @@ -32,7 +32,7 @@ Because TiDB Lightning imports data without going through TiDB, the `AUTO_INCREM Back end is the destination where TiDB Lightning sends the parsed result. Also spelled as "backend". -See [TiDB Lightning "TiDB" Back End](/dev/reference/tools/tidb-lightning/tidb-backend.md) for details. +See [TiDB Lightning TiDB-Back-End](/dev/reference/tools/tidb-lightning/tidb-backend.md) for details. ### Black-white list diff --git a/dev/reference/tools/tidb-lightning/overview.md b/dev/reference/tools/tidb-lightning/overview.md index 0dcd1ea8be426..9163d8ce9585f 100644 --- a/dev/reference/tools/tidb-lightning/overview.md +++ b/dev/reference/tools/tidb-lightning/overview.md @@ -41,4 +41,4 @@ The complete import process is as follows: 7. Finally, `tidb-lightning` switches the TiKV cluster back to "normal mode", so the cluster resumes normal services. -TiDB Lightning also supports using "TiDB" instead of "Importer" as the back end. In this configuration, `tidb-lightning` transforms data into SQL `INSERT` statements and directly executes them on the target cluster, similar to Loader. See [TiDB Lightning "TiDB" Back End](/dev/reference/tools/tidb-lightning/tidb-backend.md) for details. +TiDB Lightning also supports using TiDB instead of Importer as the back end. In this configuration, `tidb-lightning` transforms data into SQL `INSERT` statements and directly executes them on the target cluster, similar to Loader. See [TiDB Lightning TiDB-Back-End](/dev/reference/tools/tidb-lightning/tidb-backend.md) for details. diff --git a/dev/reference/tools/tidb-lightning/tidb-backend.md b/dev/reference/tools/tidb-lightning/tidb-backend.md index 3843760c385a2..c59b872b65b97 100644 --- a/dev/reference/tools/tidb-lightning/tidb-backend.md +++ b/dev/reference/tools/tidb-lightning/tidb-backend.md @@ -1,34 +1,34 @@ --- -title: TiDB Lightning "TiDB" Back End +title: TiDB Lightning TiDB-Back-End summary: Choose how to write data into the TiDB cluster. category: reference --- -# TiDB Lightning "TiDB" Back End +# TiDB Lightning TiDB-Back-End -TiDB Lightning supports two back ends: "Importer" and "TiDB". It determines how `tidb-lightning` delivers data into the target cluster. +TiDB Lightning supports two back ends: Importer and TiDB. It determines how `tidb-lightning` delivers data into the target cluster. -The "Importer" back end (default) requires `tidb-lightning` to first encode the SQL or CSV data into KV pairs, and relies on the external `tikv-importer` program to sort these KV pairs and ingest directly into the TiKV nodes. +The Importer-back-end (default) requires `tidb-lightning` to first encode the SQL or CSV data into KV pairs, and relies on the external `tikv-importer` program to sort these KV pairs and ingest directly into the TiKV nodes. -The "TiDB" back end requires `tidb-lightning` to encode these data into SQL `INSERT` statements, and has these statements executed directly on the TiDB node. +The TiDB-back-end requires `tidb-lightning` to encode these data into SQL `INSERT` statements, and has these statements executed directly on the TiDB node. -| Back end | "Importer" | "TiDB" | +| Back end | Importer | TiDB | |:---|:---|:---| | Speed | Fast (~300 GB/hr) | Slow (~50 GB/hr) | | Resource usage | High | Low | | ACID respected while importing | No | Yes | | Target tables | Must be empty | Can be populated | -## Deployment for "TiDB" back end +## Deployment for TiDB-back-end -When using the "TiDB" back end, you no longer need `tikv-importer`. Compared with the [standard deployment procedure](/dev/reference/tools/tidb-lightning/deployment.md), the "TiDB" back end deployment has the following two differences: +When using the TiDB-back-end, you no longer need `tikv-importer`. Compared with the [standard deployment procedure](/dev/reference/tools/tidb-lightning/deployment.md), the TiDB-back-end deployment has the following two differences: * Steps involving `tikv-importer` can all be skipped. -* The configuration must be changed to indicate the "TiDB" back end is used. +* The configuration must be changed to indicate the TiDB-back-end is used. ### Hardware requirements -The speed of TiDB Lightning using "TiDB" back end is limited by the SQL processing speed of TiDB. Therefore, even a lower-end machine may max out the possible performance. The recommended hardware configuration is: +The speed of TiDB Lightning using TiDB-back-end is limited by the SQL processing speed of TiDB. Therefore, even a lower-end machine may max out the possible performance. The recommended hardware configuration is: * 16 logical cores CPU * An SSD large enough to store the entire data source, preferring higher read speed @@ -80,7 +80,7 @@ or supplying the `--backend tidb` arguments when executing `tidb-lightning`. ## Conflict resolution -The "TiDB" back end supports importing to an already-populated table. However, the new data might cause a unique key conflict with the old data. You can control how to resolve the conflict by using this task configuration. +The TiDB-back-end supports importing to an already-populated table. However, the new data might cause a unique key conflict with the old data. You can control how to resolve the conflict by using this task configuration. ```toml [tikv-importer] @@ -94,9 +94,9 @@ on-duplicate = "replace" # or "error" or "ignore" | ignore | Keep old entries and ignore new ones | `INSERT IGNORE INTO ...` | | error | Abort import | `INSERT INTO ...` | -## Migrating from Loader to TiDB Lightning "TiDB" back end +## Migrating from Loader to TiDB Lightning TiDB-back-end -TiDB Lightning using the "TiDB" back end can completely replace functions of [Loader](/dev/reference/tools/loader.md). The following list shows how to translate Loader configurations into [TiDB Lightning configurations](/dev/reference/tools/tidb-lightning/config.md). +TiDB Lightning using the TiDB-back-end can completely replace functions of [Loader](/dev/reference/tools/loader.md). The following list shows how to translate Loader configurations into [TiDB Lightning configurations](/dev/reference/tools/tidb-lightning/config.md). @@ -175,7 +175,7 @@ schema = "tidb_lightning_checkpoint" ```toml [tikv-importer] -# use the "TiDB" back end +# use the TiDB-back-end backend = "tidb" ``` diff --git a/v3.0/TOC.md b/v3.0/TOC.md index 6e5b38e1a2ab1..e5e5252a0ea57 100644 --- a/v3.0/TOC.md +++ b/v3.0/TOC.md @@ -349,7 +349,7 @@ - [Checkpoints](/v3.0/reference/tools/tidb-lightning/checkpoints.md) - [Table Filter](/v3.0/reference/tools/tidb-lightning/table-filter.md) - [CSV Support](/v3.0/reference/tools/tidb-lightning/csv.md) - - ["TiDB" Back End](/v3.0/reference/tools/tidb-lightning/tidb-backend.md) + - [TiDB-Back-End](/v3.0/reference/tools/tidb-lightning/tidb-backend.md) - [Monitor](/v3.0/reference/tools/tidb-lightning/monitor.md) - [Troubleshoot](/v3.0/how-to/troubleshoot/tidb-lightning.md) - [FAQ](/v3.0/faq/tidb-lightning.md) diff --git a/v3.0/faq/tidb-lightning.md b/v3.0/faq/tidb-lightning.md index be733a859a402..3066510069d74 100644 --- a/v3.0/faq/tidb-lightning.md +++ b/v3.0/faq/tidb-lightning.md @@ -25,12 +25,12 @@ TiDB Lightning requires the following privileges: * CREATE * DROP -If the ["TiDB" back end](/v3.0/reference/tools/tidb-lightning/tidb-backend.md) is chosen, or the target database is used to store checkpoints, it additionally requires these privileges: +If the [TiDB-back-end](/v3.0/reference/tools/tidb-lightning/tidb-backend.md) is chosen, or the target database is used to store checkpoints, it additionally requires these privileges: * INSERT * DELETE -The "Importer" back end does not require these two privileges because data is ingested into TiKV directly, which bypasses the entire TiDB privilege system. This is secure as long as the ports of TiKV, TiKV Importer and TiDB Lightning are not reachable outside the cluster. +The Importer-back-end does not require these two privileges because data is ingested into TiKV directly, which bypasses the entire TiDB privilege system. This is secure as long as the ports of TiKV, TiKV Importer and TiDB Lightning are not reachable outside the cluster. If the `checksum` configuration of TiDB Lightning is set to `true`, then the admin user privileges in the downstream TiDB need to be granted to TiDB Lightning. diff --git a/v3.0/reference/tools/tidb-lightning/deployment.md b/v3.0/reference/tools/tidb-lightning/deployment.md index 5c9020845d8a2..205bf9921569c 100644 --- a/v3.0/reference/tools/tidb-lightning/deployment.md +++ b/v3.0/reference/tools/tidb-lightning/deployment.md @@ -6,9 +6,9 @@ category: reference # TiDB Lightning Deployment -This document describes the hardware requirements of TiDB Lightning using the default "Importer" back end, and how to deploy it using Ansible or manually. +This document describes the hardware requirements of TiDB Lightning using the default Importer-back-end, and how to deploy it using Ansible or manually. -If you wish to use the "TiDB" back end, also read [TiDB Lightning "TiDB" Back End](/v3.0/reference/tools/tidb-lightning/tidb-backend.md) for the changes to the deployment steps. +If you wish to use the TiDB-back-end, also read [TiDB Lightning TiDB-Back-End](/v3.0/reference/tools/tidb-lightning/tidb-backend.md) for the changes to the deployment steps. ## Notes diff --git a/v3.0/reference/tools/tidb-lightning/overview.md b/v3.0/reference/tools/tidb-lightning/overview.md index 5bdbfdf82a518..dcf647efca566 100644 --- a/v3.0/reference/tools/tidb-lightning/overview.md +++ b/v3.0/reference/tools/tidb-lightning/overview.md @@ -42,4 +42,4 @@ The complete import process is as follows: 7. Finally, `tidb-lightning` switches the TiKV cluster back to "normal mode", so the cluster resumes normal services. -TiDB Lightning also supports using "TiDB" instead of "Importer" as the back end. In this configuration, `tidb-lightning` transforms data into SQL `INSERT` statements and directly executes them on the target cluster, similar to Loader. See [TiDB Lightning "TiDB" Back End](/v3.0/reference/tools/tidb-lightning/tidb-backend.md) for details. +TiDB Lightning also supports using TiDB instead of Importer as the back end. In this configuration, `tidb-lightning` transforms data into SQL `INSERT` statements and directly executes them on the target cluster, similar to Loader. See [TiDB Lightning TiDB-Back-End](/v3.0/reference/tools/tidb-lightning/tidb-backend.md) for details. diff --git a/v3.0/reference/tools/tidb-lightning/tidb-backend.md b/v3.0/reference/tools/tidb-lightning/tidb-backend.md index 78975ed3b8ed3..86f5589599baf 100644 --- a/v3.0/reference/tools/tidb-lightning/tidb-backend.md +++ b/v3.0/reference/tools/tidb-lightning/tidb-backend.md @@ -1,34 +1,34 @@ --- -title: TiDB Lightning "TiDB" Back End +title: TiDB Lightning TiDB-Back-End summary: Choose how to write data into the TiDB cluster. category: reference --- -# TiDB Lightning "TiDB" Back End +# TiDB Lightning TiDB-Back-End -TiDB Lightning supports two back ends: "Importer" and "TiDB". It determines how `tidb-lightning` delivers data into the target cluster. +TiDB Lightning supports two back ends: Importer and TiDB. It determines how `tidb-lightning` delivers data into the target cluster. -The "Importer" back end (default) requires `tidb-lightning` to first encode the SQL or CSV data into KV pairs, and relies on the external `tikv-importer` program to sort these KV pairs and ingest directly into the TiKV nodes. +The Importer-back-end (default) requires `tidb-lightning` to first encode the SQL or CSV data into KV pairs, and relies on the external `tikv-importer` program to sort these KV pairs and ingest directly into the TiKV nodes. -The "TiDB" back end requires `tidb-lightning` to encode these data into SQL `INSERT` statements, and has these statements executed directly on the TiDB node. +The TiDB-back-end requires `tidb-lightning` to encode these data into SQL `INSERT` statements, and has these statements executed directly on the TiDB node. -| Back end | "Importer" | "TiDB" | +| Back end | Importer | TiDB | |:---|:---|:---| | Speed | Fast (~300 GB/hr) | Slow (~50 GB/hr) | | Resource usage | High | Low | | ACID respected while importing | No | Yes | | Target tables | Must be empty | Can be populated | -## Deployment for "TiDB" back end +## Deployment for TiDB-back-end -When using the "TiDB" back end, you no longer need `tikv-importer`. Compared with the [standard deployment procedure](/v3.0/reference/tools/tidb-lightning/deployment.md), the "TiDB" back end deployment has the following two differences: +When using the TiDB-back-end, you no longer need `tikv-importer`. Compared with the [standard deployment procedure](/v3.0/reference/tools/tidb-lightning/deployment.md), the TiDB-back-end deployment has the following two differences: * Steps involving `tikv-importer` can all be skipped. -* The configuration must be changed to indicate the "TiDB" back end is used. +* The configuration must be changed to indicate the TiDB-back-end is used. ### Hardware requirements -The speed of TiDB Lightning using "TiDB" back end is limited by the SQL processing speed of TiDB. Therefore, even a lower-end machine may max out the possible performance. The recommended hardware configuration is: +The speed of TiDB Lightning using TiDB-back-end is limited by the SQL processing speed of TiDB. Therefore, even a lower-end machine may max out the possible performance. The recommended hardware configuration is: * 16 logical cores CPU * An SSD large enough to store the entire data source, preferring higher read speed @@ -80,7 +80,7 @@ or supplying the `--backend tidb` arguments when executing `tidb-lightning`. ## Conflict resolution -The "TiDB" back end supports importing to an already-populated table. However, the new data might cause a unique key conflict with the old data. You can control how to resolve the conflict by using this task configuration. +The TiDB-back-end supports importing to an already-populated table. However, the new data might cause a unique key conflict with the old data. You can control how to resolve the conflict by using this task configuration. ```toml [tikv-importer] @@ -94,9 +94,9 @@ on-duplicate = "replace" # or "error" or "ignore" | ignore | Keep old entries and ignore new ones | `INSERT IGNORE INTO ...` | | error | Abort import | `INSERT INTO ...` | -## Migrating from Loader to TiDB Lightning "TiDB" back end +## Migrating from Loader to TiDB Lightning TiDB-back-end -TiDB Lightning using the "TiDB" back end can completely replace functions of [Loader](/v3.0/reference/tools/loader.md). The following list shows how to translate Loader configurations into [TiDB Lightning configurations](/v3.0/reference/tools/tidb-lightning/deployment.md#step-4-start-tidb-lightning). +TiDB Lightning using the TiDB-back-end can completely replace functions of [Loader](/v3.0/reference/tools/loader.md). The following list shows how to translate Loader configurations into [TiDB Lightning configurations](/v3.0/reference/tools/tidb-lightning/deployment.md#step-4-start-tidb-lightning).
LoaderTiDB Lightning
@@ -175,7 +175,7 @@ schema = "tidb_lightning_checkpoint" ```toml [tikv-importer] -# use the "TiDB" back end +# use the TiDB-back-end backend = "tidb" ``` diff --git a/v3.1/TOC.md b/v3.1/TOC.md index 3bacc51d79337..49a5cdc0a6a66 100644 --- a/v3.1/TOC.md +++ b/v3.1/TOC.md @@ -352,7 +352,7 @@ - [Checkpoints](/v3.1/reference/tools/tidb-lightning/checkpoints.md) - [Table Filter](/v3.1/reference/tools/tidb-lightning/table-filter.md) - [CSV Support](/v3.1/reference/tools/tidb-lightning/csv.md) - - ["TiDB" Back End](/v3.1/reference/tools/tidb-lightning/tidb-backend.md) + - [TiDB-Back-End](/v3.1/reference/tools/tidb-lightning/tidb-backend.md) - [Monitor](/v3.1/reference/tools/tidb-lightning/monitor.md) - [Troubleshoot](/v3.1/how-to/troubleshoot/tidb-lightning.md) - [FAQ](/v3.1/faq/tidb-lightning.md) diff --git a/v3.1/faq/tidb-lightning.md b/v3.1/faq/tidb-lightning.md index 47888b3cea726..4dbc1c1113908 100644 --- a/v3.1/faq/tidb-lightning.md +++ b/v3.1/faq/tidb-lightning.md @@ -24,12 +24,12 @@ TiDB Lightning requires the following privileges: * CREATE * DROP -If the ["TiDB" back end](/v3.1/reference/tools/tidb-lightning/tidb-backend.md) is chosen, or the target database is used to store checkpoints, it additionally requires these privileges: +If the [TiDB-back-end](/v3.1/reference/tools/tidb-lightning/tidb-backend.md) is chosen, or the target database is used to store checkpoints, it additionally requires these privileges: * INSERT * DELETE -The "Importer" back end does not require these two privileges because data is ingested into TiKV directly, which bypasses the entire TiDB privilege system. This is secure as long as the ports of TiKV, TiKV Importer and TiDB Lightning are not reachable outside the cluster. +The Importer-back-end does not require these two privileges because data is ingested into TiKV directly, which bypasses the entire TiDB privilege system. This is secure as long as the ports of TiKV, TiKV Importer and TiDB Lightning are not reachable outside the cluster. If the `checksum` configuration of TiDB Lightning is set to `true`, then the admin user privileges in the downstream TiDB need to be granted to TiDB Lightning. diff --git a/v3.1/reference/tools/tidb-lightning/deployment.md b/v3.1/reference/tools/tidb-lightning/deployment.md index 47cbac3d40348..43304ee1dfa0a 100644 --- a/v3.1/reference/tools/tidb-lightning/deployment.md +++ b/v3.1/reference/tools/tidb-lightning/deployment.md @@ -6,9 +6,9 @@ category: reference # TiDB Lightning Deployment -This document describes the hardware requirements of TiDB Lightning using the default "Importer" back end, and how to deploy it using Ansible or manually. +This document describes the hardware requirements of TiDB Lightning using the default Importer-back-end, and how to deploy it using Ansible or manually. -If you wish to use the "TiDB" back end, also read [TiDB Lightning "TiDB" Back End](/v3.1/reference/tools/tidb-lightning/tidb-backend.md) for the changes to the deployment steps. +If you wish to use the TiDB-back-end, also read [TiDB Lightning TiDB-Back-End](/v3.1/reference/tools/tidb-lightning/tidb-backend.md) for the changes to the deployment steps. ## Notes diff --git a/v3.1/reference/tools/tidb-lightning/overview.md b/v3.1/reference/tools/tidb-lightning/overview.md index 40f51d4dd55bb..5a2fd98f51c40 100644 --- a/v3.1/reference/tools/tidb-lightning/overview.md +++ b/v3.1/reference/tools/tidb-lightning/overview.md @@ -41,4 +41,4 @@ The complete import process is as follows: 7. Finally, `tidb-lightning` switches the TiKV cluster back to "normal mode", so the cluster resumes normal services. -TiDB Lightning also supports using "TiDB" instead of "Importer" as the back end. In this configuration, `tidb-lightning` transforms data into SQL `INSERT` statements and directly executes them on the target cluster, similar to Loader. See [TiDB Lightning "TiDB" Back End](/v3.1/reference/tools/tidb-lightning/tidb-backend.md) for details. +TiDB Lightning also supports using TiDB instead of Importer as the back end. In this configuration, `tidb-lightning` transforms data into SQL `INSERT` statements and directly executes them on the target cluster, similar to Loader. See [TiDB Lightning TiDB-Back-End](/v3.1/reference/tools/tidb-lightning/tidb-backend.md) for details. diff --git a/v3.1/reference/tools/tidb-lightning/tidb-backend.md b/v3.1/reference/tools/tidb-lightning/tidb-backend.md index 958f37dfa4a04..50276932c5409 100644 --- a/v3.1/reference/tools/tidb-lightning/tidb-backend.md +++ b/v3.1/reference/tools/tidb-lightning/tidb-backend.md @@ -1,34 +1,34 @@ --- -title: TiDB Lightning "TiDB" Back End +title: TiDB Lightning TiDB-Back-End summary: Choose how to write data into the TiDB cluster. category: reference --- -# TiDB Lightning "TiDB" Back End +# TiDB Lightning TiDB-Back-End -TiDB Lightning supports two back ends: "Importer" and "TiDB". It determines how `tidb-lightning` delivers data into the target cluster. +TiDB Lightning supports two back ends: Importer and TiDB. It determines how `tidb-lightning` delivers data into the target cluster. -The "Importer" back end (default) requires `tidb-lightning` to first encode the SQL or CSV data into KV pairs, and relies on the external `tikv-importer` program to sort these KV pairs and ingest directly into the TiKV nodes. +The Importer-back-end (default) requires `tidb-lightning` to first encode the SQL or CSV data into KV pairs, and relies on the external `tikv-importer` program to sort these KV pairs and ingest directly into the TiKV nodes. -The "TiDB" back end requires `tidb-lightning` to encode these data into SQL `INSERT` statements, and has these statements executed directly on the TiDB node. +The TiDB-back-end requires `tidb-lightning` to encode these data into SQL `INSERT` statements, and has these statements executed directly on the TiDB node. -| Back end | "Importer" | "TiDB" | +| Back end | Importer | TiDB | |:---|:---|:---| | Speed | Fast (~300 GB/hr) | Slow (~50 GB/hr) | | Resource usage | High | Low | | ACID respected while importing | No | Yes | | Target tables | Must be empty | Can be populated | -## Deployment for "TiDB" back end +## Deployment for TiDB-back-end -When using the "TiDB" back end, you no longer need `tikv-importer`. Compared with the [standard deployment procedure](/v3.1/reference/tools/tidb-lightning/deployment.md), the "TiDB" back end deployment has the following two differences: +When using the TiDB-back-end, you no longer need `tikv-importer`. Compared with the [standard deployment procedure](/v3.1/reference/tools/tidb-lightning/deployment.md), the TiDB-back-end deployment has the following two differences: * Steps involving `tikv-importer` can all be skipped. -* The configuration must be changed to indicate the "TiDB" back end is used. +* The configuration must be changed to indicate the TiDB-back-end is used. ### Hardware requirements -The speed of TiDB Lightning using "TiDB" back end is limited by the SQL processing speed of TiDB. Therefore, even a lower-end machine may max out the possible performance. The recommended hardware configuration is: +The speed of TiDB Lightning using TiDB-back-end is limited by the SQL processing speed of TiDB. Therefore, even a lower-end machine may max out the possible performance. The recommended hardware configuration is: * 16 logical cores CPU * An SSD large enough to store the entire data source, preferring higher read speed @@ -80,7 +80,7 @@ or supplying the `--backend tidb` arguments when executing `tidb-lightning`. ## Conflict resolution -The "TiDB" back end supports importing to an already-populated table. However, the new data might cause a unique key conflict with the old data. You can control how to resolve the conflict by using this task configuration. +The TiDB-back-end supports importing to an already-populated table. However, the new data might cause a unique key conflict with the old data. You can control how to resolve the conflict by using this task configuration. ```toml [tikv-importer] @@ -94,9 +94,9 @@ on-duplicate = "replace" # or "error" or "ignore" | ignore | Keep old entries and ignore new ones | `INSERT IGNORE INTO ...` | | error | Abort import | `INSERT INTO ...` | -## Migrating from Loader to TiDB Lightning "TiDB" back end +## Migrating from Loader to TiDB Lightning TiDB-back-end -TiDB Lightning using the "TiDB" back end can completely replace functions of [Loader](/v3.1/reference/tools/loader.md). The following list shows how to translate Loader configurations into [TiDB Lightning configurations](/v3.1/reference/tools/tidb-lightning/deployment.md#step-4-start-tidb-lightning). +TiDB Lightning using the TiDB-back-end can completely replace functions of [Loader](/v3.1/reference/tools/loader.md). The following list shows how to translate Loader configurations into [TiDB Lightning configurations](/v3.1/reference/tools/tidb-lightning/deployment.md#step-4-start-tidb-lightning).
LoaderTiDB Lightning
@@ -175,7 +175,7 @@ schema = "tidb_lightning_checkpoint" ```toml [tikv-importer] -# use the "TiDB" back end +# use the TiDB-back-end backend = "tidb" ``` From e576c6b959d4519be03aa0c09e7b5bc84c6347b1 Mon Sep 17 00:00:00 2001 From: kennytm Date: Thu, 9 Jan 2020 23:37:02 +0800 Subject: [PATCH 7/9] Update dev/reference/tools/tidb-lightning/glossary.md Co-Authored-By: Calvin Weng --- dev/reference/tools/tidb-lightning/glossary.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/reference/tools/tidb-lightning/glossary.md b/dev/reference/tools/tidb-lightning/glossary.md index 8d173d1289e1b..60066e2c6149f 100644 --- a/dev/reference/tools/tidb-lightning/glossary.md +++ b/dev/reference/tools/tidb-lightning/glossary.md @@ -176,7 +176,7 @@ The [checksum](/dev/reference/tools/tidb-lightning/glossary.md#checksum) of a ta ### Scattering -An operation which randomly reassigns the leader and the peers of a [region](/dev/glossary.md#regionpeerraft-group). Scattering ensures that the imported data are distributed evenly among TiKV stores. This reduces stress on PD. +An operation that randomly reassigns the leader and the peers of a [Region](/dev/glossary.md#regionpeerraft-group). Scattering ensures that the imported data are distributed evenly among TiKV stores. This reduces stress on PD. ### Splitting From c0ea049a166c67a0c19c824a3b04cd2f9bd3230d Mon Sep 17 00:00:00 2001 From: kennytm Date: Thu, 9 Jan 2020 23:50:55 +0800 Subject: [PATCH 8/9] tool/tidb-lightning: replace all "back end"/"back-end" by "backend" --- dev/TOC.md | 2 +- dev/faq/tidb-lightning.md | 4 +-- dev/reference/tools/tidb-lightning/config.md | 8 +++--- .../tools/tidb-lightning/deployment.md | 4 +-- .../tools/tidb-lightning/glossary.md | 2 +- .../tools/tidb-lightning/overview.md | 2 +- .../tools/tidb-lightning/tidb-backend.md | 26 +++++++++---------- v3.0/TOC.md | 2 +- v3.0/faq/tidb-lightning.md | 4 +-- .../tools/tidb-lightning/deployment.md | 8 +++--- .../tools/tidb-lightning/overview.md | 2 +- .../tools/tidb-lightning/tidb-backend.md | 26 +++++++++---------- v3.1/TOC.md | 2 +- v3.1/faq/tidb-lightning.md | 4 +-- .../tools/tidb-lightning/deployment.md | 8 +++--- .../tools/tidb-lightning/overview.md | 2 +- .../tools/tidb-lightning/tidb-backend.md | 26 +++++++++---------- 17 files changed, 66 insertions(+), 66 deletions(-) diff --git a/dev/TOC.md b/dev/TOC.md index 02616cd28f743..b04c8f6695bfc 100644 --- a/dev/TOC.md +++ b/dev/TOC.md @@ -358,7 +358,7 @@ - [Checkpoints](/dev/reference/tools/tidb-lightning/checkpoints.md) - [Table Filter](/dev/reference/tools/tidb-lightning/table-filter.md) - [CSV Support](/dev/reference/tools/tidb-lightning/csv.md) - - [TiDB-Back-End](/dev/reference/tools/tidb-lightning/tidb-backend.md) + - [TiDB-Backend](/dev/reference/tools/tidb-lightning/tidb-backend.md) - [Web Interface](/dev/reference/tools/tidb-lightning/web.md) - [Monitor](/dev/reference/tools/tidb-lightning/monitor.md) - [Troubleshoot](/dev/how-to/troubleshoot/tidb-lightning.md) diff --git a/dev/faq/tidb-lightning.md b/dev/faq/tidb-lightning.md index 383104d57a92e..e5ef61289e38e 100644 --- a/dev/faq/tidb-lightning.md +++ b/dev/faq/tidb-lightning.md @@ -28,12 +28,12 @@ TiDB Lightning requires the following privileges: * CREATE * DROP -If the [TiDB-back-end](/dev/reference/tools/tidb-lightning/tidb-backend.md) is chosen, or the target database is used to store checkpoints, it additionally requires these privileges: +If the [TiDB-backend](/dev/reference/tools/tidb-lightning/tidb-backend.md) is chosen, or the target database is used to store checkpoints, it additionally requires these privileges: * INSERT * DELETE -The Importer-back-end does not require these two privileges because data is ingested into TiKV directly, which bypasses the entire TiDB privilege system. This is secure as long as the ports of TiKV, TiKV Importer and TiDB Lightning are not reachable outside the cluster. +The Importer-backend does not require these two privileges because data is ingested into TiKV directly, which bypasses the entire TiDB privilege system. This is secure as long as the ports of TiKV, TiKV Importer and TiDB Lightning are not reachable outside the cluster. If the `checksum` configuration of TiDB Lightning is set to `true`, then the admin user privileges in the downstream TiDB need to be granted to TiDB Lightning. diff --git a/dev/reference/tools/tidb-lightning/config.md b/dev/reference/tools/tidb-lightning/config.md index 59d60a78381ba..52e64295ba06d 100644 --- a/dev/reference/tools/tidb-lightning/config.md +++ b/dev/reference/tools/tidb-lightning/config.md @@ -89,11 +89,11 @@ driver = "file" #keep-after-success = false [tikv-importer] -# Delivery back end, can be "importer" or "tidb". +# Delivery backend, can be "importer" or "tidb". # backend = "importer" -# The listening address of tikv-importer when back end is "importer". Change it to the actual address. +# The listening address of tikv-importer when backend is "importer". Change it to the actual address. addr = "172.16.31.10:8287" -# Action to do when trying to insert a duplicated entry in the "tidb" back end. +# Action to do when trying to insert a duplicated entry in the "tidb" backend. # - replace: new entry replaces existing entry # - ignore: keep existing entry, ignore new entry # - error: report error and quit the program @@ -298,7 +298,7 @@ min-available-ratio = 0.05 | -V | Prints program version | | | -d *directory* | Directory of the data dump to read from | `mydumper.data-source-dir` | | -L *level* | Log level: debug, info, warn, error, fatal (default = info) | `lightning.log-level` | -| --backend *backend* | [Delivery back end](/dev/reference/tools/tidb-lightning/tidb-backend.md) (`importer` or `tidb`) | `tikv-importer.backend` | +| --backend *backend* | [Delivery backend](/dev/reference/tools/tidb-lightning/tidb-backend.md) (`importer` or `tidb`) | `tikv-importer.backend` | | --log-file *file* | Log file path | `lightning.log-file` | | --status-addr *ip:port* | Listening address of the TiDB Lightning server | `lightning.status-port` | | --importer *host:port* | Address of TiKV Importer | `tikv-importer.addr` | diff --git a/dev/reference/tools/tidb-lightning/deployment.md b/dev/reference/tools/tidb-lightning/deployment.md index 0b9002bae276d..64e4fdfbb0b81 100644 --- a/dev/reference/tools/tidb-lightning/deployment.md +++ b/dev/reference/tools/tidb-lightning/deployment.md @@ -6,9 +6,9 @@ category: reference # TiDB Lightning Deployment -This document describes the hardware requirements of TiDB Lightning using the default Importer-back-end, and how to deploy it using Ansible or manually. +This document describes the hardware requirements of TiDB Lightning using the default Importer-backend, and how to deploy it using Ansible or manually. -If you wish to use the TiDB-back-end, also read [TiDB Lightning TiDB-Back-End](/dev/reference/tools/tidb-lightning/tidb-backend.md) for the changes to the deployment steps. +If you wish to use the TiDB-backend, also read [TiDB Lightning TiDB-Backend](/dev/reference/tools/tidb-lightning/tidb-backend.md) for the changes to the deployment steps. ## Notes diff --git a/dev/reference/tools/tidb-lightning/glossary.md b/dev/reference/tools/tidb-lightning/glossary.md index 60066e2c6149f..3b7e669cc2543 100644 --- a/dev/reference/tools/tidb-lightning/glossary.md +++ b/dev/reference/tools/tidb-lightning/glossary.md @@ -32,7 +32,7 @@ Because TiDB Lightning imports data without going through TiDB, the `AUTO_INCREM Back end is the destination where TiDB Lightning sends the parsed result. Also spelled as "backend". -See [TiDB Lightning TiDB-Back-End](/dev/reference/tools/tidb-lightning/tidb-backend.md) for details. +See [TiDB Lightning TiDB-Backend](/dev/reference/tools/tidb-lightning/tidb-backend.md) for details. ### Black-white list diff --git a/dev/reference/tools/tidb-lightning/overview.md b/dev/reference/tools/tidb-lightning/overview.md index 9163d8ce9585f..a1830985000e1 100644 --- a/dev/reference/tools/tidb-lightning/overview.md +++ b/dev/reference/tools/tidb-lightning/overview.md @@ -41,4 +41,4 @@ The complete import process is as follows: 7. Finally, `tidb-lightning` switches the TiKV cluster back to "normal mode", so the cluster resumes normal services. -TiDB Lightning also supports using TiDB instead of Importer as the back end. In this configuration, `tidb-lightning` transforms data into SQL `INSERT` statements and directly executes them on the target cluster, similar to Loader. See [TiDB Lightning TiDB-Back-End](/dev/reference/tools/tidb-lightning/tidb-backend.md) for details. +TiDB Lightning also supports using TiDB instead of Importer as the backend. In this configuration, `tidb-lightning` transforms data into SQL `INSERT` statements and directly executes them on the target cluster, similar to Loader. See [TiDB Lightning TiDB-Backend](/dev/reference/tools/tidb-lightning/tidb-backend.md) for details. diff --git a/dev/reference/tools/tidb-lightning/tidb-backend.md b/dev/reference/tools/tidb-lightning/tidb-backend.md index c59b872b65b97..41fa624af4057 100644 --- a/dev/reference/tools/tidb-lightning/tidb-backend.md +++ b/dev/reference/tools/tidb-lightning/tidb-backend.md @@ -1,16 +1,16 @@ --- -title: TiDB Lightning TiDB-Back-End +title: TiDB Lightning TiDB-Backend summary: Choose how to write data into the TiDB cluster. category: reference --- -# TiDB Lightning TiDB-Back-End +# TiDB Lightning TiDB-Backend -TiDB Lightning supports two back ends: Importer and TiDB. It determines how `tidb-lightning` delivers data into the target cluster. +TiDB Lightning supports two backends: Importer and TiDB. It determines how `tidb-lightning` delivers data into the target cluster. -The Importer-back-end (default) requires `tidb-lightning` to first encode the SQL or CSV data into KV pairs, and relies on the external `tikv-importer` program to sort these KV pairs and ingest directly into the TiKV nodes. +The Importer-backend (default) requires `tidb-lightning` to first encode the SQL or CSV data into KV pairs, and relies on the external `tikv-importer` program to sort these KV pairs and ingest directly into the TiKV nodes. -The TiDB-back-end requires `tidb-lightning` to encode these data into SQL `INSERT` statements, and has these statements executed directly on the TiDB node. +The TiDB-backend requires `tidb-lightning` to encode these data into SQL `INSERT` statements, and has these statements executed directly on the TiDB node. | Back end | Importer | TiDB | |:---|:---|:---| @@ -19,16 +19,16 @@ The TiDB-back-end requires `tidb-lightning` to encode these data into SQL `INSER | ACID respected while importing | No | Yes | | Target tables | Must be empty | Can be populated | -## Deployment for TiDB-back-end +## Deployment for TiDB-backend -When using the TiDB-back-end, you no longer need `tikv-importer`. Compared with the [standard deployment procedure](/dev/reference/tools/tidb-lightning/deployment.md), the TiDB-back-end deployment has the following two differences: +When using the TiDB-backend, you no longer need `tikv-importer`. Compared with the [standard deployment procedure](/dev/reference/tools/tidb-lightning/deployment.md), the TiDB-backend deployment has the following two differences: * Steps involving `tikv-importer` can all be skipped. -* The configuration must be changed to indicate the TiDB-back-end is used. +* The configuration must be changed to indicate the TiDB-backend is used. ### Hardware requirements -The speed of TiDB Lightning using TiDB-back-end is limited by the SQL processing speed of TiDB. Therefore, even a lower-end machine may max out the possible performance. The recommended hardware configuration is: +The speed of TiDB Lightning using TiDB-backend is limited by the SQL processing speed of TiDB. Therefore, even a lower-end machine may max out the possible performance. The recommended hardware configuration is: * 16 logical cores CPU * An SSD large enough to store the entire data source, preferring higher read speed @@ -80,7 +80,7 @@ or supplying the `--backend tidb` arguments when executing `tidb-lightning`. ## Conflict resolution -The TiDB-back-end supports importing to an already-populated table. However, the new data might cause a unique key conflict with the old data. You can control how to resolve the conflict by using this task configuration. +The TiDB-backend supports importing to an already-populated table. However, the new data might cause a unique key conflict with the old data. You can control how to resolve the conflict by using this task configuration. ```toml [tikv-importer] @@ -94,9 +94,9 @@ on-duplicate = "replace" # or "error" or "ignore" | ignore | Keep old entries and ignore new ones | `INSERT IGNORE INTO ...` | | error | Abort import | `INSERT INTO ...` | -## Migrating from Loader to TiDB Lightning TiDB-back-end +## Migrating from Loader to TiDB Lightning TiDB-backend -TiDB Lightning using the TiDB-back-end can completely replace functions of [Loader](/dev/reference/tools/loader.md). The following list shows how to translate Loader configurations into [TiDB Lightning configurations](/dev/reference/tools/tidb-lightning/config.md). +TiDB Lightning using the TiDB-backend can completely replace functions of [Loader](/dev/reference/tools/loader.md). The following list shows how to translate Loader configurations into [TiDB Lightning configurations](/dev/reference/tools/tidb-lightning/config.md).
LoaderTiDB Lightning
@@ -175,7 +175,7 @@ schema = "tidb_lightning_checkpoint" ```toml [tikv-importer] -# use the TiDB-back-end +# use the TiDB-backend backend = "tidb" ``` diff --git a/v3.0/TOC.md b/v3.0/TOC.md index ea11be6d89517..27e3048d0fc1a 100644 --- a/v3.0/TOC.md +++ b/v3.0/TOC.md @@ -353,7 +353,7 @@ - [Checkpoints](/v3.0/reference/tools/tidb-lightning/checkpoints.md) - [Table Filter](/v3.0/reference/tools/tidb-lightning/table-filter.md) - [CSV Support](/v3.0/reference/tools/tidb-lightning/csv.md) - - [TiDB-Back-End](/v3.0/reference/tools/tidb-lightning/tidb-backend.md) + - [TiDB-Backend](/v3.0/reference/tools/tidb-lightning/tidb-backend.md) - [Monitor](/v3.0/reference/tools/tidb-lightning/monitor.md) - [Troubleshoot](/v3.0/how-to/troubleshoot/tidb-lightning.md) - [FAQ](/v3.0/faq/tidb-lightning.md) diff --git a/v3.0/faq/tidb-lightning.md b/v3.0/faq/tidb-lightning.md index 2f4e19cb92602..1a607509022a2 100644 --- a/v3.0/faq/tidb-lightning.md +++ b/v3.0/faq/tidb-lightning.md @@ -29,12 +29,12 @@ TiDB Lightning requires the following privileges: * CREATE * DROP -If the [TiDB-back-end](/v3.0/reference/tools/tidb-lightning/tidb-backend.md) is chosen, or the target database is used to store checkpoints, it additionally requires these privileges: +If the [TiDB-backend](/v3.0/reference/tools/tidb-lightning/tidb-backend.md) is chosen, or the target database is used to store checkpoints, it additionally requires these privileges: * INSERT * DELETE -The Importer-back-end does not require these two privileges because data is ingested into TiKV directly, which bypasses the entire TiDB privilege system. This is secure as long as the ports of TiKV, TiKV Importer and TiDB Lightning are not reachable outside the cluster. +The Importer-backend does not require these two privileges because data is ingested into TiKV directly, which bypasses the entire TiDB privilege system. This is secure as long as the ports of TiKV, TiKV Importer and TiDB Lightning are not reachable outside the cluster. If the `checksum` configuration of TiDB Lightning is set to `true`, then the admin user privileges in the downstream TiDB need to be granted to TiDB Lightning. diff --git a/v3.0/reference/tools/tidb-lightning/deployment.md b/v3.0/reference/tools/tidb-lightning/deployment.md index f1100a675e7b8..29bddd01abdf7 100644 --- a/v3.0/reference/tools/tidb-lightning/deployment.md +++ b/v3.0/reference/tools/tidb-lightning/deployment.md @@ -6,9 +6,9 @@ category: reference # TiDB Lightning Deployment -This document describes the hardware requirements of TiDB Lightning using the default Importer-back-end, and how to deploy it using Ansible or manually. +This document describes the hardware requirements of TiDB Lightning using the default Importer-backend, and how to deploy it using Ansible or manually. -If you wish to use the TiDB-back-end, also read [TiDB Lightning TiDB-Back-End](/v3.0/reference/tools/tidb-lightning/tidb-backend.md) for the changes to the deployment steps. +If you wish to use the TiDB-backend, also read [TiDB Lightning TiDB-Backend](/v3.0/reference/tools/tidb-lightning/tidb-backend.md) for the changes to the deployment steps. ## Notes @@ -246,9 +246,9 @@ Refer to the [TiDB enterprise tools download page](/v3.0/reference/tools/downloa file = "tidb-lightning.log" [tikv-importer] - # Delivery back end, can be "importer" or "tidb". + # Delivery backend, can be "importer" or "tidb". # backend = "importer" - # The listening address of tikv-importer when back end is "importer". Change it to the actual address. + # The listening address of tikv-importer when backend is "importer". Change it to the actual address. addr = "172.16.31.10:8287" [mydumper] diff --git a/v3.0/reference/tools/tidb-lightning/overview.md b/v3.0/reference/tools/tidb-lightning/overview.md index dcf647efca566..d7cf052d1de8e 100644 --- a/v3.0/reference/tools/tidb-lightning/overview.md +++ b/v3.0/reference/tools/tidb-lightning/overview.md @@ -42,4 +42,4 @@ The complete import process is as follows: 7. Finally, `tidb-lightning` switches the TiKV cluster back to "normal mode", so the cluster resumes normal services. -TiDB Lightning also supports using TiDB instead of Importer as the back end. In this configuration, `tidb-lightning` transforms data into SQL `INSERT` statements and directly executes them on the target cluster, similar to Loader. See [TiDB Lightning TiDB-Back-End](/v3.0/reference/tools/tidb-lightning/tidb-backend.md) for details. +TiDB Lightning also supports using TiDB instead of Importer as the backend. In this configuration, `tidb-lightning` transforms data into SQL `INSERT` statements and directly executes them on the target cluster, similar to Loader. See [TiDB Lightning TiDB-Backend](/v3.0/reference/tools/tidb-lightning/tidb-backend.md) for details. diff --git a/v3.0/reference/tools/tidb-lightning/tidb-backend.md b/v3.0/reference/tools/tidb-lightning/tidb-backend.md index 86f5589599baf..69b6ae297d388 100644 --- a/v3.0/reference/tools/tidb-lightning/tidb-backend.md +++ b/v3.0/reference/tools/tidb-lightning/tidb-backend.md @@ -1,16 +1,16 @@ --- -title: TiDB Lightning TiDB-Back-End +title: TiDB Lightning TiDB-Backend summary: Choose how to write data into the TiDB cluster. category: reference --- -# TiDB Lightning TiDB-Back-End +# TiDB Lightning TiDB-Backend -TiDB Lightning supports two back ends: Importer and TiDB. It determines how `tidb-lightning` delivers data into the target cluster. +TiDB Lightning supports two backends: Importer and TiDB. It determines how `tidb-lightning` delivers data into the target cluster. -The Importer-back-end (default) requires `tidb-lightning` to first encode the SQL or CSV data into KV pairs, and relies on the external `tikv-importer` program to sort these KV pairs and ingest directly into the TiKV nodes. +The Importer-backend (default) requires `tidb-lightning` to first encode the SQL or CSV data into KV pairs, and relies on the external `tikv-importer` program to sort these KV pairs and ingest directly into the TiKV nodes. -The TiDB-back-end requires `tidb-lightning` to encode these data into SQL `INSERT` statements, and has these statements executed directly on the TiDB node. +The TiDB-backend requires `tidb-lightning` to encode these data into SQL `INSERT` statements, and has these statements executed directly on the TiDB node. | Back end | Importer | TiDB | |:---|:---|:---| @@ -19,16 +19,16 @@ The TiDB-back-end requires `tidb-lightning` to encode these data into SQL `INSER | ACID respected while importing | No | Yes | | Target tables | Must be empty | Can be populated | -## Deployment for TiDB-back-end +## Deployment for TiDB-backend -When using the TiDB-back-end, you no longer need `tikv-importer`. Compared with the [standard deployment procedure](/v3.0/reference/tools/tidb-lightning/deployment.md), the TiDB-back-end deployment has the following two differences: +When using the TiDB-backend, you no longer need `tikv-importer`. Compared with the [standard deployment procedure](/v3.0/reference/tools/tidb-lightning/deployment.md), the TiDB-backend deployment has the following two differences: * Steps involving `tikv-importer` can all be skipped. -* The configuration must be changed to indicate the TiDB-back-end is used. +* The configuration must be changed to indicate the TiDB-backend is used. ### Hardware requirements -The speed of TiDB Lightning using TiDB-back-end is limited by the SQL processing speed of TiDB. Therefore, even a lower-end machine may max out the possible performance. The recommended hardware configuration is: +The speed of TiDB Lightning using TiDB-backend is limited by the SQL processing speed of TiDB. Therefore, even a lower-end machine may max out the possible performance. The recommended hardware configuration is: * 16 logical cores CPU * An SSD large enough to store the entire data source, preferring higher read speed @@ -80,7 +80,7 @@ or supplying the `--backend tidb` arguments when executing `tidb-lightning`. ## Conflict resolution -The TiDB-back-end supports importing to an already-populated table. However, the new data might cause a unique key conflict with the old data. You can control how to resolve the conflict by using this task configuration. +The TiDB-backend supports importing to an already-populated table. However, the new data might cause a unique key conflict with the old data. You can control how to resolve the conflict by using this task configuration. ```toml [tikv-importer] @@ -94,9 +94,9 @@ on-duplicate = "replace" # or "error" or "ignore" | ignore | Keep old entries and ignore new ones | `INSERT IGNORE INTO ...` | | error | Abort import | `INSERT INTO ...` | -## Migrating from Loader to TiDB Lightning TiDB-back-end +## Migrating from Loader to TiDB Lightning TiDB-backend -TiDB Lightning using the TiDB-back-end can completely replace functions of [Loader](/v3.0/reference/tools/loader.md). The following list shows how to translate Loader configurations into [TiDB Lightning configurations](/v3.0/reference/tools/tidb-lightning/deployment.md#step-4-start-tidb-lightning). +TiDB Lightning using the TiDB-backend can completely replace functions of [Loader](/v3.0/reference/tools/loader.md). The following list shows how to translate Loader configurations into [TiDB Lightning configurations](/v3.0/reference/tools/tidb-lightning/deployment.md#step-4-start-tidb-lightning).
LoaderTiDB Lightning
@@ -175,7 +175,7 @@ schema = "tidb_lightning_checkpoint" ```toml [tikv-importer] -# use the TiDB-back-end +# use the TiDB-backend backend = "tidb" ``` diff --git a/v3.1/TOC.md b/v3.1/TOC.md index 9531b9eeec58f..41a40be335d51 100644 --- a/v3.1/TOC.md +++ b/v3.1/TOC.md @@ -355,7 +355,7 @@ - [Checkpoints](/v3.1/reference/tools/tidb-lightning/checkpoints.md) - [Table Filter](/v3.1/reference/tools/tidb-lightning/table-filter.md) - [CSV Support](/v3.1/reference/tools/tidb-lightning/csv.md) - - [TiDB-Back-End](/v3.1/reference/tools/tidb-lightning/tidb-backend.md) + - [TiDB-Backend](/v3.1/reference/tools/tidb-lightning/tidb-backend.md) - [Monitor](/v3.1/reference/tools/tidb-lightning/monitor.md) - [Troubleshoot](/v3.1/how-to/troubleshoot/tidb-lightning.md) - [FAQ](/v3.1/faq/tidb-lightning.md) diff --git a/v3.1/faq/tidb-lightning.md b/v3.1/faq/tidb-lightning.md index db26e7a7f0846..89f2940e76d56 100644 --- a/v3.1/faq/tidb-lightning.md +++ b/v3.1/faq/tidb-lightning.md @@ -28,12 +28,12 @@ TiDB Lightning requires the following privileges: * CREATE * DROP -If the [TiDB-back-end](/v3.1/reference/tools/tidb-lightning/tidb-backend.md) is chosen, or the target database is used to store checkpoints, it additionally requires these privileges: +If the [TiDB-backend](/v3.1/reference/tools/tidb-lightning/tidb-backend.md) is chosen, or the target database is used to store checkpoints, it additionally requires these privileges: * INSERT * DELETE -The Importer-back-end does not require these two privileges because data is ingested into TiKV directly, which bypasses the entire TiDB privilege system. This is secure as long as the ports of TiKV, TiKV Importer and TiDB Lightning are not reachable outside the cluster. +The Importer-backend does not require these two privileges because data is ingested into TiKV directly, which bypasses the entire TiDB privilege system. This is secure as long as the ports of TiKV, TiKV Importer and TiDB Lightning are not reachable outside the cluster. If the `checksum` configuration of TiDB Lightning is set to `true`, then the admin user privileges in the downstream TiDB need to be granted to TiDB Lightning. diff --git a/v3.1/reference/tools/tidb-lightning/deployment.md b/v3.1/reference/tools/tidb-lightning/deployment.md index be22ccdbb32ab..f6425e2e01c88 100644 --- a/v3.1/reference/tools/tidb-lightning/deployment.md +++ b/v3.1/reference/tools/tidb-lightning/deployment.md @@ -6,9 +6,9 @@ category: reference # TiDB Lightning Deployment -This document describes the hardware requirements of TiDB Lightning using the default Importer-back-end, and how to deploy it using Ansible or manually. +This document describes the hardware requirements of TiDB Lightning using the default Importer-backend, and how to deploy it using Ansible or manually. -If you wish to use the TiDB-back-end, also read [TiDB Lightning TiDB-Back-End](/v3.1/reference/tools/tidb-lightning/tidb-backend.md) for the changes to the deployment steps. +If you wish to use the TiDB-backend, also read [TiDB Lightning TiDB-Backend](/v3.1/reference/tools/tidb-lightning/tidb-backend.md) for the changes to the deployment steps. ## Notes @@ -246,9 +246,9 @@ Refer to the [TiDB enterprise tools download page](/v3.1/reference/tools/downloa file = "tidb-lightning.log" [tikv-importer] - # Delivery back end, can be "importer" or "tidb". + # Delivery backend, can be "importer" or "tidb". # backend = "importer" - # The listening address of tikv-importer when back end is "importer". Change it to the actual address. + # The listening address of tikv-importer when backend is "importer". Change it to the actual address. addr = "172.16.31.10:8287" [mydumper] diff --git a/v3.1/reference/tools/tidb-lightning/overview.md b/v3.1/reference/tools/tidb-lightning/overview.md index 5a2fd98f51c40..104ca06725aab 100644 --- a/v3.1/reference/tools/tidb-lightning/overview.md +++ b/v3.1/reference/tools/tidb-lightning/overview.md @@ -41,4 +41,4 @@ The complete import process is as follows: 7. Finally, `tidb-lightning` switches the TiKV cluster back to "normal mode", so the cluster resumes normal services. -TiDB Lightning also supports using TiDB instead of Importer as the back end. In this configuration, `tidb-lightning` transforms data into SQL `INSERT` statements and directly executes them on the target cluster, similar to Loader. See [TiDB Lightning TiDB-Back-End](/v3.1/reference/tools/tidb-lightning/tidb-backend.md) for details. +TiDB Lightning also supports using TiDB instead of Importer as the backend. In this configuration, `tidb-lightning` transforms data into SQL `INSERT` statements and directly executes them on the target cluster, similar to Loader. See [TiDB Lightning TiDB-Backend](/v3.1/reference/tools/tidb-lightning/tidb-backend.md) for details. diff --git a/v3.1/reference/tools/tidb-lightning/tidb-backend.md b/v3.1/reference/tools/tidb-lightning/tidb-backend.md index 50276932c5409..5a4b08d57b01f 100644 --- a/v3.1/reference/tools/tidb-lightning/tidb-backend.md +++ b/v3.1/reference/tools/tidb-lightning/tidb-backend.md @@ -1,16 +1,16 @@ --- -title: TiDB Lightning TiDB-Back-End +title: TiDB Lightning TiDB-Backend summary: Choose how to write data into the TiDB cluster. category: reference --- -# TiDB Lightning TiDB-Back-End +# TiDB Lightning TiDB-Backend -TiDB Lightning supports two back ends: Importer and TiDB. It determines how `tidb-lightning` delivers data into the target cluster. +TiDB Lightning supports two backends: Importer and TiDB. It determines how `tidb-lightning` delivers data into the target cluster. -The Importer-back-end (default) requires `tidb-lightning` to first encode the SQL or CSV data into KV pairs, and relies on the external `tikv-importer` program to sort these KV pairs and ingest directly into the TiKV nodes. +The Importer-backend (default) requires `tidb-lightning` to first encode the SQL or CSV data into KV pairs, and relies on the external `tikv-importer` program to sort these KV pairs and ingest directly into the TiKV nodes. -The TiDB-back-end requires `tidb-lightning` to encode these data into SQL `INSERT` statements, and has these statements executed directly on the TiDB node. +The TiDB-backend requires `tidb-lightning` to encode these data into SQL `INSERT` statements, and has these statements executed directly on the TiDB node. | Back end | Importer | TiDB | |:---|:---|:---| @@ -19,16 +19,16 @@ The TiDB-back-end requires `tidb-lightning` to encode these data into SQL `INSER | ACID respected while importing | No | Yes | | Target tables | Must be empty | Can be populated | -## Deployment for TiDB-back-end +## Deployment for TiDB-backend -When using the TiDB-back-end, you no longer need `tikv-importer`. Compared with the [standard deployment procedure](/v3.1/reference/tools/tidb-lightning/deployment.md), the TiDB-back-end deployment has the following two differences: +When using the TiDB-backend, you no longer need `tikv-importer`. Compared with the [standard deployment procedure](/v3.1/reference/tools/tidb-lightning/deployment.md), the TiDB-backend deployment has the following two differences: * Steps involving `tikv-importer` can all be skipped. -* The configuration must be changed to indicate the TiDB-back-end is used. +* The configuration must be changed to indicate the TiDB-backend is used. ### Hardware requirements -The speed of TiDB Lightning using TiDB-back-end is limited by the SQL processing speed of TiDB. Therefore, even a lower-end machine may max out the possible performance. The recommended hardware configuration is: +The speed of TiDB Lightning using TiDB-backend is limited by the SQL processing speed of TiDB. Therefore, even a lower-end machine may max out the possible performance. The recommended hardware configuration is: * 16 logical cores CPU * An SSD large enough to store the entire data source, preferring higher read speed @@ -80,7 +80,7 @@ or supplying the `--backend tidb` arguments when executing `tidb-lightning`. ## Conflict resolution -The TiDB-back-end supports importing to an already-populated table. However, the new data might cause a unique key conflict with the old data. You can control how to resolve the conflict by using this task configuration. +The TiDB-backend supports importing to an already-populated table. However, the new data might cause a unique key conflict with the old data. You can control how to resolve the conflict by using this task configuration. ```toml [tikv-importer] @@ -94,9 +94,9 @@ on-duplicate = "replace" # or "error" or "ignore" | ignore | Keep old entries and ignore new ones | `INSERT IGNORE INTO ...` | | error | Abort import | `INSERT INTO ...` | -## Migrating from Loader to TiDB Lightning TiDB-back-end +## Migrating from Loader to TiDB Lightning TiDB-backend -TiDB Lightning using the TiDB-back-end can completely replace functions of [Loader](/v3.1/reference/tools/loader.md). The following list shows how to translate Loader configurations into [TiDB Lightning configurations](/v3.1/reference/tools/tidb-lightning/deployment.md#step-4-start-tidb-lightning). +TiDB Lightning using the TiDB-backend can completely replace functions of [Loader](/v3.1/reference/tools/loader.md). The following list shows how to translate Loader configurations into [TiDB Lightning configurations](/v3.1/reference/tools/tidb-lightning/deployment.md#step-4-start-tidb-lightning).
LoaderTiDB Lightning
@@ -175,7 +175,7 @@ schema = "tidb_lightning_checkpoint" ```toml [tikv-importer] -# use the TiDB-back-end +# use the TiDB-backend backend = "tidb" ``` From e36e020118828ecf9d7c74ec16c43d468244c8cc Mon Sep 17 00:00:00 2001 From: kennytm Date: Mon, 3 Feb 2020 15:09:13 +0800 Subject: [PATCH 9/9] tools/tidb-lightning: addressed comments on PR 1693 --- dev/TOC.md | 2 +- dev/reference/tools/tidb-lightning/deployment.md | 2 +- dev/reference/tools/tidb-lightning/glossary.md | 6 +++--- dev/reference/tools/tidb-lightning/overview.md | 2 +- dev/reference/tools/tidb-lightning/tidb-backend.md | 4 ++-- v3.0/TOC.md | 2 +- v3.0/reference/tools/tidb-lightning/deployment.md | 2 +- v3.0/reference/tools/tidb-lightning/overview.md | 2 +- v3.0/reference/tools/tidb-lightning/tidb-backend.md | 4 ++-- v3.1/TOC.md | 2 +- v3.1/reference/tools/tidb-lightning/deployment.md | 2 +- v3.1/reference/tools/tidb-lightning/overview.md | 2 +- v3.1/reference/tools/tidb-lightning/tidb-backend.md | 4 ++-- 13 files changed, 18 insertions(+), 18 deletions(-) diff --git a/dev/TOC.md b/dev/TOC.md index 1e6206b27b1a5..f3771dd746507 100644 --- a/dev/TOC.md +++ b/dev/TOC.md @@ -358,7 +358,7 @@ - [Checkpoints](/dev/reference/tools/tidb-lightning/checkpoints.md) - [Table Filter](/dev/reference/tools/tidb-lightning/table-filter.md) - [CSV Support](/dev/reference/tools/tidb-lightning/csv.md) - - [TiDB-Backend](/dev/reference/tools/tidb-lightning/tidb-backend.md) + - [TiDB-backend](/dev/reference/tools/tidb-lightning/tidb-backend.md) - [Web Interface](/dev/reference/tools/tidb-lightning/web.md) - [Monitor](/dev/reference/tools/tidb-lightning/monitor.md) - [Troubleshoot](/dev/how-to/troubleshoot/tidb-lightning.md) diff --git a/dev/reference/tools/tidb-lightning/deployment.md b/dev/reference/tools/tidb-lightning/deployment.md index 64e4fdfbb0b81..f029f22555fbd 100644 --- a/dev/reference/tools/tidb-lightning/deployment.md +++ b/dev/reference/tools/tidb-lightning/deployment.md @@ -8,7 +8,7 @@ category: reference This document describes the hardware requirements of TiDB Lightning using the default Importer-backend, and how to deploy it using Ansible or manually. -If you wish to use the TiDB-backend, also read [TiDB Lightning TiDB-Backend](/dev/reference/tools/tidb-lightning/tidb-backend.md) for the changes to the deployment steps. +If you wish to use the TiDB-backend, also read [TiDB Lightning TiDB-backend](/dev/reference/tools/tidb-lightning/tidb-backend.md) for the changes to the deployment steps. ## Notes diff --git a/dev/reference/tools/tidb-lightning/glossary.md b/dev/reference/tools/tidb-lightning/glossary.md index 3b7e669cc2543..42098d41a8245 100644 --- a/dev/reference/tools/tidb-lightning/glossary.md +++ b/dev/reference/tools/tidb-lightning/glossary.md @@ -32,7 +32,7 @@ Because TiDB Lightning imports data without going through TiDB, the `AUTO_INCREM Back end is the destination where TiDB Lightning sends the parsed result. Also spelled as "backend". -See [TiDB Lightning TiDB-Backend](/dev/reference/tools/tidb-lightning/tidb-backend.md) for details. +See [TiDB Lightning TiDB-backend](/dev/reference/tools/tidb-lightning/tidb-backend.md) for details. ### Black-white list @@ -118,7 +118,7 @@ Regardless of number of indices, every table is associated with exactly one inde TiDB Lightning processes multiple index engines concurrently. This is controlled by the `lightning.index-concurrency` setting. Since every table has exactly one index engine, this also configures the maximum number of tables to process at the same time. -### Ingestion +### Ingest An operation which inserts the entire content of an [SST file](/dev/reference/tools/tidb-lightning/glossary.md#sst-file) into the RocksDB (TiKV) store. @@ -186,4 +186,4 @@ An engine is typically very large (around 100 GB), which is not friendly to TiKV SST is the abbreviation of "sorted string table". An SST file is RocksDB's (and thus TiKV's) native storage format of a collection of KV pairs. -TiKV Importer produces SST files from a closed [engine](/dev/reference/tools/tidb-lightning/glossary.md#engine). These SST files are uploaded and then [ingested](/dev/reference/tools/tidb-lightning/glossary.md#ingestion) into TiKV stores. +TiKV Importer produces SST files from a closed [engine](/dev/reference/tools/tidb-lightning/glossary.md#engine). These SST files are uploaded and then [ingested](/dev/reference/tools/tidb-lightning/glossary.md#ingest) into TiKV stores. diff --git a/dev/reference/tools/tidb-lightning/overview.md b/dev/reference/tools/tidb-lightning/overview.md index a1830985000e1..d7efe90e98637 100644 --- a/dev/reference/tools/tidb-lightning/overview.md +++ b/dev/reference/tools/tidb-lightning/overview.md @@ -41,4 +41,4 @@ The complete import process is as follows: 7. Finally, `tidb-lightning` switches the TiKV cluster back to "normal mode", so the cluster resumes normal services. -TiDB Lightning also supports using TiDB instead of Importer as the backend. In this configuration, `tidb-lightning` transforms data into SQL `INSERT` statements and directly executes them on the target cluster, similar to Loader. See [TiDB Lightning TiDB-Backend](/dev/reference/tools/tidb-lightning/tidb-backend.md) for details. +TiDB Lightning also supports using TiDB instead of Importer as the backend. In this configuration, `tidb-lightning` transforms data into SQL `INSERT` statements and directly executes them on the target cluster, similar to Loader. See [TiDB Lightning TiDB-backend](/dev/reference/tools/tidb-lightning/tidb-backend.md) for details. diff --git a/dev/reference/tools/tidb-lightning/tidb-backend.md b/dev/reference/tools/tidb-lightning/tidb-backend.md index 41fa624af4057..e1e6b1375adbd 100644 --- a/dev/reference/tools/tidb-lightning/tidb-backend.md +++ b/dev/reference/tools/tidb-lightning/tidb-backend.md @@ -1,10 +1,10 @@ --- -title: TiDB Lightning TiDB-Backend +title: TiDB Lightning TiDB-backend summary: Choose how to write data into the TiDB cluster. category: reference --- -# TiDB Lightning TiDB-Backend +# TiDB Lightning TiDB-backend TiDB Lightning supports two backends: Importer and TiDB. It determines how `tidb-lightning` delivers data into the target cluster. diff --git a/v3.0/TOC.md b/v3.0/TOC.md index 27e3048d0fc1a..23fa726d3dde9 100644 --- a/v3.0/TOC.md +++ b/v3.0/TOC.md @@ -353,7 +353,7 @@ - [Checkpoints](/v3.0/reference/tools/tidb-lightning/checkpoints.md) - [Table Filter](/v3.0/reference/tools/tidb-lightning/table-filter.md) - [CSV Support](/v3.0/reference/tools/tidb-lightning/csv.md) - - [TiDB-Backend](/v3.0/reference/tools/tidb-lightning/tidb-backend.md) + - [TiDB-backend](/v3.0/reference/tools/tidb-lightning/tidb-backend.md) - [Monitor](/v3.0/reference/tools/tidb-lightning/monitor.md) - [Troubleshoot](/v3.0/how-to/troubleshoot/tidb-lightning.md) - [FAQ](/v3.0/faq/tidb-lightning.md) diff --git a/v3.0/reference/tools/tidb-lightning/deployment.md b/v3.0/reference/tools/tidb-lightning/deployment.md index 29bddd01abdf7..0b3177b65d8ad 100644 --- a/v3.0/reference/tools/tidb-lightning/deployment.md +++ b/v3.0/reference/tools/tidb-lightning/deployment.md @@ -8,7 +8,7 @@ category: reference This document describes the hardware requirements of TiDB Lightning using the default Importer-backend, and how to deploy it using Ansible or manually. -If you wish to use the TiDB-backend, also read [TiDB Lightning TiDB-Backend](/v3.0/reference/tools/tidb-lightning/tidb-backend.md) for the changes to the deployment steps. +If you wish to use the TiDB-backend, also read [TiDB Lightning TiDB-backend](/v3.0/reference/tools/tidb-lightning/tidb-backend.md) for the changes to the deployment steps. ## Notes diff --git a/v3.0/reference/tools/tidb-lightning/overview.md b/v3.0/reference/tools/tidb-lightning/overview.md index d7cf052d1de8e..425f658efe4f3 100644 --- a/v3.0/reference/tools/tidb-lightning/overview.md +++ b/v3.0/reference/tools/tidb-lightning/overview.md @@ -42,4 +42,4 @@ The complete import process is as follows: 7. Finally, `tidb-lightning` switches the TiKV cluster back to "normal mode", so the cluster resumes normal services. -TiDB Lightning also supports using TiDB instead of Importer as the backend. In this configuration, `tidb-lightning` transforms data into SQL `INSERT` statements and directly executes them on the target cluster, similar to Loader. See [TiDB Lightning TiDB-Backend](/v3.0/reference/tools/tidb-lightning/tidb-backend.md) for details. +TiDB Lightning also supports using TiDB instead of Importer as the backend. In this configuration, `tidb-lightning` transforms data into SQL `INSERT` statements and directly executes them on the target cluster, similar to Loader. See [TiDB Lightning TiDB-backend](/v3.0/reference/tools/tidb-lightning/tidb-backend.md) for details. diff --git a/v3.0/reference/tools/tidb-lightning/tidb-backend.md b/v3.0/reference/tools/tidb-lightning/tidb-backend.md index 69b6ae297d388..606f7e99bd73e 100644 --- a/v3.0/reference/tools/tidb-lightning/tidb-backend.md +++ b/v3.0/reference/tools/tidb-lightning/tidb-backend.md @@ -1,10 +1,10 @@ --- -title: TiDB Lightning TiDB-Backend +title: TiDB Lightning TiDB-backend summary: Choose how to write data into the TiDB cluster. category: reference --- -# TiDB Lightning TiDB-Backend +# TiDB Lightning TiDB-backend TiDB Lightning supports two backends: Importer and TiDB. It determines how `tidb-lightning` delivers data into the target cluster. diff --git a/v3.1/TOC.md b/v3.1/TOC.md index 7259355fd81f4..ff67390492e6e 100644 --- a/v3.1/TOC.md +++ b/v3.1/TOC.md @@ -355,7 +355,7 @@ - [Checkpoints](/v3.1/reference/tools/tidb-lightning/checkpoints.md) - [Table Filter](/v3.1/reference/tools/tidb-lightning/table-filter.md) - [CSV Support](/v3.1/reference/tools/tidb-lightning/csv.md) - - [TiDB-Backend](/v3.1/reference/tools/tidb-lightning/tidb-backend.md) + - [TiDB-backend](/v3.1/reference/tools/tidb-lightning/tidb-backend.md) - [Monitor](/v3.1/reference/tools/tidb-lightning/monitor.md) - [Troubleshoot](/v3.1/how-to/troubleshoot/tidb-lightning.md) - [FAQ](/v3.1/faq/tidb-lightning.md) diff --git a/v3.1/reference/tools/tidb-lightning/deployment.md b/v3.1/reference/tools/tidb-lightning/deployment.md index f6425e2e01c88..f942e35e97b05 100644 --- a/v3.1/reference/tools/tidb-lightning/deployment.md +++ b/v3.1/reference/tools/tidb-lightning/deployment.md @@ -8,7 +8,7 @@ category: reference This document describes the hardware requirements of TiDB Lightning using the default Importer-backend, and how to deploy it using Ansible or manually. -If you wish to use the TiDB-backend, also read [TiDB Lightning TiDB-Backend](/v3.1/reference/tools/tidb-lightning/tidb-backend.md) for the changes to the deployment steps. +If you wish to use the TiDB-backend, also read [TiDB Lightning TiDB-backend](/v3.1/reference/tools/tidb-lightning/tidb-backend.md) for the changes to the deployment steps. ## Notes diff --git a/v3.1/reference/tools/tidb-lightning/overview.md b/v3.1/reference/tools/tidb-lightning/overview.md index 104ca06725aab..b595a16c3041d 100644 --- a/v3.1/reference/tools/tidb-lightning/overview.md +++ b/v3.1/reference/tools/tidb-lightning/overview.md @@ -41,4 +41,4 @@ The complete import process is as follows: 7. Finally, `tidb-lightning` switches the TiKV cluster back to "normal mode", so the cluster resumes normal services. -TiDB Lightning also supports using TiDB instead of Importer as the backend. In this configuration, `tidb-lightning` transforms data into SQL `INSERT` statements and directly executes them on the target cluster, similar to Loader. See [TiDB Lightning TiDB-Backend](/v3.1/reference/tools/tidb-lightning/tidb-backend.md) for details. +TiDB Lightning also supports using TiDB instead of Importer as the backend. In this configuration, `tidb-lightning` transforms data into SQL `INSERT` statements and directly executes them on the target cluster, similar to Loader. See [TiDB Lightning TiDB-backend](/v3.1/reference/tools/tidb-lightning/tidb-backend.md) for details. diff --git a/v3.1/reference/tools/tidb-lightning/tidb-backend.md b/v3.1/reference/tools/tidb-lightning/tidb-backend.md index 5a4b08d57b01f..4c0c7ccd4c37c 100644 --- a/v3.1/reference/tools/tidb-lightning/tidb-backend.md +++ b/v3.1/reference/tools/tidb-lightning/tidb-backend.md @@ -1,10 +1,10 @@ --- -title: TiDB Lightning TiDB-Backend +title: TiDB Lightning TiDB-backend summary: Choose how to write data into the TiDB cluster. category: reference --- -# TiDB Lightning TiDB-Backend +# TiDB Lightning TiDB-backend TiDB Lightning supports two backends: Importer and TiDB. It determines how `tidb-lightning` delivers data into the target cluster.
LoaderTiDB Lightning