From 50759482c02499f819a574194b9b143f9b0ac159 Mon Sep 17 00:00:00 2001 From: TomShawn <41534398+TomShawn@users.noreply.github.com> Date: Wed, 5 Aug 2020 16:48:05 +0800 Subject: [PATCH 1/3] cherry pick #3544 to release-3.0 Signed-off-by: ti-srebot --- loader-overview.md | 37 ++++++++++++++++++++++++++++++++++ troubleshoot-tidb-lightning.md | 4 ++++ 2 files changed, 41 insertions(+) diff --git a/loader-overview.md b/loader-overview.md index 381b10edfdd96..5fc3ed2d8ef84 100644 --- a/loader-overview.md +++ b/loader-overview.md @@ -1,7 +1,11 @@ --- title: Loader Instructions summary: Use Loader to load data to TiDB. +<<<<<<< HEAD aliases: ['/docs/v3.0/loader-overview/','/docs/v3.0/reference/tools/loader/','/docs/tools/loader/'] +======= +aliases: ['/docs/v3.1/loader-overview/','/docs/v3.1/reference/tools/loader/','/docs/v3.1/load-misuse-handling/','/docs/v3.1/reference/tools/error-case-handling/load-misuse-handling/','/tidb/v3.1/load-misuse-handling'] +>>>>>>> 597c8f1... Remove 2 docs that are not in toc (#3544) --- # Loader Instructions @@ -150,3 +154,36 @@ pattern-table = "table_*" target-schema = "example_db" target-table = "table" ``` + +### Error: ```Try adjusting the `max_allowed_packet` variable``` + +The following error is reported during full data import: + +``` +packet for query is too large. Try adjusting the 'max_allowed_packet' variable +``` + +#### Reasons + +* Both MySQL client and MySQL/TiDB Server have `max_allowed_packet` quotas. If any of the `max_allowed_packet` quotas is violated, the client receives a corresponding error message. Currently, the latest version of Loader and TiDB Server all have a default `max_allowed_packet` quota of `64M`. + * Please use the latest version, or the latest stable version of the tool. See the [download page](/download-ecosystem-tools.md). +* The full data import processing module in Loader or DM does not support splitting `dump sqls` files. This is because Mydumper has the simple code implementation, as shown in the code comment `/* Poor man's data dump code */`. To support correctly splitting `dump sqls` files, you need to implement a sound parser based on TiDB parser, but you will encounter the following issues: + * A large amount of workload. + * The task is difficult. It is not easy to ensure the correctness. + * Significant performance reduction. + +#### Solutions + +* For the above reasons, it is recommended to use the `-s, --statement-size` option which Mydumper offers to control the size of `Insert Statement`: `Attempted size of INSERT statement in bytes, default 1000000`. + + According to the default configuration of `--statement-size`, the size of `Insert Statement` that Mydumper generates is as close as `1M`. This default configuration ensures that this error will not occur in most cases. + + Sometimes the following `WARN` log appears during the dump process. The `WARN` log itself does not affect the dump process but indicates that the dumped table might be a wide table. + + ``` + Row bigger than statement_size for xxx + ``` + +* If a single row of a wide table exceeds 64M, you need to modify and enable the following two configurations: + * Execute `set @@global.max_allowed_packet=134217728` (`134217728 = 128M`) in TiDB Server. + * Add `max-allowed-packet=128M` to db configuration in the Loader configuration file according to your situation, and then restart the progress or task. diff --git a/troubleshoot-tidb-lightning.md b/troubleshoot-tidb-lightning.md index fda3a805e72da..ba16a26e4d19c 100644 --- a/troubleshoot-tidb-lightning.md +++ b/troubleshoot-tidb-lightning.md @@ -1,7 +1,11 @@ --- title: TiDB Lightning Troubleshooting summary: Learn about common errors and solutions of TiDB Lightning. +<<<<<<< HEAD aliases: ['/docs/v3.0/troubleshoot-tidb-lightning/','/docs/v3.0/how-to/troubleshoot/tidb-lightning/'] +======= +aliases: ['/docs/v3.1/troubleshoot-tidb-lightning/','/docs/v3.1/how-to/troubleshoot/tidb-lightning/','/docs/v3.1/tidb-lightning/tidb-lightning-misuse-handling/','/docs/v3.1/reference/tools/error-case-handling/lightning-misuse-handling/','/tidb/v3.1/tidb-lightning-misuse-handling'] +>>>>>>> 597c8f1... Remove 2 docs that are not in toc (#3544) --- # TiDB Lightning Troubleshooting From 6fb0ec2d31513ad0732e02541cd2526adefccd0e Mon Sep 17 00:00:00 2001 From: TomShawn <41534398+TomShawn@users.noreply.github.com> Date: Wed, 5 Aug 2020 17:07:21 +0800 Subject: [PATCH 2/3] resolve conflicts and delete files --- load-misuse-handling.md | 38 ------------------- loader-overview.md | 6 +-- .../tidb-lightning-misuse-handling.md | 36 ------------------ troubleshoot-tidb-lightning.md | 6 +-- 4 files changed, 2 insertions(+), 84 deletions(-) delete mode 100644 load-misuse-handling.md delete mode 100644 tidb-lightning/tidb-lightning-misuse-handling.md diff --git a/load-misuse-handling.md b/load-misuse-handling.md deleted file mode 100644 index fecf7692c29a9..0000000000000 --- a/load-misuse-handling.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: Common Misuses During Full Data Import -aliases: ['/docs/v3.0/load-misuse-handling/','/docs/v3.0/reference/tools/error-case-handling/load-misuse-handling/'] ---- - -# Common Misuses During Full Data Import - -This document introduces the common error scenarios in using [Loader](/loader-overview.md) or [TiDB Data Migration](https://pingcap.com/docs/tidb-data-migration/stable/overview/) (DM) during the full data import process, and also provides their reasons and solutions. - -## Error: ```Try adjusting the `max_allowed_packet` variable``` - -The following error is reported during full data import: - -``` -packet for query is too large. Try adjusting the 'max_allowed_packet' variable -``` - -### Reasons - -* Both MySQL client and MySQL/TiDB Server have `max_allowed_packet` quotas. If any of the `max_allowed_packet` quotas is violated, the client receives a corresponding error message. Currently, the latest version of Syncer, Loader, DM, and TiDB Server all have a default `max_allowed_packet` quota of `64M`. - * Please use the latest version, or the latest stable version of the tool. See the [download page](/download-ecosystem-tools.md). -* The full data import processing module in Loader or DM does not support splitting `dump sqls` files. - -### Solutions - -* For the above reasons, it is recommended to use the `-s, --statement-size` option which Mydumper offers to control the size of `Insert Statement`: `Attempted size of INSERT statement in bytes, default 1000000`. - - According to the default configuration of `--statement-size`, the size of `Insert Statement` that Mydumper generates is as close as `1M`. This default configuration ensures that this error will not occur in most cases. - - Sometimes the following `WARN` log appears during the dump process. The `WARN` log itself does not affect the dump process but indicates that the dumped table might be a wide table. - - ``` - Row bigger than statement_size for xxx - ``` - -* If a single row of a wide table exceeds 64M, you need to modify and enable the following two configurations: - * Execute `set @@global.max_allowed_packet=134217728` (`134217728 = 128M`) in TiDB Server. - * Add the configuration like `max-allowed-packet=128M` to the Loader configuration file or DB configuration in DM task configuration file, and then restart the progress or task. \ No newline at end of file diff --git a/loader-overview.md b/loader-overview.md index 5fc3ed2d8ef84..f6f2786b71bbe 100644 --- a/loader-overview.md +++ b/loader-overview.md @@ -1,11 +1,7 @@ --- title: Loader Instructions summary: Use Loader to load data to TiDB. -<<<<<<< HEAD -aliases: ['/docs/v3.0/loader-overview/','/docs/v3.0/reference/tools/loader/','/docs/tools/loader/'] -======= -aliases: ['/docs/v3.1/loader-overview/','/docs/v3.1/reference/tools/loader/','/docs/v3.1/load-misuse-handling/','/docs/v3.1/reference/tools/error-case-handling/load-misuse-handling/','/tidb/v3.1/load-misuse-handling'] ->>>>>>> 597c8f1... Remove 2 docs that are not in toc (#3544) +aliases: ['/docs/v3.0/loader-overview/','/docs/v3.0/reference/tools/loader/','/docs/tools/loader/',,'/docs/v3.0/load-misuse-handling/','/docs/v3.0/reference/tools/error-case-handling/load-misuse-handling/','/tidb/v3.0/load-misuse-handling'] --- # Loader Instructions diff --git a/tidb-lightning/tidb-lightning-misuse-handling.md b/tidb-lightning/tidb-lightning-misuse-handling.md deleted file mode 100644 index e9ea2adc34939..0000000000000 --- a/tidb-lightning/tidb-lightning-misuse-handling.md +++ /dev/null @@ -1,36 +0,0 @@ ---- -title: Common Misuses of TiDB Lightning -aliases: ['/docs/v3.0/tidb-lightning/tidb-lightning-misuse-handling/','/docs/v3.0/reference/tools/error-case-handling/lightning-misuse-handling/'] ---- - -# Common Misuses of TiDB Lightning - -This document introduces common error scenarios in using [TiDB Lightning](/tidb-lightning/tidb-lightning-overview.md), and also provides their reasons and solutions. - -## Error: `checksum mismatched remote vs local` - -The following error is reported during data import: - -```log -Error: checksum mismatched remote vs local => (checksum: 3828723015727756136 vs 7895534721177712659) (total_kvs: 1221416844 vs 1501500000) (total_bytes:237660308510 vs 292158203078) -``` - -### Reasons - -* TiDB Lightning was used to import data previously, and the corresponding [`checkpoint`](/tidb-lightning/tidb-lightning-checkpoints.md) data was not cleaned up. To confirm this, you can check the log at the first launch of TiDB Lightning: - - * When `[checkpoint] driver = file`, if the log that marks the beginning of data import using TiDB Lightning shows `open checkpoint file failed, going to create a new one`, then the `checkpoint` is cleaned correctly. Otherwise, the remaining data might lead to imported data missing. - * When `[checkpoint] driver = mysql`, you can run `curl http://{TiDBIP}:10080/schema/{checkpoint.schema}/{checkpoint.table}` through TiDB API to query the creation time of `checkpoint table`. Then you can confirm whether the `checkpoint` is cleaned correctly. - -* There is conflicting data from the data source imported by TiDB Lightning. - * Data in different rows have the same primary key or unique key. - -### Solutions - -* Delete data from tables with `checksum mismatch` error. - - ``` - tidb-lightning-ctl --config conf/tidb-lightning.toml --checkpoint-error-destroy=all - ``` - -* Find a way to detect whether there is conflicting data in the data source. TiDB Lightning generally processes large amounts of data, so currently there is no effective conflict detection mechanism or solution yet. \ No newline at end of file diff --git a/troubleshoot-tidb-lightning.md b/troubleshoot-tidb-lightning.md index ba16a26e4d19c..d91791a883223 100644 --- a/troubleshoot-tidb-lightning.md +++ b/troubleshoot-tidb-lightning.md @@ -1,11 +1,7 @@ --- title: TiDB Lightning Troubleshooting summary: Learn about common errors and solutions of TiDB Lightning. -<<<<<<< HEAD -aliases: ['/docs/v3.0/troubleshoot-tidb-lightning/','/docs/v3.0/how-to/troubleshoot/tidb-lightning/'] -======= -aliases: ['/docs/v3.1/troubleshoot-tidb-lightning/','/docs/v3.1/how-to/troubleshoot/tidb-lightning/','/docs/v3.1/tidb-lightning/tidb-lightning-misuse-handling/','/docs/v3.1/reference/tools/error-case-handling/lightning-misuse-handling/','/tidb/v3.1/tidb-lightning-misuse-handling'] ->>>>>>> 597c8f1... Remove 2 docs that are not in toc (#3544) +aliases: ['/docs/v3.0/troubleshoot-tidb-lightning/','/docs/v3.0/how-to/troubleshoot/tidb-lightning/','/docs/v3.0/tidb-lightning/tidb-lightning-misuse-handling/','/docs/v3.0/reference/tools/error-case-handling/lightning-misuse-handling/','/tidb/v3.0/tidb-lightning-misuse-handling'] --- # TiDB Lightning Troubleshooting From 06f79ec504e6fadc2071fa6fc0ecd515b961c69d Mon Sep 17 00:00:00 2001 From: TomShawn <41534398+TomShawn@users.noreply.github.com> Date: Wed, 5 Aug 2020 17:08:19 +0800 Subject: [PATCH 3/3] Update loader-overview.md --- loader-overview.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loader-overview.md b/loader-overview.md index f6f2786b71bbe..87c3ac38f3a84 100644 --- a/loader-overview.md +++ b/loader-overview.md @@ -1,7 +1,7 @@ --- title: Loader Instructions summary: Use Loader to load data to TiDB. -aliases: ['/docs/v3.0/loader-overview/','/docs/v3.0/reference/tools/loader/','/docs/tools/loader/',,'/docs/v3.0/load-misuse-handling/','/docs/v3.0/reference/tools/error-case-handling/load-misuse-handling/','/tidb/v3.0/load-misuse-handling'] +aliases: ['/docs/v3.0/loader-overview/','/docs/v3.0/reference/tools/loader/','/docs/tools/loader/','/docs/v3.0/load-misuse-handling/','/docs/v3.0/reference/tools/error-case-handling/load-misuse-handling/','/tidb/v3.0/load-misuse-handling'] --- # Loader Instructions