From dc664f24229cbdf176393c12be80f16dc84d376a Mon Sep 17 00:00:00 2001 From: TomShawn <41534398+TomShawn@users.noreply.github.com> Date: Wed, 22 Jul 2020 10:39:20 +0800 Subject: [PATCH 1/2] cherry pick #3270 to release-4.0 Signed-off-by: ti-srebot --- TOC.md | 1 + loader-overview.md | 57 ++++++++++++++++--- .../get-started-with-tidb-binlog.md | 0 3 files changed, 50 insertions(+), 8 deletions(-) rename get-started-with-tidb-binlog.md => tidb-binlog/get-started-with-tidb-binlog.md (100%) diff --git a/TOC.md b/TOC.md index e8fe83cd5a688..b6f8d8c7a496f 100644 --- a/TOC.md +++ b/TOC.md @@ -144,6 +144,7 @@ + [BR Use Cases](/br/backup-and-restore-use-cases.md) + TiDB Binlog + [Overview](/tidb-binlog/tidb-binlog-overview.md) + + [Quick Start](/tidb-binlog/get-started-with-tidb-binlog.md) + [Deploy](/tidb-binlog/deploy-tidb-binlog.md) + [Maintain](/tidb-binlog/maintain-tidb-binlog-cluster.md) + [Configure](/tidb-binlog/tidb-binlog-configuration-file.md) diff --git a/loader-overview.md b/loader-overview.md index 7acf0d4a3d1cb..76275cbbade13 100644 --- a/loader-overview.md +++ b/loader-overview.md @@ -1,11 +1,19 @@ --- title: Loader Instructions summary: Use Loader to load data to TiDB. +<<<<<<< HEAD aliases: ['/docs/stable/loader-overview/','/docs/v4.0/loader-overview/','/docs/stable/reference/tools/loader/'] +======= +aliases: ['/docs/dev/loader-overview/','/docs/dev/reference/tools/loader/','/docs/dev/load-misuse-handling/','/docs/dev/reference/tools/error-case-handling/load-misuse-handling/','/tidb/dev/load-misuse-handling'] +>>>>>>> 4405915... loader: move loader faq to loader overview (#3270) --- # Loader Instructions +> **Warning:** +> +> Loader is no longer maintained. Its features are completely superseded by [TiDB Lightning TiDB-backend](/tidb-lightning/tidb-lightning-tidb-backend.md). It is strongly recommend to use TiDB Lightning instead. + ## What is Loader? Loader is a data import tool to load data to TiDB. @@ -18,11 +26,11 @@ Since tools like mysqldump will take us days to migrate massive amounts of data, ## What can Loader do? -+ Multi-thread import data ++ Multi-threaded data import -+ Support table level concurrent import and scattered hot spot write ++ Support table-level concurrent import and scattered hotspot write -+ Support concurrent import of a single large table and scattered hot spot write ++ Support concurrent import of a single large table and scattered hotspot write + Support Mydumper data format @@ -56,23 +64,23 @@ Since tools like mysqldump will take us days to migrate massive amounts of data, -d string: the storage directory of data that need to import (default: "./") - -h string: the host of TiDB (default: "127.0.0.1") + -h string: the host IP of TiDB (default: "127.0.0.1") -p string: the account and password of TiDB -status-addr string: It can be used by Prometheus to pull Loader metrics, and is also the pprof address of Loader (default: ":8272") - -t int: the number of thread,increase this as TiKV nodes increase (default: 16) + -t int: the number of threads. Each worker restores one file at a time. (default: 16) -u string: the user name of TiDB (default: "root") ``` ### Configuration file -Apart from command line parameters, you can also use configuration files. The format is shown as below: +Apart from command-line parameters, you can also use configuration files. The format is shown as below: ```toml -# Loader log level, which can be set as "debug", "info", "warn", "error" and "fatal" (default: "info") +# Loader log level, which can be set as "debug", "info", "warn", "error", and "fatal" (default: "info") log-level = "info" # Loader log file @@ -87,7 +95,7 @@ status-addr = ":8272" # The checkpoint data is saved to TiDB, and the schema name is defined here. checkpoint-schema = "tidb_loader" -# Number of threads restoring concurrently for worker pool (default: 16). Each worker restore one file at a time. +# Number of threads restoring concurrently for worker pool (default: 16). Each worker restores one file at a time. pool-size = 16 # The target database information @@ -150,3 +158,36 @@ pattern-table = "table_*" target-schema = "example_db" target-table = "table" ``` + +## Error: ```Try adjusting the `max_allowed_packet` variable``` + +The following error is reported during full data import: + +``` +packet for query is too large. Try adjusting the 'max_allowed_packet' variable +``` + +### Reasons + +* Both MySQL client and MySQL/TiDB Server have `max_allowed_packet` quotas. If any of the `max_allowed_packet` quotas is violated, the client receives a corresponding error message. Currently, the latest version of Loader and TiDB Server all have a default `max_allowed_packet` quota of `64M`. + * Please use the latest version, or the latest stable version of the tool. See the [download page](/download-ecosystem-tools.md). +* The full data import processing module in Loader or DM does not support splitting `dump sqls` files. This is because Mydumper has the simple code implementation, as shown in the code comment `/* Poor man's data dump code */`. To support correctly splitting `dump sqls` files, you need to implement a sound parser based on TiDB parser, but you will encounter the following issues: + * A large amount of workload. + * The task is difficult. It is not easy to ensure the correctness. + * Significant performance reduction. + +### Solutions + +* For the above reasons, it is recommended to use the `-s, --statement-size` option which Mydumper offers to control the size of `Insert Statement`: `Attempted size of INSERT statement in bytes, default 1000000`. + + According to the default configuration of `--statement-size`, the size of `Insert Statement` that Mydumper generates is as close as `1M`. This default configuration ensures that this error will not occur in most cases. + + Sometimes the following `WARN` log appears during the dump process. The `WARN` log itself does not affect the dump process but indicates that the dumped table might be a wide table. + + ``` + Row bigger than statement_size for xxx + ``` + +* If a single row of a wide table exceeds 64M, you need to modify and enable the following two configurations: + * Execute `set @@global.max_allowed_packet=134217728` (`134217728 = 128M`) in TiDB Server. + * Add `max-allowed-packet=128M` to db configuration in the Loader configuration file according to your situation, and then restart the progress or task. diff --git a/get-started-with-tidb-binlog.md b/tidb-binlog/get-started-with-tidb-binlog.md similarity index 100% rename from get-started-with-tidb-binlog.md rename to tidb-binlog/get-started-with-tidb-binlog.md From 2c42723d9639383995454261eed6c9c727b48ceb Mon Sep 17 00:00:00 2001 From: TomShawn <41534398+TomShawn@users.noreply.github.com> Date: Wed, 22 Jul 2020 10:53:53 +0800 Subject: [PATCH 2/2] resolve conflict --- load-misuse-handling.md | 38 -------------------------------------- loader-overview.md | 6 +----- 2 files changed, 1 insertion(+), 43 deletions(-) delete mode 100644 load-misuse-handling.md diff --git a/load-misuse-handling.md b/load-misuse-handling.md deleted file mode 100644 index 1298fb647c56f..0000000000000 --- a/load-misuse-handling.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -title: Common Misuses During Full Data Import -aliases: ['/docs/stable/load-misuse-handling/','/docs/v4.0/load-misuse-handling/','/docs/stable/reference/tools/error-case-handling/load-misuse-handling/'] ---- - -# Common Misuses During Full Data Import - -This document introduces the common error scenarios in using [Loader](/loader-overview.md) or [TiDB Data Migration](https://docs.pingcap.com/tidb-data-migration/v1.0/overview) (DM) during the full data import process, and also provides their reasons and solutions. - -## Error: ```Try adjusting the `max_allowed_packet` variable``` - -The following error is reported during full data import: - -``` -packet for query is too large. Try adjusting the 'max_allowed_packet' variable -``` - -### Reasons - -* Both MySQL client and MySQL/TiDB Server have `max_allowed_packet` quotas. If any of the `max_allowed_packet` quotas is violated, the client receives a corresponding error message. Currently, the latest version of Syncer, Loader, DM, and TiDB Server all have a default `max_allowed_packet` quota of `64M`. - * Please use the latest version, or the latest stable version of the tool. See the [download page](/download-ecosystem-tools.md). -* The full data import processing module in Loader or DM does not support splitting `dump sqls` files. - -### Solutions - -* For the above reasons, it is recommended to use the `-s, --statement-size` option which Mydumper offers to control the size of `Insert Statement`: `Attempted size of INSERT statement in bytes, default 1000000`. - - According to the default configuration of `--statement-size`, the size of `Insert Statement` that Mydumper generates is as close as `1M`. This default configuration ensures that this error will not occur in most cases. - - Sometimes the following `WARN` log appears during the dump process. The `WARN` log itself does not affect the dump process but indicates that the dumped table might be a wide table. - - ``` - Row bigger than statement_size for xxx - ``` - -* If a single row of a wide table exceeds 64M, you need to modify and enable the following two configurations: - * Execute `set @@global.max_allowed_packet=134217728` (`134217728 = 128M`) in TiDB Server. - * Add the configuration like `max-allowed-packet=128M` to the Loader configuration file or DB configuration in DM task configuration file, and then restart the progress or task. \ No newline at end of file diff --git a/loader-overview.md b/loader-overview.md index 76275cbbade13..b57314d86793a 100644 --- a/loader-overview.md +++ b/loader-overview.md @@ -1,11 +1,7 @@ --- title: Loader Instructions summary: Use Loader to load data to TiDB. -<<<<<<< HEAD -aliases: ['/docs/stable/loader-overview/','/docs/v4.0/loader-overview/','/docs/stable/reference/tools/loader/'] -======= -aliases: ['/docs/dev/loader-overview/','/docs/dev/reference/tools/loader/','/docs/dev/load-misuse-handling/','/docs/dev/reference/tools/error-case-handling/load-misuse-handling/','/tidb/dev/load-misuse-handling'] ->>>>>>> 4405915... loader: move loader faq to loader overview (#3270) +aliases: ['/docs/stable/loader-overview/','/docs/v4.0/loader-overview/','/docs/stable/reference/tools/loader/','/docs/stable/load-misuse-handling/','/docs/v4.0/load-misuse-handling/','/docs/stable/reference/tools/error-case-handling/load-misuse-handling/','/tidb/stable/load-misuse-handling'] --- # Loader Instructions