From e79939e321b5471857d7f332012587a8301945a4 Mon Sep 17 00:00:00 2001 From: Jon Tirsen Date: Fri, 2 Oct 2020 17:42:55 +0200 Subject: [PATCH] Use the updated Writer implementation from br --- go.mod | 9 +++--- go.sum | 25 +++++++++++++++ v4/export/metadata.go | 4 +-- v4/export/test_util.go | 4 +-- v4/export/writer.go | 12 +++---- v4/export/writer_util.go | 59 ++++++++++++++++++----------------- v4/export/writer_util_test.go | 20 ++++++------ 7 files changed, 80 insertions(+), 53 deletions(-) diff --git a/go.mod b/go.mod index dd4d0d42..ba1d2954 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,8 @@ module github.com/pingcap/dumpling go 1.13 +replace github.com/pingcap/br => /Users/jontirsen/Development/br + require ( github.com/DATA-DOG/go-sqlmock v1.4.1 github.com/coreos/go-semver v0.3.0 @@ -9,18 +11,17 @@ require ( github.com/go-sql-driver/mysql v1.5.0 github.com/pingcap/br v0.0.0-20200925095602-bf9cc603382e github.com/pingcap/check v0.0.0-20200212061837-5e12011dc712 - github.com/pingcap/errors v0.11.5-0.20200902104258-eba4f1d8f6de + github.com/pingcap/errors v0.11.5-0.20200917111840-a15ef68f753d github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce - github.com/pingcap/kvproto v0.0.0-20200910095337-6b893f12be43 + github.com/pingcap/kvproto v0.0.0-20200927025644-73dc27044686 github.com/pingcap/log v0.0.0-20200828042413-fce0951f1463 github.com/pingcap/tidb-tools v4.0.5-0.20200820092506-34ea90c93237+incompatible github.com/pkg/errors v0.9.1 github.com/soheilhy/cmux v0.1.4 github.com/spf13/pflag v1.0.5 - github.com/tikv/pd v1.1.0-beta.0.20200825070655-6b09f3acbb1f + github.com/tikv/pd v1.1.0-beta.0.20200910042021-254d1345be09 go.etcd.io/etcd v0.5.0-alpha.5.0.20191023171146-3cf2f69b5738 go.uber.org/zap v1.16.0 - golang.org/x/net v0.0.0-20200822124328-c89045814202 // indirect golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8 // indirect golang.org/x/tools v0.0.0-20200823205832-c024452afbcd // indirect diff --git a/go.sum b/go.sum index 5613f560..2609087e 100644 --- a/go.sum +++ b/go.sum @@ -36,6 +36,7 @@ github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUW github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= +github.com/VividCortex/mysqlerr v0.0.0-20200629151747-c28746d985dd/go.mod h1:f3HiCrHjHBdcm6E83vGaXh1KomZMA2P6aeo3hKx/wg0= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -60,6 +61,7 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d/go.mod h1:VKt7CNAQxpFpSDz3sXyj9hY/GbVsQCr0sB3w59nE7lU= github.com/c2fo/vfs/v5 v5.5.4 h1:NNfNJA4thbMzrxN4MzVQttQyWkY/U2MW8SVOUIriHKs= github.com/c2fo/vfs/v5 v5.5.4/go.mod h1:62NkBSss3QjWL+4t/TgCuf6WTfBtq1qWTn3Y+1aIIu4= +github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5/go.mod h1:jtAfVaU/2cu1+wdSRPWE2c1N2qeAA3K4RH9pYgqwets= github.com/cenkalti/backoff/v4 v4.0.2/go.mod h1:eEew/i+1Q6OrCDZh3WiXYv3+nJwBASZ8Bog/87DQnVg= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= @@ -90,6 +92,7 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7 h1:u9SHYsPQNyt5tgDm3YN7+9dYrpK96E5wFilTFWIDZOM= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20181031085051-9002847aa142/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+cbHpyrpLDmnN1HqhBfnX7WDiW7eG2c= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -408,6 +411,7 @@ github.com/ngaut/unistore v0.0.0-20200803051709-607d96233b1d/go.mod h1:2QAH8tXCj github.com/ngaut/unistore v0.0.0-20200806113332-5b9f73333a19/go.mod h1:RtZJKyiaHRiII+b9/g/4339rSikSvfrUJmIbrUkYVi4= github.com/ngaut/unistore v0.0.0-20200820080223-c734bcc4ea53/go.mod h1:85S5ZgzoHtTMyaEYhaWnxv9OWMBfyhTNuWypXCfVn/0= github.com/ngaut/unistore v0.0.0-20200828072424-1c0ede06a3fc/go.mod h1:iSlx5Ub/926GvQn6+d2B2C16wJJwgQIsi6k/bEU0vl4= +github.com/ngaut/unistore v0.0.0-20200918091209-68a2db6bb775/go.mod h1:ZR3NH+HzqfiYetwdoAivApnIy8iefPZHTMLfrFNm8g4= github.com/nicksnyder/go-i18n v1.10.0/go.mod h1:HrK7VCrbOvQoUAQ7Vpy7i87N7JZZZ7R2xBGjv0j365Q= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= @@ -440,9 +444,11 @@ github.com/pingcap-incubator/tidb-dashboard v0.0.0-20200514075710-eecc9a4525b5/g github.com/pingcap-incubator/tidb-dashboard v0.0.0-20200710045508-523e95bc5ec9/go.mod h1:9yaAM77sPfa5/f6sdxr3jSkKfIz463KRHyiFHiGjdes= github.com/pingcap-incubator/tidb-dashboard v0.0.0-20200715070228-47f5de8a6992/go.mod h1:9yaAM77sPfa5/f6sdxr3jSkKfIz463KRHyiFHiGjdes= github.com/pingcap-incubator/tidb-dashboard v0.0.0-20200807020752-01f0abe88e93/go.mod h1:9yaAM77sPfa5/f6sdxr3jSkKfIz463KRHyiFHiGjdes= +github.com/pingcap-incubator/tidb-dashboard v0.0.0-20200908071351-a715a95c7de2/go.mod h1:X3r7/4Wr9fSC5KlsfezBh/5noeWGEJNQuSvjgS2rvdI= github.com/pingcap/badger v1.5.1-0.20200604041313-19c397305fcc/go.mod h1:LyrqUOHZrUDf9oGi1yoz1+qw9ckSIhQb5eMa1acOLNQ= github.com/pingcap/badger v1.5.1-0.20200714132513-80ba2000f159/go.mod h1:LyrqUOHZrUDf9oGi1yoz1+qw9ckSIhQb5eMa1acOLNQ= github.com/pingcap/badger v1.5.1-0.20200810065601-8c92a97807f9/go.mod h1:LyrqUOHZrUDf9oGi1yoz1+qw9ckSIhQb5eMa1acOLNQ= +github.com/pingcap/badger v1.5.1-0.20200908111422-2e78ee155d19/go.mod h1:LyrqUOHZrUDf9oGi1yoz1+qw9ckSIhQb5eMa1acOLNQ= github.com/pingcap/br v0.0.0-20200426093517-dd11ae28b885/go.mod h1:4w3meMnk7HDNpNgjuRAxavruTeKJvUiXxoEWTjzXPnA= github.com/pingcap/br v0.0.0-20200521085655-53201addd4ad/go.mod h1:SlSUHWY7QUoooiYxOKuJ8kUh2KjI29ogBh89YXz2dLA= github.com/pingcap/br v0.0.0-20200610051721-b057d65ff579/go.mod h1:Gq6o66nDReG0fMxqSdl8JHLZhMnxKhVxtHxqubUWAyg= @@ -474,6 +480,8 @@ github.com/pingcap/errors v0.11.5-0.20200820035142-66eb5bf1d1cd h1:ay+wAVWHI/Z6v github.com/pingcap/errors v0.11.5-0.20200820035142-66eb5bf1d1cd/go.mod h1:g4vx//d6VakjJ0mk7iLBlKA8LFavV/sAVINT/1PFxeQ= github.com/pingcap/errors v0.11.5-0.20200902104258-eba4f1d8f6de h1:mW8hC2yXTpflfyTeJgcN4aJQfwcYODde8YgjBgAy6do= github.com/pingcap/errors v0.11.5-0.20200902104258-eba4f1d8f6de/go.mod h1:g4vx//d6VakjJ0mk7iLBlKA8LFavV/sAVINT/1PFxeQ= +github.com/pingcap/errors v0.11.5-0.20200917111840-a15ef68f753d h1:TH18wFO5Nq/zUQuWu9ms2urgZnLP69XJYiI2JZAkUGc= +github.com/pingcap/errors v0.11.5-0.20200917111840-a15ef68f753d/go.mod h1:g4vx//d6VakjJ0mk7iLBlKA8LFavV/sAVINT/1PFxeQ= github.com/pingcap/failpoint v0.0.0-20191029060244-12f4ac2fd11d/go.mod h1:DNS3Qg7bEDhU6EXNHF+XSv/PGznQaMJ5FWvctpm6pQI= github.com/pingcap/failpoint v0.0.0-20200210140405-f8f9fb234798/go.mod h1:DNS3Qg7bEDhU6EXNHF+XSv/PGznQaMJ5FWvctpm6pQI= github.com/pingcap/failpoint v0.0.0-20200506114213-c17f16071c53/go.mod h1:w4PEZ5y16LeofeeGwdgZB4ddv9bLyDuIX+ljstgKZyk= @@ -499,9 +507,12 @@ github.com/pingcap/kvproto v0.0.0-20200715040832-c3e2e0b163ee/go.mod h1:IOdRDPLy github.com/pingcap/kvproto v0.0.0-20200803054707-ebd5de15093f/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/kvproto v0.0.0-20200810113304-6157337686b1 h1:hv22UEhdqeIqa5Jx0oeqDQNGHUBSW3LVOx02gqbd5Gg= github.com/pingcap/kvproto v0.0.0-20200810113304-6157337686b1/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= +github.com/pingcap/kvproto v0.0.0-20200827082727-23dedec2339b/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/kvproto v0.0.0-20200828054126-d677e6fd224a/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/kvproto v0.0.0-20200910095337-6b893f12be43 h1:SIElNchZjuPWQ727um6AjfUDFkqClZolUn746SdGg2w= github.com/pingcap/kvproto v0.0.0-20200910095337-6b893f12be43/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= +github.com/pingcap/kvproto v0.0.0-20200927025644-73dc27044686 h1:cf7TL5LMMPQew7vPOtvcfam6AyYxwu5uzcOrHMN8z7k= +github.com/pingcap/kvproto v0.0.0-20200927025644-73dc27044686/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI= github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20200117041106-d28c14d3b1cd/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8= github.com/pingcap/log v0.0.0-20200511115504-543df19646ad h1:SveG82rmu/GFxYanffxsSF503SiQV+2JLnWEiGiF+Tc= @@ -519,6 +530,7 @@ github.com/pingcap/parser v0.0.0-20200731033026-84f62115187c/go.mod h1:vQdbJqobJ github.com/pingcap/parser v0.0.0-20200813083329-a4bff035d3e2/go.mod h1:vQdbJqobJAgFyiRNNtXahpMoGWwPEuWciVEK5A20NS0= github.com/pingcap/parser v0.0.0-20200909072241-6dac7bb703e2 h1:ucCC+Xo6yTXbvJD2JELhHJbrZJYzJpv5lQeAxzzg9Ec= github.com/pingcap/parser v0.0.0-20200909072241-6dac7bb703e2/go.mod h1:RlLfMRJwFBSiXd2lUaWdV5pSXtrpyvZM8k5bbZWsheU= +github.com/pingcap/parser v0.0.0-20200921041333-cd2542b7a8a2/go.mod h1:RlLfMRJwFBSiXd2lUaWdV5pSXtrpyvZM8k5bbZWsheU= github.com/pingcap/pd/v4 v4.0.0-rc.1.0.20200422143320-428acd53eba2/go.mod h1:s+utZtXDznOiL24VK0qGmtoHjjXNsscJx3m1n8cC56s= github.com/pingcap/pd/v4 v4.0.0-rc.2.0.20200520083007-2c251bd8f181/go.mod h1:q4HTx/bA8aKBa4S7L+SQKHvjRPXCRV0tA0yRw0qkZSA= github.com/pingcap/pd/v4 v4.0.0-rc.2.0.20200714122454-1a64f969cb3c/go.mod h1:v/dY4mVkt3dh/Liphhk0E4ScOkZpIk0m0GvWJ9FapDs= @@ -544,6 +556,7 @@ github.com/pingcap/tidb v1.1.0-beta.0.20200810064414-d81150394f9d/go.mod h1:vLYo github.com/pingcap/tidb v1.1.0-beta.0.20200820085534-0d997f2b8b3c/go.mod h1:z7Hn1KY8Crt9cHhWtbGPKMBcjvmSJXIoOjO4rMk165w= github.com/pingcap/tidb v1.1.0-beta.0.20200910052409-5d52a34b2476 h1:tTG85OVtt1A4+HAbBAy6RKHe5QkLoGsJ5Wj/ETZ/uvc= github.com/pingcap/tidb v1.1.0-beta.0.20200910052409-5d52a34b2476/go.mod h1:UQ46/x8gOmKLyBv/XW7JkaOFMdtJOFl20PWxSc6cMh8= +github.com/pingcap/tidb v1.1.0-beta.0.20200921082409-501466fb690d/go.mod h1:n6cUCN/yzxI+2zsH/tmXAb3elV3SQIGTMqLujpzXgfA= github.com/pingcap/tidb-tools v4.0.0-beta.1.0.20200306084441-875bd09aa3d5+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= github.com/pingcap/tidb-tools v4.0.0-rc.1.0.20200421113014-507d2bb3a15e+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= github.com/pingcap/tidb-tools v4.0.0-rc.1.0.20200514040632-f76b3e428e19+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= @@ -675,6 +688,9 @@ github.com/tikv/pd v1.1.0-beta.0.20200818122340-ef1a4e920b2f/go.mod h1:mwZ3Lip1Y github.com/tikv/pd v1.1.0-beta.0.20200820084926-bcfa77a7a593/go.mod h1:quwjWtCmawAvS+YdxtSKG08sEexLzkhQgAno59wW+lI= github.com/tikv/pd v1.1.0-beta.0.20200825070655-6b09f3acbb1f h1:LwrGUM/5vEQGUhgNUOoyBTsSWWjvH2mzebKqB8c/uIA= github.com/tikv/pd v1.1.0-beta.0.20200825070655-6b09f3acbb1f/go.mod h1:quwjWtCmawAvS+YdxtSKG08sEexLzkhQgAno59wW+lI= +github.com/tikv/pd v1.1.0-beta.0.20200907085700-5b04bec39b99/go.mod h1:h0GTvNPZrjA06ToexaL13DxlzAvm/6kkJWz12baD68M= +github.com/tikv/pd v1.1.0-beta.0.20200910042021-254d1345be09 h1:5NsHTjk0O7C3/d8vfl/cWu9L6db+8YGvEj7XBGbMTbY= +github.com/tikv/pd v1.1.0-beta.0.20200910042021-254d1345be09/go.mod h1:Z+EQXV6FyfpH7olLqXH0zvYOnFcCNGJmzm+MN4W1/RE= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6 h1:lYIiVDtZnyTWlNwiAxLj0bbpTcx1BWCFhXjfsvmPdNc= github.com/tmc/grpc-websocket-proxy v0.0.0-20171017195756-830351dc03c6/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -703,6 +719,9 @@ github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX github.com/urfave/cli/v2 v2.1.1 h1:Qt8FeAtxE/vfdrLmR3rxR6JRE0RoVmbXu8+6kZtYU4k= github.com/urfave/cli/v2 v2.1.1/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= github.com/urfave/negroni v0.3.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= +github.com/vmihailenco/msgpack/v4 v4.3.11/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= +github.com/vmihailenco/msgpack/v5 v5.0.0-beta.1/go.mod h1:xlngVLeyQ/Qi05oQxhQ+oTuqa03RjMwMfk/7/TCs+QI= +github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= @@ -711,6 +730,7 @@ github.com/yookoala/realpath v1.0.0/go.mod h1:gJJMA9wuX7AcqLy1+ffPatSCySA1FQ2S8Y github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/zhangjinpeng1987/raft v0.0.0-20190624145930-deeb32d6553d/go.mod h1:1KDQ09J8MRHEtHze4at7BJZDW/doUAgkJ8w9KjEUhSo= github.com/zhangjinpeng1987/raft v0.0.0-20200819064223-df31bb68a018/go.mod h1:rTSjwgeYU2on64W50csWDlhyy0x9UYVYJUovHlYdt5s= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -830,12 +850,15 @@ golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200904194848-62affa334b73 h1:MXfv8rhZWmFeqX3GNZRsd6vOLoaCHjYEX3qkRo3YBUA= +golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190517181255-950ef44c6e07 h1:XC1K3wNjuz44KaI+cj85C9TW85w/46RH7J+DTXNH5Wk= @@ -889,6 +912,7 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200819171115-d785dc25833f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8 h1:AvbQYmiaaaza3cW3QXRyPo5kYgpFIzOAfeAAN7m3qQ4= golang.org/x/sys v0.0.0-20200824131525-c12d262b63d8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -943,6 +967,7 @@ golang.org/x/tools v0.0.0-20200527183253-8e7acdbce89d/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20200820010801-b793a1359eac/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200823205832-c024452afbcd h1:KNSumuk5eGuQV7zbOrDDZ3MIkwsQr0n5oKiH4oE0/hU= golang.org/x/tools v0.0.0-20200823205832-c024452afbcd/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= diff --git a/v4/export/metadata.go b/v4/export/metadata.go index 9d41e499..ffe2680c 100644 --- a/v4/export/metadata.go +++ b/v4/export/metadata.go @@ -195,7 +195,7 @@ func (m *globalMetadata) writeGlobalMetaData(ctx context.Context) error { if err != nil { return err } - defer tearDown(ctx) + defer tearDown() - return write(ctx, fileWriter, m.String()) + return write(fileWriter, m.String()) } diff --git a/v4/export/test_util.go b/v4/export/test_util.go index 58813c77..61aee5e8 100644 --- a/v4/export/test_util.go +++ b/v4/export/test_util.go @@ -13,7 +13,7 @@ type mockPoisonWriter struct { buf string } -func (m *mockPoisonWriter) Write(ctx context.Context, p []byte) (int, error) { +func (m *mockPoisonWriter) Write(p []byte) (int, error) { s := string(p) if s == "poison" { return 0, fmt.Errorf("poison_error") @@ -22,7 +22,7 @@ func (m *mockPoisonWriter) Write(ctx context.Context, p []byte) (int, error) { return len(s), nil } -func (m *mockPoisonWriter) Close(ctx context.Context) error { +func (m *mockPoisonWriter) Close() error { // noop return nil } diff --git a/v4/export/writer.go b/v4/export/writer.go index 7209cacc..138be12b 100644 --- a/v4/export/writer.go +++ b/v4/export/writer.go @@ -82,9 +82,9 @@ func (f SQLWriter) WriteTableData(ctx context.Context, ir TableDataIR) error { defer chunksIter.Rows().Close() for { - fileWriter, tearDown := buildInterceptFileWriter(f.cfg.ExternalStorage, fileName) + fileWriter, tearDown := buildInterceptFileWriter(ctx, f.cfg.ExternalStorage, fileName) err = WriteInsert(ctx, chunksIter, fileWriter, f.cfg.FileSize, f.cfg.StatementSize) - tearDown(ctx) + tearDown() if err != nil { return err } @@ -112,9 +112,9 @@ func writeMetaToFile(ctx context.Context, target, metaSQL string, s storage.Exte if err != nil { return err } - defer tearDown(ctx) + defer tearDown() - return WriteMeta(ctx, &metaData{ + return WriteMeta(&metaData{ target: target, metaSQL: metaSQL, specCmts: []string{ @@ -178,9 +178,9 @@ func (f CSVWriter) WriteTableData(ctx context.Context, ir TableDataIR) error { } for { - fileWriter, tearDown := buildInterceptFileWriter(f.cfg.ExternalStorage, fileName) + fileWriter, tearDown := buildInterceptFileWriter(ctx, f.cfg.ExternalStorage, fileName) err = WriteInsertInCsv(ctx, chunksIter, fileWriter, f.cfg.NoHeader, opt, f.cfg.FileSize) - tearDown(ctx) + tearDown() if err != nil { return err } diff --git a/v4/export/writer_util.go b/v4/export/writer_util.go index acfa3e10..f661fb46 100644 --- a/v4/export/writer_util.go +++ b/v4/export/writer_util.go @@ -35,10 +35,10 @@ type writerPipe struct { fileSizeLimit uint64 statementSizeLimit uint64 - w storage.Writer + w io.Writer } -func newWriterPipe(w storage.Writer, fileSizeLimit, statementSizeLimit uint64) *writerPipe { +func newWriterPipe(w io.Writer, fileSizeLimit, statementSizeLimit uint64) *writerPipe { return &writerPipe{ input: make(chan *bytes.Buffer, 8), closed: make(chan struct{}), @@ -64,7 +64,7 @@ func (b *writerPipe) Run(ctx context.Context) { if errOccurs { continue } - err := writeBytes(ctx, b.w, s.Bytes()) + err := writeBytes(b.w, s.Bytes()) s.Reset() pool.Put(s) if err != nil { @@ -100,17 +100,17 @@ func (b *writerPipe) ShouldSwitchStatement() bool { (b.statementSizeLimit != UnspecifiedSize && b.currentStatementSize >= b.statementSizeLimit) } -func WriteMeta(ctx context.Context, meta MetaIR, w storage.Writer) error { +func WriteMeta(meta MetaIR, w io.Writer) error { log.Debug("start dumping meta data", zap.String("target", meta.TargetName())) specCmtIter := meta.SpecialComments() for specCmtIter.HasNext() { - if err := write(ctx, w, fmt.Sprintf("%s\n", specCmtIter.Next())); err != nil { + if err := write(w, fmt.Sprintf("%s\n", specCmtIter.Next())); err != nil { return err } } - if err := write(ctx, w, meta.MetaSQL()); err != nil { + if err := write(w, meta.MetaSQL()); err != nil { return err } @@ -118,7 +118,7 @@ func WriteMeta(ctx context.Context, meta MetaIR, w storage.Writer) error { return nil } -func WriteInsert(pCtx context.Context, tblIR TableDataIR, w storage.Writer, fileSizeLimit, statementSizeLimit uint64) error { +func WriteInsert(pCtx context.Context, tblIR TableDataIR, w io.Writer, fileSizeLimit, statementSizeLimit uint64) error { fileRowIter := tblIR.Rows() if !fileRowIter.HasNext() { return nil @@ -230,7 +230,7 @@ func WriteInsert(pCtx context.Context, tblIR TableDataIR, w storage.Writer, file return wp.Error() } -func WriteInsertInCsv(pCtx context.Context, tblIR TableDataIR, w storage.Writer, noHeader bool, opt *csvOption, fileSizeLimit uint64) error { +func WriteInsertInCsv(pCtx context.Context, tblIR TableDataIR, w io.Writer, noHeader bool, opt *csvOption, fileSizeLimit uint64) error { fileRowIter := tblIR.Rows() if !fileRowIter.HasNext() { return nil @@ -322,8 +322,8 @@ func WriteInsertInCsv(pCtx context.Context, tblIR TableDataIR, w storage.Writer, return wp.Error() } -func write(ctx context.Context, writer storage.Writer, str string) error { - _, err := writer.Write(ctx, []byte(str)) +func write(writer io.Writer, str string) error { + _, err := writer.Write([]byte(str)) if err != nil { // str might be very long, only output the first 200 chars outputLength := len(str) @@ -337,8 +337,8 @@ func write(ctx context.Context, writer storage.Writer, str string) error { return err } -func writeBytes(ctx context.Context, writer storage.Writer, p []byte) error { - _, err := writer.Write(ctx, p) +func writeBytes(writer io.Writer, p []byte) error { + _, err := writer.Write(p) if err != nil { // str might be very long, only output the first 200 chars outputLength := len(p) @@ -353,7 +353,7 @@ func writeBytes(ctx context.Context, writer storage.Writer, p []byte) error { return err } -func buildFileWriter(ctx context.Context, s storage.ExternalStorage, path string) (storage.Writer, func(ctx context.Context), error) { +func buildFileWriter(ctx context.Context, s storage.ExternalStorage, path string) (io.Writer, func(), error) { fullPath := s.URI() + path uploader, err := s.CreateUploader(ctx, path) if err != nil { @@ -362,10 +362,10 @@ func buildFileWriter(ctx context.Context, s storage.ExternalStorage, path string zap.Error(err)) return nil, nil, err } - writer := storage.NewUploaderWriter(uploader, hardcodedS3ChunkSize) + writer := storage.NewUploaderWriter(ctx, uploader, hardcodedS3ChunkSize) log.Debug("opened file", zap.String("path", fullPath)) - tearDownRoutine := func(ctx context.Context) { - err := writer.Close(ctx) + tearDownRoutine := func() { + err := writer.Close() if err == nil { return } @@ -376,10 +376,10 @@ func buildFileWriter(ctx context.Context, s storage.ExternalStorage, path string return writer, tearDownRoutine, nil } -func buildInterceptFileWriter(s storage.ExternalStorage, path string) (storage.Writer, func(context.Context)) { - var writer storage.Writer +func buildInterceptFileWriter(ctx context.Context, s storage.ExternalStorage, path string) (io.WriteCloser, func()) { + var writer io.WriteCloser fullPath := s.URI() + path - fileWriter := &InterceptFileWriter{} + fileWriter := &InterceptFileWriter{ctx: ctx} initRoutine := func(ctx context.Context) error { uploader, err := s.CreateUploader(ctx, path) if err != nil { @@ -388,20 +388,20 @@ func buildInterceptFileWriter(s storage.ExternalStorage, path string) (storage.W zap.Error(err)) return err } - w := storage.NewUploaderWriter(uploader, hardcodedS3ChunkSize) + w := storage.NewUploaderWriter(ctx, uploader, hardcodedS3ChunkSize) writer = w log.Debug("opened file", zap.String("path", fullPath)) - fileWriter.Writer = writer + fileWriter.WriteCloser = writer return err } fileWriter.initRoutine = initRoutine - tearDownRoutine := func(ctx context.Context) { + tearDownRoutine := func() { if writer == nil { return } log.Debug("tear down lazy file writer...") - err := writer.Close(ctx) + err := writer.Close() if err != nil { log.Error("close file failed", zap.String("path", fullPath)) } @@ -427,27 +427,28 @@ func (l *LazyStringWriter) WriteString(str string) (int, error) { // InterceptFileWriter is an interceptor of os.File, // tracking whether a StringWriter has written something. type InterceptFileWriter struct { - storage.Writer + io.WriteCloser sync.Once + ctx context.Context initRoutine func(context.Context) error err error SomethingIsWritten bool } -func (w *InterceptFileWriter) Write(ctx context.Context, p []byte) (int, error) { - w.Do(func() { w.err = w.initRoutine(ctx) }) +func (w *InterceptFileWriter) Write(p []byte) (int, error) { + w.Do(func() { w.err = w.initRoutine(w.ctx) }) if len(p) > 0 { w.SomethingIsWritten = true } if w.err != nil { return 0, fmt.Errorf("open file error: %s", w.err.Error()) } - return w.Writer.Write(ctx, p) + return w.WriteCloser.Write(p) } -func (w *InterceptFileWriter) Close(ctx context.Context) error { - return w.Writer.Close(ctx) +func (w *InterceptFileWriter) Close() error { + return w.WriteCloser.Close() } func wrapBackTicks(identifier string) string { diff --git a/v4/export/writer_util_test.go b/v4/export/writer_util_test.go index 701b2744..e51ff74b 100644 --- a/v4/export/writer_util_test.go +++ b/v4/export/writer_util_test.go @@ -1,6 +1,7 @@ package export import ( + "bytes" "context" "errors" "fmt" @@ -9,7 +10,6 @@ import ( "database/sql/driver" - "github.com/pingcap/br/pkg/storage" . "github.com/pingcap/check" ) @@ -35,15 +35,15 @@ func (s *testUtilSuite) TestWriteMeta(c *C) { ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;\n" specCmts := []string{"/*!40103 SET TIME_ZONE='+00:00' */;"} meta := newMockMetaIR("t1", createTableStmt, specCmts) - writer := storage.NewBufferWriter() + strCollector := &bytes.Buffer{} - err := WriteMeta(context.Background(), meta, writer) + err := WriteMeta(meta, strCollector) c.Assert(err, IsNil) expected := "/*!40103 SET TIME_ZONE='+00:00' */;\n" + "CREATE TABLE `t1` (\n" + " `a` int(11) DEFAULT NULL\n" + ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;\n" - c.Assert(writer.String(), Equals, expected) + c.Assert(strCollector.String(), Equals, expected) } func (s *testUtilSuite) TestWriteInsert(c *C) { @@ -59,7 +59,7 @@ func (s *testUtilSuite) TestWriteInsert(c *C) { "/*!40014 SET FOREIGN_KEY_CHECKS=0*/;", } tableIR := newMockTableIR("test", "employee", data, specCmts, colTypes) - bf := storage.NewBufferWriter() + bf := &bytes.Buffer{} err := WriteInsert(context.Background(), tableIR, bf, UnspecifiedSize, UnspecifiedSize) c.Assert(err, IsNil) @@ -89,7 +89,7 @@ func (s *testUtilSuite) TestWriteInsertReturnsError(c *C) { rowErr := errors.New("mock row error") tableIR := newMockTableIR("test", "employee", data, specCmts, colTypes) tableIR.rowErr = rowErr - bf := storage.NewBufferWriter() + bf := &bytes.Buffer{} err := WriteInsert(context.Background(), tableIR, bf, UnspecifiedSize, UnspecifiedSize) c.Assert(err, Equals, rowErr) @@ -111,7 +111,7 @@ func (s *testUtilSuite) TestWriteInsertInCsv(c *C) { } colTypes := []string{"INT", "SET", "VARCHAR", "VARCHAR", "TEXT"} tableIR := newMockTableIR("test", "employee", data, nil, colTypes) - bf := storage.NewBufferWriter() + bf := &bytes.Buffer{} // test nullValue opt := &csvOption{separator: []byte(","), delimiter: doubleQuotationMark, nullValue: "\\N"} @@ -176,7 +176,7 @@ func (s *testUtilSuite) TestSQLDataTypes(c *C) { tableData := [][]driver.Value{{origin}} colType := []string{sqlType} tableIR := newMockTableIR("test", "t", tableData, nil, colType) - bf := storage.NewBufferWriter() + bf := &bytes.Buffer{} err := WriteInsert(context.Background(), tableIR, bf, UnspecifiedSize, UnspecifiedSize) c.Assert(err, IsNil) @@ -192,7 +192,7 @@ func (s *testUtilSuite) TestWrite(c *C) { exp := []string{"test", "loooooooooooooooooooong", "poison_error"} for i, s := range src { - err := write(context.Background(), mocksw, s) + err := write(mocksw, s) if err != nil { c.Assert(err.Error(), Equals, exp[i]) } else { @@ -200,6 +200,6 @@ func (s *testUtilSuite) TestWrite(c *C) { c.Assert(mocksw.buf, Equals, exp[i]) } } - err := write(context.Background(), mocksw, "test") + err := write(mocksw, "test") c.Assert(err, IsNil) }