Skip to content

Commit 37ff66f

Browse files
jacjac
authored
Add Snapshot Restore command (#947)
Snapshot Restore generates commands to restore dumped snapshots to a Sourcegraph instance
1 parent 7875b7d commit 37ff66f

File tree

3 files changed

+184
-37
lines changed

3 files changed

+184
-37
lines changed

‎cmd/src/snapshot_databases.go‎

Lines changed: 25 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"flag"
55
"fmt"
66
"os"
7+
"os/exec"
78
"strings"
89

910
"github.com/sourcegraph/sourcegraph/lib/errors"
@@ -18,7 +19,7 @@ func init() {
1819
Note that these commands are intended for use as reference - you may need to adjust the commands for your deployment.
1920
2021
USAGE
21-
src [-v] snapshot databases <pg_dump|docker|kubectl> [--targets=<docker|k8s|"targets.yaml">]
22+
src [-v] snapshot databases [--targets=<docker|k8s|"targets.yaml">] [--run] <pg_dump|docker|kubectl>
2223
2324
TARGETS FILES
2425
Predefined targets are available based on default Sourcegraph configurations ('docker', 'k8s').
@@ -38,6 +39,7 @@ TARGETS FILES
3839
`
3940
flagSet := flag.NewFlagSet("databases", flag.ExitOnError)
4041
targetsKeyFlag := flagSet.String("targets", "auto", "predefined targets ('docker' or 'k8s'), or a custom targets.yaml file")
42+
run := flagSet.Bool("run", false, "Automatically run the commands")
4143

4244
snapshotCommands = append(snapshotCommands, &command{
4345
flagSet: flagSet,
@@ -47,35 +49,13 @@ TARGETS FILES
4749
}
4850
out := output.NewOutput(flagSet.Output(), output.OutputOpts{Verbose: *verbose})
4951

50-
var builder string
51-
if len(args) > 0 {
52-
builder = args[0]
53-
}
52+
builder := flagSet.Arg(0)
5453

55-
targetKey := "docker"
56-
var commandBuilder pgdump.CommandBuilder
57-
switch builder {
58-
case "pg_dump", "":
59-
targetKey = "local"
60-
commandBuilder = func(t pgdump.Target) (string, error) {
61-
cmd := pgdump.Command(t)
62-
if t.Target != "" {
63-
return fmt.Sprintf("%s --host=%s", cmd, t.Target), nil
64-
}
65-
return cmd, nil
66-
}
67-
case "docker":
68-
commandBuilder = func(t pgdump.Target) (string, error) {
69-
return fmt.Sprintf("docker exec -it %s sh -c '%s'", t.Target, pgdump.Command(t)), nil
70-
}
71-
case "kubectl":
72-
targetKey = "k8s"
73-
commandBuilder = func(t pgdump.Target) (string, error) {
74-
return fmt.Sprintf("kubectl exec -it %s -- bash -c '%s'", t.Target, pgdump.Command(t)), nil
75-
}
76-
default:
54+
commandBuilder, targetKey := pgdump.Builder(builder, pgdump.DumpCommand)
55+
if targetKey == "" {
7756
return errors.Newf("unknown or invalid template type %q", builder)
7857
}
58+
7959
if *targetsKeyFlag != "auto" {
8060
targetKey = *targetsKeyFlag
8161
}
@@ -94,19 +74,32 @@ TARGETS FILES
9474
out.WriteLine(output.Emojif(output.EmojiInfo, "Using predefined targets for %s environments", targetKey))
9575
}
9676

97-
commands, err := pgdump.BuildCommands(srcSnapshotDir, commandBuilder, targets)
77+
commands, err := pgdump.BuildCommands(srcSnapshotDir, commandBuilder, targets, true)
9878
if err != nil {
9979
return errors.Wrap(err, "failed to build commands")
10080
}
10181

10282
_ = os.MkdirAll(srcSnapshotDir, os.ModePerm)
10383

104-
b := out.Block(output.Emoji(output.EmojiSuccess, "Run these commands to generate the required database dumps:"))
105-
b.Write("\n" + strings.Join(commands, "\n"))
106-
b.Close()
84+
if *run {
85+
for _, c := range commands {
86+
out.WriteLine(output.Emojif(output.EmojiInfo, "Running command: %q", c))
87+
command := exec.Command("bash", "-c", c)
88+
output, err := command.CombinedOutput()
89+
out.Write(string(output))
90+
if err != nil {
91+
return errors.Wrapf(err, "failed to run command: %q", c)
92+
}
93+
}
10794

108-
out.WriteLine(output.Styledf(output.StyleSuggestion, "Note that you may need to do some additional setup, such as authentication, beforehand."))
95+
out.WriteLine(output.Emoji(output.EmojiSuccess, "Successfully completed dump commands"))
96+
} else {
97+
b := out.Block(output.Emoji(output.EmojiSuccess, "Run these commands to generate the required database dumps:"))
98+
b.Write("\n" + strings.Join(commands, "\n"))
99+
b.Close()
109100

101+
out.WriteLine(output.Styledf(output.StyleSuggestion, "Note that you may need to do some additional setup, such as authentication, beforehand."))
102+
}
110103
return nil
111104
},
112105
usageFunc: func() { fmt.Fprint(flag.CommandLine.Output(), usage) },

‎cmd/src/snapshot_restore.go‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
package main
2+
3+
import (
4+
"flag"
5+
"fmt"
6+
"os"
7+
"os/exec"
8+
"strings"
9+
10+
"github.com/sourcegraph/sourcegraph/lib/errors"
11+
"github.com/sourcegraph/sourcegraph/lib/output"
12+
"gopkg.in/yaml.v3"
13+
14+
"github.com/sourcegraph/src-cli/internal/pgdump"
15+
)
16+
17+
func init() {
18+
usage := `'src snapshot restore' restores a Sourcegraph instance using Sourcegraph database dumps.
19+
Note that these commands are intended for use as reference - you may need to adjust the commands for your deployment.
20+
21+
USAGE
22+
src [-v] snapshot restore [--targets<docker|k8s|"targets.yaml">] [--run] <pg_dump|docker|kubectl>
23+
24+
TARGETS FILES
25+
Predefined targets are available based on default Sourcegraph configurations ('docker', 'k8s').
26+
Custom targets configuration can be provided in YAML format with '--targets=target.yaml', e.g.
27+
28+
primary:
29+
target: ... # the DSN of the database deployment, e.g. in docker, the name of the database container
30+
dbname: ... # name of database
31+
username: ... # username for database access
32+
password: ... # password for database access - only include password if it is non-sensitive
33+
codeintel:
34+
# same as above
35+
codeinsights:
36+
# same as above
37+
38+
See the pgdump.Targets type for more details.
39+
`
40+
41+
flagSet := flag.NewFlagSet("restore", flag.ExitOnError)
42+
targetsKeyFlag := flagSet.String("targets", "auto", "predefined targets ('docker' or 'k8s'), or a custom targets.yaml file")
43+
run := flagSet.Bool("run", false, "Automatically run the commands")
44+
45+
snapshotCommands = append(snapshotCommands, &command{
46+
flagSet: flagSet,
47+
handler: func(args []string) error {
48+
if err := flagSet.Parse(args); err != nil {
49+
return err
50+
}
51+
out := output.NewOutput(flagSet.Output(), output.OutputOpts{Verbose: *verbose})
52+
53+
builder := flagSet.Arg(0)
54+
55+
commandBuilder, targetKey := pgdump.Builder(builder, pgdump.RestoreCommand)
56+
if targetKey == "" {
57+
return errors.Newf("unknown or invalid template type %q", builder)
58+
}
59+
60+
if *targetsKeyFlag != "auto" {
61+
targetKey = *targetsKeyFlag
62+
}
63+
64+
targets, ok := predefinedDatabaseDumpTargets[targetKey]
65+
if !ok {
66+
out.WriteLine(output.Emojif(output.EmojiInfo, "Using targets defined in targets file %q", targetKey))
67+
f, err := os.Open(targetKey)
68+
if err != nil {
69+
return errors.Wrapf(err, "invalid targets file %q", targetKey)
70+
}
71+
if err := yaml.NewDecoder(f).Decode(&targets); err != nil {
72+
return errors.Wrapf(err, "invalid targets file %q", targetKey)
73+
}
74+
} else {
75+
out.WriteLine(output.Emojif(output.EmojiInfo, "Using predefined targets for %s environments", targetKey))
76+
}
77+
78+
commands, err := pgdump.BuildCommands(srcSnapshotDir, commandBuilder, targets, false)
79+
if err != nil {
80+
return errors.Wrap(err, "failed to build commands")
81+
}
82+
if *run {
83+
for _, c := range commands {
84+
out.WriteLine(output.Emojif(output.EmojiInfo, "Running command: %q", c))
85+
command := exec.Command("bash", "-c", c)
86+
output, err := command.CombinedOutput()
87+
out.Write(string(output))
88+
if err != nil {
89+
return errors.Wrapf(err, "failed to run command: %q", c)
90+
}
91+
}
92+
93+
out.WriteLine(output.Emoji(output.EmojiSuccess, "Successfully completed restore commands"))
94+
out.WriteLine(output.Styledf(output.StyleSuggestion, "It may be necessary to restart your Sourcegraph instance after restoring"))
95+
} else {
96+
b := out.Block(output.Emoji(output.EmojiSuccess, "Run these commands to restore the databases:"))
97+
b.Write("\n" + strings.Join(commands, "\n"))
98+
b.Close()
99+
100+
out.WriteLine(output.Styledf(output.StyleSuggestion, "Note that you may need to do some additional setup, such as authentication, beforehand."))
101+
}
102+
103+
return nil
104+
},
105+
106+
usageFunc: func() { fmt.Fprint(flag.CommandLine.Output(), usage) },
107+
})
108+
}

‎internal/pgdump/pgdump.go‎

Lines changed: 51 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,19 @@ type Target struct {
3030
Password string `yaml:"password"`
3131
}
3232

33-
// Command generates a pg_dump command that can be used for on-prem-to-Cloud migrations.
34-
func Command(t Target) string {
35-
dump := fmt.Sprintf("pg_dump --no-owner --format=p --no-acl --username=%s --dbname=%s",
33+
// RestoreCommand generates a psql command that can be used for migrations.
34+
func RestoreCommand(t Target) string {
35+
dump := fmt.Sprintf("psql --username=%s --dbname=%s 1>/dev/null",
36+
t.Username, t.DBName)
37+
if t.Password == "" {
38+
return dump
39+
}
40+
return fmt.Sprintf("PGPASSWORD=%s %s", t.Password, dump)
41+
}
42+
43+
// DumpCommand generates a pg_dump command that can be used for on-prem-to-Cloud migrations.
44+
func DumpCommand(t Target) string {
45+
dump := fmt.Sprintf("pg_dump --no-owner --format=p --no-acl --clean --if-exists --username=%s --dbname=%s",
3646
t.Username, t.DBName)
3747
if t.Password == "" {
3848
return dump
@@ -61,17 +71,53 @@ func Outputs(dir string, targets Targets) []Output {
6171
}
6272

6373
type CommandBuilder func(Target) (string, error)
74+
type PGCommand func(Target) string
75+
76+
// Builder generates the CommandBuilder and targetKey for a given builder and PGCommand
77+
func Builder(builder string, command PGCommand) (commandBuilder CommandBuilder, targetKey string) {
78+
switch builder {
79+
case "pg_dump", "":
80+
targetKey = "local"
81+
commandBuilder = func(t Target) (string, error) {
82+
cmd := command(t)
83+
if t.Target != "" {
84+
return fmt.Sprintf("%s --host=%s", cmd, t.Target), nil
85+
}
86+
return cmd, nil
87+
}
88+
case "docker":
89+
targetKey = "docker"
90+
commandBuilder = func(t Target) (string, error) {
91+
return fmt.Sprintf("docker exec -i %s sh -c '%s'", t.Target, command(t)), nil
92+
}
93+
case "kubectl":
94+
targetKey = "k8s"
95+
commandBuilder = func(t Target) (string, error) {
96+
return fmt.Sprintf("kubectl exec -i %s -- bash -c '%s'", t.Target, command(t)), nil
97+
}
98+
default:
99+
return commandBuilder, targetKey
100+
}
101+
return commandBuilder, targetKey
102+
}
64103

65104
// BuildCommands generates commands that output Postgres dumps and sends them to predefined
66105
// files for each target database.
67-
func BuildCommands(outDir string, commandBuilder CommandBuilder, targets Targets) ([]string, error) {
106+
func BuildCommands(outDir string, commandBuilder CommandBuilder, targets Targets, dump bool) ([]string, error) {
68107
var commands []string
69108
for _, t := range Outputs(outDir, targets) {
70109
c, err := commandBuilder(t.Target)
71110
if err != nil {
72111
return nil, errors.Wrapf(err, "generating command for %q", t.Output)
73112
}
74-
commands = append(commands, fmt.Sprintf("%s > %s", c, t.Output))
113+
114+
if dump {
115+
// When dumping use output redirection to dump command stdout to target file
116+
commands = append(commands, fmt.Sprintf("%s > %s", c, t.Output))
117+
} else {
118+
// When restoring use input redirection to pass target file to command stdin
119+
commands = append(commands, fmt.Sprintf("%s < %s", c, t.Output))
120+
}
75121
}
76122
return commands, nil
77123
}

0 commit comments

Comments
 (0)