diff --git a/.gitignore b/.gitignore index 3b735ec..0721bfc 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,10 @@ *.dll *.so *.dylib +*.back +output.html +nohup +main # Test binary, built with `go test -c` *.test diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 0000000..d3d56bb --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,50 @@ +/* +Copyright © 2024 NAME HERE +*/ +package cmd + +import ( + "os" + "webprobe/internal" + + "github.com/spf13/cobra" +) + +// rootCmd represents the base command when called without any subcommands +var rootCmd = &cobra.Command{ + Use: "webprobe", + Short: "支持IPv4、IPv6的网站探测工具， 支持探测多级子链并计算支持度", + Long: `支持IPv4、IPv6的网站探测工具，能够探测多级子链并计算支持度，同时提供网络性能优化、报告和可视化、灵活配置选项和安全性考虑。 + + `, + // Uncomment the following line if your bare application + // has an action associated with it: + // Run: func(cmd *cobra.Command, args []string) {}, + Run: func(cmd *cobra.Command, args []string) { + // Do Stuff Here + internal.ScrapyWeb() + }, +} + +// Execute adds all child commands to the root command and sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the rootCmd. +func Execute() { + err := rootCmd.Execute() + if err != nil { + os.Exit(1) + } + +} + +func init() { + // Here you will define your flags and configuration settings. + // Cobra supports persistent flags, which, if defined here, + // will be global for your application. + + // rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.webprobe.yaml)") + + // Cobra also supports local flags, which will only run + // when this action is called directly. + rootCmd.PersistentFlags().StringVarP(&internal.ConfigPath, "config", "c", ".", "config file directory") + //rootCmd.MarkPersistentFlagRequired("config") // 标记为必需 +} diff --git a/cmd/run.go b/cmd/run.go new file mode 100644 index 0000000..71330c7 --- /dev/null +++ b/cmd/run.go @@ -0,0 +1,39 @@ +/* +Copyright © 2024 NAME HERE +*/ +package cmd + +import ( + "webprobe/utility" + + "github.com/spf13/cobra" +) + +// runCmd represents the run command +var runCmd = &cobra.Command{ + Use: "run", + Short: "A brief description of your command", + Long: `A longer description that spans multiple lines and likely contains examples +and usage of using your command. For example: + +Cobra is a CLI library for Go that empowers applications. +This application is a tool to generate the needed files +to quickly create a Cobra application.`, + Run: func(cmd *cobra.Command, args []string) { + utility.Cmdline() + }, +} + +func init() { + rootCmd.AddCommand(runCmd) + + // Here you will define your flags and configuration settings. + + // Cobra supports Persistent Flags which will work for this command + // and all subcommands, e.g.: + // runCmd.PersistentFlags().String("foo", "", "A help for foo") + + // Cobra supports local flags which will only run when this command + // is called directly, e.g.: + // runCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") +} diff --git a/cmd/serve.go b/cmd/serve.go new file mode 100644 index 0000000..b99738c --- /dev/null +++ b/cmd/serve.go @@ -0,0 +1,40 @@ +/* +Copyright © 2024 NAME HERE + +*/ +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +// serveCmd represents the serve command +var serveCmd = &cobra.Command{ + Use: "serve", + Short: "A brief description of your command", + Long: `A longer description that spans multiple lines and likely contains examples +and usage of using your command. For example: + +Cobra is a CLI library for Go that empowers applications. +This application is a tool to generate the needed files +to quickly create a Cobra application.`, + Run: func(cmd *cobra.Command, args []string) { + fmt.Println("serve called") + }, +} + +func init() { + rootCmd.AddCommand(serveCmd) + + // Here you will define your flags and configuration settings. + + // Cobra supports Persistent Flags which will work for this command + // and all subcommands, e.g.: + // serveCmd.PersistentFlags().String("foo", "", "A help for foo") + + // Cobra supports local flags which will only run when this command + // is called directly, e.g.: + // serveCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") +} diff --git a/cmd/toDB.go b/cmd/toDB.go new file mode 100644 index 0000000..b23aed4 --- /dev/null +++ b/cmd/toDB.go @@ -0,0 +1,67 @@ +/* +Copyright © 2024 NAME HERE +*/ +package cmd + +import ( + "log" + "webprobe/handler" + "webprobe/internal" + "webprobe/utility" + + "github.com/spf13/cobra" +) + +// toDBCmd represents the toDB command +var toDBCmd = &cobra.Command{ + Use: "toDB", + Short: "Specify the output database type (relational or time-series). Default is time-series.", + Long: `Specify the type of database to save the scan results. The available options are: + + -- RDB: Save the scan results to a relational database. + -- TSDB: Save the scan results to a time-series database (default).`, + Run: func(cmd *cobra.Command, args []string) { + internal.ScrapyWeb() + + RDBItems, _ := cmd.Flags().GetStringArray("RDB") + TSDBItems, _ := cmd.Flags().GetStringArray("TSDB") + //for _, item := range RDBItems { + if len(RDBItems) > 0 { + + if internal.Cfg.DBConfig.UseDB { + log.Printf("正在写入数据库...") + handler.SaveData(internal.DB, internal.UrlDataWithReachability, internal.Cfg.ScannerConfig.Crawl.Depth) + log.Printf("写入关系数据库" + RDBItems[0] + "完成.") + } + } + + //for _, item := range TSDBItems { + if len(TSDBItems) > 0 { + + utility.PushToPrometheus(internal.Cfg.Push.PushGatewayURL, internal.UrlDataWithReachability) + log.Printf("写入时序数据库" + TSDBItems[0] + "完成.") + } + + }, +} + +func init() { + + rootCmd.AddCommand(toDBCmd) + // 在这里定义您的标志和配置设置。 + + // Cobra支持持久性标志，这些标志将适用于此命令 + // 和所有子命令，例如： + // toDBCmd.PersistentFlags().String("foo", "", "A help for foo") + // 创建toDBCmd命令的Flag，作用为指定关系性数据库的名称 (默认为tsdb) + toDBCmd.PersistentFlags().StringArrayP("RDB", "r", []string{""}, "result to RDB") + // 创建toDBCmd命令的Flag，作用为指定时序数据库的名称 (默认为tsdb) + toDBCmd.PersistentFlags().StringArrayP("TSDB", "t", []string{""}, "result to toTSDB") + + // Cobra支持仅在直接调用此命令时运行的本地标志，例如： + // toDBCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") + +} + +// +// diff --git a/config.yaml b/config.yaml index bce06f0..74f9e36 100644 --- a/config.yaml +++ b/config.yaml @@ -1,7 +1,7 @@ -url: ["https://www.ecnu.edu.cn", "https://www.baidu.com"] +url: ["https://eoffice.ecnu.edu.cn"] scanner: crawl: - depth: 2 # 爬虫时的链接深度，0 代表只检测url自身，以此类推 + depth: 1 # 爬虫时的链接深度，0 代表只检测url自身，以此类推 timeout: 5 # 爬虫时访问每个url的超时时间 check: ipv6: false # 默认仅使用ipv4检测，设置为true后进行ipv4和ipv6双检测 @@ -13,9 +13,16 @@ scanner: http_client_timeout: 15 # 请求每个URL的超时时间 context_timeout: 1000 # 完成所有检测的超时时间，应足够大，否则会在检测某个url上一直挂起 push: - push_to_prometheus: false - push_gateway_url: "http://10.24.70.20:9091" + push_to_prometheus: true + push_gateway_url: "http://127.0.0.1:9091" output: "output.html" db: - use_db: false - db_path: "LinkPulse.db" + use_db: true + dialect: "mysql" + username: "root" + password: "xxxxxx" + host: "127.0.0.1" + port: "3306" + db_name: "webprobe" + ssl_mode: "" + diff --git a/config/config.go b/config/config.go index 5fff321..98d9c0e 100644 --- a/config/config.go +++ b/config/config.go @@ -1,8 +1,10 @@ package config import ( - "github.com/spf13/viper" + "log" "time" + + "github.com/spf13/viper" ) type Config struct { @@ -40,15 +42,23 @@ type Push struct { PushGatewayURL string `mapstructure:"push_gateway_url"` } type DBConfig struct { - UseDB bool `mapstructure:"use_db"` // default value: true - DBPath string `mapstructure:"db_path"` // default value: "url_status.db" + UseDB bool `mapstructure:"use_db"` // 是否使用数据库，默认值为 true + Dialect string `mapstructure:"dialect"` // 数据库方言（例如：mysql、postgres） + Username string `mapstructure:"username"` // 数据库用户名 + Password string `mapstructure:"password"` // 数据库密码 + Host string `mapstructure:"host"` // 数据库主机地址 + Port int `mapstructure:"port"` // 数据库连接端口号 + DBName string `mapstructure:"db_name"` // 数据库名称 + SSLMode string `mapstructure:"ssl_mode"` // SSL 模式（仅适用于某些数据库，如 PostgreSQL） } -func LoadConfig() (*Config, error) { +var ConfigPath string + +func LoadConfig(configPath string) (*Config, error) { + log.Println(ConfigPath) viper.SetConfigName("config") viper.SetConfigType("yaml") - viper.AddConfigPath(".") - + viper.AddConfigPath(configPath) viper.AutomaticEnv() // read from environment variables // 设置默认值 diff --git a/go.mod b/go.mod index d35a08d..e0159d3 100644 --- a/go.mod +++ b/go.mod @@ -6,16 +6,27 @@ require ( github.com/mattn/go-sqlite3 v1.14.19 github.com/prometheus/client_golang v1.17.0 github.com/spf13/viper v1.18.1 - golang.org/x/net v0.19.0 - golang.org/x/sync v0.5.0 + golang.org/x/net v0.21.0 + golang.org/x/sync v0.7.0 ) require ( + filippo.io/edwards25519 v1.1.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-sql-driver/mysql v1.8.1 // indirect github.com/golang/protobuf v1.5.3 // indirect + github.com/gookit/color v1.5.4 // indirect + github.com/gookit/goutil v0.6.15 // indirect github.com/hashicorp/hcl v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect + github.com/jackc/pgx/v5 v5.5.5 // indirect + github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect @@ -28,14 +39,20 @@ require ( github.com/sourcegraph/conc v0.3.0 // indirect github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect + github.com/spf13/cobra v1.8.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.6.0 // indirect + github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect + golang.org/x/crypto v0.22.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect - golang.org/x/sys v0.15.0 // indirect + golang.org/x/sys v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect + gorm.io/driver/mysql v1.5.6 // indirect + gorm.io/driver/postgres v1.5.7 // indirect + gorm.io/gorm v1.25.10 // indirect ) diff --git a/go.sum b/go.sum index 4e2b84a..1be03c7 100644 --- a/go.sum +++ b/go.sum @@ -1,21 +1,45 @@ +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= +github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= +github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0= +github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w= +github.com/gookit/goutil v0.6.15 h1:mMQ0ElojNZoyPD0eVROk5QXJPh2uKR4g06slgPDF5Jo= +github.com/gookit/goutil v0.6.15/go.mod h1:qdKdYEHQdEtyH+4fNdQNZfJHhI0jUZzHxQVAV3DaMDY= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 h1:L0QtFUgDarD7Fpv9jeVMgy/+Ec0mtnmYuImjTz6dtDA= +github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw= +github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= +github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= +github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= @@ -39,6 +63,7 @@ github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= @@ -49,6 +74,8 @@ github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.18.1 h1:rmuU42rScKWlhhJDyXZRKJQHXFX02chSVW1IvkPGiVM= @@ -57,25 +84,36 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= +github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -90,3 +128,10 @@ gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/mysql v1.5.6 h1:Ld4mkIickM+EliaQZQx3uOJDJHtrd70MxAUqWqlx3Y8= +gorm.io/driver/mysql v1.5.6/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= +gorm.io/driver/postgres v1.5.7 h1:8ptbNJTDbEmhdr62uReG5BGkdQyeasu/FZHxI0IMGnM= +gorm.io/driver/postgres v1.5.7/go.mod h1:3e019WlBaYI5o5LIdNV+LyxCMNtLOQETBXL2h4chKpA= +gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= +gorm.io/gorm v1.25.10 h1:dQpO+33KalOA+aFYGlK+EfxcI5MbO7EP2yYygwh9h+s= +gorm.io/gorm v1.25.10/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= diff --git a/handler/save.go b/handler/save.go index a21fb72..e97d602 100644 --- a/handler/save.go +++ b/handler/save.go @@ -1,58 +1,17 @@ package handler import ( - "database/sql" - _ "github.com/mattn/go-sqlite3" "log" "time" "webprobe/scanner" -) - -func InitDB(dbPath string) *sql.DB { - db, err := sql.Open("sqlite3", dbPath) - if err != nil { - log.Fatalf("Error opening database: %v", err) - } - // 创建第一个表：url_status - createURLStatusTableSQL := ` - CREATE TABLE IF NOT EXISTS url_status ( - FatherURL TEXT, - Depth INTEGER, - URL TEXT, - IPVersion TEXT, - Up BOOLEAN, - StatusCode INTEGER, - Latency INTEGER, - CertExpire DATETIME, - CreateTime DATETIME, - PRIMARY KEY (URL, IPVersion, CreateTime) - );` - _, err = db.Exec(createURLStatusTableSQL) - if err != nil { - log.Fatalf("Error creating url_status table: %v", err) - } + "webprobe/models" - // 创建第二个表：reachability - createReachabilityTableSQL := ` - CREATE TABLE IF NOT EXISTS reachability ( - URL TEXT, - IPv4FirstLevelReach FLOAT, - IPv4SecondLevelReach FLOAT, - IPv6FirstLevelReach FLOAT, - IPv6SecondLevelReach FLOAT, - CreateTime DATETIME, - PRIMARY KEY (URL, CreateTime) - );` - _, err = db.Exec(createReachabilityTableSQL) - if err != nil { - log.Fatalf("Error creating reachability table: %v", err) - } - - return db -} + _ "github.com/mattn/go-sqlite3" + "gorm.io/gorm" +) -// 将URL状态数据保存到数据库 +/* // 将URL状态数据保存到数据库 func SaveData(db *sql.DB, urlDataWithReachability []scanner.URLDataWithReachability, maxDepth int) { const layout = "2006-01-02 15:04:05-07:00" // 设置时间格式 @@ -74,3 +33,34 @@ func SaveData(db *sql.DB, urlDataWithReachability []scanner.URLDataWithReachabil } } } +*/ + +// SaveData 保存 URL 状态数据到数据库 +func SaveData(db *gorm.DB, urlDataWithReachability []scanner.URLDataWithReachability, maxDepth int) { + currentTime := time.Now() + + for _, data := range urlDataWithReachability { + status := data.URLStatus + + // 保存 URLStatus 数据 + if err := db.Create(&status).Error; err != nil { + log.Printf("Error inserting URLStatus data: %v", err) + } + + // 如果深度小于最大深度，则保存 Reachability 数据 + if status.Depth < maxDepth { + reachability := models.Reachability{ + URL: data.URL, + IPv4FirstLevelReach: data.ReachabilityIPv4.FirstLevelReach, + IPv4SecondLevelReach: data.ReachabilityIPv4.SecondLevelReach, + IPv6FirstLevelReach: data.ReachabilityIPv6.FirstLevelReach, + IPv6SecondLevelReach: data.ReachabilityIPv6.SecondLevelReach, + CreateTime: currentTime, + } + + if err := db.Create(&reachability).Error; err != nil { + log.Printf("Error inserting Reachability data: %v", err) + } + } + } +} diff --git a/internal/getConfig.go b/internal/getConfig.go new file mode 100644 index 0000000..bb08e5f --- /dev/null +++ b/internal/getConfig.go @@ -0,0 +1,18 @@ +package internal + +import ( + "log" + "webprobe/config" +) + +var Cfg config.Config + +func GetConfig(configPath string) config.Config { + log.Printf("开始导入配置文件...") + cfg, err := config.LoadConfig(configPath) + if err != nil { + log.Fatalf("导入配置文件失败: %v", err) + } + log.Printf("配置文件导入成功.") + return *cfg +} diff --git a/internal/geturlDataWithReachability.go b/internal/geturlDataWithReachability.go new file mode 100644 index 0000000..9c87d74 --- /dev/null +++ b/internal/geturlDataWithReachability.go @@ -0,0 +1,22 @@ +package internal + +import ( + "log" + + "webprobe/config" + "webprobe/handler" + "webprobe/scanner" +) + +var UrlDataWithReachability []scanner.URLDataWithReachability + +// geturlDataWithReachability +func GeturlDataWithReachability(cfg config.Config) ([]scanner.URLDataWithReachability, error) { + + // scanner结果的接收数组 + + log.Printf("开始进行扫描...") + urlDataWithReachability := make([]scanner.URLDataWithReachability, 0) + urlDataWithReachability = handler.Scanner(cfg.URLs, cfg.ScannerConfig, urlDataWithReachability) + return urlDataWithReachability, nil +} diff --git a/internal/initDB.go b/internal/initDB.go new file mode 100644 index 0000000..21b51bb --- /dev/null +++ b/internal/initDB.go @@ -0,0 +1,50 @@ +package internal + +import ( + "fmt" + "log" + "webprobe/config" + "webprobe/models" + + "gorm.io/driver/mysql" + "gorm.io/driver/postgres" + "gorm.io/gorm" +) + +var DB *gorm.DB + +// InitDB 初始化数据库并返回一个 *gorm.DB 实例 +func InitDB(config config.Config) (*gorm.DB, error) { + var ( + db *gorm.DB + err error + ) + + switch config.DBConfig.Dialect { + case "mysql": + dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=True&loc=Local", config.DBConfig.Username, config.DBConfig.Password, config.DBConfig.Host, config.DBConfig.Port, config.DBConfig.DBName) + db, err = gorm.Open(mysql.Open(dsn), &gorm.Config{}) + case "postgres": + dsn := fmt.Sprintf("host=%s user=%s password=%s dbname=%s port=%d sslmode=%s", config.DBConfig.Host, config.DBConfig.Username, config.DBConfig.Password, config.DBConfig.DBName, config.DBConfig.Port, config.DBConfig.SSLMode) + db, err = gorm.Open(postgres.New(postgres.Config{ + DSN: dsn, + }), &gorm.Config{}) + // 可以添加其他数据库类型的处理逻辑 + default: + log.Fatalf("Unsupported database dialect: %s", config.DBConfig.Dialect) + return nil, fmt.Errorf("unsupported database dialect: %s", config.DBConfig.Dialect) + } + + if err != nil { + log.Fatalf("Error opening database: %v", err) + return nil, err + } + + // 自动迁移表结构 + if err := db.AutoMigrate(&models.URLStatus{}, &models.Reachability{}); err != nil { + log.Fatalf("Error auto migrating tables: %v", err) + return nil, err + } + + return db, nil +} diff --git a/internal/scrapyWeb.go b/internal/scrapyWeb.go new file mode 100644 index 0000000..c7b079e --- /dev/null +++ b/internal/scrapyWeb.go @@ -0,0 +1,31 @@ +package internal + +import ( + "fmt" + "log" + "webprobe/models" + + "gorm.io/gorm/schema" +) + +var ConfigPath string + +func ScrapyWeb() { + Cfg = GetConfig(ConfigPath) + DB, _ = InitDB(Cfg) + DB.NamingStrategy = schema.NamingStrategy{ + TablePrefix: "", // 表前缀，如果有的话 + SingularTable: true, // 禁用表名的复数形式 + } + + // 执行自动迁移 + err := DB.AutoMigrate(&models.Reachability{}, &models.URLStatus{}) + if err != nil { + log.Fatalf("Error performing auto migration: %v", err) + } + if err != nil { + fmt.Printf("Error initializing database: %v\n", err) + return + } + UrlDataWithReachability, _ = GeturlDataWithReachability(Cfg) +} diff --git a/main.go b/main.go index d9f4f55..300fd50 100644 --- a/main.go +++ b/main.go @@ -1,42 +1,13 @@ +/* +Copyright © 2024 NAME HERE +*/ package main import ( - "log" - "webprobe/config" - "webprobe/handler" - "webprobe/metrics" - "webprobe/scanner" + "webprobe/cmd" ) func main() { - log.Printf("开始导入配置文件...") - cfg, err := config.LoadConfig() - if err != nil { - log.Fatalf("导入配置文件失败: %v", err) - } - log.Printf("配置文件导入成功.") - // scanner结果的接收数组 - metrics.InitMetric() - log.Printf("开始进行扫描...") - urlDataWithReachability := make([]scanner.URLDataWithReachability, 0) - urlDataWithReachability = handler.Scanner(cfg.URLs, cfg.ScannerConfig, urlDataWithReachability) - log.Printf("扫描已完成.") - log.Printf("开始生成 HTML 文件...") - handler.GenerateHTMLFile(urlDataWithReachability, cfg.Output, cfg.ScannerConfig.Crawl.Depth) - log.Printf("HTML 文件已生成，请访问 %s 查看.", cfg.Output) + cmd.Execute() - if cfg.DBConfig.UseDB { - log.Printf("正在写入数据库...") - db := handler.InitDB(cfg.DBConfig.DBPath) - handler.SaveData(db, urlDataWithReachability, cfg.ScannerConfig.Crawl.Depth) - defer db.Close() - log.Printf("写入数据库完成.") - } - - if cfg.Push.PushToPrometheus { - log.Printf("开始进行结果推送...") - metrics.SaveMetrics(urlDataWithReachability) - metrics.PushMetrics(cfg.Push.PushGatewayURL) - log.Printf("结果推送已完成，请访问 pushgateway 页面 %s 查看.", cfg.Push.PushGatewayURL) - } } diff --git a/models/Reachability.go b/models/Reachability.go new file mode 100644 index 0000000..0fb323c --- /dev/null +++ b/models/Reachability.go @@ -0,0 +1,17 @@ +package models + +import "time" + +type Reachability struct { + URL string `gorm:"column:URL;primaryKey"` + IPv4FirstLevelReach float64 `gorm:"column:IPv4FirstLevelReach"` + IPv4SecondLevelReach float64 `gorm:"column:IPv4SecondLevelReach"` + IPv6FirstLevelReach float64 `gorm:"column:IPv6FirstLevelReach"` + IPv6SecondLevelReach float64 `gorm:"column:IPv6SecondLevelReach"` + CreateTime time.Time `gorm:"column:create_time;primaryKey;default:CURRENT_TIMESTAMP"` +} + +// TableName sets the table name for the model. +func (Reachability) TableName() string { + return "reachability" +} diff --git a/models/URLStatus.go b/models/URLStatus.go new file mode 100644 index 0000000..fefa95b --- /dev/null +++ b/models/URLStatus.go @@ -0,0 +1,20 @@ +package models + +import "time" + +type URLStatus struct { + FatherURL string `gorm:"column:father_url"` + Depth int `gorm:"column:depth"` + URL string `gorm:"column:URL;primaryKey"` + IPVersion string `gorm:"column:ip_version;primaryKey"` + Up bool `gorm:"column:up"` + StatusCode int `gorm:"column:status_code"` + Latency int `gorm:"column:latency"` + CertExpire time.Time `gorm:"column:cert_expire"` + CreateTime time.Time `gorm:"column:create_time;primaryKey;default:CURRENT_TIMESTAMP"` +} + +// TableName sets the table name for the model. +func (URLStatus) TableName() string { + return "url_status" +} diff --git a/utility/cmdline.go b/utility/cmdline.go new file mode 100644 index 0000000..2c1e584 --- /dev/null +++ b/utility/cmdline.go @@ -0,0 +1,35 @@ +package utility + +import ( + "webprobe/config" + "webprobe/internal" + + "github.com/gookit/goutil/dump" +) + +func Cmdline() { + cfg := internal.GetConfig(config.ConfigPath) + urlDataWithReachability, _ := internal.GeturlDataWithReachability(cfg) + /*options := dump.Options{ + Output: os.Stdout, + NoType: true, + NoColor: false, + IndentLen: 2, + IndentChar: ' ', + MaxDepth: 10, + ShowFlag: 0, + CallerSkip: 1, + } + */ + // 使用这些选项创建一个新的 dumper + d := dump.NewWithOptions(dump.WithoutType()) + + // 使用新的 dumper 打印变量 + d.Print(urlDataWithReachability) + /*for _, data := range urlDataWithReachability { + //currentTime := time.Now().Format(layout) // 格式化当前时间 + status := data.URLStatus // 从URLDataWithReachability中获取URLStatus + fmt.Printf("ipv4: %s,ipv6:%s, 状态: %s \n", data.ReachabilityIPv4, data.ReachabilityIPv6, status) + + }*/ +} diff --git a/utility/generateHTML.go b/utility/generateHTML.go new file mode 100644 index 0000000..a6ff90f --- /dev/null +++ b/utility/generateHTML.go @@ -0,0 +1,19 @@ +package utility + +import ( + "log" + + "webprobe/config" + "webprobe/handler" + + "webprobe/internal" +) + +func generateHTML() { + cfg := internal.GetConfig(config.ConfigPath) + urlDataWithReachability, _ := internal.GeturlDataWithReachability(cfg) + log.Printf("扫描已完成.") + log.Printf("开始生成 HTML 文件...") + handler.GenerateHTMLFile(urlDataWithReachability, cfg.Output, cfg.ScannerConfig.Crawl.Depth) + log.Printf("HTML 文件已生成，请访问 %s 查看.", cfg.Output) +} diff --git a/utility/pushToPrometheus.go b/utility/pushToPrometheus.go new file mode 100644 index 0000000..7ae6d66 --- /dev/null +++ b/utility/pushToPrometheus.go @@ -0,0 +1,18 @@ +package utility + +import ( + "log" + + "webprobe/metrics" + "webprobe/scanner" +) + +func PushToPrometheus(pushGatewayURL string, urlDataWithReachability []scanner.URLDataWithReachability) { + + metrics.InitMetric() + log.Printf("开始进行结果推送...") + metrics.SaveMetrics(urlDataWithReachability) + metrics.PushMetrics(pushGatewayURL) + log.Printf("结果推送已完成，请访问 pushgateway 页面 %s 查看.", pushGatewayURL) + +} diff --git a/utility/pushToRDB.go b/utility/pushToRDB.go new file mode 100644 index 0000000..aca41a4 --- /dev/null +++ b/utility/pushToRDB.go @@ -0,0 +1,42 @@ +package utility + +import ( + "log" + "time" + "webprobe/scanner" + + "webprobe/models" + + _ "github.com/mattn/go-sqlite3" + "gorm.io/gorm" +) + +// SaveData 保存 URL 状态数据到数据库 +func SaveData(db *gorm.DB, urlDataWithReachability []scanner.URLDataWithReachability, maxDepth int) { + currentTime := time.Now() + + for _, data := range urlDataWithReachability { + status := data.URLStatus + + // 保存 URLStatus 数据 + if err := db.Create(&status).Error; err != nil { + log.Printf("Error inserting URLStatus data: %v", err) + } + + // 如果深度小于最大深度，则保存 Reachability 数据 + if status.Depth < maxDepth { + reachability := models.Reachability{ + URL: data.URL, + IPv4FirstLevelReach: data.ReachabilityIPv4.FirstLevelReach, + IPv4SecondLevelReach: data.ReachabilityIPv4.SecondLevelReach, + IPv6FirstLevelReach: data.ReachabilityIPv6.FirstLevelReach, + IPv6SecondLevelReach: data.ReachabilityIPv6.SecondLevelReach, + CreateTime: currentTime, + } + + if err := db.Create(&reachability).Error; err != nil { + log.Printf("Error inserting Reachability data: %v", err) + } + } + } +} diff --git a/webprobe b/webprobe new file mode 100644 index 0000000..0fb6bb7 Binary files /dev/null and b/webprobe differ