From df32cc565433d948c713b28c28c30b009e7bdde1 Mon Sep 17 00:00:00 2001 From: Pedro Lopes Date: Mon, 2 May 2022 08:53:10 +0100 Subject: [PATCH 1/4] Update sanitize function --- README.md | 1 - internal/commands/project.go | 9 +- internal/commands/root.go | 21 +---- internal/commands/scan.go | 35 ++++---- internal/commands/util/usercount/gitlab.go | 5 +- internal/logger/utils.go | 61 ++++++++++++++ internal/params/flags.go | 16 ++-- internal/wrappers/azure-http.go | 9 +- internal/wrappers/bfl-http.go | 1 - internal/wrappers/bitbucket-http.go | 9 +- internal/wrappers/client.go | 98 ++++++---------------- internal/wrappers/github-http.go | 13 +-- internal/wrappers/gitlab-http.go | 15 ++-- internal/wrappers/predicates-http.go | 13 +-- 14 files changed, 166 insertions(+), 140 deletions(-) create mode 100644 internal/logger/utils.go diff --git a/README.md b/README.md index 925370ef4..f45805450 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,6 @@ Report Bug ยท Request Feature -

diff --git a/internal/commands/project.go b/internal/commands/project.go index 31b78cfeb..29588231c 100644 --- a/internal/commands/project.go +++ b/internal/commands/project.go @@ -9,8 +9,9 @@ import ( "github.com/MakeNowJust/heredoc" "github.com/checkmarx/ast-cli/internal/commands/util" "github.com/checkmarx/ast-cli/internal/commands/util/printer" - + "github.com/checkmarx/ast-cli/internal/logger" commonParams "github.com/checkmarx/ast-cli/internal/params" + "github.com/spf13/viper" "github.com/pkg/errors" @@ -306,7 +307,7 @@ func runCreateProjectCommand( } var payload []byte payload, _ = json.Marshal(projModel) - PrintIfVerbose(fmt.Sprintf("Payload to projects service: %s\n", string(payload))) + logger.PrintIfVerbose(fmt.Sprintf("Payload to projects service: %s\n", string(payload))) projResponseModel, errorModel, err = projectsWrapper.Create(&projModel) if err != nil { return errors.Wrapf(err, "%s", failedCreatingProj) @@ -350,6 +351,8 @@ func updateProjectConfigurationIfNeeded(cmd *cobra.Command, projectsWrapper wrap return sshErr } + viper.Set(commonParams.SshValue, sshKey) + sshKeyConf := getProjectConfiguration(sshConfKey, "sshKey", git, projOriginLevel, sshKey, "Secret", true) projectConfigurations = append(projectConfigurations, sshKeyConf) @@ -435,6 +438,8 @@ func runListProjectsCommand(projectsWrapper wrappers.ProjectsWrapper) func(cmd * if err != nil { return errors.Wrapf(err, "%s\n", failedGettingAll) } + // + // Checking the response if errorModel != nil { return errors.Errorf(ErrorCodeFormat, failedGettingAll, errorModel.Code, errorModel.Message) diff --git a/internal/commands/root.go b/internal/commands/root.go index 1164558a0..1f63c5fac 100644 --- a/internal/commands/root.go +++ b/internal/commands/root.go @@ -9,6 +9,7 @@ import ( "github.com/MakeNowJust/heredoc" "github.com/checkmarx/ast-cli/internal/commands/util" "github.com/checkmarx/ast-cli/internal/commands/util/printer" + "github.com/checkmarx/ast-cli/internal/logger" "github.com/checkmarx/ast-cli/internal/params" "github.com/pkg/errors" @@ -147,23 +148,9 @@ func NewAstCLI( const configFormatString = "%30v: %s" func PrintConfiguration() { - if viper.GetBool(params.DebugFlag) { - log.Println("CLI Configuration:") - for param := range util.Properties { - if param == "cx_client_secret" && len(viper.GetString(param)) > 0 { - log.Println(fmt.Sprintf(configFormatString, param, "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX")) - } else if param == "cx_apikey" && len(viper.GetString(param)) > 0 { - log.Println(fmt.Sprintf(configFormatString, param, "XXXXXXXXXXXXXXXXXXXXX")) - } else { - log.Println(fmt.Sprintf(configFormatString, param, viper.GetString(param))) - } - } - } -} - -func PrintIfVerbose(msg string) { - if viper.GetBool(params.DebugFlag) { - log.Println(msg) + logger.PrintIfVerbose("CLI Configuration:") + for param := range util.Properties { + logger.PrintIfVerbose(fmt.Sprintf(configFormatString, param, viper.GetString(param))) } } diff --git a/internal/commands/scan.go b/internal/commands/scan.go index de9d68b1b..a550d1ac3 100644 --- a/internal/commands/scan.go +++ b/internal/commands/scan.go @@ -18,6 +18,7 @@ import ( "github.com/checkmarx/ast-cli/internal/commands/util" "github.com/checkmarx/ast-cli/internal/commands/util/printer" + "github.com/checkmarx/ast-cli/internal/logger" "github.com/google/shlex" "github.com/google/uuid" "github.com/pkg/errors" @@ -641,7 +642,7 @@ func compressFolder(sourceDir, filter, userIncludeFilter, scaResolver string) (s if err != nil { return "", err } - PrintIfVerbose(fmt.Sprintf("Zip size: %.2fMB\n", float64(stat.Size())/mbBytes)) + logger.PrintIfVerbose(fmt.Sprintf("Zip size: %.2fMB\n", float64(stat.Size())/mbBytes)) return outputFile.Name(), err } @@ -667,13 +668,13 @@ func addDirFilesIgnoreFilter(zipWriter *zip.Writer, baseDir, parentDir string) e } for _, file := range files { if file.IsDir() { - PrintIfVerbose("Directory: " + file.Name()) + logger.PrintIfVerbose("Directory: " + file.Name()) newParent := parentDir + file.Name() + "/" newBase := baseDir + file.Name() + "/" err = addDirFilesIgnoreFilter(zipWriter, newBase, newParent) } else { fileName := parentDir + file.Name() - PrintIfVerbose("Included: " + fileName) + logger.PrintIfVerbose("Included: " + fileName) dat, _ := ioutil.ReadFile(fileName) f, _ := zipWriter.Create(baseDir + file.Name()) @@ -715,7 +716,7 @@ func handleFile( ) error { fileName := parentDir + file.Name() if filterMatched(includeFilters, file.Name()) && filterMatched(filters, file.Name()) { - PrintIfVerbose("Included: " + fileName) + logger.PrintIfVerbose("Included: " + fileName) dat, err := ioutil.ReadFile(parentDir + file.Name()) if err != nil { if os.IsNotExist(err) { @@ -732,7 +733,7 @@ func handleFile( return err } } else { - PrintIfVerbose("Excluded: " + fileName) + logger.PrintIfVerbose("Excluded: " + fileName) } return nil } @@ -747,7 +748,7 @@ func handleDir( ) error { // Check if folder belongs to the disabled exclusions if commonParams.DisabledExclusions[file.Name()] { - PrintIfVerbose("The folder " + file.Name() + " is being included") + logger.PrintIfVerbose("The folder " + file.Name() + " is being included") newParent, newBase := GetNewParentAndBase(parentDir, file, baseDir) return addDirFilesIgnoreFilter(zipWriter, newBase, newParent) } @@ -760,7 +761,7 @@ func handleDir( return err } if match { - PrintIfVerbose("Excluded: " + parentDir + file.Name() + "/") + logger.PrintIfVerbose("Excluded: " + parentDir + file.Name() + "/") return nil } } @@ -770,7 +771,7 @@ func handleDir( } func GetNewParentAndBase(parentDir string, file fs.FileInfo, baseDir string) (newParent, newBase string) { - PrintIfVerbose("Directory: " + parentDir + file.Name()) + logger.PrintIfVerbose("Directory: " + parentDir + file.Name()) newParent = parentDir + file.Name() + "/" newBase = baseDir + file.Name() + "/" return newParent, newBase @@ -829,13 +830,13 @@ func runScaResolver(sourceDir, scaResolver, scaResolverParams string) error { if err != nil { return errors.Errorf("%s", err) } - PrintIfVerbose(string(out)) + logger.PrintIfVerbose(string(out)) } return nil } func addScaResults(zipWriter *zip.Writer) error { - PrintIfVerbose("Included SCA Results: " + ".cxsca-results.json") + logger.PrintIfVerbose("Included SCA Results: " + ".cxsca-results.json") dat, err := ioutil.ReadFile(scaResolverResultsFile) _ = os.Remove(scaResolverResultsFile) if err != nil { @@ -908,7 +909,7 @@ func getUploadURLFromSource( if zipFilePathErr != nil { return "", errors.Wrapf(zipFilePathErr, "%s: Failed to upload sources file\n", failedCreating) } - PrintIfVerbose(fmt.Sprintf("Uploading file to %s\n", *preSignedURL)) + logger.PrintIfVerbose(fmt.Sprintf("Uploading file to %s\n", *preSignedURL)) return *preSignedURL, zipFilePathErr } return preSignedURL, nil @@ -932,13 +933,13 @@ func UnzipFile(f string) (string, error) { for _, f := range archive.File { filePath := filepath.Join(tempDir, f.Name) - PrintIfVerbose("unzipping file " + filePath + "...") + logger.PrintIfVerbose("unzipping file " + filePath + "...") if !strings.HasPrefix(filePath, filepath.Clean(tempDir)+string(os.PathSeparator)) { return "", errors.New("invalid file path " + filePath) } if f.FileInfo().IsDir() { - PrintIfVerbose("creating directory...") + logger.PrintIfVerbose("creating directory...") err = os.MkdirAll(filePath, os.ModePerm) if err != nil { return "", errors.Errorf("%s %s", errorUnzippingFile, err.Error()) @@ -1155,6 +1156,10 @@ func setupScanHandler( func defineSSHCredentials(sshKeyPath string, handler *wrappers.ScanHandler) error { sshKey, err := util.ReadFileAsString(sshKeyPath) + if err != nil { + return err + } + viper.Set(commonParams.SshValue, sshKey) credentials := wrappers.GitCredentials{} @@ -1163,7 +1168,7 @@ func defineSSHCredentials(sshKeyPath string, handler *wrappers.ScanHandler) erro handler.Credentials = credentials - return err + return nil } func handleWait( @@ -1238,7 +1243,7 @@ func applyThreshold( currentValue := summaryMap[key] failed := currentValue >= thresholdLimit logMessage := fmt.Sprintf(thresholdLog, key, thresholdLimit, currentValue) - PrintIfVerbose(logMessage) + logger.PrintIfVerbose(logMessage) if failed { errorBuilder.WriteString(fmt.Sprintf("%s | ", logMessage)) diff --git a/internal/commands/util/usercount/gitlab.go b/internal/commands/util/usercount/gitlab.go index d1c2d8036..5e4a270cc 100644 --- a/internal/commands/util/usercount/gitlab.go +++ b/internal/commands/util/usercount/gitlab.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/checkmarx/ast-cli/internal/commands/util/printer" + "github.com/checkmarx/ast-cli/internal/logger" "github.com/checkmarx/ast-cli/internal/params" "github.com/checkmarx/ast-cli/internal/wrappers" "github.com/pkg/errors" @@ -151,7 +152,7 @@ func collectFromGitLabGroups(gitLabWrapper wrappers.GitLabWrapper) ( repoAccessLevelDisabled) || strings.EqualFold( gitLabProject.RepoAccessLevel, repoAccessLevelPrivate) { - wrappers.PrintIfVerbose( + logger.PrintIfVerbose( fmt.Sprintf( "Skipping the project %s because of empty repository.", gitLabProject.PathWithNameSpace)) @@ -191,7 +192,7 @@ func collectFromUser(gitLabWrapper wrappers.GitLabWrapper) ( if gitLabProject.EmptyRepo || strings.EqualFold( gitLabProject.RepoAccessLevel, repoAccessLevelDisabled) || strings.EqualFold(gitLabProject.RepoAccessLevel, repoAccessLevelPrivate) { - wrappers.PrintIfVerbose( + logger.PrintIfVerbose( fmt.Sprintf( "Skipping the project %s because of empty repository.", gitLabProject.PathWithNameSpace)) } else { diff --git a/internal/logger/utils.go b/internal/logger/utils.go new file mode 100644 index 000000000..6763453b3 --- /dev/null +++ b/internal/logger/utils.go @@ -0,0 +1,61 @@ +package logger + +import ( + "fmt" + "log" + "net/http" + "net/http/httputil" + "strings" + "unicode/utf8" + + "github.com/checkmarx/ast-cli/internal/params" + "github.com/spf13/viper" +) + +var sanitizeFlags = []string{ + params.AstAPIKey, params.AccessKeyIDConfigKey, params.AccessKeySecretConfigKey, + params.AstToken, params.SshValue, + params.SCMTokenFlag, +} + +func PrintIfVerbose(msg string) { + if viper.GetBool(params.DebugFlag) { + if utf8.Valid([]byte(msg)) { + log.Print(sanitizeLogs(msg)) + } else { + log.Print("Request contains binary data and cannot be printed!") + } + } +} + +func PrintRequest(r *http.Request) { + PrintIfVerbose("Sending API request to:") + requestDump, err := httputil.DumpRequest(r, true) + if err != nil { + fmt.Println(err) + } + PrintIfVerbose(string(requestDump)) +} + +func PrintResponse(r *http.Response) { + PrintIfVerbose("Receiving API response:") + requestDump, err := httputil.DumpResponse(r, true) + if err != nil { + fmt.Println(err) + } + PrintIfVerbose(string(requestDump)) +} + +func sanitizeLogs(log string) string { + if strings.Contains(log, "access_token") && strings.Contains(log, "token_type") { + return "" + } + + for _, flag := range sanitizeFlags { + value := viper.GetString(flag) + if len(value) > 0 { + log = strings.ReplaceAll(log, value, "***") + } + } + return log +} diff --git a/internal/params/flags.go b/internal/params/flags.go index 2f6f79a49..f297307c2 100644 --- a/internal/params/flags.go +++ b/internal/params/flags.go @@ -100,14 +100,14 @@ const ( QueryIDFlag = "query-id" SSHKeyFlag = "ssh-key" RepoURLFlag = "repo-url" - - // INDIVIDUAL FILTER FLAGS - SastFilterFlag = "sast-filter" - SastFilterUsage = "SAST filter" - KicsFilterFlag = "kics-filter" - KicsFilterUsage = "KICS filter" - ScaFilterFlag = "sca-filter" - ScaFilterUsage = "SCA filter" + AstToken = "ast-token" + SshValue = "ssh-value" + SastFilterFlag = "sast-filter" + SastFilterUsage = "SAST filter" + KicsFilterFlag = "kics-filter" + KicsFilterUsage = "KICS filter" + ScaFilterFlag = "sca-filter" + ScaFilterUsage = "SCA filter" ) // Parameter values diff --git a/internal/wrappers/azure-http.go b/internal/wrappers/azure-http.go index b3bb77d13..5fc6115fa 100644 --- a/internal/wrappers/azure-http.go +++ b/internal/wrappers/azure-http.go @@ -9,6 +9,7 @@ import ( b64 "encoding/base64" + "github.com/checkmarx/ast-cli/internal/logger" "github.com/checkmarx/ast-cli/internal/params" "github.com/pkg/errors" "github.com/spf13/viper" @@ -82,8 +83,6 @@ func (g *AzureHTTPWrapper) GetProjects(url, organizationName, token string) (Azu func (g *AzureHTTPWrapper) get(url, token string, target interface{}, queryParams map[string]string, authFormat string) error { var err error - PrintIfVerbose(fmt.Sprintf("Request to %s", url)) - req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { return err @@ -99,13 +98,19 @@ func (g *AzureHTTPWrapper) get(url, token string, target interface{}, queryParam } req.URL.RawQuery = q.Encode() resp, err := g.client.Do(req) + if err != nil { return err } + logger.PrintRequest(req) + defer func() { _ = resp.Body.Close() }() + + logger.PrintResponse(resp) + switch resp.StatusCode { case http.StatusOK: err = json.NewDecoder(resp.Body).Decode(target) diff --git a/internal/wrappers/bfl-http.go b/internal/wrappers/bfl-http.go index b18ff2874..1aa578f68 100644 --- a/internal/wrappers/bfl-http.go +++ b/internal/wrappers/bfl-http.go @@ -41,7 +41,6 @@ func handleBflResponseWithBody(resp *http.Response, err error) (*BFLResponseMode return nil, nil, err } - PrintIfVerbose(fmt.Sprintf("Response : %s", resp.Status)) decoder := json.NewDecoder(resp.Body) defer func() { diff --git a/internal/wrappers/bitbucket-http.go b/internal/wrappers/bitbucket-http.go index 093ae5460..c769afe34 100644 --- a/internal/wrappers/bitbucket-http.go +++ b/internal/wrappers/bitbucket-http.go @@ -10,6 +10,7 @@ import ( "strconv" "time" + "github.com/checkmarx/ast-cli/internal/logger" "github.com/checkmarx/ast-cli/internal/params" "github.com/spf13/viper" ) @@ -144,7 +145,7 @@ func (g *BitBucketHTTPWrapper) getFromBitBucket( ) error { var err error - PrintIfVerbose(fmt.Sprintf("Request to %s", url)) + logger.PrintIfVerbose(fmt.Sprintf("Request to %s", url)) req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { @@ -297,12 +298,14 @@ func getBitBucket(client *http.Client, token, url string, target interface{}, qu if err != nil { return err } - - PrintIfVerbose(fmt.Sprintf("Request to %s", req.URL.String())) + logger.PrintRequest(req) defer func() { _ = resp.Body.Close() }() + + logger.PrintResponse(resp) + switch resp.StatusCode { case http.StatusOK: err = json.NewDecoder(resp.Body).Decode(target) diff --git a/internal/wrappers/client.go b/internal/wrappers/client.go index 322a71ad2..68051fa88 100644 --- a/internal/wrappers/client.go +++ b/internal/wrappers/client.go @@ -1,7 +1,6 @@ package wrappers import ( - "bytes" "crypto/tls" "encoding/json" "fmt" @@ -14,14 +13,13 @@ import ( "net/url" "strings" "time" - "unicode/utf8" + "github.com/checkmarx/ast-cli/internal/logger" "github.com/pkg/errors" + "github.com/spf13/viper" commonParams "github.com/checkmarx/ast-cli/internal/params" "github.com/checkmarx/ast-cli/internal/wrappers/ntlm" - - "github.com/spf13/viper" ) const ( @@ -54,32 +52,8 @@ const FailedAccessToken = "Failed to obtain access token" var cachedAccessToken string var cachedAccessTime time.Time -func PrintIfVerbose(msg string) { - if viper.GetBool(commonParams.DebugFlag) { - if utf8.Valid([]byte(msg)) { - log.Print(msg) - } else { - log.Print("Request contains binary data and cannot be printed!") - } - } -} - -func convertReqBodyToString(body io.Reader) (string, io.Reader) { - var bodyStr string - if body != nil { - b, err := ioutil.ReadAll(body) - if err != nil { - panic(err) - } - body = bytes.NewBuffer(b) - bodyStr = string(b) - } - return bodyStr, body -} - func setAgentName(req *http.Request) { agentStr := viper.GetString(commonParams.AgentNameKey) + "/" + commonParams.Version - PrintIfVerbose("Using Agent Name: " + agentStr) req.Header.Set("User-Agent", agentStr) } @@ -117,13 +91,13 @@ func basicProxyClient(timeout uint, proxyStr string) *http.Client { u, _ := url.Parse(proxyStr) var tr *http.Transport if len(proxyStr) > 0 { - PrintIfVerbose("Creating HTTP Client with Proxy: " + proxyStr) + logger.PrintIfVerbose("Creating HTTP Client with Proxy: " + proxyStr) tr = &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure}, Proxy: http.ProxyURL(u), } } else { - PrintIfVerbose("Creating HTTP Client.") + logger.PrintIfVerbose("Creating HTTP Client.") tr = &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure}, } @@ -140,7 +114,7 @@ func ntmlProxyClient(timeout uint, proxyStr string) *http.Client { domainStr := viper.GetString(commonParams.ProxyDomainKey) proxyUser := u.User.Username() proxyPass, _ := u.User.Password() - PrintIfVerbose("Creating HTTP client using NTLM Proxy using: " + proxyStr) + logger.PrintIfVerbose("Creating HTTP client using NTLM Proxy using: " + proxyStr) ntlmDialContext := ntlm.NewNTLMProxyDialContext(dialer, u, proxyUser, proxyPass, domainStr, nil) return &http.Client{ Transport: &http.Transport{ @@ -157,7 +131,6 @@ func SendHTTPRequest(method, path string, body io.Reader, auth bool, timeout uin } func SendHTTPRequestByFullURL(method, fullURL string, body io.Reader, auth bool, timeout uint) (*http.Response, error) { - bodyStr, body := convertReqBodyToString(body) req, err := http.NewRequest(method, fullURL, body) client := getClient(timeout) setAgentName(req) @@ -170,10 +143,7 @@ func SendHTTPRequestByFullURL(method, fullURL string, body io.Reader, auth bool, return nil, err } } - PrintIfVerbose("Sending API request to: " + fullURL) - if len(bodyStr) > 0 { - PrintIfVerbose(bodyStr) - } + req = addReqMonitor(req) var resp *http.Response resp, err = doRequest(client, req) @@ -222,7 +192,6 @@ func SendHTTPRequestPasswordAuth( method, path string, body io.Reader, timeout uint, username, password, adminClientID, adminClientSecret string, ) (*http.Response, error) { - bodyStr, body := convertReqBodyToString(body) u := GetAuthURL(path) req, err := http.NewRequest(method, u, body) client := getClient(timeout) @@ -236,10 +205,7 @@ func SendHTTPRequestPasswordAuth( return nil, err } var resp *http.Response - PrintIfVerbose("Requesting Password Auth with Auth URL: " + u) - if len(bodyStr) > 0 { - PrintIfVerbose(bodyStr) - } + req = addReqMonitor(req) resp, err = doRequest(client, req) if err != nil { @@ -262,7 +228,7 @@ func GetAuthURL(path string) string { } else { authURL = GetURL(path) } - PrintIfVerbose("Auth URL is: " + authURL) + logger.PrintIfVerbose("Auth URL is: " + authURL) return authURL } @@ -270,7 +236,6 @@ func SendHTTPRequestWithQueryParams( method, path string, params map[string]string, body io.Reader, timeout uint, ) (*http.Response, error) { - bodyStr, body := convertReqBodyToString(body) u := GetURL(path) req, err := http.NewRequest(method, u, body) client := getClient(timeout) @@ -287,14 +252,10 @@ func SendHTTPRequestWithQueryParams( if err != nil { return nil, err } - PrintIfVerbose("Sending API request to: " + u) - if len(bodyStr) > 0 { - PrintIfVerbose(bodyStr) - } var resp *http.Response resp, err = doRequest(client, req) if err != nil { - return resp, errors.Errorf("%s %s \n", checkmarxURLError, req.URL) + return resp, errors.Errorf("%s %s \n", checkmarxURLError, req.URL.RequestURI()) } if resp.StatusCode == http.StatusForbidden { return resp, errors.Errorf("%s", "Provided credentials do not have permissions for this command") @@ -380,7 +341,7 @@ func enrichWithPasswordCredentials( } func getClientCredentials(accessKeyID, accessKeySecret, astAPKey, authURI string) (*string, error) { - PrintIfVerbose("Fetching API access token.") + logger.PrintIfVerbose("Fetching API access token.") tokenExpirySeconds := viper.GetInt(commonParams.TokenExpirySecondsKey) var accessToken *string var err error @@ -405,37 +366,24 @@ func getClientCredentials(accessKeyID, accessKeySecret, astAPKey, authURI string } func getClientCredentialsFromCache(tokenExpirySeconds int) *string { - PrintIfVerbose("Checking cache for API access token.") + logger.PrintIfVerbose("Checking cache for API access token.") expired := time.Since(cachedAccessTime) > time.Duration(tokenExpirySeconds-expiryGraceSeconds)*time.Second if !expired { - PrintIfVerbose("Using cached API access token!") + logger.PrintIfVerbose("Using cached API access token!") return &cachedAccessToken } - PrintIfVerbose("API access token not found in cache!") + logger.PrintIfVerbose("API access token not found in cache!") return nil } func writeCredentialsToCache(accessToken *string) { - PrintIfVerbose("Storing API access token to cache.") + logger.PrintIfVerbose("Storing API access token to cache.") + viper.Set(commonParams.AstToken, *accessToken) cachedAccessToken = *accessToken cachedAccessTime = time.Now() } -func sanitizeCredentials(credentialsPayload string) string { - sanitized := credentialsPayload - if strings.Contains(credentialsPayload, "grant_type=client_credentials") { - strs := strings.Split(credentialsPayload, "client_secret") - sanitized = strs[0] + "client_secret=XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX" - } - if strings.Contains(credentialsPayload, "grant_type=refresh_token") { - sanitized = "api_key=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" - } - return sanitized -} - func getNewToken(credentialsPayload, authServerURI string) (*string, error) { - PrintIfVerbose("Getting new API access token from: " + authServerURI) - PrintIfVerbose(sanitizeCredentials(credentialsPayload)) payload := strings.NewReader(credentialsPayload) req, err := http.NewRequest(http.MethodPost, authServerURI, payload) setAgentName(req) @@ -446,6 +394,7 @@ func getNewToken(credentialsPayload, authServerURI string) (*string, error) { req.Header.Add("content-type", "application/x-www-form-urlencoded") clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) client := getClient(clientTimeout) + res, err := doRequest(client, req) if err != nil { return nil, errors.Errorf("%s %s", checkmarxURLError, GetAuthURL("")) @@ -482,22 +431,22 @@ func getNewToken(credentialsPayload, authServerURI string) (*string, error) { return nil, err } - PrintIfVerbose("Successfully retrieved API token.") + logger.PrintIfVerbose("Successfully retrieved API token.") return &credentialsInfo.AccessToken, nil } func getCredentialsPayload(accessKeyID, accessKeySecret string) string { - PrintIfVerbose("Using Client ID and secret credentials.") + logger.PrintIfVerbose("Using Client ID and secret credentials.") return fmt.Sprintf("grant_type=client_credentials&client_id=%s&client_secret=%s", accessKeyID, accessKeySecret) } func getAPIKeyPayload(astToken string) string { - PrintIfVerbose("Using API key credentials.") + logger.PrintIfVerbose("Using API key credentials.") return fmt.Sprintf("grant_type=refresh_token&client_id=ast-app&refresh_token=%s", astToken) } func getPasswordCredentialsPayload(username, password, adminClientID, adminClientSecret string) string { - PrintIfVerbose("Using username and password credentials.") + logger.PrintIfVerbose("Using username and password credentials.") return fmt.Sprintf( "scope=openid&grant_type=password&username=%s&password=%s"+ "&client_id=%s&client_secret=%s", username, password, adminClientID, adminClientSecret, @@ -510,13 +459,16 @@ func doRequest(client *http.Client, req *http.Request) (*http.Response, error) { retryLimit := int(viper.GetUint(commonParams.RetryFlag)) retryWaitTimeSeconds := viper.GetUint(commonParams.RetryDelayFlag) // try starts at -1 as we always do at least one request, retryLimit can be 0 + logger.PrintRequest(req) for try := -1; try < retryLimit; try++ { - PrintIfVerbose(fmt.Sprintf("Request attempt %d in %d", try+tryPrintOffset, retryLimit+retryLimitPrintOffset)) + logger.PrintIfVerbose(fmt.Sprintf("Request attempt %d in %d", + try+tryPrintOffset, retryLimit+retryLimitPrintOffset)) resp, err = client.Do(req) if resp != nil && err == nil { + logger.PrintResponse(resp) return resp, nil } - PrintIfVerbose(fmt.Sprintf("Request failed in attempt %d", try+tryPrintOffset)) + logger.PrintIfVerbose(fmt.Sprintf("Request failed in attempt %d", try+tryPrintOffset)) time.Sleep(time.Duration(retryWaitTimeSeconds) * time.Second) } return nil, err diff --git a/internal/wrappers/github-http.go b/internal/wrappers/github-http.go index ee61b9adc..75e7bebe1 100644 --- a/internal/wrappers/github-http.go +++ b/internal/wrappers/github-http.go @@ -7,6 +7,7 @@ import ( "net/http" "strings" + "github.com/checkmarx/ast-cli/internal/logger" "github.com/checkmarx/ast-cli/internal/params" "github.com/pkg/errors" "github.com/spf13/viper" @@ -252,32 +253,34 @@ func get(client *http.Client, url string, target interface{}, queryParams map[st } req.URL.RawQuery = q.Encode() - PrintIfVerbose(fmt.Sprintf("Request to %s", req.URL)) + logger.PrintRequest(req) resp, currentError := client.Do(req) if currentError != nil { count++ - PrintIfVerbose(fmt.Sprintf("Request to %s dropped, retrying", req.URL)) + logger.PrintIfVerbose(fmt.Sprintf("Request to %s dropped, retrying", req.URL)) err = currentError continue } + logger.PrintResponse(resp) + switch resp.StatusCode { case http.StatusOK: - PrintIfVerbose(fmt.Sprintf("Request to URL %s OK", req.URL)) + logger.PrintIfVerbose(fmt.Sprintf("Request to URL %s OK", req.URL)) currentError = json.NewDecoder(resp.Body).Decode(target) closeBody(resp) if currentError != nil { return nil, currentError } case http.StatusConflict: - PrintIfVerbose(fmt.Sprintf("Found empty repository in %s", req.URL)) + logger.PrintIfVerbose(fmt.Sprintf("Found empty repository in %s", req.URL)) closeBody(resp) return nil, nil default: body, currentError := io.ReadAll(resp.Body) closeBody(resp) if currentError != nil { - PrintIfVerbose(currentError.Error()) + logger.PrintIfVerbose(currentError.Error()) return nil, currentError } message := fmt.Sprintf("Code %d %s", resp.StatusCode, string(body)) diff --git a/internal/wrappers/gitlab-http.go b/internal/wrappers/gitlab-http.go index d6ac1b696..3dade4123 100644 --- a/internal/wrappers/gitlab-http.go +++ b/internal/wrappers/gitlab-http.go @@ -7,6 +7,7 @@ import ( "net/http" "net/url" + "github.com/checkmarx/ast-cli/internal/logger" "github.com/checkmarx/ast-cli/internal/params" "github.com/pkg/errors" "github.com/spf13/viper" @@ -71,7 +72,7 @@ func (g *GitLabHTTPWrapper) GetCommits( encodedProjectPath := url.QueryEscape(gitLabProjectPathWithNameSpace) commitsURL := fmt.Sprintf(gitLabCommitURL, gitLabBaseURL, gitLabAPIVersion, encodedProjectPath) - PrintIfVerbose(fmt.Sprintf("Getting commits for project: %s", gitLabProjectPathWithNameSpace)) + logger.PrintIfVerbose(fmt.Sprintf("Getting commits for project: %s", gitLabProjectPathWithNameSpace)) pages, err := fetchWithPagination(g.client, commitsURL, queryParams) if err != nil { @@ -101,7 +102,7 @@ func (g *GitLabHTTPWrapper) GetGitLabProjects(gitLabGroupName string, queryParam gitLabBaseURL := viper.GetString(params.GitLabURLFlag) encodedGroupName := url.QueryEscape(gitLabGroupName) - PrintIfVerbose(fmt.Sprintf("Finding the projects for group: %s", gitLabGroupName)) + logger.PrintIfVerbose(fmt.Sprintf("Finding the projects for group: %s", gitLabGroupName)) projectsURL := fmt.Sprintf(gitLabGroupProjectsURL, gitLabBaseURL, gitLabAPIVersion, encodedGroupName) pages, err := fetchWithPagination(g.client, projectsURL, queryParams) @@ -146,15 +147,19 @@ func getFromGitLab( q.Add(k, v) } req.URL.RawQuery = q.Encode() - PrintIfVerbose(fmt.Sprintf("Request to %s", req.URL)) + + logger.PrintRequest(req) + resp, currentError := client.Do(req) if currentError != nil { count++ - PrintIfVerbose(fmt.Sprintf("Request to %s dropped, retrying", req.URL)) + logger.PrintIfVerbose(fmt.Sprintf("Request to %s dropped, retrying", req.URL)) err = currentError continue } + logger.PrintResponse(resp) + switch resp.StatusCode { case http.StatusOK: currentError = json.NewDecoder(resp.Body).Decode(target) @@ -166,7 +171,7 @@ func getFromGitLab( body, currentError := io.ReadAll(resp.Body) closeResponseBody(resp) if currentError != nil { - PrintIfVerbose(currentError.Error()) + logger.PrintIfVerbose(currentError.Error()) return nil, currentError } message := fmt.Sprintf("Code %d %s", resp.StatusCode, string(body)) diff --git a/internal/wrappers/predicates-http.go b/internal/wrappers/predicates-http.go index a8a220ce2..fe175d04b 100644 --- a/internal/wrappers/predicates-http.go +++ b/internal/wrappers/predicates-http.go @@ -7,6 +7,7 @@ import ( "net/http" "strings" + "github.com/checkmarx/ast-cli/internal/logger" "github.com/checkmarx/ast-cli/internal/params" "github.com/pkg/errors" "github.com/spf13/viper" @@ -41,11 +42,11 @@ func (r *ResultsPredicatesHTTPWrapper) GetAllPredicatesForSimilarityID(similarit return nil, nil, errors.Errorf(invalidScanType, scannerType) } - PrintIfVerbose(fmt.Sprintf("Fetching the predicate history for SimilarityId : %s", similarityID)) + logger.PrintIfVerbose(fmt.Sprintf("Fetching the predicate history for SimilarityId : %s", similarityID)) r.SetPath(triageAPIPath) var request = "/" + similarityID + "?project-ids=" + projectID - PrintIfVerbose(fmt.Sprintf("Sending GET request to %s", r.path+request)) + logger.PrintIfVerbose(fmt.Sprintf("Sending GET request to %s", r.path+request)) return handleResponseWithBody(SendHTTPRequest(http.MethodGet, r.path+request, nil, true, clientTimeout)) } @@ -73,8 +74,8 @@ func (r ResultsPredicatesHTTPWrapper) PredicateSeverityAndState(predicate *Predi return nil, errors.Errorf(invalidScanType, predicate.ScannerType) } - PrintIfVerbose(fmt.Sprintf("Sending POST request to %s", triageAPIPath)) - PrintIfVerbose(fmt.Sprintf("Request Payload: %s", string(jsonBytes))) + logger.PrintIfVerbose(fmt.Sprintf("Sending POST request to %s", triageAPIPath)) + logger.PrintIfVerbose(fmt.Sprintf("Request Payload: %s", string(jsonBytes))) r.SetPath(triageAPIPath) @@ -83,7 +84,7 @@ func (r ResultsPredicatesHTTPWrapper) PredicateSeverityAndState(predicate *Predi return nil, err } - PrintIfVerbose(fmt.Sprintf("Response : %s ", resp.Status)) + logger.PrintIfVerbose(fmt.Sprintf("Response : %s ", resp.Status)) defer func() { _ = resp.Body.Close() @@ -111,7 +112,7 @@ func handleResponseWithBody(resp *http.Response, err error) (*PredicatesCollecti return nil, nil, err } - PrintIfVerbose(fmt.Sprintf("Response : %s", resp.Status)) + logger.PrintIfVerbose(fmt.Sprintf("Response : %s", resp.Status)) decoder := json.NewDecoder(resp.Body) From 082a41605e1225c304889e67bbbc6ef8072da303 Mon Sep 17 00:00:00 2001 From: Pedro Lopes Date: Mon, 2 May 2022 10:02:18 +0100 Subject: [PATCH 2/4] fix linter --- internal/commands/project.go | 2 +- internal/commands/scan.go | 2 +- internal/logger/utils.go | 10 +++++----- internal/params/flags.go | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/internal/commands/project.go b/internal/commands/project.go index 29588231c..57afcc61b 100644 --- a/internal/commands/project.go +++ b/internal/commands/project.go @@ -351,7 +351,7 @@ func updateProjectConfigurationIfNeeded(cmd *cobra.Command, projectsWrapper wrap return sshErr } - viper.Set(commonParams.SshValue, sshKey) + viper.Set(commonParams.SSHValue, sshKey) sshKeyConf := getProjectConfiguration(sshConfKey, "sshKey", git, projOriginLevel, sshKey, "Secret", true) diff --git a/internal/commands/scan.go b/internal/commands/scan.go index a550d1ac3..a11ba9df2 100644 --- a/internal/commands/scan.go +++ b/internal/commands/scan.go @@ -1159,7 +1159,7 @@ func defineSSHCredentials(sshKeyPath string, handler *wrappers.ScanHandler) erro if err != nil { return err } - viper.Set(commonParams.SshValue, sshKey) + viper.Set(commonParams.SSHValue, sshKey) credentials := wrappers.GitCredentials{} diff --git a/internal/logger/utils.go b/internal/logger/utils.go index 6763453b3..df191e85e 100644 --- a/internal/logger/utils.go +++ b/internal/logger/utils.go @@ -14,7 +14,7 @@ import ( var sanitizeFlags = []string{ params.AstAPIKey, params.AccessKeyIDConfigKey, params.AccessKeySecretConfigKey, - params.AstToken, params.SshValue, + params.AstToken, params.SSHValue, params.SCMTokenFlag, } @@ -46,16 +46,16 @@ func PrintResponse(r *http.Response) { PrintIfVerbose(string(requestDump)) } -func sanitizeLogs(log string) string { - if strings.Contains(log, "access_token") && strings.Contains(log, "token_type") { +func sanitizeLogs(msg string) string { + if strings.Contains(msg, "access_token") && strings.Contains(msg, "token_type") { return "" } for _, flag := range sanitizeFlags { value := viper.GetString(flag) if len(value) > 0 { - log = strings.ReplaceAll(log, value, "***") + msg = strings.ReplaceAll(msg, value, "***") } } - return log + return msg } diff --git a/internal/params/flags.go b/internal/params/flags.go index f297307c2..4cb7ec34a 100644 --- a/internal/params/flags.go +++ b/internal/params/flags.go @@ -101,7 +101,7 @@ const ( SSHKeyFlag = "ssh-key" RepoURLFlag = "repo-url" AstToken = "ast-token" - SshValue = "ssh-value" + SSHValue = "ssh-value" SastFilterFlag = "sast-filter" SastFilterUsage = "SAST filter" KicsFilterFlag = "kics-filter" From c61a3eab93712f9627e9c1169db03f1977bbbb33 Mon Sep 17 00:00:00 2001 From: Pedro Lopes Date: Mon, 2 May 2022 14:23:11 +0100 Subject: [PATCH 3/4] improve body print --- internal/commands/auth.go | 2 ++ internal/logger/utils.go | 9 +++------ internal/wrappers/azure-http.go | 2 +- internal/wrappers/bitbucket-http.go | 2 +- internal/wrappers/client.go | 12 ++++++++++-- internal/wrappers/github-http.go | 2 +- internal/wrappers/gitlab-http.go | 2 +- 7 files changed, 19 insertions(+), 12 deletions(-) diff --git a/internal/commands/auth.go b/internal/commands/auth.go index 35be84f64..fa8378dd0 100644 --- a/internal/commands/auth.go +++ b/internal/commands/auth.go @@ -134,11 +134,13 @@ func runRegister(authWrapper wrappers.AuthWrapper) func(cmd *cobra.Command, args if username == "" { return errors.Errorf(pleaseProvideFlag, failedCreatingClient, params.UsernameFlag) } + viper.Set(params.UsernameFlag, username) password, _ := cmd.Flags().GetString(params.PasswordFlag) if password == "" { return errors.Errorf(pleaseProvideFlag, failedCreatingClient, params.PasswordFlag) } + viper.Set(params.PasswordFlag, password) roles, _ := cmd.Flags().GetStringSlice(params.ClientRolesFlag) err := validateRoles(roles) diff --git a/internal/logger/utils.go b/internal/logger/utils.go index df191e85e..dd5349193 100644 --- a/internal/logger/utils.go +++ b/internal/logger/utils.go @@ -14,6 +14,7 @@ import ( var sanitizeFlags = []string{ params.AstAPIKey, params.AccessKeyIDConfigKey, params.AccessKeySecretConfigKey, + params.UsernameFlag, params.PasswordFlag, params.AstToken, params.SSHValue, params.SCMTokenFlag, } @@ -37,9 +38,9 @@ func PrintRequest(r *http.Request) { PrintIfVerbose(string(requestDump)) } -func PrintResponse(r *http.Response) { +func PrintResponse(r *http.Response, body bool) { PrintIfVerbose("Receiving API response:") - requestDump, err := httputil.DumpResponse(r, true) + requestDump, err := httputil.DumpResponse(r, body) if err != nil { fmt.Println(err) } @@ -47,10 +48,6 @@ func PrintResponse(r *http.Response) { } func sanitizeLogs(msg string) string { - if strings.Contains(msg, "access_token") && strings.Contains(msg, "token_type") { - return "" - } - for _, flag := range sanitizeFlags { value := viper.GetString(flag) if len(value) > 0 { diff --git a/internal/wrappers/azure-http.go b/internal/wrappers/azure-http.go index 5fc6115fa..ba4f610ca 100644 --- a/internal/wrappers/azure-http.go +++ b/internal/wrappers/azure-http.go @@ -109,7 +109,7 @@ func (g *AzureHTTPWrapper) get(url, token string, target interface{}, queryParam _ = resp.Body.Close() }() - logger.PrintResponse(resp) + logger.PrintResponse(resp, true) switch resp.StatusCode { case http.StatusOK: diff --git a/internal/wrappers/bitbucket-http.go b/internal/wrappers/bitbucket-http.go index c769afe34..73c5e4381 100644 --- a/internal/wrappers/bitbucket-http.go +++ b/internal/wrappers/bitbucket-http.go @@ -304,7 +304,7 @@ func getBitBucket(client *http.Client, token, url string, target interface{}, qu _ = resp.Body.Close() }() - logger.PrintResponse(resp) + logger.PrintResponse(resp, true) switch resp.StatusCode { case http.StatusOK: diff --git a/internal/wrappers/client.go b/internal/wrappers/client.go index 68051fa88..d4d236b63 100644 --- a/internal/wrappers/client.go +++ b/internal/wrappers/client.go @@ -395,7 +395,7 @@ func getNewToken(credentialsPayload, authServerURI string) (*string, error) { clientTimeout := viper.GetUint(commonParams.ClientTimeoutKey) client := getClient(clientTimeout) - res, err := doRequest(client, req) + res, err := doPrivateRequest(client, req) if err != nil { return nil, errors.Errorf("%s %s", checkmarxURLError, GetAuthURL("")) } @@ -453,7 +453,15 @@ func getPasswordCredentialsPayload(username, password, adminClientID, adminClien ) } +func doPrivateRequest(client *http.Client, req *http.Request) (*http.Response, error) { + return request(client, req, false) +} + func doRequest(client *http.Client, req *http.Request) (*http.Response, error) { + return request(client, req, true) +} + +func request(client *http.Client, req *http.Request, responseBody bool) (*http.Response, error) { var err error var resp *http.Response retryLimit := int(viper.GetUint(commonParams.RetryFlag)) @@ -465,7 +473,7 @@ func doRequest(client *http.Client, req *http.Request) (*http.Response, error) { try+tryPrintOffset, retryLimit+retryLimitPrintOffset)) resp, err = client.Do(req) if resp != nil && err == nil { - logger.PrintResponse(resp) + logger.PrintResponse(resp, responseBody) return resp, nil } logger.PrintIfVerbose(fmt.Sprintf("Request failed in attempt %d", try+tryPrintOffset)) diff --git a/internal/wrappers/github-http.go b/internal/wrappers/github-http.go index 75e7bebe1..7f4221d9c 100644 --- a/internal/wrappers/github-http.go +++ b/internal/wrappers/github-http.go @@ -262,7 +262,7 @@ func get(client *http.Client, url string, target interface{}, queryParams map[st continue } - logger.PrintResponse(resp) + logger.PrintResponse(resp, true) switch resp.StatusCode { case http.StatusOK: diff --git a/internal/wrappers/gitlab-http.go b/internal/wrappers/gitlab-http.go index 3dade4123..c215981ae 100644 --- a/internal/wrappers/gitlab-http.go +++ b/internal/wrappers/gitlab-http.go @@ -158,7 +158,7 @@ func getFromGitLab( continue } - logger.PrintResponse(resp) + logger.PrintResponse(resp, true) switch resp.StatusCode { case http.StatusOK: From c2219d4eacba9af729d9a3c30c209bc345f03c1d Mon Sep 17 00:00:00 2001 From: Pedro Lopes Date: Tue, 3 May 2022 09:39:47 +0100 Subject: [PATCH 4/4] remove comment --- internal/commands/project.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/commands/project.go b/internal/commands/project.go index 57afcc61b..0cf907ff2 100644 --- a/internal/commands/project.go +++ b/internal/commands/project.go @@ -438,7 +438,6 @@ func runListProjectsCommand(projectsWrapper wrappers.ProjectsWrapper) func(cmd * if err != nil { return errors.Wrapf(err, "%s\n", failedGettingAll) } - // // Checking the response if errorModel != nil {