diff --git a/.pre-commit-hooks.yaml b/.pre-commit-hooks.yaml index ab69bf3f..c50e4d73 100644 --- a/.pre-commit-hooks.yaml +++ b/.pre-commit-hooks.yaml @@ -10,3 +10,9 @@ language_version: python3 entry: cycode args: [ '-o', 'text', '--no-progress-meter', 'scan', '-t', 'sca', 'pre-commit' ] +- id: cycode-sast + name: Cycode SAST pre-commit defender + language: python + language_version: python3 + entry: cycode + args: [ '-o', 'text', '--no-progress-meter', 'scan', '-t', 'sast', 'pre-commit' ] diff --git a/README.md b/README.md index f5945134..f3e57c10 100644 --- a/README.md +++ b/README.md @@ -86,9 +86,15 @@ To install the Cycode CLI application on your local machine, perform the followi brew install cycode ``` -3. Navigate to the top directory of the local repository you wish to scan. + - To install from [GitHub Releases](https://github.com/cycodehq/cycode-cli/releases) navigate and download executable for your operating system and architecture, then run the following command: -4. There are three methods to set the Cycode client ID and client secret: + ```bash + cd /path/to/downloaded/cycode-cli + chmod +x cycode + ./cycode + ``` + +3. Authenticate CLI. There are three methods to set the Cycode client ID and client secret: - [cycode auth](#using-the-auth-command) (**Recommended**) - [cycode configure](#using-the-configure-command) @@ -205,7 +211,7 @@ export CYCODE_CLIENT_SECRET={your Cycode Secret Key} Cycodeโ€™s pre-commit hook can be set up within your local repository so that the Cycode CLI application will identify any issues with your code automatically before you commit it to your codebase. > [!NOTE] -> pre-commit hook is only available to Secrets and SCA scans. +> pre-commit hook is not available for IaC scans. Perform the following steps to install the pre-commit hook: @@ -222,19 +228,19 @@ Perform the following steps to install the pre-commit hook: ```yaml repos: - repo: https://github.com/cycodehq/cycode-cli - rev: v3.0.0 + rev: v3.2.0 hooks: - id: cycode stages: - pre-commit ``` -4. Modify the created file for your specific needs. Use hook ID `cycode` to enable scan for Secrets. Use hook ID `cycode-sca` to enable SCA scan. If you want to enable both, use this configuration: +4. Modify the created file for your specific needs. Use hook ID `cycode` to enable scan for Secrets. Use hook ID `cycode-sca` to enable SCA scan. Use hook ID `cycode-sast` to enable SAST scan. If you want to enable all scanning types, use this configuration: ```yaml repos: - repo: https://github.com/cycodehq/cycode-cli - rev: v3.0.0 + rev: v3.2.0 hooks: - id: cycode stages: @@ -242,6 +248,9 @@ Perform the following steps to install the pre-commit hook: - id: cycode-sca stages: - pre-commit + - id: cycode-sast + stages: + - pre-commit ``` 5. Install Cycodeโ€™s hook: @@ -268,14 +277,17 @@ Perform the following steps to install the pre-commit hook: The following are the options and commands available with the Cycode CLI application: -| Option | Description | -|--------------------------------------|------------------------------------------------------------------------| -| `-v`, `--verbose` | Show detailed logs. | -| `--no-progress-meter` | Do not show the progress meter. | -| `--no-update-notifier` | Do not check CLI for updates. | -| `-o`, `--output [text\|json\|table]` | Specify the output (`text`/`json`/`table`). The default is `text`. | -| `--user-agent TEXT` | Characteristic JSON object that lets servers identify the application. | -| `--help` | Show options for given command. | +| Option | Description | +|-------------------------------------------------------------------|------------------------------------------------------------------------------------| +| `-v`, `--verbose` | Show detailed logs. | +| `--no-progress-meter` | Do not show the progress meter. | +| `--no-update-notifier` | Do not check CLI for updates. | +| `-o`, `--output [rich\|text\|json\|table]` | Specify the output type. The default is `rich`. | +| `--client-id TEXT` | Specify a Cycode client ID for this specific scan execution. | +| `--client-secret TEXT` | Specify a Cycode client secret for this specific scan execution. | +| `--install-completion` | Install completion for the current shell.. | +| `--show-completion [bash\|zsh\|fish\|powershell\|pwsh]` | Show completion for the specified shell, to copy it or customize the installation. | +| `-h`, `--help` | Show options for given command. | | Command | Description | |-------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------| @@ -295,8 +307,6 @@ The Cycode CLI application offers several types of scans so that you can choose | Option | Description | |------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------| | `-t, --scan-type [secret\|iac\|sca\|sast]` | Specify the scan you wish to execute (`secret`/`iac`/`sca`/`sast`), the default is `secret`. | -| `--client-secret TEXT` | Specify a Cycode client secret for this specific scan execution. | -| `--client-id TEXT` | Specify a Cycode client ID for this specific scan execution. | | `--show-secret BOOLEAN` | Show secrets in plain text. See [Show/Hide Secrets](#showhide-secrets) section for more details. | | `--soft-fail BOOLEAN` | Run scan without failing, always return a non-error status code. See [Soft Fail](#soft-fail) section for more details. | | `--severity-threshold [INFO\|LOW\|MEDIUM\|HIGH\|CRITICAL]` | Show only violations at the specified level or higher. | @@ -500,15 +510,7 @@ If no issues are found, the scan ends with the following success message: `Good job! No issues were found!!! ๐Ÿ‘๐Ÿ‘๐Ÿ‘` -If an issue is found, a `Found issue of type:` message appears upon completion instead: - -```bash -โ›” Found issue of type: generic-password (rule ID: ce3a4de0-9dfc-448b-a004-c538cf8b4710) in file: config/my_config.py -Secret SHA: a44081db3296c84b82d12a35c446a3cba19411dddfa0380134c75f7b3973bff0 โ›” -0 | @@ -0,0 +1 @@ -1 | +my_password = 'h3l***********350' -2 | \ No newline at end of file -``` +If an issue is found, a violation card appears upon completion instead. If an issue is found, review the file in question for the specific line highlighted by the result message. Implement any changes required to resolve the issue, then execute the scan again. @@ -524,15 +526,7 @@ In the following example, a Path Scan is executed against the `cli` subdirectory `cycode scan --show-secret path ./cli` -The result would then not be obfuscated: - -```bash -โ›” Found issue of type: generic-password (rule ID: ce3a4de0-9dfc-448b-a004-c538cf8b4710) in file: config/my_config.py -Secret SHA: a44081db3296c84b82d12a35c446a3cba19411dddfa0380134c75f7b3973bff0 โ›” -0 | @@ -0,0 +1 @@ -1 | +my_password = 'h3110w0r1d!@#$350' -2 | \ No newline at end of file -``` +The result would then not be obfuscated. ### Soft Fail @@ -548,41 +542,92 @@ Scan results are assigned with a value of exit code `1` when issues are found in #### Secrets Result Example ```bash -โ›” Found issue of type: generic-password (rule ID: ce3a4de0-9dfc-448b-a004-c538cf8b4710) in file: config/my_config.py -Secret SHA: a44081db3296c84b82d12a35c446a3cba19411dddfa0380134c75f7b3973bff0 โ›” -0 | @@ -0,0 +1 @@ -1 | +my_password = 'h3l***********350' -2 | \ No newline at end of file +โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ Hardcoded generic-password is used โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ +โ”‚ Violation 12 of 12 โ”‚ +โ”‚ โ•ญโ”€ ๐Ÿ” Details โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ โ•ญโ”€ ๐Ÿ’ป Code Snippet โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ โ”‚ +โ”‚ โ”‚ Severity ๐ŸŸ  MEDIUM โ”‚ โ”‚ 34 }; โ”‚ โ”‚ +โ”‚ โ”‚ In file /Users/cycodemacuser/NodeGoat/test/s โ”‚ โ”‚ 35 โ”‚ โ”‚ +โ”‚ โ”‚ ecurity/profile-test.js โ”‚ โ”‚ 36 var sutUserName = "user1"; โ”‚ โ”‚ +โ”‚ โ”‚ Secret SHA b4ea3116d868b7c982ee6812cce61727856b โ”‚ โ”‚ โฑ 37 var sutUserPassword = "Us*****23"; โ”‚ โ”‚ +โ”‚ โ”‚ 802b3063cd5aebe7d796988552e0 โ”‚ โ”‚ 38 โ”‚ โ”‚ +โ”‚ โ”‚ Rule ID 68b6a876-4890-4e62-9531-0e687223579f โ”‚ โ”‚ 39 chrome.setDefaultService(service); โ”‚ โ”‚ +โ”‚ โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ โ”‚ 40 โ”‚ โ”‚ +โ”‚ โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ โ”‚ +โ”‚ โ•ญโ”€ ๐Ÿ“ Summary โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ โ”‚ +โ”‚ โ”‚ A generic secret or password is an authentication token used to access a computer or application and is assigned to a password variable. โ”‚ โ”‚ +โ”‚ โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ โ”‚ +โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ ``` #### IaC Result Example ```bash -โ›” Found issue of type: Resource should use non-default namespace (rule ID: bdaa88e2-5e7c-46ff-ac2a-29721418c59c) in file: ./k8s/k8s.yaml โ›” - -7 | name: secrets-file -8 | namespace: default -9 | resourceVersion: "4228" +โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ Enable Content Encoding through the attribute 'MinimumCompressionSize'. This value should be greater than -1 and smaller than 10485760. โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ +โ”‚ Violation 45 of 110 โ”‚ +โ”‚ โ•ญโ”€ ๐Ÿ” Details โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ โ•ญโ”€ ๐Ÿ’ป Code Snippet โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ โ”‚ +โ”‚ โ”‚ Severity ๐ŸŸ  MEDIUM โ”‚ โ”‚ 20 BinaryMediaTypes: โ”‚ โ”‚ +โ”‚ โ”‚ In file ...ads-copy/iac/cft/api-gateway/ap โ”‚ โ”‚ 21 - !Ref binaryMediaType1 โ”‚ โ”‚ +โ”‚ โ”‚ i-gateway-rest-api/deploy.yml โ”‚ โ”‚ 22 - !Ref binaryMediaType2 โ”‚ โ”‚ +โ”‚ โ”‚ IaC Provider CloudFormation โ”‚ โ”‚ โฑ 23 MinimumCompressionSize: -1 โ”‚ โ”‚ +โ”‚ โ”‚ Rule ID 33c4b90c-3270-4337-a075-d3109c141b โ”‚ โ”‚ 24 EndpointConfiguration: โ”‚ โ”‚ +โ”‚ โ”‚ 53 โ”‚ โ”‚ 25 Types: โ”‚ โ”‚ +โ”‚ โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ โ”‚ 26 - EDGE โ”‚ โ”‚ +โ”‚ โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ โ”‚ +โ”‚ โ•ญโ”€ ๐Ÿ“ Summary โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ โ”‚ +โ”‚ โ”‚ This policy validates the proper configuration of content encoding in AWS API Gateway. Specifically, the policy checks for the attribute โ”‚ โ”‚ +โ”‚ โ”‚ 'minimum_compression_size' in API Gateway REST APIs. Correct configuration of this attribute is important for enabling content encoding of API responses for โ”‚ โ”‚ +โ”‚ โ”‚ improved API performance and reduced payload sizes. โ”‚ โ”‚ +โ”‚ โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ โ”‚ +โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ ``` #### SCA Result Example ```bash -โ›” Found issue of type: Security vulnerability in package 'pyyaml' referenced in project 'Users/myuser/my-test-repo': Improper Input Validation in PyYAML (rule ID: d003b23a-a2eb-42f3-83c9-7a84505603e5) in file: Users/myuser/my-test-repo/requirements.txt โ›” - -1 | PyYAML~=5.3.1 -2 | vyper==0.3.1 -3 | cleo==1.0.0a5 +โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ [CVE-2019-10795] Prototype Pollution in undefsafe โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ +โ”‚ Violation 172 of 195 โ”‚ +โ”‚ โ•ญโ”€ ๐Ÿ” Details โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ โ•ญโ”€ ๐Ÿ’ป Code Snippet โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ โ”‚ +โ”‚ โ”‚ Severity ๐ŸŸ  MEDIUM โ”‚ โ”‚ 26758 "integrity": "sha1-5z3T17DXxe2G+6xrCufYxqadUPo=", โ”‚ โ”‚ +โ”‚ โ”‚ In file /Users/cycodemacuser/Node โ”‚ โ”‚ 26759 "dev": true โ”‚ โ”‚ +โ”‚ โ”‚ Goat/package-lock.json โ”‚ โ”‚ 26760 }, โ”‚ โ”‚ +โ”‚ โ”‚ CVEs CVE-2019-10795 โ”‚ โ”‚ โฑ 26761 "undefsafe": { โ”‚ โ”‚ +โ”‚ โ”‚ Package undefsafe โ”‚ โ”‚ 26762 "version": "2.0.2", โ”‚ โ”‚ +โ”‚ โ”‚ Version 2.0.2 โ”‚ โ”‚ 26763 "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.2.tgz", โ”‚ โ”‚ +โ”‚ โ”‚ First patched version Not fixed โ”‚ โ”‚ 26764 "integrity": "sha1-Il9rngM3Zj4Njnz9aG/Cg2zKznY=", โ”‚ โ”‚ +โ”‚ โ”‚ Dependency path nodemon 1.19.1 -> โ”‚ โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ โ”‚ +โ”‚ โ”‚ undefsafe 2.0.2 โ”‚ โ”‚ +โ”‚ โ”‚ Rule ID 9c6a8911-e071-4616-86db-4 โ”‚ โ”‚ +โ”‚ โ”‚ 943f2e1df81 โ”‚ โ”‚ +โ”‚ โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ โ”‚ +โ”‚ โ•ญโ”€ ๐Ÿ“ Summary โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ โ”‚ +โ”‚ โ”‚ undefsafe before 2.0.3 is vulnerable to Prototype Pollution. The 'a' function could be tricked into adding or modifying properties of Object.prototype using โ”‚ โ”‚ +โ”‚ โ”‚ a __proto__ payload. โ”‚ โ”‚ +โ”‚ โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ โ”‚ +โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ ``` #### SAST Result Example ```bash -โ›” Found issue of type: Detected a request using 'http://'. This request will be unencrypted, and attackers could listen into traffic on the network and be able to obtain sensitive information. Use 'https://' instead. (rule ID: 3fbbd34b-b00d-4415-b9d9-f861c076b9f2) in file: ./requests.py โ›” - -2 | -3 | res = requests.get('http://example.com', timeout=1) -4 | print(res.content) +โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ [CWE-208: Observable Timing Discrepancy] Observable Timing Discrepancy โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ +โ”‚ Violation 24 of 49 โ”‚ +โ”‚ โ•ญโ”€ ๐Ÿ” Details โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ โ•ญโ”€ ๐Ÿ’ป Code Snippet โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ โ”‚ +โ”‚ โ”‚ Severity ๐ŸŸ  MEDIUM โ”‚ โ”‚ 173 " including numbers, lowercase and uppercase letters."; โ”‚ โ”‚ +โ”‚ โ”‚ In file /Users/cycodemacuser/NodeGoat/app โ”‚ โ”‚ 174 return false; โ”‚ โ”‚ +โ”‚ โ”‚ /routes/session.js โ”‚ โ”‚ 175 } โ”‚ โ”‚ +โ”‚ โ”‚ CWE CWE-208 โ”‚ โ”‚ โฑ 176 if (password !== verify) { โ”‚ โ”‚ +โ”‚ โ”‚ Subcategory Security โ”‚ โ”‚ 177 errors.verifyError = "Password must match"; โ”‚ โ”‚ +โ”‚ โ”‚ Language js โ”‚ โ”‚ 178 return false; โ”‚ โ”‚ +โ”‚ โ”‚ Security Tool Bearer (Powered by Cycode) โ”‚ โ”‚ 179 } โ”‚ โ”‚ +โ”‚ โ”‚ Rule ID 19fbca07-a8e7-4fa6-92ac-a36d15509 โ”‚ โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ โ”‚ +โ”‚ โ”‚ fa9 โ”‚ โ”‚ +โ”‚ โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ โ”‚ +โ”‚ โ•ญโ”€ ๐Ÿ“ Summary โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ โ”‚ +โ”‚ โ”‚ Observable Timing Discrepancy occurs when the time it takes for certain operations to complete can be measured and observed by attackers. This vulnerability โ”‚ โ”‚ +โ”‚ โ”‚ is particularly concerning when operations involve sensitive information, such as password checks or secret comparisons. If attackers can analyze how long โ”‚ โ”‚ +โ”‚ โ”‚ these operations take, they might be able to deduce confidential details, putting your data at risk. โ”‚ โ”‚ +โ”‚ โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ โ”‚ +โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ ``` ### Companyโ€™s Custom Remediation Guidelines @@ -609,18 +654,6 @@ The following are the options available for the `cycode ignore` command: | `-t, --scan-type [secret\|iac\|sca\|sast]` | Specify the scan you wish to execute (`secret`/`iac`/`sca`/`sast`). The default value is `secret`. | | `-g, --global` | Add an ignore rule and update it in the global `.cycode` config file. | -In the following example, a pre-commit scan runs and finds the following: - -```bash -โ›” Found issue of type: generic-password (rule ID: ce3a4de0-9dfc-448b-a004-c538cf8b4710) in file: config/my_config.py -Secret SHA: a44081db3296c84b82d12a35c446a3cba19411dddfa0380134c75f7b3973bff0 โ›” -0 | @@ -0,0 +1 @@ -1 | +my_password = 'h3l***********350' -2 | \ No newline at end of file -``` - -If this is a value that is not a valid secret, then use the `cycode ignore` command to ignore the secret by its value, SHA value, specific path, or rule ID. If this is an IaC scan, then you can ignore that result by its path or rule ID. - ### Ignoring a Secret Value To ignore a specific secret value, you will need to use the `--by-value` flag. This will ignore the given secret value from all future scans. Use the following command to add a secret value to be ignored: diff --git a/cycode/cli/apps/report/sbom/path/path_command.py b/cycode/cli/apps/report/sbom/path/path_command.py index 9741aa73..9c839b08 100644 --- a/cycode/cli/apps/report/sbom/path/path_command.py +++ b/cycode/cli/apps/report/sbom/path/path_command.py @@ -8,7 +8,7 @@ from cycode.cli.apps.report.sbom.common import create_sbom_report, send_report_feedback from cycode.cli.exceptions.handle_report_sbom_errors import handle_report_exception from cycode.cli.files_collector.path_documents import get_relevant_documents -from cycode.cli.files_collector.sca.sca_code_scanner import perform_pre_scan_documents_actions +from cycode.cli.files_collector.sca.sca_file_collector import add_sca_dependencies_tree_documents_if_needed from cycode.cli.files_collector.zip_documents import zip_documents from cycode.cli.utils.get_api_client import get_report_cycode_client from cycode.cli.utils.progress_bar import SbomReportProgressBarSection @@ -41,7 +41,7 @@ def path_command( ) # TODO(MarshalX): combine perform_pre_scan_documents_actions with get_relevant_document. # unhardcode usage of context in perform_pre_scan_documents_actions - perform_pre_scan_documents_actions(ctx, consts.SCA_SCAN_TYPE, documents) + add_sca_dependencies_tree_documents_if_needed(ctx, consts.SCA_SCAN_TYPE, documents) zipped_documents = zip_documents(consts.SCA_SCAN_TYPE, documents) report_execution = client.request_sbom_report_execution(report_parameters, zip_file=zipped_documents) diff --git a/cycode/cli/apps/scan/aggregation_report.py b/cycode/cli/apps/scan/aggregation_report.py new file mode 100644 index 00000000..45b891ed --- /dev/null +++ b/cycode/cli/apps/scan/aggregation_report.py @@ -0,0 +1,42 @@ +from typing import TYPE_CHECKING, Optional + +import typer + +from cycode.logger import get_logger + +if TYPE_CHECKING: + from cycode.cyclient.scan_client import ScanClient + +logger = get_logger('Aggregation Report URL') + + +def _set_aggregation_report_url(ctx: typer.Context, aggregation_report_url: Optional[str] = None) -> None: + ctx.obj['aggregation_report_url'] = aggregation_report_url + + +def try_get_aggregation_report_url_if_needed( + scan_parameters: dict, cycode_client: 'ScanClient', scan_type: str +) -> Optional[str]: + if not scan_parameters.get('report', False): + return None + + aggregation_id = scan_parameters.get('aggregation_id') + if aggregation_id is None: + return None + + try: + report_url_response = cycode_client.get_scan_aggregation_report_url(aggregation_id, scan_type) + return report_url_response.report_url + except Exception as e: + logger.debug('Failed to get aggregation report url: %s', str(e)) + + +def try_set_aggregation_report_url_if_needed( + ctx: typer.Context, scan_parameters: dict, cycode_client: 'ScanClient', scan_type: str +) -> None: + aggregation_report_url = try_get_aggregation_report_url_if_needed(scan_parameters, cycode_client, scan_type) + if aggregation_report_url: + _set_aggregation_report_url(ctx, aggregation_report_url) + logger.debug('Aggregation report URL set successfully', {'aggregation_report_url': aggregation_report_url}) + else: + logger.debug('No aggregation report URL found or report generation is disabled') diff --git a/cycode/cli/apps/scan/code_scanner.py b/cycode/cli/apps/scan/code_scanner.py index 21b5959e..19b43733 100644 --- a/cycode/cli/apps/scan/code_scanner.py +++ b/cycode/cli/apps/scan/code_scanner.py @@ -1,45 +1,33 @@ -import logging -import os -import sys import time from platform import platform from typing import TYPE_CHECKING, Callable, Optional -from uuid import UUID, uuid4 -import click import typer from cycode.cli import consts -from cycode.cli.cli_types import SeverityOption +from cycode.cli.apps.scan.aggregation_report import try_set_aggregation_report_url_if_needed +from cycode.cli.apps.scan.scan_parameters import get_scan_parameters +from cycode.cli.apps.scan.scan_result import ( + create_local_scan_result, + get_scan_result, + get_sync_scan_result, + print_local_scan_results, +) from cycode.cli.config import configuration_manager -from cycode.cli.console import console from cycode.cli.exceptions import custom_exceptions from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception -from cycode.cli.files_collector.excluder import excluder -from cycode.cli.files_collector.models.in_memory_zip import InMemoryZip from cycode.cli.files_collector.path_documents import get_relevant_documents -from cycode.cli.files_collector.repository_documents import ( - get_commit_range_modified_documents, - get_diff_file_path, - get_pre_commit_modified_documents, - parse_commit_range, -) -from cycode.cli.files_collector.sca import sca_code_scanner -from cycode.cli.files_collector.sca.sca_code_scanner import perform_pre_scan_documents_actions +from cycode.cli.files_collector.sca.sca_file_collector import add_sca_dependencies_tree_documents_if_needed from cycode.cli.files_collector.zip_documents import zip_documents -from cycode.cli.models import CliError, Document, DocumentDetections, LocalScanResult -from cycode.cli.utils import scan_utils -from cycode.cli.utils.git_proxy import git_proxy -from cycode.cli.utils.path_utils import get_path_by_os +from cycode.cli.models import CliError, Document, LocalScanResult from cycode.cli.utils.progress_bar import ScanProgressBarSection from cycode.cli.utils.scan_batch import run_parallel_batched_scan -from cycode.cli.utils.scan_utils import set_issue_detected -from cycode.cli.utils.shell_executor import shell -from cycode.cyclient.models import Detection, DetectionSchema, DetectionsPerFile, ZippedFileScanResult -from cycode.logger import get_logger, set_logging_level +from cycode.cli.utils.scan_utils import generate_unique_scan_id, set_issue_detected_by_scan_results +from cycode.cyclient.models import ZippedFileScanResult +from cycode.logger import get_logger if TYPE_CHECKING: - from cycode.cyclient.models import ScanDetailsResponse + from cycode.cli.files_collector.models.in_memory_zip import InMemoryZip from cycode.cyclient.scan_client import ScanClient start_scan_time = time.time() @@ -48,60 +36,18 @@ logger = get_logger('Code Scanner') -def scan_sca_pre_commit(ctx: typer.Context, repo_path: str) -> None: - scan_type = ctx.obj['scan_type'] - scan_parameters = get_scan_parameters(ctx) - git_head_documents, pre_committed_documents = get_pre_commit_modified_documents( - progress_bar=ctx.obj['progress_bar'], - progress_bar_section=ScanProgressBarSection.PREPARE_LOCAL_FILES, - repo_path=repo_path, - ) - git_head_documents = excluder.exclude_irrelevant_documents_to_scan(scan_type, git_head_documents) - pre_committed_documents = excluder.exclude_irrelevant_documents_to_scan(scan_type, pre_committed_documents) - sca_code_scanner.perform_pre_hook_range_scan_actions(repo_path, git_head_documents, pre_committed_documents) - scan_commit_range_documents( - ctx, - git_head_documents, - pre_committed_documents, - scan_parameters, - configuration_manager.get_sca_pre_commit_timeout_in_seconds(), - ) - - -def scan_sca_commit_range(ctx: typer.Context, path: str, commit_range: str) -> None: - scan_type = ctx.obj['scan_type'] - progress_bar = ctx.obj['progress_bar'] - - scan_parameters = get_scan_parameters(ctx, (path,)) - from_commit_rev, to_commit_rev = parse_commit_range(commit_range, path) - from_commit_documents, to_commit_documents = get_commit_range_modified_documents( - progress_bar, ScanProgressBarSection.PREPARE_LOCAL_FILES, path, from_commit_rev, to_commit_rev - ) - from_commit_documents = excluder.exclude_irrelevant_documents_to_scan(scan_type, from_commit_documents) - to_commit_documents = excluder.exclude_irrelevant_documents_to_scan(scan_type, to_commit_documents) - sca_code_scanner.perform_pre_commit_range_scan_actions( - path, from_commit_documents, from_commit_rev, to_commit_documents, to_commit_rev - ) - - scan_commit_range_documents(ctx, from_commit_documents, to_commit_documents, scan_parameters=scan_parameters) - - def scan_disk_files(ctx: typer.Context, paths: tuple[str, ...]) -> None: scan_type = ctx.obj['scan_type'] progress_bar = ctx.obj['progress_bar'] try: documents = get_relevant_documents(progress_bar, ScanProgressBarSection.PREPARE_LOCAL_FILES, scan_type, paths) - perform_pre_scan_documents_actions(ctx, scan_type, documents) + add_sca_dependencies_tree_documents_if_needed(ctx, scan_type, documents) scan_documents(ctx, documents, get_scan_parameters(ctx, paths)) except Exception as e: handle_scan_exception(ctx, e) -def set_issue_detected_by_scan_results(ctx: typer.Context, scan_results: list[LocalScanResult]) -> None: - set_issue_detected(ctx, any(scan_result.issue_detected for scan_result in scan_results)) - - def _should_use_sync_flow(command_scan_type: str, scan_type: str, sync_option: bool) -> bool: """Decide whether to use sync flow or async flow for the scan. @@ -175,7 +121,7 @@ def _scan_batch_thread_func(batch: list[Document]) -> tuple[str, CliError, Local local_scan_result = error = error_message = None detections_count = relevant_detections_count = zip_file_size = 0 - scan_id = str(_generate_unique_id()) + scan_id = str(generate_unique_scan_id()) scan_completed = False should_use_sync_flow = _should_use_sync_flow(command_scan_type, scan_type, sync_option) @@ -184,7 +130,7 @@ def _scan_batch_thread_func(batch: list[Document]) -> tuple[str, CliError, Local logger.debug('Preparing local files, %s', {'batch_files_count': len(batch)}) zipped_documents = zip_documents(scan_type, batch) zip_file_size = zipped_documents.size - scan_result = perform_scan( + scan_result = _perform_scan( cycode_client, zipped_documents, scan_type, @@ -219,7 +165,7 @@ def _scan_batch_thread_func(batch: list[Document]) -> tuple[str, CliError, Local 'zip_file_size': zip_file_size, }, ) - _report_scan_status( + report_scan_status( cycode_client, scan_type, scan_id, @@ -237,66 +183,6 @@ def _scan_batch_thread_func(batch: list[Document]) -> tuple[str, CliError, Local return _scan_batch_thread_func -def scan_commit_range( - ctx: typer.Context, path: str, commit_range: str, max_commits_count: Optional[int] = None -) -> None: - scan_type = ctx.obj['scan_type'] - - progress_bar = ctx.obj['progress_bar'] - progress_bar.start() - - if scan_type not in consts.COMMIT_RANGE_SCAN_SUPPORTED_SCAN_TYPES: - raise click.ClickException(f'Commit range scanning for {str.upper(scan_type)} is not supported') - - if scan_type == consts.SCA_SCAN_TYPE: - return scan_sca_commit_range(ctx, path, commit_range) - - documents_to_scan = [] - commit_ids_to_scan = [] - - repo = git_proxy.get_repo(path) - total_commits_count = int(repo.git.rev_list('--count', commit_range)) - logger.debug('Calculating diffs for %s commits in the commit range %s', total_commits_count, commit_range) - - progress_bar.set_section_length(ScanProgressBarSection.PREPARE_LOCAL_FILES, total_commits_count) - - for scanned_commits_count, commit in enumerate(repo.iter_commits(rev=commit_range)): - if _does_reach_to_max_commits_to_scan_limit(commit_ids_to_scan, max_commits_count): - logger.debug('Reached to max commits to scan count. Going to scan only %s last commits', max_commits_count) - progress_bar.update(ScanProgressBarSection.PREPARE_LOCAL_FILES, total_commits_count - scanned_commits_count) - break - - progress_bar.update(ScanProgressBarSection.PREPARE_LOCAL_FILES) - - commit_id = commit.hexsha - commit_ids_to_scan.append(commit_id) - parent = commit.parents[0] if commit.parents else git_proxy.get_null_tree() - diff_index = commit.diff(parent, create_patch=True, R=True) - commit_documents_to_scan = [] - for diff in diff_index: - commit_documents_to_scan.append( - Document( - path=get_path_by_os(get_diff_file_path(diff)), - content=diff.diff.decode('UTF-8', errors='replace'), - is_git_diff_format=True, - unique_id=commit_id, - ) - ) - - logger.debug( - 'Found all relevant files in commit %s', - {'path': path, 'commit_range': commit_range, 'commit_id': commit_id}, - ) - - documents_to_scan.extend(excluder.exclude_irrelevant_documents_to_scan(scan_type, commit_documents_to_scan)) - - logger.debug('List of commit ids to scan, %s', {'commit_ids': commit_ids_to_scan}) - logger.debug('Starting to scan commit range (it may take a few minutes)') - - scan_documents(ctx, documents_to_scan, get_scan_parameters(ctx, (path,)), is_git_diff=True, is_commit_range=True) - return None - - def scan_documents( ctx: typer.Context, documents_to_scan: list[Document], @@ -324,155 +210,17 @@ def scan_documents( scan_batch_thread_func, scan_type, documents_to_scan, progress_bar=progress_bar ) - aggregation_report_url = _try_get_aggregation_report_url_if_needed(scan_parameters, ctx.obj['client'], scan_type) - _set_aggregation_report_url(ctx, aggregation_report_url) + try_set_aggregation_report_url_if_needed(ctx, scan_parameters, ctx.obj['client'], scan_type) progress_bar.set_section_length(ScanProgressBarSection.GENERATE_REPORT, 1) progress_bar.update(ScanProgressBarSection.GENERATE_REPORT) progress_bar.stop() set_issue_detected_by_scan_results(ctx, local_scan_results) - print_results(ctx, local_scan_results, errors) - - -def scan_commit_range_documents( - ctx: typer.Context, - from_documents_to_scan: list[Document], - to_documents_to_scan: list[Document], - scan_parameters: Optional[dict] = None, - timeout: Optional[int] = None, -) -> None: - """In use by SCA only.""" - cycode_client = ctx.obj['client'] - scan_type = ctx.obj['scan_type'] - severity_threshold = ctx.obj['severity_threshold'] - scan_command_type = ctx.info_name - progress_bar = ctx.obj['progress_bar'] - - local_scan_result = error_message = None - scan_completed = False - scan_id = str(_generate_unique_id()) - from_commit_zipped_documents = InMemoryZip() - to_commit_zipped_documents = InMemoryZip() - - try: - progress_bar.set_section_length(ScanProgressBarSection.SCAN, 1) - - scan_result = init_default_scan_result(scan_id) - if should_scan_documents(from_documents_to_scan, to_documents_to_scan): - logger.debug('Preparing from-commit zip') - from_commit_zipped_documents = zip_documents(scan_type, from_documents_to_scan) - - logger.debug('Preparing to-commit zip') - to_commit_zipped_documents = zip_documents(scan_type, to_documents_to_scan) + print_local_scan_results(ctx, local_scan_results, errors) - scan_result = perform_commit_range_scan_async( - cycode_client, - from_commit_zipped_documents, - to_commit_zipped_documents, - scan_type, - scan_parameters, - timeout, - ) - - progress_bar.update(ScanProgressBarSection.SCAN) - progress_bar.set_section_length(ScanProgressBarSection.GENERATE_REPORT, 1) - - local_scan_result = create_local_scan_result( - scan_result, to_documents_to_scan, scan_command_type, scan_type, severity_threshold - ) - set_issue_detected_by_scan_results(ctx, [local_scan_result]) - - progress_bar.update(ScanProgressBarSection.GENERATE_REPORT) - progress_bar.stop() - # errors will be handled with try-except block; printing will not occur on errors - print_results(ctx, [local_scan_result]) - - scan_completed = True - except Exception as e: - handle_scan_exception(ctx, e) - error_message = str(e) - - zip_file_size = from_commit_zipped_documents.size + to_commit_zipped_documents.size - - detections_count = relevant_detections_count = 0 - if local_scan_result: - detections_count = local_scan_result.detections_count - relevant_detections_count = local_scan_result.relevant_detections_count - scan_id = local_scan_result.scan_id - - logger.debug( - 'Processing commit range scan results, %s', - { - 'all_violations_count': detections_count, - 'relevant_violations_count': relevant_detections_count, - 'scan_id': scan_id, - 'zip_file_size': zip_file_size, - }, - ) - _report_scan_status( - cycode_client, - scan_type, - scan_id, - scan_completed, - relevant_detections_count, - detections_count, - len(to_documents_to_scan), - zip_file_size, - scan_command_type, - error_message, - ) - - -def should_scan_documents(from_documents_to_scan: list[Document], to_documents_to_scan: list[Document]) -> bool: - return len(from_documents_to_scan) > 0 or len(to_documents_to_scan) > 0 - - -def create_local_scan_result( - scan_result: ZippedFileScanResult, - documents_to_scan: list[Document], - command_scan_type: str, - scan_type: str, - severity_threshold: str, -) -> LocalScanResult: - document_detections = get_document_detections(scan_result, documents_to_scan) - relevant_document_detections_list = exclude_irrelevant_document_detections( - document_detections, scan_type, command_scan_type, severity_threshold - ) - - detections_count = sum([len(document_detection.detections) for document_detection in document_detections]) - relevant_detections_count = sum( - [len(document_detections.detections) for document_detections in relevant_document_detections_list] - ) - - return LocalScanResult( - scan_id=scan_result.scan_id, - report_url=scan_result.report_url, - document_detections=relevant_document_detections_list, - issue_detected=len(relevant_document_detections_list) > 0, - detections_count=detections_count, - relevant_detections_count=relevant_detections_count, - ) - - -def perform_scan( - cycode_client: 'ScanClient', - zipped_documents: 'InMemoryZip', - scan_type: str, - is_git_diff: bool, - is_commit_range: bool, - scan_parameters: dict, - should_use_sync_flow: bool = False, -) -> ZippedFileScanResult: - if should_use_sync_flow: - # it does not support commit range scans; should_use_sync_flow handles it - return perform_scan_sync(cycode_client, zipped_documents, scan_type, scan_parameters, is_git_diff) - - return perform_scan_async(cycode_client, zipped_documents, scan_type, scan_parameters, is_commit_range) - - -def perform_scan_async( +def _perform_scan_async( cycode_client: 'ScanClient', zipped_documents: 'InMemoryZip', scan_type: str, @@ -492,38 +240,32 @@ def perform_scan_async( ) -def perform_scan_sync( +def _perform_scan_sync( cycode_client: 'ScanClient', zipped_documents: 'InMemoryZip', scan_type: str, scan_parameters: dict, is_git_diff: bool = False, -) -> ZippedFileScanResult: +) -> 'ZippedFileScanResult': scan_results = cycode_client.zipped_file_scan_sync(zipped_documents, scan_type, scan_parameters, is_git_diff) logger.debug('Sync scan request has been triggered successfully, %s', {'scan_id': scan_results.id}) - return ZippedFileScanResult( - did_detect=True, - detections_per_file=_map_detections_per_file_and_commit_id(scan_type, scan_results.detection_messages), - scan_id=scan_results.id, - ) + return get_sync_scan_result(scan_type, scan_results) -def perform_commit_range_scan_async( +def _perform_scan( cycode_client: 'ScanClient', - from_commit_zipped_documents: 'InMemoryZip', - to_commit_zipped_documents: 'InMemoryZip', + zipped_documents: 'InMemoryZip', scan_type: str, + is_git_diff: bool, + is_commit_range: bool, scan_parameters: dict, - timeout: Optional[int] = None, + should_use_sync_flow: bool = False, ) -> ZippedFileScanResult: - scan_async_result = cycode_client.multiple_zipped_file_scan_async( - from_commit_zipped_documents, to_commit_zipped_documents, scan_type, scan_parameters - ) + if should_use_sync_flow: + # it does not support commit range scans; should_use_sync_flow handles it + return _perform_scan_sync(cycode_client, zipped_documents, scan_type, scan_parameters, is_git_diff) - logger.debug( - 'Async commit range scan request has been triggered successfully, %s', {'scan_id': scan_async_result.scan_id} - ) - return poll_scan_results(cycode_client, scan_async_result.scan_id, scan_type, scan_parameters, timeout) + return _perform_scan_async(cycode_client, zipped_documents, scan_type, scan_parameters, is_commit_range) def poll_scan_results( @@ -532,7 +274,7 @@ def poll_scan_results( scan_type: str, scan_parameters: dict, polling_timeout: Optional[int] = None, -) -> ZippedFileScanResult: +) -> 'ZippedFileScanResult': if polling_timeout is None: polling_timeout = configuration_manager.get_scan_polling_timeout_in_seconds() @@ -544,10 +286,13 @@ def poll_scan_results( if scan_details.scan_update_at is not None and scan_details.scan_update_at != last_scan_update_at: last_scan_update_at = scan_details.scan_update_at - print_debug_scan_details(scan_details) + logger.debug('Scan update, %s', {'scan_id': scan_details.id, 'scan_status': scan_details.scan_status}) + + if scan_details.message: + logger.debug('Scan message: %s', scan_details.message) if scan_details.scan_status == consts.SCAN_STATUS_COMPLETED: - return _get_scan_result(cycode_client, scan_type, scan_id, scan_details, scan_parameters) + return get_scan_result(cycode_client, scan_type, scan_id, scan_details, scan_parameters) if scan_details.scan_status == consts.SCAN_STATUS_ERROR: raise custom_exceptions.ScanAsyncError( @@ -559,350 +304,7 @@ def poll_scan_results( raise custom_exceptions.ScanAsyncError(f'Failed to complete scan after {polling_timeout} seconds') -def print_debug_scan_details(scan_details_response: 'ScanDetailsResponse') -> None: - logger.debug( - 'Scan update, %s', {'scan_id': scan_details_response.id, 'scan_status': scan_details_response.scan_status} - ) - - if scan_details_response.message: - logger.debug('Scan message: %s', scan_details_response.message) - - -def print_results( - ctx: typer.Context, local_scan_results: list[LocalScanResult], errors: Optional[dict[str, 'CliError']] = None -) -> None: - printer = ctx.obj.get('console_printer') - printer.update_ctx(ctx) - printer.print_scan_results(local_scan_results, errors) - - -def get_document_detections( - scan_result: ZippedFileScanResult, documents_to_scan: list[Document] -) -> list[DocumentDetections]: - logger.debug('Getting document detections') - - document_detections = [] - for detections_per_file in scan_result.detections_per_file: - file_name = get_path_by_os(detections_per_file.file_name) - commit_id = detections_per_file.commit_id - - logger.debug( - 'Going to find the document of the violated file, %s', {'file_name': file_name, 'commit_id': commit_id} - ) - - document = _get_document_by_file_name(documents_to_scan, file_name, commit_id) - document_detections.append(DocumentDetections(document=document, detections=detections_per_file.detections)) - - return document_detections - - -def exclude_irrelevant_document_detections( - document_detections_list: list[DocumentDetections], - scan_type: str, - command_scan_type: str, - severity_threshold: str, -) -> list[DocumentDetections]: - relevant_document_detections_list = [] - for document_detections in document_detections_list: - relevant_detections = exclude_irrelevant_detections( - document_detections.detections, scan_type, command_scan_type, severity_threshold - ) - if relevant_detections: - relevant_document_detections_list.append( - DocumentDetections(document=document_detections.document, detections=relevant_detections) - ) - - return relevant_document_detections_list - - -def parse_pre_receive_input() -> str: - """Parse input to pushed branch update details. - - Example input: - old_value new_value refname - ----------------------------------------------- - 0000000000000000000000000000000000000000 9cf90954ef26e7c58284f8ebf7dcd0fcf711152a refs/heads/main - 973a96d3e925b65941f7c47fa16129f1577d499f 0000000000000000000000000000000000000000 refs/heads/feature-branch - 59564ef68745bca38c42fc57a7822efd519a6bd9 3378e52dcfa47fb11ce3a4a520bea5f85d5d0bf3 refs/heads/develop - - :return: First branch update details (input's first line) - """ - # FIXME(MarshalX): this blocks main thread forever if called outside of pre-receive hook - pre_receive_input = sys.stdin.read().strip() - if not pre_receive_input: - raise ValueError( - 'Pre receive input was not found. Make sure that you are using this command only in pre-receive hook' - ) - - # each line represents a branch update request, handle the first one only - # TODO(MichalBor): support case of multiple update branch requests - return pre_receive_input.splitlines()[0] - - -def _get_default_scan_parameters(ctx: typer.Context) -> dict: - return { - 'monitor': ctx.obj.get('monitor'), - 'report': ctx.obj.get('report'), - 'package_vulnerabilities': ctx.obj.get('package-vulnerabilities'), - 'license_compliance': ctx.obj.get('license-compliance'), - 'command_type': ctx.info_name.replace('-', '_'), # save backward compatibility - 'aggregation_id': str(_generate_unique_id()), - } - - -def get_scan_parameters(ctx: typer.Context, paths: Optional[tuple[str, ...]] = None) -> dict: - scan_parameters = _get_default_scan_parameters(ctx) - - if not paths: - return scan_parameters - - scan_parameters['paths'] = paths - - if len(paths) != 1: - logger.debug('Multiple paths provided, going to ignore remote url') - return scan_parameters - - if not os.path.isdir(paths[0]): - logger.debug('Path is not a directory, going to ignore remote url') - return scan_parameters - - remote_url = try_get_git_remote_url(paths[0]) - if not remote_url: - remote_url = try_to_get_plastic_remote_url(paths[0]) - - if remote_url: - # TODO(MarshalX): remove hardcode in context - ctx.obj['remote_url'] = remote_url - scan_parameters['remote_url'] = remote_url - - return scan_parameters - - -def try_get_git_remote_url(path: str) -> Optional[str]: - try: - remote_url = git_proxy.get_repo(path).remotes[0].config_reader.get('url') - logger.debug('Found Git remote URL, %s', {'remote_url': remote_url, 'path': path}) - return remote_url - except Exception: - logger.debug('Failed to get Git remote URL. Probably not a Git repository') - return None - - -def _get_plastic_repository_name(path: str) -> Optional[str]: - """Get the name of the Plastic repository from the current working directory. - - The command to execute is: - cm status --header --machinereadable --fieldseparator=":::" - - Example of status header in machine-readable format: - STATUS:::0:::Project/RepoName:::OrgName@ServerInfo - """ - try: - command = [ - 'cm', - 'status', - '--header', - '--machinereadable', - f'--fieldseparator={consts.PLASTIC_VCS_DATA_SEPARATOR}', - ] - - status = shell( - command=command, timeout=consts.PLASTIC_VSC_CLI_TIMEOUT, working_directory=path, silent_exc_info=True - ) - if not status: - logger.debug('Failed to get Plastic repository name (command failed)') - return None - - status_parts = status.split(consts.PLASTIC_VCS_DATA_SEPARATOR) - if len(status_parts) < 2: - logger.debug('Failed to parse Plastic repository name (command returned unexpected format)') - return None - - return status_parts[2].strip() - except Exception: - logger.debug('Failed to get Plastic repository name. Probably not a Plastic repository') - return None - - -def _get_plastic_repository_list(working_dir: Optional[str] = None) -> dict[str, str]: - """Get the list of Plastic repositories and their GUIDs. - - The command to execute is: - cm repo list --format="{repname}:::{repguid}" - - Example line with data: - Project/RepoName:::tapo1zqt-wn99-4752-h61m-7d9k79d40r4v - - Each line represents an individual repository. - """ - repo_name_to_guid = {} - - try: - command = ['cm', 'repo', 'ls', f'--format={{repname}}{consts.PLASTIC_VCS_DATA_SEPARATOR}{{repguid}}'] - - status = shell( - command=command, timeout=consts.PLASTIC_VSC_CLI_TIMEOUT, working_directory=working_dir, silent_exc_info=True - ) - if not status: - logger.debug('Failed to get Plastic repository list (command failed)') - return repo_name_to_guid - - status_lines = status.splitlines() - for line in status_lines: - data_parts = line.split(consts.PLASTIC_VCS_DATA_SEPARATOR) - if len(data_parts) < 2: - logger.debug('Failed to parse Plastic repository list line (unexpected format), %s', {'line': line}) - continue - - repo_name, repo_guid = data_parts - repo_name_to_guid[repo_name.strip()] = repo_guid.strip() - - return repo_name_to_guid - except Exception as e: - logger.debug('Failed to get Plastic repository list', exc_info=e) - return repo_name_to_guid - - -def try_to_get_plastic_remote_url(path: str) -> Optional[str]: - repository_name = _get_plastic_repository_name(path) - if not repository_name: - return None - - repository_map = _get_plastic_repository_list(path) - if repository_name not in repository_map: - logger.debug('Failed to get Plastic repository GUID (repository not found in the list)') - return None - - repository_guid = repository_map[repository_name] - return f'{consts.PLASTIC_VCS_REMOTE_URI_PREFIX}{repository_guid}' - - -def exclude_irrelevant_detections( - detections: list[Detection], scan_type: str, command_scan_type: str, severity_threshold: str -) -> list[Detection]: - relevant_detections = _exclude_detections_by_exclusions_configuration(detections, scan_type) - relevant_detections = _exclude_detections_by_scan_type(relevant_detections, scan_type, command_scan_type) - return _exclude_detections_by_severity(relevant_detections, severity_threshold) - - -def _exclude_detections_by_severity(detections: list[Detection], severity_threshold: str) -> list[Detection]: - relevant_detections = [] - for detection in detections: - severity = detection.severity - - if _does_severity_match_severity_threshold(severity, severity_threshold): - relevant_detections.append(detection) - else: - logger.debug( - 'Going to ignore violations because they are below the severity threshold, %s', - {'severity': severity, 'severity_threshold': severity_threshold}, - ) - - return relevant_detections - - -def _exclude_detections_by_scan_type( - detections: list[Detection], scan_type: str, command_scan_type: str -) -> list[Detection]: - if command_scan_type == consts.PRE_COMMIT_COMMAND_SCAN_TYPE: - return exclude_detections_in_deleted_lines(detections) - - exclude_in_deleted_lines = configuration_manager.get_should_exclude_detections_in_deleted_lines(command_scan_type) - if ( - command_scan_type in consts.COMMIT_RANGE_BASED_COMMAND_SCAN_TYPES - and scan_type == consts.SECRET_SCAN_TYPE - and exclude_in_deleted_lines - ): - return exclude_detections_in_deleted_lines(detections) - - return detections - - -def exclude_detections_in_deleted_lines(detections: list[Detection]) -> list[Detection]: - return [detection for detection in detections if detection.detection_details.get('line_type') != 'Removed'] - - -def _exclude_detections_by_exclusions_configuration(detections: list[Detection], scan_type: str) -> list[Detection]: - exclusions = configuration_manager.get_exclusions_by_scan_type(scan_type) - return [detection for detection in detections if not _should_exclude_detection(detection, exclusions)] - - -def _should_exclude_detection(detection: Detection, exclusions: dict) -> bool: - # FIXME(MarshalX): what the difference between by_value and by_sha? - exclusions_by_value = exclusions.get(consts.EXCLUSIONS_BY_VALUE_SECTION_NAME, []) - if _is_detection_sha_configured_in_exclusions(detection, exclusions_by_value): - logger.debug( - 'Ignoring violation because its value is on the ignore list, %s', - {'value_sha': detection.detection_details.get('sha512')}, - ) - return True - - exclusions_by_sha = exclusions.get(consts.EXCLUSIONS_BY_SHA_SECTION_NAME, []) - if _is_detection_sha_configured_in_exclusions(detection, exclusions_by_sha): - logger.debug( - 'Ignoring violation because its SHA value is on the ignore list, %s', - {'sha': detection.detection_details.get('sha512')}, - ) - return True - - exclusions_by_rule = exclusions.get(consts.EXCLUSIONS_BY_RULE_SECTION_NAME, []) - detection_rule_id = detection.detection_rule_id - if detection_rule_id in exclusions_by_rule: - logger.debug( - 'Ignoring violation because its Detection Rule ID is on the ignore list, %s', - {'detection_rule_id': detection_rule_id}, - ) - return True - - exclusions_by_package = exclusions.get(consts.EXCLUSIONS_BY_PACKAGE_SECTION_NAME, []) - package = _get_package_name(detection) - if package and package in exclusions_by_package: - logger.debug('Ignoring violation because its package@version is on the ignore list, %s', {'package': package}) - return True - - exclusions_by_cve = exclusions.get(consts.EXCLUSIONS_BY_CVE_SECTION_NAME, []) - cve = _get_cve_identifier(detection) - if cve and cve in exclusions_by_cve: - logger.debug('Ignoring violation because its CVE is on the ignore list, %s', {'cve': cve}) - return True - - return False - - -def _is_detection_sha_configured_in_exclusions(detection: Detection, exclusions: list[str]) -> bool: - detection_sha = detection.detection_details.get('sha512') - return detection_sha in exclusions - - -def _get_package_name(detection: Detection) -> Optional[str]: - package_name = detection.detection_details.get('vulnerable_component') - package_version = detection.detection_details.get('vulnerable_component_version') - - if package_name is None: - package_name = detection.detection_details.get('package_name') - package_version = detection.detection_details.get('package_version') - - if package_name and package_version: - return f'{package_name}@{package_version}' - - return None - - -def _get_cve_identifier(detection: Detection) -> Optional[str]: - return detection.detection_details.get('alert', {}).get('cve_identifier') - - -def _get_document_by_file_name( - documents: list[Document], file_name: str, unique_id: Optional[str] = None -) -> Optional[Document]: - for document in documents: - if _normalize_file_path(document.path) == _normalize_file_path(file_name) and document.unique_id == unique_id: - return document - - return None - - -def _report_scan_status( +def report_scan_status( cycode_client: 'ScanClient', scan_type: str, scan_id: str, @@ -932,162 +334,3 @@ def _report_scan_status( cycode_client.report_scan_status(scan_type, scan_id, scan_status) except Exception as e: logger.debug('Failed to report scan status', exc_info=e) - - -def _generate_unique_id() -> UUID: - if 'PYTEST_TEST_UNIQUE_ID' in os.environ: - return UUID(os.environ['PYTEST_TEST_UNIQUE_ID']) - - return uuid4() - - -def _does_severity_match_severity_threshold(severity: str, severity_threshold: str) -> bool: - detection_severity_value = SeverityOption.get_member_weight(severity) - severity_threshold_value = SeverityOption.get_member_weight(severity_threshold) - if detection_severity_value < 0 or severity_threshold_value < 0: - return True - - return detection_severity_value >= severity_threshold_value - - -def _get_scan_result( - cycode_client: 'ScanClient', - scan_type: str, - scan_id: str, - scan_details: 'ScanDetailsResponse', - scan_parameters: dict, -) -> ZippedFileScanResult: - if not scan_details.detections_count: - return init_default_scan_result(scan_id) - - scan_raw_detections = cycode_client.get_scan_raw_detections(scan_id) - - return ZippedFileScanResult( - did_detect=True, - detections_per_file=_map_detections_per_file_and_commit_id(scan_type, scan_raw_detections), - scan_id=scan_id, - report_url=_try_get_aggregation_report_url_if_needed(scan_parameters, cycode_client, scan_type), - ) - - -def init_default_scan_result(scan_id: str) -> ZippedFileScanResult: - return ZippedFileScanResult( - did_detect=False, - detections_per_file=[], - scan_id=scan_id, - ) - - -def _set_aggregation_report_url(ctx: typer.Context, aggregation_report_url: Optional[str] = None) -> None: - ctx.obj['aggregation_report_url'] = aggregation_report_url - - -def _try_get_aggregation_report_url_if_needed( - scan_parameters: dict, cycode_client: 'ScanClient', scan_type: str -) -> Optional[str]: - if not scan_parameters.get('report', False): - return None - - aggregation_id = scan_parameters.get('aggregation_id') - if aggregation_id is None: - return None - - try: - report_url_response = cycode_client.get_scan_aggregation_report_url(aggregation_id, scan_type) - return report_url_response.report_url - except Exception as e: - logger.debug('Failed to get aggregation report url: %s', str(e)) - - -def _map_detections_per_file_and_commit_id(scan_type: str, raw_detections: list[dict]) -> list[DetectionsPerFile]: - """Convert a list of detections (async flow) to list of DetectionsPerFile objects (sync flow). - - Args: - scan_type: Type of the scan. - raw_detections: List of detections as is returned from the server. - - Note: - This method fakes server response structure - to be able to use the same logic for both async and sync scans. - - Note: - Aggregation is performed by file name and commit ID (if available) - - """ - detections_per_files = {} - for raw_detection in raw_detections: - try: - # FIXME(MarshalX): investigate this field mapping - raw_detection['message'] = raw_detection['correlation_message'] - - file_name = _get_file_name_from_detection(scan_type, raw_detection) - detection: Detection = DetectionSchema().load(raw_detection) - commit_id: Optional[str] = detection.detection_details.get('commit_id') # could be None - group_by_key = (file_name, commit_id) - - if group_by_key in detections_per_files: - detections_per_files[group_by_key].append(detection) - else: - detections_per_files[group_by_key] = [detection] - except Exception as e: - logger.debug('Failed to parse detection', exc_info=e) - continue - - return [ - DetectionsPerFile(file_name=file_name, detections=file_detections, commit_id=commit_id) - for (file_name, commit_id), file_detections in detections_per_files.items() - ] - - -def _get_file_name_from_detection(scan_type: str, raw_detection: dict) -> str: - if scan_type == consts.SAST_SCAN_TYPE: - return raw_detection['detection_details']['file_path'] - if scan_type == consts.SECRET_SCAN_TYPE: - return _get_secret_file_name_from_detection(raw_detection) - - return raw_detection['detection_details']['file_name'] - - -def _get_secret_file_name_from_detection(raw_detection: dict) -> str: - file_path: str = raw_detection['detection_details']['file_path'] - file_name: str = raw_detection['detection_details']['file_name'] - return os.path.join(file_path, file_name) - - -def _does_reach_to_max_commits_to_scan_limit(commit_ids: list[str], max_commits_count: Optional[int]) -> bool: - if max_commits_count is None: - return False - - return len(commit_ids) >= max_commits_count - - -def _normalize_file_path(path: str) -> str: - if path.startswith('/'): - return path[1:] - if path.startswith('./'): - return path[2:] - return path - - -def perform_post_pre_receive_scan_actions(ctx: typer.Context) -> None: - if scan_utils.is_scan_failed(ctx): - console.print(consts.PRE_RECEIVE_REMEDIATION_MESSAGE) - - -def enable_verbose_mode(ctx: typer.Context) -> None: - ctx.obj['verbose'] = True - set_logging_level(logging.DEBUG) - - -def is_verbose_mode_requested_in_pre_receive_scan() -> bool: - return does_git_push_option_have_value(consts.VERBOSE_SCAN_FLAG) - - -def should_skip_pre_receive_scan() -> bool: - return does_git_push_option_have_value(consts.SKIP_SCAN_FLAG) - - -def does_git_push_option_have_value(value: str) -> bool: - option_count_env_value = os.getenv(consts.GIT_PUSH_OPTION_COUNT_ENV_VAR_NAME, '') - option_count = int(option_count_env_value) if option_count_env_value.isdigit() else 0 - return any(os.getenv(f'{consts.GIT_PUSH_OPTION_ENV_VAR_PREFIX}{i}') == value for i in range(option_count)) diff --git a/cycode/cli/apps/scan/commit_history/commit_history_command.py b/cycode/cli/apps/scan/commit_history/commit_history_command.py index fc1ef23f..5935cf59 100644 --- a/cycode/cli/apps/scan/commit_history/commit_history_command.py +++ b/cycode/cli/apps/scan/commit_history/commit_history_command.py @@ -3,7 +3,7 @@ import typer -from cycode.cli.apps.scan.code_scanner import scan_commit_range +from cycode.cli.apps.scan.commit_range_scanner import scan_commit_range from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception from cycode.cli.logger import logger from cycode.cli.utils.sentry import add_breadcrumb @@ -28,6 +28,6 @@ def commit_history_command( add_breadcrumb('commit_history') logger.debug('Starting commit history scan process, %s', {'path': path, 'commit_range': commit_range}) - scan_commit_range(ctx, path=str(path), commit_range=commit_range) + scan_commit_range(ctx, repo_path=str(path), commit_range=commit_range) except Exception as e: handle_scan_exception(ctx, e) diff --git a/cycode/cli/apps/scan/commit_range_scanner.py b/cycode/cli/apps/scan/commit_range_scanner.py new file mode 100644 index 00000000..b191611f --- /dev/null +++ b/cycode/cli/apps/scan/commit_range_scanner.py @@ -0,0 +1,311 @@ +import os +from typing import TYPE_CHECKING, Optional + +import click +import typer + +from cycode.cli import consts +from cycode.cli.apps.scan.code_scanner import ( + poll_scan_results, + report_scan_status, + scan_documents, +) +from cycode.cli.apps.scan.scan_parameters import get_scan_parameters +from cycode.cli.apps.scan.scan_result import ( + create_local_scan_result, + init_default_scan_result, + print_local_scan_results, +) +from cycode.cli.config import configuration_manager +from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception +from cycode.cli.files_collector.commit_range_documents import ( + collect_commit_range_diff_documents, + get_commit_range_modified_documents, + get_diff_file_content, + get_diff_file_path, + get_pre_commit_modified_documents, + parse_commit_range_sast, + parse_commit_range_sca, +) +from cycode.cli.files_collector.file_excluder import excluder +from cycode.cli.files_collector.models.in_memory_zip import InMemoryZip +from cycode.cli.files_collector.sca.sca_file_collector import ( + perform_sca_pre_commit_range_scan_actions, + perform_sca_pre_hook_range_scan_actions, +) +from cycode.cli.files_collector.zip_documents import zip_documents +from cycode.cli.models import Document +from cycode.cli.utils.git_proxy import git_proxy +from cycode.cli.utils.path_utils import get_path_by_os +from cycode.cli.utils.progress_bar import ScanProgressBarSection +from cycode.cli.utils.scan_utils import generate_unique_scan_id, set_issue_detected_by_scan_results +from cycode.cyclient.models import ZippedFileScanResult +from cycode.logger import get_logger + +if TYPE_CHECKING: + from cycode.cyclient.scan_client import ScanClient + +logger = get_logger('Commit Range Scanner') + + +def _does_git_push_option_have_value(value: str) -> bool: + option_count_env_value = os.getenv(consts.GIT_PUSH_OPTION_COUNT_ENV_VAR_NAME, '') + option_count = int(option_count_env_value) if option_count_env_value.isdigit() else 0 + return any(os.getenv(f'{consts.GIT_PUSH_OPTION_ENV_VAR_PREFIX}{i}') == value for i in range(option_count)) + + +def is_verbose_mode_requested_in_pre_receive_scan() -> bool: + return _does_git_push_option_have_value(consts.VERBOSE_SCAN_FLAG) + + +def should_skip_pre_receive_scan() -> bool: + return _does_git_push_option_have_value(consts.SKIP_SCAN_FLAG) + + +def _perform_commit_range_scan_async( + cycode_client: 'ScanClient', + from_commit_zipped_documents: 'InMemoryZip', + to_commit_zipped_documents: 'InMemoryZip', + scan_type: str, + scan_parameters: dict, + timeout: Optional[int] = None, +) -> ZippedFileScanResult: + scan_async_result = cycode_client.commit_range_scan_async( + from_commit_zipped_documents, to_commit_zipped_documents, scan_type, scan_parameters + ) + + logger.debug( + 'Async commit range scan request has been triggered successfully, %s', {'scan_id': scan_async_result.scan_id} + ) + return poll_scan_results(cycode_client, scan_async_result.scan_id, scan_type, scan_parameters, timeout) + + +def _scan_commit_range_documents( + ctx: typer.Context, + from_documents_to_scan: list[Document], + to_documents_to_scan: list[Document], + scan_parameters: Optional[dict] = None, + timeout: Optional[int] = None, +) -> None: + cycode_client = ctx.obj['client'] + scan_type = ctx.obj['scan_type'] + severity_threshold = ctx.obj['severity_threshold'] + scan_command_type = ctx.info_name + progress_bar = ctx.obj['progress_bar'] + + local_scan_result = error_message = None + scan_completed = False + scan_id = str(generate_unique_scan_id()) + from_commit_zipped_documents = InMemoryZip() + to_commit_zipped_documents = InMemoryZip() + + try: + progress_bar.set_section_length(ScanProgressBarSection.SCAN, 1) + + scan_result = init_default_scan_result(scan_id) + if len(from_documents_to_scan) > 0 or len(to_documents_to_scan) > 0: + logger.debug('Preparing from-commit zip') + # for SAST it is files from to_commit with actual content to scan + from_commit_zipped_documents = zip_documents(scan_type, from_documents_to_scan) + + logger.debug('Preparing to-commit zip') + # for SAST it is files with diff between from_commit and to_commit + to_commit_zipped_documents = zip_documents(scan_type, to_documents_to_scan) + + scan_result = _perform_commit_range_scan_async( + cycode_client, + from_commit_zipped_documents, + to_commit_zipped_documents, + scan_type, + scan_parameters, + timeout, + ) + + progress_bar.update(ScanProgressBarSection.SCAN) + progress_bar.set_section_length(ScanProgressBarSection.GENERATE_REPORT, 1) + + local_scan_result = create_local_scan_result( + scan_result, to_documents_to_scan, scan_command_type, scan_type, severity_threshold + ) + set_issue_detected_by_scan_results(ctx, [local_scan_result]) + + progress_bar.update(ScanProgressBarSection.GENERATE_REPORT) + progress_bar.stop() + + # errors will be handled with try-except block; printing will not occur on errors + print_local_scan_results(ctx, [local_scan_result]) + + scan_completed = True + except Exception as e: + handle_scan_exception(ctx, e) + error_message = str(e) + + zip_file_size = from_commit_zipped_documents.size + to_commit_zipped_documents.size + + detections_count = relevant_detections_count = 0 + if local_scan_result: + detections_count = local_scan_result.detections_count + relevant_detections_count = local_scan_result.relevant_detections_count + scan_id = local_scan_result.scan_id + + logger.debug( + 'Processing commit range scan results, %s', + { + 'all_violations_count': detections_count, + 'relevant_violations_count': relevant_detections_count, + 'scan_id': scan_id, + 'zip_file_size': zip_file_size, + }, + ) + report_scan_status( + cycode_client, + scan_type, + scan_id, + scan_completed, + relevant_detections_count, + detections_count, + len(to_documents_to_scan), + zip_file_size, + scan_command_type, + error_message, + ) + + +def _scan_sca_commit_range(ctx: typer.Context, repo_path: str, commit_range: str, **_) -> None: + scan_parameters = get_scan_parameters(ctx, (repo_path,)) + + from_commit_rev, to_commit_rev = parse_commit_range_sca(commit_range, repo_path) + from_commit_documents, to_commit_documents, _ = get_commit_range_modified_documents( + ctx.obj['progress_bar'], ScanProgressBarSection.PREPARE_LOCAL_FILES, repo_path, from_commit_rev, to_commit_rev + ) + from_commit_documents = excluder.exclude_irrelevant_documents_to_scan(consts.SCA_SCAN_TYPE, from_commit_documents) + to_commit_documents = excluder.exclude_irrelevant_documents_to_scan(consts.SCA_SCAN_TYPE, to_commit_documents) + + perform_sca_pre_commit_range_scan_actions( + repo_path, from_commit_documents, from_commit_rev, to_commit_documents, to_commit_rev + ) + + _scan_commit_range_documents(ctx, from_commit_documents, to_commit_documents, scan_parameters=scan_parameters) + + +def _scan_secret_commit_range( + ctx: typer.Context, repo_path: str, commit_range: str, max_commits_count: Optional[int] = None +) -> None: + commit_diff_documents_to_scan = collect_commit_range_diff_documents(ctx, repo_path, commit_range, max_commits_count) + diff_documents_to_scan = excluder.exclude_irrelevant_documents_to_scan( + consts.SECRET_SCAN_TYPE, commit_diff_documents_to_scan + ) + + scan_documents( + ctx, diff_documents_to_scan, get_scan_parameters(ctx, (repo_path,)), is_git_diff=True, is_commit_range=True + ) + + +def _scan_sast_commit_range(ctx: typer.Context, repo_path: str, commit_range: str, **_) -> None: + scan_parameters = get_scan_parameters(ctx, (repo_path,)) + + from_commit_rev, to_commit_rev = parse_commit_range_sast(commit_range, repo_path) + _, commit_documents, diff_documents = get_commit_range_modified_documents( + ctx.obj['progress_bar'], + ScanProgressBarSection.PREPARE_LOCAL_FILES, + repo_path, + from_commit_rev, + to_commit_rev, + reverse_diff=False, + ) + commit_documents = excluder.exclude_irrelevant_documents_to_scan(consts.SAST_SCAN_TYPE, commit_documents) + diff_documents = excluder.exclude_irrelevant_documents_to_scan(consts.SAST_SCAN_TYPE, diff_documents) + + _scan_commit_range_documents(ctx, commit_documents, diff_documents, scan_parameters=scan_parameters) + + +_SCAN_TYPE_TO_COMMIT_RANGE_HANDLER = { + consts.SCA_SCAN_TYPE: _scan_sca_commit_range, + consts.SECRET_SCAN_TYPE: _scan_secret_commit_range, + consts.SAST_SCAN_TYPE: _scan_sast_commit_range, +} + + +def scan_commit_range(ctx: typer.Context, repo_path: str, commit_range: str, **kwargs) -> None: + scan_type = ctx.obj['scan_type'] + + progress_bar = ctx.obj['progress_bar'] + progress_bar.start() + + if scan_type not in _SCAN_TYPE_TO_COMMIT_RANGE_HANDLER: + raise click.ClickException(f'Commit range scanning for {scan_type.upper()} is not supported') + + _SCAN_TYPE_TO_COMMIT_RANGE_HANDLER[scan_type](ctx, repo_path, commit_range, **kwargs) + + +def _scan_sca_pre_commit(ctx: typer.Context, repo_path: str) -> None: + scan_parameters = get_scan_parameters(ctx) + + git_head_documents, pre_committed_documents, _ = get_pre_commit_modified_documents( + progress_bar=ctx.obj['progress_bar'], + progress_bar_section=ScanProgressBarSection.PREPARE_LOCAL_FILES, + repo_path=repo_path, + ) + git_head_documents = excluder.exclude_irrelevant_documents_to_scan(consts.SCA_SCAN_TYPE, git_head_documents) + pre_committed_documents = excluder.exclude_irrelevant_documents_to_scan( + consts.SCA_SCAN_TYPE, pre_committed_documents + ) + + perform_sca_pre_hook_range_scan_actions(repo_path, git_head_documents, pre_committed_documents) + + _scan_commit_range_documents( + ctx, + git_head_documents, + pre_committed_documents, + scan_parameters, + configuration_manager.get_sca_pre_commit_timeout_in_seconds(), + ) + + +def _scan_secret_pre_commit(ctx: typer.Context, repo_path: str) -> None: + progress_bar = ctx.obj['progress_bar'] + diff_index = git_proxy.get_repo(repo_path).index.diff(consts.GIT_HEAD_COMMIT_REV, create_patch=True, R=True) + + progress_bar.set_section_length(ScanProgressBarSection.PREPARE_LOCAL_FILES, len(diff_index)) + + documents_to_scan = [] + for diff in diff_index: + progress_bar.update(ScanProgressBarSection.PREPARE_LOCAL_FILES) + documents_to_scan.append( + Document(get_path_by_os(get_diff_file_path(diff)), get_diff_file_content(diff), is_git_diff_format=True) + ) + documents_to_scan = excluder.exclude_irrelevant_documents_to_scan(consts.SECRET_SCAN_TYPE, documents_to_scan) + + scan_documents(ctx, documents_to_scan, get_scan_parameters(ctx), is_git_diff=True) + + +def _scan_sast_pre_commit(ctx: typer.Context, repo_path: str, **_) -> None: + scan_parameters = get_scan_parameters(ctx, (repo_path,)) + + _, pre_committed_documents, diff_documents = get_pre_commit_modified_documents( + progress_bar=ctx.obj['progress_bar'], + progress_bar_section=ScanProgressBarSection.PREPARE_LOCAL_FILES, + repo_path=repo_path, + ) + pre_committed_documents = excluder.exclude_irrelevant_documents_to_scan( + consts.SAST_SCAN_TYPE, pre_committed_documents + ) + diff_documents = excluder.exclude_irrelevant_documents_to_scan(consts.SAST_SCAN_TYPE, diff_documents) + + _scan_commit_range_documents(ctx, pre_committed_documents, diff_documents, scan_parameters=scan_parameters) + + +_SCAN_TYPE_TO_PRE_COMMIT_HANDLER = { + consts.SCA_SCAN_TYPE: _scan_sca_pre_commit, + consts.SECRET_SCAN_TYPE: _scan_secret_pre_commit, + consts.SAST_SCAN_TYPE: _scan_sast_pre_commit, +} + + +def scan_pre_commit(ctx: typer.Context, repo_path: str) -> None: + scan_type = ctx.obj['scan_type'] + if scan_type not in _SCAN_TYPE_TO_PRE_COMMIT_HANDLER: + raise click.ClickException(f'Pre-commit scanning for {scan_type.upper()} is not supported') + + _SCAN_TYPE_TO_PRE_COMMIT_HANDLER[scan_type](ctx, repo_path) + logger.debug('Pre-commit scan completed successfully') diff --git a/cycode/cli/apps/scan/detection_excluder.py b/cycode/cli/apps/scan/detection_excluder.py new file mode 100644 index 00000000..3697bcaf --- /dev/null +++ b/cycode/cli/apps/scan/detection_excluder.py @@ -0,0 +1,153 @@ +from typing import Optional + +from cycode.cli import consts +from cycode.cli.cli_types import SeverityOption +from cycode.cli.config import configuration_manager +from cycode.cli.models import DocumentDetections +from cycode.cyclient.models import Detection +from cycode.logger import get_logger + +logger = get_logger('Detection Excluder') + + +def _does_severity_match_severity_threshold(severity: str, severity_threshold: str) -> bool: + detection_severity_value = SeverityOption.get_member_weight(severity) + severity_threshold_value = SeverityOption.get_member_weight(severity_threshold) + if detection_severity_value < 0 or severity_threshold_value < 0: + return True + + return detection_severity_value >= severity_threshold_value + + +def _exclude_irrelevant_detections( + detections: list[Detection], scan_type: str, command_scan_type: str, severity_threshold: str +) -> list[Detection]: + relevant_detections = _exclude_detections_by_exclusions_configuration(detections, scan_type) + relevant_detections = _exclude_detections_by_scan_type(relevant_detections, scan_type, command_scan_type) + return _exclude_detections_by_severity(relevant_detections, severity_threshold) + + +def _exclude_detections_by_severity(detections: list[Detection], severity_threshold: str) -> list[Detection]: + relevant_detections = [] + for detection in detections: + severity = detection.severity + + if _does_severity_match_severity_threshold(severity, severity_threshold): + relevant_detections.append(detection) + else: + logger.debug( + 'Going to ignore violations because they are below the severity threshold, %s', + {'severity': severity, 'severity_threshold': severity_threshold}, + ) + + return relevant_detections + + +def _exclude_detections_by_scan_type( + detections: list[Detection], scan_type: str, command_scan_type: str +) -> list[Detection]: + if command_scan_type == consts.PRE_COMMIT_COMMAND_SCAN_TYPE: + return _exclude_detections_in_deleted_lines(detections) + + exclude_in_deleted_lines = configuration_manager.get_should_exclude_detections_in_deleted_lines(command_scan_type) + if ( + command_scan_type in consts.COMMIT_RANGE_BASED_COMMAND_SCAN_TYPES + and scan_type == consts.SECRET_SCAN_TYPE + and exclude_in_deleted_lines + ): + return _exclude_detections_in_deleted_lines(detections) + + return detections + + +def _exclude_detections_in_deleted_lines(detections: list[Detection]) -> list[Detection]: + return [detection for detection in detections if detection.detection_details.get('line_type') != 'Removed'] + + +def _exclude_detections_by_exclusions_configuration(detections: list[Detection], scan_type: str) -> list[Detection]: + exclusions = configuration_manager.get_exclusions_by_scan_type(scan_type) + return [detection for detection in detections if not _should_exclude_detection(detection, exclusions)] + + +def _should_exclude_detection(detection: Detection, exclusions: dict) -> bool: + # FIXME(MarshalX): what the difference between by_value and by_sha? + exclusions_by_value = exclusions.get(consts.EXCLUSIONS_BY_VALUE_SECTION_NAME, []) + if _is_detection_sha_configured_in_exclusions(detection, exclusions_by_value): + logger.debug( + 'Ignoring violation because its value is on the ignore list, %s', + {'value_sha': detection.detection_details.get('sha512')}, + ) + return True + + exclusions_by_sha = exclusions.get(consts.EXCLUSIONS_BY_SHA_SECTION_NAME, []) + if _is_detection_sha_configured_in_exclusions(detection, exclusions_by_sha): + logger.debug( + 'Ignoring violation because its SHA value is on the ignore list, %s', + {'sha': detection.detection_details.get('sha512')}, + ) + return True + + exclusions_by_rule = exclusions.get(consts.EXCLUSIONS_BY_RULE_SECTION_NAME, []) + detection_rule_id = detection.detection_rule_id + if detection_rule_id in exclusions_by_rule: + logger.debug( + 'Ignoring violation because its Detection Rule ID is on the ignore list, %s', + {'detection_rule_id': detection_rule_id}, + ) + return True + + exclusions_by_package = exclusions.get(consts.EXCLUSIONS_BY_PACKAGE_SECTION_NAME, []) + package = _get_package_name(detection) + if package and package in exclusions_by_package: + logger.debug('Ignoring violation because its package@version is on the ignore list, %s', {'package': package}) + return True + + exclusions_by_cve = exclusions.get(consts.EXCLUSIONS_BY_CVE_SECTION_NAME, []) + cve = _get_cve_identifier(detection) + if cve and cve in exclusions_by_cve: + logger.debug('Ignoring violation because its CVE is on the ignore list, %s', {'cve': cve}) + return True + + return False + + +def _is_detection_sha_configured_in_exclusions(detection: Detection, exclusions: list[str]) -> bool: + detection_sha = detection.detection_details.get('sha512') + return detection_sha in exclusions + + +def _get_package_name(detection: Detection) -> Optional[str]: + package_name = detection.detection_details.get('vulnerable_component') + package_version = detection.detection_details.get('vulnerable_component_version') + + if package_name is None: + package_name = detection.detection_details.get('package_name') + package_version = detection.detection_details.get('package_version') + + if package_name and package_version: + return f'{package_name}@{package_version}' + + return None + + +def _get_cve_identifier(detection: Detection) -> Optional[str]: + return detection.detection_details.get('alert', {}).get('cve_identifier') + + +def exclude_irrelevant_document_detections( + document_detections_list: list[DocumentDetections], + scan_type: str, + command_scan_type: str, + severity_threshold: str, +) -> list[DocumentDetections]: + relevant_document_detections_list = [] + for document_detections in document_detections_list: + relevant_detections = _exclude_irrelevant_detections( + document_detections.detections, scan_type, command_scan_type, severity_threshold + ) + if relevant_detections: + relevant_document_detections_list.append( + DocumentDetections(document=document_detections.document, detections=relevant_detections) + ) + + return relevant_document_detections_list diff --git a/cycode/cli/apps/scan/pre_commit/pre_commit_command.py b/cycode/cli/apps/scan/pre_commit/pre_commit_command.py index 9242b450..5693412f 100644 --- a/cycode/cli/apps/scan/pre_commit/pre_commit_command.py +++ b/cycode/cli/apps/scan/pre_commit/pre_commit_command.py @@ -3,19 +3,7 @@ import typer -from cycode.cli import consts -from cycode.cli.apps.scan.code_scanner import get_scan_parameters, scan_documents, scan_sca_pre_commit -from cycode.cli.files_collector.excluder import excluder -from cycode.cli.files_collector.repository_documents import ( - get_diff_file_content, - get_diff_file_path, -) -from cycode.cli.models import Document -from cycode.cli.utils.git_proxy import git_proxy -from cycode.cli.utils.path_utils import ( - get_path_by_os, -) -from cycode.cli.utils.progress_bar import ScanProgressBarSection +from cycode.cli.apps.scan.commit_range_scanner import scan_pre_commit from cycode.cli.utils.sentry import add_breadcrumb @@ -25,25 +13,9 @@ def pre_commit_command( ) -> None: add_breadcrumb('pre_commit') - scan_type = ctx.obj['scan_type'] - repo_path = os.getcwd() # change locally for easy testing progress_bar = ctx.obj['progress_bar'] progress_bar.start() - if scan_type == consts.SCA_SCAN_TYPE: - scan_sca_pre_commit(ctx, repo_path) - return - - diff_files = git_proxy.get_repo(repo_path).index.diff(consts.GIT_HEAD_COMMIT_REV, create_patch=True, R=True) - - progress_bar.set_section_length(ScanProgressBarSection.PREPARE_LOCAL_FILES, len(diff_files)) - - documents_to_scan = [] - for file in diff_files: - progress_bar.update(ScanProgressBarSection.PREPARE_LOCAL_FILES) - documents_to_scan.append(Document(get_path_by_os(get_diff_file_path(file)), get_diff_file_content(file))) - - documents_to_scan = excluder.exclude_irrelevant_documents_to_scan(scan_type, documents_to_scan) - scan_documents(ctx, documents_to_scan, get_scan_parameters(ctx), is_git_diff=True) + scan_pre_commit(ctx, repo_path) diff --git a/cycode/cli/apps/scan/pre_receive/pre_receive_command.py b/cycode/cli/apps/scan/pre_receive/pre_receive_command.py index eb4f1420..ef30ee8f 100644 --- a/cycode/cli/apps/scan/pre_receive/pre_receive_command.py +++ b/cycode/cli/apps/scan/pre_receive/pre_receive_command.py @@ -1,26 +1,27 @@ +import logging import os from typing import Annotated, Optional -import click import typer from cycode.cli import consts -from cycode.cli.apps.scan.code_scanner import ( - enable_verbose_mode, +from cycode.cli.apps.scan.commit_range_scanner import ( is_verbose_mode_requested_in_pre_receive_scan, - parse_pre_receive_input, - perform_post_pre_receive_scan_actions, scan_commit_range, should_skip_pre_receive_scan, ) from cycode.cli.config import configuration_manager +from cycode.cli.console import console from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception -from cycode.cli.files_collector.repository_documents import ( +from cycode.cli.files_collector.commit_range_documents import ( calculate_pre_receive_commit_range, + parse_pre_receive_input, ) from cycode.cli.logger import logger +from cycode.cli.utils import scan_utils from cycode.cli.utils.sentry import add_breadcrumb from cycode.cli.utils.task_timer import TimeoutAfter +from cycode.logger import set_logging_level def pre_receive_command( @@ -30,10 +31,6 @@ def pre_receive_command( try: add_breadcrumb('pre_receive') - scan_type = ctx.obj['scan_type'] - if scan_type != consts.SECRET_SCAN_TYPE: - raise click.ClickException(f'Commit range scanning for {scan_type.upper()} is not supported') - if should_skip_pre_receive_scan(): logger.info( 'A scan has been skipped as per your request. ' @@ -42,15 +39,13 @@ def pre_receive_command( return if is_verbose_mode_requested_in_pre_receive_scan(): - enable_verbose_mode(ctx) + ctx.obj['verbose'] = True + set_logging_level(logging.DEBUG) logger.debug('Verbose mode enabled: all log levels will be displayed.') command_scan_type = ctx.info_name timeout = configuration_manager.get_pre_receive_command_timeout(command_scan_type) with TimeoutAfter(timeout): - if scan_type not in consts.COMMIT_RANGE_SCAN_SUPPORTED_SCAN_TYPES: - raise click.ClickException(f'Commit range scanning for {scan_type.upper()} is not supported') - branch_update_details = parse_pre_receive_input() commit_range = calculate_pre_receive_commit_range(branch_update_details) if not commit_range: @@ -60,8 +55,14 @@ def pre_receive_command( ) return - max_commits_to_scan = configuration_manager.get_pre_receive_max_commits_to_scan_count(command_scan_type) - scan_commit_range(ctx, os.getcwd(), commit_range, max_commits_count=max_commits_to_scan) - perform_post_pre_receive_scan_actions(ctx) + scan_commit_range( + ctx=ctx, + repo_path=os.getcwd(), + commit_range=commit_range, + max_commits_count=configuration_manager.get_pre_receive_max_commits_to_scan_count(command_scan_type), + ) + + if scan_utils.is_scan_failed(ctx): + console.print(consts.PRE_RECEIVE_REMEDIATION_MESSAGE) except Exception as e: handle_scan_exception(ctx, e) diff --git a/cycode/cli/apps/scan/remote_url_resolver.py b/cycode/cli/apps/scan/remote_url_resolver.py new file mode 100644 index 00000000..5f96328d --- /dev/null +++ b/cycode/cli/apps/scan/remote_url_resolver.py @@ -0,0 +1,115 @@ +from typing import Optional + +from cycode.cli import consts +from cycode.cli.utils.git_proxy import git_proxy +from cycode.cli.utils.shell_executor import shell +from cycode.logger import get_logger + +logger = get_logger('Remote URL Resolver') + + +def _get_plastic_repository_name(path: str) -> Optional[str]: + """Get the name of the Plastic repository from the current working directory. + + The command to execute is: + cm status --header --machinereadable --fieldseparator=":::" + + Example of status header in machine-readable format: + STATUS:::0:::Project/RepoName:::OrgName@ServerInfo + """ + try: + command = [ + 'cm', + 'status', + '--header', + '--machinereadable', + f'--fieldseparator={consts.PLASTIC_VCS_DATA_SEPARATOR}', + ] + + status = shell( + command=command, timeout=consts.PLASTIC_VSC_CLI_TIMEOUT, working_directory=path, silent_exc_info=True + ) + if not status: + logger.debug('Failed to get Plastic repository name (command failed)') + return None + + status_parts = status.split(consts.PLASTIC_VCS_DATA_SEPARATOR) + if len(status_parts) < 2: + logger.debug('Failed to parse Plastic repository name (command returned unexpected format)') + return None + + return status_parts[2].strip() + except Exception: + logger.debug('Failed to get Plastic repository name. Probably not a Plastic repository') + return None + + +def _get_plastic_repository_list(working_dir: Optional[str] = None) -> dict[str, str]: + """Get the list of Plastic repositories and their GUIDs. + + The command to execute is: + cm repo list --format="{repname}:::{repguid}" + + Example line with data: + Project/RepoName:::tapo1zqt-wn99-4752-h61m-7d9k79d40r4v + + Each line represents an individual repository. + """ + repo_name_to_guid = {} + + try: + command = ['cm', 'repo', 'ls', f'--format={{repname}}{consts.PLASTIC_VCS_DATA_SEPARATOR}{{repguid}}'] + + status = shell( + command=command, timeout=consts.PLASTIC_VSC_CLI_TIMEOUT, working_directory=working_dir, silent_exc_info=True + ) + if not status: + logger.debug('Failed to get Plastic repository list (command failed)') + return repo_name_to_guid + + status_lines = status.splitlines() + for line in status_lines: + data_parts = line.split(consts.PLASTIC_VCS_DATA_SEPARATOR) + if len(data_parts) < 2: + logger.debug('Failed to parse Plastic repository list line (unexpected format), %s', {'line': line}) + continue + + repo_name, repo_guid = data_parts + repo_name_to_guid[repo_name.strip()] = repo_guid.strip() + + return repo_name_to_guid + except Exception as e: + logger.debug('Failed to get Plastic repository list', exc_info=e) + return repo_name_to_guid + + +def _try_to_get_plastic_remote_url(path: str) -> Optional[str]: + repository_name = _get_plastic_repository_name(path) + if not repository_name: + return None + + repository_map = _get_plastic_repository_list(path) + if repository_name not in repository_map: + logger.debug('Failed to get Plastic repository GUID (repository not found in the list)') + return None + + repository_guid = repository_map[repository_name] + return f'{consts.PLASTIC_VCS_REMOTE_URI_PREFIX}{repository_guid}' + + +def _try_get_git_remote_url(path: str) -> Optional[str]: + try: + remote_url = git_proxy.get_repo(path).remotes[0].config_reader.get('url') + logger.debug('Found Git remote URL, %s', {'remote_url': remote_url, 'path': path}) + return remote_url + except Exception: + logger.debug('Failed to get Git remote URL. Probably not a Git repository') + return None + + +def try_get_any_remote_url(path: str) -> Optional[str]: + remote_url = _try_get_git_remote_url(path) + if not remote_url: + remote_url = _try_to_get_plastic_remote_url(path) + + return remote_url diff --git a/cycode/cli/apps/scan/repository/repository_command.py b/cycode/cli/apps/scan/repository/repository_command.py index e9a3f63d..6fc77bee 100644 --- a/cycode/cli/apps/scan/repository/repository_command.py +++ b/cycode/cli/apps/scan/repository/repository_command.py @@ -5,11 +5,12 @@ import typer from cycode.cli import consts -from cycode.cli.apps.scan.code_scanner import get_scan_parameters, scan_documents +from cycode.cli.apps.scan.code_scanner import scan_documents +from cycode.cli.apps.scan.scan_parameters import get_scan_parameters from cycode.cli.exceptions.handle_scan_errors import handle_scan_exception -from cycode.cli.files_collector.excluder import excluder +from cycode.cli.files_collector.file_excluder import excluder from cycode.cli.files_collector.repository_documents import get_git_repository_tree_file_entries -from cycode.cli.files_collector.sca.sca_code_scanner import perform_pre_scan_documents_actions +from cycode.cli.files_collector.sca.sca_file_collector import add_sca_dependencies_tree_documents_if_needed from cycode.cli.logger import logger from cycode.cli.models import Document from cycode.cli.utils.path_utils import get_path_by_os @@ -59,7 +60,7 @@ def repository_command( documents_to_scan = excluder.exclude_irrelevant_documents_to_scan(scan_type, documents_to_scan) - perform_pre_scan_documents_actions(ctx, scan_type, documents_to_scan) + add_sca_dependencies_tree_documents_if_needed(ctx, scan_type, documents_to_scan) logger.debug('Found all relevant files for scanning %s', {'path': path, 'branch': branch}) scan_documents(ctx, documents_to_scan, get_scan_parameters(ctx, (str(path),))) diff --git a/cycode/cli/apps/scan/scan_ci/scan_ci_command.py b/cycode/cli/apps/scan/scan_ci/scan_ci_command.py index cbfebb72..4303cda2 100644 --- a/cycode/cli/apps/scan/scan_ci/scan_ci_command.py +++ b/cycode/cli/apps/scan/scan_ci/scan_ci_command.py @@ -3,7 +3,7 @@ import click import typer -from cycode.cli.apps.scan.code_scanner import scan_commit_range +from cycode.cli.apps.scan.commit_range_scanner import scan_commit_range from cycode.cli.apps.scan.scan_ci.ci_integrations import get_commit_range from cycode.cli.utils.sentry import add_breadcrumb @@ -17,4 +17,4 @@ @click.pass_context def scan_ci_command(ctx: typer.Context) -> None: add_breadcrumb('ci') - scan_commit_range(ctx, path=os.getcwd(), commit_range=get_commit_range()) + scan_commit_range(ctx, repo_path=os.getcwd(), commit_range=get_commit_range()) diff --git a/cycode/cli/apps/scan/scan_command.py b/cycode/cli/apps/scan/scan_command.py index 7c2de1a6..363c409a 100644 --- a/cycode/cli/apps/scan/scan_command.py +++ b/cycode/cli/apps/scan/scan_command.py @@ -9,7 +9,7 @@ ISSUE_DETECTED_STATUS_CODE, NO_ISSUES_STATUS_CODE, ) -from cycode.cli.files_collector.excluder import excluder +from cycode.cli.files_collector.file_excluder import excluder from cycode.cli.utils import scan_utils from cycode.cli.utils.get_api_client import get_scan_cycode_client from cycode.cli.utils.sentry import add_breadcrumb diff --git a/cycode/cli/apps/scan/scan_parameters.py b/cycode/cli/apps/scan/scan_parameters.py new file mode 100644 index 00000000..c3c4ecbe --- /dev/null +++ b/cycode/cli/apps/scan/scan_parameters.py @@ -0,0 +1,46 @@ +import os +from typing import Optional + +import typer + +from cycode.cli.apps.scan.remote_url_resolver import try_get_any_remote_url +from cycode.cli.utils.scan_utils import generate_unique_scan_id +from cycode.logger import get_logger + +logger = get_logger('Scan Parameters') + + +def _get_default_scan_parameters(ctx: typer.Context) -> dict: + return { + 'monitor': ctx.obj.get('monitor'), + 'report': ctx.obj.get('report'), + 'package_vulnerabilities': ctx.obj.get('package-vulnerabilities'), + 'license_compliance': ctx.obj.get('license-compliance'), + 'command_type': ctx.info_name.replace('-', '_'), # save backward compatibility + 'aggregation_id': str(generate_unique_scan_id()), + } + + +def get_scan_parameters(ctx: typer.Context, paths: Optional[tuple[str, ...]] = None) -> dict: + scan_parameters = _get_default_scan_parameters(ctx) + + if not paths: + return scan_parameters + + scan_parameters['paths'] = paths + + if len(paths) != 1: + logger.debug('Multiple paths provided, going to ignore remote url') + return scan_parameters + + if not os.path.isdir(paths[0]): + logger.debug('Path is not a directory, going to ignore remote url') + return scan_parameters + + remote_url = try_get_any_remote_url(paths[0]) + if remote_url: + # TODO(MarshalX): remove hardcode in context + ctx.obj['remote_url'] = remote_url + scan_parameters['remote_url'] = remote_url + + return scan_parameters diff --git a/cycode/cli/apps/scan/scan_result.py b/cycode/cli/apps/scan/scan_result.py new file mode 100644 index 00000000..88bc6320 --- /dev/null +++ b/cycode/cli/apps/scan/scan_result.py @@ -0,0 +1,181 @@ +import os +from typing import TYPE_CHECKING, Optional + +import typer + +from cycode.cli import consts +from cycode.cli.apps.scan.aggregation_report import try_get_aggregation_report_url_if_needed +from cycode.cli.apps.scan.detection_excluder import exclude_irrelevant_document_detections +from cycode.cli.models import Document, DocumentDetections, LocalScanResult +from cycode.cli.utils.path_utils import get_path_by_os, normalize_file_path +from cycode.cyclient.models import ( + Detection, + DetectionSchema, + DetectionsPerFile, + ScanResultsSyncFlow, + ZippedFileScanResult, +) +from cycode.logger import get_logger + +if TYPE_CHECKING: + from cycode.cli.models import CliError + from cycode.cyclient.models import ScanDetailsResponse + from cycode.cyclient.scan_client import ScanClient + +logger = get_logger('Scan Results') + + +def _get_document_by_file_name( + documents: list[Document], file_name: str, unique_id: Optional[str] = None +) -> Optional[Document]: + for document in documents: + if normalize_file_path(document.path) == normalize_file_path(file_name) and document.unique_id == unique_id: + return document + + return None + + +def _get_document_detections( + scan_result: ZippedFileScanResult, documents_to_scan: list[Document] +) -> list[DocumentDetections]: + logger.debug('Getting document detections') + + document_detections = [] + for detections_per_file in scan_result.detections_per_file: + file_name = get_path_by_os(detections_per_file.file_name) + commit_id = detections_per_file.commit_id + + logger.debug( + 'Going to find the document of the violated file, %s', {'file_name': file_name, 'commit_id': commit_id} + ) + + document = _get_document_by_file_name(documents_to_scan, file_name, commit_id) + document_detections.append(DocumentDetections(document=document, detections=detections_per_file.detections)) + + return document_detections + + +def create_local_scan_result( + scan_result: ZippedFileScanResult, + documents_to_scan: list[Document], + command_scan_type: str, + scan_type: str, + severity_threshold: str, +) -> LocalScanResult: + document_detections = _get_document_detections(scan_result, documents_to_scan) + relevant_document_detections_list = exclude_irrelevant_document_detections( + document_detections, scan_type, command_scan_type, severity_threshold + ) + + detections_count = sum([len(document_detection.detections) for document_detection in document_detections]) + relevant_detections_count = sum( + [len(document_detections.detections) for document_detections in relevant_document_detections_list] + ) + + return LocalScanResult( + scan_id=scan_result.scan_id, + report_url=scan_result.report_url, + document_detections=relevant_document_detections_list, + issue_detected=len(relevant_document_detections_list) > 0, + detections_count=detections_count, + relevant_detections_count=relevant_detections_count, + ) + + +def _get_file_name_from_detection(scan_type: str, raw_detection: dict) -> str: + if scan_type == consts.SAST_SCAN_TYPE: + return raw_detection['detection_details']['file_path'] + if scan_type == consts.SECRET_SCAN_TYPE: + return _get_secret_file_name_from_detection(raw_detection) + + return raw_detection['detection_details']['file_name'] + + +def _get_secret_file_name_from_detection(raw_detection: dict) -> str: + file_path: str = raw_detection['detection_details']['file_path'] + file_name: str = raw_detection['detection_details']['file_name'] + return os.path.join(file_path, file_name) + + +def _map_detections_per_file_and_commit_id(scan_type: str, raw_detections: list[dict]) -> list[DetectionsPerFile]: + """Convert a list of detections (async flow) to list of DetectionsPerFile objects (sync flow). + + Args: + scan_type: Type of the scan. + raw_detections: List of detections as is returned from the server. + + Note: + This method fakes server response structure + to be able to use the same logic for both async and sync scans. + + Note: + Aggregation is performed by file name and commit ID (if available) + + """ + detections_per_files = {} + for raw_detection in raw_detections: + try: + # FIXME(MarshalX): investigate this field mapping + raw_detection['message'] = raw_detection['correlation_message'] + + file_name = _get_file_name_from_detection(scan_type, raw_detection) + detection: Detection = DetectionSchema().load(raw_detection) + commit_id: Optional[str] = detection.detection_details.get('commit_id') # could be None + group_by_key = (file_name, commit_id) + + if group_by_key in detections_per_files: + detections_per_files[group_by_key].append(detection) + else: + detections_per_files[group_by_key] = [detection] + except Exception as e: + logger.debug('Failed to parse detection', exc_info=e) + continue + + return [ + DetectionsPerFile(file_name=file_name, detections=file_detections, commit_id=commit_id) + for (file_name, commit_id), file_detections in detections_per_files.items() + ] + + +def init_default_scan_result(scan_id: str) -> ZippedFileScanResult: + return ZippedFileScanResult( + did_detect=False, + detections_per_file=[], + scan_id=scan_id, + ) + + +def get_scan_result( + cycode_client: 'ScanClient', + scan_type: str, + scan_id: str, + scan_details: 'ScanDetailsResponse', + scan_parameters: dict, +) -> ZippedFileScanResult: + if not scan_details.detections_count: + return init_default_scan_result(scan_id) + + scan_raw_detections = cycode_client.get_scan_raw_detections(scan_id) + + return ZippedFileScanResult( + did_detect=True, + detections_per_file=_map_detections_per_file_and_commit_id(scan_type, scan_raw_detections), + scan_id=scan_id, + report_url=try_get_aggregation_report_url_if_needed(scan_parameters, cycode_client, scan_type), + ) + + +def get_sync_scan_result(scan_type: str, scan_results: 'ScanResultsSyncFlow') -> ZippedFileScanResult: + return ZippedFileScanResult( + did_detect=True, + detections_per_file=_map_detections_per_file_and_commit_id(scan_type, scan_results.detection_messages), + scan_id=scan_results.id, + ) + + +def print_local_scan_results( + ctx: typer.Context, local_scan_results: list[LocalScanResult], errors: Optional[dict[str, 'CliError']] = None +) -> None: + printer = ctx.obj.get('console_printer') + printer.update_ctx(ctx) + printer.print_scan_results(local_scan_results, errors) diff --git a/cycode/cli/consts.py b/cycode/cli/consts.py index c0ed33f0..74a9758c 100644 --- a/cycode/cli/consts.py +++ b/cycode/cli/consts.py @@ -140,9 +140,11 @@ 'conan': ['conanfile.py', 'conanfile.txt', 'conan.lock'], } -COMMIT_RANGE_SCAN_SUPPORTED_SCAN_TYPES = [SECRET_SCAN_TYPE, SCA_SCAN_TYPE] +COMMIT_RANGE_SCAN_SUPPORTED_SCAN_TYPES = [SECRET_SCAN_TYPE, SCA_SCAN_TYPE, SAST_SCAN_TYPE] COMMIT_RANGE_BASED_COMMAND_SCAN_TYPES = [ + PRE_COMMIT_COMMAND_SCAN_TYPE, + PRE_COMMIT_COMMAND_SCAN_TYPE_OLD, PRE_RECEIVE_COMMAND_SCAN_TYPE, PRE_RECEIVE_COMMAND_SCAN_TYPE_OLD, COMMIT_HISTORY_COMMAND_SCAN_TYPE, diff --git a/cycode/cli/files_collector/commit_range_documents.py b/cycode/cli/files_collector/commit_range_documents.py new file mode 100644 index 00000000..68d18978 --- /dev/null +++ b/cycode/cli/files_collector/commit_range_documents.py @@ -0,0 +1,289 @@ +import os +import sys +from typing import TYPE_CHECKING, Optional + +import typer + +from cycode.cli import consts +from cycode.cli.files_collector.repository_documents import ( + get_file_content_from_commit_path, +) +from cycode.cli.models import Document +from cycode.cli.utils.git_proxy import git_proxy +from cycode.cli.utils.path_utils import get_file_content, get_path_by_os +from cycode.cli.utils.progress_bar import ScanProgressBarSection +from cycode.logger import get_logger + +if TYPE_CHECKING: + from git import Diff, Repo + + from cycode.cli.utils.progress_bar import BaseProgressBar, ProgressBarSection + +logger = get_logger('Commit Range Collector') + + +def _does_reach_to_max_commits_to_scan_limit(commit_ids: list[str], max_commits_count: Optional[int]) -> bool: + if max_commits_count is None: + return False + + return len(commit_ids) >= max_commits_count + + +def collect_commit_range_diff_documents( + ctx: typer.Context, path: str, commit_range: str, max_commits_count: Optional[int] = None +) -> list[Document]: + """Collects documents from a specified commit range in a Git repository. + + Return a list of Document objects containing the diffs of files changed in each commit. + """ + progress_bar = ctx.obj['progress_bar'] + + commit_ids_to_scan = [] + commit_documents_to_scan = [] + + repo = git_proxy.get_repo(path) + total_commits_count = int(repo.git.rev_list('--count', commit_range)) + logger.debug('Calculating diffs for %s commits in the commit range %s', total_commits_count, commit_range) + + progress_bar.set_section_length(ScanProgressBarSection.PREPARE_LOCAL_FILES, total_commits_count) + + for scanned_commits_count, commit in enumerate(repo.iter_commits(rev=commit_range)): + if _does_reach_to_max_commits_to_scan_limit(commit_ids_to_scan, max_commits_count): + logger.debug('Reached to max commits to scan count. Going to scan only %s last commits', max_commits_count) + progress_bar.update(ScanProgressBarSection.PREPARE_LOCAL_FILES, total_commits_count - scanned_commits_count) + break + + progress_bar.update(ScanProgressBarSection.PREPARE_LOCAL_FILES) + + commit_id = commit.hexsha + commit_ids_to_scan.append(commit_id) + parent = commit.parents[0] if commit.parents else git_proxy.get_null_tree() + diff_index = commit.diff(parent, create_patch=True, R=True) + for diff in diff_index: + commit_documents_to_scan.append( + Document( + path=get_path_by_os(get_diff_file_path(diff)), + content=get_diff_file_content(diff), + is_git_diff_format=True, + unique_id=commit_id, + ) + ) + + logger.debug( + 'Found all relevant files in commit %s', + {'path': path, 'commit_range': commit_range, 'commit_id': commit_id}, + ) + + logger.debug('List of commit ids to scan, %s', {'commit_ids': commit_ids_to_scan}) + + return commit_documents_to_scan + + +def calculate_pre_receive_commit_range(branch_update_details: str) -> Optional[str]: + end_commit = _get_end_commit_from_branch_update_details(branch_update_details) + + # branch is deleted, no need to perform scan + if end_commit == consts.EMPTY_COMMIT_SHA: + return None + + start_commit = _get_oldest_unupdated_commit_for_branch(end_commit) + + # no new commit to update found + if not start_commit: + return None + + return f'{start_commit}~1...{end_commit}' + + +def _get_end_commit_from_branch_update_details(update_details: str) -> str: + # update details pattern: + _, end_commit, _ = update_details.split() + return end_commit + + +def _get_oldest_unupdated_commit_for_branch(commit: str) -> Optional[str]: + # get a list of commits by chronological order that are not in the remote repository yet + # more info about rev-list command: https://git-scm.com/docs/git-rev-list + repo = git_proxy.get_repo(os.getcwd()) + not_updated_commits = repo.git.rev_list(commit, '--topo-order', '--reverse', '--not', '--all') + + commits = not_updated_commits.splitlines() + if not commits: + return None + + return commits[0] + + +def _get_file_content_from_commit_diff(repo: 'Repo', commit: str, diff: 'Diff') -> Optional[str]: + file_path = get_diff_file_path(diff, relative=True) + return get_file_content_from_commit_path(repo, commit, file_path) + + +def get_commit_range_modified_documents( + progress_bar: 'BaseProgressBar', + progress_bar_section: 'ProgressBarSection', + path: str, + from_commit_rev: str, + to_commit_rev: str, + reverse_diff: bool = True, +) -> tuple[list[Document], list[Document], list[Document]]: + from_commit_documents = [] + to_commit_documents = [] + diff_documents = [] + + repo = git_proxy.get_repo(path) + diff_index = repo.commit(from_commit_rev).diff(to_commit_rev, create_patch=True, R=reverse_diff) + + modified_files_diff = [ + diff for diff in diff_index if diff.change_type != consts.COMMIT_DIFF_DELETED_FILE_CHANGE_TYPE + ] + progress_bar.set_section_length(progress_bar_section, len(modified_files_diff)) + for diff in modified_files_diff: + progress_bar.update(progress_bar_section) + + file_path = get_path_by_os(get_diff_file_path(diff)) + + diff_documents.append( + Document( + path=file_path, + content=get_diff_file_content(diff), + is_git_diff_format=True, + ) + ) + + file_content = _get_file_content_from_commit_diff(repo, from_commit_rev, diff) + if file_content is not None: + from_commit_documents.append(Document(file_path, file_content)) + + file_content = _get_file_content_from_commit_diff(repo, to_commit_rev, diff) + if file_content is not None: + to_commit_documents.append(Document(file_path, file_content)) + + return from_commit_documents, to_commit_documents, diff_documents + + +def parse_pre_receive_input() -> str: + """Parse input to pushed branch update details. + + Example input: + old_value new_value refname + ----------------------------------------------- + 0000000000000000000000000000000000000000 9cf90954ef26e7c58284f8ebf7dcd0fcf711152a refs/heads/main + 973a96d3e925b65941f7c47fa16129f1577d499f 0000000000000000000000000000000000000000 refs/heads/feature-branch + 59564ef68745bca38c42fc57a7822efd519a6bd9 3378e52dcfa47fb11ce3a4a520bea5f85d5d0bf3 refs/heads/develop + + :return: First branch update details (input's first line) + """ + # FIXME(MarshalX): this blocks main thread forever if called outside of pre-receive hook + pre_receive_input = sys.stdin.read().strip() + if not pre_receive_input: + raise ValueError( + 'Pre receive input was not found. Make sure that you are using this command only in pre-receive hook' + ) + + # each line represents a branch update request, handle the first one only + # TODO(MichalBor): support case of multiple update branch requests + return pre_receive_input.splitlines()[0] + + +def get_diff_file_path(diff: 'Diff', relative: bool = False) -> Optional[str]: + if relative: + # relative to the repository root + return diff.b_path if diff.b_path else diff.a_path + + if diff.b_blob: + return diff.b_blob.abspath + return diff.a_blob.abspath + + +def get_diff_file_content(diff: 'Diff') -> str: + return diff.diff.decode('UTF-8', errors='replace') + + +def get_pre_commit_modified_documents( + progress_bar: 'BaseProgressBar', + progress_bar_section: 'ProgressBarSection', + repo_path: str, +) -> tuple[list[Document], list[Document], list[Document]]: + git_head_documents = [] + pre_committed_documents = [] + diff_documents = [] + + repo = git_proxy.get_repo(repo_path) + diff_index = repo.index.diff(consts.GIT_HEAD_COMMIT_REV, create_patch=True, R=True) + progress_bar.set_section_length(progress_bar_section, len(diff_index)) + for diff in diff_index: + progress_bar.update(progress_bar_section) + + file_path = get_path_by_os(get_diff_file_path(diff)) + + diff_documents.append( + Document( + path=file_path, + content=get_diff_file_content(diff), + is_git_diff_format=True, + ) + ) + + file_content = _get_file_content_from_commit_diff(repo, consts.GIT_HEAD_COMMIT_REV, diff) + if file_content: + git_head_documents.append(Document(file_path, file_content)) + + if os.path.exists(file_path): + file_content = get_file_content(file_path) + if file_content: + pre_committed_documents.append(Document(file_path, file_content)) + + return git_head_documents, pre_committed_documents, diff_documents + + +def parse_commit_range_sca(commit_range: str, path: str) -> tuple[Optional[str], Optional[str]]: + # FIXME(MarshalX): i truly believe that this function does NOT work as expected + # it does not handle cases like 'A..B' correctly + # i leave it as it for SCA to not break anything + # the more correct approach is implemented for SAST + from_commit_rev = to_commit_rev = None + + for commit in git_proxy.get_repo(path).iter_commits(rev=commit_range): + if not to_commit_rev: + to_commit_rev = commit.hexsha + from_commit_rev = commit.hexsha + + return from_commit_rev, to_commit_rev + + +def parse_commit_range_sast(commit_range: str, path: str) -> tuple[Optional[str], Optional[str]]: + """Parses a git commit range string and returns the full SHAs for the 'from' and 'to' commits. + + Supports: + - 'from..to' + - 'from...to' + - 'commit' (interpreted as 'commit..HEAD') + - '..to' (interpreted as 'HEAD..to') + - 'from..' (interpreted as 'from..HEAD') + """ + repo = git_proxy.get_repo(path) + + if '...' in commit_range: + from_spec, to_spec = commit_range.split('...', 1) + elif '..' in commit_range: + from_spec, to_spec = commit_range.split('..', 1) + else: + # Git commands like 'git diff ' compare against HEAD. + from_spec = commit_range + to_spec = 'HEAD' + + # If a spec is empty (e.g., from '..master'), default it to 'HEAD' + if not from_spec: + from_spec = 'HEAD' + if not to_spec: + to_spec = 'HEAD' + + try: + # Use rev_parse to resolve each specifier to its full commit SHA + from_commit_rev = repo.rev_parse(from_spec).hexsha + to_commit_rev = repo.rev_parse(to_spec).hexsha + return from_commit_rev, to_commit_rev + except git_proxy.get_git_command_error() as e: + logger.warning("Failed to parse commit range '%s'", commit_range, exc_info=e) + return None, None diff --git a/cycode/cli/files_collector/excluder.py b/cycode/cli/files_collector/file_excluder.py similarity index 100% rename from cycode/cli/files_collector/excluder.py rename to cycode/cli/files_collector/file_excluder.py diff --git a/cycode/cli/files_collector/path_documents.py b/cycode/cli/files_collector/path_documents.py index 556a8cf8..73cd0768 100644 --- a/cycode/cli/files_collector/path_documents.py +++ b/cycode/cli/files_collector/path_documents.py @@ -1,7 +1,7 @@ import os from typing import TYPE_CHECKING -from cycode.cli.files_collector.excluder import excluder +from cycode.cli.files_collector.file_excluder import excluder from cycode.cli.files_collector.iac.tf_content_generator import ( generate_tf_content_from_tfplan, generate_tfplan_document_name, diff --git a/cycode/cli/files_collector/repository_documents.py b/cycode/cli/files_collector/repository_documents.py index 379346f8..935d3db1 100644 --- a/cycode/cli/files_collector/repository_documents.py +++ b/cycode/cli/files_collector/repository_documents.py @@ -1,146 +1,26 @@ -import os from collections.abc import Iterator from typing import TYPE_CHECKING, Optional, Union -from cycode.cli import consts -from cycode.cli.files_collector.sca.sca_code_scanner import get_file_content_from_commit_diff -from cycode.cli.models import Document from cycode.cli.utils.git_proxy import git_proxy -from cycode.cli.utils.path_utils import get_file_content, get_path_by_os if TYPE_CHECKING: - from git import Blob, Diff + from git import Blob, Repo from git.objects.base import IndexObjUnion from git.objects.tree import TraversedTreeTup - from cycode.cli.utils.progress_bar import BaseProgressBar, ProgressBarSection - -def should_process_git_object(obj: 'Blob', _: int) -> bool: +def _should_process_git_object(obj: 'Blob', _: int) -> bool: return obj.type == 'blob' and obj.size > 0 def get_git_repository_tree_file_entries( path: str, branch: str ) -> Union[Iterator['IndexObjUnion'], Iterator['TraversedTreeTup']]: - return git_proxy.get_repo(path).tree(branch).traverse(predicate=should_process_git_object) - - -def parse_commit_range(commit_range: str, path: str) -> tuple[str, str]: - from_commit_rev = None - to_commit_rev = None - - for commit in git_proxy.get_repo(path).iter_commits(rev=commit_range): - if not to_commit_rev: - to_commit_rev = commit.hexsha - from_commit_rev = commit.hexsha - - return from_commit_rev, to_commit_rev - - -def get_diff_file_path(file: 'Diff', relative: bool = False) -> Optional[str]: - if relative: - # relative to the repository root - return file.b_path if file.b_path else file.a_path - - if file.b_blob: - return file.b_blob.abspath - return file.a_blob.abspath - - -def get_diff_file_content(file: 'Diff') -> str: - return file.diff.decode('UTF-8', errors='replace') - - -def get_pre_commit_modified_documents( - progress_bar: 'BaseProgressBar', - progress_bar_section: 'ProgressBarSection', - repo_path: str, -) -> tuple[list[Document], list[Document]]: - git_head_documents = [] - pre_committed_documents = [] - - repo = git_proxy.get_repo(repo_path) - diff_index = repo.index.diff(consts.GIT_HEAD_COMMIT_REV, create_patch=True, R=True) - progress_bar.set_section_length(progress_bar_section, len(diff_index)) - for diff in diff_index: - progress_bar.update(progress_bar_section) - - file_path = get_path_by_os(get_diff_file_path(diff)) - file_content = get_file_content_from_commit_diff(repo, consts.GIT_HEAD_COMMIT_REV, diff) - if file_content is not None: - git_head_documents.append(Document(file_path, file_content)) - - if os.path.exists(file_path): - file_content = get_file_content(file_path) - pre_committed_documents.append(Document(file_path, file_content)) - - return git_head_documents, pre_committed_documents - - -def get_commit_range_modified_documents( - progress_bar: 'BaseProgressBar', - progress_bar_section: 'ProgressBarSection', - path: str, - from_commit_rev: str, - to_commit_rev: str, -) -> tuple[list[Document], list[Document]]: - from_commit_documents = [] - to_commit_documents = [] + return git_proxy.get_repo(path).tree(branch).traverse(predicate=_should_process_git_object) - repo = git_proxy.get_repo(path) - diff = repo.commit(from_commit_rev).diff(to_commit_rev) - modified_files_diff = [ - change for change in diff if change.change_type != consts.COMMIT_DIFF_DELETED_FILE_CHANGE_TYPE - ] - progress_bar.set_section_length(progress_bar_section, len(modified_files_diff)) - for blob in modified_files_diff: - progress_bar.update(progress_bar_section) - - file_path = get_path_by_os(get_diff_file_path(blob)) - - file_content = get_file_content_from_commit_diff(repo, from_commit_rev, blob) - if file_content is not None: - from_commit_documents.append(Document(file_path, file_content)) - - file_content = get_file_content_from_commit_diff(repo, to_commit_rev, blob) - if file_content is not None: - to_commit_documents.append(Document(file_path, file_content)) - - return from_commit_documents, to_commit_documents - - -def calculate_pre_receive_commit_range(branch_update_details: str) -> Optional[str]: - end_commit = _get_end_commit_from_branch_update_details(branch_update_details) - - # branch is deleted, no need to perform scan - if end_commit == consts.EMPTY_COMMIT_SHA: - return None - - start_commit = _get_oldest_unupdated_commit_for_branch(end_commit) - - # no new commit to update found - if not start_commit: +def get_file_content_from_commit_path(repo: 'Repo', commit: str, file_path: str) -> Optional[str]: + try: + return repo.git.show(f'{commit}:{file_path}') + except git_proxy.get_git_command_error(): return None - - return f'{start_commit}~1...{end_commit}' - - -def _get_end_commit_from_branch_update_details(update_details: str) -> str: - # update details pattern: - _, end_commit, _ = update_details.split() - return end_commit - - -def _get_oldest_unupdated_commit_for_branch(commit: str) -> Optional[str]: - # get a list of commits by chronological order that are not in the remote repository yet - # more info about rev-list command: https://git-scm.com/docs/git-rev-list - repo = git_proxy.get_repo(os.getcwd()) - not_updated_commits = repo.git.rev_list(commit, '--topo-order', '--reverse', '--not', '--all') - - commits = not_updated_commits.splitlines() - if not commits: - return None - - return commits[0] diff --git a/cycode/cli/files_collector/sca/sca_code_scanner.py b/cycode/cli/files_collector/sca/sca_file_collector.py similarity index 70% rename from cycode/cli/files_collector/sca/sca_code_scanner.py rename to cycode/cli/files_collector/sca/sca_file_collector.py index febd8858..e3ed22f8 100644 --- a/cycode/cli/files_collector/sca/sca_code_scanner.py +++ b/cycode/cli/files_collector/sca/sca_file_collector.py @@ -3,6 +3,7 @@ import typer from cycode.cli import consts +from cycode.cli.files_collector.repository_documents import get_file_content_from_commit_path from cycode.cli.files_collector.sca.base_restore_dependencies import BaseRestoreDependencies from cycode.cli.files_collector.sca.go.restore_go_dependencies import RestoreGoDependencies from cycode.cli.files_collector.sca.maven.restore_gradle_dependencies import RestoreGradleDependencies @@ -17,15 +18,30 @@ from cycode.logger import get_logger if TYPE_CHECKING: - from git import Diff, Repo + from git import Repo BUILD_DEP_TREE_TIMEOUT = 180 -logger = get_logger('SCA Code Scanner') +logger = get_logger('SCA File Collector') -def perform_pre_commit_range_scan_actions( +def _add_ecosystem_related_files_if_exists( + documents: list[Document], repo: Optional['Repo'] = None, commit_rev: Optional[str] = None +) -> None: + documents_to_add: list[Document] = [] + for doc in documents: + ecosystem = _get_project_file_ecosystem(doc) + if ecosystem is None: + logger.debug('Failed to resolve project file ecosystem: %s', doc.path) + continue + + documents_to_add.extend(_get_doc_ecosystem_related_project_files(doc, documents, ecosystem, commit_rev, repo)) + + documents.extend(documents_to_add) + + +def perform_sca_pre_commit_range_scan_actions( path: str, from_commit_documents: list[Document], from_commit_rev: str, @@ -33,40 +49,25 @@ def perform_pre_commit_range_scan_actions( to_commit_rev: str, ) -> None: repo = git_proxy.get_repo(path) - add_ecosystem_related_files_if_exists(from_commit_documents, repo, from_commit_rev) - add_ecosystem_related_files_if_exists(to_commit_documents, repo, to_commit_rev) + _add_ecosystem_related_files_if_exists(from_commit_documents, repo, from_commit_rev) + _add_ecosystem_related_files_if_exists(to_commit_documents, repo, to_commit_rev) -def perform_pre_hook_range_scan_actions( +def perform_sca_pre_hook_range_scan_actions( repo_path: str, git_head_documents: list[Document], pre_committed_documents: list[Document] ) -> None: repo = git_proxy.get_repo(repo_path) - add_ecosystem_related_files_if_exists(git_head_documents, repo, consts.GIT_HEAD_COMMIT_REV) - add_ecosystem_related_files_if_exists(pre_committed_documents) - - -def add_ecosystem_related_files_if_exists( - documents: list[Document], repo: Optional['Repo'] = None, commit_rev: Optional[str] = None -) -> None: - documents_to_add: list[Document] = [] - for doc in documents: - ecosystem = get_project_file_ecosystem(doc) - if ecosystem is None: - logger.debug('Failed to resolve project file ecosystem: %s', doc.path) - continue + _add_ecosystem_related_files_if_exists(git_head_documents, repo, consts.GIT_HEAD_COMMIT_REV) + _add_ecosystem_related_files_if_exists(pre_committed_documents) - documents_to_add.extend(get_doc_ecosystem_related_project_files(doc, documents, ecosystem, commit_rev, repo)) - - documents.extend(documents_to_add) - -def get_doc_ecosystem_related_project_files( +def _get_doc_ecosystem_related_project_files( doc: Document, documents: list[Document], ecosystem: str, commit_rev: Optional[str], repo: Optional['Repo'] ) -> list[Document]: documents_to_add: list[Document] = [] for ecosystem_project_file in consts.PROJECT_FILES_BY_ECOSYSTEM_MAP.get(ecosystem): file_to_search = join_paths(get_file_dir(doc.path), ecosystem_project_file) - if not is_project_file_exists_in_documents(documents, file_to_search): + if not _is_project_file_exists_in_documents(documents, file_to_search): if repo: file_content = get_file_content_from_commit_path(repo, commit_rev, file_to_search) else: @@ -78,11 +79,11 @@ def get_doc_ecosystem_related_project_files( return documents_to_add -def is_project_file_exists_in_documents(documents: list[Document], file: str) -> bool: +def _is_project_file_exists_in_documents(documents: list[Document], file: str) -> bool: return any(doc for doc in documents if file == doc.path) -def get_project_file_ecosystem(document: Document) -> Optional[str]: +def _get_project_file_ecosystem(document: Document) -> Optional[str]: for ecosystem, project_files in consts.PROJECT_FILES_BY_ECOSYSTEM_MAP.items(): for project_file in project_files: if document.path.endswith(project_file): @@ -90,7 +91,11 @@ def get_project_file_ecosystem(document: Document) -> Optional[str]: return None -def try_restore_dependencies( +def _get_manifest_file_path(document: Document, is_monitor_action: bool, project_path: str) -> str: + return join_paths(project_path, document.path) if is_monitor_action else document.path + + +def _try_restore_dependencies( ctx: typer.Context, restore_dependencies: 'BaseRestoreDependencies', document: Document, @@ -110,34 +115,13 @@ def try_restore_dependencies( is_monitor_action = ctx.obj.get('monitor', False) project_path = get_path_from_context(ctx) - manifest_file_path = get_manifest_file_path(document, is_monitor_action, project_path) + manifest_file_path = _get_manifest_file_path(document, is_monitor_action, project_path) logger.debug('Succeeded to generate dependencies tree on path: %s', manifest_file_path) return restore_dependencies_document -def add_dependencies_tree_document( - ctx: typer.Context, documents_to_scan: list[Document], is_git_diff: bool = False -) -> None: - documents_to_add: dict[str, Document] = {document.path: document for document in documents_to_scan} - restore_dependencies_list = restore_handlers(ctx, is_git_diff) - - for restore_dependencies in restore_dependencies_list: - for document in documents_to_scan: - restore_dependencies_document = try_restore_dependencies(ctx, restore_dependencies, document) - if restore_dependencies_document is None: - continue - - if restore_dependencies_document.path in documents_to_add: - logger.debug('Duplicate document on restore for path: %s', restore_dependencies_document.path) - else: - documents_to_add[restore_dependencies_document.path] = restore_dependencies_document - - # mutate original list using slice assignment - documents_to_scan[:] = list(documents_to_add.values()) - - -def restore_handlers(ctx: typer.Context, is_git_diff: bool) -> list[BaseRestoreDependencies]: +def _get_restore_handlers(ctx: typer.Context, is_git_diff: bool) -> list[BaseRestoreDependencies]: return [ RestoreGradleDependencies(ctx, is_git_diff, BUILD_DEP_TREE_TIMEOUT), RestoreMavenDependencies(ctx, is_git_diff, BUILD_DEP_TREE_TIMEOUT), @@ -149,28 +133,38 @@ def restore_handlers(ctx: typer.Context, is_git_diff: bool) -> list[BaseRestoreD ] -def get_manifest_file_path(document: Document, is_monitor_action: bool, project_path: str) -> str: - return join_paths(project_path, document.path) if is_monitor_action else document.path +def _add_dependencies_tree_documents( + ctx: typer.Context, documents_to_scan: list[Document], is_git_diff: bool = False +) -> None: + logger.debug( + 'Adding dependencies tree documents, %s', + {'documents_count': len(documents_to_scan), 'is_git_diff': is_git_diff}, + ) + documents_to_add: dict[str, Document] = {document.path: document for document in documents_to_scan} + restore_dependencies_list = _get_restore_handlers(ctx, is_git_diff) -def get_file_content_from_commit_path(repo: 'Repo', commit: str, file_path: str) -> Optional[str]: - try: - return repo.git.show(f'{commit}:{file_path}') - except git_proxy.get_git_command_error(): - return None + for restore_dependencies in restore_dependencies_list: + for document in documents_to_scan: + restore_dependencies_document = _try_restore_dependencies(ctx, restore_dependencies, document) + if restore_dependencies_document is None: + continue + if restore_dependencies_document.path in documents_to_add: + logger.debug('Duplicate document on restore for path: %s', restore_dependencies_document.path) + else: + logger.debug('Adding dependencies tree document, %s', restore_dependencies_document.path) + documents_to_add[restore_dependencies_document.path] = restore_dependencies_document -def get_file_content_from_commit_diff(repo: 'Repo', commit: str, diff: 'Diff') -> Optional[str]: - from cycode.cli.files_collector.repository_documents import get_diff_file_path + logger.debug('Finished adding dependencies tree documents, %s', {'documents_count': len(documents_to_add)}) - file_path = get_diff_file_path(diff, relative=True) - return get_file_content_from_commit_path(repo, commit, file_path) + # mutate original list using slice assignment + documents_to_scan[:] = list(documents_to_add.values()) -def perform_pre_scan_documents_actions( +def add_sca_dependencies_tree_documents_if_needed( ctx: typer.Context, scan_type: str, documents_to_scan: list[Document], is_git_diff: bool = False ) -> None: no_restore = ctx.params.get('no-restore', False) if scan_type == consts.SCA_SCAN_TYPE and not no_restore: - logger.debug('Perform pre-scan document add_dependencies_tree_document action') - add_dependencies_tree_document(ctx, documents_to_scan, is_git_diff) + _add_dependencies_tree_documents(ctx, documents_to_scan, is_git_diff) diff --git a/cycode/cli/files_collector/zip_documents.py b/cycode/cli/files_collector/zip_documents.py index 770121fa..6f5edd81 100644 --- a/cycode/cli/files_collector/zip_documents.py +++ b/cycode/cli/files_collector/zip_documents.py @@ -27,7 +27,7 @@ def zip_documents(scan_type: str, documents: list[Document], zip_file: Optional[ _validate_zip_file_size(scan_type, zip_file.size) logger.debug( - 'Adding file, %s', + 'Adding file to ZIP, %s', {'index': index, 'filename': document.path, 'unique_id': document.unique_id}, ) zip_file.append(document.path, document.unique_id, document.content) @@ -37,13 +37,13 @@ def zip_documents(scan_type: str, documents: list[Document], zip_file: Optional[ end_zip_creation_time = timeit.default_timer() zip_creation_time = int(end_zip_creation_time - start_zip_creation_time) logger.debug( - 'Finished to create file, %s', + 'Finished to create ZIP file, %s', {'zip_creation_time': zip_creation_time, 'zip_size': zip_file.size, 'documents_count': len(documents)}, ) if zip_file.configuration_manager.get_debug_flag(): zip_file_path = Path.joinpath(Path.cwd(), f'{scan_type}_scan_{end_zip_creation_time}.zip') - logger.debug('Writing file to disk, %s', {'zip_file_path': zip_file_path}) + logger.debug('Writing ZIP file to disk, %s', {'zip_file_path': zip_file_path}) zip_file.write_on_disk(zip_file_path) return zip_file diff --git a/cycode/cli/printers/tables/sca_table_printer.py b/cycode/cli/printers/tables/sca_table_printer.py index 1bf358c8..c0bedcc7 100644 --- a/cycode/cli/printers/tables/sca_table_printer.py +++ b/cycode/cli/printers/tables/sca_table_printer.py @@ -63,7 +63,7 @@ def _get_table(self, policy_id: str) -> Table: elif policy_id == LICENSE_COMPLIANCE_POLICY_ID: table.add_column(LICENSE_COLUMN) - if is_git_diff_based_scan(self.scan_type, self.command_scan_type): + if is_git_diff_based_scan(self.command_scan_type): table.add_column(REPOSITORY_COLUMN) table.add_column(SEVERITY_COLUMN) diff --git a/cycode/cli/printers/tables/table_printer.py b/cycode/cli/printers/tables/table_printer.py index 6fc85a1b..6a5dd198 100644 --- a/cycode/cli/printers/tables/table_printer.py +++ b/cycode/cli/printers/tables/table_printer.py @@ -48,7 +48,7 @@ def _get_table(self) -> Table: table.add_column(LINE_NUMBER_COLUMN) table.add_column(COLUMN_NUMBER_COLUMN) - if is_git_diff_based_scan(self.scan_type, self.command_scan_type): + if is_git_diff_based_scan(self.command_scan_type): table.add_column(COMMIT_SHA_COLUMN) if self.scan_type == SECRET_SCAN_TYPE: diff --git a/cycode/cli/printers/utils/__init__.py b/cycode/cli/printers/utils/__init__.py index e1676c35..d1ee86c0 100644 --- a/cycode/cli/printers/utils/__init__.py +++ b/cycode/cli/printers/utils/__init__.py @@ -1,8 +1,5 @@ from cycode.cli import consts -def is_git_diff_based_scan(scan_type: str, command_scan_type: str) -> bool: - return ( - command_scan_type in consts.COMMIT_RANGE_BASED_COMMAND_SCAN_TYPES - and scan_type in consts.COMMIT_RANGE_SCAN_SUPPORTED_SCAN_TYPES - ) +def is_git_diff_based_scan(command_scan_type: str) -> bool: + return command_scan_type in consts.COMMIT_RANGE_BASED_COMMAND_SCAN_TYPES diff --git a/cycode/cli/printers/utils/code_snippet_syntax.py b/cycode/cli/printers/utils/code_snippet_syntax.py index 12501544..d9ea3af2 100644 --- a/cycode/cli/printers/utils/code_snippet_syntax.py +++ b/cycode/cli/printers/utils/code_snippet_syntax.py @@ -108,7 +108,7 @@ def get_code_snippet_syntax( lines_to_display_after: int = 1, obfuscate: bool = True, ) -> Syntax: - if is_git_diff_based_scan(scan_type, command_scan_type): + if is_git_diff_based_scan(command_scan_type): # it will return syntax with just one line return _get_code_snippet_syntax_from_git_diff(scan_type, detection, document, obfuscate) diff --git a/cycode/cli/utils/path_utils.py b/cycode/cli/utils/path_utils.py index 7d525e56..ce60b0da 100644 --- a/cycode/cli/utils/path_utils.py +++ b/cycode/cli/utils/path_utils.py @@ -111,3 +111,11 @@ def get_path_from_context(ctx: typer.Context) -> Optional[str]: if path is None and 'paths' in ctx.params: path = ctx.params['paths'][0] return path + + +def normalize_file_path(path: str) -> str: + if path.startswith('/'): + return path[1:] + if path.startswith('./'): + return path[2:] + return path diff --git a/cycode/cli/utils/scan_utils.py b/cycode/cli/utils/scan_utils.py index 8c9dcca7..57586b51 100644 --- a/cycode/cli/utils/scan_utils.py +++ b/cycode/cli/utils/scan_utils.py @@ -1,11 +1,29 @@ +import os +from typing import TYPE_CHECKING +from uuid import UUID, uuid4 + import typer +if TYPE_CHECKING: + from cycode.cli.models import LocalScanResult + def set_issue_detected(ctx: typer.Context, issue_detected: bool) -> None: ctx.obj['issue_detected'] = issue_detected +def set_issue_detected_by_scan_results(ctx: typer.Context, scan_results: list['LocalScanResult']) -> None: + set_issue_detected(ctx, any(scan_result.issue_detected for scan_result in scan_results)) + + def is_scan_failed(ctx: typer.Context) -> bool: did_fail = ctx.obj.get('did_fail') issue_detected = ctx.obj.get('issue_detected') return did_fail or issue_detected + + +def generate_unique_scan_id() -> UUID: + if 'PYTEST_TEST_UNIQUE_ID' in os.environ: + return UUID(os.environ['PYTEST_TEST_UNIQUE_ID']) + + return uuid4() diff --git a/cycode/cyclient/scan_client.py b/cycode/cyclient/scan_client.py index b1c697c6..6ddce8d5 100644 --- a/cycode/cyclient/scan_client.py +++ b/cycode/cyclient/scan_client.py @@ -92,6 +92,15 @@ def zipped_file_scan_sync( ) return models.ScanResultsSyncFlowSchema().load(response.json()) + @staticmethod + def _create_compression_manifest_string(zip_file: InMemoryZip) -> str: + return json.dumps( + { + 'file_count_by_extension': zip_file.extension_statistics, + 'file_count': zip_file.files_count, + } + ) + def zipped_file_scan_async( self, zip_file: InMemoryZip, @@ -102,24 +111,19 @@ def zipped_file_scan_async( ) -> models.ScanInitializationResponse: files = {'file': ('multiple_files_scan.zip', zip_file.read())} - compression_manifest = { - 'file_count_by_extension': zip_file.extension_statistics, - 'file_count': zip_file.files_count, - } - response = self.scan_cycode_client.post( url_path=self.get_zipped_file_scan_async_url_path(scan_type), data={ 'is_git_diff': is_git_diff, 'scan_parameters': json.dumps(scan_parameters), 'is_commit_range': is_commit_range, - 'compression_manifest': json.dumps(compression_manifest), + 'compression_manifest': self._create_compression_manifest_string(zip_file), }, files=files, ) return models.ScanInitializationResponseSchema().load(response.json()) - def multiple_zipped_file_scan_async( + def commit_range_scan_async( self, from_commit_zip_file: InMemoryZip, to_commit_zip_file: InMemoryZip, @@ -127,6 +131,20 @@ def multiple_zipped_file_scan_async( scan_parameters: dict, is_git_diff: bool = False, ) -> models.ScanInitializationResponse: + """Commit range scan. + Used by SCA and SAST scans. + + For SCA: + - from_commit_zip_file is file content + - to_commit_zip_file is file content + + For SAST: + - from_commit_zip_file is file content + - to_commit_zip_file is diff content + + Note: + Compression manifest is supported only for SAST scans. + """ url_path = f'{self.get_scan_service_url_path(scan_type)}/{scan_type}/repository/commit-range' files = { 'file_from_commit': ('multiple_files_scan.zip', from_commit_zip_file.read()), @@ -134,7 +152,11 @@ def multiple_zipped_file_scan_async( } response = self.scan_cycode_client.post( url_path=url_path, - data={'is_git_diff': is_git_diff, 'scan_parameters': json.dumps(scan_parameters)}, + data={ + 'is_git_diff': is_git_diff, + 'scan_parameters': json.dumps(scan_parameters), + 'compression_manifest': self._create_compression_manifest_string(from_commit_zip_file), + }, files=files, ) return models.ScanInitializationResponseSchema().load(response.json()) diff --git a/images/sca_report_url.png b/images/sca_report_url.png index f438180e..6513966e 100644 Binary files a/images/sca_report_url.png and b/images/sca_report_url.png differ diff --git a/tests/cli/commands/scan/test_code_scanner.py b/tests/cli/commands/scan/test_code_scanner.py index 3151684e..bf4d3574 100644 --- a/tests/cli/commands/scan/test_code_scanner.py +++ b/tests/cli/commands/scan/test_code_scanner.py @@ -1,8 +1,7 @@ import os from cycode.cli import consts -from cycode.cli.apps.scan.code_scanner import _does_severity_match_severity_threshold -from cycode.cli.files_collector.excluder import _is_file_relevant_for_sca_scan +from cycode.cli.files_collector.file_excluder import _is_file_relevant_for_sca_scan from cycode.cli.files_collector.path_documents import _generate_document from cycode.cli.models import Document @@ -73,24 +72,3 @@ def test_generate_document() -> None: assert isinstance(generated_tfplan_document, Document) assert generated_tfplan_document.path.endswith('.tf') assert generated_tfplan_document.is_git_diff_format == is_git_diff - - -def test_does_severity_match_severity_threshold() -> None: - assert _does_severity_match_severity_threshold('INFO', 'LOW') is False - - assert _does_severity_match_severity_threshold('LOW', 'LOW') is True - assert _does_severity_match_severity_threshold('LOW', 'MEDIUM') is False - - assert _does_severity_match_severity_threshold('MEDIUM', 'LOW') is True - assert _does_severity_match_severity_threshold('MEDIUM', 'MEDIUM') is True - assert _does_severity_match_severity_threshold('MEDIUM', 'HIGH') is False - - assert _does_severity_match_severity_threshold('HIGH', 'MEDIUM') is True - assert _does_severity_match_severity_threshold('HIGH', 'HIGH') is True - assert _does_severity_match_severity_threshold('HIGH', 'CRITICAL') is False - - assert _does_severity_match_severity_threshold('CRITICAL', 'HIGH') is True - assert _does_severity_match_severity_threshold('CRITICAL', 'CRITICAL') is True - - assert _does_severity_match_severity_threshold('NON_EXISTENT', 'LOW') is True - assert _does_severity_match_severity_threshold('LOW', 'NON_EXISTENT') is True diff --git a/tests/cli/commands/scan/test_detection_excluder.py b/tests/cli/commands/scan/test_detection_excluder.py new file mode 100644 index 00000000..d35787ab --- /dev/null +++ b/tests/cli/commands/scan/test_detection_excluder.py @@ -0,0 +1,22 @@ +from cycode.cli.apps.scan.detection_excluder import _does_severity_match_severity_threshold + + +def test_does_severity_match_severity_threshold() -> None: + assert _does_severity_match_severity_threshold('INFO', 'LOW') is False + + assert _does_severity_match_severity_threshold('LOW', 'LOW') is True + assert _does_severity_match_severity_threshold('LOW', 'MEDIUM') is False + + assert _does_severity_match_severity_threshold('MEDIUM', 'LOW') is True + assert _does_severity_match_severity_threshold('MEDIUM', 'MEDIUM') is True + assert _does_severity_match_severity_threshold('MEDIUM', 'HIGH') is False + + assert _does_severity_match_severity_threshold('HIGH', 'MEDIUM') is True + assert _does_severity_match_severity_threshold('HIGH', 'HIGH') is True + assert _does_severity_match_severity_threshold('HIGH', 'CRITICAL') is False + + assert _does_severity_match_severity_threshold('CRITICAL', 'HIGH') is True + assert _does_severity_match_severity_threshold('CRITICAL', 'CRITICAL') is True + + assert _does_severity_match_severity_threshold('NON_EXISTENT', 'LOW') is True + assert _does_severity_match_severity_threshold('LOW', 'NON_EXISTENT') is True diff --git a/tests/test_code_scanner.py b/tests/test_aggregation_report.py similarity index 81% rename from tests/test_code_scanner.py rename to tests/test_aggregation_report.py index 01234d1d..b09c4d69 100644 --- a/tests/test_code_scanner.py +++ b/tests/test_aggregation_report.py @@ -5,11 +5,9 @@ import responses from cycode.cli import consts -from cycode.cli.apps.scan.code_scanner import ( - _try_get_aggregation_report_url_if_needed, -) +from cycode.cli.apps.scan.aggregation_report import try_get_aggregation_report_url_if_needed from cycode.cli.cli_types import ScanTypeOption -from cycode.cli.files_collector.excluder import excluder +from cycode.cli.files_collector.file_excluder import excluder from cycode.cyclient.scan_client import ScanClient from tests.conftest import TEST_FILES_PATH from tests.cyclient.mocked_responses.scan_client import ( @@ -29,7 +27,7 @@ def test_try_get_aggregation_report_url_if_no_report_command_needed_return_none( ) -> None: aggregation_id = uuid4().hex scan_parameter = {'aggregation_id': aggregation_id} - result = _try_get_aggregation_report_url_if_needed(scan_parameter, scan_client, scan_type) + result = try_get_aggregation_report_url_if_needed(scan_parameter, scan_client, scan_type) assert result is None @@ -38,7 +36,7 @@ def test_try_get_aggregation_report_url_if_no_aggregation_id_needed_return_none( scan_type: ScanTypeOption, scan_client: ScanClient ) -> None: scan_parameter = {'report': True} - result = _try_get_aggregation_report_url_if_needed(scan_parameter, scan_client, scan_type) + result = try_get_aggregation_report_url_if_needed(scan_parameter, scan_client, scan_type) assert result is None @@ -55,5 +53,5 @@ def test_try_get_aggregation_report_url_if_needed_return_result( scan_aggregation_report_url_response = scan_client.get_scan_aggregation_report_url(str(aggregation_id), scan_type) - result = _try_get_aggregation_report_url_if_needed(scan_parameter, scan_client, scan_type) + result = try_get_aggregation_report_url_if_needed(scan_parameter, scan_client, scan_type) assert result == scan_aggregation_report_url_response.report_url