v0.2.0

README.md

@@ -21,6 +21,7 @@ CodeQL Agent CLI is a tool that automates the process of using CodeQL, a semanti
 - Automated CodeQL from detect language, create database and scan.
 - Scan remote target (e.g. GitHub repository) or local target (e.g. source code folder).
 - Support running on Docker which prepackaged and precompiled CodeQL for running code scanning (*under development*).
+- Send results to Discord webhook.
 
 ## Requirements
 
@@ -69,30 +70,32 @@ Usage: codeql-agent scan [options] <target>
 scan a source code folder or remote repository (e.g. GitHub repository)
 
 Arguments:
-  target                      source code folder or remote repository.
-  
+  target                          source code folder or remote repository.
+
   Examples:
-        codeql-agent src/sammple 
+        codeql-agent scan src/sammple
         codeql-agent scan src/sammple --use-docker
         codeql-agent scan https://github.com/OWASP/NodeGoat
 
 Options:
-  -l, --language <language>   language of source code. Supported languages: go, java, cpp, csharp, cpp, javascript, ruby. Omitting this option to auto-detect the language.
-  -o, --output <output>       output folder. Default: <target>-codeql-results
-  -c, --command <command>     command to create database for compiled languages, omit if the only languages requested are Python and JavaScript. This specifies the build commands
-                              needed to invoke the compiler. If you don't set this variable, CodeQL will attempt to detect the build system automatically, using a built-in autobuilder
-  -t, --threads <number>      number of threads to use. Pass 0 to use one threads per core on the machine. Default: 1 (default: 1)
-  --query <query>             CodeQL query to run. Default: <language>-security-extended.qls
-  --format <format>           output format. Default: sarif-latest (default: "sarif-latest")
-  --overwrite                 overwrite existing database.
-  --no-download               do not download missing queries before analyzing.
-  --remove-remote-repository  remove the remote repository after cloning.
-  --db-output <dbOutput>      database folder path.
-  --remove-database           remove the CodeQL database after scanning.
-  --create-db-only            only create CodeQL database, do not scan.
-  --use-docker                use docker to isolated run CodeQL.
-  -v, --verbose               verbose output
-  -h, --help                  display help for command
+  -l, --language <language>       language of source code. Supported languages: go, java, cpp, csharp, cpp, javascript, ruby. Omitting this option to auto-detect the language.
+  -o, --output <output>           output folder. Default: <target>-codeql-results
+  -c, --command <command>         command to create database for compiled languages, omit if the only languages requested are Python and JavaScript. This specifies the build commands needed to invoke the compiler. If
+                                  you don't set this variable, CodeQL will attempt to detect the build system automatically, using a built-in autobuilder
+  -t, --threads <number>          number of threads to use. Pass 0 to use one threads per core on the machine. Default: 1 (default: 1)
+  --query <query>                 CodeQL query to run. Default: <language>-security-extended.qls
+  --format <format>               output format. Default: sarif-latest (default: "sarif-latest")
+  --overwrite                     overwrite existing database.
+  --download                      download missing queries before analyzing.
+  --remove-remote-repository      remove the remote repository after cloning.
+  --db-output <dbOutput>          database folder path.
+  --remove-database               remove the CodeQL database after scanning.
+  --create-db-only                only create CodeQL database, do not scan.
+  --enable-file-logging           enable file logging.
+  --discord-webhook <webhookUrl>  discord web hook to send the result to.
+  --use-docker                    use docker to isolated run CodeQL.
+  -v, --verbose                   verbose output
+  -h, --help                      display help for command
 ```
 
 ## Using CodeQL Agent on VSCode

cli.js

@@ -35,11 +35,13 @@ program.command('scan')
     .option('--query <query>', 'CodeQL query to run. Default: <language>-security-extended.qls')
     .option('--format <format>', 'output format. Default: sarif-latest', 'sarif-latest')
     .option('--overwrite', 'overwrite existing database.')
-    .option('--no-download', 'do not download missing queries before analyzing.')
+    .option('--download', 'download missing queries before analyzing.')
     .option('--remove-remote-repository', 'remove the remote repository after cloning.')
     .option('--db-output <dbOutput>', 'database folder path. ')
     .option('--remove-database', 'remove the CodeQL database after scanning.')
     .option('--create-db-only', 'only create CodeQL database, do not scan.')
+    .option('--enable-file-logging', 'enable file logging.')
+    .option('--discord-webhook <webhookUrl>', 'discord web hook to send the result to.')
     .option('--use-docker', 'use docker to isolated run CodeQL.')
     .option('-v, --verbose', 'verbose output')
     .action(scan);

index.js

@@ -1,4 +1,4 @@
-const logger = require('./logger');
+const { defaultLogger, bugLogger } = require('./logger');
 const utils = require('./utils');
 const fs = require('fs');
 const config = require('./config');
@@ -9,34 +9,36 @@ module.exports = {
 }
 
 async function scanAction(sourceTarget, options) {
-    if (options.verbose) { logger.setLevel('verbose') }
+    if (options.verbose) { defaultLogger.setLevel('verbose') }
     if (options.useDocker) {
-        await utils.isCommandExist('docker', logger);
-        logger.error('Docker is not supported yet.');
+        await utils.isCommandExist('docker', defaultLogger);
+        defaultLogger.error('Docker is not supported yet.');
         return;
     }
-    await utils.isCommandExist('codeql', logger);
+    if (options.enableFileLogging) { defaultLogger.enableFileTransport() }
+    if (options.discordWebhook) { bugLogger.enableDiscordTransport(options.discordWebhook) }
+    await utils.isCommandExist('codeql', defaultLogger);
     // Create Database
     var createDbOptions = { ...options };
     createDbOptions.output = options.dbOutput;
     var isRemoteRepository = utils.isRemoteRepository(sourceTarget);
     if (isRemoteRepository) {
-        logger.info(`Cloning remote repository ${sourceTarget}`)
-        sourceFolderPath = await utils.cloneRemoteRepository(sourceTarget, logger);
+        defaultLogger.info(`Cloning remote repository ${sourceTarget}`)
+        sourceFolderPath = await utils.cloneRemoteRepository(sourceTarget, defaultLogger);
     } else sourceFolderPath = sourceTarget;
     sourceFolderPath = fs.realpathSync(sourceFolderPath);
-    logger.info(`Creating CodeQL database for ${sourceFolderPath}...`)
-    var { args: createDbArgs, databasePath } = await utils.setupCreateDatabaseCommandArgs(sourceFolderPath, createDbOptions, logger);
-    logger.verbose(`Options:`);
+    defaultLogger.info(`Creating CodeQL database for ${sourceFolderPath}...`)
+    var { args: createDbArgs, databasePath } = await utils.setupCreateDatabaseCommandArgs(sourceFolderPath, createDbOptions, defaultLogger);
+    defaultLogger.verbose(`Options:`);
     for (const key in options) {
         const element = options[key];
-        logger.verbose(`[+] ${key}: ${element}`);
+        defaultLogger.verbose(`[+] ${key}: ${element}`);
     }
-    createDbExitCode = await utils.executeCommand('codeql', createDbArgs, 'Create CodeQL database', logger);
-    logger.info(`CodeQL database created at ${databasePath}.`)
+    createDbExitCode = await utils.executeCommand('codeql', createDbArgs, 'Create CodeQL database', defaultLogger);
+    defaultLogger.info(`CodeQL database created at ${databasePath}.`)
     if (isRemoteRepository && options.removeRemoteRepository) {
-        logger.info(`Removing remote repository ${sourceFolderPath}`)
-        await utils.removeFolder(sourceFolderPath, logger);
+        defaultLogger.info(`Removing remote repository ${sourceFolderPath}`)
+        await utils.removeFolder(sourceFolderPath, defaultLogger);
     }
     if (options.createDbOnly) {
         return databasePath;
@@ -46,26 +48,39 @@ async function scanAction(sourceTarget, options) {
     if (!fs.existsSync(outputFolderPath)) {
         fs.mkdirSync(outputFolderPath);
     }
-    const languages = await utils.getDatabaseLanguages(databasePath, logger);
+    const languages = await utils.getDatabaseLanguages(databasePath, defaultLogger);
+    if (!languages) {
+        defaultLogger.error('Can not detect languages. Please specify the language using --language option');
+        return;
+    }
     for (const language of languages) {
         options.language = language;
         languageDatabasePath = path.resolve(`${databasePath}${path.sep}${language}`);
         options.output = path.resolve(outputFolderPath, `${language}-codeql-result.sarif`)
-        logger.info(`Scanning ${language} code in ${databasePath}...`)
-        var { args: scanArgs } = await utils.setupScanCommandArgs(languageDatabasePath, options, logger);
-        await utils.executeCommand('codeql', scanArgs, 'Scan CodeQL database', logger);
+        defaultLogger.info(`Scanning ${language} code in ${databasePath}...`)
+        var { args: scanArgs } = await utils.setupScanCommandArgs(languageDatabasePath, options, defaultLogger);
+        await utils.executeCommand('codeql', scanArgs, 'Scan CodeQL database', defaultLogger);
     }
-    logger.info(`CodeQL scan results saved at ${outputFolderPath}.`)
+    defaultLogger.info(`CodeQL scan results saved at ${outputFolderPath}.`)
     const resultFiles = fs.readdirSync(outputFolderPath);
+    var alerts = [];
     for (const resultFile of resultFiles) {
-        const alerts = await utils.parseSarif(path.resolve(outputFolderPath, resultFile), logger);
-        for (const alert of alerts) {
-            logger.error(`[${alert.id}][${alert.level}][precision:${alert.precision}][severity:${alert.severity}] ${alert.title}: ${alert.location}`);
-        }
+        alerts = alerts.concat(await utils.parseSarif(path.resolve(outputFolderPath, resultFile), defaultLogger));
+    }
+    for (const alert of alerts) {
+        defaultLogger.log({
+            level: utils.castBugLevelToLogLevel(alert.level),
+            message: `[${alert.id}][${alert.level}][precision:${alert.precision}][severity:${alert.severity}][${alert.location}] ${alert.title}`
+        });
+        bugLogger.log({
+            level: utils.castBugLevelToLogLevel(alert.level),
+            message: path.basename(sourceFolderPath),
+            meta: alert
+        });
     }
     if (options.removeDatabase) {
-        logger.info(`Removing database folder ${databasePath}`)
-        await utils.removeFolder(databasePath, logger);
+        defaultLogger.info(`Removing database folder ${databasePath}`)
+        await utils.removeFolder(databasePath, defaultLogger);
     }
-    return outputFolderPath;
+    return alerts;
 }

logger.js

@@ -1,29 +1,84 @@
 const winston = require('winston');
+const { createLogger, format, transports } = winston;
+const Logger = winston.Logger;
+const { combine, timestamp, printf, colorize } = format;
+const DailyRotateFile = require('winston-daily-rotate-file');
+const { DiscordTransport } = require('winston-transport-discord');
+const { HiddenLogger } = require('winston/lib/winston/logger');
 
-// Default logger
-const logger = winston.createLogger({
-    level: 'info',
-    format: winston.format.json(),
-    transports: [
-        new winston.transports.Console({
-            level: 'info',
-            format: winston.format.combine(
-                winston.format.colorize(),
-                winston.format.simple()
-            )
-        })
-    ],
+/*
+    * Custom logger
+*/
+const defaultLogFormat = printf(({ level, message, timestamp }) => {
+    return `[${level}][${timestamp}]: ${message}`;
 });
 
-logger.setLevel = function (logLevel) {
-    this.clear()
-    this.add(new winston.transports.Console({
-        level: logLevel,
+const discordLogFormat = printf(({ level, message }) => {
+    return `[${level}] ${message}`;
+});
+
+const setLevel = function (logLevel) {
+    this.level = logLevel;
+}
+const enableConsoleTransport = function () {
+    this.add(new transports.Console({
         format: winston.format.combine(
-            winston.format.colorize(),
-            winston.format.simple()
+            colorize({ all: true }),
+            timestamp(),
+            defaultLogFormat
         )
     }))
 }
+const enableFileTransport = function () {
+    this.add(new DailyRotateFile({
+        format: winston.format.combine(
+            timestamp(),
+            defaultLogFormat
+        ),
+        filename: 'logs/application-%DATE%.log',
+        datePattern: 'YYYY-MM-DD',
+        zippedArchive: true,
+        maxSize: '20m',
+        maxFiles: '30d'
+    }))
+}
+const enableDiscordTransport = function (webhookUrl) {
+    this.add(new DiscordTransport({
+        metadata: {
+            host: 'CodeQL Agent'
+        },
+        discord: {
+            webhook: {
+                url: webhookUrl
+            }
+        },
+        format: combine(discordLogFormat),
+        level: 'info'
+    }))
+}
+
+/*
+    * There are two logger instances
+    * 1. Default logger: defaultLogger. This logger is used for logging to console and file
+    * 2. Bug logger: bugLogger. This logger is used for logging bugs found.
+*/
+var defaultLogger = new createLogger({});
+var bugLogger = new createLogger({});
+
+defaultLogger.setLevel = setLevel;
+defaultLogger.enableConsoleTransport = enableConsoleTransport;
+defaultLogger.enableFileTransport = enableFileTransport;
+defaultLogger.enableDiscordTransport = enableDiscordTransport;
+
+bugLogger.setLevel = setLevel;
+bugLogger.enableConsoleTransport = enableConsoleTransport;
+bugLogger.enableFileTransport = enableFileTransport;
+bugLogger.enableDiscordTransport = enableDiscordTransport;
+
+defaultLogger.setLevel('info')
+defaultLogger.enableConsoleTransport()
+
+bugLogger.setLevel('info')
 
-module.exports = logger;
+module.exports.defaultLogger = defaultLogger;
+module.exports.bugLogger = bugLogger;