diff --git a/cmd/fleet/cron.go b/cmd/fleet/cron.go index 9c73d991b5c..a3eebd6ac6f 100644 --- a/cmd/fleet/cron.go +++ b/cmd/fleet/cron.go @@ -41,6 +41,7 @@ import ( "github.com/fleetdm/fleet/v4/server/vulnerabilities/macoffice" "github.com/fleetdm/fleet/v4/server/vulnerabilities/msrc" "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd" + "github.com/fleetdm/fleet/v4/server/vulnerabilities/osv" "github.com/fleetdm/fleet/v4/server/vulnerabilities/oval" "github.com/fleetdm/fleet/v4/server/vulnerabilities/utils" "github.com/fleetdm/fleet/v4/server/webhooks" @@ -193,7 +194,16 @@ func scanVulnerabilities( } nvdVulns := checkNVDVulnerabilities(ctx, ds, logger, vulnPath, config, vulnAutomationEnabled != "", startTime) - ovalVulns := checkOvalVulnerabilities(ctx, ds, logger, vulnPath, config, vulnAutomationEnabled != "") + + // Use OSV or OVAL for Ubuntu vulnerabilities based on feature flag + var ovalVulns []fleet.SoftwareVulnerability + var osvVulns []fleet.SoftwareVulnerability + if config.OSVForUbuntu { + osvVulns = checkOSVVulnerabilities(ctx, ds, logger, vulnPath, config, vulnAutomationEnabled != "") + } else { + ovalVulns = checkOvalVulnerabilities(ctx, ds, logger, vulnPath, config, vulnAutomationEnabled != "") + } + govalDictVulns := checkGovalDictionaryVulnerabilities(ctx, ds, logger, vulnPath, config, vulnAutomationEnabled != "") macOfficeVulns := checkMacOfficeVulnerabilities(ctx, ds, logger, vulnPath, config, vulnAutomationEnabled != "") customVulns := checkCustomVulnerabilities(ctx, ds, logger, vulnAutomationEnabled != "", startTime) @@ -209,9 +219,10 @@ func scanVulnerabilities( trace.WithAttributes(attribute.String("automation_type", vulnAutomationEnabled))) defer automationSpan.End() - vulns := make([]fleet.SoftwareVulnerability, 0, len(nvdVulns)+len(ovalVulns)+len(macOfficeVulns)) + vulns := make([]fleet.SoftwareVulnerability, 0, len(nvdVulns)+len(ovalVulns)+len(osvVulns)+len(macOfficeVulns)) vulns = append(vulns, nvdVulns...) vulns = append(vulns, ovalVulns...) + vulns = append(vulns, osvVulns...) vulns = append(vulns, macOfficeVulns...) vulns = append(vulns, govalDictVulns...) vulns = append(vulns, customVulns...) @@ -430,6 +441,52 @@ func checkOvalVulnerabilities( return results } +func checkOSVVulnerabilities( + ctx context.Context, + ds fleet.Datastore, + logger *logging.Logger, + vulnPath string, + config *config.VulnerabilitiesConfig, + collectVulns bool, +) []fleet.SoftwareVulnerability { + ctx, span := tracer.Start(ctx, "vuln.check_osv") + defer span.End() + + var results []fleet.SoftwareVulnerability + + // Get Platforms + versions, err := ds.OSVersions(ctx, nil, nil, nil, nil) + if err != nil { + errHandler(ctx, logger, "getting os versions for OSV", err) + return nil + } + + // Analyze all supported os versions using the OSV artifacts + analyzeCtx, analyzeSpan := tracer.Start(ctx, "vuln.osv.analyze", + trace.WithAttributes(attribute.Int("os_count", len(versions.OSVersions)))) + for _, version := range versions.OSVersions { + start := time.Now() + r, err := osv.Analyze(analyzeCtx, ds, version, vulnPath, collectVulns) + if err != nil && errors.Is(err, osv.ErrUnsupportedPlatform) { + logger.DebugContext(analyzeCtx, "osv-analysis-unsupported", "platform", version.Name) + continue + } + + elapsed := time.Since(start) + logger.DebugContext(analyzeCtx, "osv-analysis-done", + "platform", version.Name, + "elapsed", elapsed, + "found new", len(r)) + results = append(results, r...) + if err != nil { + errHandler(analyzeCtx, logger, "analyzing osv definitions", err) + } + } + analyzeSpan.End() + + return results +} + func checkGovalDictionaryVulnerabilities( ctx context.Context, ds fleet.Datastore, diff --git a/cmd/osv-processor/main.go b/cmd/osv-processor/main.go new file mode 100644 index 00000000000..e8aa6cd5459 --- /dev/null +++ b/cmd/osv-processor/main.go @@ -0,0 +1,408 @@ +package main + +import ( + "bufio" + "compress/gzip" + "encoding/json" + "flag" + "fmt" + "log" + "os" + "path/filepath" + "strings" + "time" +) + +type OSVData struct { + SchemaVersion string `json:"schema_version"` + ID string `json:"id"` + Published string `json:"published"` + Modified string `json:"modified"` + Details string `json:"details"` + Affected []Affected `json:"affected"` + Upstream []string `json:"upstream,omitempty"` + Related []string `json:"related,omitempty"` +} + +type Affected struct { + Package Package `json:"package"` + Ranges []Range `json:"ranges"` + Versions []string `json:"versions,omitempty"` + EcosystemSpecific map[string]interface{} `json:"ecosystem_specific,omitempty"` + DatabaseSpecific map[string]interface{} `json:"database_specific,omitempty"` +} + +type Package struct { + Ecosystem string `json:"ecosystem"` + Name string `json:"name"` + Purl string `json:"purl,omitempty"` +} + +type Range struct { + Type string `json:"type"` + Events []Event `json:"events"` +} + +type Event struct { + Introduced string `json:"introduced,omitempty"` + Fixed string `json:"fixed,omitempty"` +} + +type ProcessedVuln struct { + CVE string `json:"cve"` + Published string `json:"published"` + Modified string `json:"modified"` + Details string `json:"details"` + Introduced string `json:"introduced,omitempty"` + Fixed string `json:"fixed,omitempty"` + Versions []string `json:"versions,omitempty"` +} + +type ArtifactData struct { + SchemaVersion string `json:"schema_version"` + UbuntuVersion string `json:"ubuntu_version"` + Generated string `json:"generated"` + TotalCVEs int `json:"total_cves"` + TotalPackages int `json:"total_packages"` + Vulnerabilities map[string][]ProcessedVuln `json:"vulnerabilities"` +} + +func main() { + inputDir := flag.String("input", "/tmp/ubuntu-osv", "Input directory with OSV JSON files") + outputDir := flag.String("output", "./artifacts", "Output directory for artifacts") + versions := flag.String("versions", "", "Comma-separated Ubuntu versions to process") + changedFilesToday := flag.String("changed-files-today", "", "Path to file containing CVE files changed today (generates today's deltas)") + changedFilesYesterday := flag.String("changed-files-yesterday", "", "Path to file containing CVE files changed yesterday (generates yesterday's deltas)") + flag.Parse() + + if err := os.MkdirAll(*outputDir, 0o755); err != nil { + log.Fatalf("Failed to create output directory: %v", err) + } + + autoDetect := *versions == "" + var targetVersions map[string]bool + if !autoDetect { + targetVersions = make(map[string]bool) + for _, ver := range strings.Split(*versions, ",") { + targetVersions[strings.TrimSpace(ver)] = true + } + log.Printf("Processing OSV files from %s for versions: %s", *inputDir, *versions) + } else { + log.Printf("Processing OSV files from %s (auto-detecting versions)", *inputDir) + } + + // Load changed CVE files for delta generation + var todayCVEFiles, yesterdayCVEFiles map[string]bool + generateTodayDeltas := *changedFilesToday != "" + generateYesterdayDeltas := *changedFilesYesterday != "" + + if generateTodayDeltas { + log.Printf("Loading today's changed CVE files from %s", *changedFilesToday) + var err error + todayCVEFiles, err = loadChangedFiles(*changedFilesToday) + if err != nil { + log.Fatalf("Failed to load today's changed files: %v", err) + } + log.Printf("Found %d CVE files changed today", len(todayCVEFiles)) + } + + if generateYesterdayDeltas { + log.Printf("Loading yesterday's changed CVE files from %s", *changedFilesYesterday) + var err error + yesterdayCVEFiles, err = loadChangedFiles(*changedFilesYesterday) + if err != nil { + log.Fatalf("Failed to load yesterday's changed files: %v", err) + } + log.Printf("Found %d CVE files changed yesterday", len(yesterdayCVEFiles)) + } + + artifacts := make(map[string]*ArtifactData) + todayArtifacts := make(map[string]*ArtifactData) + yesterdayArtifacts := make(map[string]*ArtifactData) + + filesProcessed := 0 + filesSkipped := 0 + + err := filepath.Walk(*inputDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.IsDir() || !strings.HasSuffix(path, ".json") { + return nil + } + + osvData, err := parseOSVFile(path) + if err != nil { + log.Printf("Failed to parse %s: %v", path, err) + filesSkipped++ + return nil + } + + inToday := false + inYesterday := false + if generateTodayDeltas || generateYesterdayDeltas { + relPath, err := filepath.Rel(*inputDir, path) + if err == nil { + fullRelPath := filepath.Join("osv/cve", relPath) + if generateTodayDeltas { + inToday = todayCVEFiles[fullRelPath] + } + if generateYesterdayDeltas { + inYesterday = yesterdayCVEFiles[fullRelPath] + } + } + } + + for _, affected := range osvData.Affected { + ecosystem := affected.Package.Ecosystem + packageName := affected.Package.Name + + ubuntuVer := extractUbuntuVersion(ecosystem) + if ubuntuVer == "" { + continue + } + + // If specific versions requested, filter + if !autoDetect && !targetVersions[ubuntuVer] { + continue + } + + cveID := extractCVEID(osvData) + if cveID == "" { + cveID = osvData.ID + } + + introduced, fixed := extractVersionRange(affected.Ranges) + + vuln := ProcessedVuln{ + CVE: cveID, + Published: osvData.Published, + Modified: osvData.Modified, + Details: osvData.Details, + Introduced: introduced, + Fixed: fixed, + Versions: affected.Versions, + } + + // Add to full artifact + if _, exists := artifacts[ubuntuVer]; !exists { + artifacts[ubuntuVer] = &ArtifactData{ + SchemaVersion: "1.0", + UbuntuVersion: ubuntuVer, + Generated: time.Now().UTC().Format(time.RFC3339), + Vulnerabilities: make(map[string][]ProcessedVuln), + } + } + artifacts[ubuntuVer].Vulnerabilities[packageName] = append(artifacts[ubuntuVer].Vulnerabilities[packageName], vuln) + + // Add to today's delta artifact if this file was changed today + if inToday { + if _, exists := todayArtifacts[ubuntuVer]; !exists { + todayArtifacts[ubuntuVer] = &ArtifactData{ + SchemaVersion: "1.0", + UbuntuVersion: ubuntuVer, + Generated: time.Now().UTC().Format(time.RFC3339), + Vulnerabilities: make(map[string][]ProcessedVuln), + } + } + todayArtifacts[ubuntuVer].Vulnerabilities[packageName] = append(todayArtifacts[ubuntuVer].Vulnerabilities[packageName], vuln) + } + + // Add to yesterday's delta artifact if this file was changed yesterday + if inYesterday { + if _, exists := yesterdayArtifacts[ubuntuVer]; !exists { + yesterdayArtifacts[ubuntuVer] = &ArtifactData{ + SchemaVersion: "1.0", + UbuntuVersion: ubuntuVer, + Generated: time.Now().UTC().Format(time.RFC3339), + Vulnerabilities: make(map[string][]ProcessedVuln), + } + } + yesterdayArtifacts[ubuntuVer].Vulnerabilities[packageName] = append(yesterdayArtifacts[ubuntuVer].Vulnerabilities[packageName], vuln) + } + } + + filesProcessed++ + if filesProcessed%1000 == 0 { + log.Printf("Processed %d files...", filesProcessed) + } + + return nil + }) + if err != nil { + log.Fatalf("Error walking directory: %v", err) + } + + log.Printf("Processed %d files, skipped %d files", filesProcessed, filesSkipped) + log.Printf("Discovered %d Ubuntu versions", len(artifacts)) + + // Write full artifacts + for ver, artifact := range artifacts { + artifact.TotalCVEs = countTotalCVEs(artifact) + artifact.TotalPackages = len(artifact.Vulnerabilities) + + outputFile := filepath.Join(*outputDir, fmt.Sprintf("osv-ubuntu-%s-%s.json.gz", + strings.Replace(ver, ".", "", -1), + time.Now().Format("2006-01-02"))) + + if err := writeArtifact(outputFile, artifact); err != nil { + log.Fatalf("Failed to write artifact for Ubuntu %s: %v", ver, err) + } + + log.Printf("Ubuntu %s: %d packages, %d CVEs -> %s", + ver, artifact.TotalPackages, artifact.TotalCVEs, outputFile) + } + + // Write delta artifacts (if any were generated) + if generateTodayDeltas && len(todayArtifacts) > 0 { + today := time.Now().UTC().Format("2006-01-02") + log.Printf("\nWriting today's delta artifacts (%s)...", today) + for ver, artifact := range todayArtifacts { + artifact.TotalCVEs = countTotalCVEs(artifact) + artifact.TotalPackages = len(artifact.Vulnerabilities) + + outputFile := filepath.Join(*outputDir, fmt.Sprintf("osv-ubuntu-%s-delta-%s.json.gz", + strings.Replace(ver, ".", "", -1), today)) + + if err := writeArtifact(outputFile, artifact); err != nil { + log.Fatalf("Failed to write today's delta for Ubuntu %s: %v", ver, err) + } + + log.Printf("Ubuntu %s (today): %d packages, %d CVEs -> %s", + ver, artifact.TotalPackages, artifact.TotalCVEs, outputFile) + } + } + + if generateYesterdayDeltas && len(yesterdayArtifacts) > 0 { + yesterday := time.Now().UTC().AddDate(0, 0, -1).Format("2006-01-02") + log.Printf("\nWriting yesterday's delta artifacts (%s)...", yesterday) + for ver, artifact := range yesterdayArtifacts { + artifact.TotalCVEs = countTotalCVEs(artifact) + artifact.TotalPackages = len(artifact.Vulnerabilities) + + outputFile := filepath.Join(*outputDir, fmt.Sprintf("osv-ubuntu-%s-delta-%s.json.gz", + strings.Replace(ver, ".", "", -1), yesterday)) + + if err := writeArtifact(outputFile, artifact); err != nil { + log.Fatalf("Failed to write yesterday's delta for Ubuntu %s: %v", ver, err) + } + + log.Printf("Ubuntu %s (yesterday): %d packages, %d CVEs -> %s", + ver, artifact.TotalPackages, artifact.TotalCVEs, outputFile) + } + } +} + +func parseOSVFile(path string) (*OSVData, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + var osv OSVData + if err := json.Unmarshal(data, &osv); err != nil { + return nil, err + } + + return &osv, nil +} + +func extractUbuntuVersion(ecosystem string) string { + // Example: "Ubuntu:24.04:LTS" -> "24.04" + // Example: "Ubuntu:Pro:22.04:LTS" -> "22.04" + parts := strings.Split(ecosystem, ":") + for _, part := range parts { + // Look for version pattern like "24.04", "22.04", "20.04" + if len(part) == 5 && strings.Contains(part, ".") { + return part + } + } + return "" +} + +func extractCVEID(osv *OSVData) string { + for _, upstream := range osv.Upstream { + if strings.HasPrefix(upstream, "CVE-") { + return upstream + } + } + + if strings.HasPrefix(osv.ID, "CVE-") { + return osv.ID + } + + if strings.HasPrefix(osv.ID, "UBUNTU-CVE-") { + return strings.TrimPrefix(osv.ID, "UBUNTU-") + } + + return "" +} + +func extractVersionRange(ranges []Range) (introduced string, fixed string) { + for _, r := range ranges { + if r.Type == "ECOSYSTEM" { + for _, event := range r.Events { + if event.Introduced != "" && introduced == "" { + introduced = event.Introduced + } + if event.Fixed != "" && fixed == "" { + fixed = event.Fixed + } + } + } + } + return +} + +func countTotalCVEs(artifact *ArtifactData) int { + seen := make(map[string]bool) + for _, vulns := range artifact.Vulnerabilities { + for _, vuln := range vulns { + seen[vuln.CVE] = true + } + } + return len(seen) +} + +func writeArtifact(path string, artifact *ArtifactData) error { + file, err := os.Create(path) + if err != nil { + return err + } + defer file.Close() + + gzWriter := gzip.NewWriter(file) + defer gzWriter.Close() + + encoder := json.NewEncoder(gzWriter) + encoder.SetIndent("", " ") + + return encoder.Encode(artifact) +} + +func loadChangedFiles(changedFilesPath string) (map[string]bool, error) { + file, err := os.Open(changedFilesPath) + if err != nil { + return nil, fmt.Errorf("failed to open changed files list: %w", err) + } + defer file.Close() + + changedFiles := make(map[string]bool) + scanner := bufio.NewScanner(file) + + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + + changedFiles[line] = true + } + + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("error reading changed files: %w", err) + } + + return changedFiles, nil +} diff --git a/cmd/osv-processor/sync-and-detect-changes.sh b/cmd/osv-processor/sync-and-detect-changes.sh new file mode 100755 index 00000000000..ce7686aed35 --- /dev/null +++ b/cmd/osv-processor/sync-and-detect-changes.sh @@ -0,0 +1,95 @@ +#!/bin/bash +# Sync Canonical OSV repository using shallow clone with rolling window +# Usage: ./sync-and-detect-changes.sh +# +# Outputs: +# - Creates/updates ubuntu-security-notices directory (shallow clone) +# - Writes changed_files_today.txt and changed_files_yesterday.txt +# - Prints metadata to stdout +# +# Exit codes: +# 0: Success +# 1: Error occurred + +set -e + +# Configuration +REPO_URL="https://github.com/canonical/ubuntu-security-notices.git" +REPO_DIR="ubuntu-security-notices" +DAYS_TO_KEEP=3 # how much git history to keep +DELTA_DAYS=2 # how many delta files to generate (Generate today + yesterday deltas) + +echo "=== OSV Repository Sync ===" +echo "" + +if [ -d "$REPO_DIR/.git" ]; then + echo "Repository exists, updating with rolling window..." + cd "$REPO_DIR" + + OLD_SHA=$(git rev-parse HEAD) + OLD_COUNT=$(git log --oneline | wc -l | xargs) + + git fetch --update-shallow --shallow-since="${DAYS_TO_KEEP} days ago" origin main --quiet + + NEW_SHA=$(git rev-parse origin/main) + + if [ "$OLD_SHA" = "$NEW_SHA" ]; then + echo " No new commits (already at $NEW_SHA)" + else + echo " Updating: $OLD_SHA -> $NEW_SHA" + git reset --hard origin/main --quiet + fi + + NEW_COUNT=$(git log --oneline | wc -l | xargs) + echo " History: $OLD_COUNT commits -> $NEW_COUNT commits" + + cd .. +else + echo "Cloning repository (shallow since ${DAYS_TO_KEEP} days ago)..." + git clone --shallow-since="${DAYS_TO_KEEP} days ago" --quiet "$REPO_URL" "$REPO_DIR" + + cd "$REPO_DIR" + COMMIT_SHA=$(git rev-parse HEAD) + COMMIT_COUNT=$(git log --oneline | wc -l | xargs) + cd .. + + echo " Cloned at: $COMMIT_SHA" + echo " History: $COMMIT_COUNT commits" + du -sh "$REPO_DIR" | awk '{print " Size: " $1}' +fi + +cd "$REPO_DIR" + +# Get files changed today (since midnight UTC today) +TODAY_UTC=$(date -u +%Y-%m-%d) +git log --since="${TODAY_UTC}T00:00:00Z" --name-only --pretty="" -- osv/cve \ + | sort -u > "../changed_files_today.txt" + +# Get files changed yesterday (from midnight yesterday to midnight today UTC) +YESTERDAY_UTC=$(date -u -v-1d +%Y-%m-%d 2>/dev/null || date -u -d "yesterday" +%Y-%m-%d) +git log --since="${YESTERDAY_UTC}T00:00:00Z" --until="${TODAY_UTC}T00:00:00Z" --name-only --pretty="" -- osv/cve \ + | sort -u > "../changed_files_yesterday.txt" + +TODAY_COUNT=$(wc -l < "../changed_files_today.txt" | xargs) +YESTERDAY_COUNT=$(wc -l < "../changed_files_yesterday.txt" | xargs) +cd .. + +echo " Today: $TODAY_COUNT CVE files changed" +echo " Yesterday: $YESTERDAY_COUNT CVE files changed" + +echo "" +echo "=== Sync Complete ===" +cd "$REPO_DIR" +FINAL_SHA=$(git rev-parse HEAD) +FINAL_COUNT=$(git log --oneline | wc -l | xargs) +cd .. + +echo "REPO_SHA=$FINAL_SHA" +echo "REPO_COMMITS=$FINAL_COUNT" +echo "OSV_DIR=$REPO_DIR/osv/cve" +echo "CHANGED_FILES_TODAY=changed_files_today.txt" +echo "CHANGED_FILES_YESTERDAY=changed_files_yesterday.txt" +echo "TODAY_COUNT=$TODAY_COUNT" +echo "YESTERDAY_COUNT=$YESTERDAY_COUNT" + +exit 0 diff --git a/server/config/config.go b/server/config/config.go index 46a12512313..be291a1e793 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -590,6 +590,7 @@ type VulnerabilitiesConfig struct { RecentVulnerabilityMaxAge time.Duration `json:"recent_vulnerability_max_age" yaml:"recent_vulnerability_max_age"` DisableWinOSVulnerabilities bool `json:"disable_win_os_vulnerabilities" yaml:"disable_win_os_vulnerabilities"` MaxConcurrency int `json:"max_concurrency" yaml:"max_concurrency"` + OSVForUbuntu bool `json:"osv_for_ubuntu" yaml:"osv_for_ubuntu"` } // UpgradesConfig defines configs related to fleet server upgrades. @@ -1529,6 +1530,11 @@ func (man Manager) addConfigs() { 1, "Maximum number of concurrent database queries to use for processing vulnerabilities.", ) + man.addConfigBool( + "vulnerabilities.osv_for_ubuntu", + false, + "Use OSV (Open Source Vulnerability) format instead of OVAL for Ubuntu vulnerability detection.", + ) // Upgrades man.addConfigBool("upgrades.allow_missing_migrations", false, @@ -1866,6 +1872,7 @@ func (man Manager) LoadConfig() FleetConfig { RecentVulnerabilityMaxAge: man.getConfigDuration("vulnerabilities.recent_vulnerability_max_age"), DisableWinOSVulnerabilities: man.getConfigBool("vulnerabilities.disable_win_os_vulnerabilities"), MaxConcurrency: man.getConfigInt("vulnerabilities.max_concurrency"), + OSVForUbuntu: man.getConfigBool("vulnerabilities.osv_for_ubuntu"), }, Upgrades: UpgradesConfig{ AllowMissingMigrations: man.getConfigBool("upgrades.allow_missing_migrations"), diff --git a/server/fleet/app.go b/server/fleet/app.go index 9eb1d287be3..b83526aebf0 100644 --- a/server/fleet/app.go +++ b/server/fleet/app.go @@ -1569,6 +1569,7 @@ type VulnerabilitiesConfig struct { DisableDataSync bool `json:"disable_data_sync"` RecentVulnerabilityMaxAge time.Duration `json:"recent_vulnerability_max_age"` DisableWinOSVulnerabilities bool `json:"disable_win_os_vulnerabilities"` + OSVForUbuntu bool `json:"osv_for_ubuntu"` } type LoggingPlugin struct { diff --git a/server/vulnerabilities/osv/analyzer.go b/server/vulnerabilities/osv/analyzer.go new file mode 100644 index 00000000000..8f2a01dd47d --- /dev/null +++ b/server/vulnerabilities/osv/analyzer.go @@ -0,0 +1,350 @@ +package osv + +import ( + "compress/gzip" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/fleetdm/fleet/v4/server/fleet" + feednvd "github.com/fleetdm/fleet/v4/server/vulnerabilities/nvd/tools/cvefeed/nvd" +) + +var ErrUnsupportedPlatform = errors.New("unsupported platform") + +// OSVArtifact represents the processed OSV data for a specific Ubuntu version +type OSVArtifact struct { + SchemaVersion string `json:"schema_version"` + UbuntuVersion string `json:"ubuntu_version"` + Generated time.Time `json:"generated"` + TotalCVEs int `json:"total_cves"` + TotalPackages int `json:"total_packages"` + Vulnerabilities map[string][]OSVVulnerability `json:"vulnerabilities"` +} + +// OSVVulnerability represents a single vulnerability for a package +type OSVVulnerability struct { + CVE string `json:"cve"` + Published time.Time `json:"published"` + Modified time.Time `json:"modified"` + Details string `json:"details"` + Introduced string `json:"introduced"` + Fixed string `json:"fixed,omitempty"` + Versions []string `json:"versions,omitempty"` +} + +// Analyze scans all hosts for vulnerabilities based on the OSV artifacts for their platform. +// Returns new vulnerabilities found. +func Analyze( + ctx context.Context, + ds fleet.Datastore, + ver fleet.OSVersion, + vulnPath string, + collectVulns bool, +) ([]fleet.SoftwareVulnerability, error) { + if ver.Platform != "ubuntu" { + return nil, ErrUnsupportedPlatform + } + + artifact, err := loadOSVArtifact(ver, vulnPath) + if err != nil { + return nil, fmt.Errorf("loading OSV artifact: %w", err) + } + + fmt.Printf("[OSV DEBUG] Loaded artifact for Ubuntu %s: %d packages, %d total CVEs\n", + artifact.UbuntuVersion, len(artifact.Vulnerabilities), artifact.TotalCVEs) + + // Get all hosts with this OS version + hostIDs, err := ds.HostIDsByOSVersion(ctx, ver, 0, 10000) + if err != nil { + return nil, fmt.Errorf("getting host IDs: %w", err) + } + + if len(hostIDs) == 0 { + return nil, nil + } + + allVulns := make(map[string]fleet.SoftwareVulnerability) + + for _, hostID := range hostIDs { + software, err := ds.ListSoftwareForVulnDetection(ctx, fleet.VulnSoftwareFilter{ + HostID: &hostID, + }) + if err != nil { + return nil, fmt.Errorf("listing software for host %d: %w", hostID, err) + } + + fmt.Printf("[OSV DEBUG] Host %d has %d software packages\n", hostID, len(software)) + + vulns := matchSoftwareToOSV(software, artifact) + + fmt.Printf("[OSV DEBUG] Host %d matched %d vulnerabilities\n", hostID, len(vulns)) + + for _, v := range vulns { + key := v.Key() + allVulns[key] = v + } + } + + vulnsList := make([]fleet.SoftwareVulnerability, 0, len(allVulns)) + for _, v := range allVulns { + vulnsList = append(vulnsList, v) + } + + source := fleet.VulnerabilitySource(7) + newVulns, err := ds.InsertSoftwareVulnerabilities(ctx, vulnsList, source) + if err != nil { + return nil, fmt.Errorf("inserting software vulnerabilities: %w", err) + } + + if !collectVulns { + return nil, nil + } + + return newVulns, nil +} + +// loadOSVArtifact loads the OSV artifact for the given Ubuntu version +func loadOSVArtifact(ver fleet.OSVersion, _ string) (*OSVArtifact, error) { + // Extract Ubuntu version (e.g., "22.04.8 LTS" -> "2204") + ubuntuVer := extractUbuntuVersion(ver.Version) + if ubuntuVer == "" { + return nil, fmt.Errorf("could not extract Ubuntu version from %s", ver.Version) + } + + // Hardcoded path for POC + artifactsPath := "/Users/ksykulev/projects/fleet-main/cmd/osv-processor/test-artifacts-final" + + // Find the latest OSV artifact file for this version + // Pattern: osv-ubuntu-2204-YYYY-MM-DD.json.gz + pattern := fmt.Sprintf("osv-ubuntu-%s-*.json.gz", ubuntuVer) + matches, err := filepath.Glob(filepath.Join(artifactsPath, pattern)) + if err != nil { + return nil, fmt.Errorf("globbing for OSV artifacts: %w", err) + } + + var fullArtifacts []string + for _, match := range matches { + if !strings.Contains(filepath.Base(match), "-delta-") { + fullArtifacts = append(fullArtifacts, match) + } + } + + if len(fullArtifacts) == 0 { + return nil, fmt.Errorf("no OSV artifact found for Ubuntu %s in %s", ubuntuVer, artifactsPath) + } + + latestFile := fullArtifacts[len(fullArtifacts)-1] + + fmt.Printf("[OSV DEBUG] Loading artifact file: %s\n", filepath.Base(latestFile)) + + f, err := os.Open(latestFile) + if err != nil { + return nil, fmt.Errorf("opening OSV artifact: %w", err) + } + defer f.Close() + + gz, err := gzip.NewReader(f) + if err != nil { + return nil, fmt.Errorf("creating gzip reader: %w", err) + } + defer gz.Close() + + var artifact OSVArtifact + if err := json.NewDecoder(gz).Decode(&artifact); err != nil { + return nil, fmt.Errorf("decoding OSV artifact: %w", err) + } + + return &artifact, nil +} + +// extractUbuntuVersion extracts the numeric version from Ubuntu version strings +// Examples: "22.04.8 LTS" -> "2204", "20.04.1 LTS" -> "2004" +func extractUbuntuVersion(version string) string { + // Remove " LTS" suffix if present + version = strings.TrimSuffix(version, " LTS") + version = strings.TrimSpace(version) + + parts := strings.Split(version, ".") + if len(parts) < 2 { + return "" + } + + return parts[0] + parts[1] +} + +// matchSoftwareToOSV matches software against OSV vulnerabilities +func matchSoftwareToOSV(software []fleet.Software, artifact *OSVArtifact) []fleet.SoftwareVulnerability { + var result []fleet.SoftwareVulnerability + + matchedPackages := 0 + checkedPackages := 0 + linuxPackagesChecked := 0 + + for _, sw := range software { + checkedPackages++ + + if strings.HasPrefix(sw.Name, "linux") { + linuxPackagesChecked++ + if linuxPackagesChecked <= 5 { + fmt.Printf("[OSV DEBUG] Linux package found: %s (version: %s)\n", sw.Name, sw.Version) + } + } + + packageName := sw.Name + + // Kernel package mapping: Only linux-image-* packages (actual kernel binaries) + // should be checked against kernel CVEs, not headers/modules/tools + if strings.HasPrefix(sw.Name, "linux-image-") || strings.HasPrefix(sw.Name, "linux-signed-image-") { + packageName = "linux" + } + + vulns, ok := artifact.Vulnerabilities[packageName] + if !ok { + if checkedPackages <= 20 { + fmt.Printf("[OSV DEBUG] No match for package: %s (version: %s, source: %s)\n", + sw.Name, sw.Version, sw.Source) + } + continue + } + + matchedPackages++ + + isLinuxPackage := strings.HasPrefix(sw.Name, "linux") + if isLinuxPackage { + fmt.Printf("[OSV DEBUG] MATCHED LINUX package: %s (version: %s) mapped to '%s' has %d potential CVEs\n", + sw.Name, sw.Version, packageName, len(vulns)) + } + + if matchedPackages <= 10 && !isLinuxPackage { + fmt.Printf("[OSV DEBUG] MATCHED package: %s (version: %s) has %d potential CVEs\n", + sw.Name, sw.Version, len(vulns)) + } + + vulnerableCount := 0 + + // For kernel packages, we need to normalize the version + // osquery: 5.15.0-94-generic -> 5.15.0-94 + // OSV: 5.15.0-94.104 + isKernelPackage := packageName == "linux" + + for _, vuln := range vulns { + var vulnerable bool + if matchedPackages == 1 { + vulnerable = isVulnerableDebug(sw.Version, vuln, isKernelPackage) + } else { + vulnerable = isVulnerable(sw.Version, vuln, isKernelPackage) + } + + if vulnerable { + result = append(result, fleet.SoftwareVulnerability{ + SoftwareID: sw.ID, + CVE: vuln.CVE, + }) + vulnerableCount++ + } + } + + if matchedPackages <= 10 { + fmt.Printf("[OSV DEBUG] -> After version check: %d actually vulnerable\n", vulnerableCount) + } + } + + fmt.Printf("[OSV DEBUG] Checked %d packages, matched %d packages, found %d vulnerabilities (linux packages checked: %d)\n", + checkedPackages, matchedPackages, len(result), linuxPackagesChecked) + + return result +} + +// normalizeKernelVersion extracts the base kernel version for matching +// Example: "5.15.0-94-generic" -> "5.15.0-94" +func normalizeKernelVersion(version string) string { + parts := strings.Split(version, "-") + if len(parts) >= 2 { + return parts[0] + "-" + parts[1] + } + return version +} + +// kernelVersionMatches checks if a kernel version matches an OSV version +// Handles both exact matches and prefix matches +// osquery: "5.15.0-94-generic" normalizes to "5.15.0-94" +// OSV: "5.15.0-94.104" starts with "5.15.0-94" +func kernelVersionMatches(softwareVersion, osvVersion string) bool { + normalized := normalizeKernelVersion(softwareVersion) + + if normalized == osvVersion { + return true + } + + if strings.HasPrefix(osvVersion, normalized+".") { + return true + } + + return false +} + +// isVulnerable checks if a software version is vulnerable based on OSV data +func isVulnerable(softwareVersion string, vuln OSVVulnerability, isKernelPackage bool) bool { + if len(vuln.Versions) > 0 { + for _, v := range vuln.Versions { + if isKernelPackage { + if kernelVersionMatches(softwareVersion, v) { + return true + } + } else { + if softwareVersion == v { + return true + } + } + } + return false + } + + // No explicit versions list - use range-based matching + introduced := vuln.Introduced + if introduced == "" { + introduced = "0" + } + + if introduced != "0" { + cmp := feednvd.SmartVerCmp(softwareVersion, introduced) + if cmp == -1 { // softwareVersion < introduced + return false + } + } + + if vuln.Fixed != "" { + cmp := feednvd.SmartVerCmp(softwareVersion, vuln.Fixed) + if cmp != -1 { // softwareVersion >= fixed (not vulnerable) + return false + } + } + + return true +} + +var debugVersionCheckCount = 0 + +// isVulnerableDebug is a debug wrapper around isVulnerable +func isVulnerableDebug(softwareVersion string, vuln OSVVulnerability, isKernelPackage bool) bool { + result := isVulnerable(softwareVersion, vuln, isKernelPackage) + + debugVersionCheckCount++ + if debugVersionCheckCount <= 5 { + hasVersions := len(vuln.Versions) > 0 + normalized := "" + if isKernelPackage { + normalized = normalizeKernelVersion(softwareVersion) + } + fmt.Printf("[OSV DEBUG] Version check: %s (normalized: %s, is_kernel: %v) vs CVE %s (introduced: %s, fixed: %s, has_versions_list: %v, versions_count: %d) -> %v\n", + softwareVersion, normalized, isKernelPackage, vuln.CVE, vuln.Introduced, vuln.Fixed, hasVersions, len(vuln.Versions), result) + } + + return result +}