From 075349ac575aea24a142c531d5ee244ac177c604 Mon Sep 17 00:00:00 2001 From: Caleb Gross Date: Sun, 1 Mar 2026 08:30:39 -0500 Subject: [PATCH 1/2] Add adaptive learning from heuristic rejections (tier 3 noise suppression) Track filesystem events rejected by the heuristic filter per path prefix. When a prefix (e.g. .config/Code/) hits 50 rejections within 1 hour, it is automatically promoted to a watcher exclusion pattern. Learned exclusions are persisted to ~/.mnemonic/learned-exclusions.txt and reloaded on restart. - Add ExcludableWatcher interface for runtime exclusion additions - Add AddExclusion() to both platform watchers with proper RWMutex locking - Add rejection_tracker.go with prefix extraction, threshold promotion, persistence - Wire promoteExclusion callback from perception agent to watchers - Add LearnedExclusionsPath config field with path expansion - Add 4 tests covering prefix extraction, promotion, cap, and persistence Closes #28 Co-Authored-By: Claude Opus 4.6 --- cmd/mnemonic/main.go | 3 +- internal/agent/perception/agent.go | 38 ++- .../agent/perception/rejection_tracker.go | 246 ++++++++++++++++++ .../perception/rejection_tracker_test.go | 205 +++++++++++++++ internal/config/config.go | 26 +- internal/watcher/filesystem/watcher_darwin.go | 12 +- internal/watcher/filesystem/watcher_other.go | 17 +- internal/watcher/watcher.go | 7 + 8 files changed, 540 insertions(+), 14 deletions(-) create mode 100644 internal/agent/perception/rejection_tracker.go create mode 100644 internal/agent/perception/rejection_tracker_test.go diff --git a/cmd/mnemonic/main.go b/cmd/mnemonic/main.go index ecf65236..944a9e56 100644 --- a/cmd/mnemonic/main.go +++ b/cmd/mnemonic/main.go @@ -973,7 +973,8 @@ func serveCommand(configPath string) { FrequencyThreshold: cfg.Perception.Heuristics.FrequencyThreshold, FrequencyWindowMin: cfg.Perception.Heuristics.FrequencyWindowMin, }, - LLMGatingEnabled: cfg.Perception.LLMGatingEnabled, + LLMGatingEnabled: cfg.Perception.LLMGatingEnabled, + LearnedExclusionsPath: cfg.Perception.LearnedExclusionsPath, }, log, ) diff --git a/internal/agent/perception/agent.go b/internal/agent/perception/agent.go index 17377283..cbf102ea 100644 --- a/internal/agent/perception/agent.go +++ b/internal/agent/perception/agent.go @@ -17,8 +17,9 @@ import ( // PerceptionConfig configures the perception agent. type PerceptionConfig struct { - HeuristicConfig HeuristicConfig - LLMGatingEnabled bool // if false, skip LLM and use heuristic score as salience + HeuristicConfig HeuristicConfig + LLMGatingEnabled bool // if false, skip LLM and use heuristic score as salience + LearnedExclusionsPath string // file path for persisting learned watcher exclusions } // PerceptionAgent orchestrates the perception pipeline: watchers → heuristic → LLM → memory. @@ -30,6 +31,7 @@ type PerceptionAgent struct { cfg PerceptionConfig log *slog.Logger heuristicFilter *HeuristicFilter + rejectionTracker *rejectionTracker bus events.Bus mu sync.RWMutex running bool @@ -75,6 +77,13 @@ func (pa *PerceptionAgent) Start(ctx context.Context, bus events.Bus) error { pa.bus = bus pa.running = true pa.heuristicFilter = NewHeuristicFilter(pa.cfg.HeuristicConfig, pa.log) + pa.rejectionTracker = newRejectionTracker( + rejectionTrackerConfig{ + PersistPath: pa.cfg.LearnedExclusionsPath, + }, + pa.log, + pa.promoteExclusion, + ) pa.watcherStopChans = make([]chan struct{}, len(pa.watchers)) pa.mu.Unlock() @@ -88,6 +97,13 @@ func (pa *PerceptionAgent) Start(ctx context.Context, bus events.Bus) error { } } + // Apply any previously learned exclusions to watchers + if pa.rejectionTracker != nil { + for _, pattern := range pa.rejectionTracker.learnedExclusions() { + pa.promoteExclusion(pattern) + } + } + // Launch a processing goroutine for each watcher for i, w := range pa.watchers { stopChan := make(chan struct{}) @@ -207,6 +223,10 @@ func (pa *PerceptionAgent) processEvent(ctx context.Context, event Event) { "path", event.Path, "rationale", heuristicResult.Rationale, ) + // Track filesystem rejections for adaptive exclusion learning + if event.Source == "filesystem" && event.Path != "" && pa.rejectionTracker != nil { + pa.rejectionTracker.recordRejection(event.Path) + } return } @@ -369,6 +389,20 @@ type llmGateResult struct { Reason string `json:"reason"` } +// promoteExclusion pushes a learned exclusion pattern to all watchers that +// support runtime exclusion updates. +func (pa *PerceptionAgent) promoteExclusion(pattern string) { + for _, w := range pa.watchers { + if ew, ok := w.(watcher.ExcludableWatcher); ok { + ew.AddExclusion(pattern) + pa.log.Info("promoted learned exclusion to watcher", + "pattern", pattern, + "watcher", w.Name(), + ) + } + } +} + // truncateContent truncates content to a maximum length. func (pa *PerceptionAgent) truncateContent(content string, maxLen int) string { if len(content) <= maxLen { diff --git a/internal/agent/perception/rejection_tracker.go b/internal/agent/perception/rejection_tracker.go new file mode 100644 index 00000000..7ddb3da1 --- /dev/null +++ b/internal/agent/perception/rejection_tracker.go @@ -0,0 +1,246 @@ +package perception + +import ( + "bufio" + "log/slog" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +// rejectionTracker monitors heuristic rejections by path prefix and promotes +// frequently-rejected prefixes to watcher exclusions. +type rejectionTracker struct { + mu sync.Mutex + counts map[string]int // path prefix → rejection count + firstSeen map[string]time.Time // path prefix → first rejection timestamp + promoted map[string]bool // path prefixes already promoted + + threshold int // rejections required before promotion + window time.Duration // time window for threshold + maxPromoted int // cap on auto-exclusions per session + + log *slog.Logger + onPromote func(pattern string) // callback when a prefix is promoted + persistPath string // file path for persisting learned exclusions +} + +// rejectionTrackerConfig holds tunable parameters for the tracker. +type rejectionTrackerConfig struct { + Threshold int // rejections to trigger promotion (default: 50) + Window time.Duration // time window (default: 1 hour) + MaxPromoted int // max auto-exclusions per session (default: 20) + PersistPath string // file to persist learned exclusions (empty = no persistence) +} + +func defaultRejectionTrackerConfig() rejectionTrackerConfig { + return rejectionTrackerConfig{ + Threshold: 50, + Window: 1 * time.Hour, + MaxPromoted: 20, + } +} + +func newRejectionTracker(cfg rejectionTrackerConfig, log *slog.Logger, onPromote func(string)) *rejectionTracker { + if cfg.Threshold == 0 { + cfg.Threshold = 50 + } + if cfg.Window == 0 { + cfg.Window = 1 * time.Hour + } + if cfg.MaxPromoted == 0 { + cfg.MaxPromoted = 20 + } + + rt := &rejectionTracker{ + counts: make(map[string]int), + firstSeen: make(map[string]time.Time), + promoted: make(map[string]bool), + threshold: cfg.Threshold, + window: cfg.Window, + maxPromoted: cfg.MaxPromoted, + log: log, + onPromote: onPromote, + persistPath: cfg.PersistPath, + } + + // Load previously learned exclusions + if cfg.PersistPath != "" { + rt.loadPersisted() + } + + return rt +} + +// extractPrefix extracts a 2-level directory prefix from a path under known +// base directories. For example: +// +// /home/user/.config/Code/WebStorage/foo → .config/Code/ +// /home/user/.local/share/gnome-shell/x → .local/share/gnome-shell/ +// +// Returns empty string if no prefix can be extracted. +func extractPrefix(path string) string { + home, err := os.UserHomeDir() + if err != nil { + return "" + } + + // Ensure path is under home + if !strings.HasPrefix(path, home) { + return "" + } + + rel := path[len(home):] + if len(rel) > 0 && rel[0] == '/' { + rel = rel[1:] + } + + // Known base directories with their depth (how many levels before app dir) + bases := []struct { + prefix string + depth int // number of path segments in the base prefix + }{ + {prefix: ".config/", depth: 1}, + {prefix: ".local/share/", depth: 2}, + {prefix: "Library/Application Support/", depth: 2}, + {prefix: "Library/Caches/", depth: 2}, + } + + for _, base := range bases { + if strings.HasPrefix(rel, base.prefix) { + after := rel[len(base.prefix):] + // Get the first directory component after the base + idx := strings.Index(after, "/") + if idx <= 0 { + continue + } + appDir := after[:idx] + return "." + "/" + rel[:len(base.prefix)+len(appDir)] + "/" + } + } + + return "" +} + +// recordRejection records a heuristic rejection for the given path. +// If the path prefix hits the threshold, it's promoted to an exclusion. +func (rt *rejectionTracker) recordRejection(path string) { + prefix := extractPrefix(path) + if prefix == "" { + return + } + + rt.mu.Lock() + defer rt.mu.Unlock() + + // Already promoted + if rt.promoted[prefix] { + return + } + + // Cap on total promotions + if len(rt.promoted) >= rt.maxPromoted { + return + } + + now := time.Now() + + // Initialize or check window + if first, ok := rt.firstSeen[prefix]; ok { + if now.Sub(first) > rt.window { + // Window expired, reset counter + rt.counts[prefix] = 0 + rt.firstSeen[prefix] = now + } + } else { + rt.firstSeen[prefix] = now + } + + rt.counts[prefix]++ + + if rt.counts[prefix] >= rt.threshold { + rt.promoted[prefix] = true + rt.log.Info("auto-excluded noisy path", + "pattern", prefix, + "rejections", rt.counts[prefix], + "window", rt.window, + ) + + // Persist + if rt.persistPath != "" { + rt.appendPersisted(prefix) + } + + // Notify watcher + if rt.onPromote != nil { + rt.onPromote(prefix) + } + + // Clean up tracking state + delete(rt.counts, prefix) + delete(rt.firstSeen, prefix) + } +} + +// learnedExclusions returns all exclusion patterns that have been promoted +// (both from this session and loaded from persistence). +func (rt *rejectionTracker) learnedExclusions() []string { + rt.mu.Lock() + defer rt.mu.Unlock() + result := make([]string, 0, len(rt.promoted)) + for pattern := range rt.promoted { + result = append(result, pattern) + } + return result +} + +// loadPersisted reads previously learned exclusions from disk. +func (rt *rejectionTracker) loadPersisted() { + f, err := os.Open(rt.persistPath) + if err != nil { + if !os.IsNotExist(err) { + rt.log.Warn("failed to load learned exclusions", "path", rt.persistPath, "error", err) + } + return + } + defer f.Close() + + count := 0 + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + rt.promoted[line] = true + count++ + } + if err := scanner.Err(); err != nil { + rt.log.Warn("error reading learned exclusions", "path", rt.persistPath, "error", err) + } + if count > 0 { + rt.log.Info("loaded learned exclusions", "count", count, "path", rt.persistPath) + } +} + +// appendPersisted appends a single pattern to the persistence file. +func (rt *rejectionTracker) appendPersisted(pattern string) { + dir := filepath.Dir(rt.persistPath) + if err := os.MkdirAll(dir, 0o755); err != nil { + rt.log.Warn("failed to create directory for learned exclusions", "error", err) + return + } + + f, err := os.OpenFile(rt.persistPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + rt.log.Warn("failed to persist learned exclusion", "pattern", pattern, "error", err) + return + } + defer f.Close() + + if _, err := f.WriteString(pattern + "\n"); err != nil { + rt.log.Warn("failed to write learned exclusion", "pattern", pattern, "error", err) + } +} diff --git a/internal/agent/perception/rejection_tracker_test.go b/internal/agent/perception/rejection_tracker_test.go new file mode 100644 index 00000000..472ab7ec --- /dev/null +++ b/internal/agent/perception/rejection_tracker_test.go @@ -0,0 +1,205 @@ +package perception + +import ( + "log/slog" + "os" + "path/filepath" + "testing" + "time" +) + +func testLogger() *slog.Logger { + return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn})) +} + +func TestExtractPrefix(t *testing.T) { + home, err := os.UserHomeDir() + if err != nil { + t.Fatalf("could not get home dir: %v", err) + } + + tests := []struct { + name string + path string + want string + }{ + { + name: "config app dir", + path: filepath.Join(home, ".config/Code/User/settings.json"), + want: "./.config/Code/", + }, + { + name: "local share app dir", + path: filepath.Join(home, ".local/share/gnome-shell/extensions/foo"), + want: "./.local/share/gnome-shell/", + }, + { + name: "not under home", + path: "/tmp/foo/bar", + want: "", + }, + { + name: "no recognizable base", + path: filepath.Join(home, "Documents/projects/foo.go"), + want: "", + }, + { + name: "config dir with no app subdir", + path: filepath.Join(home, ".config/somefile"), + want: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := extractPrefix(tt.path) + if got != tt.want { + t.Errorf("extractPrefix(%q) = %q, want %q", tt.path, got, tt.want) + } + }) + } +} + +func TestRejectionTracker_PromotesAfterThreshold(t *testing.T) { + home, err := os.UserHomeDir() + if err != nil { + t.Fatalf("could not get home dir: %v", err) + } + + var promoted []string + onPromote := func(pattern string) { + promoted = append(promoted, pattern) + } + + rt := newRejectionTracker( + rejectionTrackerConfig{ + Threshold: 5, + Window: 1 * time.Hour, + MaxPromoted: 10, + }, + testLogger(), + onPromote, + ) + + path := filepath.Join(home, ".config/Code/User/settings.json") + + // Record 4 rejections — should not promote yet + for i := 0; i < 4; i++ { + rt.recordRejection(path) + } + if len(promoted) != 0 { + t.Errorf("expected no promotions after 4 rejections, got %d", len(promoted)) + } + + // 5th rejection should trigger promotion + rt.recordRejection(path) + if len(promoted) != 1 { + t.Fatalf("expected 1 promotion after 5 rejections, got %d", len(promoted)) + } + if promoted[0] != "./.config/Code/" { + t.Errorf("promoted pattern = %q, want %q", promoted[0], "./.config/Code/") + } + + // Further rejections for the same prefix should be no-ops + rt.recordRejection(path) + if len(promoted) != 1 { + t.Errorf("expected no additional promotions, got %d", len(promoted)) + } +} + +func TestRejectionTracker_MaxPromotedCap(t *testing.T) { + home, err := os.UserHomeDir() + if err != nil { + t.Fatalf("could not get home dir: %v", err) + } + + var promoted []string + onPromote := func(pattern string) { + promoted = append(promoted, pattern) + } + + rt := newRejectionTracker( + rejectionTrackerConfig{ + Threshold: 1, + Window: 1 * time.Hour, + MaxPromoted: 2, + }, + testLogger(), + onPromote, + ) + + // Promote 2 distinct prefixes + rt.recordRejection(filepath.Join(home, ".config/AppA/file")) + rt.recordRejection(filepath.Join(home, ".config/AppB/file")) + + if len(promoted) != 2 { + t.Fatalf("expected 2 promotions, got %d", len(promoted)) + } + + // 3rd distinct prefix should be capped + rt.recordRejection(filepath.Join(home, ".config/AppC/file")) + if len(promoted) != 2 { + t.Errorf("expected cap at 2 promotions, got %d", len(promoted)) + } +} + +func TestRejectionTracker_Persistence(t *testing.T) { + tmpDir := t.TempDir() + persistPath := filepath.Join(tmpDir, "learned.txt") + + var promoted []string + onPromote := func(pattern string) { + promoted = append(promoted, pattern) + } + + home, err := os.UserHomeDir() + if err != nil { + t.Fatalf("could not get home dir: %v", err) + } + + // First tracker: promote a pattern + rt1 := newRejectionTracker( + rejectionTrackerConfig{ + Threshold: 1, + Window: 1 * time.Hour, + MaxPromoted: 10, + PersistPath: persistPath, + }, + testLogger(), + onPromote, + ) + rt1.recordRejection(filepath.Join(home, ".config/Code/User/settings.json")) + + if len(promoted) != 1 { + t.Fatalf("expected 1 promotion, got %d", len(promoted)) + } + + // Verify file was written + data, err := os.ReadFile(persistPath) + if err != nil { + t.Fatalf("failed to read persist file: %v", err) + } + if string(data) == "" { + t.Fatal("persist file is empty") + } + + // Second tracker: should load the persisted exclusion + rt2 := newRejectionTracker( + rejectionTrackerConfig{ + Threshold: 1, + Window: 1 * time.Hour, + MaxPromoted: 10, + PersistPath: persistPath, + }, + testLogger(), + nil, + ) + + exclusions := rt2.learnedExclusions() + if len(exclusions) != 1 { + t.Fatalf("expected 1 loaded exclusion, got %d", len(exclusions)) + } + if exclusions[0] != "./.config/Code/" { + t.Errorf("loaded exclusion = %q, want %q", exclusions[0], "./.config/Code/") + } +} diff --git a/internal/config/config.go b/internal/config/config.go index e5f761fe..28f96be2 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -59,12 +59,13 @@ type MemoryConfig struct { // PerceptionConfig holds perception settings. type PerceptionConfig struct { - Enabled bool `yaml:"enabled"` - LLMGatingEnabled bool `yaml:"llm_gating_enabled"` - Filesystem FilesystemPerceptionConfig `yaml:"filesystem"` - Terminal TerminalPerceptionConfig `yaml:"terminal"` - Clipboard ClipboardPerceptionConfig `yaml:"clipboard"` - Heuristics HeuristicsConfig `yaml:"heuristics"` + Enabled bool `yaml:"enabled"` + LLMGatingEnabled bool `yaml:"llm_gating_enabled"` + LearnedExclusionsPath string `yaml:"learned_exclusions_path"` + Filesystem FilesystemPerceptionConfig `yaml:"filesystem"` + Terminal TerminalPerceptionConfig `yaml:"terminal"` + Clipboard ClipboardPerceptionConfig `yaml:"clipboard"` + Heuristics HeuristicsConfig `yaml:"heuristics"` } // FilesystemPerceptionConfig holds filesystem perception settings. @@ -280,8 +281,9 @@ func Default() *Config { MaxWorkingMemory: 7, }, Perception: PerceptionConfig{ - Enabled: true, - LLMGatingEnabled: false, + Enabled: true, + LLMGatingEnabled: false, + LearnedExclusionsPath: "~/.mnemonic/learned-exclusions.txt", Filesystem: FilesystemPerceptionConfig{ Enabled: true, WatchDirs: []string{ @@ -436,6 +438,14 @@ func (c *Config) process(configDir string) error { c.Perception.Filesystem.WatchDirs[i] = expanded } + // Expand Perception learned exclusions path + if c.Perception.LearnedExclusionsPath != "" { + c.Perception.LearnedExclusionsPath, err = resolvePath(c.Perception.LearnedExclusionsPath, configDir) + if err != nil { + return fmt.Errorf("expanding perception.learned_exclusions_path: %w", err) + } + } + // Expand Logging file path c.Logging.File, err = resolvePath(c.Logging.File, configDir) if err != nil { diff --git a/internal/watcher/filesystem/watcher_darwin.go b/internal/watcher/filesystem/watcher_darwin.go index d4db0102..974a9319 100644 --- a/internal/watcher/filesystem/watcher_darwin.go +++ b/internal/watcher/filesystem/watcher_darwin.go @@ -116,6 +116,13 @@ func (fw *FilesystemWatcher) Events() <-chan watcher.Event { return fw.events } +// AddExclusion adds an exclusion pattern at runtime. Thread-safe. +func (fw *FilesystemWatcher) AddExclusion(pattern string) { + fw.mu.Lock() + defer fw.mu.Unlock() + fw.cfg.ExcludePatterns = append(fw.cfg.ExcludePatterns, pattern) +} + func (fw *FilesystemWatcher) Health(ctx context.Context) error { fw.mu.RLock() defer fw.mu.RUnlock() @@ -149,7 +156,10 @@ func (fw *FilesystemWatcher) processEvent(event fsevents.Event) { path := "/" + event.Path // FSEvents strips leading slash // Skip excluded paths - if MatchesExcludePattern(path, fw.cfg.ExcludePatterns) { + fw.mu.RLock() + excluded := MatchesExcludePattern(path, fw.cfg.ExcludePatterns) + fw.mu.RUnlock() + if excluded { return } diff --git a/internal/watcher/filesystem/watcher_other.go b/internal/watcher/filesystem/watcher_other.go index cbc2b908..42c206df 100644 --- a/internal/watcher/filesystem/watcher_other.go +++ b/internal/watcher/filesystem/watcher_other.go @@ -108,6 +108,13 @@ func (fw *FilesystemWatcher) Events() <-chan watcher.Event { return fw.events } +// AddExclusion adds an exclusion pattern at runtime. Thread-safe. +func (fw *FilesystemWatcher) AddExclusion(pattern string) { + fw.mu.Lock() + defer fw.mu.Unlock() + fw.cfg.ExcludePatterns = append(fw.cfg.ExcludePatterns, pattern) +} + func (fw *FilesystemWatcher) Health(ctx context.Context) error { fw.mu.RLock() defer fw.mu.RUnlock() @@ -125,7 +132,10 @@ func (fw *FilesystemWatcher) addDirRecursive(dir string) error { if !d.IsDir() { return nil } - if MatchesExcludePattern(path, fw.cfg.ExcludePatterns) { + fw.mu.RLock() + excluded := MatchesExcludePattern(path, fw.cfg.ExcludePatterns) + fw.mu.RUnlock() + if excluded { return filepath.SkipDir } if err := fw.fsw.Add(path); err != nil { @@ -148,7 +158,10 @@ func (fw *FilesystemWatcher) handleEvents(ctx context.Context) { if !ok { return } - if MatchesExcludePattern(event.Name, fw.cfg.ExcludePatterns) { + fw.mu.RLock() + excluded := MatchesExcludePattern(event.Name, fw.cfg.ExcludePatterns) + fw.mu.RUnlock() + if excluded { continue } if event.Has(fsnotify.Create) { diff --git a/internal/watcher/watcher.go b/internal/watcher/watcher.go index ec3cd1b1..03fc834f 100644 --- a/internal/watcher/watcher.go +++ b/internal/watcher/watcher.go @@ -33,3 +33,10 @@ type Watcher interface { // Health checks if the watcher is functioning. Health(ctx context.Context) error } + +// ExcludableWatcher is an optional interface for watchers that support +// adding exclusion patterns at runtime (e.g., learned from repeated rejections). +type ExcludableWatcher interface { + Watcher + AddExclusion(pattern string) +} From 9398cf916873212c4cfba15a8ac2b66c78244608 Mon Sep 17 00:00:00 2001 From: Caleb Gross Date: Sun, 1 Mar 2026 08:34:11 -0500 Subject: [PATCH 2/2] Remove unused defaultRejectionTrackerConfig function Fixes golangci-lint unused function error in CI. Co-Authored-By: Claude Opus 4.6 --- internal/agent/perception/rejection_tracker.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/internal/agent/perception/rejection_tracker.go b/internal/agent/perception/rejection_tracker.go index 7ddb3da1..52dab158 100644 --- a/internal/agent/perception/rejection_tracker.go +++ b/internal/agent/perception/rejection_tracker.go @@ -35,14 +35,6 @@ type rejectionTrackerConfig struct { PersistPath string // file to persist learned exclusions (empty = no persistence) } -func defaultRejectionTrackerConfig() rejectionTrackerConfig { - return rejectionTrackerConfig{ - Threshold: 50, - Window: 1 * time.Hour, - MaxPromoted: 20, - } -} - func newRejectionTracker(cfg rejectionTrackerConfig, log *slog.Logger, onPromote func(string)) *rejectionTracker { if cfg.Threshold == 0 { cfg.Threshold = 50