feat: EXP-25 faithfulness probe, encoding fixes, MCP HTTP transport

cmd/mnemonic/serve.go

@@ -685,6 +685,33 @@ func serveCommand(configPath string) {
 			return cfg.MemoryDefaults.SalienceForType(memType)
 		}
 
+		// Create MCP session manager for HTTP transport
+		mcpResolver := config.NewProjectResolver(cfg.Projects)
+		mcpSessions := mcp.NewSessionManager(mcp.SessionManagerConfig{
+			Store:           memStore,
+			Retriever:       retriever,
+			Bus:             bus,
+			Log:             log,
+			Version:         Version,
+			CoachingFile:    cfg.Coaching.CoachingFile,
+			ExcludePatterns: cfg.Perception.Filesystem.ExcludePatterns,
+			MaxContentBytes: cfg.Perception.Filesystem.MaxContentBytes,
+			Resolver:        mcpResolver,
+			DaemonURL:       fmt.Sprintf("http://%s:%d", cfg.API.Host, cfg.API.Port),
+			MemDefaults: mcp.MemoryDefaults{
+				SalienceGeneral:       cfg.MemoryDefaults.InitialSalienceGeneral,
+				SalienceDecision:      cfg.MemoryDefaults.InitialSalienceDecision,
+				SalienceError:         cfg.MemoryDefaults.InitialSalienceError,
+				SalienceInsight:       cfg.MemoryDefaults.InitialSalienceInsight,
+				SalienceLearning:      cfg.MemoryDefaults.InitialSalienceLearning,
+				SalienceHandoff:       cfg.MemoryDefaults.InitialSalienceHandoff,
+				FeedbackStrengthDelta: cfg.MemoryDefaults.FeedbackStrengthDelta,
+				FeedbackSalienceBoost: cfg.MemoryDefaults.FeedbackSalienceBoost,
+			},
+		})
+		apiDeps.MCPSessions = mcpSessions
+		defer mcpSessions.Stop(rootCtx)
+
 		apiServer := api.NewServer(api.ServerConfig{
 			Host:              cfg.API.Host,
 			Port:              cfg.API.Port,

internal/agent/encoding/agent.go

@@ -1147,10 +1147,9 @@ func (ea *EncodingAgent) compressAndExtractConcepts(ctx context.Context, raw sto
 
 	// Gather contextual information for richer encoding
 	episodeCtx := ea.getEpisodeContext(ctx, raw)
-	relatedCtx := ea.getRelatedContext(ctx, raw)
 
 	// Build the LLM prompt
-	prompt := buildCompressionPrompt(truncatedContent, raw.Source, raw.Type, episodeCtx, relatedCtx, ea.coachingInstructions, ea.config.ConceptVocabulary)
+	prompt := buildCompressionPrompt(truncatedContent, raw.Source, raw.Type, episodeCtx, ea.coachingInstructions, ea.config.ConceptVocabulary)
 
 	req := llm.CompletionRequest{
 		Messages: []llm.Message{
@@ -1221,7 +1220,7 @@ func (ea *EncodingAgent) compressAndExtractConcepts(ctx context.Context, raw sto
 // NOTE: The prompt deliberately avoids showing a JSON template because the local LLM model
 // echoes template placeholder text verbatim into the output fields. Structured output
 // (response_format with json_schema) enforces the JSON structure instead.
-func buildCompressionPrompt(content, source, memType, episodeCtx, relatedCtx, coachingInstructions string, conceptVocabulary []string) string {
+func buildCompressionPrompt(content, source, memType, episodeCtx, coachingInstructions string, conceptVocabulary []string) string {
 	var b strings.Builder
 
 	if source == "ingest" {
@@ -1268,10 +1267,6 @@ Fill in every JSON field based on the actual event content below:
 	if episodeCtx != "" {
 		b.WriteString(episodeCtx)
 	}
-	if relatedCtx != "" {
-		b.WriteString(relatedCtx)
-	}
-
 	if coachingInstructions != "" {
 		b.WriteString(coachingInstructions)
 		b.WriteString("\n\n")
@@ -1779,35 +1774,6 @@ func (ea *EncodingAgent) getEpisodeContext(ctx context.Context, raw store.RawMem
 	return result
 }
 
-// getRelatedContext gathers semantically similar existing memories for context.
-func (ea *EncodingAgent) getRelatedContext(ctx context.Context, raw store.RawMemory) string {
-	// Use concept-based search with keywords from the raw content
-	words := extractKeywords(raw.Content)
-	if len(words) == 0 {
-		return ""
-	}
-
-	if len(words) > 5 {
-		words = words[:5]
-	}
-
-	related, err := ea.store.SearchByConcepts(ctx, words, 3)
-	if err != nil || len(related) == 0 {
-		return ""
-	}
-
-	result := "RELATED EXISTING MEMORIES:\n"
-	for _, mem := range related {
-		result += fmt.Sprintf("  - [%s] %s (concepts: %s)\n",
-			mem.Timestamp.Format("2006-01-02 15:04"),
-			mem.Summary,
-			joinConcepts(mem.Concepts),
-		)
-	}
-	result += "\n"
-	return result
-}
-
 // getEpisodeIDForRaw finds which episode a raw memory belongs to.
 // Checks both open and recently closed episodes since encoding is async
 // and the episode may close before encoding completes.
@@ -1836,47 +1802,6 @@ func getEpisodeIDForRaw(ea *EncodingAgent, ctx context.Context, raw store.RawMem
 	return ""
 }
 
-// extractKeywords pulls significant words from content for concept search.
-func extractKeywords(content string) []string {
-	// Simple keyword extraction: split, filter short/common words
-	words := strings.Fields(strings.ToLower(content))
-	seen := make(map[string]bool)
-	var keywords []string
-
-	stopWords := map[string]bool{
-		"the": true, "a": true, "an": true, "is": true, "was": true,
-		"are": true, "were": true, "be": true, "been": true, "being": true,
-		"have": true, "has": true, "had": true, "do": true, "does": true,
-		"did": true, "will": true, "would": true, "could": true, "should": true,
-		"may": true, "might": true, "shall": true, "can": true, "to": true,
-		"of": true, "in": true, "for": true, "on": true, "with": true,
-		"at": true, "by": true, "from": true, "as": true, "into": true,
-		"through": true, "during": true, "before": true, "after": true,
-		"it": true, "its": true, "this": true, "that": true, "these": true,
-		"and": true, "but": true, "or": true, "nor": true, "not": true,
-	}
-
-	for _, w := range words {
-		if len(w) < 3 || stopWords[w] || seen[w] {
-			continue
-		}
-		seen[w] = true
-		keywords = append(keywords, w)
-		if len(keywords) >= 10 {
-			break
-		}
-	}
-	return keywords
-}
-
-// joinConcepts joins concepts with commas.
-func joinConcepts(concepts []string) string {
-	if len(concepts) == 0 {
-		return "none"
-	}
-	return strings.Join(concepts, ", ")
-}
-
 // truncateString truncates a string to maxLen characters.
 // Uses rune-aware slicing to avoid splitting multi-byte UTF-8 characters.
 func truncateString(s string, maxLen int) string {

internal/agent/encoding/agent_test.go

@@ -539,91 +539,6 @@ func TestHeuristicSalience(t *testing.T) {
 	})
 }
 
-// ---------------------------------------------------------------------------
-// Tests for extractKeywords
-// ---------------------------------------------------------------------------
-
-func TestExtractKeywords(t *testing.T) {
-	t.Run("extracts meaningful words", func(t *testing.T) {
-		keywords := extractKeywords("debugging the authentication module for error handling")
-
-		if len(keywords) == 0 {
-			t.Fatal("expected at least one keyword")
-		}
-		// Should not contain stop words
-		for _, kw := range keywords {
-			if kw == "the" || kw == "for" {
-				t.Errorf("unexpected stop word %q in keywords", kw)
-			}
-		}
-	})
-
-	t.Run("limits to 10 keywords", func(t *testing.T) {
-		longContent := strings.Repeat("alpha bravo charlie delta echo foxtrot golf hotel india juliet kilo lima ", 5)
-		keywords := extractKeywords(longContent)
-
-		if len(keywords) > 10 {
-			t.Errorf("expected at most 10 keywords, got %d", len(keywords))
-		}
-	})
-
-	t.Run("deduplicates words", func(t *testing.T) {
-		keywords := extractKeywords("testing testing testing testing")
-		count := 0
-		for _, kw := range keywords {
-			if kw == "testing" {
-				count++
-			}
-		}
-		if count > 1 {
-			t.Errorf("expected 'testing' to appear at most once, appeared %d times", count)
-		}
-	})
-
-	t.Run("empty content returns empty", func(t *testing.T) {
-		keywords := extractKeywords("")
-		if len(keywords) != 0 {
-			t.Errorf("expected empty keywords for empty content, got %v", keywords)
-		}
-	})
-
-	t.Run("filters short words", func(t *testing.T) {
-		keywords := extractKeywords("go is ok to do it")
-		for _, kw := range keywords {
-			if len(kw) < 3 {
-				t.Errorf("unexpected short word %q in keywords", kw)
-			}
-		}
-	})
-}
-
-// ---------------------------------------------------------------------------
-// Tests for joinConcepts
-// ---------------------------------------------------------------------------
-
-func TestJoinConcepts(t *testing.T) {
-	t.Run("joins concepts with comma", func(t *testing.T) {
-		result := joinConcepts([]string{"go", "testing", "memory"})
-		if result != "go, testing, memory" {
-			t.Errorf("expected 'go, testing, memory', got %q", result)
-		}
-	})
-
-	t.Run("empty returns none", func(t *testing.T) {
-		result := joinConcepts([]string{})
-		if result != "none" {
-			t.Errorf("expected 'none', got %q", result)
-		}
-	})
-
-	t.Run("single concept", func(t *testing.T) {
-		result := joinConcepts([]string{"single"})
-		if result != "single" {
-			t.Errorf("expected 'single', got %q", result)
-		}
-	})
-}
-
 // ---------------------------------------------------------------------------
 // Tests for isTemporalRelationship
 // ---------------------------------------------------------------------------

internal/api/routes/mcp.go

@@ -0,0 +1,92 @@
+package routes
+
+import (
+	"encoding/json"
+	"io"
+	"log/slog"
+	"net/http"
+
+	"github.com/appsprout-dev/mnemonic/internal/mcp"
+)
+
+// HandleMCP returns an HTTP handler for the MCP JSON-RPC protocol.
+//
+// Session lifecycle follows the MCP streamable HTTP transport spec:
+//   - First request (initialize): no Mcp-Session-Id header needed.
+//     Server creates a session and returns the ID in the response header.
+//   - Subsequent requests: client includes Mcp-Session-Id from the
+//     initialize response. Server routes to the existing session.
+//   - DELETE with Mcp-Session-Id: explicitly ends the session.
+//   - Idle sessions are reaped by the session manager after timeout.
+func HandleMCP(sm *mcp.SessionManager, log *slog.Logger) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if r.Method == http.MethodDelete {
+			handleMCPDelete(sm, log, w, r)
+			return
+		}
+		if r.Method != http.MethodPost {
+			http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+			return
+		}
+
+		// Read and parse the JSON-RPC request
+		body, err := io.ReadAll(io.LimitReader(r.Body, 1<<20)) // 1MB limit
+		if err != nil {
+			writeJSONRPCError(w, nil, -32700, "Failed to read request body")
+			return
+		}
+
+		var req mcp.JSONRPCRequest
+		if err := json.Unmarshal(body, &req); err != nil {
+			writeJSONRPCError(w, nil, -32700, "Parse error")
+			return
+		}
+
+		// Resolve session: use client header if present, otherwise create new
+		clientSessionID := r.Header.Get("Mcp-Session-Id")
+		srv, sessionKey := sm.GetOrCreate(clientSessionID)
+
+		resp := srv.HandleSingleRequest(r.Context(), &req)
+
+		// Always return the session ID so the client can include it in subsequent requests
+		w.Header().Set("Mcp-Session-Id", sessionKey)
+
+		// Notifications return nil — respond with 202 Accepted
+		if resp == nil {
+			w.WriteHeader(http.StatusAccepted)
+			return
+		}
+
+		w.Header().Set("Content-Type", "application/json")
+		if err := json.NewEncoder(w).Encode(resp); err != nil {
+			log.Warn("failed to encode MCP HTTP response", "error", err)
+		}
+	}
+}
+
+// handleMCPDelete explicitly ends an MCP session.
+func handleMCPDelete(sm *mcp.SessionManager, log *slog.Logger, w http.ResponseWriter, r *http.Request) {
+	sessionID := r.Header.Get("Mcp-Session-Id")
+	if sessionID == "" {
+		http.Error(w, "Mcp-Session-Id header is required", http.StatusBadRequest)
+		return
+	}
+
+	sm.EndSession(r.Context(), sessionID)
+	log.Info("MCP session explicitly ended via DELETE", "session_id", sessionID)
+	w.WriteHeader(http.StatusNoContent)
+}
+
+// writeJSONRPCError writes a JSON-RPC error response.
+func writeJSONRPCError(w http.ResponseWriter, id interface{}, code int, message string) {
+	w.Header().Set("Content-Type", "application/json")
+	w.WriteHeader(http.StatusOK) // JSON-RPC errors are still 200
+	_ = json.NewEncoder(w).Encode(map[string]interface{}{
+		"jsonrpc": "2.0",
+		"id":      id,
+		"error": map[string]interface{}{
+			"code":    code,
+			"message": message,
+		},
+	})
+}

internal/api/server.go

@@ -13,6 +13,7 @@ import (
 	"github.com/appsprout-dev/mnemonic/internal/api/routes"
 	"github.com/appsprout-dev/mnemonic/internal/events"
 	"github.com/appsprout-dev/mnemonic/internal/llm"
+	"github.com/appsprout-dev/mnemonic/internal/mcp"
 	"github.com/appsprout-dev/mnemonic/internal/store"
 	"github.com/appsprout-dev/mnemonic/internal/web"
 )
@@ -30,7 +31,7 @@ type ServerConfig struct {
 type ServerDeps struct {
 	Store                 store.Store
 	LLM                   llm.Provider
-	ModelManager          llm.ModelManager        // can be nil if not using embedded provider
+	ModelManager          llm.ModelManager // can be nil if not using embedded provider
 	Bus                   events.Bus
 	Retriever             *retrieval.RetrievalAgent
 	Consolidator          routes.ConsolidationRunner // can be nil if disabled
@@ -43,6 +44,7 @@ type ServerDeps struct {
 	ServiceRestarter      routes.ServiceRestarter // can be nil if not installed as service
 	PIDRestart            routes.PIDRestartFunc   // fallback restart when service manager unavailable
 	MCPToolCount          int                     // number of registered MCP tools
+	MCPSessions           *mcp.SessionManager     // HTTP MCP session manager (nil = disabled)
 	StartTime             time.Time               // daemon start time for uptime calculation
 	Log                   *slog.Logger
 }
@@ -173,6 +175,13 @@ func (s *Server) registerRoutes() {
 	s.mux.HandleFunc("PATCH /api/v1/forum/posts/{id}", routes.HandleUpdateForumPost(s.deps.Store, s.deps.Log))
 	s.mux.HandleFunc("POST /api/v1/forum/posts/{id}/internalize", routes.HandleInternalizeForumPost(s.deps.Store, s.deps.Bus, s.deps.Log))
 
+	// MCP over HTTP transport (shares daemon's LLM, store, agents — no subprocess needed)
+	if s.deps.MCPSessions != nil {
+		mcpHandler := routes.HandleMCP(s.deps.MCPSessions, s.deps.Log)
+		s.mux.HandleFunc("POST /mcp", mcpHandler)
+		s.mux.HandleFunc("DELETE /mcp", mcpHandler)
+	}
+
 	// WebSocket
 	s.mux.HandleFunc("GET /ws", routes.HandleWebSocket(s.deps.Bus, s.deps.Log))