Generic scope ID extraction and remote-scope API client (#115)

* Initial plan

* feat: generic scope ID extraction and remote-scope API client

- Add ScopeIDField and HasRepoScopes to ConnectionDef; set on github/gh-copilot entries
- Replace ScopeListEntry typed struct with json.RawMessage + extractScopeID helper
- Add RemoteScopeChild/RemoteScopeResponse types and ListRemoteScopes/SearchRemoteScopes client methods
- Remove hardcoded if-plugin-github checks from listConnectionScopes, scope list, scope delete
- Add table-driven tests for ExtractScopeID, ScopeListWrapper helpers, and new client methods

Co-authored-by: ewega <26189114+ewega@users.noreply.github.com>

* fix: ScopeName empty-string fallthrough and deduplicate ScopeFullName call

- ScopeName() now skips empty strings (like ExtractScopeID does), so a
  JSON field of "" correctly falls through to the next candidate key
- configure_projects.go: call ScopeFullName() once per scope and reuse
  the value for both the display name and repo-tracking check
- Add "empty fullName falls through to name" test case

Co-authored-by: ewega <26189114+ewega@users.noreply.github.com>

* fix: cache parseScope, ScopeFullName empty-string, ID fallback, pagination test assertions

- ScopeListWrapper gains a shared parseScope() helper with lazy map caching;
  ScopeName() and ScopeFullName() now unmarshal RawScope at most once per item
- ScopeFullName() now treats "" as absent (consistent with ScopeName/ExtractScopeID)
- configure_scope_list: deduplicate FindConnectionDef; add scopeIDFor() closure
  that falls back to ScopeName() when ExtractScopeID returns ""
- configure_scope_delete: add id=="" fallback to name so interactive delete
  always has a usable identifier even when ScopeIDField payload is missing
- TestSearchRemoteScopes: assert page, pageSize, and search query params are
  sent when non-zero and absent when zero

Co-authored-by: ewega <26189114+ewega@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: ewega <26189114+ewega@users.noreply.github.com>

Author: Copilot

cmd/configure_projects.go

@@ -233,23 +233,27 @@ func listConnectionScopes(client *devlake.Client, c connChoice) (*addedConnectio
 
 	var bpScopes []devlake.BlueprintScope
 	var repos []string
+	def := FindConnectionDef(c.plugin)
 	for _, w := range resp.Scopes {
-		s := w.Scope
-		// Resolve scope ID: GitHub uses githubId (int), Copilot uses id (string)
-		scopeID := s.ID
-		if c.plugin == "github" && s.GithubID > 0 {
-			scopeID = fmt.Sprintf("%d", s.GithubID)
+		// Generic scope ID extraction using the plugin's configured ScopeIDField.
+		var scopeID string
+		if def != nil && def.ScopeIDField != "" {
+			scopeID = devlake.ExtractScopeID(w.RawScope, def.ScopeIDField)
 		}
-		scopeName := s.FullName
+		fullName := w.ScopeFullName()
+		scopeName := fullName
 		if scopeName == "" {
-			scopeName = s.Name
+			scopeName = w.ScopeName()
+		}
+		if scopeID == "" {
+			scopeID = scopeName
 		}
 		bpScopes = append(bpScopes, devlake.BlueprintScope{
 			ScopeID:   scopeID,
 			ScopeName: scopeName,
 		})
-		if c.plugin == "github" && s.FullName != "" {
-			repos = append(repos, s.FullName)
+		if def != nil && def.HasRepoScopes && fullName != "" {
+			repos = append(repos, fullName)
 		}
 		fmt.Printf("   %s (ID: %s)\n", scopeName, scopeID)
 	}

cmd/configure_scope_delete.go

@@ -2,11 +2,11 @@ package cmd
 
 import (
 	"fmt"
-	"strconv"
 	"strings"
 
 	"github.com/spf13/cobra"
 
+	"github.com/DevExpGBB/gh-devlake/internal/devlake"
 	"github.com/DevExpGBB/gh-devlake/internal/prompt"
 )
 
@@ -101,14 +101,18 @@ func runScopeDelete(cmd *cobra.Command, args []string) error {
 		}
 		var entries []scopeEntry
 		var labels []string
+		def := FindConnectionDef(selectedPlugin)
 		for _, s := range resp.Scopes {
-			id := s.Scope.ID
-			if id == "" {
-				id = strconv.Itoa(s.Scope.GithubID)
-			}
-			name := s.Scope.FullName
+			name := s.ScopeFullName()
 			if name == "" {
-				name = s.Scope.Name
+				name = s.ScopeName()
+			}
+			var id string
+			if def != nil && def.ScopeIDField != "" {
+				id = devlake.ExtractScopeID(s.RawScope, def.ScopeIDField)
+			}
+			if id == "" {
+				id = name
 			}
 			label := fmt.Sprintf("[%s] %s", id, name)
 			entries = append(entries, scopeEntry{id: id, label: label})

cmd/configure_scope_list.go

@@ -2,7 +2,6 @@ package cmd
 
 import (
 	"fmt"
-	"strconv"
 	"strings"
 	"text/tabwriter"
 
@@ -107,18 +106,28 @@ func runScopeList(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("failed to list scopes: %w", err)
 	}
 
+	def := FindConnectionDef(selectedPlugin)
+
+	// scopeIDFor extracts the scope ID using the plugin's ScopeIDField (def.ScopeIDField).
+	// When ExtractScopeID returns "" (field absent or plugin has no ScopeIDField configured),
+	// it falls back to the display name so the output always shows a usable identifier.
+	scopeIDFor := func(s *devlake.ScopeListWrapper) string {
+		if def != nil && def.ScopeIDField != "" {
+			if id := devlake.ExtractScopeID(s.RawScope, def.ScopeIDField); id != "" {
+				return id
+			}
+		}
+		return s.ScopeName()
+	}
+
 	// JSON output path
 	if outputJSON {
 		items := make([]scopeListItem, len(resp.Scopes))
 		for i, s := range resp.Scopes {
-			scopeID := s.Scope.ID
-			if scopeID == "" {
-				scopeID = strconv.Itoa(s.Scope.GithubID)
-			}
 			items[i] = scopeListItem{
-				ID:       scopeID,
-				Name:     s.Scope.Name,
-				FullName: s.Scope.FullName,
+				ID:       scopeIDFor(&s),
+				Name:     s.ScopeName(),
+				FullName: s.ScopeFullName(),
 			}
 		}
 		return printJSON(items)
@@ -133,11 +142,7 @@ func runScopeList(cmd *cobra.Command, args []string) error {
 	fmt.Fprintln(w, "Scope ID\tName\tFull Name")
 	fmt.Fprintln(w, strings.Repeat("\u2500", 10)+"\t"+strings.Repeat("\u2500", 20)+"\t"+strings.Repeat("\u2500", 30))
 	for _, s := range resp.Scopes {
-		scopeID := s.Scope.ID
-		if scopeID == "" {
-			scopeID = strconv.Itoa(s.Scope.GithubID)
-		}
-		fmt.Fprintf(w, "%s\t%s\t%s\n", scopeID, s.Scope.Name, s.Scope.FullName)
+		fmt.Fprintf(w, "%s\t%s\t%s\n", scopeIDFor(&s), s.ScopeName(), s.ScopeFullName())
 	}
 	w.Flush()
 	fmt.Println()

cmd/connection_types.go

@@ -35,6 +35,8 @@ type ConnectionDef struct {
 	EnvVarNames      []string     // environment variable names for token resolution
 	EnvFileKeys      []string     // .devlake.env keys for token resolution
 	ScopeFunc        ScopeHandler // nil = scope configuration not yet supported
+	ScopeIDField     string       // JSON field name for the scope ID (e.g. "githubId", "id")
+	HasRepoScopes    bool         // true = scopes carry a FullName that should be tracked as repos
 }
 
 // MenuLabel returns the label for interactive menus.
@@ -144,6 +146,8 @@ var connectionRegistry = []*ConnectionDef{
 		EnvVarNames:      []string{"GITHUB_PAT", "GITHUB_TOKEN", "GH_TOKEN"},
 		EnvFileKeys:      []string{"GITHUB_PAT", "GITHUB_TOKEN", "GH_TOKEN"},
 		ScopeFunc:        scopeGitHubHandler,
+		ScopeIDField:     "githubId",
+		HasRepoScopes:    true,
 	},
 	{
 		Plugin:           "gh-copilot",
@@ -161,6 +165,7 @@ var connectionRegistry = []*ConnectionDef{
 		EnvVarNames:      []string{"GITHUB_PAT", "GITHUB_TOKEN", "GH_TOKEN"},
 		EnvFileKeys:      []string{"GITHUB_PAT", "GITHUB_TOKEN", "GH_TOKEN"},
 		ScopeFunc:        scopeCopilotHandler,
+		ScopeIDField:     "id",
 	},
 	{
 		Plugin:      "gitlab",

internal/devlake/client.go

@@ -462,6 +462,43 @@ func (c *Client) GetPipeline(id int) (*Pipeline, error) {
 	return doGet[Pipeline](c, fmt.Sprintf("/pipelines/%d", id))
 }
 
+// ListRemoteScopes queries the DevLake remote-scope API for a plugin connection.
+// groupID and pageToken are optional (pass "" to omit).
+func (c *Client) ListRemoteScopes(plugin string, connID int, groupID, pageToken string) (*RemoteScopeResponse, error) {
+	path := fmt.Sprintf("/plugins/%s/connections/%d/remote-scopes", plugin, connID)
+	q := url.Values{}
+	if groupID != "" {
+		q.Set("groupId", groupID)
+	}
+	if pageToken != "" {
+		q.Set("pageToken", pageToken)
+	}
+	if len(q) > 0 {
+		path += "?" + q.Encode()
+	}
+	return doGet[RemoteScopeResponse](c, path)
+}
+
+// SearchRemoteScopes queries the DevLake search-remote-scopes API for a plugin connection.
+// page and pageSize control pagination; pass 0 to use DevLake defaults.
+func (c *Client) SearchRemoteScopes(plugin string, connID int, search string, page, pageSize int) (*RemoteScopeResponse, error) {
+	path := fmt.Sprintf("/plugins/%s/connections/%d/search-remote-scopes", plugin, connID)
+	q := url.Values{}
+	if search != "" {
+		q.Set("search", search)
+	}
+	if page > 0 {
+		q.Set("page", fmt.Sprintf("%d", page))
+	}
+	if pageSize > 0 {
+		q.Set("pageSize", fmt.Sprintf("%d", pageSize))
+	}
+	if len(q) > 0 {
+		path += "?" + q.Encode()
+	}
+	return doGet[RemoteScopeResponse](c, path)
+}
+
 // TriggerMigration triggers the DevLake database migration endpoint.
 func (c *Client) TriggerMigration() error {
 	resp, err := c.HTTPClient.Get(c.BaseURL + "/proceed-db-migration")

internal/devlake/client_test.go

@@ -589,8 +589,8 @@ func TestListScopes(t *testing.T) {
 	if len(result.Scopes) != 1 {
 		t.Fatalf("len(Scopes) = %d, want 1", len(result.Scopes))
 	}
-	if result.Scopes[0].Scope.Name != "repo1" {
-		t.Errorf("Name = %q, want %q", result.Scopes[0].Scope.Name, "repo1")
+	if result.Scopes[0].ScopeName() != "repo1" {
+		t.Errorf("Name = %q, want %q", result.Scopes[0].ScopeName(), "repo1")
 	}
 }
 

internal/devlake/types.go

@@ -1,5 +1,10 @@
 package devlake
 
+import (
+	"encoding/json"
+	"strconv"
+)
+
 // ScopeConfig represents a DevLake scope configuration (e.g., DORA settings).
 type ScopeConfig struct {
 	ID                int            `json:"id,omitempty"`
@@ -46,18 +51,80 @@ type ScopeBatchRequest struct {
 
 // ScopeListWrapper wraps a scope object as returned by the DevLake GET scopes API.
 // The API nests each scope inside a "scope" key: { "scope": { ... } }.
+// RawScope preserves the full plugin-specific payload for generic ID extraction.
 type ScopeListWrapper struct {
-	Scope ScopeListEntry `json:"scope"`
-}
-
-// ScopeListEntry represents a scope object returned inside the wrapper.
-// ID fields vary by plugin (githubId for GitHub, id for Copilot), so we
-// capture both and resolve in the caller.
-type ScopeListEntry struct {
-	GithubID int    `json:"githubId,omitempty"`
-	ID       string `json:"id,omitempty"`
-	Name     string `json:"name"`
-	FullName string `json:"fullName,omitempty"`
+	RawScope json.RawMessage            `json:"scope"`
+	parsed   map[string]json.RawMessage // lazily populated by parseScope
+}
+
+// parseScope unmarshals RawScope into a map exactly once per wrapper instance,
+// caching the result so callers that invoke both ScopeName and ScopeFullName on
+// the same item do not unmarshal the same JSON twice.
+func (w *ScopeListWrapper) parseScope() map[string]json.RawMessage {
+	if w.parsed == nil {
+		var m map[string]json.RawMessage
+		if err := json.Unmarshal(w.RawScope, &m); err != nil || m == nil {
+			m = make(map[string]json.RawMessage)
+		}
+		w.parsed = m
+	}
+	return w.parsed
+}
+
+// ScopeName returns the display name from the raw scope JSON (checks "fullName" then "name").
+// Empty string values are skipped so the next candidate key is tried.
+// Parsing is cached via parseScope() so calling ScopeName and ScopeFullName on the
+// same instance only unmarshals the JSON once.
+func (w *ScopeListWrapper) ScopeName() string {
+	m := w.parseScope()
+	for _, key := range []string{"fullName", "name"} {
+		if v, ok := m[key]; ok {
+			var s string
+			if err := json.Unmarshal(v, &s); err == nil && s != "" {
+				return s
+			}
+		}
+	}
+	return ""
+}
+
+// ScopeFullName returns the "fullName" field from the raw scope JSON, or "".
+// An empty string value is treated as absent (returns "").
+func (w *ScopeListWrapper) ScopeFullName() string {
+	m := w.parseScope()
+	if v, ok := m["fullName"]; ok {
+		var s string
+		if err := json.Unmarshal(v, &s); err == nil && s != "" {
+			return s
+		}
+	}
+	return ""
+}
+
+// ExtractScopeID extracts the scope ID from a raw JSON scope object using the
+// given field name.  It tries to decode the value as a string first, then as
+// an integer (converted to its decimal string representation).
+func ExtractScopeID(raw json.RawMessage, fieldName string) string {
+	if fieldName == "" {
+		return ""
+	}
+	var m map[string]json.RawMessage
+	if err := json.Unmarshal(raw, &m); err != nil {
+		return ""
+	}
+	v, ok := m[fieldName]
+	if !ok {
+		return ""
+	}
+	var s string
+	if err := json.Unmarshal(v, &s); err == nil && s != "" {
+		return s
+	}
+	var n int64
+	if err := json.Unmarshal(v, &n); err == nil && n != 0 {
+		return strconv.FormatInt(n, 10)
+	}
+	return ""
 }
 
 // ScopeListResponse is the response from GET /plugins/{plugin}/connections/{id}/scopes.
@@ -66,6 +133,22 @@ type ScopeListResponse struct {
 	Count  int                `json:"count"`
 }
 
+// RemoteScopeChild represents one item (group or scope) from the remote-scope API.
+type RemoteScopeChild struct {
+	Type     string          `json:"type"`     // "group" or "scope"
+	ID       string          `json:"id"`
+	ParentID string          `json:"parentId"`
+	Name     string          `json:"name"`
+	FullName string          `json:"fullName"`
+	Data     json.RawMessage `json:"data"`
+}
+
+// RemoteScopeResponse is the response from GET /plugins/{plugin}/connections/{id}/remote-scopes.
+type RemoteScopeResponse struct {
+	Children      []RemoteScopeChild `json:"children"`
+	NextPageToken string             `json:"nextPageToken"`
+}
+
 // Project represents a DevLake project.
 type Project struct {
 	Name        string          `json:"name"`