diff --git a/.gitignore b/.gitignore index b89b1ab..5e0b07f 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ someguy +.autoconf-cache diff --git a/CHANGELOG.md b/CHANGELOG.md index fd2efac..7b2722b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,10 @@ The following emojis are used to highlight certain changes: ### Added +- AutoConf support: automatic configuration of bootstrap peers and delegated routing endpoints ([#123](https://github.com/ipfs/someguy/pull/123)). When enabled (default), the `auto` placeholder is replaced with network-recommended values. + - All endpoint flags (`--provider-endpoints`, `--peer-endpoints`, `--ipns-endpoints`) default to `auto` + - See [environment-variables.md](docs/environment-variables.md#someguy_autoconf) for configuration details + ### Changed - [go-libp2p v0.45.0](https://github.com/libp2p/go-libp2p/releases/tag/v0.45.0) diff --git a/README.md b/README.md index aea245b..7ba550a 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,27 @@ If you don't want to run a server yourself, but want to query some other server, For more details run `someguy ask --help`. +### AutoConf + +Automatic configuration of bootstrap peers and delegated routing endpoints. When enabled (default), the `auto` placeholder is replaced with network-recommended values fetched from a remote URL. + +Configuration: +- `--autoconf` / [`SOMEGUY_AUTOCONF`](docs/environment-variables.md#someguy_autoconf) +- `--autoconf-url` / [`SOMEGUY_AUTOCONF_URL`](docs/environment-variables.md#someguy_autoconf_url) +- `--autoconf-refresh` / [`SOMEGUY_AUTOCONF_REFRESH`](docs/environment-variables.md#someguy_autoconf_refresh) + +Endpoint flags (default to `auto`): +- `--provider-endpoints` / [`SOMEGUY_PROVIDER_ENDPOINTS`](docs/environment-variables.md#someguy_provider_endpoints) +- `--peer-endpoints` / [`SOMEGUY_PEER_ENDPOINTS`](docs/environment-variables.md#someguy_peer_endpoints) +- `--ipns-endpoints` / [`SOMEGUY_IPNS_ENDPOINTS`](docs/environment-variables.md#someguy_ipns_endpoints) + +To use custom endpoints instead of `auto`: +```bash +someguy start --ipns-endpoints https://example.com +``` + +See [environment-variables.md](docs/environment-variables.md) for URL formats and configuration details. + ## Deployment Suggested method for self-hosting is to run a [prebuilt Docker image](#docker). diff --git a/autoconf.go b/autoconf.go new file mode 100644 index 0000000..c193b61 --- /dev/null +++ b/autoconf.go @@ -0,0 +1,264 @@ +// autoconf.go implements automatic configuration for someguy. +// +// Autoconf fetches network configuration from a remote JSON endpoint to automatically +// configure bootstrap peers and delegated routing endpoints. +// +// The autoconf system: +// - Fetches configuration from a remote URL (configurable) +// - Caches configuration locally and refreshes periodically +// - Falls back to embedded defaults if fetching fails +// - Expands "auto" placeholder in endpoint configuration +// - Filters out endpoints for systems running natively (e.g., DHT) +// - Validates and normalizes endpoint URLs +// +// See https://github.com/ipfs/someguy/blob/main/docs/environment-variables.md +// for configuration options and defaults. +package main + +import ( + "context" + "fmt" + "path/filepath" + "slices" + "strings" + "time" + + "github.com/ipfs/boxo/autoconf" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/multiformats/go-multiaddr" +) + +// autoConfConfig contains the configuration for the autoconf subsystem. +type autoConfConfig struct { + // enabled determines whether to use autoconf + // Default: true + enabled bool + + // url is the HTTP(S) URL to fetch the autoconf.json from + // Default: https://conf.ipfs-mainnet.org/autoconf.json + url string + + // refreshInterval is how often to refresh autoconf data + // Default: 24h + refreshInterval time.Duration + + // cacheDir is the directory to cache autoconf data + // Default: $SOMEGUY_DATADIR/.autoconf-cache + cacheDir string +} + +func startAutoConf(ctx context.Context, cfg *config) (*autoconf.Config, error) { + var autoConf *autoconf.Config + if cfg.autoConf.enabled && cfg.autoConf.url != "" { + client, err := createAutoConfClient(cfg.autoConf) + if err != nil { + return nil, fmt.Errorf("failed to create autoconf client: %w", err) + } + // Start primes cache and starts background updater + // Note: Start() always returns a config (using fallback if needed) + autoConf, err = client.Start(ctx) + if err != nil { + return nil, fmt.Errorf("failed to start autoconf updater: %w", err) + } + } + return autoConf, nil +} + +func getBootstrapPeerAddrInfos(cfg *config, autoConf *autoconf.Config) []peer.AddrInfo { + if autoConf != nil { + nativeSystems := getNativeSystems(cfg.dhtType) + return stringsToPeerAddrInfos(autoConf.GetBootstrapPeers(nativeSystems...)) + } + // Fallback to autoconf fallback bootstrappers. + return stringsToPeerAddrInfos(autoconf.FallbackBootstrapPeers) +} + +// normalizeEndpointURL validates and normalizes a single endpoint URL for a specific routing type. +// Returns the base URL (with routing path stripped if present) and an error if the URL has a mismatched path. +func normalizeEndpointURL(url, expectedPath, flagName string) (string, error) { + // "auto" placeholder passes through unchanged + if url == autoconf.AutoPlaceholder { + return url, nil + } + + // Check if URL has the expected routing path + if strings.HasSuffix(url, expectedPath) { + // Strip the expected path to get base URL + return strings.TrimSuffix(url, expectedPath), nil + } + + // Check if URL has a different routing path (potential misconfiguration) + routingPaths := [...]string{ + autoconf.RoutingV1ProvidersPath, + autoconf.RoutingV1PeersPath, + autoconf.RoutingV1IPNSPath, + } + for _, path := range routingPaths { + if strings.HasSuffix(url, path) { + return "", fmt.Errorf("URL %q has path %q which doesn't match %s (expected %q or no path)", url, path, flagName, expectedPath) + } + } + + // URL has no routing path or unknown path - treat as base URL + return url, nil +} + +// validateEndpointURLs validates and normalizes a list of endpoint URLs for a specific routing type +func validateEndpointURLs(urls []string, expectedPath, flagName, envVar string) ([]string, error) { + normalized := make([]string, 0, len(urls)) + for _, url := range urls { + baseURL, err := normalizeEndpointURL(url, expectedPath, flagName) + if err != nil { + return nil, fmt.Errorf("%s: %w. Use %s or %s to fix", envVar, err, envVar, flagName) + } + normalized = append(normalized, baseURL) + } + return normalized, nil +} + +// expandDelegatedRoutingEndpoints expands autoconf placeholders and categorizes endpoints by path +func expandDelegatedRoutingEndpoints(cfg *config, autoConf *autoconf.Config) error { + // Validate and normalize each flag's URLs separately + normalizedProviders, err := validateEndpointURLs(cfg.contentEndpoints, autoconf.RoutingV1ProvidersPath, "--provider-endpoints", "SOMEGUY_PROVIDER_ENDPOINTS") + if err != nil { + return err + } + + normalizedPeers, err := validateEndpointURLs(cfg.peerEndpoints, autoconf.RoutingV1PeersPath, "--peer-endpoints", "SOMEGUY_PEER_ENDPOINTS") + if err != nil { + return err + } + + normalizedIPNS, err := validateEndpointURLs(cfg.ipnsEndpoints, autoconf.RoutingV1IPNSPath, "--ipns-endpoints", "SOMEGUY_IPNS_ENDPOINTS") + if err != nil { + return err + } + + if !cfg.autoConf.enabled { + // Check for "auto" placeholder when autoconf is disabled + if slices.Contains(normalizedProviders, autoconf.AutoPlaceholder) || + slices.Contains(normalizedPeers, autoconf.AutoPlaceholder) || + slices.Contains(normalizedIPNS, autoconf.AutoPlaceholder) { + return fmt.Errorf("'auto' placeholder found in endpoint option but autoconf is disabled. Set explicit endpoint option with SOMEGUY_PROVIDER_ENDPOINTS/SOMEGUY_PEER_ENDPOINTS/SOMEGUY_IPNS_ENDPOINTS or --provider-endpoints/--peer-endpoints/--ipns-endpoints, or re-enable autoconf") + } + // No autoconf, keep normalized endpoints as configured + cfg.contentEndpoints = deduplicateEndpoints(normalizedProviders) + cfg.peerEndpoints = deduplicateEndpoints(normalizedPeers) + cfg.ipnsEndpoints = deduplicateEndpoints(normalizedIPNS) + return nil + } + + nativeSystems := getNativeSystems(cfg.dhtType) + + // Expand each routing type separately to maintain category information + expandedProviders := autoconf.ExpandDelegatedEndpoints( + normalizedProviders, + autoConf, + nativeSystems, + autoconf.RoutingV1ProvidersPath, + ) + + expandedPeers := autoconf.ExpandDelegatedEndpoints( + normalizedPeers, + autoConf, + nativeSystems, + autoconf.RoutingV1PeersPath, + ) + + expandedIPNS := autoconf.ExpandDelegatedEndpoints( + normalizedIPNS, + autoConf, + nativeSystems, + autoconf.RoutingV1IPNSPath, + ) + + // Strip routing paths from expanded URLs to get base URLs + cfg.contentEndpoints = stripRoutingPaths(expandedProviders, autoconf.RoutingV1ProvidersPath) + cfg.peerEndpoints = stripRoutingPaths(expandedPeers, autoconf.RoutingV1PeersPath) + cfg.ipnsEndpoints = stripRoutingPaths(expandedIPNS, autoconf.RoutingV1IPNSPath) + + logger.Debugf("expanded endpoints - providers: %v, peers: %v, IPNS: %v", + cfg.contentEndpoints, cfg.peerEndpoints, cfg.ipnsEndpoints) + + return nil +} + +// stripRoutingPaths strips the routing path from URLs and deduplicates +// URLs without the expected path are kept as base URLs (from custom config) +// Handles trailing slashes by normalizing before comparison +func stripRoutingPaths(urls []string, expectedPath string) []string { + result := make([]string, 0, len(urls)) + for _, url := range urls { + // Trim trailing slash for comparison + normalized := strings.TrimSuffix(url, "/") + // Strip path from autoconf-expanded URL or keep normalized base URL. + result = append(result, strings.TrimSuffix(normalized, expectedPath)) + } + return deduplicateEndpoints(result) +} + +// deduplicateEndpoints removes duplicate endpoints from a list +func deduplicateEndpoints(endpoints []string) []string { + if len(endpoints) == 0 { + return endpoints + } + slices.Sort(endpoints) + return slices.Compact(endpoints) +} + +func stringsToPeerAddrInfos(addrs []string) []peer.AddrInfo { + addrInfos := make([]peer.AddrInfo, 0, len(addrs)) + + for _, s := range addrs { + ma, err := multiaddr.NewMultiaddr(s) + if err != nil { + logger.Error("bad multiaddr in bootstrapper autoconf data", "err", err) + continue + } + + info, err := peer.AddrInfoFromP2pAddr(ma) + if err != nil { + logger.Errorw("failed to convert bootstrapper address to peer addr info", "address", ma.String(), err, "err") + continue + } + addrInfos = append(addrInfos, *info) + } + + return addrInfos +} + +// createAutoConfClient creates an autoconf client with the given configuration +func createAutoConfClient(cfg autoConfConfig) (*autoconf.Client, error) { + if cfg.cacheDir == "" { + cfg.cacheDir = filepath.Join(".", ".autoconf-cache") + } + if cfg.refreshInterval == 0 { + cfg.refreshInterval = autoconf.DefaultRefreshInterval + } + if cfg.url == "" { + cfg.url = autoconf.MainnetAutoConfURL + } + + return autoconf.NewClient( + autoconf.WithCacheDir(cfg.cacheDir), + autoconf.WithUserAgent("someguy/"+version), + autoconf.WithCacheSize(autoconf.DefaultCacheSize), + autoconf.WithTimeout(autoconf.DefaultTimeout), + autoconf.WithURL(cfg.url), + autoconf.WithRefreshInterval(cfg.refreshInterval), + autoconf.WithFallback(autoconf.GetMainnetFallbackConfig), + ) +} + +// getNativeSystems returns the list of systems that should be used natively based on routing type +func getNativeSystems(routingType string) []string { + switch routingType { + case "dht", "accelerated", "standard", "auto": + return []string{autoconf.SystemAminoDHT} + case "disabled", "off", "none", "delegated", "custom": + return []string{} + default: + logger.Warnf("getNativeSystems: unknown routing type %q, assuming no native systems", routingType) + return []string{} + } +} diff --git a/autoconf_test.go b/autoconf_test.go new file mode 100644 index 0000000..b7557c9 --- /dev/null +++ b/autoconf_test.go @@ -0,0 +1,662 @@ +package main + +import ( + "testing" + + autoconf "github.com/ipfs/boxo/autoconf" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestGetNativeSystems verifies that routing types are correctly mapped to native systems +func TestGetNativeSystems(t *testing.T) { + tests := []struct { + name string + routingType string + expectedSystems []string + }{ + { + name: "DHT routing", + routingType: "dht", + expectedSystems: []string{autoconf.SystemAminoDHT}, + }, + { + name: "Accelerated routing", + routingType: "accelerated", + expectedSystems: []string{autoconf.SystemAminoDHT}, + }, + { + name: "Standard routing", + routingType: "standard", + expectedSystems: []string{autoconf.SystemAminoDHT}, + }, + { + name: "Auto routing", + routingType: "auto", + expectedSystems: []string{autoconf.SystemAminoDHT}, + }, + { + name: "Off routing", + routingType: "off", + expectedSystems: []string{}, + }, + { + name: "None routing", + routingType: "none", + expectedSystems: []string{}, + }, + { + name: "Unknown routing type", + routingType: "custom", + expectedSystems: []string{}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + systems := getNativeSystems(tt.routingType) + assert.Equal(t, tt.expectedSystems, systems) + }) + } +} + +// TestExpandDelegatedRoutingEndpoints verifies endpoint expansion and path categorization +func TestExpandDelegatedRoutingEndpoints(t *testing.T) { + t.Run("auto placeholder errors when autoconf disabled", func(t *testing.T) { + cfg := config{ + autoConf: autoConfConfig{ + enabled: false, + }, + contentEndpoints: []string{autoconf.AutoPlaceholder}, + } + err := expandDelegatedRoutingEndpoints(&cfg, nil) + require.Error(t, err, "should error when 'auto' is used with autoconf disabled") + assert.Contains(t, err.Error(), "'auto' placeholder found in endpoint option") + assert.Contains(t, err.Error(), "SOMEGUY_PROVIDER_ENDPOINTS") + }) + + t.Run("custom endpoints without paths preserved", func(t *testing.T) { + cfg := config{ + autoConf: autoConfConfig{ + enabled: false, + }, + contentEndpoints: []string{"https://example.com"}, + } + err := expandDelegatedRoutingEndpoints(&cfg, nil) + require.NoError(t, err) + // Without autoconf, endpoints pass through unchanged + assert.Equal(t, []string{"https://example.com"}, cfg.contentEndpoints) + }) + + t.Run("separate flags with matching paths", func(t *testing.T) { + cfg := config{ + autoConf: autoConfConfig{ + enabled: true, + }, + dhtType: "none", // no native systems to exclude + contentEndpoints: []string{ + "https://provider-only.example.com/routing/v1/providers", + "https://all-in-one.example.com/routing/v1/providers", + }, + peerEndpoints: []string{ + "https://peer-only.example.com/routing/v1/peers", + "https://all-in-one.example.com/routing/v1/peers", + }, + ipnsEndpoints: []string{ + "https://ipns-only.example.com/routing/v1/ipns", + "https://all-in-one.example.com/routing/v1/ipns", + }, + } + + mockAutoConf := &autoconf.Config{} + err := expandDelegatedRoutingEndpoints(&cfg, mockAutoConf) + require.NoError(t, err) + + // Verify paths stripped to base URLs + assert.ElementsMatch(t, []string{ + "https://all-in-one.example.com", + "https://provider-only.example.com", + }, cfg.contentEndpoints, "provider endpoints should have paths stripped") + + assert.ElementsMatch(t, []string{ + "https://all-in-one.example.com", + "https://peer-only.example.com", + }, cfg.peerEndpoints, "peer endpoints should have paths stripped") + + assert.ElementsMatch(t, []string{ + "https://all-in-one.example.com", + "https://ipns-only.example.com", + }, cfg.ipnsEndpoints, "IPNS endpoints should have paths stripped") + }) + + t.Run("base URLs and unknown paths accepted", func(t *testing.T) { + cfg := config{ + autoConf: autoConfConfig{ + enabled: true, + }, + dhtType: "none", + contentEndpoints: []string{ + "https://example.com/routing/v1/providers", + "https://example.com/custom/path", + "https://example.com", + }, + } + + mockAutoConf := &autoconf.Config{} + err := expandDelegatedRoutingEndpoints(&cfg, mockAutoConf) + require.NoError(t, err) + + // All URLs accepted: known path stripped, unknown path and base URL kept + assert.ElementsMatch(t, []string{ + "https://example.com", + "https://example.com/custom/path", + }, cfg.contentEndpoints) + assert.Empty(t, cfg.peerEndpoints) + assert.Empty(t, cfg.ipnsEndpoints) + }) + + t.Run("deduplication works", func(t *testing.T) { + cfg := config{ + autoConf: autoConfConfig{ + enabled: true, + }, + dhtType: "none", + contentEndpoints: []string{ + "https://example.com/routing/v1/providers", + "https://example.com/routing/v1/providers", // duplicate + "https://example.com", // duplicate after path stripping + }, + peerEndpoints: []string{ + "https://example.com/routing/v1/peers", + "https://example.com", // duplicate after path stripping + }, + } + + mockAutoConf := &autoconf.Config{} + err := expandDelegatedRoutingEndpoints(&cfg, mockAutoConf) + require.NoError(t, err) + + // Duplicates removed, paths stripped + assert.Equal(t, []string{"https://example.com"}, cfg.contentEndpoints) + assert.Equal(t, []string{"https://example.com"}, cfg.peerEndpoints) + }) + + t.Run("mismatched path errors", func(t *testing.T) { + cfg := config{ + autoConf: autoConfConfig{ + enabled: true, + }, + dhtType: "none", + contentEndpoints: []string{ + "https://example.com/routing/v1/peers", // wrong path for provider endpoints + }, + } + + mockAutoConf := &autoconf.Config{} + err := expandDelegatedRoutingEndpoints(&cfg, mockAutoConf) + require.Error(t, err) + assert.Contains(t, err.Error(), "/routing/v1/peers") + assert.Contains(t, err.Error(), "--provider-endpoints") + }) + + t.Run("mixing auto with custom URLs", func(t *testing.T) { + cfg := config{ + autoConf: autoConfConfig{ + enabled: true, + }, + dhtType: "none", + contentEndpoints: []string{ + autoconf.AutoPlaceholder, + "https://custom-provider.example.com", + }, + peerEndpoints: []string{ + "https://custom-peer.example.com", + }, + ipnsEndpoints: []string{ + autoconf.AutoPlaceholder, + }, + } + + // Empty autoConf means "auto" expands to nothing, custom URLs preserved + mockAutoConf := &autoconf.Config{} + err := expandDelegatedRoutingEndpoints(&cfg, mockAutoConf) + require.NoError(t, err) + + // Custom URLs should be preserved + assert.Equal(t, []string{"https://custom-provider.example.com"}, cfg.contentEndpoints) + assert.Equal(t, []string{"https://custom-peer.example.com"}, cfg.peerEndpoints) + assert.Empty(t, cfg.ipnsEndpoints) // auto expanded to nothing + }) + + t.Run("multiple custom URLs in one flag", func(t *testing.T) { + cfg := config{ + autoConf: autoConfConfig{ + enabled: true, + }, + dhtType: "none", + contentEndpoints: []string{ + "https://a.example.com", + "https://b.example.com/routing/v1/providers", + "https://c.example.com", + }, + peerEndpoints: []string{ + "https://peer1.example.com/routing/v1/peers", + "https://peer2.example.com", + }, + } + + mockAutoConf := &autoconf.Config{} + err := expandDelegatedRoutingEndpoints(&cfg, mockAutoConf) + require.NoError(t, err) + + // All URLs should be processed (paths stripped where present) + assert.ElementsMatch(t, []string{ + "https://a.example.com", + "https://b.example.com", + "https://c.example.com", + }, cfg.contentEndpoints) + + assert.ElementsMatch(t, []string{ + "https://peer1.example.com", + "https://peer2.example.com", + }, cfg.peerEndpoints) + }) + + t.Run("trailing slashes handled consistently", func(t *testing.T) { + cfg := config{ + autoConf: autoConfConfig{ + enabled: true, + }, + dhtType: "none", + contentEndpoints: []string{ + "https://example.com/routing/v1/providers/", // with trailing slash + "https://another.com/routing/v1/providers", // without trailing slash + "https://base.com/", // base URL with trailing slash + "https://clean.com", // base URL without trailing slash + }, + } + + mockAutoConf := &autoconf.Config{} + err := expandDelegatedRoutingEndpoints(&cfg, mockAutoConf) + require.NoError(t, err) + + // Verify trailing slashes are normalized (removed) consistently + // Paths should be stripped and trailing slashes removed + assert.ElementsMatch(t, []string{ + "https://another.com", + "https://base.com", // trailing slash removed + "https://clean.com", + "https://example.com", // path stripped and trailing slash removed + }, cfg.contentEndpoints) + }) +} + +// TestNormalizeEndpointURL verifies URL normalization and path handling +func TestNormalizeEndpointURL(t *testing.T) { + tests := []struct { + name string + url string + expectedPath string + flagName string + want string + wantErr bool + errContains string + }{ + { + name: "auto placeholder passes through", + url: autoconf.AutoPlaceholder, + expectedPath: autoconf.RoutingV1ProvidersPath, + flagName: "--provider-endpoints", + want: autoconf.AutoPlaceholder, + wantErr: false, + }, + { + name: "URL with expected path stripped", + url: "https://example.com/routing/v1/providers", + expectedPath: autoconf.RoutingV1ProvidersPath, + flagName: "--provider-endpoints", + want: "https://example.com", + wantErr: false, + }, + { + name: "URL with trailing slash and expected path passes through", + url: "https://example.com/routing/v1/providers/", + expectedPath: autoconf.RoutingV1ProvidersPath, + flagName: "--provider-endpoints", + want: "https://example.com/routing/v1/providers/", + wantErr: false, + }, + { + name: "base URL without path", + url: "https://example.com", + expectedPath: autoconf.RoutingV1ProvidersPath, + flagName: "--provider-endpoints", + want: "https://example.com", + wantErr: false, + }, + { + name: "base URL with trailing slash", + url: "https://example.com/", + expectedPath: autoconf.RoutingV1ProvidersPath, + flagName: "--provider-endpoints", + want: "https://example.com/", + wantErr: false, + }, + { + name: "peers path in provider flag errors with mismatch message", + url: "https://example.com/routing/v1/peers", + expectedPath: autoconf.RoutingV1ProvidersPath, + flagName: "--provider-endpoints", + want: "", + wantErr: true, + errContains: "has path \"/routing/v1/peers\" which doesn't match --provider-endpoints (expected \"/routing/v1/providers\"", + }, + { + name: "IPNS path in provider flag errors with mismatch message", + url: "https://example.com/routing/v1/ipns", + expectedPath: autoconf.RoutingV1ProvidersPath, + flagName: "--provider-endpoints", + want: "", + wantErr: true, + errContains: "has path \"/routing/v1/ipns\" which doesn't match --provider-endpoints (expected \"/routing/v1/providers\"", + }, + { + name: "provider path in peer flag errors with mismatch message", + url: "https://example.com/routing/v1/providers", + expectedPath: autoconf.RoutingV1PeersPath, + flagName: "--peer-endpoints", + want: "", + wantErr: true, + errContains: "has path \"/routing/v1/providers\" which doesn't match --peer-endpoints (expected \"/routing/v1/peers\"", + }, + { + name: "provider path in IPNS flag errors with mismatch message", + url: "https://example.com/routing/v1/providers", + expectedPath: autoconf.RoutingV1IPNSPath, + flagName: "--ipns-endpoints", + want: "", + wantErr: true, + errContains: "has path \"/routing/v1/providers\" which doesn't match --ipns-endpoints (expected \"/routing/v1/ipns\"", + }, + { + name: "peer path in IPNS flag errors with mismatch message", + url: "https://example.com/routing/v1/peers", + expectedPath: autoconf.RoutingV1IPNSPath, + flagName: "--ipns-endpoints", + want: "", + wantErr: true, + errContains: "has path \"/routing/v1/peers\" which doesn't match --ipns-endpoints (expected \"/routing/v1/ipns\"", + }, + { + name: "IPNS path in peer flag errors with mismatch message", + url: "https://example.com/routing/v1/ipns", + expectedPath: autoconf.RoutingV1PeersPath, + flagName: "--peer-endpoints", + want: "", + wantErr: true, + errContains: "has path \"/routing/v1/ipns\" which doesn't match --peer-endpoints (expected \"/routing/v1/peers\"", + }, + { + name: "custom path accepted", + url: "https://example.com/custom/path", + expectedPath: autoconf.RoutingV1ProvidersPath, + flagName: "--provider-endpoints", + want: "https://example.com/custom/path", + wantErr: false, + }, + { + name: "empty URL passes through", + url: "", + expectedPath: autoconf.RoutingV1ProvidersPath, + flagName: "--provider-endpoints", + want: "", + wantErr: false, + }, + { + name: "peer path works for peer endpoints", + url: "https://example.com/routing/v1/peers", + expectedPath: autoconf.RoutingV1PeersPath, + flagName: "--peer-endpoints", + want: "https://example.com", + wantErr: false, + }, + { + name: "IPNS path works for IPNS endpoints", + url: "https://example.com/routing/v1/ipns", + expectedPath: autoconf.RoutingV1IPNSPath, + flagName: "--ipns-endpoints", + want: "https://example.com", + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := normalizeEndpointURL(tt.url, tt.expectedPath, tt.flagName) + if tt.wantErr { + require.Error(t, err) + if tt.errContains != "" { + assert.Contains(t, err.Error(), tt.errContains) + } + } else { + require.NoError(t, err) + assert.Equal(t, tt.want, got) + } + }) + } +} + +// TestValidateEndpointURLs verifies batch URL validation with proper error messages +func TestValidateEndpointURLs(t *testing.T) { + tests := []struct { + name string + urls []string + expectedPath string + flagName string + envVar string + want []string + wantErr bool + errContains []string + }{ + { + name: "valid URLs pass validation", + urls: []string{"https://a.com", "https://b.com/routing/v1/providers"}, + expectedPath: autoconf.RoutingV1ProvidersPath, + flagName: "--provider-endpoints", + envVar: "SOMEGUY_PROVIDER_ENDPOINTS", + want: []string{"https://a.com", "https://b.com"}, + wantErr: false, + }, + { + name: "auto placeholder passes validation", + urls: []string{autoconf.AutoPlaceholder}, + expectedPath: autoconf.RoutingV1ProvidersPath, + flagName: "--provider-endpoints", + envVar: "SOMEGUY_PROVIDER_ENDPOINTS", + want: []string{autoconf.AutoPlaceholder}, + wantErr: false, + }, + { + name: "mixed auto and custom URLs", + urls: []string{autoconf.AutoPlaceholder, "https://custom.com"}, + expectedPath: autoconf.RoutingV1ProvidersPath, + flagName: "--provider-endpoints", + envVar: "SOMEGUY_PROVIDER_ENDPOINTS", + want: []string{autoconf.AutoPlaceholder, "https://custom.com"}, + wantErr: false, + }, + { + name: "mismatched path error includes flag and env var", + urls: []string{"https://example.com/routing/v1/peers"}, + expectedPath: autoconf.RoutingV1ProvidersPath, + flagName: "--provider-endpoints", + envVar: "SOMEGUY_PROVIDER_ENDPOINTS", + want: nil, + wantErr: true, + errContains: []string{"SOMEGUY_PROVIDER_ENDPOINTS", "--provider-endpoints", "/routing/v1/peers"}, + }, + { + name: "empty URLs list", + urls: []string{}, + expectedPath: autoconf.RoutingV1ProvidersPath, + flagName: "--provider-endpoints", + envVar: "SOMEGUY_PROVIDER_ENDPOINTS", + want: []string{}, + wantErr: false, + }, + { + name: "URLs with trailing slashes pass through", + urls: []string{"https://a.com/routing/v1/providers/", "https://b.com/"}, + expectedPath: autoconf.RoutingV1ProvidersPath, + flagName: "--provider-endpoints", + envVar: "SOMEGUY_PROVIDER_ENDPOINTS", + want: []string{"https://a.com/routing/v1/providers/", "https://b.com/"}, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := validateEndpointURLs(tt.urls, tt.expectedPath, tt.flagName, tt.envVar) + if tt.wantErr { + require.Error(t, err) + for _, contains := range tt.errContains { + assert.Contains(t, err.Error(), contains) + } + } else { + require.NoError(t, err) + assert.Equal(t, tt.want, got) + } + }) + } +} + +// TestStripRoutingPaths verifies path stripping and deduplication logic +func TestStripRoutingPaths(t *testing.T) { + tests := []struct { + name string + urls []string + expectedPath string + want []string + }{ + { + name: "strips expected paths", + urls: []string{"https://a.com/routing/v1/providers", "https://b.com/routing/v1/providers"}, + expectedPath: autoconf.RoutingV1ProvidersPath, + want: []string{"https://a.com", "https://b.com"}, + }, + { + name: "normalizes trailing slashes", + urls: []string{"https://a.com/routing/v1/providers/", "https://b.com/routing/v1/providers"}, + expectedPath: autoconf.RoutingV1ProvidersPath, + want: []string{"https://a.com", "https://b.com"}, + }, + { + name: "preserves base URLs without paths", + urls: []string{"https://a.com", "https://b.com"}, + expectedPath: autoconf.RoutingV1ProvidersPath, + want: []string{"https://a.com", "https://b.com"}, + }, + { + name: "preserves custom paths", + urls: []string{"https://a.com/custom/path", "https://b.com"}, + expectedPath: autoconf.RoutingV1ProvidersPath, + want: []string{"https://a.com/custom/path", "https://b.com"}, + }, + { + name: "deduplicates after stripping", + urls: []string{"https://a.com/routing/v1/providers", "https://a.com", "https://a.com/routing/v1/providers"}, + expectedPath: autoconf.RoutingV1ProvidersPath, + want: []string{"https://a.com"}, + }, + { + name: "mixed URLs with and without paths", + urls: []string{"https://a.com/routing/v1/providers", "https://b.com", "https://c.com/custom"}, + expectedPath: autoconf.RoutingV1ProvidersPath, + want: []string{"https://a.com", "https://b.com", "https://c.com/custom"}, + }, + { + name: "removes trailing slashes from base URLs", + urls: []string{"https://a.com/", "https://b.com"}, + expectedPath: autoconf.RoutingV1ProvidersPath, + want: []string{"https://a.com", "https://b.com"}, + }, + { + name: "deduplicates URLs with and without trailing slashes", + urls: []string{"https://a.com/", "https://a.com", "https://b.com/"}, + expectedPath: autoconf.RoutingV1ProvidersPath, + want: []string{"https://a.com", "https://b.com"}, + }, + { + name: "empty URLs list", + urls: []string{}, + expectedPath: autoconf.RoutingV1ProvidersPath, + want: []string{}, + }, + { + name: "single URL with path", + urls: []string{"https://example.com/routing/v1/providers"}, + expectedPath: autoconf.RoutingV1ProvidersPath, + want: []string{"https://example.com"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := stripRoutingPaths(tt.urls, tt.expectedPath) + assert.Equal(t, tt.want, got) + }) + } +} + +// TestDeduplicateEndpoints verifies deduplication and sorting behavior +func TestDeduplicateEndpoints(t *testing.T) { + tests := []struct { + name string + input []string + want []string + }{ + { + name: "empty slice", + input: []string{}, + want: []string{}, + }, + { + name: "no duplicates", + input: []string{"https://a.com", "https://b.com"}, + want: []string{"https://a.com", "https://b.com"}, + }, + { + name: "with duplicates", + input: []string{"https://a.com", "https://b.com", "https://a.com"}, + want: []string{"https://a.com", "https://b.com"}, + }, + { + name: "all duplicates", + input: []string{"https://a.com", "https://a.com", "https://a.com"}, + want: []string{"https://a.com"}, + }, + { + name: "unsorted input gets sorted", + input: []string{"https://c.com", "https://a.com", "https://b.com"}, + want: []string{"https://a.com", "https://b.com", "https://c.com"}, + }, + { + name: "duplicates with unsorted input", + input: []string{"https://c.com", "https://a.com", "https://b.com", "https://a.com", "https://c.com"}, + want: []string{"https://a.com", "https://b.com", "https://c.com"}, + }, + { + name: "single element", + input: []string{"https://example.com"}, + want: []string{"https://example.com"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := deduplicateEndpoints(tt.input) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/client.go b/client.go index 89dcaf3..b83db7f 100644 --- a/client.go +++ b/client.go @@ -9,14 +9,13 @@ import ( "time" "github.com/ipfs/boxo/ipns" - "github.com/ipfs/boxo/routing/http/client" "github.com/ipfs/boxo/routing/http/types" "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p/core/peer" ) func findProviders(ctx context.Context, key cid.Cid, endpoint string, prettyOutput bool) error { - drc, err := client.New(endpoint) + drc, err := newDelegatedRoutingClient(endpoint) if err != nil { return err } @@ -77,7 +76,7 @@ func findProviders(ctx context.Context, key cid.Cid, endpoint string, prettyOutp } func findPeers(ctx context.Context, pid peer.ID, endpoint string, prettyOutput bool) error { - drc, err := client.New(endpoint) + drc, err := newDelegatedRoutingClient(endpoint) if err != nil { return err } @@ -118,7 +117,7 @@ func findPeers(ctx context.Context, pid peer.ID, endpoint string, prettyOutput b } func getIPNS(ctx context.Context, name ipns.Name, endpoint string, prettyOutput bool) error { - drc, err := client.New(endpoint) + drc, err := newDelegatedRoutingClient(endpoint) if err != nil { return err } @@ -172,7 +171,7 @@ func getIPNS(ctx context.Context, name ipns.Name, endpoint string, prettyOutput } func putIPNS(ctx context.Context, name ipns.Name, record []byte, endpoint string) error { - drc, err := client.New(endpoint) + drc, err := newDelegatedRoutingClient(endpoint) if err != nil { return err } diff --git a/docs/environment-variables.md b/docs/environment-variables.md index 823a7c1..a61394b 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -11,6 +11,9 @@ - [`SOMEGUY_PROVIDER_ENDPOINTS`](#someguy_provider_endpoints) - [`SOMEGUY_PEER_ENDPOINTS`](#someguy_peer_endpoints) - [`SOMEGUY_IPNS_ENDPOINTS`](#someguy_ipns_endpoints) + - [`SOMEGUY_AUTOCONF`](#someguy_autoconf) + - [`SOMEGUY_AUTOCONF_URL`](#someguy_autoconf_url) + - [`SOMEGUY_AUTOCONF_REFRESH`](#someguy_autoconf_refresh) - [`SOMEGUY_HTTP_BLOCK_PROVIDER_ENDPOINTS`](#someguy_http_block_provider_endpoints) - [`SOMEGUY_HTTP_BLOCK_PROVIDER_PEERIDS`](#someguy_http_block_provider_peerids) - [`SOMEGUY_LIBP2P_LISTEN_ADDRS`](#someguy_libp2p_listen_addrs) @@ -62,21 +65,51 @@ Default: `true` ### `SOMEGUY_PROVIDER_ENDPOINTS` -Comma-separated list of other Delegated Routing V1 endpoints to proxy provider requests to. +Comma-separated list of [Delegated Routing V1](https://specs.ipfs.tech/routing/http-routing-v1/) endpoints for provider lookups. -Default: `https://cid.contact` +Supports two URL formats: +- Base URL without path: `https://example.com` +- Full URL with path: `https://example.com/routing/v1/providers` + +When using the `auto` placeholder (default), endpoints are automatically configured from the network configuration at [`SOMEGUY_AUTOCONF_URL`](#someguy_autoconf_url). + +Default: `auto` ### `SOMEGUY_PEER_ENDPOINTS` -Comma-separated list of other Delegated Routing V1 endpoints to proxy peer requests to. +Comma-separated list of Delegated Routing V1 endpoints for peer routing. -Default: none +URL formats: same as [`SOMEGUY_PROVIDER_ENDPOINTS`](#someguy_provider_endpoints) (use `/routing/v1/peers` path). + +Default: `auto` ### `SOMEGUY_IPNS_ENDPOINTS` -Comma-separated list of other Delegated Routing V1 endpoints to proxy IPNS requests to. +Comma-separated list of Delegated Routing V1 endpoints for IPNS records. -Default: none +URL formats: same as [`SOMEGUY_PROVIDER_ENDPOINTS`](#someguy_provider_endpoints) (use `/routing/v1/ipns` path). + +Default: `auto` + +### `SOMEGUY_AUTOCONF` + +Enable or disable automatic configuration (autoconf) of delegated routing endpoints and bootstrap peers. + +When enabled, the `auto` placeholder in endpoint configuration is replaced with network-recommended values fetched from the autoconf URL. + +Default: `true` + +### `SOMEGUY_AUTOCONF_URL` + +URL to fetch autoconf data from. Defaults to the service that provides configuration for [IPFS Mainnet](https://docs.ipfs.tech/concepts/glossary/#mainnet). + +Default: `https://conf.ipfs-mainnet.org/autoconf.json` + +### `SOMEGUY_AUTOCONF_REFRESH` + +How often to refresh the autoconf data. The configuration is cached and updated at this interval. + +Default: `24h` ### `SOMEGUY_HTTP_BLOCK_PROVIDER_ENDPOINTS` diff --git a/main.go b/main.go index dbe244e..3c8f7d4 100644 --- a/main.go +++ b/main.go @@ -6,9 +6,11 @@ import ( "fmt" "log" "os" + "path/filepath" "strings" "time" + "github.com/ipfs/boxo/autoconf" "github.com/ipfs/boxo/ipns" "github.com/ipfs/go-cid" "github.com/libp2p/go-libp2p/core/peer" @@ -17,8 +19,6 @@ import ( "github.com/urfave/cli/v2" ) -const cidContactEndpoint = "https://cid.contact" - func main() { app := &cli.App{ Name: name, @@ -61,7 +61,7 @@ func main() { }, &cli.StringSliceFlag{ Name: "provider-endpoints", - Value: cli.NewStringSlice(cidContactEndpoint), + Value: cli.NewStringSlice(autoconf.AutoPlaceholder), EnvVars: []string{"SOMEGUY_PROVIDER_ENDPOINTS"}, Usage: "other Delegated Routing V1 endpoints to proxy provider requests to", }, @@ -79,13 +79,13 @@ func main() { }, &cli.StringSliceFlag{ Name: "peer-endpoints", - Value: cli.NewStringSlice(), + Value: cli.NewStringSlice(autoconf.AutoPlaceholder), EnvVars: []string{"SOMEGUY_PEER_ENDPOINTS"}, Usage: "other Delegated Routing V1 endpoints to proxy peer requests to", }, &cli.StringSliceFlag{ Name: "ipns-endpoints", - Value: cli.NewStringSlice(), + Value: cli.NewStringSlice(autoconf.AutoPlaceholder), EnvVars: []string{"SOMEGUY_IPNS_ENDPOINTS"}, Usage: "other Delegated Routing V1 endpoints to proxy IPNS requests to", }, @@ -145,6 +145,30 @@ func main() { EnvVars: []string{"SOMEGUY_SAMPLING_FRACTION"}, Usage: "Rate at which to sample gateway requests. Does not include requests with traceheaders which will always sample", }, + &cli.StringFlag{ + Name: "datadir", + Value: "", + EnvVars: []string{"SOMEGUY_DATADIR"}, + Usage: "Directory for persistent data (autoconf cache)", + }, + &cli.BoolFlag{ + Name: "autoconf", + Value: true, + EnvVars: []string{"SOMEGUY_AUTOCONF"}, + Usage: "Enable autoconf for bootstrap, DNS resolvers, and HTTP routers", + }, + &cli.StringFlag{ + Name: "autoconf-url", + Value: "https://conf.ipfs-mainnet.org/autoconf.json", + EnvVars: []string{"SOMEGUY_AUTOCONF_URL"}, + Usage: "URL to fetch autoconf data from", + }, + &cli.DurationFlag{ + Name: "autoconf-refresh", + Value: 24 * time.Hour, + EnvVars: []string{"SOMEGUY_AUTOCONF_REFRESH"}, + Usage: "How often to refresh autoconf data", + }, }, Action: func(ctx *cli.Context) error { cfg := &config{ @@ -169,6 +193,13 @@ func main() { tracingAuth: ctx.String("tracing-auth"), samplingFraction: ctx.Float64("sampling-fraction"), + + autoConf: autoConfConfig{ + enabled: ctx.Bool("autoconf"), + url: ctx.String("autoconf-url"), + refreshInterval: ctx.Duration("autoconf-refresh"), + cacheDir: filepath.Join(ctx.String("datadir"), ".autoconf-cache"), + }, } fmt.Printf("Starting %s %s\n", name, version) @@ -209,7 +240,7 @@ func main() { Flags: []cli.Flag{ &cli.StringFlag{ Name: "endpoint", - Value: cidContactEndpoint, + Value: autoconf.AutoPlaceholder, Usage: "the Delegated Routing V1 endpoint to ask", }, &cli.BoolFlag{ @@ -217,6 +248,30 @@ func main() { Value: false, Usage: "output data in a prettier format that may convey less information", }, + &cli.StringFlag{ + Name: "datadir", + Value: "", + EnvVars: []string{"SOMEGUY_DATADIR"}, + Usage: "Directory for persistent data (autoconf cache)", + }, + &cli.BoolFlag{ + Name: "autoconf", + Value: true, + EnvVars: []string{"SOMEGUY_AUTOCONF"}, + Usage: "Enable autoconf for bootstrap, DNS resolvers, and HTTP routers", + }, + &cli.StringFlag{ + Name: "autoconf-url", + Value: "https://conf.ipfs-mainnet.org/autoconf.json", + EnvVars: []string{"SOMEGUY_AUTOCONF_URL"}, + Usage: "URL to fetch autoconf data from", + }, + &cli.DurationFlag{ + Name: "autoconf-refresh", + Value: 24 * time.Hour, + EnvVars: []string{"SOMEGUY_AUTOCONF_REFRESH"}, + Usage: "How often to refresh autoconf data", + }, }, Subcommands: []*cli.Command{ { @@ -232,7 +287,34 @@ func main() { if err != nil { return err } - return findProviders(ctx.Context, c, ctx.String("endpoint"), ctx.Bool("pretty")) + + cfg := &config{ + dhtType: "none", + contentEndpoints: []string{ctx.String("endpoint")}, + autoConf: autoConfConfig{ + enabled: ctx.Bool("autoconf"), + url: ctx.String("autoconf-url"), + refreshInterval: ctx.Duration("autoconf-refresh"), + cacheDir: filepath.Join(ctx.String("datadir"), ".autoconf-cache"), + }, + } + + autoConf, err := startAutoConf(ctx.Context, cfg) + if err != nil { + logger.Error(err.Error()) + } + + if err = expandDelegatedRoutingEndpoints(cfg, autoConf); err != nil { + return err + } + if len(cfg.contentEndpoints) == 0 { + return errors.New("no delegated routing endpoint configured, use --endpoint to specify") + } + + endPoint := cfg.contentEndpoints[0] + logger.Debugf("delegated routing endpoint: %s", endPoint) + + return findProviders(ctx.Context, c, endPoint, ctx.Bool("pretty")) }, }, { @@ -248,7 +330,33 @@ func main() { if err != nil { return err } - return findPeers(ctx.Context, pid, ctx.String("endpoint"), ctx.Bool("pretty")) + cfg := &config{ + dhtType: "none", + contentEndpoints: []string{ctx.String("endpoint")}, + autoConf: autoConfConfig{ + enabled: ctx.Bool("autoconf"), + url: ctx.String("autoconf-url"), + refreshInterval: ctx.Duration("autoconf-refresh"), + cacheDir: filepath.Join(ctx.String("datadir"), ".autoconf-cache"), + }, + } + + autoConf, err := startAutoConf(ctx.Context, cfg) + if err != nil { + logger.Error(err.Error()) + } + + if err = expandDelegatedRoutingEndpoints(cfg, autoConf); err != nil { + return err + } + if len(cfg.contentEndpoints) == 0 { + return errors.New("no delegated routing endpoint configured, use --endpoint to specify") + } + + endPoint := cfg.contentEndpoints[0] + logger.Debugf("delegated routing endpoint: %s", endPoint) + + return findPeers(ctx.Context, pid, endPoint, ctx.Bool("pretty")) }, }, { diff --git a/server.go b/server.go index 4a92b68..330cc18 100644 --- a/server.go +++ b/server.go @@ -18,7 +18,6 @@ import ( "github.com/CAFxX/httpcompression" sddaemon "github.com/coreos/go-systemd/v22/daemon" "github.com/felixge/httpsnoop" - drclient "github.com/ipfs/boxo/routing/http/client" "github.com/ipfs/boxo/routing/http/server" logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p" @@ -97,6 +96,8 @@ type config struct { tracingAuth string samplingFraction float64 + + autoConf autoConfConfig } func start(ctx context.Context, cfg *config) error { @@ -105,17 +106,40 @@ func start(ctx context.Context, cfg *config) error { return err } + autoConf, err := startAutoConf(ctx, cfg) + if err != nil { + logger.Error(err.Error()) + } + + bootstrapAddrInfos := getBootstrapPeerAddrInfos(cfg, autoConf) + + // Expand delegated routing endpoints and categorize by path + if err = expandDelegatedRoutingEndpoints(cfg, autoConf); err != nil { + return err + } + + // Print delegated routing endpoints + if len(cfg.contentEndpoints) > 0 { + fmt.Printf("Delegated routing endpoints for /routing/v1/providers: %v\n", cfg.contentEndpoints) + } + if len(cfg.peerEndpoints) > 0 { + fmt.Printf("Delegated routing endpoints for /routing/v1/peers: %v\n", cfg.peerEndpoints) + } + if len(cfg.ipnsEndpoints) > 0 { + fmt.Printf("Delegated routing endpoints for /routing/v1/ipns: %v\n", cfg.ipnsEndpoints) + } + fmt.Printf("Someguy libp2p host listening on %v\n", h.Addrs()) var dhtRouting routing.Routing switch cfg.dhtType { case "accelerated": - wrappedDHT, err := newBundledDHT(ctx, h) + wrappedDHT, err := newBundledDHT(ctx, h, bootstrapAddrInfos) if err != nil { return err } dhtRouting = wrappedDHT case "standard": - standardDHT, err := dht.New(ctx, h, dht.Mode(dht.ModeClient), dht.BootstrapPeers(dht.GetDefaultBootstrapPeerAddrInfos()...)) + standardDHT, err := dht.New(ctx, h, dht.Mode(dht.ModeClient), dht.BootstrapPeers(bootstrapAddrInfos...)) if err != nil { return err } @@ -163,20 +187,16 @@ func start(ctx context.Context, cfg *config) error { } } - crRouters, err := getCombinedRouting(cfg.contentEndpoints, dhtRouting, cachedAddrBook, blockProviderRouters) + // Create deduplicated HTTP routers - one client per unique base URL + providerHTTPRouters, peerHTTPRouters, ipnsHTTPRouters, err := createDelegatedHTTPRouters(cfg) if err != nil { return err } - prRouters, err := getCombinedRouting(cfg.peerEndpoints, dhtRouting, cachedAddrBook, nil) - if err != nil { - return err - } - - ipnsRouters, err := getCombinedRouting(cfg.ipnsEndpoints, dhtRouting, cachedAddrBook, nil) - if err != nil { - return err - } + // Combine HTTP routers with DHT and additional routers + crRouters := combineRouters(dhtRouting, cachedAddrBook, providerHTTPRouters, blockProviderRouters) + prRouters := combineRouters(dhtRouting, cachedAddrBook, peerHTTPRouters, nil) + ipnsRouters := combineRouters(dhtRouting, cachedAddrBook, ipnsHTTPRouters, nil) _, port, err := net.SplitHostPort(cfg.listenAddress) if err != nil { @@ -303,7 +323,9 @@ func newHost(cfg *config) (host.Host, error) { return h, nil } -func getCombinedRouting(endpoints []string, dht routing.Routing, cachedAddrBook *cachedAddrBook, additionalRouters []router) (router, error) { +// combineRouters combines delegated HTTP routers with DHT and additional routers. +// It no longer creates HTTP clients (that's done in createDelegatedHTTPRouters). +func combineRouters(dht routing.Routing, cachedAddrBook *cachedAddrBook, delegatedRouters, additionalRouters []router) router { var dhtRouter router if cachedAddrBook != nil { @@ -313,31 +335,11 @@ func getCombinedRouting(endpoints []string, dht routing.Routing, cachedAddrBook dhtRouter = sanitizeRouter{libp2pRouter{routing: dht}} } - if len(endpoints) == 0 && len(additionalRouters) == 0 { + if len(delegatedRouters) == 0 && len(additionalRouters) == 0 { if dhtRouter == nil { - return composableRouter{}, nil + return composableRouter{} } - return dhtRouter, nil - } - - var delegatedRouters []router - - for _, endpoint := range endpoints { - drclient, err := drclient.New(endpoint, - drclient.WithUserAgent("someguy/"+buildVersion()), - // override default filters, we want all results from remote endpoint, then someguy's user can use IPIP-484 to narrow them down - drclient.WithProtocolFilter([]string{}), - drclient.WithDisabledLocalFiltering(true), - ) - if err != nil { - return nil, err - } - delegatedRouters = append(delegatedRouters, clientRouter{Client: drclient}) - } - - // setup delegated routing client metrics - if err := view.Register(drclient.OpenCensusViews...); err != nil { - return nil, fmt.Errorf("registering HTTP delegated routing views: %w", err) + return dhtRouter } var routers []router @@ -349,7 +351,7 @@ func getCombinedRouting(endpoints []string, dht routing.Routing, cachedAddrBook return parallelRouter{ routers: routers, - }, nil + } } func withTracingAndDebug(next http.Handler, authToken string) http.Handler { diff --git a/server_delegated_routing.go b/server_delegated_routing.go new file mode 100644 index 0000000..d30b631 --- /dev/null +++ b/server_delegated_routing.go @@ -0,0 +1,154 @@ +// server_delegated_routing.go implements HTTP delegated routing for the server. +// +// This file contains code for creating and managing HTTP clients that talk to +// remote delegated routing endpoints (e.g., cid.contact, delegated-ipfs.dev). +// The server uses these HTTP clients to perform content, peer, and IPNS lookups +// when delegated routing is enabled. +// +// Key components: +// - newDelegatedRoutingClient: creates HTTP client with consistent options +// - collectEndpoints: deduplicates URLs and aggregates capabilities +// - createDelegatedHTTPRouters: creates one client per unique base URL +package main + +import ( + "context" + "fmt" + + drclient "github.com/ipfs/boxo/routing/http/client" + "github.com/ipfs/boxo/routing/http/types" + "github.com/ipfs/boxo/routing/http/types/iter" + "github.com/ipfs/go-cid" + "github.com/libp2p/go-libp2p/core/peer" + "go.opencensus.io/stats/view" +) + +// clientRouter wraps an HTTP delegated routing client to implement the router interface. +// Only FindProviders and FindPeers are explicitly implemented to adapt the signature +// (our interface includes a limit parameter). The IPNS methods (GetIPNS/PutIPNS) are +// inherited from the embedded drclient.Client as their signatures already match. +var _ router = clientRouter{} + +type clientRouter struct { + *drclient.Client +} + +func (d clientRouter) FindProviders(ctx context.Context, cid cid.Cid, limit int) (iter.ResultIter[types.Record], error) { + return d.Client.FindProviders(ctx, cid) +} + +func (d clientRouter) FindPeers(ctx context.Context, pid peer.ID, limit int) (iter.ResultIter[*types.PeerRecord], error) { + return d.Client.FindPeers(ctx, pid) +} + +// newDelegatedRoutingClient creates an HTTP delegated routing client with consistent options +func newDelegatedRoutingClient(endpoint string) (*drclient.Client, error) { + return drclient.New( + endpoint, + drclient.WithUserAgent("someguy/"+buildVersion()), + drclient.WithProtocolFilter([]string{}), + drclient.WithDisabledLocalFiltering(true), + ) +} + +// endpointConfig tracks which routing capabilities a base URL should provide +type endpointConfig struct { + baseURL string + providers bool // FindProviders capability + peers bool // FindPeer capability + ipns bool // GetIPNS/PutIPNS capability +} + +// collectEndpoints deduplicates base URLs across all endpoint types and +// aggregates their capabilities. This ensures we create only one HTTP client +// per unique base URL, even if it appears in multiple endpoint configurations. +func collectEndpoints(cfg *config) []endpointConfig { + capabilities := make(map[string]*endpointConfig) + + // Collect provider endpoints + for _, url := range cfg.contentEndpoints { + if url == "" { + continue // skip empty strings + } + if caps := capabilities[url]; caps != nil { + caps.providers = true + } else { + capabilities[url] = &endpointConfig{baseURL: url, providers: true} + } + } + + // Collect peer endpoints + for _, url := range cfg.peerEndpoints { + if url == "" { + continue // skip empty strings + } + if caps := capabilities[url]; caps != nil { + caps.peers = true + } else { + capabilities[url] = &endpointConfig{baseURL: url, peers: true} + } + } + + // Collect IPNS endpoints + for _, url := range cfg.ipnsEndpoints { + if url == "" { + continue // skip empty strings + } + if caps := capabilities[url]; caps != nil { + caps.ipns = true + } else { + capabilities[url] = &endpointConfig{baseURL: url, ipns: true} + } + } + + // Convert map to slice + result := make([]endpointConfig, 0, len(capabilities)) + for _, caps := range capabilities { + result = append(result, *caps) + } + + return result +} + +// createDelegatedHTTPRouters creates deduplicated HTTP routing clients. +// It ensures that each unique base URL gets exactly one HTTP client, even if +// that URL appears in multiple endpoint configurations (provider/peer/ipns). +// The same client instance is added to multiple router lists based on its +// aggregated capabilities. +func createDelegatedHTTPRouters(cfg *config) (providers, peers, ipns []router, err error) { + endpoints := collectEndpoints(cfg) + + var providerRouters, peerRouters, ipnsRouters []router + + for _, endpoint := range endpoints { + // Create ONE HTTP client per unique base URL + client, err := newDelegatedRoutingClient(endpoint.baseURL) + if err != nil { + return nil, nil, nil, err + } + + // Wrap in clientRouter - this implements all routing interfaces + router := clientRouter{Client: client} + + // Add the same router instance to appropriate lists based on capabilities + if endpoint.providers { + providerRouters = append(providerRouters, router) + } + if endpoint.peers { + peerRouters = append(peerRouters, router) + } + if endpoint.ipns { + ipnsRouters = append(ipnsRouters, router) + } + } + + // Register delegated routing client metrics only once for all HTTP clients. + // We must avoid registering multiple times since view.Register() is a global operation. + if len(providerRouters) > 0 || len(peerRouters) > 0 || len(ipnsRouters) > 0 { + if err := view.Register(drclient.OpenCensusViews...); err != nil { + return nil, nil, nil, fmt.Errorf("registering HTTP delegated routing views: %w", err) + } + } + + return providerRouters, peerRouters, ipnsRouters, nil +} diff --git a/server_delegated_routing_test.go b/server_delegated_routing_test.go new file mode 100644 index 0000000..78acd08 --- /dev/null +++ b/server_delegated_routing_test.go @@ -0,0 +1,116 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestCollectEndpoints(t *testing.T) { + t.Run("deduplicates same URL across multiple endpoint types", func(t *testing.T) { + cfg := &config{ + contentEndpoints: []string{"https://example.com"}, + peerEndpoints: []string{"https://example.com"}, + ipnsEndpoints: []string{"https://example.com"}, + } + + endpoints := collectEndpoints(cfg) + + require.Len(t, endpoints, 1, "should have exactly one endpoint") + assert.Equal(t, "https://example.com", endpoints[0].baseURL) + assert.True(t, endpoints[0].providers, "should support providers") + assert.True(t, endpoints[0].peers, "should support peers") + assert.True(t, endpoints[0].ipns, "should support ipns") + }) + + t.Run("handles different URLs separately", func(t *testing.T) { + cfg := &config{ + contentEndpoints: []string{"https://a.com", "https://b.com"}, + peerEndpoints: []string{"https://b.com", "https://c.com"}, + } + + endpoints := collectEndpoints(cfg) + + require.Len(t, endpoints, 3, "should have three separate endpoints") + + // Convert to map for easier testing + urlMap := make(map[string]endpointConfig) + for _, ep := range endpoints { + urlMap[ep.baseURL] = ep + } + + // Verify a.com (providers only) + assert.True(t, urlMap["https://a.com"].providers) + assert.False(t, urlMap["https://a.com"].peers) + assert.False(t, urlMap["https://a.com"].ipns) + + // Verify b.com (providers and peers) + assert.True(t, urlMap["https://b.com"].providers) + assert.True(t, urlMap["https://b.com"].peers) + assert.False(t, urlMap["https://b.com"].ipns) + + // Verify c.com (peers only) + assert.False(t, urlMap["https://c.com"].providers) + assert.True(t, urlMap["https://c.com"].peers) + assert.False(t, urlMap["https://c.com"].ipns) + }) + + t.Run("skips empty strings", func(t *testing.T) { + cfg := &config{ + contentEndpoints: []string{"https://example.com", "", "https://another.com"}, + peerEndpoints: []string{""}, + } + + endpoints := collectEndpoints(cfg) + + require.Len(t, endpoints, 2, "should skip empty strings") + + urlMap := make(map[string]endpointConfig) + for _, ep := range endpoints { + urlMap[ep.baseURL] = ep + } + + assert.Contains(t, urlMap, "https://example.com") + assert.Contains(t, urlMap, "https://another.com") + assert.NotContains(t, urlMap, "") + }) + + t.Run("handles all three endpoint types for different URLs", func(t *testing.T) { + cfg := &config{ + contentEndpoints: []string{"https://provider.com"}, + peerEndpoints: []string{"https://peer.com"}, + ipnsEndpoints: []string{"https://ipns.com"}, + } + + endpoints := collectEndpoints(cfg) + + require.Len(t, endpoints, 3) + + urlMap := make(map[string]endpointConfig) + for _, ep := range endpoints { + urlMap[ep.baseURL] = ep + } + + // Each URL should have only one capability enabled + assert.True(t, urlMap["https://provider.com"].providers) + assert.False(t, urlMap["https://provider.com"].peers) + assert.False(t, urlMap["https://provider.com"].ipns) + + assert.False(t, urlMap["https://peer.com"].providers) + assert.True(t, urlMap["https://peer.com"].peers) + assert.False(t, urlMap["https://peer.com"].ipns) + + assert.False(t, urlMap["https://ipns.com"].providers) + assert.False(t, urlMap["https://ipns.com"].peers) + assert.True(t, urlMap["https://ipns.com"].ipns) + }) + + t.Run("empty config returns empty list", func(t *testing.T) { + cfg := &config{} + + endpoints := collectEndpoints(cfg) + + assert.Empty(t, endpoints) + }) +} diff --git a/server_dht.go b/server_dht.go index 70cf291..7f42aab 100644 --- a/server_dht.go +++ b/server_dht.go @@ -18,8 +18,8 @@ type bundledDHT struct { fullRT *fullrt.FullRT } -func newBundledDHT(ctx context.Context, h host.Host) (routing.Routing, error) { - standardDHT, err := dht.New(ctx, h, dht.Mode(dht.ModeClient), dht.BootstrapPeers(dht.GetDefaultBootstrapPeerAddrInfos()...)) +func newBundledDHT(ctx context.Context, h host.Host, bootstrapAddrInfos []peer.AddrInfo) (routing.Routing, error) { + standardDHT, err := dht.New(ctx, h, dht.Mode(dht.ModeClient), dht.BootstrapPeers(bootstrapAddrInfos...)) if err != nil { return nil, err } @@ -31,7 +31,7 @@ func newBundledDHT(ctx context.Context, h host.Host) (routing.Routing, error) { "pk": record.PublicKeyValidator{}, "ipns": ipns.Validator{}, }), - dht.BootstrapPeers(dht.GetDefaultBootstrapPeerAddrInfos()...), + dht.BootstrapPeers(bootstrapAddrInfos...), dht.Mode(dht.ModeClient), )) if err != nil { diff --git a/server_routers.go b/server_routers.go index 486951e..607e971 100644 --- a/server_routers.go +++ b/server_routers.go @@ -8,7 +8,6 @@ import ( "time" "github.com/ipfs/boxo/ipns" - "github.com/ipfs/boxo/routing/http/client" "github.com/ipfs/boxo/routing/http/server" "github.com/ipfs/boxo/routing/http/types" "github.com/ipfs/boxo/routing/http/types/iter" @@ -412,20 +411,6 @@ func (it *peerChanIter) Close() error { return nil } -var _ router = clientRouter{} - -type clientRouter struct { - *client.Client -} - -func (d clientRouter) FindProviders(ctx context.Context, cid cid.Cid, limit int) (iter.ResultIter[types.Record], error) { - return d.Client.FindProviders(ctx, cid) -} - -func (d clientRouter) FindPeers(ctx context.Context, pid peer.ID, limit int) (iter.ResultIter[*types.PeerRecord], error) { - return d.Client.FindPeers(ctx, pid) -} - var _ server.ContentRouter = sanitizeRouter{} type sanitizeRouter struct { diff --git a/server_test.go b/server_test.go index a09dd6c..e4ffbb9 100644 --- a/server_test.go +++ b/server_test.go @@ -6,19 +6,21 @@ import ( "github.com/stretchr/testify/require" ) -func TestGetCombinedRouting(t *testing.T) { +func TestCombineRouters(t *testing.T) { t.Parallel() - // Check of the result of get combined routing is a sanitize router. - v, err := getCombinedRouting(nil, &bundledDHT{}, nil, nil) - require.NoError(t, err) + // Mock router for testing + mockRouter := composableRouter{} + + // Check that combineRouters with DHT only returns sanitizeRouter + v := combineRouters(&bundledDHT{}, nil, nil, nil) require.IsType(t, sanitizeRouter{}, v) - v, err = getCombinedRouting([]string{"https://example.com/"}, nil, nil, nil) - require.NoError(t, err) + // Check that combineRouters with delegated routers only returns parallelRouter + v = combineRouters(nil, nil, []router{mockRouter}, nil) require.IsType(t, parallelRouter{}, v) - v, err = getCombinedRouting([]string{"https://example.com/"}, &bundledDHT{}, nil, nil) - require.NoError(t, err) + // Check that combineRouters with both DHT and delegated routers returns parallelRouter + v = combineRouters(&bundledDHT{}, nil, []router{mockRouter}, nil) require.IsType(t, parallelRouter{}, v) }