[language-platform] update src-cli to handle scip file upload (#897)

Author: cesrjimenez

CHANGELOG.md

@@ -13,6 +13,8 @@ All notable changes to `src-cli` are documented in this file.
 
 ### Added
 
+- `src codeintel upload` will now upload SCIP indexes (over LSIF indexes) when the target instance supports it. [#897](https://github.com/sourcegraph/src-cli/pull/897)
+
 ### Changed
 
 ### Fixed
@@ -36,7 +38,7 @@ All notable changes to `src-cli` are documented in this file.
 
 ### Fixed
 
-- Fix network timeout in `src users clean` occuring in instances with many users  [#901](https://github.com/sourcegraph/src-cli/pull/901)
+- Fix network timeout in `src users clean` occuring in instances with many users [#901](https://github.com/sourcegraph/src-cli/pull/901)
 - Aligned parsing of spec file parameter of `src batch repos` with other commands. [#919](https://github.com/sourcegraph/src-cli/pull/919)
 - Remove empty log outputs during batch spec execution. [#923](https://github.com/sourcegraph/src-cli/pull/923)
 
@@ -135,7 +137,6 @@ No noteworthy changes, mechanical release to match Sourcegraph release.
 
 - INTERNAL ONLY: Fixed src batch exec not logging errors.
 
-
 ## 3.42.2
 
 ### Fixed
@@ -179,7 +180,6 @@ No noteworthy changes, mechanical release to match Sourcegraph release.
 
 - The preview link shown when running `src batch remote` to create a new batch change no longer 404s. [sourcegraph/src-cli](https://github.com/sourcegraph/src-cli/pull/787)
 
-
 ## 3.40.11
 
 ### Changed

cmd/src/code_intel_upload.go

@@ -9,6 +9,7 @@ import (
 	"io"
 	"net/url"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -70,7 +71,12 @@ Examples:
 func handleCodeIntelUpload(args []string) error {
 	ctx := context.Background()
 
-	out, err := parseAndValidateCodeIntelUploadFlags(args)
+	isSCIPAvailable, err := isSCIPAvailable()
+	if err != nil {
+		return err
+	}
+
+	out, err := parseAndValidateCodeIntelUploadFlags(args, isSCIPAvailable)
 	if !codeintelUploadFlags.json {
 		if out != nil {
 			printInferredArguments(out)
@@ -88,7 +94,7 @@ func handleCodeIntelUpload(args []string) error {
 		Flags: codeintelUploadFlags.apiFlags,
 	})
 
-	uploadID, err := upload.UploadIndex(ctx, codeintelUploadFlags.file, client, codeintelUploadOptions(out))
+	uploadID, err := upload.UploadIndex(ctx, codeintelUploadFlags.file, client, codeintelUploadOptions(out, isSCIPAvailable))
 	if err != nil {
 		return handleUploadError(out, err)
 	}
@@ -132,12 +138,19 @@ func handleCodeIntelUpload(args []string) error {
 }
 
 // codeintelUploadOptions creates a set of upload options given the values in the flags.
-func codeintelUploadOptions(out *output.Output) upload.UploadOptions {
+func codeintelUploadOptions(out *output.Output, isSCIPAvailable bool) upload.UploadOptions {
 	var associatedIndexID *int
 	if codeintelUploadFlags.associatedIndexID != -1 {
 		associatedIndexID = &codeintelUploadFlags.associatedIndexID
 	}
 
+	cfg.AdditionalHeaders["Content-Type"] = "application/x-ndjson+lsif"
+	path := codeintelUploadFlags.uploadRoute
+	if isSCIPAvailable && filepath.Ext(codeintelUploadFlags.file) == ".scip" {
+		cfg.AdditionalHeaders["Content-Type"] = "application/x-protobuf+scip"
+		path = "/.api/scip/upload"
+	}
+
 	logger := upload.NewRequestLogger(
 		os.Stdout,
 		// Don't need to check upper bounds as we only compare verbosity ranges
@@ -162,7 +175,7 @@ func codeintelUploadOptions(out *output.Output) upload.UploadOptions {
 			AdditionalHeaders:   cfg.AdditionalHeaders,
 			MaxRetries:          5,
 			RetryInterval:       time.Second,
-			Path:                codeintelUploadFlags.uploadRoute,
+			Path:                path,
 			MaxPayloadSizeBytes: codeintelUploadFlags.maxPayloadSizeMb * 1000 * 1000,
 			GitHubToken:         codeintelUploadFlags.gitHubToken,
 			GitLabToken:         codeintelUploadFlags.gitLabToken,

cmd/src/code_intel_upload_flags.go

@@ -1,20 +1,23 @@
 package main
 
 import (
+	"context"
 	"flag"
 	"fmt"
+	"net/http"
 	"os"
 	"path"
 	"path/filepath"
 	"strings"
+	"time"
 
+	"github.com/sourcegraph/scip/bindings/go/scip"
+	libscip "github.com/sourcegraph/sourcegraph/lib/codeintel/lsif/scip"
+	"github.com/sourcegraph/sourcegraph/lib/codeintel/upload"
 	"github.com/sourcegraph/sourcegraph/lib/errors"
 	"github.com/sourcegraph/sourcegraph/lib/output"
 	"google.golang.org/protobuf/proto"
 
-	"github.com/sourcegraph/scip/bindings/go/scip"
-	"github.com/sourcegraph/sourcegraph/lib/codeintel/upload"
-
 	"github.com/sourcegraph/src-cli/internal/api"
 	"github.com/sourcegraph/src-cli/internal/codeintel"
 )
@@ -53,10 +56,15 @@ var (
 	// Used to include the insecure-skip-verify flag in the help output, as we don't use any of the
 	// other api.Client methods, so only the insecureSkipVerify flag is relevant here.
 	dummyflag bool
+
+	// Used to skip the LSIF -> SCIP conversion during the migration. Not expected to be used outside
+	// of codeintel-qa pipelines and not expected to last much longer than a few releases while we
+	// deprecate all LSIF code paths.
+	skipConversionToSCIP bool
 )
 
 func init() {
-	codeintelUploadFlagSet.StringVar(&codeintelUploadFlags.file, "file", "./dump.lsif", `The path to the LSIF dump file.`)
+	codeintelUploadFlagSet.StringVar(&codeintelUploadFlags.file, "file", "", `The path to the LSIF dump file.`)
 
 	// UploadRecordOptions
 	codeintelUploadFlagSet.StringVar(&codeintelUploadFlags.repo, "repo", "", `The name of the repository (e.g. github.com/gorilla/mux). By default, derived from the origin remote.`)
@@ -81,6 +89,9 @@ func init() {
 	codeintelUploadFlagSet.BoolVar(&codeintelUploadFlags.json, "json", false, `Output relevant state in JSON on success.`)
 	codeintelUploadFlagSet.BoolVar(&codeintelUploadFlags.open, "open", false, `Open the LSIF upload page in your browser.`)
 	codeintelUploadFlagSet.BoolVar(&dummyflag, "insecure-skip-verify", false, "Skip validation of TLS certificates against trusted chains")
+
+	// Testing flags
+	codeintelUploadFlagSet.BoolVar(&skipConversionToSCIP, "skip-scip", false, "Skip converting LSIF index to SCIP if the instance supports it; this option should only used for debugging")
 }
 
 // parseAndValidateCodeIntelUploadFlags calls codeintelUploadFlagset.Parse, then infers values for
@@ -89,7 +100,7 @@ func init() {
 //
 // On success, the global codeintelUploadFlags object will be populated with valid values. An
 // error is returned on failure.
-func parseAndValidateCodeIntelUploadFlags(args []string) (*output.Output, error) {
+func parseAndValidateCodeIntelUploadFlags(args []string, isSCIPAvailable bool) (*output.Output, error) {
 	if err := codeintelUploadFlagSet.Parse(args); err != nil {
 		return nil, err
 	}
@@ -112,22 +123,43 @@ func parseAndValidateCodeIntelUploadFlags(args []string) (*output.Output, error)
 		return nil, err
 	}
 
-	if err := handleSCIP(out); err != nil {
-		return nil, err
+	if !isFlagSet(codeintelUploadFlagSet, "file") {
+		defaultFile, err := inferDefaultFile()
+		if err != nil {
+			return nil, err
+		}
+		codeintelUploadFlags.file = defaultFile
 	}
 
-	if inferenceErrors := inferMissingCodeIntelUploadFlags(); len(inferenceErrors) > 0 {
-		return nil, errorWithHint{
-			err: inferenceErrors[0].err, hint: strings.Join([]string{
-				fmt.Sprintf(
-					"Unable to determine %s from environment. Check your working directory or supply -%s={value} explicitly",
-					inferenceErrors[0].argument,
-					inferenceErrors[0].argument,
-				),
-			}, "\n"),
+	// Check to see if input file exists
+	if _, err := os.Stat(codeintelUploadFlags.file); os.IsNotExist(err) {
+		if !isFlagSet(codeintelUploadFlagSet, "file") {
+			return nil, formatInferenceError(argumentInferenceError{"file", err})
+		}
+
+		return nil, errors.Newf("file %q does not exist", codeintelUploadFlags.file)
+	}
+
+	if !isSCIPAvailable {
+		if err := handleSCIP(out); err != nil {
+			return nil, err
+		}
+	} else {
+		if err := handleLSIF(out); err != nil {
+			return nil, err
 		}
 	}
 
+	// Check for new file existence after transformation
+	if _, err := os.Stat(codeintelUploadFlags.file); os.IsNotExist(err) {
+		return nil, errors.Newf("file %q does not exist", codeintelUploadFlags.file)
+	}
+
+	// Infer the remaining default arguments (may require reading from new file)
+	if inferenceErrors := inferMissingCodeIntelUploadFlags(); len(inferenceErrors) > 0 {
+		return nil, formatInferenceError(inferenceErrors[0])
+	}
+
 	if err := validateCodeIntelUploadFlags(); err != nil {
 		return nil, err
 	}
@@ -151,6 +183,22 @@ func codeintelUploadOutput() (out *output.Output) {
 	})
 }
 
+func isSCIPAvailable() (bool, error) {
+	client := cfg.apiClient(codeintelUploadFlags.apiFlags, codeintelUploadFlagSet.Output())
+	req, err := client.NewHTTPRequest(context.Background(), "HEAD", "/.api/scip/upload", nil)
+	if err != nil {
+		return false, err
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return false, err
+	}
+	defer resp.Body.Close()
+
+	return resp.StatusCode == http.StatusOK, nil
+}
+
 type argumentInferenceError struct {
 	argument string
 	err      error
@@ -174,7 +222,7 @@ func replaceBaseName(oldPath string, newBaseName string) string {
 func handleSCIP(out *output.Output) error {
 	fileExt := path.Ext(codeintelUploadFlags.file)
 	if len(fileExt) == 0 {
-		return errors.Newf("missing file extension for %s; expected .scip", codeintelUploadFlags.file)
+		return errors.Newf("missing file extension for %s; expected .scip or .lsif", codeintelUploadFlags.file)
 	}
 	inputFile := codeintelUploadFlags.file
 	if fileExt == ".scip" || fileExt == ".lsif-typed" {
@@ -251,16 +299,114 @@ func convertSCIPToLSIFGraph(out *output.Output, inputFile, outputFile string) er
 	return nil
 }
 
+func inferDefaultFile() (string, error) {
+	hasSCIP := true
+	const scipFilename = "index.scip"
+	if _, err := os.Stat(scipFilename); err != nil {
+		if os.IsNotExist(err) {
+			hasSCIP = false
+		} else {
+			return "", err
+		}
+	}
+
+	hasLSIF := true
+	const lsifFilename = "dump.lsif"
+	if _, err := os.Stat(lsifFilename); err != nil {
+		if os.IsNotExist(err) {
+			hasLSIF = false
+		} else {
+			return "", err
+		}
+	}
+
+	if hasSCIP && hasLSIF {
+		return "", errors.Newf("both %s and %s exists - cannot determine unambiguous choice", scipFilename, lsifFilename)
+	}
+	if !(hasSCIP || hasLSIF) {
+		return "", formatInferenceError(argumentInferenceError{"file", errors.Newf("neither %s nor %s exists", scipFilename, lsifFilename)})
+	}
+
+	if hasSCIP {
+		return scipFilename, nil
+	}
+
+	return lsifFilename, nil
+}
+
+func formatInferenceError(inferenceErr argumentInferenceError) error {
+	return errorWithHint{
+		err: inferenceErr.err, hint: strings.Join([]string{
+			fmt.Sprintf(
+				"Unable to determine %s from environment. Check your working directory or supply -%s={value} explicitly",
+				inferenceErr.argument,
+				inferenceErr.argument,
+			),
+		}, "\n"),
+	}
+}
+
+func handleLSIF(out *output.Output) error {
+	if skipConversionToSCIP {
+		return nil
+	}
+
+	fileExt := path.Ext(codeintelUploadFlags.file)
+	if len(fileExt) == 0 {
+		return errors.Newf("missing file extension for %s; expected .scip or .lsif", codeintelUploadFlags.file)
+	}
+
+	if fileExt == ".lsif" || fileExt == ".dump" {
+		inputFile := codeintelUploadFlags.file
+		outputFile := replaceExtension(inputFile, ".scip")
+		codeintelUploadFlags.file = outputFile
+		return convertLSIFToSCIP(out, inputFile, outputFile)
+	}
+
+	return nil
+}
+
+// Reads the LSIF encoded input file and writes the corresponding SCIP encoded output file.
+func convertLSIFToSCIP(out *output.Output, inputFile, outputFile string) error {
+	if out != nil {
+		out.Writef("%s  Converting %s into %s", output.EmojiInfo, inputFile, outputFile)
+	}
+
+	ctx := context.Background()
+	uploadID := -time.Now().Nanosecond()
+	root := codeintelUploadFlags.root
+
+	if !isFlagSet(codeintelUploadFlagSet, "root") {
+		// Best-effort infer the root; we have a strange cyclic init order where we're
+		// currently trying to determine the filename that determines the root, but we
+		// need the root when converting from LSIF to SCIP.
+		root, _ = inferIndexRoot()
+	}
+
+	rc, err := os.Open(inputFile)
+	if err != nil {
+		return err
+	}
+	defer rc.Close()
+
+	index, err := libscip.ConvertLSIF(ctx, uploadID, rc, root)
+	if err != nil {
+		return err
+	}
+	serialized, err := proto.Marshal(index)
+	if err != nil {
+		return err
+	}
+
+	return os.WriteFile(outputFile, serialized, os.ModePerm)
+}
+
 // inferMissingCodeIntelUploadFlags updates the flags values which were not explicitly
 // supplied by the user with default values inferred from the current git state and
 // filesystem.
 //
 // Note: This function must not be called before codeintelUploadFlagset.Parse.
 func inferMissingCodeIntelUploadFlags() (inferErrors []argumentInferenceError) {
-	if _, err := os.Stat(codeintelUploadFlags.file); os.IsNotExist(err) {
-		inferErrors = append(inferErrors, argumentInferenceError{"file", err})
-	}
-
 	indexerName, indexerVersion, readIndexerNameAndVersionErr := readIndexerNameAndVersion()
 	getIndexerName := func() (string, error) { return indexerName, readIndexerNameAndVersionErr }
 	getIndexerVersion := func() (string, error) { return indexerVersion, readIndexerNameAndVersionErr }

go.mod

@@ -21,9 +21,9 @@ require (
 	github.com/sourcegraph/go-diff v0.6.1
 	github.com/sourcegraph/jsonx v0.0.0-20200629203448-1a936bd500cf
 	github.com/sourcegraph/scip v0.2.3
-	github.com/sourcegraph/sourcegraph/lib v0.0.0-20230110145349-bfa074f6c6d7
+	github.com/sourcegraph/sourcegraph/lib v0.0.0-20230113014457-46187b849239
 	github.com/stretchr/testify v1.8.0
-	golang.org/x/net v0.2.0
+	golang.org/x/net v0.5.0
 	golang.org/x/sync v0.1.0
 	google.golang.org/api v0.102.0
 	google.golang.org/protobuf v1.28.1
@@ -114,10 +114,10 @@ require (
 	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
 	golang.org/x/mod v0.7.0 // indirect
 	golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect
-	golang.org/x/sys v0.2.0 // indirect
-	golang.org/x/term v0.2.0 // indirect
-	golang.org/x/text v0.4.0 // indirect
-	golang.org/x/tools v0.3.0 // indirect
+	golang.org/x/sys v0.4.0 // indirect
+	golang.org/x/term v0.4.0 // indirect
+	golang.org/x/text v0.6.0 // indirect
+	golang.org/x/tools v0.5.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20221024183307-1bc688fe9f3e // indirect