Skip to content

Commit eabf693

Browse files
cmaglieper1234
cmaglie
and
per1234
authored
[skip-changelog] [breaking] Refactoring of download subroutines (#1697)
* Moved downloaders subroutines in 'resources' package * Moved function to the correct file * Merge Download subroutine for resources * Factored all download subroutines * Allow passing nil Config in resource.Download method * Moved some package in their appropriate place. httpclient is now under arduino/httpclient. Some configuration values have been moved under 'configuration' package, and the related subroutines have been refactored. * Created IndexResource and factored out all download utilities * Fix regression in unit-tests * Adjusted integration tests * Fixed linter problems * Moved DownloadFile from 'resources' package into 'httpclient' * Factored all progress reports callback definitions in the rpc package * Updated UPGRADING.md * Applied suggestions from code review Co-authored-by: per1234 <accounts@perglass.com> * Fixed typos Co-authored-by: per1234 <accounts@perglass.com>
1 parent f07d9ea commit eabf693

File tree

33 files changed

+474
-431
lines changed

33 files changed

+474
-431
lines changed

‎arduino/cores/packagemanager/download.go‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919
"fmt"
2020

2121
"github.com/arduino/arduino-cli/arduino/cores"
22+
rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
2223
"go.bug.st/downloader/v2"
2324
semver "go.bug.st/relaxed-semver"
2425
)
@@ -117,16 +118,16 @@ func (pm *PackageManager) FindPlatformReleaseDependencies(item *PlatformReferenc
117118

118119
// DownloadToolRelease downloads a ToolRelease. If the tool is already downloaded a nil Downloader
119120
// is returned.
120-
func (pm *PackageManager) DownloadToolRelease(tool *cores.ToolRelease, config *downloader.Config) (*downloader.Downloader, error) {
121+
func (pm *PackageManager) DownloadToolRelease(tool *cores.ToolRelease, config *downloader.Config, label string, progressCB rpc.DownloadProgressCB) error {
121122
resource := tool.GetCompatibleFlavour()
122123
if resource == nil {
123-
return nil, fmt.Errorf(tr("tool not available for your OS"))
124+
return fmt.Errorf(tr("tool not available for your OS"))
124125
}
125-
return resource.Download(pm.DownloadDir, config)
126+
return resource.Download(pm.DownloadDir, config, label, progressCB)
126127
}
127128

128129
// DownloadPlatformRelease downloads a PlatformRelease. If the platform is already downloaded a
129130
// nil Downloader is returned.
130-
func (pm *PackageManager) DownloadPlatformRelease(platform *cores.PlatformRelease, config *downloader.Config) (*downloader.Downloader, error) {
131-
return platform.Resource.Download(pm.DownloadDir, config)
131+
func (pm *PackageManager) DownloadPlatformRelease(platform *cores.PlatformRelease, config *downloader.Config, label string, progressCB rpc.DownloadProgressCB) error {
132+
return platform.Resource.Download(pm.DownloadDir, config, label, progressCB)
132133
}

‎arduino/httpclient/httpclient.go‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
// This file is part of arduino-cli.
2+
//
3+
// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
4+
//
5+
// This software is released under the GNU General Public License version 3,
6+
// which covers the main part of arduino-cli.
7+
// The terms of this license can be found at:
8+
// https://www.gnu.org/licenses/gpl-3.0.en.html
9+
//
10+
// You can be released from the requirements of the above licenses by purchasing
11+
// a commercial license. Buying such a license is mandatory if you want to
12+
// modify or otherwise use the software for commercial activities involving the
13+
// Arduino software without disclosing the source code of your own applications.
14+
// To purchase a commercial license, send an email to license@arduino.cc.
15+
16+
package httpclient
17+
18+
import (
19+
"net/http"
20+
"net/url"
21+
"time"
22+
23+
"github.com/arduino/arduino-cli/arduino"
24+
"github.com/arduino/arduino-cli/configuration"
25+
"github.com/arduino/arduino-cli/i18n"
26+
rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
27+
"github.com/arduino/go-paths-helper"
28+
"go.bug.st/downloader/v2"
29+
)
30+
31+
var tr = i18n.Tr
32+
33+
// DownloadFile downloads a file from a URL into the specified path. An optional config and options may be passed (or nil to use the defaults).
34+
// A DownloadProgressCB callback function must be passed to monitor download progress.
35+
func DownloadFile(path *paths.Path, URL string, label string, downloadCB rpc.DownloadProgressCB, config *downloader.Config, options ...downloader.DownloadOptions) error {
36+
if config == nil {
37+
c, err := GetDownloaderConfig()
38+
if err != nil {
39+
return err
40+
}
41+
config = c
42+
}
43+
44+
d, err := downloader.DownloadWithConfig(path.String(), URL, *config, options...)
45+
if err != nil {
46+
return err
47+
}
48+
downloadCB(&rpc.DownloadProgress{
49+
File: label,
50+
Url: d.URL,
51+
TotalSize: d.Size(),
52+
})
53+
54+
err = d.RunAndPoll(func(downloaded int64) {
55+
downloadCB(&rpc.DownloadProgress{Downloaded: downloaded})
56+
}, 250*time.Millisecond)
57+
if err != nil {
58+
return err
59+
}
60+
61+
// The URL is not reachable for some reason
62+
if d.Resp.StatusCode >= 400 && d.Resp.StatusCode <= 599 {
63+
return &arduino.FailedDownloadError{Message: tr("Server responded with: %s", d.Resp.Status)}
64+
}
65+
66+
downloadCB(&rpc.DownloadProgress{Completed: true})
67+
return nil
68+
}
69+
70+
// Config is the configuration of the http client
71+
type Config struct {
72+
UserAgent string
73+
Proxy *url.URL
74+
}
75+
76+
// New returns a default http client for use in the arduino-cli
77+
func New() (*http.Client, error) {
78+
userAgent := configuration.UserAgent(configuration.Settings)
79+
proxy, err := configuration.NetworkProxy(configuration.Settings)
80+
if err != nil {
81+
return nil, err
82+
}
83+
return NewWithConfig(&Config{UserAgent: userAgent, Proxy: proxy}), nil
84+
}
85+
86+
// NewWithConfig creates a http client for use in the arduino-cli, with a given configuration
87+
func NewWithConfig(config *Config) *http.Client {
88+
return &http.Client{
89+
Transport: &httpClientRoundTripper{
90+
transport: &http.Transport{
91+
Proxy: http.ProxyURL(config.Proxy),
92+
},
93+
userAgent: config.UserAgent,
94+
},
95+
}
96+
}
97+
98+
// GetDownloaderConfig returns the downloader configuration based on current settings.
99+
func GetDownloaderConfig() (*downloader.Config, error) {
100+
httpClient, err := New()
101+
if err != nil {
102+
return nil, &arduino.InvalidArgumentError{Message: tr("Could not connect via HTTP"), Cause: err}
103+
}
104+
return &downloader.Config{
105+
HttpClient: *httpClient,
106+
}, nil
107+
}
108+
109+
type httpClientRoundTripper struct {
110+
transport http.RoundTripper
111+
userAgent string
112+
}
113+
114+
func (h *httpClientRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
115+
req.Header.Add("User-Agent", h.userAgent)
116+
return h.transport.RoundTrip(req)
117+
}

httpclient/httpclient_test.go renamed to arduino/httpclient/httpclient_test.go

File renamed without changes.

‎arduino/resources/download.go‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
// This file is part of arduino-cli.
2+
//
3+
// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
4+
//
5+
// This software is released under the GNU General Public License version 3,
6+
// which covers the main part of arduino-cli.
7+
// The terms of this license can be found at:
8+
// https://www.gnu.org/licenses/gpl-3.0.en.html
9+
//
10+
// You can be released from the requirements of the above licenses by purchasing
11+
// a commercial license. Buying such a license is mandatory if you want to
12+
// modify or otherwise use the software for commercial activities involving the
13+
// Arduino software without disclosing the source code of your own applications.
14+
// To purchase a commercial license, send an email to license@arduino.cc.
15+
16+
package resources
17+
18+
import (
19+
"fmt"
20+
"os"
21+
22+
"github.com/arduino/arduino-cli/arduino/httpclient"
23+
rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
24+
paths "github.com/arduino/go-paths-helper"
25+
"go.bug.st/downloader/v2"
26+
)
27+
28+
// Download performs a download loop using the provided downloader.Config.
29+
// Messages are passed back to the DownloadProgressCB using label as text for the File field.
30+
func (r *DownloadResource) Download(downloadDir *paths.Path, config *downloader.Config, label string, downloadCB rpc.DownloadProgressCB) error {
31+
path, err := r.ArchivePath(downloadDir)
32+
if err != nil {
33+
return fmt.Errorf(tr("getting archive path: %s"), err)
34+
}
35+
36+
if _, err := path.Stat(); os.IsNotExist(err) {
37+
// normal download
38+
} else if err == nil {
39+
// check local file integrity
40+
ok, err := r.TestLocalArchiveIntegrity(downloadDir)
41+
if err != nil || !ok {
42+
if err := path.Remove(); err != nil {
43+
return fmt.Errorf(tr("removing corrupted archive file: %s"), err)
44+
}
45+
} else {
46+
// File is cached, nothing to do here
47+
48+
// This signal means that the file is already downloaded
49+
downloadCB(&rpc.DownloadProgress{
50+
File: label,
51+
Completed: true,
52+
})
53+
return nil
54+
}
55+
} else {
56+
return fmt.Errorf(tr("getting archive file info: %s"), err)
57+
}
58+
return httpclient.DownloadFile(path, r.URL, label, downloadCB, config)
59+
}

‎arduino/resources/helpers.go‎

Lines changed: 0 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,8 @@ package resources
1717

1818
import (
1919
"fmt"
20-
"os"
2120

2221
"github.com/arduino/go-paths-helper"
23-
"go.bug.st/downloader/v2"
2422
)
2523

2624
// ArchivePath returns the path of the Archive of the specified DownloadResource relative
@@ -41,30 +39,3 @@ func (r *DownloadResource) IsCached(downloadDir *paths.Path) (bool, error) {
4139
}
4240
return archivePath.Exist(), nil
4341
}
44-
45-
// Download a DownloadResource.
46-
func (r *DownloadResource) Download(downloadDir *paths.Path, config *downloader.Config) (*downloader.Downloader, error) {
47-
path, err := r.ArchivePath(downloadDir)
48-
if err != nil {
49-
return nil, fmt.Errorf(tr("getting archive path: %s"), err)
50-
}
51-
52-
if _, err := path.Stat(); os.IsNotExist(err) {
53-
// normal download
54-
} else if err == nil {
55-
// check local file integrity
56-
ok, err := r.TestLocalArchiveIntegrity(downloadDir)
57-
if err != nil || !ok {
58-
if err := path.Remove(); err != nil {
59-
return nil, fmt.Errorf(tr("removing corrupted archive file: %s"), err)
60-
}
61-
} else {
62-
// File is cached, nothing to do here
63-
return nil, nil
64-
}
65-
} else {
66-
return nil, fmt.Errorf(tr("getting archive file info: %s"), err)
67-
}
68-
69-
return downloader.DownloadWithConfig(path.String(), r.URL, *config)
70-
}

‎arduino/resources/helpers_test.go‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@ import (
2222
"strings"
2323
"testing"
2424

25-
"github.com/arduino/arduino-cli/httpclient"
25+
"github.com/arduino/arduino-cli/arduino/httpclient"
26+
rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
2627
"github.com/arduino/go-paths-helper"
2728
"github.com/stretchr/testify/require"
2829
"go.bug.st/downloader/v2"
@@ -55,9 +56,7 @@ func TestDownloadApplyUserAgentHeaderUsingConfig(t *testing.T) {
5556

5657
httpClient := httpclient.NewWithConfig(&httpclient.Config{UserAgent: goldUserAgentValue})
5758

58-
d, err := r.Download(tmp, &downloader.Config{HttpClient: *httpClient})
59-
require.NoError(t, err)
60-
err = d.Run()
59+
err = r.Download(tmp, &downloader.Config{HttpClient: *httpClient}, "", func(progress *rpc.DownloadProgress) {})
6160
require.NoError(t, err)
6261

6362
// leverage the download helper to download the echo for the request made by the downloader itself

‎arduino/resources/index.go‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
// This file is part of arduino-cli.
2+
//
3+
// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
4+
//
5+
// This software is released under the GNU General Public License version 3,
6+
// which covers the main part of arduino-cli.
7+
// The terms of this license can be found at:
8+
// https://www.gnu.org/licenses/gpl-3.0.en.html
9+
//
10+
// You can be released from the requirements of the above licenses by purchasing
11+
// a commercial license. Buying such a license is mandatory if you want to
12+
// modify or otherwise use the software for commercial activities involving the
13+
// Arduino software without disclosing the source code of your own applications.
14+
// To purchase a commercial license, send an email to license@arduino.cc.
15+
16+
package resources
17+
18+
import (
19+
"net/url"
20+
"path"
21+
"strings"
22+
23+
"github.com/arduino/arduino-cli/arduino"
24+
"github.com/arduino/arduino-cli/arduino/httpclient"
25+
"github.com/arduino/arduino-cli/arduino/security"
26+
rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
27+
"github.com/arduino/go-paths-helper"
28+
"go.bug.st/downloader/v2"
29+
)
30+
31+
// IndexResource is a reference to an index file URL with an optional signature.
32+
type IndexResource struct {
33+
URL *url.URL
34+
SignatureURL *url.URL
35+
}
36+
37+
// Download will download the index and possibly check the signature using the Arduino's public key.
38+
// If the file is in .gz format it will be unpacked first.
39+
func (res *IndexResource) Download(destDir *paths.Path, downloadCB rpc.DownloadProgressCB) error {
40+
// Create destination directory
41+
if err := destDir.MkdirAll(); err != nil {
42+
return &arduino.PermissionDeniedError{Message: tr("Can't create data directory %s", destDir), Cause: err}
43+
}
44+
45+
// Create a temp dir to stage all downloads
46+
tmp, err := paths.MkTempDir("", "library_index_download")
47+
if err != nil {
48+
return &arduino.TempDirCreationFailedError{Cause: err}
49+
}
50+
defer tmp.RemoveAll()
51+
52+
// Download index file
53+
indexFileName := path.Base(res.URL.Path) // == package_index.json[.gz]
54+
tmpIndexPath := tmp.Join(indexFileName)
55+
if err := httpclient.DownloadFile(tmpIndexPath, res.URL.String(), tr("Downloading index: %s", indexFileName), downloadCB, nil, downloader.NoResume); err != nil {
56+
return &arduino.FailedDownloadError{Message: tr("Error downloading index '%s'", res.URL), Cause: err}
57+
}
58+
59+
// Expand the index if it is compressed
60+
if strings.HasSuffix(indexFileName, ".gz") {
61+
indexFileName = strings.TrimSuffix(indexFileName, ".gz") // == package_index.json
62+
tmpUnzippedIndexPath := tmp.Join(indexFileName)
63+
if err := paths.GUnzip(tmpIndexPath, tmpUnzippedIndexPath); err != nil {
64+
return &arduino.PermissionDeniedError{Message: tr("Error extracting %s", indexFileName), Cause: err}
65+
}
66+
tmpIndexPath = tmpUnzippedIndexPath
67+
}
68+
69+
// Check the signature if needed
70+
var signaturePath, tmpSignaturePath *paths.Path
71+
if res.SignatureURL != nil {
72+
// Compose signature URL
73+
signatureFileName := path.Base(res.SignatureURL.Path)
74+
75+
// Download signature
76+
signaturePath = destDir.Join(signatureFileName)
77+
tmpSignaturePath = tmp.Join(signatureFileName)
78+
if err := httpclient.DownloadFile(tmpSignaturePath, res.SignatureURL.String(), tr("Downloading index signature: %s", signatureFileName), downloadCB, nil, downloader.NoResume); err != nil {
79+
return &arduino.FailedDownloadError{Message: tr("Error downloading index signature '%s'", res.SignatureURL), Cause: err}
80+
}
81+
82+
// Check signature...
83+
if valid, _, err := security.VerifyArduinoDetachedSignature(tmpIndexPath, tmpSignaturePath); err != nil {
84+
return &arduino.PermissionDeniedError{Message: tr("Error verifying signature"), Cause: err}
85+
} else if !valid {
86+
return &arduino.SignatureVerificationFailedError{File: res.URL.String()}
87+
}
88+
}
89+
90+
// TODO: Implement a ResourceValidator
91+
// if !validate(tmpIndexPath) { return error }
92+
93+
// Make a backup copy of old index and signature so the defer function can rollback in case of errors.
94+
indexPath := destDir.Join(indexFileName)
95+
oldIndex := tmp.Join("old_index")
96+
if indexPath.Exist() {
97+
if err := indexPath.CopyTo(oldIndex); err != nil {
98+
return &arduino.PermissionDeniedError{Message: tr("Error saving downloaded index"), Cause: err}
99+
}
100+
defer oldIndex.CopyTo(indexPath) // will silently fail in case of success
101+
}
102+
oldSignature := tmp.Join("old_signature")
103+
if oldSignature.Exist() {
104+
if err := signaturePath.CopyTo(oldSignature); err != nil {
105+
return &arduino.PermissionDeniedError{Message: tr("Error saving downloaded index signature"), Cause: err}
106+
}
107+
defer oldSignature.CopyTo(signaturePath) // will silently fail in case of success
108+
}
109+
if err := tmpIndexPath.CopyTo(indexPath); err != nil {
110+
return &arduino.PermissionDeniedError{Message: tr("Error saving downloaded index"), Cause: err}
111+
}
112+
if res.SignatureURL != nil {
113+
if err := tmpSignaturePath.CopyTo(signaturePath); err != nil {
114+
return &arduino.PermissionDeniedError{Message: tr("Error saving downloaded index signature"), Cause: err}
115+
}
116+
}
117+
oldIndex.Remove()
118+
oldSignature.Remove()
119+
return nil
120+
}

0 commit comments

Comments
 (0)