1
|
|
|
package repo |
2
|
|
|
|
3
|
|
|
import ( |
4
|
|
|
"encoding/json" |
5
|
|
|
"fmt" |
6
|
|
|
"io/ioutil" |
7
|
|
|
"net/http" |
8
|
|
|
"net/url" |
9
|
|
|
"os" |
10
|
|
|
"path/filepath" |
11
|
|
|
"runtime" |
12
|
|
|
"sort" |
13
|
|
|
"strings" |
14
|
|
|
|
15
|
|
|
cp "github.com/Masterminds/glide/cache" |
16
|
|
|
"github.com/Masterminds/glide/cfg" |
17
|
|
|
"github.com/Masterminds/glide/msg" |
18
|
|
|
gpath "github.com/Masterminds/glide/path" |
19
|
|
|
"github.com/Masterminds/semver" |
20
|
|
|
v "github.com/Masterminds/vcs" |
21
|
|
|
) |
22
|
|
|
|
23
|
|
|
// VcsUpdate updates to a particular checkout based on the VCS setting. |
24
|
|
|
func VcsUpdate(dep *cfg.Dependency, force bool, updated *UpdateTracker) error { |
25
|
|
|
|
26
|
|
|
// If the dependency has already been pinned we can skip it. This is a |
27
|
|
|
// faster path so we don't need to resolve it again. |
28
|
|
|
if dep.Pin != "" { |
29
|
|
|
msg.Debug("Dependency %s has already been pinned. Fetching updates skipped", dep.Name) |
30
|
|
|
return nil |
31
|
|
|
} |
32
|
|
|
|
33
|
|
|
if updated.Check(dep.Name) { |
34
|
|
|
msg.Debug("%s was already updated, skipping", dep.Name) |
35
|
|
|
return nil |
36
|
|
|
} |
37
|
|
|
updated.Add(dep.Name) |
38
|
|
|
|
39
|
|
|
if filterArchOs(dep) { |
40
|
|
|
msg.Info("%s is not used for %s/%s.\n", dep.Name, runtime.GOOS, runtime.GOARCH) |
41
|
|
|
return nil |
42
|
|
|
} |
43
|
|
|
|
44
|
|
|
key, err := cp.Key(dep.Remote()) |
45
|
|
|
if err != nil { |
46
|
|
|
msg.Die("Cache key generation error: %s", err) |
47
|
|
|
} |
48
|
|
|
location := cp.Location() |
49
|
|
|
dest := filepath.Join(location, "src", key) |
50
|
|
|
|
51
|
|
|
// If destination doesn't exist we need to perform an initial checkout. |
52
|
|
|
if _, err := os.Stat(dest); os.IsNotExist(err) { |
53
|
|
|
msg.Info("--> Fetching %s", dep.Name) |
54
|
|
|
if err = VcsGet(dep); err != nil { |
55
|
|
|
msg.Warn("Unable to checkout %s\n", dep.Name) |
56
|
|
|
return err |
57
|
|
|
} |
58
|
|
|
} else { |
59
|
|
|
// At this point we have a directory for the package. |
60
|
|
|
msg.Info("--> Fetching updates for %s", dep.Name) |
61
|
|
|
|
62
|
|
|
// When the directory is not empty and has no VCS directory it's |
63
|
|
|
// a vendored files situation. |
64
|
|
|
empty, err := gpath.IsDirectoryEmpty(dest) |
65
|
|
|
if err != nil { |
66
|
|
|
return err |
67
|
|
|
} |
68
|
|
|
_, err = v.DetectVcsFromFS(dest) |
69
|
|
|
if empty == true && err == v.ErrCannotDetectVCS { |
70
|
|
|
msg.Warn("Cached version of %s is an empty directory. Fetching a new copy of the dependency", dep.Name) |
71
|
|
|
msg.Debug("Removing empty directory %s", dest) |
72
|
|
|
err := os.RemoveAll(dest) |
73
|
|
|
if err != nil { |
74
|
|
|
return err |
75
|
|
|
} |
76
|
|
|
if err = VcsGet(dep); err != nil { |
77
|
|
|
msg.Warn("Unable to checkout %s\n", dep.Name) |
78
|
|
|
return err |
79
|
|
|
} |
80
|
|
|
} else { |
81
|
|
|
repo, err := dep.GetRepo(dest) |
82
|
|
|
|
83
|
|
|
// Tried to checkout a repo to a path that does not work. Either the |
84
|
|
|
// type or endpoint has changed. Force is being passed in so the old |
85
|
|
|
// location can be removed and replaced with the new one. |
86
|
|
|
// Warning, any changes in the old location will be deleted. |
87
|
|
|
// TODO: Put dirty checking in on the existing local checkout. |
88
|
|
|
if (err == v.ErrWrongVCS || err == v.ErrWrongRemote) && force == true { |
89
|
|
|
newRemote := dep.Remote() |
90
|
|
|
|
91
|
|
|
msg.Warn("Replacing %s with contents from %s\n", dep.Name, newRemote) |
92
|
|
|
rerr := os.RemoveAll(dest) |
93
|
|
|
if rerr != nil { |
94
|
|
|
return rerr |
95
|
|
|
} |
96
|
|
|
if err = VcsGet(dep); err != nil { |
97
|
|
|
msg.Warn("Unable to checkout %s\n", dep.Name) |
98
|
|
|
return err |
99
|
|
|
} |
100
|
|
|
|
101
|
|
|
repo, err = dep.GetRepo(dest) |
102
|
|
|
if err != nil { |
103
|
|
|
return err |
104
|
|
|
} |
105
|
|
|
} else if err != nil { |
106
|
|
|
return err |
107
|
|
|
} else if repo.IsDirty() { |
108
|
|
|
return fmt.Errorf("%s contains uncommitted changes. Skipping update", dep.Name) |
109
|
|
|
} |
110
|
|
|
|
111
|
|
|
ver := dep.Reference |
112
|
|
|
if ver == "" { |
113
|
|
|
ver = defaultBranch(repo) |
114
|
|
|
} |
115
|
|
|
// Check if the current version is a tag or commit id. If it is |
116
|
|
|
// and that version is already checked out we can skip updating |
117
|
|
|
// which is faster than going out to the Internet to perform |
118
|
|
|
// an update. |
119
|
|
|
if ver != "" { |
120
|
|
|
version, err := repo.Version() |
121
|
|
|
if err != nil { |
122
|
|
|
return err |
123
|
|
|
} |
124
|
|
|
ib, err := isBranch(ver, repo) |
125
|
|
|
if err != nil { |
126
|
|
|
return err |
127
|
|
|
} |
128
|
|
|
|
129
|
|
|
// If the current version equals the ref and it's not a |
130
|
|
|
// branch it's a tag or commit id so we can skip |
131
|
|
|
// performing an update. |
132
|
|
|
if version == ver && !ib { |
133
|
|
|
msg.Debug("%s is already set to version %s. Skipping update", dep.Name, dep.Reference) |
134
|
|
|
return nil |
135
|
|
|
} |
136
|
|
|
} |
137
|
|
|
|
138
|
|
|
if err := repo.Update(); err != nil { |
139
|
|
|
msg.Warn("Download failed.\n") |
140
|
|
|
return err |
141
|
|
|
} |
142
|
|
|
} |
143
|
|
|
} |
144
|
|
|
|
145
|
|
|
return nil |
146
|
|
|
} |
147
|
|
|
|
148
|
|
|
// VcsVersion set the VCS version for a checkout. |
149
|
|
|
func VcsVersion(dep *cfg.Dependency) error { |
150
|
|
|
|
151
|
|
|
// If the dependency has already been pinned we can skip it. This is a |
152
|
|
|
// faster path so we don't need to resolve it again. |
153
|
|
|
if dep.Pin != "" { |
154
|
|
|
msg.Debug("Dependency %s has already been pinned. Setting version skipped", dep.Name) |
155
|
|
|
return nil |
156
|
|
|
} |
157
|
|
|
|
158
|
|
|
key, err := cp.Key(dep.Remote()) |
159
|
|
|
if err != nil { |
160
|
|
|
msg.Die("Cache key generation error: %s", err) |
161
|
|
|
} |
162
|
|
|
location := cp.Location() |
163
|
|
|
cwd := filepath.Join(location, "src", key) |
164
|
|
|
|
165
|
|
|
// If there is no reference configured there is nothing to set. |
166
|
|
|
if dep.Reference == "" { |
167
|
|
|
// Before exiting update the pinned version |
168
|
|
|
repo, err := dep.GetRepo(cwd) |
169
|
|
|
if err != nil { |
170
|
|
|
return err |
171
|
|
|
} |
172
|
|
|
dep.Pin, err = repo.Version() |
173
|
|
|
if err != nil { |
174
|
|
|
return err |
175
|
|
|
} |
176
|
|
|
return nil |
177
|
|
|
} |
178
|
|
|
|
179
|
|
|
// When the directory is not empty and has no VCS directory it's |
180
|
|
|
// a vendored files situation. |
181
|
|
|
empty, err := gpath.IsDirectoryEmpty(cwd) |
182
|
|
|
if err != nil { |
183
|
|
|
return err |
184
|
|
|
} |
185
|
|
|
_, err = v.DetectVcsFromFS(cwd) |
186
|
|
|
if empty == false && err == v.ErrCannotDetectVCS { |
187
|
|
|
return fmt.Errorf("Cache directory missing VCS information for %s", dep.Name) |
188
|
|
|
} |
189
|
|
|
|
190
|
|
|
repo, err := dep.GetRepo(cwd) |
191
|
|
|
if err != nil { |
192
|
|
|
return err |
193
|
|
|
} |
194
|
|
|
|
195
|
|
|
ver := dep.Reference |
196
|
|
|
// References in Git can begin with a ^ which is similar to semver. |
197
|
|
|
// If there is a ^ prefix we assume it's a semver constraint rather than |
198
|
|
|
// part of the git/VCS commit id. |
199
|
|
|
if repo.IsReference(ver) && !strings.HasPrefix(ver, "^") { |
200
|
|
|
msg.Info("--> Setting version for %s to %s.\n", dep.Name, ver) |
201
|
|
|
} else { |
202
|
|
|
|
203
|
|
|
// Create the constraint first to make sure it's valid before |
204
|
|
|
// working on the repo. |
205
|
|
|
constraint, err := semver.NewConstraint(ver) |
206
|
|
|
|
207
|
|
|
// Make sure the constriant is valid. At this point it's not a valid |
208
|
|
|
// reference so if it's not a valid constrint we can exit early. |
209
|
|
|
if err != nil { |
210
|
|
|
msg.Warn("The reference '%s' is not valid\n", ver) |
211
|
|
|
return err |
212
|
|
|
} |
213
|
|
|
|
214
|
|
|
// Get the tags and branches (in that order) |
215
|
|
|
refs, err := getAllVcsRefs(repo) |
216
|
|
|
if err != nil { |
217
|
|
|
return err |
218
|
|
|
} |
219
|
|
|
|
220
|
|
|
// Convert and filter the list to semver.Version instances |
221
|
|
|
semvers := getSemVers(refs) |
222
|
|
|
|
223
|
|
|
// Sort semver list |
224
|
|
|
sort.Sort(sort.Reverse(semver.Collection(semvers))) |
225
|
|
|
found := false |
226
|
|
|
for _, v := range semvers { |
227
|
|
|
if constraint.Check(v) { |
228
|
|
|
found = true |
229
|
|
|
// If the constrint passes get the original reference |
230
|
|
|
ver = v.Original() |
231
|
|
|
break |
232
|
|
|
} |
233
|
|
|
} |
234
|
|
|
if found { |
235
|
|
|
msg.Info("--> Detected semantic version. Setting version for %s to %s", dep.Name, ver) |
236
|
|
|
} else { |
237
|
|
|
msg.Warn("--> Unable to find semantic version for constraint %s %s", dep.Name, ver) |
238
|
|
|
} |
239
|
|
|
} |
240
|
|
|
if err := repo.UpdateVersion(ver); err != nil { |
241
|
|
|
return err |
242
|
|
|
} |
243
|
|
|
dep.Pin, err = repo.Version() |
244
|
|
|
if err != nil { |
245
|
|
|
return err |
246
|
|
|
} |
247
|
|
|
|
248
|
|
|
return nil |
249
|
|
|
} |
250
|
|
|
|
251
|
|
|
// VcsGet figures out how to fetch a dependency, and then gets it. |
252
|
|
|
// |
253
|
|
|
// VcsGet installs into the cache. |
254
|
|
|
func VcsGet(dep *cfg.Dependency) error { |
255
|
|
|
|
256
|
|
|
key, err := cp.Key(dep.Remote()) |
257
|
|
|
if err != nil { |
258
|
|
|
msg.Die("Cache key generation error: %s", err) |
259
|
|
|
} |
260
|
|
|
location := cp.Location() |
261
|
|
|
d := filepath.Join(location, "src", key) |
262
|
|
|
|
263
|
|
|
repo, err := dep.GetRepo(d) |
264
|
|
|
if err != nil { |
265
|
|
|
return err |
266
|
|
|
} |
267
|
|
|
// If the directory does not exist this is a first cache. |
268
|
|
|
if _, err = os.Stat(d); os.IsNotExist(err) { |
269
|
|
|
msg.Debug("Adding %s to the cache for the first time", dep.Name) |
270
|
|
|
err = repo.Get() |
271
|
|
|
if err != nil { |
272
|
|
|
return err |
273
|
|
|
} |
274
|
|
|
branch := findCurrentBranch(repo) |
275
|
|
|
if branch != "" { |
276
|
|
|
msg.Debug("Saving default branch for %s", repo.Remote()) |
277
|
|
|
c := cp.RepoInfo{DefaultBranch: branch} |
278
|
|
|
err = cp.SaveRepoData(key, c) |
279
|
|
|
if err == cp.ErrCacheDisabled { |
280
|
|
|
msg.Debug("Unable to cache default branch because caching is disabled") |
281
|
|
|
} else if err != nil { |
282
|
|
|
msg.Debug("Error saving %s to cache. Error: %s", repo.Remote(), err) |
283
|
|
|
} |
284
|
|
|
} |
285
|
|
|
} else { |
286
|
|
|
msg.Debug("Updating %s in the cache", dep.Name) |
287
|
|
|
err = repo.Update() |
288
|
|
|
if err != nil { |
289
|
|
|
return err |
290
|
|
|
} |
291
|
|
|
} |
292
|
|
|
|
293
|
|
|
return nil |
294
|
|
|
} |
295
|
|
|
|
296
|
|
|
// filterArchOs indicates a dependency should be filtered out because it is |
297
|
|
|
// the wrong GOOS or GOARCH. |
298
|
|
|
// |
299
|
|
|
// FIXME: Should this be moved to the dependency package? |
300
|
|
|
func filterArchOs(dep *cfg.Dependency) bool { |
301
|
|
|
found := false |
302
|
|
|
if len(dep.Arch) > 0 { |
303
|
|
|
for _, a := range dep.Arch { |
304
|
|
|
if a == runtime.GOARCH { |
305
|
|
|
found = true |
306
|
|
|
} |
307
|
|
|
} |
308
|
|
|
// If it's not found, it should be filtered out. |
309
|
|
|
if !found { |
310
|
|
|
return true |
311
|
|
|
} |
312
|
|
|
} |
313
|
|
|
|
314
|
|
|
found = false |
315
|
|
|
if len(dep.Os) > 0 { |
316
|
|
|
for _, o := range dep.Os { |
317
|
|
|
if o == runtime.GOOS { |
318
|
|
|
found = true |
319
|
|
|
} |
320
|
|
|
} |
321
|
|
|
if !found { |
322
|
|
|
return true |
323
|
|
|
} |
324
|
|
|
|
325
|
|
|
} |
326
|
|
|
|
327
|
|
|
return false |
328
|
|
|
} |
329
|
|
|
|
330
|
|
|
// isBranch returns true if the given string is a branch in VCS. |
331
|
|
|
func isBranch(branch string, repo v.Repo) (bool, error) { |
332
|
|
|
branches, err := repo.Branches() |
333
|
|
|
if err != nil { |
334
|
|
|
return false, err |
335
|
|
|
} |
336
|
|
|
for _, b := range branches { |
337
|
|
|
if b == branch { |
338
|
|
|
return true, nil |
339
|
|
|
} |
340
|
|
|
} |
341
|
|
|
return false, nil |
342
|
|
|
} |
343
|
|
|
|
344
|
|
|
// defaultBranch tries to ascertain the default branch for the given repo. |
345
|
|
|
// Some repos will have multiple branches in them (e.g. Git) while others |
346
|
|
|
// (e.g. Svn) will not. |
347
|
|
|
func defaultBranch(repo v.Repo) string { |
348
|
|
|
|
349
|
|
|
// Svn and Bzr use different locations (paths or entire locations) |
350
|
|
|
// for branches so we won't have a default branch. |
351
|
|
|
if repo.Vcs() == v.Svn || repo.Vcs() == v.Bzr { |
352
|
|
|
return "" |
353
|
|
|
} |
354
|
|
|
|
355
|
|
|
// Check the cache for a value. |
356
|
|
|
key, kerr := cp.Key(repo.Remote()) |
357
|
|
|
var d cp.RepoInfo |
358
|
|
|
if kerr == nil { |
359
|
|
|
d, err := cp.RepoData(key) |
360
|
|
|
if err == nil { |
361
|
|
|
if d.DefaultBranch != "" { |
362
|
|
|
return d.DefaultBranch |
363
|
|
|
} |
364
|
|
|
} |
365
|
|
|
} |
366
|
|
|
|
367
|
|
|
// If we don't have it in the store try some APIs |
368
|
|
|
r := repo.Remote() |
369
|
|
|
u, err := url.Parse(r) |
370
|
|
|
if err != nil { |
371
|
|
|
return "" |
372
|
|
|
} |
373
|
|
|
if u.Scheme == "" { |
374
|
|
|
// Where there is no scheme we try urls like [email protected]:foo/bar |
375
|
|
|
r = strings.Replace(r, ":", "/", -1) |
376
|
|
|
r = "ssh://" + r |
377
|
|
|
u, err = url.Parse(r) |
378
|
|
|
if err != nil { |
379
|
|
|
return "" |
380
|
|
|
} |
381
|
|
|
u.Scheme = "" |
382
|
|
|
} |
383
|
|
|
if u.Host == "github.com" { |
384
|
|
|
parts := strings.Split(u.Path, "/") |
385
|
|
|
if len(parts) != 2 { |
386
|
|
|
return "" |
387
|
|
|
} |
388
|
|
|
api := fmt.Sprintf("https://api.github.com/repos/%s/%s", parts[0], parts[1]) |
389
|
|
|
resp, err := http.Get(api) |
390
|
|
|
if err != nil { |
391
|
|
|
return "" |
392
|
|
|
} |
393
|
|
|
defer resp.Body.Close() |
394
|
|
|
if resp.StatusCode >= 300 || resp.StatusCode < 200 { |
395
|
|
|
return "" |
396
|
|
|
} |
397
|
|
|
body, err := ioutil.ReadAll(resp.Body) |
398
|
|
|
var data interface{} |
399
|
|
|
err = json.Unmarshal(body, &data) |
400
|
|
|
if err != nil { |
401
|
|
|
return "" |
402
|
|
|
} |
403
|
|
|
gh := data.(map[string]interface{}) |
404
|
|
|
db := gh["default_branch"].(string) |
405
|
|
|
if kerr == nil { |
406
|
|
|
d.DefaultBranch = db |
407
|
|
|
err := cp.SaveRepoData(key, d) |
408
|
|
|
if err == cp.ErrCacheDisabled { |
409
|
|
|
msg.Debug("Unable to cache default branch because caching is disabled") |
410
|
|
|
} else if err != nil { |
411
|
|
|
msg.Debug("Error saving %s to cache. Error: %s", repo.Remote(), err) |
412
|
|
|
} |
413
|
|
|
} |
414
|
|
|
return db |
415
|
|
|
} |
416
|
|
|
|
417
|
|
|
if u.Host == "bitbucket.org" { |
418
|
|
|
parts := strings.Split(u.Path, "/") |
419
|
|
|
if len(parts) != 2 { |
420
|
|
|
return "" |
421
|
|
|
} |
422
|
|
|
api := fmt.Sprintf("https://bitbucket.org/api/1.0/repositories/%s/%s/main-branch/", parts[0], parts[1]) |
423
|
|
|
resp, err := http.Get(api) |
424
|
|
|
if err != nil { |
425
|
|
|
return "" |
426
|
|
|
} |
427
|
|
|
defer resp.Body.Close() |
428
|
|
|
if resp.StatusCode >= 300 || resp.StatusCode < 200 { |
429
|
|
|
return "" |
430
|
|
|
} |
431
|
|
|
body, err := ioutil.ReadAll(resp.Body) |
432
|
|
|
var data interface{} |
433
|
|
|
err = json.Unmarshal(body, &data) |
434
|
|
|
if err != nil { |
435
|
|
|
return "" |
436
|
|
|
} |
437
|
|
|
bb := data.(map[string]interface{}) |
438
|
|
|
db := bb["name"].(string) |
439
|
|
|
if kerr == nil { |
440
|
|
|
d.DefaultBranch = db |
441
|
|
|
err := cp.SaveRepoData(key, d) |
442
|
|
|
if err == cp.ErrCacheDisabled { |
443
|
|
|
msg.Debug("Unable to cache default branch because caching is disabled") |
444
|
|
|
} else if err != nil { |
445
|
|
|
msg.Debug("Error saving %s to cache. Error: %s", repo.Remote(), err) |
446
|
|
|
} |
447
|
|
|
} |
448
|
|
|
return db |
449
|
|
|
} |
450
|
|
|
|
451
|
|
|
return "" |
452
|
|
|
} |
453
|
|
|
|
454
|
|
|
// From a local repo find out the current branch name if there is one. |
455
|
|
|
// Note, this should only be used right after a fresh clone to get accurate |
456
|
|
|
// information. |
457
|
|
|
func findCurrentBranch(repo v.Repo) string { |
458
|
|
|
msg.Debug("Attempting to find current branch for %s", repo.Remote()) |
459
|
|
|
// Svn and Bzr don't have default branches. |
460
|
|
|
if repo.Vcs() == v.Svn || repo.Vcs() == v.Bzr { |
461
|
|
|
return "" |
462
|
|
|
} |
463
|
|
|
|
464
|
|
|
if repo.Vcs() == v.Git || repo.Vcs() == v.Hg { |
465
|
|
|
ver, err := repo.Current() |
466
|
|
|
if err != nil { |
467
|
|
|
msg.Debug("Unable to find current branch for %s, error: %s", repo.Remote(), err) |
468
|
|
|
return "" |
469
|
|
|
} |
470
|
|
|
return ver |
471
|
|
|
} |
472
|
|
|
|
473
|
|
|
return "" |
474
|
|
|
} |
475
|
|
|
|
476
|
|
|
func envForDir(dir string) []string { |
477
|
|
|
env := os.Environ() |
478
|
|
|
return mergeEnvLists([]string{"PWD=" + dir}, env) |
479
|
|
|
} |
480
|
|
|
|
481
|
|
|
func mergeEnvLists(in, out []string) []string { |
482
|
|
|
NextVar: |
483
|
|
|
for _, inkv := range in { |
484
|
|
|
k := strings.SplitAfterN(inkv, "=", 2)[0] |
485
|
|
|
for i, outkv := range out { |
486
|
|
|
if strings.HasPrefix(outkv, k) { |
487
|
|
|
out[i] = inkv |
488
|
|
|
continue NextVar |
489
|
|
|
} |
490
|
|
|
} |
491
|
|
|
out = append(out, inkv) |
492
|
|
|
} |
493
|
|
|
return out |
494
|
|
|
} |
495
|
|
|
|