1
2
3
4
5
6
7
8
9
10 package org.eclipse.jgit.gitrepo;
11
12 import static java.nio.charset.StandardCharsets.UTF_8;
13 import static org.eclipse.jgit.lib.Constants.DEFAULT_REMOTE_NAME;
14 import static org.eclipse.jgit.lib.Constants.R_REMOTES;
15
16 import java.io.File;
17 import java.io.FileInputStream;
18 import java.io.IOException;
19 import java.io.InputStream;
20 import java.net.URI;
21 import java.text.MessageFormat;
22 import java.util.ArrayList;
23 import java.util.List;
24 import java.util.Map;
25 import java.util.Objects;
26 import java.util.StringJoiner;
27 import java.util.TreeMap;
28
29 import org.eclipse.jgit.annotations.NonNull;
30 import org.eclipse.jgit.annotations.Nullable;
31 import org.eclipse.jgit.api.Git;
32 import org.eclipse.jgit.api.GitCommand;
33 import org.eclipse.jgit.api.SubmoduleAddCommand;
34 import org.eclipse.jgit.api.errors.ConcurrentRefUpdateException;
35 import org.eclipse.jgit.api.errors.GitAPIException;
36 import org.eclipse.jgit.api.errors.InvalidRefNameException;
37 import org.eclipse.jgit.api.errors.JGitInternalException;
38 import org.eclipse.jgit.dircache.DirCache;
39 import org.eclipse.jgit.dircache.DirCacheBuilder;
40 import org.eclipse.jgit.dircache.DirCacheEntry;
41 import org.eclipse.jgit.gitrepo.ManifestParser.IncludedFileReader;
42 import org.eclipse.jgit.gitrepo.RepoProject.CopyFile;
43 import org.eclipse.jgit.gitrepo.RepoProject.LinkFile;
44 import org.eclipse.jgit.gitrepo.internal.RepoText;
45 import org.eclipse.jgit.internal.JGitText;
46 import org.eclipse.jgit.lib.CommitBuilder;
47 import org.eclipse.jgit.lib.Config;
48 import org.eclipse.jgit.lib.Constants;
49 import org.eclipse.jgit.lib.FileMode;
50 import org.eclipse.jgit.lib.ObjectId;
51 import org.eclipse.jgit.lib.ObjectInserter;
52 import org.eclipse.jgit.lib.PersonIdent;
53 import org.eclipse.jgit.lib.ProgressMonitor;
54 import org.eclipse.jgit.lib.Ref;
55 import org.eclipse.jgit.lib.RefDatabase;
56 import org.eclipse.jgit.lib.RefUpdate;
57 import org.eclipse.jgit.lib.RefUpdate.Result;
58 import org.eclipse.jgit.lib.Repository;
59 import org.eclipse.jgit.revwalk.RevCommit;
60 import org.eclipse.jgit.revwalk.RevWalk;
61 import org.eclipse.jgit.treewalk.TreeWalk;
62 import org.eclipse.jgit.util.FileUtils;
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81 public class RepoCommand extends GitCommand<RevCommit> {
82 private String manifestPath;
83 private String baseUri;
84 private URI targetUri;
85 private String groupsParam;
86 private String branch;
87 private String targetBranch = Constants.HEAD;
88 private boolean recordRemoteBranch = true;
89 private boolean recordSubmoduleLabels = true;
90 private boolean recordShallowSubmodules = true;
91 private PersonIdent author;
92 private RemoteReader callback;
93 private InputStream inputStream;
94 private IncludedFileReader includedReader;
95 private boolean ignoreRemoteFailures = false;
96
97 private ProgressMonitor monitor;
98
99
100
101
102
103
104
105
106
107
108
109 public interface RemoteReader {
110
111
112
113
114
115
116
117
118
119
120
121
122
123 @Nullable
124 public ObjectId sha1(String uri, String ref) throws GitAPIException;
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143 @Deprecated
144 public default byte[] readFile(String uri, String ref, String path)
145 throws GitAPIException, IOException {
146 return readFileWithMode(uri, ref, path).getContents();
147 }
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170 @NonNull
171 public RemoteFile readFileWithMode(String uri, String ref, String path)
172 throws GitAPIException, IOException;
173 }
174
175
176
177
178
179
180
181 public static final class RemoteFile {
182 @NonNull
183 private final byte[] contents;
184
185 @NonNull
186 private final FileMode fileMode;
187
188
189
190
191
192
193
194 public RemoteFile(@NonNull byte[] contents,
195 @NonNull FileMode fileMode) {
196 this.contents = Objects.requireNonNull(contents);
197 this.fileMode = Objects.requireNonNull(fileMode);
198 }
199
200
201
202
203
204
205
206
207
208 @NonNull
209 public byte[] getContents() {
210 return contents;
211 }
212
213
214
215
216 @NonNull
217 public FileMode getFileMode() {
218 return fileMode;
219 }
220
221 }
222
223
224 public static class DefaultRemoteReader implements RemoteReader {
225
226 @Override
227 public ObjectId sha1(String uri, String ref) throws GitAPIException {
228 Map<String, Ref> map = Git
229 .lsRemoteRepository()
230 .setRemote(uri)
231 .callAsMap();
232 Ref r = RefDatabase.findRef(map, ref);
233 return r != null ? r.getObjectId() : null;
234 }
235
236 @Override
237 public RemoteFile readFileWithMode(String uri, String ref, String path)
238 throws GitAPIException, IOException {
239 File dir = FileUtils.createTempDir("jgit_", ".git", null);
240 try (Git git = Git.cloneRepository().setBare(true).setDirectory(dir)
241 .setURI(uri).call()) {
242 Repository repo = git.getRepository();
243 ObjectId refCommitId = sha1(uri, ref);
244 if (refCommitId == null) {
245 throw new InvalidRefNameException(MessageFormat
246 .format(JGitText.get().refNotResolved, ref));
247 }
248 RevCommit commit = repo.parseCommit(refCommitId);
249 TreeWalk tw = TreeWalk.forPath(repo, path, commit.getTree());
250
251
252
253 return new RemoteFile(
254 tw.getObjectReader().open(tw.getObjectId(0))
255 .getCachedBytes(Integer.MAX_VALUE),
256 tw.getFileMode(0));
257 } finally {
258 FileUtils.delete(dir, FileUtils.RECURSIVE);
259 }
260 }
261 }
262
263 @SuppressWarnings("serial")
264 private static class ManifestErrorException extends GitAPIException {
265 ManifestErrorException(Throwable cause) {
266 super(RepoText.get().invalidManifest, cause);
267 }
268 }
269
270 @SuppressWarnings("serial")
271 private static class RemoteUnavailableException extends GitAPIException {
272 RemoteUnavailableException(String uri) {
273 super(MessageFormat.format(RepoText.get().errorRemoteUnavailable, uri));
274 }
275 }
276
277
278
279
280
281
282
283 public RepoCommand(Repository repo) {
284 super(repo);
285 }
286
287
288
289
290
291
292
293
294
295
296 public RepoCommand setPath(String path) {
297 this.manifestPath = path;
298 return this;
299 }
300
301
302
303
304
305
306
307
308
309
310
311 public RepoCommand setInputStream(InputStream inputStream) {
312 this.inputStream = inputStream;
313 return this;
314 }
315
316
317
318
319
320
321
322
323
324
325
326
327 public RepoCommand setURI(String uri) {
328 this.baseUri = uri;
329 return this;
330 }
331
332
333
334
335
336
337
338
339
340
341 public RepoCommand setTargetURI(String uri) {
342
343
344
345
346 this.targetUri = URI.create(uri + "/");
347 return this;
348 }
349
350
351
352
353
354
355
356 public RepoCommand setGroups(String groups) {
357 this.groupsParam = groups;
358 return this;
359 }
360
361
362
363
364
365
366
367
368
369
370
371
372 public RepoCommand setBranch(String branch) {
373 this.branch = branch;
374 return this;
375 }
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391 public RepoCommand setTargetBranch(String branch) {
392 this.targetBranch = Constants.R_HEADS + branch;
393 return this;
394 }
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415 public RepoCommand setRecordRemoteBranch(boolean enable) {
416 this.recordRemoteBranch = enable;
417 return this;
418 }
419
420
421
422
423
424
425
426
427
428
429
430 public RepoCommand setRecordSubmoduleLabels(boolean enable) {
431 this.recordSubmoduleLabels = enable;
432 return this;
433 }
434
435
436
437
438
439
440
441
442
443
444
445 public RepoCommand setRecommendShallow(boolean enable) {
446 this.recordShallowSubmodules = enable;
447 return this;
448 }
449
450
451
452
453
454
455
456
457
458
459 public RepoCommand setProgressMonitor(ProgressMonitor monitor) {
460 this.monitor = monitor;
461 return this;
462 }
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479 public RepoCommand setIgnoreRemoteFailures(boolean ignore) {
480 this.ignoreRemoteFailures = ignore;
481 return this;
482 }
483
484
485
486
487
488
489
490
491
492
493
494 public RepoCommand setAuthor(PersonIdent author) {
495 this.author = author;
496 return this;
497 }
498
499
500
501
502
503
504
505
506
507
508
509 public RepoCommand setRemoteReader(RemoteReader callback) {
510 this.callback = callback;
511 return this;
512 }
513
514
515
516
517
518
519
520
521
522
523
524 public RepoCommand setIncludedFileReader(IncludedFileReader reader) {
525 this.includedReader = reader;
526 return this;
527 }
528
529
530 @Override
531 public RevCommit call() throws GitAPIException {
532 checkCallable();
533 if (baseUri == null) {
534 baseUri = "";
535 }
536 if (inputStream == null) {
537 if (manifestPath == null || manifestPath.length() == 0)
538 throw new IllegalArgumentException(
539 JGitText.get().pathNotConfigured);
540 try {
541 inputStream = new FileInputStream(manifestPath);
542 } catch (IOException e) {
543 throw new IllegalArgumentException(
544 JGitText.get().pathNotConfigured, e);
545 }
546 }
547
548 List<RepoProject> filteredProjects;
549 try {
550 ManifestParser parser = new ManifestParser(includedReader,
551 manifestPath, branch, baseUri, groupsParam, repo);
552 parser.read(inputStream);
553 filteredProjects = parser.getFilteredProjects();
554 } catch (IOException e) {
555 throw new ManifestErrorException(e);
556 } finally {
557 try {
558 inputStream.close();
559 } catch (IOException e) {
560
561 }
562 }
563
564 if (repo.isBare()) {
565 if (author == null)
566 author = new PersonIdent(repo);
567 if (callback == null)
568 callback = new DefaultRemoteReader();
569 List<RepoProject> renamedProjects = renameProjects(filteredProjects);
570
571 DirCache index = DirCache.newInCore();
572 DirCacheBuilder builder = index.builder();
573 ObjectInserter inserter = repo.newObjectInserter();
574 try (RevWalkRevWalk.html#RevWalk">RevWalk rw = new RevWalk(repo)) {
575 Config cfg = new Config();
576 StringBuilder attributes = new StringBuilder();
577 for (RepoProject proj : renamedProjects) {
578 String name = proj.getName();
579 String path = proj.getPath();
580 String url = proj.getUrl();
581 ObjectId objectId;
582 if (ObjectId.isId(proj.getRevision())) {
583 objectId = ObjectId.fromString(proj.getRevision());
584 } else {
585 objectId = callback.sha1(url, proj.getRevision());
586 if (objectId == null && !ignoreRemoteFailures) {
587 throw new RemoteUnavailableException(url);
588 }
589 if (recordRemoteBranch) {
590
591 cfg.setString("submodule", name, "branch",
592 proj.getRevision());
593 }
594
595 if (recordShallowSubmodules && proj.getRecommendShallow() != null) {
596
597
598
599
600
601 cfg.setBoolean("submodule", name, "shallow",
602 true);
603 }
604 }
605 if (recordSubmoduleLabels) {
606 StringBuilder rec = new StringBuilder();
607 rec.append("/");
608 rec.append(path);
609 for (String group : proj.getGroups()) {
610 rec.append(" ");
611 rec.append(group);
612 }
613 rec.append("\n");
614 attributes.append(rec.toString());
615 }
616
617 URI submodUrl = URI.create(url);
618 if (targetUri != null) {
619 submodUrl = relativize(targetUri, submodUrl);
620 }
621 cfg.setString("submodule", name, "path", path);
622 cfg.setString("submodule", name, "url",
623 submodUrl.toString());
624
625
626 if (objectId != null) {
627 DirCacheEntry dcEntry = new DirCacheEntry(path);
628 dcEntry.setObjectId(objectId);
629 dcEntry.setFileMode(FileMode.GITLINK);
630 builder.add(dcEntry);
631
632 for (CopyFile copyfile : proj.getCopyFiles()) {
633 RemoteFile rf = callback.readFileWithMode(
634 url, proj.getRevision(), copyfile.src);
635 objectId = inserter.insert(Constants.OBJ_BLOB,
636 rf.getContents());
637 dcEntry = new DirCacheEntry(copyfile.dest);
638 dcEntry.setObjectId(objectId);
639 dcEntry.setFileMode(rf.getFileMode());
640 builder.add(dcEntry);
641 }
642 for (LinkFile linkfile : proj.getLinkFiles()) {
643 String link;
644 if (linkfile.dest.contains("/")) {
645 link = FileUtils.relativizeGitPath(
646 linkfile.dest.substring(0,
647 linkfile.dest.lastIndexOf('/')),
648 proj.getPath() + "/" + linkfile.src);
649 } else {
650 link = proj.getPath() + "/" + linkfile.src;
651 }
652
653 objectId = inserter.insert(Constants.OBJ_BLOB,
654 link.getBytes(UTF_8));
655 dcEntry = new DirCacheEntry(linkfile.dest);
656 dcEntry.setObjectId(objectId);
657 dcEntry.setFileMode(FileMode.SYMLINK);
658 builder.add(dcEntry);
659 }
660 }
661 }
662 String content = cfg.toText();
663
664
665 final DirCacheEntrytry.html#DirCacheEntry">DirCacheEntry dcEntry = new DirCacheEntry(Constants.DOT_GIT_MODULES);
666 ObjectId objectId = inserter.insert(Constants.OBJ_BLOB,
667 content.getBytes(UTF_8));
668 dcEntry.setObjectId(objectId);
669 dcEntry.setFileMode(FileMode.REGULAR_FILE);
670 builder.add(dcEntry);
671
672 if (recordSubmoduleLabels) {
673
674 final DirCacheEntryhtml#DirCacheEntry">DirCacheEntry dcEntryAttr = new DirCacheEntry(Constants.DOT_GIT_ATTRIBUTES);
675 ObjectId attrId = inserter.insert(Constants.OBJ_BLOB,
676 attributes.toString().getBytes(UTF_8));
677 dcEntryAttr.setObjectId(attrId);
678 dcEntryAttr.setFileMode(FileMode.REGULAR_FILE);
679 builder.add(dcEntryAttr);
680 }
681
682 builder.finish();
683 ObjectId treeId = index.writeTree(inserter);
684
685
686 ObjectId headId = repo.resolve(targetBranch + "^{commit}");
687 if (headId != null && rw.parseCommit(headId).getTree().getId().equals(treeId)) {
688
689 return rw.parseCommit(headId);
690 }
691
692 CommitBuilder commit = new CommitBuilder();
693 commit.setTreeId(treeId);
694 if (headId != null)
695 commit.setParentIds(headId);
696 commit.setAuthor(author);
697 commit.setCommitter(author);
698 commit.setMessage(RepoText.get().repoCommitMessage);
699
700 ObjectId commitId = inserter.insert(commit);
701 inserter.flush();
702
703 RefUpdate ru = repo.updateRef(targetBranch);
704 ru.setNewObjectId(commitId);
705 ru.setExpectedOldObjectId(headId != null ? headId : ObjectId.zeroId());
706 Result rc = ru.update(rw);
707
708 switch (rc) {
709 case NEW:
710 case FORCED:
711 case FAST_FORWARD:
712
713 break;
714 case REJECTED:
715 case LOCK_FAILURE:
716 throw new ConcurrentRefUpdateException(
717 MessageFormat.format(
718 JGitText.get().cannotLock, targetBranch),
719 ru.getRef(),
720 rc);
721 default:
722 throw new JGitInternalException(MessageFormat.format(
723 JGitText.get().updatingRefFailed,
724 targetBranch, commitId.name(), rc));
725 }
726
727 return rw.parseCommit(commitId);
728 } catch (GitAPIException | IOException e) {
729 throw new ManifestErrorException(e);
730 }
731 }
732 try (Gitit.html#Git">Git git = new Git(repo)) {
733 for (RepoProject proj : filteredProjects) {
734 addSubmodule(proj.getName(), proj.getUrl(), proj.getPath(),
735 proj.getRevision(), proj.getCopyFiles(),
736 proj.getLinkFiles(), git);
737 }
738 return git.commit().setMessage(RepoText.get().repoCommitMessage)
739 .call();
740 } catch (GitAPIException | IOException e) {
741 throw new ManifestErrorException(e);
742 }
743 }
744
745 private void addSubmodule(String name, String url, String path,
746 String revision, List<CopyFile> copyfiles, List<LinkFile> linkfiles,
747 Git git) throws GitAPIException, IOException {
748 assert (!repo.isBare());
749 assert (git != null);
750 if (!linkfiles.isEmpty()) {
751 throw new UnsupportedOperationException(
752 JGitText.get().nonBareLinkFilesNotSupported);
753 }
754
755 SubmoduleAddCommand add = git.submoduleAdd().setName(name).setPath(path)
756 .setURI(url);
757 if (monitor != null)
758 add.setProgressMonitor(monitor);
759
760 Repository subRepo = add.call();
761 if (revision != null) {
762 try (Gitit.html#Git">Git sub = new Git(subRepo)) {
763 sub.checkout().setName(findRef(revision, subRepo)).call();
764 }
765 subRepo.close();
766 git.add().addFilepattern(path).call();
767 }
768 for (CopyFile copyfile : copyfiles) {
769 copyfile.copy();
770 git.add().addFilepattern(copyfile.dest).call();
771 }
772 }
773
774
775
776
777
778
779
780
781 private List<RepoProject> renameProjects(List<RepoProject> projects) {
782 Map<String, List<RepoProject>> m = new TreeMap<>();
783 for (RepoProject proj : projects) {
784 List<RepoProject> l = m.get(proj.getName());
785 if (l == null) {
786 l = new ArrayList<>();
787 m.put(proj.getName(), l);
788 }
789 l.add(proj);
790 }
791
792 List<RepoProject> ret = new ArrayList<>();
793 for (List<RepoProject> ps : m.values()) {
794 boolean nameConflict = ps.size() != 1;
795 for (RepoProject proj : ps) {
796 String name = proj.getName();
797 if (nameConflict) {
798 name += SLASH + proj.getPath();
799 }
800 RepoProject p = new RepoProject(name,
801 proj.getPath(), proj.getRevision(), null,
802 proj.getGroups(), proj.getRecommendShallow());
803 p.setUrl(proj.getUrl());
804 p.addCopyFiles(proj.getCopyFiles());
805 p.addLinkFiles(proj.getLinkFiles());
806 ret.add(p);
807 }
808 }
809 return ret;
810 }
811
812
813
814
815
816
817 private static final String SLASH = "/";
818 static URI relativize(URI current, URI target) {
819 if (!Objects.equals(current.getHost(), target.getHost())) {
820 return target;
821 }
822
823 String cur = current.normalize().getPath();
824 String dest = target.normalize().getPath();
825
826
827 if (cur.startsWith(SLASH) != dest.startsWith(SLASH)) {
828 return target;
829 }
830
831 while (cur.startsWith(SLASH)) {
832 cur = cur.substring(1);
833 }
834 while (dest.startsWith(SLASH)) {
835 dest = dest.substring(1);
836 }
837
838 if (cur.indexOf('/') == -1 || dest.indexOf('/') == -1) {
839
840 String prefix = "prefix/";
841 cur = prefix + cur;
842 dest = prefix + dest;
843 }
844
845 if (!cur.endsWith(SLASH)) {
846
847 int lastSlash = cur.lastIndexOf('/');
848 cur = cur.substring(0, lastSlash);
849 }
850 String destFile = "";
851 if (!dest.endsWith(SLASH)) {
852
853 int lastSlash = dest.lastIndexOf('/');
854 destFile = dest.substring(lastSlash + 1, dest.length());
855 dest = dest.substring(0, dest.lastIndexOf('/'));
856 }
857
858 String[] cs = cur.split(SLASH);
859 String[] ds = dest.split(SLASH);
860
861 int common = 0;
862 while (common < cs.length && common < ds.length && cs[common].equals(ds[common])) {
863 common++;
864 }
865
866 StringJoiner j = new StringJoiner(SLASH);
867 for (int i = common; i < cs.length; i++) {
868 j.add("..");
869 }
870 for (int i = common; i < ds.length; i++) {
871 j.add(ds[i]);
872 }
873
874 j.add(destFile);
875 return URI.create(j.toString());
876 }
877
878 private static String findRef(String ref, Repository repo)
879 throws IOException {
880 if (!ObjectId.isId(ref)) {
881 Ref r = repo.exactRef(R_REMOTES + DEFAULT_REMOTE_NAME + "/" + ref);
882 if (r != null)
883 return r.getName();
884 }
885 return ref;
886 }
887 }