1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46 package org.eclipse.jgit.transport;
47
48 import java.io.IOException;
49 import java.io.InputStream;
50 import java.io.OutputStream;
51 import java.text.MessageFormat;
52 import java.util.Collection;
53 import java.util.Collections;
54 import java.util.Date;
55 import java.util.Set;
56
57 import org.eclipse.jgit.errors.PackProtocolException;
58 import org.eclipse.jgit.errors.TransportException;
59 import org.eclipse.jgit.internal.JGitText;
60 import org.eclipse.jgit.internal.storage.file.PackLock;
61 import org.eclipse.jgit.lib.AnyObjectId;
62 import org.eclipse.jgit.lib.Config;
63 import org.eclipse.jgit.lib.Constants;
64 import org.eclipse.jgit.lib.MutableObjectId;
65 import org.eclipse.jgit.lib.NullProgressMonitor;
66 import org.eclipse.jgit.lib.ObjectId;
67 import org.eclipse.jgit.lib.ObjectInserter;
68 import org.eclipse.jgit.lib.ProgressMonitor;
69 import org.eclipse.jgit.lib.Ref;
70 import org.eclipse.jgit.revwalk.RevCommit;
71 import org.eclipse.jgit.revwalk.RevCommitList;
72 import org.eclipse.jgit.revwalk.RevFlag;
73 import org.eclipse.jgit.revwalk.RevObject;
74 import org.eclipse.jgit.revwalk.RevSort;
75 import org.eclipse.jgit.revwalk.RevWalk;
76 import org.eclipse.jgit.revwalk.filter.CommitTimeRevFilter;
77 import org.eclipse.jgit.revwalk.filter.RevFilter;
78 import org.eclipse.jgit.transport.GitProtocolConstants.MultiAck;
79 import org.eclipse.jgit.transport.PacketLineIn.AckNackResult;
80 import org.eclipse.jgit.util.TemporaryBuffer;
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104 public abstract class BasePackFetchConnection extends BasePackConnection
105 implements FetchConnection {
106
107
108
109
110
111
112
113 private static final int MAX_HAVES = 256;
114
115
116
117
118
119
120
121
122
123 protected static final int MIN_CLIENT_BUFFER = 2 * 32 * 46 + 8;
124
125
126
127
128
129 public static final String OPTION_INCLUDE_TAG = GitProtocolConstants.OPTION_INCLUDE_TAG;
130
131
132
133
134
135 public static final String OPTION_MULTI_ACK = GitProtocolConstants.OPTION_MULTI_ACK;
136
137
138
139
140
141 public static final String OPTION_MULTI_ACK_DETAILED = GitProtocolConstants.OPTION_MULTI_ACK_DETAILED;
142
143
144
145
146
147 public static final String OPTION_THIN_PACK = GitProtocolConstants.OPTION_THIN_PACK;
148
149
150
151
152
153 public static final String OPTION_SIDE_BAND = GitProtocolConstants.OPTION_SIDE_BAND;
154
155
156
157
158
159 public static final String OPTION_SIDE_BAND_64K = GitProtocolConstants.OPTION_SIDE_BAND_64K;
160
161
162
163
164
165 public static final String OPTION_OFS_DELTA = GitProtocolConstants.OPTION_OFS_DELTA;
166
167
168
169
170
171 public static final String OPTION_SHALLOW = GitProtocolConstants.OPTION_SHALLOW;
172
173
174
175
176
177 public static final String OPTION_NO_PROGRESS = GitProtocolConstants.OPTION_NO_PROGRESS;
178
179
180
181
182
183 public static final String OPTION_NO_DONE = GitProtocolConstants.OPTION_NO_DONE;
184
185
186
187
188
189
190 public static final String OPTION_ALLOW_TIP_SHA1_IN_WANT = GitProtocolConstants.OPTION_ALLOW_TIP_SHA1_IN_WANT;
191
192
193
194
195
196
197 public static final String OPTION_ALLOW_REACHABLE_SHA1_IN_WANT = GitProtocolConstants.OPTION_ALLOW_REACHABLE_SHA1_IN_WANT;
198
199
200
201
202
203
204 public static final String OPTION_FILTER = GitProtocolConstants.OPTION_FILTER;
205
206 private final RevWalk walk;
207
208
209 private RevCommitList<RevCommit> reachableCommits;
210
211
212 final RevFlag REACHABLE;
213
214
215 final RevFlag COMMON;
216
217
218 private final RevFlag STATE;
219
220
221 final RevFlag ADVERTISED;
222
223 private MultiAck multiAck = MultiAck.OFF;
224
225 private boolean thinPack;
226
227 private boolean sideband;
228
229 private boolean includeTags;
230
231 private boolean allowOfsDelta;
232
233 private boolean noDone;
234
235 private boolean noProgress;
236
237 private String lockMessage;
238
239 private PackLock packLock;
240
241 private int maxHaves;
242
243
244 private TemporaryBuffer.Heap state;
245
246 private PacketLineOut pckState;
247
248
249
250
251
252
253 private final FilterSpec filterSpec;
254
255
256
257
258
259
260
261 public BasePackFetchConnection(PackTransport packTransport) {
262 super(packTransport);
263
264 if (local != null) {
265 final FetchConfig cfg = getFetchConfig();
266 allowOfsDelta = cfg.allowOfsDelta;
267 maxHaves = cfg.maxHaves;
268 } else {
269 allowOfsDelta = true;
270 maxHaves = Integer.MAX_VALUE;
271 }
272
273 includeTags = transport.getTagOpt() != TagOpt.NO_TAGS;
274 thinPack = transport.isFetchThin();
275 filterSpec = transport.getFilterSpec();
276
277 if (local != null) {
278 walk = new RevWalk(local);
279 walk.setRetainBody(false);
280 reachableCommits = new RevCommitList<>();
281 REACHABLE = walk.newFlag("REACHABLE");
282 COMMON = walk.newFlag("COMMON");
283 STATE = walk.newFlag("STATE");
284 ADVERTISED = walk.newFlag("ADVERTISED");
285
286 walk.carry(COMMON);
287 walk.carry(REACHABLE);
288 walk.carry(ADVERTISED);
289 } else {
290 walk = null;
291 REACHABLE = null;
292 COMMON = null;
293 STATE = null;
294 ADVERTISED = null;
295 }
296 }
297
298 static class FetchConfig {
299 final boolean allowOfsDelta;
300
301 final int maxHaves;
302
303 FetchConfig(Config c) {
304 allowOfsDelta = c.getBoolean("repack", "usedeltabaseoffset", true);
305 maxHaves = c.getInt("fetch", "maxhaves", Integer.MAX_VALUE);
306 }
307
308 FetchConfig(boolean allowOfsDelta, int maxHaves) {
309 this.allowOfsDelta = allowOfsDelta;
310 this.maxHaves = maxHaves;
311 }
312 }
313
314
315 @Override
316 public final void fetch(final ProgressMonitor monitor,
317 final Collection<Ref> want, final Set<ObjectId> have)
318 throws TransportException {
319 fetch(monitor, want, have, null);
320 }
321
322
323 @Override
324 public final void fetch(final ProgressMonitor monitor,
325 final Collection<Ref> want, final Set<ObjectId> have,
326 OutputStream outputStream) throws TransportException {
327 markStartedOperation();
328 doFetch(monitor, want, have, outputStream);
329 }
330
331
332 @Override
333 public boolean didFetchIncludeTags() {
334 return false;
335 }
336
337
338 @Override
339 public boolean didFetchTestConnectivity() {
340 return false;
341 }
342
343
344 @Override
345 public void setPackLockMessage(String message) {
346 lockMessage = message;
347 }
348
349
350 @Override
351 public Collection<PackLock> getPackLocks() {
352 if (packLock != null)
353 return Collections.singleton(packLock);
354 return Collections.<PackLock> emptyList();
355 }
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376 protected void doFetch(final ProgressMonitor monitor,
377 final Collection<Ref> want, final Set<ObjectId> have,
378 OutputStream outputStream) throws TransportException {
379 try {
380 noProgress = monitor == NullProgressMonitor.INSTANCE;
381
382 markRefsAdvertised();
383 markReachable(have, maxTimeWanted(want));
384
385 if (statelessRPC) {
386 state = new TemporaryBuffer.Heap(Integer.MAX_VALUE);
387 pckState = new PacketLineOut(state);
388 }
389
390 if (sendWants(want)) {
391 negotiate(monitor);
392
393 walk.dispose();
394 reachableCommits = null;
395 state = null;
396 pckState = null;
397
398 receivePack(monitor, outputStream);
399 }
400 } catch (CancelledException ce) {
401 close();
402 return;
403 } catch (IOException | RuntimeException err) {
404 close();
405 throw new TransportException(err.getMessage(), err);
406 }
407 }
408
409
410 @Override
411 public void close() {
412 if (walk != null)
413 walk.close();
414 super.close();
415 }
416
417 FetchConfig getFetchConfig() {
418 return local.getConfig().get(FetchConfig::new);
419 }
420
421 private int maxTimeWanted(Collection<Ref> wants) {
422 int maxTime = 0;
423 for (Ref r : wants) {
424 try {
425 final RevObject obj = walk.parseAny(r.getObjectId());
426 if (obj instanceof RevCommit) {
427 final int cTime = ((RevCommit) obj).getCommitTime();
428 if (maxTime < cTime)
429 maxTime = cTime;
430 }
431 } catch (IOException error) {
432
433 }
434 }
435 return maxTime;
436 }
437
438 private void markReachable(Set<ObjectId> have, int maxTime)
439 throws IOException {
440 for (Ref r : local.getRefDatabase().getRefs()) {
441 ObjectId id = r.getPeeledObjectId();
442 if (id == null)
443 id = r.getObjectId();
444 if (id == null)
445 continue;
446 parseReachable(id);
447 }
448
449 for (ObjectId id : local.getAdditionalHaves())
450 parseReachable(id);
451
452 for (ObjectId id : have)
453 parseReachable(id);
454
455 if (maxTime > 0) {
456
457
458
459
460 final Date maxWhen = new Date(maxTime * 1000L);
461 walk.sort(RevSort.COMMIT_TIME_DESC);
462 walk.markStart(reachableCommits);
463 walk.setRevFilter(CommitTimeRevFilter.after(maxWhen));
464 for (;;) {
465 final RevCommit c = walk.next();
466 if (c == null)
467 break;
468 if (c.has(ADVERTISED) && !c.has(COMMON)) {
469
470
471
472 c.add(COMMON);
473 c.carry(COMMON);
474 reachableCommits.add(c);
475 }
476 }
477 }
478 }
479
480 private void parseReachable(ObjectId id) {
481 try {
482 RevCommit o = walk.parseCommit(id);
483 if (!o.has(REACHABLE)) {
484 o.add(REACHABLE);
485 reachableCommits.add(o);
486 }
487 } catch (IOException readError) {
488
489 }
490 }
491
492 private boolean sendWants(Collection<Ref> want) throws IOException {
493 final PacketLineOut p = statelessRPC ? pckState : pckOut;
494 boolean first = true;
495 for (Ref r : want) {
496 ObjectId objectId = r.getObjectId();
497 if (objectId == null) {
498 continue;
499 }
500 try {
501 if (walk.parseAny(objectId).has(REACHABLE)) {
502
503
504
505 continue;
506 }
507 } catch (IOException err) {
508
509
510 }
511
512 final StringBuilder line = new StringBuilder(46);
513 line.append("want ");
514 line.append(objectId.name());
515 if (first) {
516 line.append(enableCapabilities());
517 first = false;
518 }
519 line.append('\n');
520 p.writeString(line.toString());
521 }
522 if (first) {
523 return false;
524 }
525 if (!filterSpec.isNoOp()) {
526 p.writeString(filterSpec.filterLine());
527 }
528 p.end();
529 outNeedsEnd = false;
530 return true;
531 }
532
533 private String enableCapabilities() throws TransportException {
534 final StringBuilder line = new StringBuilder();
535 if (noProgress)
536 wantCapability(line, OPTION_NO_PROGRESS);
537 if (includeTags)
538 includeTags = wantCapability(line, OPTION_INCLUDE_TAG);
539 if (allowOfsDelta)
540 wantCapability(line, OPTION_OFS_DELTA);
541
542 if (wantCapability(line, OPTION_MULTI_ACK_DETAILED)) {
543 multiAck = MultiAck.DETAILED;
544 if (statelessRPC)
545 noDone = wantCapability(line, OPTION_NO_DONE);
546 } else if (wantCapability(line, OPTION_MULTI_ACK))
547 multiAck = MultiAck.CONTINUE;
548 else
549 multiAck = MultiAck.OFF;
550
551 if (thinPack)
552 thinPack = wantCapability(line, OPTION_THIN_PACK);
553 if (wantCapability(line, OPTION_SIDE_BAND_64K))
554 sideband = true;
555 else if (wantCapability(line, OPTION_SIDE_BAND))
556 sideband = true;
557
558 if (statelessRPC && multiAck != MultiAck.DETAILED) {
559
560
561
562
563 throw new PackProtocolException(uri, MessageFormat.format(
564 JGitText.get().statelessRPCRequiresOptionToBeEnabled,
565 OPTION_MULTI_ACK_DETAILED));
566 }
567
568 if (!filterSpec.isNoOp() && !wantCapability(line, OPTION_FILTER)) {
569 throw new PackProtocolException(uri,
570 JGitText.get().filterRequiresCapability);
571 }
572
573 addUserAgentCapability(line);
574 return line.toString();
575 }
576
577 private void negotiate(ProgressMonitor monitor) throws IOException,
578 CancelledException {
579 final MutableObjectId.html#MutableObjectId">MutableObjectId ackId = new MutableObjectId();
580 int resultsPending = 0;
581 int havesSent = 0;
582 int havesSinceLastContinue = 0;
583 boolean receivedContinue = false;
584 boolean receivedAck = false;
585 boolean receivedReady = false;
586
587 if (statelessRPC) {
588 state.writeTo(out, null);
589 }
590
591 negotiateBegin();
592 SEND_HAVES: for (;;) {
593 final RevCommit c = walk.next();
594 if (c == null) {
595 break SEND_HAVES;
596 }
597
598 ObjectId o = c.getId();
599 pckOut.writeString("have " + o.name() + "\n");
600 havesSent++;
601 havesSinceLastContinue++;
602
603 if ((31 & havesSent) != 0) {
604
605
606
607
608 continue;
609 }
610
611 if (monitor.isCancelled()) {
612 throw new CancelledException();
613 }
614
615 pckOut.end();
616 resultsPending++;
617
618 if (havesSent == 32 && !statelessRPC) {
619
620
621
622
623
624 continue;
625 }
626
627 READ_RESULT: for (;;) {
628 final AckNackResult anr = pckIn.readACK(ackId);
629 switch (anr) {
630 case NAK:
631
632
633
634 resultsPending--;
635 break READ_RESULT;
636
637 case ACK:
638
639
640
641
642 multiAck = MultiAck.OFF;
643 resultsPending = 0;
644 receivedAck = true;
645 if (statelessRPC) {
646 state.writeTo(out, null);
647 }
648 break SEND_HAVES;
649
650 case ACK_CONTINUE:
651 case ACK_COMMON:
652 case ACK_READY:
653
654
655
656
657
658 markCommon(walk.parseAny(ackId), anr);
659 receivedAck = true;
660 receivedContinue = true;
661 havesSinceLastContinue = 0;
662 if (anr == AckNackResult.ACK_READY) {
663 receivedReady = true;
664 }
665 break;
666 }
667
668 if (monitor.isCancelled()) {
669 throw new CancelledException();
670 }
671 }
672
673 if (noDone & receivedReady) {
674 break SEND_HAVES;
675 }
676 if (statelessRPC) {
677 state.writeTo(out, null);
678 }
679
680 if ((receivedContinue && havesSinceLastContinue > MAX_HAVES)
681 || havesSent >= maxHaves) {
682
683
684
685
686
687 break SEND_HAVES;
688 }
689 }
690
691
692
693 if (monitor.isCancelled()) {
694 throw new CancelledException();
695 }
696
697 if (!receivedReady || !noDone) {
698
699
700
701
702 pckOut.writeString("done\n");
703 pckOut.flush();
704 }
705
706 if (!receivedAck) {
707
708
709
710
711 multiAck = MultiAck.OFF;
712 resultsPending++;
713 }
714
715 READ_RESULT: while (resultsPending > 0 || multiAck != MultiAck.OFF) {
716 final AckNackResult anr = pckIn.readACK(ackId);
717 resultsPending--;
718 switch (anr) {
719 case NAK:
720
721
722
723 break;
724
725 case ACK:
726
727
728
729 break READ_RESULT;
730
731 case ACK_CONTINUE:
732 case ACK_COMMON:
733 case ACK_READY:
734
735
736 multiAck = MultiAck.CONTINUE;
737 break;
738 }
739
740 if (monitor.isCancelled()) {
741 throw new CancelledException();
742 }
743 }
744 }
745
746 private void negotiateBegin() throws IOException {
747 walk.resetRetain(REACHABLE, ADVERTISED);
748 walk.markStart(reachableCommits);
749 walk.sort(RevSort.COMMIT_TIME_DESC);
750 walk.setRevFilter(new RevFilter() {
751 @Override
752 public RevFilter clone() {
753 return this;
754 }
755
756 @Override
757 public boolean include(RevWalk walker, RevCommit c) {
758 final boolean remoteKnowsIsCommon = c.has(COMMON);
759 if (c.has(ADVERTISED)) {
760
761
762
763
764
765 c.add(COMMON);
766 }
767 return !remoteKnowsIsCommon;
768 }
769
770 @Override
771 public boolean requiresCommitBody() {
772 return false;
773 }
774 });
775 }
776
777 private void markRefsAdvertised() {
778 for (Ref r : getRefs()) {
779 markAdvertised(r.getObjectId());
780 if (r.getPeeledObjectId() != null)
781 markAdvertised(r.getPeeledObjectId());
782 }
783 }
784
785 private void markAdvertised(AnyObjectId id) {
786 try {
787 walk.parseAny(id).add(ADVERTISED);
788 } catch (IOException readError) {
789
790 }
791 }
792
793 private void markCommon(RevObject obj, AckNackResult anr)
794 throws IOException {
795 if (statelessRPC && anr == AckNackResult.ACK_COMMON && !obj.has(STATE)) {
796 StringBuilder s;
797
798 s = new StringBuilder(6 + Constants.OBJECT_ID_STRING_LENGTH);
799 s.append("have ");
800 s.append(obj.name());
801 s.append('\n');
802 pckState.writeString(s.toString());
803 obj.add(STATE);
804 }
805 obj.add(COMMON);
806 if (obj instanceof RevCommit)
807 ((RevCommit) obj).carry(COMMON);
808 }
809
810 private void receivePack(final ProgressMonitor monitor,
811 OutputStream outputStream) throws IOException {
812 onReceivePack();
813 InputStream input = in;
814 if (sideband)
815 input = new SideBandInputStream(input, monitor, getMessageWriter(),
816 outputStream);
817
818 try (ObjectInserter ins = local.newObjectInserter()) {
819 PackParser parser = ins.newPackParser(input);
820 parser.setAllowThin(thinPack);
821 parser.setObjectChecker(transport.getObjectChecker());
822 parser.setLockMessage(lockMessage);
823 packLock = parser.parse(monitor);
824 ins.flush();
825 }
826 }
827
828
829
830
831
832
833
834
835 protected void onReceivePack() {
836
837 }
838
839 private static class CancelledException extends Exception {
840 private static final long serialVersionUID = 1L;
841 }
842 }