1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46 package org.eclipse.jgit.transport;
47
48 import java.io.IOException;
49 import java.io.InputStream;
50 import java.io.OutputStream;
51 import java.text.MessageFormat;
52 import java.util.Collection;
53 import java.util.Collections;
54 import java.util.Date;
55 import java.util.Set;
56
57 import org.eclipse.jgit.errors.PackProtocolException;
58 import org.eclipse.jgit.errors.TransportException;
59 import org.eclipse.jgit.internal.JGitText;
60 import org.eclipse.jgit.internal.storage.file.PackLock;
61 import org.eclipse.jgit.lib.AnyObjectId;
62 import org.eclipse.jgit.lib.Config;
63 import org.eclipse.jgit.lib.Constants;
64 import org.eclipse.jgit.lib.MutableObjectId;
65 import org.eclipse.jgit.lib.NullProgressMonitor;
66 import org.eclipse.jgit.lib.ObjectId;
67 import org.eclipse.jgit.lib.ObjectInserter;
68 import org.eclipse.jgit.lib.ProgressMonitor;
69 import org.eclipse.jgit.lib.Ref;
70 import org.eclipse.jgit.revwalk.RevCommit;
71 import org.eclipse.jgit.revwalk.RevCommitList;
72 import org.eclipse.jgit.revwalk.RevFlag;
73 import org.eclipse.jgit.revwalk.RevObject;
74 import org.eclipse.jgit.revwalk.RevSort;
75 import org.eclipse.jgit.revwalk.RevWalk;
76 import org.eclipse.jgit.revwalk.filter.CommitTimeRevFilter;
77 import org.eclipse.jgit.revwalk.filter.RevFilter;
78 import org.eclipse.jgit.transport.GitProtocolConstants.MultiAck;
79 import org.eclipse.jgit.transport.PacketLineIn.AckNackResult;
80 import org.eclipse.jgit.util.TemporaryBuffer;
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104 public abstract class BasePackFetchConnection extends BasePackConnection
105 implements FetchConnection {
106
107
108
109
110
111
112
113 private static final int MAX_HAVES = 256;
114
115
116
117
118
119
120
121
122
123 protected static final int MIN_CLIENT_BUFFER = 2 * 32 * 46 + 8;
124
125
126
127
128
129 public static final String OPTION_INCLUDE_TAG = GitProtocolConstants.OPTION_INCLUDE_TAG;
130
131
132
133
134
135 public static final String OPTION_MULTI_ACK = GitProtocolConstants.OPTION_MULTI_ACK;
136
137
138
139
140
141 public static final String OPTION_MULTI_ACK_DETAILED = GitProtocolConstants.OPTION_MULTI_ACK_DETAILED;
142
143
144
145
146
147 public static final String OPTION_THIN_PACK = GitProtocolConstants.OPTION_THIN_PACK;
148
149
150
151
152
153 public static final String OPTION_SIDE_BAND = GitProtocolConstants.OPTION_SIDE_BAND;
154
155
156
157
158
159 public static final String OPTION_SIDE_BAND_64K = GitProtocolConstants.OPTION_SIDE_BAND_64K;
160
161
162
163
164
165 public static final String OPTION_OFS_DELTA = GitProtocolConstants.OPTION_OFS_DELTA;
166
167
168
169
170
171 public static final String OPTION_SHALLOW = GitProtocolConstants.OPTION_SHALLOW;
172
173
174
175
176
177 public static final String OPTION_NO_PROGRESS = GitProtocolConstants.OPTION_NO_PROGRESS;
178
179
180
181
182
183 public static final String OPTION_NO_DONE = GitProtocolConstants.OPTION_NO_DONE;
184
185
186
187
188
189
190 public static final String OPTION_ALLOW_TIP_SHA1_IN_WANT = GitProtocolConstants.OPTION_ALLOW_TIP_SHA1_IN_WANT;
191
192
193
194
195
196
197 public static final String OPTION_ALLOW_REACHABLE_SHA1_IN_WANT = GitProtocolConstants.OPTION_ALLOW_REACHABLE_SHA1_IN_WANT;
198
199
200
201
202
203
204 public static final String OPTION_FILTER = GitProtocolConstants.OPTION_FILTER;
205
206 private final RevWalk walk;
207
208
209 private RevCommitList<RevCommit> reachableCommits;
210
211
212 final RevFlag REACHABLE;
213
214
215 final RevFlag COMMON;
216
217
218 private final RevFlag STATE;
219
220
221 final RevFlag ADVERTISED;
222
223 private MultiAck multiAck = MultiAck.OFF;
224
225 private boolean thinPack;
226
227 private boolean sideband;
228
229 private boolean includeTags;
230
231 private boolean allowOfsDelta;
232
233 private boolean noDone;
234
235 private boolean noProgress;
236
237 private String lockMessage;
238
239 private PackLock packLock;
240
241 private int maxHaves;
242
243
244 private TemporaryBuffer.Heap state;
245
246 private PacketLineOut pckState;
247
248
249 private final long filterBlobLimit;
250
251
252
253
254
255
256
257 public BasePackFetchConnection(PackTransport packTransport) {
258 super(packTransport);
259
260 if (local != null) {
261 final FetchConfig cfg = getFetchConfig();
262 allowOfsDelta = cfg.allowOfsDelta;
263 maxHaves = cfg.maxHaves;
264 } else {
265 allowOfsDelta = true;
266 maxHaves = Integer.MAX_VALUE;
267 }
268
269 includeTags = transport.getTagOpt() != TagOpt.NO_TAGS;
270 thinPack = transport.isFetchThin();
271 filterBlobLimit = transport.getFilterBlobLimit();
272
273 if (local != null) {
274 walk = new RevWalk(local);
275 reachableCommits = new RevCommitList<>();
276 REACHABLE = walk.newFlag("REACHABLE");
277 COMMON = walk.newFlag("COMMON");
278 STATE = walk.newFlag("STATE");
279 ADVERTISED = walk.newFlag("ADVERTISED");
280
281 walk.carry(COMMON);
282 walk.carry(REACHABLE);
283 walk.carry(ADVERTISED);
284 } else {
285 walk = null;
286 REACHABLE = null;
287 COMMON = null;
288 STATE = null;
289 ADVERTISED = null;
290 }
291 }
292
293 static class FetchConfig {
294 final boolean allowOfsDelta;
295
296 final int maxHaves;
297
298 FetchConfig(Config c) {
299 allowOfsDelta = c.getBoolean("repack", "usedeltabaseoffset", true);
300 maxHaves = c.getInt("fetch", "maxhaves", Integer.MAX_VALUE);
301 }
302
303 FetchConfig(boolean allowOfsDelta, int maxHaves) {
304 this.allowOfsDelta = allowOfsDelta;
305 this.maxHaves = maxHaves;
306 }
307 }
308
309
310 @Override
311 public final void fetch(final ProgressMonitor monitor,
312 final Collection<Ref> want, final Set<ObjectId> have)
313 throws TransportException {
314 fetch(monitor, want, have, null);
315 }
316
317
318 @Override
319 public final void fetch(final ProgressMonitor monitor,
320 final Collection<Ref> want, final Set<ObjectId> have,
321 OutputStream outputStream) throws TransportException {
322 markStartedOperation();
323 doFetch(monitor, want, have, outputStream);
324 }
325
326
327 @Override
328 public boolean didFetchIncludeTags() {
329 return false;
330 }
331
332
333 @Override
334 public boolean didFetchTestConnectivity() {
335 return false;
336 }
337
338
339 @Override
340 public void setPackLockMessage(String message) {
341 lockMessage = message;
342 }
343
344
345 @Override
346 public Collection<PackLock> getPackLocks() {
347 if (packLock != null)
348 return Collections.singleton(packLock);
349 return Collections.<PackLock> emptyList();
350 }
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371 protected void doFetch(final ProgressMonitor monitor,
372 final Collection<Ref> want, final Set<ObjectId> have,
373 OutputStream outputStream) throws TransportException {
374 try {
375 noProgress = monitor == NullProgressMonitor.INSTANCE;
376
377 markRefsAdvertised();
378 markReachable(have, maxTimeWanted(want));
379
380 if (statelessRPC) {
381 state = new TemporaryBuffer.Heap(Integer.MAX_VALUE);
382 pckState = new PacketLineOut(state);
383 }
384
385 if (sendWants(want)) {
386 negotiate(monitor);
387
388 walk.dispose();
389 reachableCommits = null;
390 state = null;
391 pckState = null;
392
393 receivePack(monitor, outputStream);
394 }
395 } catch (CancelledException ce) {
396 close();
397 return;
398 } catch (IOException err) {
399 close();
400 throw new TransportException(err.getMessage(), err);
401 } catch (RuntimeException err) {
402 close();
403 throw new TransportException(err.getMessage(), err);
404 }
405 }
406
407
408 @Override
409 public void close() {
410 if (walk != null)
411 walk.close();
412 super.close();
413 }
414
415 FetchConfig getFetchConfig() {
416 return local.getConfig().get(FetchConfig::new);
417 }
418
419 private int maxTimeWanted(Collection<Ref> wants) {
420 int maxTime = 0;
421 for (Ref r : wants) {
422 try {
423 final RevObject obj = walk.parseAny(r.getObjectId());
424 if (obj instanceof RevCommit) {
425 final int cTime = ((RevCommit) obj).getCommitTime();
426 if (maxTime < cTime)
427 maxTime = cTime;
428 }
429 } catch (IOException error) {
430
431 }
432 }
433 return maxTime;
434 }
435
436 private void markReachable(Set<ObjectId> have, int maxTime)
437 throws IOException {
438 for (Ref r : local.getRefDatabase().getRefs()) {
439 ObjectId id = r.getPeeledObjectId();
440 if (id == null)
441 id = r.getObjectId();
442 if (id == null)
443 continue;
444 parseReachable(id);
445 }
446
447 for (ObjectId id : local.getAdditionalHaves())
448 parseReachable(id);
449
450 for (ObjectId id : have)
451 parseReachable(id);
452
453 if (maxTime > 0) {
454
455
456
457
458 final Date maxWhen = new Date(maxTime * 1000L);
459 walk.sort(RevSort.COMMIT_TIME_DESC);
460 walk.markStart(reachableCommits);
461 walk.setRevFilter(CommitTimeRevFilter.after(maxWhen));
462 for (;;) {
463 final RevCommit c = walk.next();
464 if (c == null)
465 break;
466 if (c.has(ADVERTISED) && !c.has(COMMON)) {
467
468
469
470 c.add(COMMON);
471 c.carry(COMMON);
472 reachableCommits.add(c);
473 }
474 }
475 }
476 }
477
478 private void parseReachable(ObjectId id) {
479 try {
480 RevCommit o = walk.parseCommit(id);
481 if (!o.has(REACHABLE)) {
482 o.add(REACHABLE);
483 reachableCommits.add(o);
484 }
485 } catch (IOException readError) {
486
487 }
488 }
489
490 private boolean sendWants(Collection<Ref> want) throws IOException {
491 final PacketLineOut p = statelessRPC ? pckState : pckOut;
492 boolean first = true;
493 for (Ref r : want) {
494 ObjectId objectId = r.getObjectId();
495 if (objectId == null) {
496 continue;
497 }
498 try {
499 if (walk.parseAny(objectId).has(REACHABLE)) {
500
501
502
503 continue;
504 }
505 } catch (IOException err) {
506
507
508 }
509
510 final StringBuilder line = new StringBuilder(46);
511 line.append("want ");
512 line.append(objectId.name());
513 if (first) {
514 line.append(enableCapabilities());
515 first = false;
516 }
517 line.append('\n');
518 p.writeString(line.toString());
519 }
520 if (first) {
521 return false;
522 }
523 if (filterBlobLimit == 0) {
524 p.writeString(OPTION_FILTER + " blob:none");
525 } else if (filterBlobLimit > 0) {
526 p.writeString(OPTION_FILTER + " blob:limit=" + filterBlobLimit);
527 }
528 p.end();
529 outNeedsEnd = false;
530 return true;
531 }
532
533 private String enableCapabilities() throws TransportException {
534 final StringBuilder line = new StringBuilder();
535 if (noProgress)
536 wantCapability(line, OPTION_NO_PROGRESS);
537 if (includeTags)
538 includeTags = wantCapability(line, OPTION_INCLUDE_TAG);
539 if (allowOfsDelta)
540 wantCapability(line, OPTION_OFS_DELTA);
541
542 if (wantCapability(line, OPTION_MULTI_ACK_DETAILED)) {
543 multiAck = MultiAck.DETAILED;
544 if (statelessRPC)
545 noDone = wantCapability(line, OPTION_NO_DONE);
546 } else if (wantCapability(line, OPTION_MULTI_ACK))
547 multiAck = MultiAck.CONTINUE;
548 else
549 multiAck = MultiAck.OFF;
550
551 if (thinPack)
552 thinPack = wantCapability(line, OPTION_THIN_PACK);
553 if (wantCapability(line, OPTION_SIDE_BAND_64K))
554 sideband = true;
555 else if (wantCapability(line, OPTION_SIDE_BAND))
556 sideband = true;
557
558 if (statelessRPC && multiAck != MultiAck.DETAILED) {
559
560
561
562
563 throw new PackProtocolException(uri, MessageFormat.format(
564 JGitText.get().statelessRPCRequiresOptionToBeEnabled,
565 OPTION_MULTI_ACK_DETAILED));
566 }
567
568 if (filterBlobLimit >= 0 && !wantCapability(line, OPTION_FILTER)) {
569 throw new PackProtocolException(uri,
570 JGitText.get().filterRequiresCapability);
571 }
572
573 addUserAgentCapability(line);
574 return line.toString();
575 }
576
577 private void negotiate(ProgressMonitor monitor) throws IOException,
578 CancelledException {
579 final MutableObjectId ackId = new MutableObjectId();
580 int resultsPending = 0;
581 int havesSent = 0;
582 int havesSinceLastContinue = 0;
583 boolean receivedContinue = false;
584 boolean receivedAck = false;
585 boolean receivedReady = false;
586
587 if (statelessRPC) {
588 state.writeTo(out, null);
589 }
590
591 negotiateBegin();
592 SEND_HAVES: for (;;) {
593 final RevCommit c = walk.next();
594 if (c == null) {
595 break SEND_HAVES;
596 }
597
598 ObjectId o = c.getId();
599 pckOut.writeString("have " + o.name() + "\n");
600 havesSent++;
601 havesSinceLastContinue++;
602
603 if ((31 & havesSent) != 0) {
604
605
606
607
608 continue;
609 }
610
611 if (monitor.isCancelled()) {
612 throw new CancelledException();
613 }
614
615 pckOut.end();
616 resultsPending++;
617
618 if (havesSent == 32 && !statelessRPC) {
619
620
621
622
623
624 continue;
625 }
626
627 READ_RESULT: for (;;) {
628 final AckNackResult anr = pckIn.readACK(ackId);
629 switch (anr) {
630 case NAK:
631
632
633
634 resultsPending--;
635 break READ_RESULT;
636
637 case ACK:
638
639
640
641
642 multiAck = MultiAck.OFF;
643 resultsPending = 0;
644 receivedAck = true;
645 if (statelessRPC) {
646 state.writeTo(out, null);
647 }
648 break SEND_HAVES;
649
650 case ACK_CONTINUE:
651 case ACK_COMMON:
652 case ACK_READY:
653
654
655
656
657
658 markCommon(walk.parseAny(ackId), anr);
659 receivedAck = true;
660 receivedContinue = true;
661 havesSinceLastContinue = 0;
662 if (anr == AckNackResult.ACK_READY) {
663 receivedReady = true;
664 }
665 break;
666 }
667
668 if (monitor.isCancelled()) {
669 throw new CancelledException();
670 }
671 }
672
673 if (noDone & receivedReady) {
674 break SEND_HAVES;
675 }
676 if (statelessRPC) {
677 state.writeTo(out, null);
678 }
679
680 if (receivedContinue && havesSinceLastContinue > MAX_HAVES
681 || havesSent >= maxHaves) {
682
683
684
685
686
687 break SEND_HAVES;
688 }
689 }
690
691
692
693 if (monitor.isCancelled()) {
694 throw new CancelledException();
695 }
696
697 if (!receivedReady || !noDone) {
698
699
700
701
702 pckOut.writeString("done\n");
703 pckOut.flush();
704 }
705
706 if (!receivedAck) {
707
708
709
710
711 multiAck = MultiAck.OFF;
712 resultsPending++;
713 }
714
715 READ_RESULT: while (resultsPending > 0 || multiAck != MultiAck.OFF) {
716 final AckNackResult anr = pckIn.readACK(ackId);
717 resultsPending--;
718 switch (anr) {
719 case NAK:
720
721
722
723 break;
724
725 case ACK:
726
727
728
729 break READ_RESULT;
730
731 case ACK_CONTINUE:
732 case ACK_COMMON:
733 case ACK_READY:
734
735
736 multiAck = MultiAck.CONTINUE;
737 break;
738 }
739
740 if (monitor.isCancelled()) {
741 throw new CancelledException();
742 }
743 }
744 }
745
746 private void negotiateBegin() throws IOException {
747 walk.resetRetain(REACHABLE, ADVERTISED);
748 walk.markStart(reachableCommits);
749 walk.sort(RevSort.COMMIT_TIME_DESC);
750 walk.setRevFilter(new RevFilter() {
751 @Override
752 public RevFilter clone() {
753 return this;
754 }
755
756 @Override
757 public boolean include(RevWalk walker, RevCommit c) {
758 final boolean remoteKnowsIsCommon = c.has(COMMON);
759 if (c.has(ADVERTISED)) {
760
761
762
763
764
765 c.add(COMMON);
766 }
767 return !remoteKnowsIsCommon;
768 }
769
770 @Override
771 public boolean requiresCommitBody() {
772 return false;
773 }
774 });
775 }
776
777 private void markRefsAdvertised() {
778 for (Ref r : getRefs()) {
779 markAdvertised(r.getObjectId());
780 if (r.getPeeledObjectId() != null)
781 markAdvertised(r.getPeeledObjectId());
782 }
783 }
784
785 private void markAdvertised(AnyObjectId id) {
786 try {
787 walk.parseAny(id).add(ADVERTISED);
788 } catch (IOException readError) {
789
790 }
791 }
792
793 private void markCommon(RevObject obj, AckNackResult anr)
794 throws IOException {
795 if (statelessRPC && anr == AckNackResult.ACK_COMMON && !obj.has(STATE)) {
796 StringBuilder s;
797
798 s = new StringBuilder(6 + Constants.OBJECT_ID_STRING_LENGTH);
799 s.append("have ");
800 s.append(obj.name());
801 s.append('\n');
802 pckState.writeString(s.toString());
803 obj.add(STATE);
804 }
805 obj.add(COMMON);
806 if (obj instanceof RevCommit)
807 ((RevCommit) obj).carry(COMMON);
808 }
809
810 private void receivePack(final ProgressMonitor monitor,
811 OutputStream outputStream) throws IOException {
812 onReceivePack();
813 InputStream input = in;
814 if (sideband)
815 input = new SideBandInputStream(input, monitor, getMessageWriter(),
816 outputStream);
817
818 try (ObjectInserter ins = local.newObjectInserter()) {
819 PackParser parser = ins.newPackParser(input);
820 parser.setAllowThin(thinPack);
821 parser.setObjectChecker(transport.getObjectChecker());
822 parser.setLockMessage(lockMessage);
823 packLock = parser.parse(monitor);
824 ins.flush();
825 }
826 }
827
828
829
830
831
832
833
834
835 protected void onReceivePack() {
836
837 }
838
839 private static class CancelledException extends Exception {
840 private static final long serialVersionUID = 1L;
841 }
842 }