1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46 package org.eclipse.jgit.transport;
47
48 import java.io.IOException;
49 import java.io.InputStream;
50 import java.io.OutputStream;
51 import java.text.MessageFormat;
52 import java.util.Collection;
53 import java.util.Collections;
54 import java.util.Date;
55 import java.util.HashSet;
56 import java.util.Set;
57
58 import org.eclipse.jgit.errors.PackProtocolException;
59 import org.eclipse.jgit.errors.TransportException;
60 import org.eclipse.jgit.internal.JGitText;
61 import org.eclipse.jgit.internal.storage.file.PackLock;
62 import org.eclipse.jgit.lib.AnyObjectId;
63 import org.eclipse.jgit.lib.Config;
64 import org.eclipse.jgit.lib.Constants;
65 import org.eclipse.jgit.lib.MutableObjectId;
66 import org.eclipse.jgit.lib.NullProgressMonitor;
67 import org.eclipse.jgit.lib.ObjectId;
68 import org.eclipse.jgit.lib.ObjectInserter;
69 import org.eclipse.jgit.lib.ProgressMonitor;
70 import org.eclipse.jgit.lib.Ref;
71 import org.eclipse.jgit.revwalk.RevCommit;
72 import org.eclipse.jgit.revwalk.RevCommitList;
73 import org.eclipse.jgit.revwalk.RevFlag;
74 import org.eclipse.jgit.revwalk.RevObject;
75 import org.eclipse.jgit.revwalk.RevSort;
76 import org.eclipse.jgit.revwalk.RevWalk;
77 import org.eclipse.jgit.revwalk.filter.CommitTimeRevFilter;
78 import org.eclipse.jgit.revwalk.filter.RevFilter;
79 import org.eclipse.jgit.transport.GitProtocolConstants.MultiAck;
80 import org.eclipse.jgit.transport.PacketLineIn.AckNackResult;
81 import org.eclipse.jgit.util.TemporaryBuffer;
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105 public abstract class BasePackFetchConnection extends BasePackConnection
106 implements FetchConnection {
107
108
109
110
111
112
113
114 private static final int MAX_HAVES = 256;
115
116
117
118
119
120
121
122
123
124 protected static final int MIN_CLIENT_BUFFER = 2 * 32 * 46 + 8;
125
126
127
128
129
130 public static final String OPTION_INCLUDE_TAG = GitProtocolConstants.OPTION_INCLUDE_TAG;
131
132
133
134
135
136 public static final String OPTION_MULTI_ACK = GitProtocolConstants.OPTION_MULTI_ACK;
137
138
139
140
141
142 public static final String OPTION_MULTI_ACK_DETAILED = GitProtocolConstants.OPTION_MULTI_ACK_DETAILED;
143
144
145
146
147
148 public static final String OPTION_THIN_PACK = GitProtocolConstants.OPTION_THIN_PACK;
149
150
151
152
153
154 public static final String OPTION_SIDE_BAND = GitProtocolConstants.OPTION_SIDE_BAND;
155
156
157
158
159
160 public static final String OPTION_SIDE_BAND_64K = GitProtocolConstants.OPTION_SIDE_BAND_64K;
161
162
163
164
165
166 public static final String OPTION_OFS_DELTA = GitProtocolConstants.OPTION_OFS_DELTA;
167
168
169
170
171
172 public static final String OPTION_SHALLOW = GitProtocolConstants.OPTION_SHALLOW;
173
174
175
176
177
178 public static final String OPTION_NO_PROGRESS = GitProtocolConstants.OPTION_NO_PROGRESS;
179
180
181
182
183
184 public static final String OPTION_NO_DONE = GitProtocolConstants.OPTION_NO_DONE;
185
186
187
188
189
190
191 public static final String OPTION_ALLOW_TIP_SHA1_IN_WANT = GitProtocolConstants.OPTION_ALLOW_TIP_SHA1_IN_WANT;
192
193
194
195
196
197
198 public static final String OPTION_ALLOW_REACHABLE_SHA1_IN_WANT = GitProtocolConstants.OPTION_ALLOW_REACHABLE_SHA1_IN_WANT;
199
200
201
202
203
204
205 public static final String OPTION_FILTER = GitProtocolConstants.OPTION_FILTER;
206
207 private final RevWalk walk;
208
209
210 private RevCommitList<RevCommit> reachableCommits;
211
212
213 final RevFlag REACHABLE;
214
215
216 final RevFlag COMMON;
217
218
219 private final RevFlag STATE;
220
221
222 final RevFlag ADVERTISED;
223
224 private MultiAck multiAck = MultiAck.OFF;
225
226 private boolean thinPack;
227
228 private boolean sideband;
229
230 private boolean includeTags;
231
232 private boolean allowOfsDelta;
233
234 private boolean noDone;
235
236 private boolean noProgress;
237
238 private Set<AnyObjectId> minimalNegotiationSet;
239
240 private String lockMessage;
241
242 private PackLock packLock;
243
244
245 private TemporaryBuffer.Heap state;
246
247 private PacketLineOut pckState;
248
249
250 private final long filterBlobLimit;
251
252
253
254
255
256
257
258 public BasePackFetchConnection(PackTransport packTransport) {
259 super(packTransport);
260
261 if (local != null) {
262 final FetchConfig cfg = getFetchConfig();
263 allowOfsDelta = cfg.allowOfsDelta;
264 if (cfg.minimalNegotiation) {
265 minimalNegotiationSet = new HashSet<>();
266 }
267 } else {
268 allowOfsDelta = true;
269 }
270 includeTags = transport.getTagOpt() != TagOpt.NO_TAGS;
271 thinPack = transport.isFetchThin();
272 filterBlobLimit = transport.getFilterBlobLimit();
273
274 if (local != null) {
275 walk = new RevWalk(local);
276 reachableCommits = new RevCommitList<>();
277 REACHABLE = walk.newFlag("REACHABLE");
278 COMMON = walk.newFlag("COMMON");
279 STATE = walk.newFlag("STATE");
280 ADVERTISED = walk.newFlag("ADVERTISED");
281
282 walk.carry(COMMON);
283 walk.carry(REACHABLE);
284 walk.carry(ADVERTISED);
285 } else {
286 walk = null;
287 REACHABLE = null;
288 COMMON = null;
289 STATE = null;
290 ADVERTISED = null;
291 }
292 }
293
294 static class FetchConfig {
295 final boolean allowOfsDelta;
296
297 final boolean minimalNegotiation;
298
299 FetchConfig(Config c) {
300 allowOfsDelta = c.getBoolean("repack", "usedeltabaseoffset", true);
301 minimalNegotiation = c.getBoolean("fetch", "useminimalnegotiation",
302 false);
303 }
304
305 FetchConfig(boolean allowOfsDelta, boolean minimalNegotiation) {
306 this.allowOfsDelta = allowOfsDelta;
307 this.minimalNegotiation = minimalNegotiation;
308 }
309 }
310
311
312 @Override
313 public final void fetch(final ProgressMonitor monitor,
314 final Collection<Ref> want, final Set<ObjectId> have)
315 throws TransportException {
316 fetch(monitor, want, have, null);
317 }
318
319
320 @Override
321 public final void fetch(final ProgressMonitor monitor,
322 final Collection<Ref> want, final Set<ObjectId> have,
323 OutputStream outputStream) throws TransportException {
324 markStartedOperation();
325 doFetch(monitor, want, have, outputStream);
326 }
327
328
329 @Override
330 public boolean didFetchIncludeTags() {
331 return false;
332 }
333
334
335 @Override
336 public boolean didFetchTestConnectivity() {
337 return false;
338 }
339
340
341 @Override
342 public void setPackLockMessage(String message) {
343 lockMessage = message;
344 }
345
346
347 @Override
348 public Collection<PackLock> getPackLocks() {
349 if (packLock != null)
350 return Collections.singleton(packLock);
351 return Collections.<PackLock> emptyList();
352 }
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373 protected void doFetch(final ProgressMonitor monitor,
374 final Collection<Ref> want, final Set<ObjectId> have,
375 OutputStream outputStream) throws TransportException {
376 try {
377 noProgress = monitor == NullProgressMonitor.INSTANCE;
378
379 markRefsAdvertised();
380 markReachable(have, maxTimeWanted(want));
381
382 if (statelessRPC) {
383 state = new TemporaryBuffer.Heap(Integer.MAX_VALUE);
384 pckState = new PacketLineOut(state);
385 }
386
387 if (sendWants(want)) {
388 negotiate(monitor);
389
390 walk.dispose();
391 reachableCommits = null;
392 state = null;
393 pckState = null;
394
395 receivePack(monitor, outputStream);
396 }
397 } catch (CancelledException ce) {
398 close();
399 return;
400 } catch (IOException err) {
401 close();
402 throw new TransportException(err.getMessage(), err);
403 } catch (RuntimeException err) {
404 close();
405 throw new TransportException(err.getMessage(), err);
406 }
407 }
408
409
410 @Override
411 public void close() {
412 if (walk != null)
413 walk.close();
414 super.close();
415 }
416
417 FetchConfig getFetchConfig() {
418 return local.getConfig().get(FetchConfig::new);
419 }
420
421 private int maxTimeWanted(Collection<Ref> wants) {
422 int maxTime = 0;
423 for (Ref r : wants) {
424 try {
425 final RevObject obj = walk.parseAny(r.getObjectId());
426 if (obj instanceof RevCommit) {
427 final int cTime = ((RevCommit) obj).getCommitTime();
428 if (maxTime < cTime)
429 maxTime = cTime;
430 }
431 } catch (IOException error) {
432
433 }
434 }
435 return maxTime;
436 }
437
438 private void markReachable(Set<ObjectId> have, int maxTime)
439 throws IOException {
440 for (Ref r : local.getRefDatabase().getRefs()) {
441 ObjectId id = r.getPeeledObjectId();
442 if (id == null)
443 id = r.getObjectId();
444 if (id == null)
445 continue;
446 parseReachable(id);
447 }
448
449 for (ObjectId id : local.getAdditionalHaves())
450 parseReachable(id);
451
452 for (ObjectId id : have)
453 parseReachable(id);
454
455 if (maxTime > 0) {
456
457
458
459
460 final Date maxWhen = new Date(maxTime * 1000L);
461 walk.sort(RevSort.COMMIT_TIME_DESC);
462 walk.markStart(reachableCommits);
463 walk.setRevFilter(CommitTimeRevFilter.after(maxWhen));
464 for (;;) {
465 final RevCommit c = walk.next();
466 if (c == null)
467 break;
468 if (c.has(ADVERTISED) && !c.has(COMMON)) {
469
470
471
472 c.add(COMMON);
473 c.carry(COMMON);
474 reachableCommits.add(c);
475 }
476 }
477 }
478 }
479
480 private void parseReachable(ObjectId id) {
481 try {
482 RevCommit o = walk.parseCommit(id);
483 if (!o.has(REACHABLE)) {
484 o.add(REACHABLE);
485 reachableCommits.add(o);
486 }
487 } catch (IOException readError) {
488
489 }
490 }
491
492 private boolean sendWants(Collection<Ref> want) throws IOException {
493 final PacketLineOut p = statelessRPC ? pckState : pckOut;
494 boolean first = true;
495 for (Ref r : want) {
496 ObjectId objectId = r.getObjectId();
497 if (objectId == null) {
498 continue;
499 }
500 try {
501 if (walk.parseAny(objectId).has(REACHABLE)) {
502
503
504
505 continue;
506 }
507 } catch (IOException err) {
508
509
510 }
511
512 final StringBuilder line = new StringBuilder(46);
513 line.append("want ");
514 line.append(objectId.name());
515 if (first) {
516 line.append(enableCapabilities());
517 first = false;
518 }
519 line.append('\n');
520 p.writeString(line.toString());
521 if (minimalNegotiationSet != null) {
522 Ref current = local.exactRef(r.getName());
523 if (current != null) {
524 ObjectId o = current.getObjectId();
525 if (o != null && !o.equals(ObjectId.zeroId())) {
526 minimalNegotiationSet.add(o);
527 }
528 }
529 }
530 }
531 if (first) {
532 return false;
533 }
534 if (filterBlobLimit == 0) {
535 p.writeString(OPTION_FILTER + " blob:none");
536 } else if (filterBlobLimit > 0) {
537 p.writeString(OPTION_FILTER + " blob:limit=" + filterBlobLimit);
538 }
539 p.end();
540 outNeedsEnd = false;
541 return true;
542 }
543
544 private String enableCapabilities() throws TransportException {
545 final StringBuilder line = new StringBuilder();
546 if (noProgress)
547 wantCapability(line, OPTION_NO_PROGRESS);
548 if (includeTags)
549 includeTags = wantCapability(line, OPTION_INCLUDE_TAG);
550 if (allowOfsDelta)
551 wantCapability(line, OPTION_OFS_DELTA);
552
553 if (wantCapability(line, OPTION_MULTI_ACK_DETAILED)) {
554 multiAck = MultiAck.DETAILED;
555 if (statelessRPC)
556 noDone = wantCapability(line, OPTION_NO_DONE);
557 } else if (wantCapability(line, OPTION_MULTI_ACK))
558 multiAck = MultiAck.CONTINUE;
559 else
560 multiAck = MultiAck.OFF;
561
562 if (thinPack)
563 thinPack = wantCapability(line, OPTION_THIN_PACK);
564 if (wantCapability(line, OPTION_SIDE_BAND_64K))
565 sideband = true;
566 else if (wantCapability(line, OPTION_SIDE_BAND))
567 sideband = true;
568
569 if (statelessRPC && multiAck != MultiAck.DETAILED) {
570
571
572
573
574 throw new PackProtocolException(uri, MessageFormat.format(
575 JGitText.get().statelessRPCRequiresOptionToBeEnabled,
576 OPTION_MULTI_ACK_DETAILED));
577 }
578
579 if (filterBlobLimit >= 0 && !wantCapability(line, OPTION_FILTER)) {
580 throw new PackProtocolException(uri,
581 JGitText.get().filterRequiresCapability);
582 }
583
584 addUserAgentCapability(line);
585 return line.toString();
586 }
587
588 private void negotiate(ProgressMonitor monitor) throws IOException,
589 CancelledException {
590 final MutableObjectId ackId = new MutableObjectId();
591 int resultsPending = 0;
592 int havesSent = 0;
593 int havesSinceLastContinue = 0;
594 boolean receivedContinue = false;
595 boolean receivedAck = false;
596 boolean receivedReady = false;
597
598 if (statelessRPC) {
599 state.writeTo(out, null);
600 }
601
602 negotiateBegin();
603 SEND_HAVES: for (;;) {
604 final RevCommit c = walk.next();
605 if (c == null) {
606 break SEND_HAVES;
607 }
608
609 ObjectId o = c.getId();
610 pckOut.writeString("have " + o.name() + "\n");
611 havesSent++;
612 havesSinceLastContinue++;
613 if (minimalNegotiationSet != null) {
614 minimalNegotiationSet.remove(o);
615 }
616
617 if ((31 & havesSent) != 0) {
618
619
620
621
622 continue;
623 }
624
625 if (monitor.isCancelled()) {
626 throw new CancelledException();
627 }
628
629 pckOut.end();
630 resultsPending++;
631
632 if (havesSent == 32 && !statelessRPC) {
633
634
635
636
637
638 continue;
639 }
640
641 READ_RESULT: for (;;) {
642 final AckNackResult anr = pckIn.readACK(ackId);
643 switch (anr) {
644 case NAK:
645
646
647
648 resultsPending--;
649 if (minimalNegotiationSet != null
650 && minimalNegotiationSet.isEmpty()) {
651
652
653
654 if (statelessRPC) {
655 state.writeTo(out, null);
656 }
657 break SEND_HAVES;
658 }
659 break READ_RESULT;
660
661 case ACK:
662
663
664
665
666 multiAck = MultiAck.OFF;
667 resultsPending = 0;
668 receivedAck = true;
669 if (statelessRPC) {
670 state.writeTo(out, null);
671 }
672 break SEND_HAVES;
673
674 case ACK_CONTINUE:
675 case ACK_COMMON:
676 case ACK_READY:
677
678
679
680
681
682 markCommon(walk.parseAny(ackId), anr);
683 receivedAck = true;
684 receivedContinue = true;
685 havesSinceLastContinue = 0;
686 if (anr == AckNackResult.ACK_READY) {
687 receivedReady = true;
688 }
689 if (minimalNegotiationSet != null && minimalNegotiationSet.isEmpty()) {
690
691
692 if (statelessRPC) {
693 state.writeTo(out, null);
694 }
695 break SEND_HAVES;
696 }
697 break;
698 }
699
700 if (monitor.isCancelled()) {
701 throw new CancelledException();
702 }
703 }
704
705 if (noDone & receivedReady) {
706 break SEND_HAVES;
707 }
708 if (statelessRPC) {
709 state.writeTo(out, null);
710 }
711
712 if (receivedContinue && havesSinceLastContinue > MAX_HAVES) {
713
714
715
716
717
718 break SEND_HAVES;
719 }
720 }
721
722
723
724 if (monitor.isCancelled()) {
725 throw new CancelledException();
726 }
727
728 if (!receivedReady || !noDone) {
729
730
731
732
733 pckOut.writeString("done\n");
734 pckOut.flush();
735 }
736
737 if (!receivedAck) {
738
739
740
741
742 multiAck = MultiAck.OFF;
743 resultsPending++;
744 }
745
746 READ_RESULT: while (resultsPending > 0 || multiAck != MultiAck.OFF) {
747 final AckNackResult anr = pckIn.readACK(ackId);
748 resultsPending--;
749 switch (anr) {
750 case NAK:
751
752
753
754 break;
755
756 case ACK:
757
758
759
760 break READ_RESULT;
761
762 case ACK_CONTINUE:
763 case ACK_COMMON:
764 case ACK_READY:
765
766
767 multiAck = MultiAck.CONTINUE;
768 break;
769 }
770
771 if (monitor.isCancelled()) {
772 throw new CancelledException();
773 }
774 }
775 }
776
777 private void negotiateBegin() throws IOException {
778 walk.resetRetain(REACHABLE, ADVERTISED);
779 walk.markStart(reachableCommits);
780 walk.sort(RevSort.COMMIT_TIME_DESC);
781 walk.setRevFilter(new RevFilter() {
782 @Override
783 public RevFilter clone() {
784 return this;
785 }
786
787 @Override
788 public boolean include(RevWalk walker, RevCommit c) {
789 final boolean remoteKnowsIsCommon = c.has(COMMON);
790 if (c.has(ADVERTISED)) {
791
792
793
794
795
796 c.add(COMMON);
797 }
798 return !remoteKnowsIsCommon;
799 }
800
801 @Override
802 public boolean requiresCommitBody() {
803 return false;
804 }
805 });
806 }
807
808 private void markRefsAdvertised() {
809 for (Ref r : getRefs()) {
810 markAdvertised(r.getObjectId());
811 if (r.getPeeledObjectId() != null)
812 markAdvertised(r.getPeeledObjectId());
813 }
814 }
815
816 private void markAdvertised(AnyObjectId id) {
817 try {
818 walk.parseAny(id).add(ADVERTISED);
819 } catch (IOException readError) {
820
821 }
822 }
823
824 private void markCommon(RevObject obj, AckNackResult anr)
825 throws IOException {
826 if (statelessRPC && anr == AckNackResult.ACK_COMMON && !obj.has(STATE)) {
827 StringBuilder s;
828
829 s = new StringBuilder(6 + Constants.OBJECT_ID_STRING_LENGTH);
830 s.append("have ");
831 s.append(obj.name());
832 s.append('\n');
833 pckState.writeString(s.toString());
834 obj.add(STATE);
835 }
836 obj.add(COMMON);
837 if (obj instanceof RevCommit)
838 ((RevCommit) obj).carry(COMMON);
839 }
840
841 private void receivePack(final ProgressMonitor monitor,
842 OutputStream outputStream) throws IOException {
843 onReceivePack();
844 InputStream input = in;
845 if (sideband)
846 input = new SideBandInputStream(input, monitor, getMessageWriter(),
847 outputStream);
848
849 try (ObjectInserter ins = local.newObjectInserter()) {
850 PackParser parser = ins.newPackParser(input);
851 parser.setAllowThin(thinPack);
852 parser.setObjectChecker(transport.getObjectChecker());
853 parser.setLockMessage(lockMessage);
854 packLock = parser.parse(monitor);
855 ins.flush();
856 }
857 }
858
859
860
861
862
863
864
865
866 protected void onReceivePack() {
867
868 }
869
870 private static class CancelledException extends Exception {
871 private static final long serialVersionUID = 1L;
872 }
873 }