1
2
3
4
5
6
7
8
9
10
11
12
13 package org.eclipse.jgit.transport;
14
15 import java.io.IOException;
16 import java.io.InputStream;
17 import java.io.OutputStream;
18 import java.text.MessageFormat;
19 import java.util.Collection;
20 import java.util.Collections;
21 import java.util.Date;
22 import java.util.Set;
23
24 import org.eclipse.jgit.errors.PackProtocolException;
25 import org.eclipse.jgit.errors.TransportException;
26 import org.eclipse.jgit.internal.JGitText;
27 import org.eclipse.jgit.internal.storage.file.PackLock;
28 import org.eclipse.jgit.lib.AnyObjectId;
29 import org.eclipse.jgit.lib.Config;
30 import org.eclipse.jgit.lib.Constants;
31 import org.eclipse.jgit.lib.MutableObjectId;
32 import org.eclipse.jgit.lib.NullProgressMonitor;
33 import org.eclipse.jgit.lib.ObjectId;
34 import org.eclipse.jgit.lib.ObjectInserter;
35 import org.eclipse.jgit.lib.ProgressMonitor;
36 import org.eclipse.jgit.lib.Ref;
37 import org.eclipse.jgit.revwalk.RevCommit;
38 import org.eclipse.jgit.revwalk.RevCommitList;
39 import org.eclipse.jgit.revwalk.RevFlag;
40 import org.eclipse.jgit.revwalk.RevObject;
41 import org.eclipse.jgit.revwalk.RevSort;
42 import org.eclipse.jgit.revwalk.RevWalk;
43 import org.eclipse.jgit.revwalk.filter.CommitTimeRevFilter;
44 import org.eclipse.jgit.revwalk.filter.RevFilter;
45 import org.eclipse.jgit.transport.GitProtocolConstants.MultiAck;
46 import org.eclipse.jgit.transport.PacketLineIn.AckNackResult;
47 import org.eclipse.jgit.util.TemporaryBuffer;
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71 public abstract class BasePackFetchConnection extends BasePackConnection
72 implements FetchConnection {
73
74
75
76
77
78
79
80 private static final int MAX_HAVES = 256;
81
82
83
84
85
86
87
88
89
90 protected static final int MIN_CLIENT_BUFFER = 2 * 32 * 46 + 8;
91
92
93
94
95
96 public static final String OPTION_INCLUDE_TAG = GitProtocolConstants.OPTION_INCLUDE_TAG;
97
98
99
100
101
102 public static final String OPTION_MULTI_ACK = GitProtocolConstants.OPTION_MULTI_ACK;
103
104
105
106
107
108 public static final String OPTION_MULTI_ACK_DETAILED = GitProtocolConstants.OPTION_MULTI_ACK_DETAILED;
109
110
111
112
113
114 public static final String OPTION_THIN_PACK = GitProtocolConstants.OPTION_THIN_PACK;
115
116
117
118
119
120 public static final String OPTION_SIDE_BAND = GitProtocolConstants.OPTION_SIDE_BAND;
121
122
123
124
125
126 public static final String OPTION_SIDE_BAND_64K = GitProtocolConstants.OPTION_SIDE_BAND_64K;
127
128
129
130
131
132 public static final String OPTION_OFS_DELTA = GitProtocolConstants.OPTION_OFS_DELTA;
133
134
135
136
137
138 public static final String OPTION_SHALLOW = GitProtocolConstants.OPTION_SHALLOW;
139
140
141
142
143
144 public static final String OPTION_NO_PROGRESS = GitProtocolConstants.OPTION_NO_PROGRESS;
145
146
147
148
149
150 public static final String OPTION_NO_DONE = GitProtocolConstants.OPTION_NO_DONE;
151
152
153
154
155
156
157 public static final String OPTION_ALLOW_TIP_SHA1_IN_WANT = GitProtocolConstants.OPTION_ALLOW_TIP_SHA1_IN_WANT;
158
159
160
161
162
163
164 public static final String OPTION_ALLOW_REACHABLE_SHA1_IN_WANT = GitProtocolConstants.OPTION_ALLOW_REACHABLE_SHA1_IN_WANT;
165
166
167
168
169
170
171 public static final String OPTION_FILTER = GitProtocolConstants.OPTION_FILTER;
172
173 private final RevWalk walk;
174
175
176 private RevCommitList<RevCommit> reachableCommits;
177
178
179 final RevFlag REACHABLE;
180
181
182 final RevFlag COMMON;
183
184
185 private final RevFlag STATE;
186
187
188 final RevFlag ADVERTISED;
189
190 private MultiAck multiAck = MultiAck.OFF;
191
192 private boolean thinPack;
193
194 private boolean sideband;
195
196 private boolean includeTags;
197
198 private boolean allowOfsDelta;
199
200 private boolean noDone;
201
202 private boolean noProgress;
203
204 private String lockMessage;
205
206 private PackLock packLock;
207
208 private int maxHaves;
209
210
211 private TemporaryBuffer.Heap state;
212
213 private PacketLineOut pckState;
214
215
216
217
218
219
220 private final FilterSpec filterSpec;
221
222
223
224
225
226
227
228 public BasePackFetchConnection(PackTransport packTransport) {
229 super(packTransport);
230
231 if (local != null) {
232 final FetchConfig cfg = getFetchConfig();
233 allowOfsDelta = cfg.allowOfsDelta;
234 maxHaves = cfg.maxHaves;
235 } else {
236 allowOfsDelta = true;
237 maxHaves = Integer.MAX_VALUE;
238 }
239
240 includeTags = transport.getTagOpt() != TagOpt.NO_TAGS;
241 thinPack = transport.isFetchThin();
242 filterSpec = transport.getFilterSpec();
243
244 if (local != null) {
245 walk = new RevWalk(local);
246 walk.setRetainBody(false);
247 reachableCommits = new RevCommitList<>();
248 REACHABLE = walk.newFlag("REACHABLE");
249 COMMON = walk.newFlag("COMMON");
250 STATE = walk.newFlag("STATE");
251 ADVERTISED = walk.newFlag("ADVERTISED");
252
253 walk.carry(COMMON);
254 walk.carry(REACHABLE);
255 walk.carry(ADVERTISED);
256 } else {
257 walk = null;
258 REACHABLE = null;
259 COMMON = null;
260 STATE = null;
261 ADVERTISED = null;
262 }
263 }
264
265 static class FetchConfig {
266 final boolean allowOfsDelta;
267
268 final int maxHaves;
269
270 FetchConfig(Config c) {
271 allowOfsDelta = c.getBoolean("repack", "usedeltabaseoffset", true);
272 maxHaves = c.getInt("fetch", "maxhaves", Integer.MAX_VALUE);
273 }
274
275 FetchConfig(boolean allowOfsDelta, int maxHaves) {
276 this.allowOfsDelta = allowOfsDelta;
277 this.maxHaves = maxHaves;
278 }
279 }
280
281
282 @Override
283 public final void fetch(final ProgressMonitor monitor,
284 final Collection<Ref> want, final Set<ObjectId> have)
285 throws TransportException {
286 fetch(monitor, want, have, null);
287 }
288
289
290 @Override
291 public final void fetch(final ProgressMonitor monitor,
292 final Collection<Ref> want, final Set<ObjectId> have,
293 OutputStream outputStream) throws TransportException {
294 markStartedOperation();
295 doFetch(monitor, want, have, outputStream);
296 }
297
298
299 @Override
300 public boolean didFetchIncludeTags() {
301 return false;
302 }
303
304
305 @Override
306 public boolean didFetchTestConnectivity() {
307 return false;
308 }
309
310
311 @Override
312 public void setPackLockMessage(String message) {
313 lockMessage = message;
314 }
315
316
317 @Override
318 public Collection<PackLock> getPackLocks() {
319 if (packLock != null)
320 return Collections.singleton(packLock);
321 return Collections.<PackLock> emptyList();
322 }
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343 protected void doFetch(final ProgressMonitor monitor,
344 final Collection<Ref> want, final Set<ObjectId> have,
345 OutputStream outputStream) throws TransportException {
346 try {
347 noProgress = monitor == NullProgressMonitor.INSTANCE;
348
349 markRefsAdvertised();
350 markReachable(have, maxTimeWanted(want));
351
352 if (statelessRPC) {
353 state = new TemporaryBuffer.Heap(Integer.MAX_VALUE);
354 pckState = new PacketLineOut(state);
355 }
356
357 if (sendWants(want)) {
358 negotiate(monitor);
359
360 walk.dispose();
361 reachableCommits = null;
362 state = null;
363 pckState = null;
364
365 receivePack(monitor, outputStream);
366 }
367 } catch (CancelledException ce) {
368 close();
369 return;
370 } catch (IOException | RuntimeException err) {
371 close();
372 throw new TransportException(err.getMessage(), err);
373 }
374 }
375
376
377 @Override
378 public void close() {
379 if (walk != null)
380 walk.close();
381 super.close();
382 }
383
384 FetchConfig getFetchConfig() {
385 return local.getConfig().get(FetchConfig::new);
386 }
387
388 private int maxTimeWanted(Collection<Ref> wants) {
389 int maxTime = 0;
390 for (Ref r : wants) {
391 try {
392 final RevObject obj = walk.parseAny(r.getObjectId());
393 if (obj instanceof RevCommit) {
394 final int cTime = ((RevCommit) obj).getCommitTime();
395 if (maxTime < cTime)
396 maxTime = cTime;
397 }
398 } catch (IOException error) {
399
400 }
401 }
402 return maxTime;
403 }
404
405 private void markReachable(Set<ObjectId> have, int maxTime)
406 throws IOException {
407 for (Ref r : local.getRefDatabase().getRefs()) {
408 ObjectId id = r.getPeeledObjectId();
409 if (id == null)
410 id = r.getObjectId();
411 if (id == null)
412 continue;
413 parseReachable(id);
414 }
415
416 for (ObjectId id : local.getAdditionalHaves())
417 parseReachable(id);
418
419 for (ObjectId id : have)
420 parseReachable(id);
421
422 if (maxTime > 0) {
423
424
425
426
427 final Date maxWhen = new Date(maxTime * 1000L);
428 walk.sort(RevSort.COMMIT_TIME_DESC);
429 walk.markStart(reachableCommits);
430 walk.setRevFilter(CommitTimeRevFilter.after(maxWhen));
431 for (;;) {
432 final RevCommit c = walk.next();
433 if (c == null)
434 break;
435 if (c.has(ADVERTISED) && !c.has(COMMON)) {
436
437
438
439 c.add(COMMON);
440 c.carry(COMMON);
441 reachableCommits.add(c);
442 }
443 }
444 }
445 }
446
447 private void parseReachable(ObjectId id) {
448 try {
449 RevCommit o = walk.parseCommit(id);
450 if (!o.has(REACHABLE)) {
451 o.add(REACHABLE);
452 reachableCommits.add(o);
453 }
454 } catch (IOException readError) {
455
456 }
457 }
458
459 private boolean sendWants(Collection<Ref> want) throws IOException {
460 final PacketLineOut p = statelessRPC ? pckState : pckOut;
461 boolean first = true;
462 for (Ref r : want) {
463 ObjectId objectId = r.getObjectId();
464 if (objectId == null) {
465 continue;
466 }
467 try {
468 if (walk.parseAny(objectId).has(REACHABLE)) {
469
470
471
472 continue;
473 }
474 } catch (IOException err) {
475
476
477 }
478
479 final StringBuilder line = new StringBuilder(46);
480 line.append("want ");
481 line.append(objectId.name());
482 if (first) {
483 line.append(enableCapabilities());
484 first = false;
485 }
486 line.append('\n');
487 p.writeString(line.toString());
488 }
489 if (first) {
490 return false;
491 }
492 if (!filterSpec.isNoOp()) {
493 p.writeString(filterSpec.filterLine());
494 }
495 p.end();
496 outNeedsEnd = false;
497 return true;
498 }
499
500 private String enableCapabilities() throws TransportException {
501 final StringBuilder line = new StringBuilder();
502 if (noProgress)
503 wantCapability(line, OPTION_NO_PROGRESS);
504 if (includeTags)
505 includeTags = wantCapability(line, OPTION_INCLUDE_TAG);
506 if (allowOfsDelta)
507 wantCapability(line, OPTION_OFS_DELTA);
508
509 if (wantCapability(line, OPTION_MULTI_ACK_DETAILED)) {
510 multiAck = MultiAck.DETAILED;
511 if (statelessRPC)
512 noDone = wantCapability(line, OPTION_NO_DONE);
513 } else if (wantCapability(line, OPTION_MULTI_ACK))
514 multiAck = MultiAck.CONTINUE;
515 else
516 multiAck = MultiAck.OFF;
517
518 if (thinPack)
519 thinPack = wantCapability(line, OPTION_THIN_PACK);
520 if (wantCapability(line, OPTION_SIDE_BAND_64K))
521 sideband = true;
522 else if (wantCapability(line, OPTION_SIDE_BAND))
523 sideband = true;
524
525 if (statelessRPC && multiAck != MultiAck.DETAILED) {
526
527
528
529
530 throw new PackProtocolException(uri, MessageFormat.format(
531 JGitText.get().statelessRPCRequiresOptionToBeEnabled,
532 OPTION_MULTI_ACK_DETAILED));
533 }
534
535 if (!filterSpec.isNoOp() && !wantCapability(line, OPTION_FILTER)) {
536 throw new PackProtocolException(uri,
537 JGitText.get().filterRequiresCapability);
538 }
539
540 addUserAgentCapability(line);
541 return line.toString();
542 }
543
544 private void negotiate(ProgressMonitor monitor) throws IOException,
545 CancelledException {
546 final MutableObjectId.html#MutableObjectId">MutableObjectId ackId = new MutableObjectId();
547 int resultsPending = 0;
548 int havesSent = 0;
549 int havesSinceLastContinue = 0;
550 boolean receivedContinue = false;
551 boolean receivedAck = false;
552 boolean receivedReady = false;
553
554 if (statelessRPC) {
555 state.writeTo(out, null);
556 }
557
558 negotiateBegin();
559 SEND_HAVES: for (;;) {
560 final RevCommit c = walk.next();
561 if (c == null) {
562 break SEND_HAVES;
563 }
564
565 ObjectId o = c.getId();
566 pckOut.writeString("have " + o.name() + "\n");
567 havesSent++;
568 havesSinceLastContinue++;
569
570 if ((31 & havesSent) != 0) {
571
572
573
574
575 continue;
576 }
577
578 if (monitor.isCancelled()) {
579 throw new CancelledException();
580 }
581
582 pckOut.end();
583 resultsPending++;
584
585 if (havesSent == 32 && !statelessRPC) {
586
587
588
589
590
591 continue;
592 }
593
594 READ_RESULT: for (;;) {
595 final AckNackResult anr = pckIn.readACK(ackId);
596 switch (anr) {
597 case NAK:
598
599
600
601 resultsPending--;
602 break READ_RESULT;
603
604 case ACK:
605
606
607
608
609 multiAck = MultiAck.OFF;
610 resultsPending = 0;
611 receivedAck = true;
612 if (statelessRPC) {
613 state.writeTo(out, null);
614 }
615 break SEND_HAVES;
616
617 case ACK_CONTINUE:
618 case ACK_COMMON:
619 case ACK_READY:
620
621
622
623
624
625 markCommon(walk.parseAny(ackId), anr);
626 receivedAck = true;
627 receivedContinue = true;
628 havesSinceLastContinue = 0;
629 if (anr == AckNackResult.ACK_READY) {
630 receivedReady = true;
631 }
632 break;
633 }
634
635 if (monitor.isCancelled()) {
636 throw new CancelledException();
637 }
638 }
639
640 if (noDone && receivedReady) {
641 break SEND_HAVES;
642 }
643 if (statelessRPC) {
644 state.writeTo(out, null);
645 }
646
647 if ((receivedContinue && havesSinceLastContinue > MAX_HAVES)
648 || havesSent >= maxHaves) {
649
650
651
652
653
654 break SEND_HAVES;
655 }
656 }
657
658
659
660 if (monitor.isCancelled()) {
661 throw new CancelledException();
662 }
663
664 if (!receivedReady || !noDone) {
665
666
667
668
669 pckOut.writeString("done\n");
670 pckOut.flush();
671 }
672
673 if (!receivedAck) {
674
675
676
677
678 multiAck = MultiAck.OFF;
679 resultsPending++;
680 }
681
682 READ_RESULT: while (resultsPending > 0 || multiAck != MultiAck.OFF) {
683 final AckNackResult anr = pckIn.readACK(ackId);
684 resultsPending--;
685 switch (anr) {
686 case NAK:
687
688
689
690 break;
691
692 case ACK:
693
694
695
696 break READ_RESULT;
697
698 case ACK_CONTINUE:
699 case ACK_COMMON:
700 case ACK_READY:
701
702
703 multiAck = MultiAck.CONTINUE;
704 break;
705 }
706
707 if (monitor.isCancelled()) {
708 throw new CancelledException();
709 }
710 }
711 }
712
713 private void negotiateBegin() throws IOException {
714 walk.resetRetain(REACHABLE, ADVERTISED);
715 walk.markStart(reachableCommits);
716 walk.sort(RevSort.COMMIT_TIME_DESC);
717 walk.setRevFilter(new RevFilter() {
718 @Override
719 public RevFilter clone() {
720 return this;
721 }
722
723 @Override
724 public boolean include(RevWalk walker, RevCommit c) {
725 final boolean remoteKnowsIsCommon = c.has(COMMON);
726 if (c.has(ADVERTISED)) {
727
728
729
730
731
732 c.add(COMMON);
733 }
734 return !remoteKnowsIsCommon;
735 }
736
737 @Override
738 public boolean requiresCommitBody() {
739 return false;
740 }
741 });
742 }
743
744 private void markRefsAdvertised() {
745 for (Ref r : getRefs()) {
746 markAdvertised(r.getObjectId());
747 if (r.getPeeledObjectId() != null)
748 markAdvertised(r.getPeeledObjectId());
749 }
750 }
751
752 private void markAdvertised(AnyObjectId id) {
753 try {
754 walk.parseAny(id).add(ADVERTISED);
755 } catch (IOException readError) {
756
757 }
758 }
759
760 private void markCommon(RevObject obj, AckNackResult anr)
761 throws IOException {
762 if (statelessRPC && anr == AckNackResult.ACK_COMMON && !obj.has(STATE)) {
763 StringBuilder s;
764
765 s = new StringBuilder(6 + Constants.OBJECT_ID_STRING_LENGTH);
766 s.append("have ");
767 s.append(obj.name());
768 s.append('\n');
769 pckState.writeString(s.toString());
770 obj.add(STATE);
771 }
772 obj.add(COMMON);
773 if (obj instanceof RevCommit)
774 ((RevCommit) obj).carry(COMMON);
775 }
776
777 private void receivePack(final ProgressMonitor monitor,
778 OutputStream outputStream) throws IOException {
779 onReceivePack();
780 InputStream input = in;
781 if (sideband)
782 input = new SideBandInputStream(input, monitor, getMessageWriter(),
783 outputStream);
784
785 try (ObjectInserter ins = local.newObjectInserter()) {
786 PackParser parser = ins.newPackParser(input);
787 parser.setAllowThin(thinPack);
788 parser.setObjectChecker(transport.getObjectChecker());
789 parser.setLockMessage(lockMessage);
790 packLock = parser.parse(monitor);
791 ins.flush();
792 }
793 }
794
795
796
797
798
799
800
801
802 protected void onReceivePack() {
803
804 }
805
806 private static class CancelledException extends Exception {
807 private static final long serialVersionUID = 1L;
808 }
809 }