1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46 package org.eclipse.jgit.transport;
47
48 import static org.eclipse.jgit.lib.RefDatabase.ALL;
49
50 import java.io.IOException;
51 import java.io.InputStream;
52 import java.io.OutputStream;
53 import java.text.MessageFormat;
54 import java.util.Collection;
55 import java.util.Collections;
56 import java.util.Date;
57 import java.util.Map;
58 import java.util.Set;
59
60 import org.eclipse.jgit.errors.PackProtocolException;
61 import org.eclipse.jgit.errors.TransportException;
62 import org.eclipse.jgit.internal.JGitText;
63 import org.eclipse.jgit.internal.storage.file.PackLock;
64 import org.eclipse.jgit.lib.AnyObjectId;
65 import org.eclipse.jgit.lib.Config;
66 import org.eclipse.jgit.lib.Config.SectionParser;
67 import org.eclipse.jgit.lib.Constants;
68 import org.eclipse.jgit.lib.MutableObjectId;
69 import org.eclipse.jgit.lib.NullProgressMonitor;
70 import org.eclipse.jgit.lib.ObjectId;
71 import org.eclipse.jgit.lib.ObjectInserter;
72 import org.eclipse.jgit.lib.ProgressMonitor;
73 import org.eclipse.jgit.lib.Ref;
74 import org.eclipse.jgit.transport.GitProtocolConstants.MultiAck;
75 import org.eclipse.jgit.revwalk.RevCommit;
76 import org.eclipse.jgit.revwalk.RevCommitList;
77 import org.eclipse.jgit.revwalk.RevFlag;
78 import org.eclipse.jgit.revwalk.RevObject;
79 import org.eclipse.jgit.revwalk.RevSort;
80 import org.eclipse.jgit.revwalk.RevWalk;
81 import org.eclipse.jgit.revwalk.filter.CommitTimeRevFilter;
82 import org.eclipse.jgit.revwalk.filter.RevFilter;
83 import org.eclipse.jgit.transport.PacketLineIn.AckNackResult;
84 import org.eclipse.jgit.util.TemporaryBuffer;
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108 public abstract class BasePackFetchConnection extends BasePackConnection
109 implements FetchConnection {
110
111
112
113
114
115
116
117 private static final int MAX_HAVES = 256;
118
119
120
121
122
123
124
125
126
127 protected static final int MIN_CLIENT_BUFFER = 2 * 32 * 46 + 8;
128
129
130
131
132
133 public static final String OPTION_INCLUDE_TAG = GitProtocolConstants.OPTION_INCLUDE_TAG;
134
135
136
137
138
139 public static final String OPTION_MULTI_ACK = GitProtocolConstants.OPTION_MULTI_ACK;
140
141
142
143
144
145 public static final String OPTION_MULTI_ACK_DETAILED = GitProtocolConstants.OPTION_MULTI_ACK_DETAILED;
146
147
148
149
150
151 public static final String OPTION_THIN_PACK = GitProtocolConstants.OPTION_THIN_PACK;
152
153
154
155
156
157 public static final String OPTION_SIDE_BAND = GitProtocolConstants.OPTION_SIDE_BAND;
158
159
160
161
162
163 public static final String OPTION_SIDE_BAND_64K = GitProtocolConstants.OPTION_SIDE_BAND_64K;
164
165
166
167
168
169 public static final String OPTION_OFS_DELTA = GitProtocolConstants.OPTION_OFS_DELTA;
170
171
172
173
174
175 public static final String OPTION_SHALLOW = GitProtocolConstants.OPTION_SHALLOW;
176
177
178
179
180
181 public static final String OPTION_NO_PROGRESS = GitProtocolConstants.OPTION_NO_PROGRESS;
182
183
184
185
186
187 public static final String OPTION_NO_DONE = GitProtocolConstants.OPTION_NO_DONE;
188
189
190
191
192
193
194 public static final String OPTION_ALLOW_TIP_SHA1_IN_WANT = GitProtocolConstants.OPTION_ALLOW_TIP_SHA1_IN_WANT;
195
196 private final RevWalk walk;
197
198
199 private RevCommitList<RevCommit> reachableCommits;
200
201
202 final RevFlag REACHABLE;
203
204
205 final RevFlag COMMON;
206
207
208 private final RevFlag STATE;
209
210
211 final RevFlag ADVERTISED;
212
213 private MultiAck multiAck = MultiAck.OFF;
214
215 private boolean thinPack;
216
217 private boolean sideband;
218
219 private boolean includeTags;
220
221 private boolean allowOfsDelta;
222
223 private boolean noDone;
224
225 private boolean noProgress;
226
227 private String lockMessage;
228
229 private PackLock packLock;
230
231
232 private TemporaryBuffer.Heap state;
233
234 private PacketLineOut pckState;
235
236
237
238
239
240
241
242 public BasePackFetchConnection(final PackTransport packTransport) {
243 super(packTransport);
244
245 if (local != null) {
246 final FetchConfig cfg = local.getConfig().get(FetchConfig.KEY);
247 allowOfsDelta = cfg.allowOfsDelta;
248 } else {
249 allowOfsDelta = true;
250 }
251 includeTags = transport.getTagOpt() != TagOpt.NO_TAGS;
252 thinPack = transport.isFetchThin();
253
254 if (local != null) {
255 walk = new RevWalk(local);
256 reachableCommits = new RevCommitList<RevCommit>();
257 REACHABLE = walk.newFlag("REACHABLE");
258 COMMON = walk.newFlag("COMMON");
259 STATE = walk.newFlag("STATE");
260 ADVERTISED = walk.newFlag("ADVERTISED");
261
262 walk.carry(COMMON);
263 walk.carry(REACHABLE);
264 walk.carry(ADVERTISED);
265 } else {
266 walk = null;
267 REACHABLE = null;
268 COMMON = null;
269 STATE = null;
270 ADVERTISED = null;
271 }
272 }
273
274 private static class FetchConfig {
275 static final SectionParser<FetchConfig> KEY = new SectionParser<FetchConfig>() {
276 public FetchConfig parse(final Config cfg) {
277 return new FetchConfig(cfg);
278 }
279 };
280
281 final boolean allowOfsDelta;
282
283 FetchConfig(final Config c) {
284 allowOfsDelta = c.getBoolean("repack", "usedeltabaseoffset", true);
285 }
286 }
287
288 public final void fetch(final ProgressMonitor monitor,
289 final Collection<Ref> want, final Set<ObjectId> have)
290 throws TransportException {
291 fetch(monitor, want, have, null);
292 }
293
294
295
296
297 public final void fetch(final ProgressMonitor monitor,
298 final Collection<Ref> want, final Set<ObjectId> have,
299 OutputStream outputStream) throws TransportException {
300 markStartedOperation();
301 doFetch(monitor, want, have, outputStream);
302 }
303
304 public boolean didFetchIncludeTags() {
305 return false;
306 }
307
308 public boolean didFetchTestConnectivity() {
309 return false;
310 }
311
312 public void setPackLockMessage(final String message) {
313 lockMessage = message;
314 }
315
316 public Collection<PackLock> getPackLocks() {
317 if (packLock != null)
318 return Collections.singleton(packLock);
319 return Collections.<PackLock> emptyList();
320 }
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341 protected void doFetch(final ProgressMonitor monitor,
342 final Collection<Ref> want, final Set<ObjectId> have,
343 OutputStream outputStream) throws TransportException {
344 try {
345 noProgress = monitor == NullProgressMonitor.INSTANCE;
346
347 markRefsAdvertised();
348 markReachable(have, maxTimeWanted(want));
349
350 if (statelessRPC) {
351 state = new TemporaryBuffer.Heap(Integer.MAX_VALUE);
352 pckState = new PacketLineOut(state);
353 }
354
355 if (sendWants(want)) {
356 negotiate(monitor);
357
358 walk.dispose();
359 reachableCommits = null;
360 state = null;
361 pckState = null;
362
363 receivePack(monitor, outputStream);
364 }
365 } catch (CancelledException ce) {
366 close();
367 return;
368 } catch (IOException err) {
369 close();
370 throw new TransportException(err.getMessage(), err);
371 } catch (RuntimeException err) {
372 close();
373 throw new TransportException(err.getMessage(), err);
374 }
375 }
376
377 @Override
378 public void close() {
379 if (walk != null)
380 walk.close();
381 super.close();
382 }
383
384 private int maxTimeWanted(final Collection<Ref> wants) {
385 int maxTime = 0;
386 for (final Ref r : wants) {
387 try {
388 final RevObject obj = walk.parseAny(r.getObjectId());
389 if (obj instanceof RevCommit) {
390 final int cTime = ((RevCommit) obj).getCommitTime();
391 if (maxTime < cTime)
392 maxTime = cTime;
393 }
394 } catch (IOException error) {
395
396 }
397 }
398 return maxTime;
399 }
400
401 private void markReachable(final Set<ObjectId> have, final int maxTime)
402 throws IOException {
403 Map<String, Ref> refs = local.getRefDatabase().getRefs(ALL);
404 for (final Ref r : refs.values()) {
405 ObjectId id = r.getPeeledObjectId();
406 if (id == null)
407 id = r.getObjectId();
408 if (id == null)
409 continue;
410 parseReachable(id);
411 }
412
413 for (ObjectId id : local.getAdditionalHaves())
414 parseReachable(id);
415
416 for (ObjectId id : have)
417 parseReachable(id);
418
419 if (maxTime > 0) {
420
421
422
423
424 final Date maxWhen = new Date(maxTime * 1000L);
425 walk.sort(RevSort.COMMIT_TIME_DESC);
426 walk.markStart(reachableCommits);
427 walk.setRevFilter(CommitTimeRevFilter.after(maxWhen));
428 for (;;) {
429 final RevCommit c = walk.next();
430 if (c == null)
431 break;
432 if (c.has(ADVERTISED) && !c.has(COMMON)) {
433
434
435
436 c.add(COMMON);
437 c.carry(COMMON);
438 reachableCommits.add(c);
439 }
440 }
441 }
442 }
443
444 private void parseReachable(ObjectId id) {
445 try {
446 RevCommit o = walk.parseCommit(id);
447 if (!o.has(REACHABLE)) {
448 o.add(REACHABLE);
449 reachableCommits.add(o);
450 }
451 } catch (IOException readError) {
452
453 }
454 }
455
456 private boolean sendWants(final Collection<Ref> want) throws IOException {
457 final PacketLineOut p = statelessRPC ? pckState : pckOut;
458 boolean first = true;
459 for (final Ref r : want) {
460 try {
461 if (walk.parseAny(r.getObjectId()).has(REACHABLE)) {
462
463
464
465 continue;
466 }
467 } catch (IOException err) {
468
469
470 }
471
472 final StringBuilder line = new StringBuilder(46);
473 line.append("want ");
474 line.append(r.getObjectId().name());
475 if (first) {
476 line.append(enableCapabilities());
477 first = false;
478 }
479 line.append('\n');
480 p.writeString(line.toString());
481 }
482 if (first)
483 return false;
484 p.end();
485 outNeedsEnd = false;
486 return true;
487 }
488
489 private String enableCapabilities() throws TransportException {
490 final StringBuilder line = new StringBuilder();
491 if (noProgress)
492 wantCapability(line, OPTION_NO_PROGRESS);
493 if (includeTags)
494 includeTags = wantCapability(line, OPTION_INCLUDE_TAG);
495 if (allowOfsDelta)
496 wantCapability(line, OPTION_OFS_DELTA);
497
498 if (wantCapability(line, OPTION_MULTI_ACK_DETAILED)) {
499 multiAck = MultiAck.DETAILED;
500 if (statelessRPC)
501 noDone = wantCapability(line, OPTION_NO_DONE);
502 } else if (wantCapability(line, OPTION_MULTI_ACK))
503 multiAck = MultiAck.CONTINUE;
504 else
505 multiAck = MultiAck.OFF;
506
507 if (thinPack)
508 thinPack = wantCapability(line, OPTION_THIN_PACK);
509 if (wantCapability(line, OPTION_SIDE_BAND_64K))
510 sideband = true;
511 else if (wantCapability(line, OPTION_SIDE_BAND))
512 sideband = true;
513
514 if (statelessRPC && multiAck != MultiAck.DETAILED) {
515
516
517
518
519 throw new PackProtocolException(uri, MessageFormat.format(
520 JGitText.get().statelessRPCRequiresOptionToBeEnabled,
521 OPTION_MULTI_ACK_DETAILED));
522 }
523
524 addUserAgentCapability(line);
525 return line.toString();
526 }
527
528 private void negotiate(final ProgressMonitor monitor) throws IOException,
529 CancelledException {
530 final MutableObjectId ackId = new MutableObjectId();
531 int resultsPending = 0;
532 int havesSent = 0;
533 int havesSinceLastContinue = 0;
534 boolean receivedContinue = false;
535 boolean receivedAck = false;
536 boolean receivedReady = false;
537
538 if (statelessRPC)
539 state.writeTo(out, null);
540
541 negotiateBegin();
542 SEND_HAVES: for (;;) {
543 final RevCommit c = walk.next();
544 if (c == null)
545 break SEND_HAVES;
546
547 pckOut.writeString("have " + c.getId().name() + "\n");
548 havesSent++;
549 havesSinceLastContinue++;
550
551 if ((31 & havesSent) != 0) {
552
553
554
555
556 continue;
557 }
558
559 if (monitor.isCancelled())
560 throw new CancelledException();
561
562 pckOut.end();
563 resultsPending++;
564
565 if (havesSent == 32 && !statelessRPC) {
566
567
568
569
570
571 continue;
572 }
573
574 READ_RESULT: for (;;) {
575 final AckNackResult anr = pckIn.readACK(ackId);
576 switch (anr) {
577 case NAK:
578
579
580
581 resultsPending--;
582 break READ_RESULT;
583
584 case ACK:
585
586
587
588
589 multiAck = MultiAck.OFF;
590 resultsPending = 0;
591 receivedAck = true;
592 if (statelessRPC)
593 state.writeTo(out, null);
594 break SEND_HAVES;
595
596 case ACK_CONTINUE:
597 case ACK_COMMON:
598 case ACK_READY:
599
600
601
602
603
604 markCommon(walk.parseAny(ackId), anr);
605 receivedAck = true;
606 receivedContinue = true;
607 havesSinceLastContinue = 0;
608 if (anr == AckNackResult.ACK_READY)
609 receivedReady = true;
610 break;
611 }
612
613 if (monitor.isCancelled())
614 throw new CancelledException();
615 }
616
617 if (noDone & receivedReady)
618 break SEND_HAVES;
619 if (statelessRPC)
620 state.writeTo(out, null);
621
622 if (receivedContinue && havesSinceLastContinue > MAX_HAVES) {
623
624
625
626
627
628 break SEND_HAVES;
629 }
630 }
631
632
633
634 if (monitor.isCancelled())
635 throw new CancelledException();
636
637 if (!receivedReady || !noDone) {
638
639
640
641
642 pckOut.writeString("done\n");
643 pckOut.flush();
644 }
645
646 if (!receivedAck) {
647
648
649
650
651 multiAck = MultiAck.OFF;
652 resultsPending++;
653 }
654
655 READ_RESULT: while (resultsPending > 0 || multiAck != MultiAck.OFF) {
656 final AckNackResult anr = pckIn.readACK(ackId);
657 resultsPending--;
658 switch (anr) {
659 case NAK:
660
661
662
663 break;
664
665 case ACK:
666
667
668
669 break READ_RESULT;
670
671 case ACK_CONTINUE:
672 case ACK_COMMON:
673 case ACK_READY:
674
675
676 multiAck = MultiAck.CONTINUE;
677 break;
678 }
679
680 if (monitor.isCancelled())
681 throw new CancelledException();
682 }
683 }
684
685 private void negotiateBegin() throws IOException {
686 walk.resetRetain(REACHABLE, ADVERTISED);
687 walk.markStart(reachableCommits);
688 walk.sort(RevSort.COMMIT_TIME_DESC);
689 walk.setRevFilter(new RevFilter() {
690 @Override
691 public RevFilter clone() {
692 return this;
693 }
694
695 @Override
696 public boolean include(final RevWalk walker, final RevCommit c) {
697 final boolean remoteKnowsIsCommon = c.has(COMMON);
698 if (c.has(ADVERTISED)) {
699
700
701
702
703
704 c.add(COMMON);
705 }
706 return !remoteKnowsIsCommon;
707 }
708
709 @Override
710 public boolean requiresCommitBody() {
711 return false;
712 }
713 });
714 }
715
716 private void markRefsAdvertised() {
717 for (final Ref r : getRefs()) {
718 markAdvertised(r.getObjectId());
719 if (r.getPeeledObjectId() != null)
720 markAdvertised(r.getPeeledObjectId());
721 }
722 }
723
724 private void markAdvertised(final AnyObjectId id) {
725 try {
726 walk.parseAny(id).add(ADVERTISED);
727 } catch (IOException readError) {
728
729 }
730 }
731
732 private void markCommon(final RevObject obj, final AckNackResult anr)
733 throws IOException {
734 if (statelessRPC && anr == AckNackResult.ACK_COMMON && !obj.has(STATE)) {
735 StringBuilder s;
736
737 s = new StringBuilder(6 + Constants.OBJECT_ID_STRING_LENGTH);
738 s.append("have ");
739 s.append(obj.name());
740 s.append('\n');
741 pckState.writeString(s.toString());
742 obj.add(STATE);
743 }
744 obj.add(COMMON);
745 if (obj instanceof RevCommit)
746 ((RevCommit) obj).carry(COMMON);
747 }
748
749 private void receivePack(final ProgressMonitor monitor,
750 OutputStream outputStream) throws IOException {
751 onReceivePack();
752 InputStream input = in;
753 if (sideband)
754 input = new SideBandInputStream(input, monitor, getMessageWriter(),
755 outputStream);
756
757 try (ObjectInserter ins = local.newObjectInserter()) {
758 PackParser parser = ins.newPackParser(input);
759 parser.setAllowThin(thinPack);
760 parser.setObjectChecker(transport.getObjectChecker());
761 parser.setLockMessage(lockMessage);
762 packLock = parser.parse(monitor);
763 ins.flush();
764 }
765 }
766
767
768
769
770
771
772
773
774 protected void onReceivePack() {
775
776 }
777
778 private static class CancelledException extends Exception {
779 private static final long serialVersionUID = 1L;
780 }
781 }