1 /*
2 * Copyright (C) 2015, Google Inc.
3 *
4 * This program and the accompanying materials are made available
5 * under the terms of the Eclipse Distribution License v1.0 which
6 * accompanies this distribution, is reproduced below, and is
7 * available at http://www.eclipse.org/org/documents/edl-v10.php
8 *
9 * All rights reserved.
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above copyright
16 * notice, this list of conditions and the following disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials provided
21 * with the distribution.
22 *
23 * - Neither the name of the Eclipse Foundation, Inc. nor the
24 * names of its contributors may be used to endorse or promote
25 * products derived from this software without specific prior
26 * written permission.
27 *
28 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
29 * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
30 * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
31 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
32 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
33 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
34 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
35 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
36 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
37 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
38 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
39 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
40 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
41 */
42
43 package org.eclipse.jgit.storage.pack;
44
45 import static org.eclipse.jgit.lib.Constants.OBJ_BLOB;
46 import static org.eclipse.jgit.lib.Constants.OBJ_COMMIT;
47 import static org.eclipse.jgit.lib.Constants.OBJ_TAG;
48 import static org.eclipse.jgit.lib.Constants.OBJ_TREE;
49
50 import java.text.MessageFormat;
51 import java.util.HashMap;
52 import java.util.List;
53 import java.util.Map;
54 import java.util.Set;
55
56 import org.eclipse.jgit.internal.JGitText;
57 import org.eclipse.jgit.internal.storage.pack.CachedPack;
58 import org.eclipse.jgit.lib.ObjectId;
59
60 /**
61 * Statistics about {@link org.eclipse.jgit.internal.storage.pack.PackWriter}
62 * pack creation.
63 *
64 * @since 4.1
65 */
66 public class PackStatistics {
67 /**
68 * Statistics about a single type of object (commits, tags, trees and
69 * blobs).
70 */
71 public static class ObjectType {
72 /**
73 * POJO for accumulating the ObjectType statistics.
74 */
75 public static class Accumulator {
76 /** Count of objects of this type. */
77 public long cntObjects;
78
79 /** Count of deltas of this type. */
80 public long cntDeltas;
81
82 /** Count of reused objects of this type. */
83 public long reusedObjects;
84
85 /** Count of reused deltas of this type. */
86 public long reusedDeltas;
87
88 /** Count of bytes for all objects of this type. */
89 public long bytes;
90
91 /** Count of delta bytes for objects of this type. */
92 public long deltaBytes;
93 }
94
95 private ObjectType.Accumulator objectType;
96
97 /**
98 * Creates a new {@link ObjectType} object from the accumulator.
99 *
100 * @param accumulator
101 * the accumulator of the statistics
102 */
103 public ObjectType(ObjectType.Accumulator accumulator) {
104 /*
105 * For efficiency this wraps and serves up the Accumulator object
106 * rather than making a deep clone. Normal usage of PackWriter is to
107 * create a single pack/index/bitmap and only call getStatistics()
108 * after all work is complete.
109 */
110 objectType = accumulator;
111 }
112
113 /**
114 * @return total number of objects output. This total includes the value
115 * of {@link #getDeltas()}.
116 */
117 public long getObjects() {
118 return objectType.cntObjects;
119 }
120
121 /**
122 * @return total number of deltas output. This may be lower than the
123 * actual number of deltas if a cached pack was reused.
124 */
125 public long getDeltas() {
126 return objectType.cntDeltas;
127 }
128
129 /**
130 * @return number of objects whose existing representation was reused in
131 * the output. This count includes {@link #getReusedDeltas()}.
132 */
133 public long getReusedObjects() {
134 return objectType.reusedObjects;
135 }
136
137 /**
138 * @return number of deltas whose existing representation was reused in
139 * the output, as their base object was also output or was
140 * assumed present for a thin pack. This may be lower than the
141 * actual number of reused deltas if a cached pack was reused.
142 */
143 public long getReusedDeltas() {
144 return objectType.reusedDeltas;
145 }
146
147 /**
148 * @return total number of bytes written. This size includes the object
149 * headers as well as the compressed data. This size also
150 * includes all of {@link #getDeltaBytes()}.
151 */
152 public long getBytes() {
153 return objectType.bytes;
154 }
155
156 /**
157 * @return number of delta bytes written. This size includes the object
158 * headers for the delta objects.
159 */
160 public long getDeltaBytes() {
161 return objectType.deltaBytes;
162 }
163 }
164
165 /**
166 * POJO for accumulating the statistics.
167 */
168 public static class Accumulator {
169 /**
170 * The count of references in the ref advertisement.
171 *
172 * @since 4.11
173 */
174 public long advertised;
175
176 /**
177 * The count of client wants.
178 *
179 * @since 4.11
180 */
181 public long wants;
182
183 /**
184 * The count of client haves.
185 *
186 * @since 4.11
187 */
188 public long haves;
189
190 /**
191 * The count of wants that were not advertised by the server.
192 *
193 * @since 5.10
194 */
195 public long notAdvertisedWants;
196
197 /**
198 * Time in ms spent in the negotiation phase. For non-bidirectional
199 * transports (e.g., HTTP), this is only for the final request that
200 * sends back the pack file.
201 *
202 * @since 4.11
203 */
204 public long timeNegotiating;
205
206 /** The set of objects to be included in the pack. */
207 public Set<ObjectId> interestingObjects;
208
209 /** The set of objects to be excluded from the pack. */
210 public Set<ObjectId> uninterestingObjects;
211
212 /** The set of shallow commits on the client. */
213 public Set<ObjectId> clientShallowCommits;
214
215 /** The collection of reused packs in the upload. */
216 public List<CachedPack> reusedPacks;
217
218 /** Commits with no parents. */
219 public Set<ObjectId> rootCommits;
220
221 /** If a shallow pack, the depth in commits. */
222 public int depth;
223
224 /**
225 * The count of objects in the pack that went through the delta search
226 * process in order to find a potential delta base.
227 */
228 public int deltaSearchNonEdgeObjects;
229
230 /**
231 * The count of objects in the pack that went through delta base search
232 * and found a suitable base. This is a subset of
233 * deltaSearchNonEdgeObjects.
234 */
235 public int deltasFound;
236
237 /** The total count of objects in the pack. */
238 public long totalObjects;
239
240 /**
241 * The count of objects that needed to be discovered through an object
242 * walk because they were not found in bitmap indices.
243 */
244 public long bitmapIndexMisses;
245
246 /** The total count of deltas output. */
247 public long totalDeltas;
248
249 /** The count of reused objects in the pack. */
250 public long reusedObjects;
251
252 /** The count of reused deltas in the pack. */
253 public long reusedDeltas;
254
255 /** The count of total bytes in the pack. */
256 public long totalBytes;
257
258 /** The size of the thin pack in bytes, if a thin pack was generated. */
259 public long thinPackBytes;
260
261 /** Time in ms spent counting the objects that will go into the pack. */
262 public long timeCounting;
263
264 /** Time in ms spent searching for objects to reuse. */
265 public long timeSearchingForReuse;
266
267 /** Time in ms spent searching for sizes of objects. */
268 public long timeSearchingForSizes;
269
270 /** Time in ms spent compressing the pack. */
271 public long timeCompressing;
272
273 /** Time in ms spent writing the pack. */
274 public long timeWriting;
275
276 /** Time in ms spent checking reachability.
277 *
278 * @since 5.10
279 */
280 public long reachabilityCheckDuration;
281
282 /** Number of trees traversed in the walk when writing the pack.
283 *
284 * @since 5.4
285 */
286 public long treesTraversed;
287
288 /**
289 * Amount of packfile uris sent to the client to download via HTTP.
290 *
291 * @since 5.6
292 */
293 public long offloadedPackfiles;
294
295 /**
296 * Total size (in bytes) offloaded to HTTP downloads.
297 *
298 * @since 5.6
299 */
300 public long offloadedPackfileSize;
301
302 /**
303 * Statistics about each object type in the pack (commits, tags, trees
304 * and blobs.)
305 */
306 public ObjectType.Accumulator[] objectTypes;
307
308 {
309 objectTypes = new ObjectType.Accumulator[5];
310 objectTypes[OBJ_COMMIT] = new ObjectType.Accumulator();
311 objectTypes[OBJ_TREE] = new ObjectType.Accumulator();
312 objectTypes[OBJ_BLOB] = new ObjectType.Accumulator();
313 objectTypes[OBJ_TAG] = new ObjectType.Accumulator();
314 }
315 }
316
317 private Accumulator statistics;
318
319 /**
320 * Creates a new {@link org.eclipse.jgit.storage.pack.PackStatistics} object
321 * from the accumulator.
322 *
323 * @param accumulator
324 * the accumulator of the statistics
325 */
326 public PackStatistics(Accumulator accumulator) {
327 /*
328 * For efficiency this wraps and serves up the Accumulator object rather
329 * than making a deep clone. Normal usage of PackWriter is to create a
330 * single pack/index/bitmap and only call getStatistics() after all work
331 * is complete.
332 */
333 statistics = accumulator;
334 }
335
336 /**
337 * Get the count of references in the ref advertisement.
338 *
339 * @return count of refs in the ref advertisement.
340 * @since 4.11
341 */
342 public long getAdvertised() {
343 return statistics.advertised;
344 }
345
346 /**
347 * Get the count of client wants.
348 *
349 * @return count of client wants.
350 * @since 4.11
351 */
352 public long getWants() {
353 return statistics.wants;
354 }
355
356 /**
357 * Get the count of client haves.
358 *
359 * @return count of client haves.
360 * @since 4.11
361 */
362 public long getHaves() {
363 return statistics.haves;
364 }
365
366 /**
367 * Get the count of client wants that were not advertised by the server.
368 *
369 * @return count of client wants that were not advertised by the server.
370 * @since 5.10
371 */
372 public long getNotAdvertisedWants() {
373 return statistics.notAdvertisedWants;
374 }
375
376 /**
377 * Time in ms spent in the negotiation phase. For non-bidirectional
378 * transports (e.g., HTTP), this is only for the final request that sends
379 * back the pack file.
380 *
381 * @return time for ref advertisement in ms.
382 * @since 4.11
383 */
384 public long getTimeNegotiating() {
385 return statistics.timeNegotiating;
386 }
387
388 /**
389 * Get unmodifiable collection of objects to be included in the pack.
390 *
391 * @return unmodifiable collection of objects to be included in the pack.
392 * May be {@code null} if the pack was hand-crafted in a unit test.
393 */
394 public Set<ObjectId> getInterestingObjects() {
395 return statistics.interestingObjects;
396 }
397
398 /**
399 * Get unmodifiable collection of objects that should be excluded from the
400 * pack
401 *
402 * @return unmodifiable collection of objects that should be excluded from
403 * the pack, as the peer that will receive the pack already has
404 * these objects.
405 */
406 public Set<ObjectId> getUninterestingObjects() {
407 return statistics.uninterestingObjects;
408 }
409
410 /**
411 * Get unmodifiable collection of objects that were shallow commits on the
412 * client.
413 *
414 * @return unmodifiable collection of objects that were shallow commits on
415 * the client.
416 */
417 public Set<ObjectId> getClientShallowCommits() {
418 return statistics.clientShallowCommits;
419 }
420
421 /**
422 * Get unmodifiable list of the cached packs that were reused in the output
423 *
424 * @return unmodifiable list of the cached packs that were reused in the
425 * output, if any were selected for reuse.
426 */
427 public List<CachedPack> getReusedPacks() {
428 return statistics.reusedPacks;
429 }
430
431 /**
432 * Get unmodifiable collection of the root commits of the history.
433 *
434 * @return unmodifiable collection of the root commits of the history.
435 */
436 public Set<ObjectId> getRootCommits() {
437 return statistics.rootCommits;
438 }
439
440 /**
441 * Get number of objects in the output pack that went through the delta
442 * search process in order to find a potential delta base.
443 *
444 * @return number of objects in the output pack that went through the delta
445 * search process in order to find a potential delta base.
446 */
447 public int getDeltaSearchNonEdgeObjects() {
448 return statistics.deltaSearchNonEdgeObjects;
449 }
450
451 /**
452 * Get number of objects in the output pack that went through delta base
453 * search and found a suitable base.
454 *
455 * @return number of objects in the output pack that went through delta base
456 * search and found a suitable base. This is a subset of
457 * {@link #getDeltaSearchNonEdgeObjects()}.
458 */
459 public int getDeltasFound() {
460 return statistics.deltasFound;
461 }
462
463 /**
464 * Get total number of objects output.
465 *
466 * @return total number of objects output. This total includes the value of
467 * {@link #getTotalDeltas()}.
468 */
469 public long getTotalObjects() {
470 return statistics.totalObjects;
471 }
472
473 /**
474 * Get the count of objects that needed to be discovered through an object
475 * walk because they were not found in bitmap indices.
476 *
477 * @return the count of objects that needed to be discovered through an
478 * object walk because they were not found in bitmap indices.
479 * Returns -1 if no bitmap indices were found.
480 */
481 public long getBitmapIndexMisses() {
482 return statistics.bitmapIndexMisses;
483 }
484
485 /**
486 * Get total number of deltas output.
487 *
488 * @return total number of deltas output. This may be lower than the actual
489 * number of deltas if a cached pack was reused.
490 */
491 public long getTotalDeltas() {
492 return statistics.totalDeltas;
493 }
494
495 /**
496 * Get number of objects whose existing representation was reused in the
497 * output.
498 *
499 * @return number of objects whose existing representation was reused in the
500 * output. This count includes {@link #getReusedDeltas()}.
501 */
502 public long getReusedObjects() {
503 return statistics.reusedObjects;
504 }
505
506 /**
507 * Get number of deltas whose existing representation was reused in the
508 * output.
509 *
510 * @return number of deltas whose existing representation was reused in the
511 * output, as their base object was also output or was assumed
512 * present for a thin pack. This may be lower than the actual number
513 * of reused deltas if a cached pack was reused.
514 */
515 public long getReusedDeltas() {
516 return statistics.reusedDeltas;
517 }
518
519 /**
520 * Get total number of bytes written.
521 *
522 * @return total number of bytes written. This size includes the pack
523 * header, trailer, thin pack, and reused cached pack(s).
524 */
525 public long getTotalBytes() {
526 return statistics.totalBytes;
527 }
528
529 /**
530 * Get size of the thin pack in bytes.
531 *
532 * @return size of the thin pack in bytes, if a thin pack was generated. A
533 * thin pack is created when the client already has objects and some
534 * deltas are created against those objects, or if a cached pack is
535 * being used and some deltas will reference objects in the cached
536 * pack. This size does not include the pack header or trailer.
537 */
538 public long getThinPackBytes() {
539 return statistics.thinPackBytes;
540 }
541
542 /**
543 * Get information about this type of object in the pack.
544 *
545 * @param typeCode
546 * object type code, e.g. OBJ_COMMIT or OBJ_TREE.
547 * @return information about this type of object in the pack.
548 */
549 public ObjectType byObjectType(int typeCode) {
550 return new ObjectType(statistics.objectTypes[typeCode]);
551 }
552
553 /**
554 * Whether the resulting pack file was a shallow pack.
555 *
556 * @return {@code true} if the resulting pack file was a shallow pack.
557 */
558 public boolean isShallow() {
559 return statistics.depth > 0;
560 }
561
562 /**
563 * Get depth (in commits) the pack includes if shallow.
564 *
565 * @return depth (in commits) the pack includes if shallow.
566 */
567 public int getDepth() {
568 return statistics.depth;
569 }
570
571 /**
572 * Get time in milliseconds spent enumerating the objects that need to be
573 * included in the output.
574 *
575 * @return time in milliseconds spent enumerating the objects that need to
576 * be included in the output. This time includes any restarts that
577 * occur when a cached pack is selected for reuse.
578 */
579 public long getTimeCounting() {
580 return statistics.timeCounting;
581 }
582
583 /**
584 * Get time in milliseconds spent matching existing representations against
585 * objects that will be transmitted.
586 *
587 * @return time in milliseconds spent matching existing representations
588 * against objects that will be transmitted, or that the client can
589 * be assumed to already have.
590 */
591 public long getTimeSearchingForReuse() {
592 return statistics.timeSearchingForReuse;
593 }
594
595 /**
596 * Get time in milliseconds spent finding the sizes of all objects that will
597 * enter the delta compression search window.
598 *
599 * @return time in milliseconds spent finding the sizes of all objects that
600 * will enter the delta compression search window. The sizes need to
601 * be known to better match similar objects together and improve
602 * delta compression ratios.
603 */
604 public long getTimeSearchingForSizes() {
605 return statistics.timeSearchingForSizes;
606 }
607
608 /**
609 * Get time in milliseconds spent on delta compression.
610 *
611 * @return time in milliseconds spent on delta compression. This is observed
612 * wall-clock time and does not accurately track CPU time used when
613 * multiple threads were used to perform the delta compression.
614 */
615 public long getTimeCompressing() {
616 return statistics.timeCompressing;
617 }
618
619 /**
620 * Get time in milliseconds spent writing the pack output, from start of
621 * header until end of trailer.
622 *
623 * @return time in milliseconds spent writing the pack output, from start of
624 * header until end of trailer. The transfer speed can be
625 * approximated by dividing {@link #getTotalBytes()} by this value.
626 */
627 public long getTimeWriting() {
628 return statistics.timeWriting;
629 }
630
631 /**
632 * Get time in milliseconds spent checking if the client has access to the
633 * commits they are requesting.
634 *
635 * @return time in milliseconds spent checking if the client has access to the
636 * commits they are requesting.
637 * @since 5.10
638 */
639 public long getReachabilityCheckDuration() {
640 return statistics.reachabilityCheckDuration;
641 }
642
643 /**
644 * @return number of trees traversed in the walk when writing the pack.
645 * @since 5.4
646 */
647 public long getTreesTraversed() {
648 return statistics.treesTraversed;
649 }
650
651 /**
652 * @return amount of packfiles offloaded (sent as "packfile-uri")/
653 * @since 5.6
654 */
655 public long getOffloadedPackfiles() {
656 return statistics.offloadedPackfiles;
657 }
658
659 /**
660 * @return total size (in bytes) offloaded to HTTP downloads.
661 * @since 5.6
662 */
663 public long getOffloadedPackfilesSize() {
664 return statistics.offloadedPackfileSize;
665 }
666
667 /**
668 * Get total time spent processing this pack.
669 *
670 * @return total time spent processing this pack.
671 */
672 public long getTimeTotal() {
673 return statistics.timeCounting + statistics.timeSearchingForReuse
674 + statistics.timeSearchingForSizes + statistics.timeCompressing
675 + statistics.timeWriting;
676 }
677
678 /**
679 * Get the average output speed in terms of bytes-per-second.
680 *
681 * @return the average output speed in terms of bytes-per-second.
682 * {@code getTotalBytes() / (getTimeWriting() / 1000.0)}.
683 */
684 public double getTransferRate() {
685 return getTotalBytes() / (getTimeWriting() / 1000.0);
686 }
687
688 /**
689 * Get formatted message string for display to clients.
690 *
691 * @return formatted message string for display to clients.
692 */
693 public String getMessage() {
694 return MessageFormat.format(JGitText.get().packWriterStatistics,
695 Long.valueOf(statistics.totalObjects),
696 Long.valueOf(statistics.totalDeltas),
697 Long.valueOf(statistics.reusedObjects),
698 Long.valueOf(statistics.reusedDeltas));
699 }
700
701 /**
702 * Get a map containing ObjectType statistics.
703 *
704 * @return a map containing ObjectType statistics.
705 */
706 public Map<Integer, ObjectType> getObjectTypes() {
707 HashMap<Integer, ObjectType> map = new HashMap<>();
708 map.put(Integer.valueOf(OBJ_BLOB), new ObjectType(
709 statistics.objectTypes[OBJ_BLOB]));
710 map.put(Integer.valueOf(OBJ_COMMIT), new ObjectType(
711 statistics.objectTypes[OBJ_COMMIT]));
712 map.put(Integer.valueOf(OBJ_TAG), new ObjectType(
713 statistics.objectTypes[OBJ_TAG]));
714 map.put(Integer.valueOf(OBJ_TREE), new ObjectType(
715 statistics.objectTypes[OBJ_TREE]));
716 return map;
717 }
718 }