1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44 package org.eclipse.jgit.pgm.debug;
45
46 import static java.nio.charset.StandardCharsets.UTF_8;
47 import static org.eclipse.jgit.lib.Constants.HEAD;
48 import static org.eclipse.jgit.lib.Constants.MASTER;
49 import static org.eclipse.jgit.lib.Constants.R_HEADS;
50 import static org.eclipse.jgit.lib.Ref.Storage.NEW;
51 import static org.eclipse.jgit.lib.Ref.Storage.PACKED;
52
53 import java.io.BufferedReader;
54 import java.io.FileInputStream;
55 import java.io.FileNotFoundException;
56 import java.io.FileOutputStream;
57 import java.io.IOException;
58 import java.io.InputStreamReader;
59 import java.io.OutputStream;
60 import java.util.ArrayList;
61 import java.util.Collections;
62 import java.util.List;
63 import java.util.regex.Matcher;
64 import java.util.regex.Pattern;
65
66 import org.eclipse.jgit.internal.storage.reftable.ReftableConfig;
67 import org.eclipse.jgit.internal.storage.reftable.ReftableWriter;
68 import org.eclipse.jgit.lib.ObjectId;
69 import org.eclipse.jgit.lib.ObjectIdRef;
70 import org.eclipse.jgit.lib.PersonIdent;
71 import org.eclipse.jgit.lib.Ref;
72 import org.eclipse.jgit.lib.SymbolicRef;
73 import org.eclipse.jgit.pgm.Command;
74 import org.eclipse.jgit.pgm.TextBuiltin;
75 import org.kohsuke.args4j.Argument;
76 import org.kohsuke.args4j.Option;
77
78 @Command
79 class WriteReftable extends TextBuiltin {
80 private static final int KIB = 1 << 10;
81 private static final int MIB = 1 << 20;
82
83 @Option(name = "--block-size")
84 private int refBlockSize;
85
86 @Option(name = "--log-block-size")
87 private int logBlockSize;
88
89 @Option(name = "--restart-interval")
90 private int restartInterval;
91
92 @Option(name = "--index-levels")
93 private int indexLevels;
94
95 @Option(name = "--reflog-in")
96 private String reflogIn;
97
98 @Option(name = "--no-index-objects")
99 private boolean noIndexObjects;
100
101 @Argument(index = 0)
102 private String in;
103
104 @Argument(index = 1)
105 private String out;
106
107 @SuppressWarnings({ "nls", "boxing" })
108 @Override
109 protected void run() throws Exception {
110 List<Ref> refs = readRefs(in);
111 List<LogEntry> logs = readLog(reflogIn);
112
113 ReftableWriter.Stats stats;
114 try (OutputStream os = new FileOutputStream(out)) {
115 ReftableConfig cfg = new ReftableConfig();
116 cfg.setIndexObjects(!noIndexObjects);
117 if (refBlockSize > 0) {
118 cfg.setRefBlockSize(refBlockSize);
119 }
120 if (logBlockSize > 0) {
121 cfg.setLogBlockSize(logBlockSize);
122 }
123 if (restartInterval > 0) {
124 cfg.setRestartInterval(restartInterval);
125 }
126 if (indexLevels > 0) {
127 cfg.setMaxIndexLevels(indexLevels);
128 }
129
130 ReftableWriter w = new ReftableWriter(cfg);
131 w.setMinUpdateIndex(min(logs)).setMaxUpdateIndex(max(logs));
132 w.begin(os);
133 w.sortAndWriteRefs(refs);
134 for (LogEntry e : logs) {
135 w.writeLog(e.ref, e.updateIndex, e.who,
136 e.oldId, e.newId, e.message);
137 }
138 stats = w.finish().getStats();
139 }
140
141 double fileMiB = ((double) stats.totalBytes()) / MIB;
142 printf("Summary:");
143 printf(" file sz : %.1f MiB (%d bytes)", fileMiB, stats.totalBytes());
144 printf(" padding : %d KiB", stats.paddingBytes() / KIB);
145 errw.println();
146
147 printf("Refs:");
148 printf(" ref blk : %d", stats.refBlockSize());
149 printf(" restarts: %d", stats.restartInterval());
150 printf(" refs : %d", stats.refCount());
151 if (stats.refIndexLevels() > 0) {
152 int idxSize = (int) Math.round(((double) stats.refIndexSize()) / KIB);
153 printf(" idx sz : %d KiB", idxSize);
154 printf(" idx lvl : %d", stats.refIndexLevels());
155 }
156 printf(" avg ref : %d bytes", stats.refBytes() / refs.size());
157 errw.println();
158
159 if (stats.objCount() > 0) {
160 int objMiB = (int) Math.round(((double) stats.objBytes()) / MIB);
161 int idLen = stats.objIdLength();
162 printf("Objects:");
163 printf(" obj blk : %d", stats.refBlockSize());
164 printf(" restarts: %d", stats.restartInterval());
165 printf(" objects : %d", stats.objCount());
166 printf(" obj sz : %d MiB (%d bytes)", objMiB, stats.objBytes());
167 if (stats.objIndexSize() > 0) {
168 int s = (int) Math.round(((double) stats.objIndexSize()) / KIB);
169 printf(" idx sz : %d KiB", s);
170 printf(" idx lvl : %d", stats.objIndexLevels());
171 }
172 printf(" id len : %d bytes (%d hex digits)", idLen, 2 * idLen);
173 printf(" avg obj : %d bytes", stats.objBytes() / stats.objCount());
174 errw.println();
175 }
176 if (stats.logCount() > 0) {
177 int logMiB = (int) Math.round(((double) stats.logBytes()) / MIB);
178 printf("Log:");
179 printf(" log blk : %d", stats.logBlockSize());
180 printf(" logs : %d", stats.logCount());
181 printf(" log sz : %d MiB (%d bytes)", logMiB, stats.logBytes());
182 printf(" avg log : %d bytes", stats.logBytes() / logs.size());
183 errw.println();
184 }
185 }
186
187 private void printf(String fmt, Object... args) throws IOException {
188 errw.println(String.format(fmt, args));
189 }
190
191 static List<Ref> readRefs(String inputFile) throws IOException {
192 List<Ref> refs = new ArrayList<>();
193 try (BufferedReader br = new BufferedReader(
194 new InputStreamReader(new FileInputStream(inputFile), UTF_8))) {
195 String line;
196 while ((line = br.readLine()) != null) {
197 ObjectId id = ObjectId.fromString(line.substring(0, 40));
198 String name = line.substring(41, line.length());
199 if (name.endsWith("^{}")) {
200 int lastIdx = refs.size() - 1;
201 Ref last = refs.get(lastIdx);
202 refs.set(lastIdx, new ObjectIdRef.PeeledTag(PACKED,
203 last.getName(), last.getObjectId(), id));
204 continue;
205 }
206
207 Ref ref;
208 if (name.equals(HEAD)) {
209 ref = new SymbolicRef(name, new ObjectIdRef.Unpeeled(NEW,
210 R_HEADS + MASTER, null));
211 } else {
212 ref = new ObjectIdRef.PeeledNonTag(PACKED, name, id);
213 }
214 refs.add(ref);
215 }
216 }
217 Collections.sort(refs, (a, b) -> a.getName().compareTo(b.getName()));
218 return refs;
219 }
220
221 private static List<LogEntry> readLog(String logPath)
222 throws FileNotFoundException, IOException {
223 if (logPath == null) {
224 return Collections.emptyList();
225 }
226
227 List<LogEntry> log = new ArrayList<>();
228 try (BufferedReader br = new BufferedReader(
229 new InputStreamReader(new FileInputStream(logPath), UTF_8))) {
230 @SuppressWarnings("nls")
231 Pattern pattern = Pattern.compile("([^,]+)"
232 + ",([0-9]+(?:[.][0-9]+)?)"
233 + ",([^,]+)"
234 + ",([^,]+)"
235 + ",([^,]+)"
236 + ",(.*)");
237 String line;
238 while ((line = br.readLine()) != null) {
239 Matcher m = pattern.matcher(line);
240 if (!m.matches()) {
241 throw new IOException("unparsed line: " + line);
242 }
243 String ref = m.group(1);
244 double t = Double.parseDouble(m.group(2));
245 long time = ((long) t) * 1000L;
246 long index = (long) (t * 1e6);
247 String user = m.group(3);
248 ObjectId oldId = parseId(m.group(4));
249 ObjectId newId = parseId(m.group(5));
250 String msg = m.group(6);
251 String email = user + "@gerrit";
252 PersonIdent who = new PersonIdent(user, email, time, -480);
253 log.add(new LogEntry(ref, index, who, oldId, newId, msg));
254 }
255 }
256 Collections.sort(log, LogEntry::compare);
257 return log;
258 }
259
260 private static long min(List<LogEntry> log) {
261 return log.stream().mapToLong(e -> e.updateIndex).min().orElse(0);
262 }
263
264 private static long max(List<LogEntry> log) {
265 return log.stream().mapToLong(e -> e.updateIndex).max().orElse(0);
266 }
267
268 private static ObjectId parseId(String s) {
269 if ("NULL".equals(s)) {
270 return ObjectId.zeroId();
271 }
272 return ObjectId.fromString(s);
273 }
274
275 private static class LogEntry {
276 static int compare(LogEntry a, LogEntry b) {
277 int cmp = a.ref.compareTo(b.ref);
278 if (cmp == 0) {
279 cmp = Long.signum(b.updateIndex - a.updateIndex);
280 }
281 return cmp;
282 }
283
284 final String ref;
285 final long updateIndex;
286 final PersonIdent who;
287 final ObjectId oldId;
288 final ObjectId newId;
289 final String message;
290
291 LogEntry(String ref, long updateIndex, PersonIdent who,
292 ObjectId oldId, ObjectId newId, String message) {
293 this.ref = ref;
294 this.updateIndex = updateIndex;
295 this.who = who;
296 this.oldId = oldId;
297 this.newId = newId;
298 this.message = message;
299 }
300 }
301 }