View Javadoc
1   /*
2    * Copyright (C) 2017, Google Inc.
3    * and other copyright owners as documented in the project's IP log.
4    *
5    * This program and the accompanying materials are made available
6    * under the terms of the Eclipse Distribution License v1.0 which
7    * accompanies this distribution, is reproduced below, and is
8    * available at http://www.eclipse.org/org/documents/edl-v10.php
9    *
10   * All rights reserved.
11   *
12   * Redistribution and use in source and binary forms, with or
13   * without modification, are permitted provided that the following
14   * conditions are met:
15   *
16   * - Redistributions of source code must retain the above copyright
17   *   notice, this list of conditions and the following disclaimer.
18   *
19   * - Redistributions in binary form must reproduce the above
20   *   copyright notice, this list of conditions and the following
21   *   disclaimer in the documentation and/or other materials provided
22   *   with the distribution.
23   *
24   * - Neither the name of the Eclipse Foundation, Inc. nor the
25   *   names of its contributors may be used to endorse or promote
26   *   products derived from this software without specific prior
27   *   written permission.
28   *
29   * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
30   * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
31   * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
32   * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
33   * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
34   * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
35   * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
36   * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
37   * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
38   * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
39   * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
40   * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
41   * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42   */
43  
44  package org.eclipse.jgit.pgm.debug;
45  
46  import static java.nio.charset.StandardCharsets.UTF_8;
47  import static org.eclipse.jgit.lib.Constants.HEAD;
48  import static org.eclipse.jgit.lib.Constants.MASTER;
49  import static org.eclipse.jgit.lib.Constants.R_HEADS;
50  import static org.eclipse.jgit.lib.Ref.Storage.NEW;
51  import static org.eclipse.jgit.lib.Ref.Storage.PACKED;
52  
53  import java.io.BufferedReader;
54  import java.io.FileInputStream;
55  import java.io.FileNotFoundException;
56  import java.io.FileOutputStream;
57  import java.io.IOException;
58  import java.io.InputStreamReader;
59  import java.io.OutputStream;
60  import java.util.ArrayList;
61  import java.util.Collections;
62  import java.util.List;
63  import java.util.regex.Matcher;
64  import java.util.regex.Pattern;
65  
66  import org.eclipse.jgit.internal.storage.reftable.ReftableConfig;
67  import org.eclipse.jgit.internal.storage.reftable.ReftableWriter;
68  import org.eclipse.jgit.lib.ObjectId;
69  import org.eclipse.jgit.lib.ObjectIdRef;
70  import org.eclipse.jgit.lib.PersonIdent;
71  import org.eclipse.jgit.lib.Ref;
72  import org.eclipse.jgit.lib.SymbolicRef;
73  import org.eclipse.jgit.pgm.Command;
74  import org.eclipse.jgit.pgm.TextBuiltin;
75  import org.kohsuke.args4j.Argument;
76  import org.kohsuke.args4j.Option;
77  
78  @Command
79  class WriteReftable extends TextBuiltin {
80  	private static final int KIB = 1 << 10;
81  	private static final int MIB = 1 << 20;
82  
83  	@Option(name = "--block-size")
84  	private int refBlockSize;
85  
86  	@Option(name = "--log-block-size")
87  	private int logBlockSize;
88  
89  	@Option(name = "--restart-interval")
90  	private int restartInterval;
91  
92  	@Option(name = "--index-levels")
93  	private int indexLevels;
94  
95  	@Option(name = "--reflog-in")
96  	private String reflogIn;
97  
98  	@Option(name = "--no-index-objects")
99  	private boolean noIndexObjects;
100 
101 	@Argument(index = 0)
102 	private String in;
103 
104 	@Argument(index = 1)
105 	private String out;
106 
107 	/** {@inheritDoc} */
108 	@SuppressWarnings({ "nls", "boxing" })
109 	@Override
110 	protected void run() throws Exception {
111 		List<Ref> refs = readRefs(in);
112 		List<LogEntry> logs = readLog(reflogIn);
113 
114 		ReftableWriter.Stats stats;
115 		try (OutputStream os = new FileOutputStream(out)) {
116 			ReftableConfig cfg = new ReftableConfig();
117 			cfg.setIndexObjects(!noIndexObjects);
118 			if (refBlockSize > 0) {
119 				cfg.setRefBlockSize(refBlockSize);
120 			}
121 			if (logBlockSize > 0) {
122 				cfg.setLogBlockSize(logBlockSize);
123 			}
124 			if (restartInterval > 0) {
125 				cfg.setRestartInterval(restartInterval);
126 			}
127 			if (indexLevels > 0) {
128 				cfg.setMaxIndexLevels(indexLevels);
129 			}
130 
131 			ReftableWriter w = new ReftableWriter(cfg);
132 			w.setMinUpdateIndex(min(logs)).setMaxUpdateIndex(max(logs));
133 			w.begin(os);
134 			w.sortAndWriteRefs(refs);
135 			for (LogEntry e : logs) {
136 				w.writeLog(e.ref, e.updateIndex, e.who,
137 						e.oldId, e.newId, e.message);
138 			}
139 			stats = w.finish().getStats();
140 		}
141 
142 		double fileMiB = ((double) stats.totalBytes()) / MIB;
143 		printf("Summary:");
144 		printf("  file sz : %.1f MiB (%d bytes)", fileMiB, stats.totalBytes());
145 		printf("  padding : %d KiB", stats.paddingBytes() / KIB);
146 		errw.println();
147 
148 		printf("Refs:");
149 		printf("  ref blk : %d", stats.refBlockSize());
150 		printf("  restarts: %d", stats.restartInterval());
151 		printf("  refs    : %d", stats.refCount());
152 		if (stats.refIndexLevels() > 0) {
153 			int idxSize = (int) Math.round(((double) stats.refIndexSize()) / KIB);
154 			printf("  idx sz  : %d KiB", idxSize);
155 			printf("  idx lvl : %d", stats.refIndexLevels());
156 		}
157 		printf("  avg ref : %d bytes", stats.refBytes() / refs.size());
158 		errw.println();
159 
160 		if (stats.objCount() > 0) {
161 			int objMiB = (int) Math.round(((double) stats.objBytes()) / MIB);
162 			int idLen = stats.objIdLength();
163 			printf("Objects:");
164 			printf("  obj blk : %d", stats.refBlockSize());
165 			printf("  restarts: %d", stats.restartInterval());
166 			printf("  objects : %d", stats.objCount());
167 			printf("  obj sz  : %d MiB (%d bytes)", objMiB, stats.objBytes());
168 			if (stats.objIndexSize() > 0) {
169 				int s = (int) Math.round(((double) stats.objIndexSize()) / KIB);
170 				printf("  idx sz  : %d KiB", s);
171 				printf("  idx lvl : %d", stats.objIndexLevels());
172 			}
173 			printf("  id len  : %d bytes (%d hex digits)", idLen, 2 * idLen);
174 			printf("  avg obj : %d bytes", stats.objBytes() / stats.objCount());
175 			errw.println();
176 		}
177 		if (stats.logCount() > 0) {
178 			int logMiB = (int) Math.round(((double) stats.logBytes()) / MIB);
179 			printf("Log:");
180 			printf("  log blk : %d", stats.logBlockSize());
181 			printf("  logs    : %d", stats.logCount());
182 			printf("  log sz  : %d MiB (%d bytes)", logMiB, stats.logBytes());
183 			printf("  avg log : %d bytes", stats.logBytes() / logs.size());
184 			errw.println();
185 		}
186 	}
187 
188 	private void printf(String fmt, Object... args) throws IOException {
189 		errw.println(String.format(fmt, args));
190 	}
191 
192 	static List<Ref> readRefs(String inputFile) throws IOException {
193 		List<Ref> refs = new ArrayList<>();
194 		try (BufferedReader br = new BufferedReader(
195 				new InputStreamReader(new FileInputStream(inputFile), UTF_8))) {
196 			String line;
197 			while ((line = br.readLine()) != null) {
198 				ObjectId id = ObjectId.fromString(line.substring(0, 40));
199 				String name = line.substring(41, line.length());
200 				if (name.endsWith("^{}")) { //$NON-NLS-1$
201 					int lastIdx = refs.size() - 1;
202 					Ref last = refs.get(lastIdx);
203 					refs.set(lastIdx, new ObjectIdRef.PeeledTag(PACKED,
204 							last.getName(), last.getObjectId(), id));
205 					continue;
206 				}
207 
208 				Ref ref;
209 				if (name.equals(HEAD)) {
210 					ref = new SymbolicRef(name, new ObjectIdRef.Unpeeled(NEW,
211 							R_HEADS + MASTER, null));
212 				} else {
213 					ref = new ObjectIdRef.PeeledNonTag(PACKED, name, id);
214 				}
215 				refs.add(ref);
216 			}
217 		}
218 		Collections.sort(refs, (a, b) -> a.getName().compareTo(b.getName()));
219 		return refs;
220 	}
221 
222 	private static List<LogEntry> readLog(String logPath)
223 			throws FileNotFoundException, IOException {
224 		if (logPath == null) {
225 			return Collections.emptyList();
226 		}
227 
228 		List<LogEntry> log = new ArrayList<>();
229 		try (BufferedReader br = new BufferedReader(
230 				new InputStreamReader(new FileInputStream(logPath), UTF_8))) {
231 			@SuppressWarnings("nls")
232 			Pattern pattern = Pattern.compile("([^,]+)" // 1: ref
233 					+ ",([0-9]+(?:[.][0-9]+)?)" // 2: time
234 					+ ",([^,]+)" // 3: who
235 					+ ",([^,]+)" // 4: old
236 					+ ",([^,]+)" // 5: new
237 					+ ",(.*)"); // 6: msg
238 			String line;
239 			while ((line = br.readLine()) != null) {
240 				Matcher m = pattern.matcher(line);
241 				if (!m.matches()) {
242 					throw new IOException("unparsed line: " + line); //$NON-NLS-1$
243 				}
244 				String ref = m.group(1);
245 				double t = Double.parseDouble(m.group(2));
246 				long time = ((long) t) * 1000L;
247 				long index = (long) (t * 1e6);
248 				String user = m.group(3);
249 				ObjectId oldId = parseId(m.group(4));
250 				ObjectId newId = parseId(m.group(5));
251 				String msg = m.group(6);
252 				String email = user + "@gerrit"; //$NON-NLS-1$
253 				PersonIdent who = new PersonIdent(user, email, time, -480);
254 				log.add(new LogEntry(ref, index, who, oldId, newId, msg));
255 			}
256 		}
257 		Collections.sort(log, LogEntry::compare);
258 		return log;
259 	}
260 
261 	private static long min(List<LogEntry> log) {
262 		return log.stream().mapToLong(e -> e.updateIndex).min().orElse(0);
263 	}
264 
265 	private static long max(List<LogEntry> log) {
266 		return log.stream().mapToLong(e -> e.updateIndex).max().orElse(0);
267 	}
268 
269 	private static ObjectId parseId(String s) {
270 		if ("NULL".equals(s)) { //$NON-NLS-1$
271 			return ObjectId.zeroId();
272 		}
273 		return ObjectId.fromString(s);
274 	}
275 
276 	private static class LogEntry {
277 		static int compare(LogEntry a, LogEntry b) {
278 			int cmp = a.ref.compareTo(b.ref);
279 			if (cmp == 0) {
280 				cmp = Long.signum(b.updateIndex - a.updateIndex);
281 			}
282 			return cmp;
283 		}
284 
285 		final String ref;
286 		final long updateIndex;
287 		final PersonIdent who;
288 		final ObjectId oldId;
289 		final ObjectId newId;
290 		final String message;
291 
292 		LogEntry(String ref, long updateIndex, PersonIdent who,
293 				ObjectId oldId, ObjectId newId, String message) {
294 			this.ref = ref;
295 			this.updateIndex = updateIndex;
296 			this.who = who;
297 			this.oldId = oldId;
298 			this.newId = newId;
299 			this.message = message;
300 		}
301 	}
302 }