View Javadoc
1   /*
2    * Copyright (C) 2022, Google Inc. and others
3    *
4    * This program and the accompanying materials are made available under the
5    * terms of the Eclipse Distribution License v. 1.0 which is available at
6    * https://www.eclipse.org/org/documents/edl-v10.php.
7    *
8    * SPDX-License-Identifier: BSD-3-Clause
9    */
10  package org.eclipse.jgit.merge;
11  
12  import static org.eclipse.jgit.lib.Constants.OBJ_BLOB;
13  
14  import java.io.Closeable;
15  import java.io.File;
16  import java.io.FileOutputStream;
17  import java.io.IOException;
18  import java.io.InputStream;
19  import java.io.OutputStream;
20  import java.time.Instant;
21  import java.util.HashMap;
22  import java.util.LinkedList;
23  import java.util.List;
24  import java.util.Map;
25  import java.util.Objects;
26  import java.util.TreeMap;
27  
28  import org.eclipse.jgit.annotations.NonNull;
29  import org.eclipse.jgit.annotations.Nullable;
30  import org.eclipse.jgit.attributes.Attribute;
31  import org.eclipse.jgit.attributes.Attributes;
32  import org.eclipse.jgit.dircache.DirCache;
33  import org.eclipse.jgit.dircache.DirCacheBuildIterator;
34  import org.eclipse.jgit.dircache.DirCacheBuilder;
35  import org.eclipse.jgit.dircache.DirCacheCheckout;
36  import org.eclipse.jgit.dircache.DirCacheCheckout.StreamSupplier;
37  import org.eclipse.jgit.dircache.DirCacheCheckout.CheckoutMetadata;
38  import org.eclipse.jgit.dircache.DirCacheEntry;
39  import org.eclipse.jgit.errors.IndexWriteException;
40  import org.eclipse.jgit.errors.NoWorkTreeException;
41  import org.eclipse.jgit.internal.JGitText;
42  import org.eclipse.jgit.lib.Config;
43  import org.eclipse.jgit.lib.ConfigConstants;
44  import org.eclipse.jgit.lib.CoreConfig.EolStreamType;
45  import org.eclipse.jgit.lib.FileMode;
46  import org.eclipse.jgit.lib.ObjectId;
47  import org.eclipse.jgit.lib.ObjectInserter;
48  import org.eclipse.jgit.lib.ObjectReader;
49  import org.eclipse.jgit.lib.Repository;
50  import org.eclipse.jgit.treewalk.TreeWalk.OperationType;
51  import org.eclipse.jgit.treewalk.WorkingTreeOptions;
52  import org.eclipse.jgit.util.LfsFactory;
53  import org.eclipse.jgit.util.LfsFactory.LfsInputStream;
54  import org.eclipse.jgit.util.io.EolStreamTypeUtil;
55  
56  /**
57   * Handles work tree updates on both the checkout and the index.
58   * <p>
59   * You should use a single instance for all of your file changes. In case of an
60   * error, make sure your instance is released, and initiate a new one if
61   * necessary.
62   */
63  class WorkTreeUpdater implements Closeable {
64  
65  	/**
66  	 * The result of writing the index changes.
67  	 */
68  	public static class Result {
69  
70  		private final List<String> modifiedFiles = new LinkedList<>();
71  
72  
73  		private final List<String> failedToDelete = new LinkedList<>();
74  
75  		private ObjectId treeId = null;
76  
77  		/**
78  		 * @return Modified tree ID if any, or null otherwise.
79  		 */
80  		public ObjectId getTreeId() {
81  			return treeId;
82  		}
83  
84  		/**
85  		 * @return Files that couldn't be deleted.
86  		 */
87  		public List<String> getFailedToDelete() {
88  			return failedToDelete;
89  		}
90  
91  		/**
92  		 * @return Files modified during this operation.
93  		 */
94  		public List<String> getModifiedFiles() {
95  			return modifiedFiles;
96  		}
97  	}
98  
99  	Result result = new Result();
100 
101 	/**
102 	 * The repository this handler operates on.
103 	 */
104 	@Nullable
105 	private final Repository repo;
106 
107 	/**
108 	 * Set to true if this operation should work in-memory. The repo's dircache
109 	 * and workingtree are not touched by this method. Eventually needed files
110 	 * are created as temporary files and a new empty, in-memory dircache will
111 	 * be used instead the repo's one. Often used for bare repos where the repo
112 	 * doesn't even have a workingtree and dircache.
113 	 */
114 	private final boolean inCore;
115 
116 	private final ObjectInserter inserter;
117 
118 	private final ObjectReader reader;
119 
120 	private DirCache dirCache;
121 
122 	private boolean implicitDirCache = false;
123 
124 	/**
125 	 * Builder to update the dir cache during this operation.
126 	 */
127 	private DirCacheBuilder builder;
128 
129 	/**
130 	 * The {@link WorkingTreeOptions} are needed to determine line endings for
131 	 * affected files.
132 	 */
133 	private WorkingTreeOptions workingTreeOptions;
134 
135 	/**
136 	 * The size limit (bytes) which controls a file to be stored in {@code Heap}
137 	 * or {@code LocalFile} during the operation.
138 	 */
139 	private int inCoreFileSizeLimit;
140 
141 	/**
142 	 * If the operation has nothing to do for a file but check it out at the end
143 	 * of the operation, it can be added here.
144 	 */
145 	private final Map<String, DirCacheEntry> toBeCheckedOut = new HashMap<>();
146 
147 	/**
148 	 * Files in this list will be deleted from the local copy at the end of the
149 	 * operation.
150 	 */
151 	private final TreeMap<String, File> toBeDeleted = new TreeMap<>();
152 
153 	/**
154 	 * Keeps {@link CheckoutMetadata} for {@link #checkout()}.
155 	 */
156 	private Map<String, CheckoutMetadata> checkoutMetadataByPath;
157 
158 	/**
159 	 * Keeps {@link CheckoutMetadata} for {@link #revertModifiedFiles()}.
160 	 */
161 	private Map<String, CheckoutMetadata> cleanupMetadataByPath;
162 
163 	/**
164 	 * Whether the changes were successfully written.
165 	 */
166 	private boolean indexChangesWritten;
167 
168 	/**
169 	 * @param repo
170 	 *            the {@link Repository}.
171 	 * @param dirCache
172 	 *            if set, use the provided dir cache. Otherwise, use the default
173 	 *            repository one
174 	 */
175 	private WorkTreeUpdater(Repository repo, DirCache dirCache) {
176 		this.repo = repo;
177 		this.dirCache = dirCache;
178 
179 		this.inCore = false;
180 		this.inserter = repo.newObjectInserter();
181 		this.reader = inserter.newReader();
182 		Config config = repo.getConfig();
183 		this.workingTreeOptions = config.get(WorkingTreeOptions.KEY);
184 		this.inCoreFileSizeLimit = getInCoreFileSizeLimit(config);
185 		this.checkoutMetadataByPath = new HashMap<>();
186 		this.cleanupMetadataByPath = new HashMap<>();
187 	}
188 
189 	/**
190 	 * Creates a new {@link WorkTreeUpdater} for the given repository.
191 	 *
192 	 * @param repo
193 	 *            the {@link Repository}.
194 	 * @param dirCache
195 	 *            if set, use the provided dir cache. Otherwise, use the default
196 	 *            repository one
197 	 * @return the {@link WorkTreeUpdater}.
198 	 */
199 	public static WorkTreeUpdater createWorkTreeUpdater(Repository repo,
200 			DirCache dirCache) {
201 		return new WorkTreeUpdater(repo, dirCache);
202 	}
203 
204 	/**
205 	 * @param repo
206 	 *            the {@link Repository}.
207 	 * @param dirCache
208 	 *            if set, use the provided dir cache. Otherwise, creates a new
209 	 *            one
210 	 * @param oi
211 	 *            to use for writing the modified objects with.
212 	 */
213 	private WorkTreeUpdater(Repository repo, DirCache dirCache,
214 			ObjectInserter oi) {
215 		this.repo = repo;
216 		this.dirCache = dirCache;
217 		this.inserter = oi;
218 
219 		this.inCore = true;
220 		this.reader = oi.newReader();
221 		if (repo != null) {
222 			this.inCoreFileSizeLimit = getInCoreFileSizeLimit(repo.getConfig());
223 		}
224 	}
225 
226 	/**
227 	 * Creates a new {@link WorkTreeUpdater} that works in memory only.
228 	 *
229 	 * @param repo
230 	 *            the {@link Repository}.
231 	 * @param dirCache
232 	 *            if set, use the provided dir cache. Otherwise, creates a new
233 	 *            one
234 	 * @param oi
235 	 *            to use for writing the modified objects with.
236 	 * @return the {@link WorkTreeUpdater}
237 	 */
238 	public static WorkTreeUpdater createInCoreWorkTreeUpdater(Repository repo,
239 			DirCache dirCache, ObjectInserter oi) {
240 		return new WorkTreeUpdater(repo, dirCache, oi);
241 	}
242 
243 	private static int getInCoreFileSizeLimit(Config config) {
244 		return config.getInt(ConfigConstants.CONFIG_MERGE_SECTION,
245 				ConfigConstants.CONFIG_KEY_IN_CORE_LIMIT, 10 << 20);
246 	}
247 
248 	/**
249 	 * Gets the size limit for in-core files in this config.
250 	 *
251 	 * @return the size
252 	 */
253 	public int getInCoreFileSizeLimit() {
254 		return inCoreFileSizeLimit;
255 	}
256 
257 	/**
258 	 * Gets dir cache for the repo. Locked if not inCore.
259 	 *
260 	 * @return the result dir cache
261 	 * @throws IOException
262 	 *             is case the dir cache cannot be read
263 	 */
264 	public DirCache getLockedDirCache() throws IOException {
265 		if (dirCache == null) {
266 			implicitDirCache = true;
267 			if (inCore) {
268 				dirCache = DirCache.newInCore();
269 			} else {
270 				dirCache = nonNullRepo().lockDirCache();
271 			}
272 		}
273 		if (builder == null) {
274 			builder = dirCache.builder();
275 		}
276 		return dirCache;
277 	}
278 
279 	/**
280 	 * Creates a {@link DirCacheBuildIterator} for the builder of this
281 	 * {@link WorkTreeUpdater}.
282 	 *
283 	 * @return the {@link DirCacheBuildIterator}
284 	 */
285 	public DirCacheBuildIterator createDirCacheBuildIterator() {
286 		return new DirCacheBuildIterator(builder);
287 	}
288 
289 	/**
290 	 * Writes the changes to the working tree (but not to the index).
291 	 *
292 	 * @param shouldCheckoutTheirs
293 	 *            before committing the changes
294 	 * @throws IOException
295 	 *             if any of the writes fail
296 	 */
297 	public void writeWorkTreeChanges(boolean shouldCheckoutTheirs)
298 			throws IOException {
299 		handleDeletedFiles();
300 
301 		if (inCore) {
302 			builder.finish();
303 			return;
304 		}
305 		if (shouldCheckoutTheirs) {
306 			// No problem found. The only thing left to be done is to
307 			// check out all files from "theirs" which have been selected to
308 			// go into the new index.
309 			checkout();
310 		}
311 
312 		// All content operations are successfully done. If we can now write the
313 		// new index we are on quite safe ground. Even if the checkout of
314 		// files coming from "theirs" fails the user can work around such
315 		// failures by checking out the index again.
316 		if (!builder.commit()) {
317 			revertModifiedFiles();
318 			throw new IndexWriteException();
319 		}
320 	}
321 
322 	/**
323 	 * Writes the changes to the index.
324 	 *
325 	 * @return the {@link Result} of the operation.
326 	 * @throws IOException
327 	 *             if any of the writes fail
328 	 */
329 	public Result writeIndexChanges() throws IOException {
330 		result.treeId = getLockedDirCache().writeTree(inserter);
331 		indexChangesWritten = true;
332 		return result;
333 	}
334 
335 	/**
336 	 * Adds a {@link DirCacheEntry} for direct checkout and remembers its
337 	 * {@link CheckoutMetadata}.
338 	 *
339 	 * @param path
340 	 *            of the entry
341 	 * @param entry
342 	 *            to add
343 	 * @param cleanupStreamType
344 	 *            to use for the cleanup metadata
345 	 * @param cleanupSmudgeCommand
346 	 *            to use for the cleanup metadata
347 	 * @param checkoutStreamType
348 	 *            to use for the checkout metadata
349 	 * @param checkoutSmudgeCommand
350 	 *            to use for the checkout metadata
351 	 */
352 	public void addToCheckout(String path, DirCacheEntry entry,
353 			EolStreamType cleanupStreamType, String cleanupSmudgeCommand,
354 			EolStreamType checkoutStreamType, String checkoutSmudgeCommand) {
355 		if (entry != null) {
356 			// In some cases, we just want to add the metadata.
357 			toBeCheckedOut.put(path, entry);
358 		}
359 		addCheckoutMetadata(cleanupMetadataByPath, path, cleanupStreamType,
360 				cleanupSmudgeCommand);
361 		addCheckoutMetadata(checkoutMetadataByPath, path, checkoutStreamType,
362 				checkoutSmudgeCommand);
363 	}
364 
365 	/**
366 	 * Gets a map which maps the paths of files which have to be checked out
367 	 * because the operation created new fully-merged content for this file into
368 	 * the index.
369 	 * <p>
370 	 * This means: the operation wrote a new stage 0 entry for this path.
371 	 * </p>
372 	 *
373 	 * @return the map
374 	 */
375 	public Map<String, DirCacheEntry> getToBeCheckedOut() {
376 		return toBeCheckedOut;
377 	}
378 
379 	/**
380 	 * Remembers the given file to be deleted.
381 	 * <p>
382 	 * Note the actual deletion is only done in {@link #writeWorkTreeChanges}.
383 	 *
384 	 * @param path
385 	 *            of the file to be deleted
386 	 * @param file
387 	 *            to be deleted
388 	 * @param streamType
389 	 *            to use for cleanup metadata
390 	 * @param smudgeCommand
391 	 *            to use for cleanup metadata
392 	 */
393 	public void deleteFile(String path, File file, EolStreamType streamType,
394 			String smudgeCommand) {
395 		toBeDeleted.put(path, file);
396 		if (file != null && file.isFile()) {
397 			addCheckoutMetadata(cleanupMetadataByPath, path, streamType,
398 					smudgeCommand);
399 		}
400 	}
401 
402 	/**
403 	 * Remembers the {@link CheckoutMetadata} for the given path; it may be
404 	 * needed in {@link #checkout()} or in {@link #revertModifiedFiles()}.
405 	 *
406 	 * @param map
407 	 *            to add the metadata to
408 	 * @param path
409 	 *            of the current node
410 	 * @param streamType
411 	 *            to use for the metadata
412 	 * @param smudgeCommand
413 	 *            to use for the metadata
414 	 */
415 	private void addCheckoutMetadata(Map<String, CheckoutMetadata> map,
416 			String path, EolStreamType streamType, String smudgeCommand) {
417 		if (inCore || map == null) {
418 			return;
419 		}
420 		map.put(path, new CheckoutMetadata(streamType, smudgeCommand));
421 	}
422 
423 	/**
424 	 * Detects if CRLF conversion has been configured.
425 	 * <p>
426 	 * </p>
427 	 * See {@link EolStreamTypeUtil#detectStreamType} for more info.
428 	 *
429 	 * @param attributes
430 	 *            of the file for which the type is to be detected
431 	 * @return the detected type
432 	 */
433 	public EolStreamType detectCheckoutStreamType(Attributes attributes) {
434 		if (inCore) {
435 			return null;
436 		}
437 		return EolStreamTypeUtil.detectStreamType(OperationType.CHECKOUT_OP,
438 				workingTreeOptions, attributes);
439 	}
440 
441 	private void handleDeletedFiles() {
442 		// Iterate in reverse so that "folder/file" is deleted before
443 		// "folder". Otherwise, this could result in a failing path because
444 		// of a non-empty directory, for which delete() would fail.
445 		for (String path : toBeDeleted.descendingKeySet()) {
446 			File file = inCore ? null : toBeDeleted.get(path);
447 			if (file != null && !file.delete()) {
448 				if (!file.isDirectory()) {
449 					result.failedToDelete.add(path);
450 				}
451 			}
452 		}
453 	}
454 
455 	/**
456 	 * Marks the given path as modified in the operation.
457 	 *
458 	 * @param path
459 	 *            to mark as modified
460 	 */
461 	public void markAsModified(String path) {
462 		result.modifiedFiles.add(path);
463 	}
464 
465 	/**
466 	 * Gets the list of files which were modified in this operation.
467 	 *
468 	 * @return the list
469 	 */
470 	public List<String> getModifiedFiles() {
471 		return result.modifiedFiles;
472 	}
473 
474 	private void checkout() throws NoWorkTreeException, IOException {
475 		for (Map.Entry<String, DirCacheEntry> entry : toBeCheckedOut
476 				.entrySet()) {
477 			DirCacheEntry dirCacheEntry = entry.getValue();
478 			if (dirCacheEntry.getFileMode() == FileMode.GITLINK) {
479 				new File(nonNullRepo().getWorkTree(), entry.getKey())
480 						.mkdirs();
481 			} else {
482 				DirCacheCheckout.checkoutEntry(repo, dirCacheEntry, reader,
483 						false, checkoutMetadataByPath.get(entry.getKey()),
484 						workingTreeOptions);
485 				result.modifiedFiles.add(entry.getKey());
486 			}
487 		}
488 	}
489 
490 	/**
491 	 * Reverts any uncommitted changes in the worktree. We know that for all
492 	 * modified files the old content was in the old index and the index
493 	 * contained only stage 0. In case of inCore operation just clear the
494 	 * history of modified files.
495 	 *
496 	 * @throws IOException
497 	 *             in case the cleaning up failed
498 	 */
499 	public void revertModifiedFiles() throws IOException {
500 		if (inCore) {
501 			result.modifiedFiles.clear();
502 			return;
503 		}
504 		if (indexChangesWritten) {
505 			return;
506 		}
507 		for (String path : result.modifiedFiles) {
508 			DirCacheEntry entry = dirCache.getEntry(path);
509 			if (entry != null) {
510 				DirCacheCheckout.checkoutEntry(repo, entry, reader, false,
511 						cleanupMetadataByPath.get(path), workingTreeOptions);
512 			}
513 		}
514 	}
515 
516 	@Override
517 	public void close() throws IOException {
518 		if (implicitDirCache) {
519 			dirCache.unlock();
520 		}
521 	}
522 
523 	/**
524 	 * Updates the file in the checkout with the given content.
525 	 *
526 	 * @param inputStream
527 	 *            the content to be updated
528 	 * @param streamType
529 	 *            for parsing the content
530 	 * @param smudgeCommand
531 	 *            for formatting the content
532 	 * @param path
533 	 *            of the file to be updated
534 	 * @param file
535 	 *            to be updated
536 	 * @throws IOException
537 	 *             if the file cannot be updated
538 	 */
539 	public void updateFileWithContent(StreamSupplier inputStream,
540 			EolStreamType streamType, String smudgeCommand, String path,
541 			File file) throws IOException {
542 		if (inCore) {
543 			return;
544 		}
545 		CheckoutMetadata metadata = new CheckoutMetadata(streamType,
546 				smudgeCommand);
547 
548 		try (OutputStream outputStream = new FileOutputStream(file)) {
549 			DirCacheCheckout.getContent(repo, path, metadata,
550 					inputStream, workingTreeOptions, outputStream);
551 		}
552 	}
553 
554 	/**
555 	 * Creates a path with the given content, and adds it to the specified stage
556 	 * to the index builder.
557 	 *
558 	 * @param input
559 	 *            the content to be updated
560 	 * @param path
561 	 *            of the file to be updated
562 	 * @param fileMode
563 	 *            of the modified file
564 	 * @param entryStage
565 	 *            of the new entry
566 	 * @param lastModified
567 	 *            instant of the modified file
568 	 * @param len
569 	 *            of the content
570 	 * @param lfsAttribute
571 	 *            for checking for LFS enablement
572 	 * @return the entry which was added to the index
573 	 * @throws IOException
574 	 *             if inserting the content fails
575 	 */
576 	public DirCacheEntry insertToIndex(InputStream input,
577 			byte[] path, FileMode fileMode, int entryStage,
578 			Instant lastModified, int len, Attribute lfsAttribute)
579 			throws IOException {
580 		return addExistingToIndex(insertResult(input, lfsAttribute, len), path,
581 				fileMode, entryStage, lastModified, len);
582 	}
583 
584 	/**
585 	 * Adds a path with the specified stage to the index builder.
586 	 *
587 	 * @param objectId
588 	 *            of the existing object to add
589 	 * @param path
590 	 *            of the modified file
591 	 * @param fileMode
592 	 *            of the modified file
593 	 * @param entryStage
594 	 *            of the new entry
595 	 * @param lastModified
596 	 *            instant of the modified file
597 	 * @param len
598 	 *            of the modified file content
599 	 * @return the entry which was added to the index
600 	 */
601 	public DirCacheEntry addExistingToIndex(ObjectId objectId, byte[] path,
602 			FileMode fileMode, int entryStage, Instant lastModified, int len) {
603 		DirCacheEntry dce = new DirCacheEntry(path, entryStage);
604 		dce.setFileMode(fileMode);
605 		if (lastModified != null) {
606 			dce.setLastModified(lastModified);
607 		}
608 		dce.setLength(inCore ? 0 : len);
609 		dce.setObjectId(objectId);
610 		builder.add(dce);
611 		return dce;
612 	}
613 
614 	private ObjectId insertResult(InputStream input,
615 			Attribute lfsAttribute, long length) throws IOException {
616 		try (LfsInputStream is = LfsFactory.getInstance().applyCleanFilter(repo,
617 				input, length,
618 				lfsAttribute)) {
619 			return inserter.insert(OBJ_BLOB, is.getLength(), is);
620 		}
621 	}
622 
623 	/**
624 	 * Gets the non-null repository instance of this {@link WorkTreeUpdater}.
625 	 *
626 	 * @return non-null repository instance
627 	 * @throws NullPointerException
628 	 *             if the handler was constructed without a repository.
629 	 */
630 	@NonNull
631 	private Repository nonNullRepo() throws NullPointerException {
632 		return Objects.requireNonNull(repo,
633 				() -> JGitText.get().repositoryIsRequired);
634 	}
635 }