View Javadoc
1   /*
2    * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org>
3    * and other copyright owners as documented in the project's IP log.
4    *
5    * This program and the accompanying materials are made available
6    * under the terms of the Eclipse Distribution License v1.0 which
7    * accompanies this distribution, is reproduced below, and is
8    * available at http://www.eclipse.org/org/documents/edl-v10.php
9    *
10   * All rights reserved.
11   *
12   * Redistribution and use in source and binary forms, with or
13   * without modification, are permitted provided that the following
14   * conditions are met:
15   *
16   * - Redistributions of source code must retain the above copyright
17   *   notice, this list of conditions and the following disclaimer.
18   *
19   * - Redistributions in binary form must reproduce the above
20   *   copyright notice, this list of conditions and the following
21   *   disclaimer in the documentation and/or other materials provided
22   *   with the distribution.
23   *
24   * - Neither the name of the Eclipse Foundation, Inc. nor the
25   *   names of its contributors may be used to endorse or promote
26   *   products derived from this software without specific prior
27   *   written permission.
28   *
29   * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
30   * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
31   * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
32   * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
33   * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
34   * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
35   * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
36   * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
37   * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
38   * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
39   * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
40   * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
41   * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42   */
43  
44  package org.eclipse.jgit.transport;
45  
46  import java.io.BufferedReader;
47  import java.io.FileNotFoundException;
48  import java.io.IOException;
49  import java.io.OutputStream;
50  import java.text.MessageFormat;
51  import java.util.ArrayList;
52  import java.util.Collection;
53  import java.util.Collections;
54  import java.util.Comparator;
55  import java.util.EnumSet;
56  import java.util.HashMap;
57  import java.util.List;
58  import java.util.Map;
59  import java.util.Set;
60  import java.util.TreeMap;
61  
62  import org.eclipse.jgit.errors.NotSupportedException;
63  import org.eclipse.jgit.errors.TransportException;
64  import org.eclipse.jgit.internal.JGitText;
65  import org.eclipse.jgit.lib.Constants;
66  import org.eclipse.jgit.lib.ObjectId;
67  import org.eclipse.jgit.lib.ObjectIdRef;
68  import org.eclipse.jgit.lib.ProgressMonitor;
69  import org.eclipse.jgit.lib.Ref;
70  import org.eclipse.jgit.lib.Ref.Storage;
71  import org.eclipse.jgit.lib.Repository;
72  import org.eclipse.jgit.lib.SymbolicRef;
73  
74  import com.jcraft.jsch.Channel;
75  import com.jcraft.jsch.ChannelSftp;
76  import com.jcraft.jsch.JSchException;
77  import com.jcraft.jsch.SftpATTRS;
78  import com.jcraft.jsch.SftpException;
79  
80  /**
81   * Transport over the non-Git aware SFTP (SSH based FTP) protocol.
82   * <p>
83   * The SFTP transport does not require any specialized Git support on the remote
84   * (server side) repository. Object files are retrieved directly through secure
85   * shell's FTP protocol, making it possible to copy objects from a remote
86   * repository that is available over SSH, but whose remote host does not have
87   * Git installed.
88   * <p>
89   * Unlike the HTTP variant (see {@link TransportHttp}) we rely upon being able
90   * to list files in directories, as the SFTP protocol supports this function. By
91   * listing files through SFTP we can avoid needing to have current
92   * <code>objects/info/packs</code> or <code>info/refs</code> files on the
93   * remote repository and access the data directly, much as Git itself would.
94   * <p>
95   * Concurrent pushing over this transport is not supported. Multiple concurrent
96   * push operations may cause confusion in the repository state.
97   *
98   * @see WalkFetchConnection
99   */
100 public class TransportSftp extends SshTransport implements WalkTransport {
101 	static final TransportProtocol PROTO_SFTP = new TransportProtocol() {
102 		public String getName() {
103 			return JGitText.get().transportProtoSFTP;
104 		}
105 
106 		public Set<String> getSchemes() {
107 			return Collections.singleton("sftp"); //$NON-NLS-1$
108 		}
109 
110 		public Set<URIishField> getRequiredFields() {
111 			return Collections.unmodifiableSet(EnumSet.of(URIishField.HOST,
112 					URIishField.PATH));
113 		}
114 
115 		public Set<URIishField> getOptionalFields() {
116 			return Collections.unmodifiableSet(EnumSet.of(URIishField.USER,
117 					URIishField.PASS, URIishField.PORT));
118 		}
119 
120 		public int getDefaultPort() {
121 			return 22;
122 		}
123 
124 		public Transport open(URIish uri, Repository local, String remoteName)
125 				throws NotSupportedException {
126 			return new TransportSftp(local, uri);
127 		}
128 	};
129 
130 	TransportSftp(final Repository local, final URIish uri) {
131 		super(local, uri);
132 	}
133 
134 	@Override
135 	public FetchConnection openFetch() throws TransportException {
136 		final SftpObjectDB c = new SftpObjectDB(uri.getPath());
137 		final WalkFetchConnection r = new WalkFetchConnection(this, c);
138 		r.available(c.readAdvertisedRefs());
139 		return r;
140 	}
141 
142 	@Override
143 	public PushConnection openPush() throws TransportException {
144 		final SftpObjectDB c = new SftpObjectDB(uri.getPath());
145 		final WalkPushConnection r = new WalkPushConnection(this, c);
146 		r.available(c.readAdvertisedRefs());
147 		return r;
148 	}
149 
150 	ChannelSftp newSftp() throws TransportException {
151 		final int tms = getTimeout() > 0 ? getTimeout() * 1000 : 0;
152 		try {
153 			// @TODO: Fix so that this operation is generic and casting to
154 			// JschSession is no longer necessary.
155 			final Channel channel = ((JschSession) getSession())
156 					.getSftpChannel();
157 			channel.connect(tms);
158 			return (ChannelSftp) channel;
159 		} catch (JSchException je) {
160 			throw new TransportException(uri, je.getMessage(), je);
161 		}
162 	}
163 
164 	class SftpObjectDB extends WalkRemoteObjectDatabase {
165 		private final String objectsPath;
166 
167 		private ChannelSftp ftp;
168 
169 		SftpObjectDB(String path) throws TransportException {
170 			if (path.startsWith("/~")) //$NON-NLS-1$
171 				path = path.substring(1);
172 			if (path.startsWith("~/")) //$NON-NLS-1$
173 				path = path.substring(2);
174 			try {
175 				ftp = newSftp();
176 				ftp.cd(path);
177 				ftp.cd("objects"); //$NON-NLS-1$
178 				objectsPath = ftp.pwd();
179 			} catch (TransportException err) {
180 				close();
181 				throw err;
182 			} catch (SftpException je) {
183 				throw new TransportException(MessageFormat.format(
184 						JGitText.get().cannotEnterObjectsPath, path,
185 						je.getMessage()), je);
186 			}
187 		}
188 
189 		SftpObjectDB(final SftpObjectDB parent, final String p)
190 				throws TransportException {
191 			try {
192 				ftp = newSftp();
193 				ftp.cd(parent.objectsPath);
194 				ftp.cd(p);
195 				objectsPath = ftp.pwd();
196 			} catch (TransportException err) {
197 				close();
198 				throw err;
199 			} catch (SftpException je) {
200 				throw new TransportException(MessageFormat.format(
201 						JGitText.get().cannotEnterPathFromParent, p,
202 						parent.objectsPath, je.getMessage()), je);
203 			}
204 		}
205 
206 		@Override
207 		URIish getURI() {
208 			return uri.setPath(objectsPath);
209 		}
210 
211 		@Override
212 		Collection<WalkRemoteObjectDatabase> getAlternates() throws IOException {
213 			try {
214 				return readAlternates(INFO_ALTERNATES);
215 			} catch (FileNotFoundException err) {
216 				return null;
217 			}
218 		}
219 
220 		@Override
221 		WalkRemoteObjectDatabase openAlternate(final String location)
222 				throws IOException {
223 			return new SftpObjectDB(this, location);
224 		}
225 
226 		@Override
227 		Collection<String> getPackNames() throws IOException {
228 			final List<String> packs = new ArrayList<String>();
229 			try {
230 				@SuppressWarnings("unchecked")
231 				final Collection<ChannelSftp.LsEntry> list = ftp.ls("pack"); //$NON-NLS-1$
232 				final HashMap<String, ChannelSftp.LsEntry> files;
233 				final HashMap<String, Integer> mtimes;
234 
235 				files = new HashMap<String, ChannelSftp.LsEntry>();
236 				mtimes = new HashMap<String, Integer>();
237 
238 				for (final ChannelSftp.LsEntry ent : list)
239 					files.put(ent.getFilename(), ent);
240 				for (final ChannelSftp.LsEntry ent : list) {
241 					final String n = ent.getFilename();
242 					if (!n.startsWith("pack-") || !n.endsWith(".pack")) //$NON-NLS-1$ //$NON-NLS-2$
243 						continue;
244 
245 					final String in = n.substring(0, n.length() - 5) + ".idx"; //$NON-NLS-1$
246 					if (!files.containsKey(in))
247 						continue;
248 
249 					mtimes.put(n, Integer.valueOf(ent.getAttrs().getMTime()));
250 					packs.add(n);
251 				}
252 
253 				Collections.sort(packs, new Comparator<String>() {
254 					public int compare(final String o1, final String o2) {
255 						return mtimes.get(o2).intValue()
256 								- mtimes.get(o1).intValue();
257 					}
258 				});
259 			} catch (SftpException je) {
260 				throw new TransportException(
261 						MessageFormat.format(JGitText.get().cannotListPackPath,
262 								objectsPath, je.getMessage()),
263 						je);
264 			}
265 			return packs;
266 		}
267 
268 		@Override
269 		FileStream open(final String path) throws IOException {
270 			try {
271 				final SftpATTRS a = ftp.lstat(path);
272 				return new FileStream(ftp.get(path), a.getSize());
273 			} catch (SftpException je) {
274 				if (je.id == ChannelSftp.SSH_FX_NO_SUCH_FILE)
275 					throw new FileNotFoundException(path);
276 				throw new TransportException(MessageFormat.format(
277 						JGitText.get().cannotGetObjectsPath, objectsPath, path,
278 						je.getMessage()), je);
279 			}
280 		}
281 
282 		@Override
283 		void deleteFile(final String path) throws IOException {
284 			try {
285 				ftp.rm(path);
286 			} catch (SftpException je) {
287 				if (je.id == ChannelSftp.SSH_FX_NO_SUCH_FILE)
288 					return;
289 				throw new TransportException(MessageFormat.format(
290 						JGitText.get().cannotDeleteObjectsPath, objectsPath,
291 						path, je.getMessage()), je);
292 			}
293 
294 			// Prune any now empty directories.
295 			//
296 			String dir = path;
297 			int s = dir.lastIndexOf('/');
298 			while (s > 0) {
299 				try {
300 					dir = dir.substring(0, s);
301 					ftp.rmdir(dir);
302 					s = dir.lastIndexOf('/');
303 				} catch (SftpException je) {
304 					// If we cannot delete it, leave it alone. It may have
305 					// entries still in it, or maybe we lack write access on
306 					// the parent. Either way it isn't a fatal error.
307 					//
308 					break;
309 				}
310 			}
311 		}
312 
313 		@Override
314 		OutputStream writeFile(final String path,
315 				final ProgressMonitor monitor, final String monitorTask)
316 				throws IOException {
317 			try {
318 				return ftp.put(path);
319 			} catch (SftpException je) {
320 				if (je.id == ChannelSftp.SSH_FX_NO_SUCH_FILE) {
321 					mkdir_p(path);
322 					try {
323 						return ftp.put(path);
324 					} catch (SftpException je2) {
325 						je = je2;
326 					}
327 				}
328 
329 				throw new TransportException(MessageFormat.format(
330 						JGitText.get().cannotWriteObjectsPath, objectsPath,
331 						path, je.getMessage()), je);
332 			}
333 		}
334 
335 		@Override
336 		void writeFile(final String path, final byte[] data) throws IOException {
337 			final String lock = path + ".lock"; //$NON-NLS-1$
338 			try {
339 				super.writeFile(lock, data);
340 				try {
341 					ftp.rename(lock, path);
342 				} catch (SftpException je) {
343 					throw new TransportException(MessageFormat.format(
344 							JGitText.get().cannotWriteObjectsPath, objectsPath,
345 							path, je.getMessage()), je);
346 				}
347 			} catch (IOException err) {
348 				try {
349 					ftp.rm(lock);
350 				} catch (SftpException e) {
351 					// Ignore deletion failure, we are already
352 					// failing anyway.
353 				}
354 				throw err;
355 			}
356 		}
357 
358 		private void mkdir_p(String path) throws IOException {
359 			final int s = path.lastIndexOf('/');
360 			if (s <= 0)
361 				return;
362 
363 			path = path.substring(0, s);
364 			try {
365 				ftp.mkdir(path);
366 			} catch (SftpException je) {
367 				if (je.id == ChannelSftp.SSH_FX_NO_SUCH_FILE) {
368 					mkdir_p(path);
369 					try {
370 						ftp.mkdir(path);
371 						return;
372 					} catch (SftpException je2) {
373 						je = je2;
374 					}
375 				}
376 
377 				throw new TransportException(MessageFormat.format(
378 						JGitText.get().cannotMkdirObjectPath, objectsPath, path,
379 						je.getMessage()), je);
380 			}
381 		}
382 
383 		Map<String, Ref> readAdvertisedRefs() throws TransportException {
384 			final TreeMap<String, Ref> avail = new TreeMap<String, Ref>();
385 			readPackedRefs(avail);
386 			readRef(avail, ROOT_DIR + Constants.HEAD, Constants.HEAD);
387 			readLooseRefs(avail, ROOT_DIR + "refs", "refs/"); //$NON-NLS-1$ //$NON-NLS-2$
388 			return avail;
389 		}
390 
391 		@SuppressWarnings("unchecked")
392 		private void readLooseRefs(final TreeMap<String, Ref> avail,
393 				final String dir, final String prefix)
394 				throws TransportException {
395 			final Collection<ChannelSftp.LsEntry> list;
396 			try {
397 				list = ftp.ls(dir);
398 			} catch (SftpException je) {
399 				throw new TransportException(MessageFormat.format(
400 						JGitText.get().cannotListObjectsPath, objectsPath, dir,
401 						je.getMessage()), je);
402 			}
403 
404 			for (final ChannelSftp.LsEntry ent : list) {
405 				final String n = ent.getFilename();
406 				if (".".equals(n) || "..".equals(n)) //$NON-NLS-1$ //$NON-NLS-2$
407 					continue;
408 
409 				final String nPath = dir + "/" + n; //$NON-NLS-1$
410 				if (ent.getAttrs().isDir())
411 					readLooseRefs(avail, nPath, prefix + n + "/"); //$NON-NLS-1$
412 				else
413 					readRef(avail, nPath, prefix + n);
414 			}
415 		}
416 
417 		private Ref readRef(final TreeMap<String, Ref> avail,
418 				final String path, final String name) throws TransportException {
419 			final String line;
420 			try {
421 				final BufferedReader br = openReader(path);
422 				try {
423 					line = br.readLine();
424 				} finally {
425 					br.close();
426 				}
427 			} catch (FileNotFoundException noRef) {
428 				return null;
429 			} catch (IOException err) {
430 				throw new TransportException(MessageFormat.format(
431 						JGitText.get().cannotReadObjectsPath, objectsPath, path,
432 						err.getMessage()), err);
433 			}
434 
435 			if (line == null)
436 				throw new TransportException(
437 						MessageFormat.format(JGitText.get().emptyRef, name));
438 
439 			if (line.startsWith("ref: ")) { //$NON-NLS-1$
440 				final String target = line.substring("ref: ".length()); //$NON-NLS-1$
441 				Ref r = avail.get(target);
442 				if (r == null)
443 					r = readRef(avail, ROOT_DIR + target, target);
444 				if (r == null)
445 					r = new ObjectIdRef.Unpeeled(Ref.Storage.NEW, target, null);
446 				r = new SymbolicRef(name, r);
447 				avail.put(r.getName(), r);
448 				return r;
449 			}
450 
451 			if (ObjectId.isId(line)) {
452 				final Ref r = new ObjectIdRef.Unpeeled(loose(avail.get(name)),
453 						name, ObjectId.fromString(line));
454 				avail.put(r.getName(), r);
455 				return r;
456 			}
457 
458 			throw new TransportException(
459 					MessageFormat.format(JGitText.get().badRef, name, line));
460 		}
461 
462 		private Storage loose(final Ref r) {
463 			if (r != null && r.getStorage() == Storage.PACKED)
464 				return Storage.LOOSE_PACKED;
465 			return Storage.LOOSE;
466 		}
467 
468 		@Override
469 		void close() {
470 			if (ftp != null) {
471 				try {
472 					if (ftp.isConnected())
473 						ftp.disconnect();
474 				} finally {
475 					ftp = null;
476 				}
477 			}
478 		}
479 	}
480 }