View Javadoc
1   /*
2    * Copyright (C) 2017, Markus Duft <markus.duft@ssi-schaefer.com>
3    * and other copyright owners as documented in the project's IP log.
4    *
5    * This program and the accompanying materials are made available
6    * under the terms of the Eclipse Distribution License v1.0 which
7    * accompanies this distribution, is reproduced below, and is
8    * available at http://www.eclipse.org/org/documents/edl-v10.php
9    *
10   * All rights reserved.
11   *
12   * Redistribution and use in source and binary forms, with or
13   * without modification, are permitted provided that the following
14   * conditions are met:
15   *
16   * - Redistributions of source code must retain the above copyright
17   *   notice, this list of conditions and the following disclaimer.
18   *
19   * - Redistributions in binary form must reproduce the above
20   *   copyright notice, this list of conditions and the following
21   *   disclaimer in the documentation and/or other materials provided
22   *   with the distribution.
23   *
24   * - Neither the name of the Eclipse Foundation, Inc. nor the
25   *   names of its contributors may be used to endorse or promote
26   *   products derived from this software without specific prior
27   *   written permission.
28   *
29   * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
30   * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
31   * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
32   * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
33   * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
34   * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
35   * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
36   * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
37   * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
38   * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
39   * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
40   * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
41   * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42   */
43  package org.eclipse.jgit.lfs.internal;
44  
45  import static org.eclipse.jgit.util.HttpSupport.ENCODING_GZIP;
46  import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT;
47  import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT_ENCODING;
48  import static org.eclipse.jgit.util.HttpSupport.HDR_CONTENT_TYPE;
49  
50  import java.io.IOException;
51  import java.net.ProxySelector;
52  import java.net.URISyntaxException;
53  import java.net.URL;
54  import java.text.SimpleDateFormat;
55  import java.util.LinkedList;
56  import java.util.Map;
57  import java.util.TreeMap;
58  
59  import org.eclipse.jgit.annotations.NonNull;
60  import org.eclipse.jgit.errors.CommandFailedException;
61  import org.eclipse.jgit.lfs.LfsPointer;
62  import org.eclipse.jgit.lfs.Protocol;
63  import org.eclipse.jgit.lfs.errors.LfsConfigInvalidException;
64  import org.eclipse.jgit.lib.ConfigConstants;
65  import org.eclipse.jgit.lib.Repository;
66  import org.eclipse.jgit.lib.StoredConfig;
67  import org.eclipse.jgit.transport.HttpConfig;
68  import org.eclipse.jgit.transport.HttpTransport;
69  import org.eclipse.jgit.transport.URIish;
70  import org.eclipse.jgit.transport.http.HttpConnection;
71  import org.eclipse.jgit.util.HttpSupport;
72  import org.eclipse.jgit.util.SshSupport;
73  
74  /**
75   * Provides means to get a valid LFS connection for a given repository.
76   */
77  public class LfsConnectionFactory {
78  
79  	private static final int SSH_AUTH_TIMEOUT_SECONDS = 30;
80  	private static final String SCHEME_HTTPS = "https"; //$NON-NLS-1$
81  	private static final String SCHEME_SSH = "ssh"; //$NON-NLS-1$
82  	private static final Map<String, AuthCache> sshAuthCache = new TreeMap<>();
83  
84  	/**
85  	 * Determine URL of LFS server by looking into config parameters lfs.url,
86  	 * lfs.[remote].url or remote.[remote].url. The LFS server URL is computed
87  	 * from remote.[remote].url by appending "/info/lfs". In case there is no
88  	 * URL configured, a SSH remote URI can be used to auto-detect the LFS URI
89  	 * by using the remote "git-lfs-authenticate" command.
90  	 *
91  	 * @param db
92  	 *            the repository to work with
93  	 * @param method
94  	 *            the method (GET,PUT,...) of the request this connection will
95  	 *            be used for
96  	 * @param purpose
97  	 *            the action, e.g. Protocol.OPERATION_DOWNLOAD
98  	 * @return the url for the lfs server. e.g.
99  	 *         "https://github.com/github/git-lfs.git/info/lfs"
100 	 * @throws IOException
101 	 */
102 	public static HttpConnection getLfsConnection(Repository db, String method,
103 			String purpose) throws IOException {
104 		StoredConfig config = db.getConfig();
105 		Map<String, String> additionalHeaders = new TreeMap<>();
106 		String lfsUrl = getLfsUrl(db, purpose, additionalHeaders);
107 		URL url = new URL(lfsUrl + Protocol.OBJECTS_LFS_ENDPOINT);
108 		HttpConnection connection = HttpTransport.getConnectionFactory().create(
109 				url, HttpSupport.proxyFor(ProxySelector.getDefault(), url));
110 		connection.setDoOutput(true);
111 		if (url.getProtocol().equals(SCHEME_HTTPS)
112 				&& !config.getBoolean(HttpConfig.HTTP,
113 						HttpConfig.SSL_VERIFY_KEY, true)) {
114 			HttpSupport.disableSslVerify(connection);
115 		}
116 		connection.setRequestMethod(method);
117 		connection.setRequestProperty(HDR_ACCEPT,
118 				Protocol.CONTENTTYPE_VND_GIT_LFS_JSON);
119 		connection.setRequestProperty(HDR_CONTENT_TYPE,
120 				Protocol.CONTENTTYPE_VND_GIT_LFS_JSON);
121 		additionalHeaders
122 				.forEach((k, v) -> connection.setRequestProperty(k, v));
123 		return connection;
124 	}
125 
126 	private static String getLfsUrl(Repository db, String purpose,
127 			Map<String, String> additionalHeaders)
128 			throws LfsConfigInvalidException {
129 		StoredConfig config = db.getConfig();
130 		String lfsUrl = config.getString(ConfigConstants.CONFIG_SECTION_LFS,
131 				null,
132 				ConfigConstants.CONFIG_KEY_URL);
133 		Exception ex = null;
134 		if (lfsUrl == null) {
135 			String remoteUrl = null;
136 			for (String remote : db.getRemoteNames()) {
137 				lfsUrl = config.getString(ConfigConstants.CONFIG_SECTION_LFS,
138 						remote,
139 						ConfigConstants.CONFIG_KEY_URL);
140 				// This could be done better (more precise logic), but according
141 				// to https://github.com/git-lfs/git-lfs/issues/1759 git-lfs
142 				// generally only supports 'origin' in an integrated workflow.
143 				if (lfsUrl == null && (remote.equals(
144 						org.eclipse.jgit.lib.Constants.DEFAULT_REMOTE_NAME))) {
145 					remoteUrl = config.getString(
146 							ConfigConstants.CONFIG_KEY_REMOTE, remote,
147 							ConfigConstants.CONFIG_KEY_URL);
148 				}
149 				break;
150 			}
151 			if (lfsUrl == null && remoteUrl != null) {
152 				try {
153 					lfsUrl = discoverLfsUrl(db, purpose, additionalHeaders,
154 							remoteUrl);
155 				} catch (URISyntaxException | IOException
156 						| CommandFailedException e) {
157 					ex = e;
158 				}
159 			} else {
160 				lfsUrl = lfsUrl + Protocol.INFO_LFS_ENDPOINT;
161 			}
162 		}
163 		if (lfsUrl == null) {
164 			if (ex != null) {
165 				throw new LfsConfigInvalidException(
166 						LfsText.get().lfsNoDownloadUrl, ex);
167 			}
168 			throw new LfsConfigInvalidException(LfsText.get().lfsNoDownloadUrl);
169 		}
170 		return lfsUrl;
171 	}
172 
173 	private static String discoverLfsUrl(Repository db, String purpose,
174 			Map<String, String> additionalHeaders, String remoteUrl)
175 			throws URISyntaxException, IOException, CommandFailedException {
176 		URIish u = new URIish(remoteUrl);
177 		if (u.getScheme() == null || SCHEME_SSH.equals(u.getScheme())) {
178 			Protocol.ExpiringAction action = getSshAuthentication(db, purpose,
179 					remoteUrl, u);
180 			additionalHeaders.putAll(action.header);
181 			return action.href;
182 		} else {
183 			return remoteUrl + Protocol.INFO_LFS_ENDPOINT;
184 		}
185 	}
186 
187 	private static Protocol.ExpiringAction getSshAuthentication(
188 			Repository db, String purpose, String remoteUrl, URIish u)
189 			throws IOException, CommandFailedException {
190 		AuthCache cached = sshAuthCache.get(remoteUrl);
191 		Protocol.ExpiringAction action = null;
192 		if (cached != null && cached.validUntil > System.currentTimeMillis()) {
193 			action = cached.cachedAction;
194 		}
195 
196 		if (action == null) {
197 			// discover and authenticate; git-lfs does "ssh
198 			// -p <port> -- <host> git-lfs-authenticate
199 			// <project> <upload/download>"
200 			String json = SshSupport.runSshCommand(u.setPath(""), //$NON-NLS-1$
201 					null, db.getFS(),
202 					"git-lfs-authenticate " + extractProjectName(u) + " " //$NON-NLS-1$//$NON-NLS-2$
203 							+ purpose,
204 					SSH_AUTH_TIMEOUT_SECONDS);
205 
206 			action = Protocol.gson().fromJson(json,
207 					Protocol.ExpiringAction.class);
208 
209 			// cache the result as long as possible.
210 			AuthCache c = new AuthCache(action);
211 			sshAuthCache.put(remoteUrl, c);
212 		}
213 		return action;
214 	}
215 
216 	/**
217 	 * Create a connection for the specified
218 	 * {@link org.eclipse.jgit.lfs.Protocol.Action}.
219 	 *
220 	 * @param repo
221 	 *            the repo to fetch required configuration from
222 	 * @param action
223 	 *            the action for which to create a connection
224 	 * @param method
225 	 *            the target method (GET or PUT)
226 	 * @return a connection. output mode is not set.
227 	 * @throws IOException
228 	 *             in case of any error.
229 	 */
230 	public static @NonNull HttpConnection getLfsContentConnection(
231 			Repository repo, Protocol.Action action, String method)
232 			throws IOException {
233 		URL contentUrl = new URL(action.href);
234 		HttpConnection contentServerConn = HttpTransport.getConnectionFactory()
235 				.create(contentUrl, HttpSupport
236 						.proxyFor(ProxySelector.getDefault(), contentUrl));
237 		contentServerConn.setRequestMethod(method);
238 		if (action.header != null) {
239 			action.header.forEach(
240 					(k, v) -> contentServerConn.setRequestProperty(k, v));
241 		}
242 		if (contentUrl.getProtocol().equals(SCHEME_HTTPS)
243 				&& !repo.getConfig().getBoolean(HttpConfig.HTTP,
244 						HttpConfig.SSL_VERIFY_KEY, true)) {
245 			HttpSupport.disableSslVerify(contentServerConn);
246 		}
247 
248 		contentServerConn.setRequestProperty(HDR_ACCEPT_ENCODING,
249 				ENCODING_GZIP);
250 
251 		return contentServerConn;
252 	}
253 
254 	private static String extractProjectName(URIish u) {
255 		String path = u.getPath();
256 
257 		// begins with a slash if the url contains a port (gerrit vs. github).
258 		if (path.startsWith("/")) { //$NON-NLS-1$
259 			path = path.substring(1);
260 		}
261 
262 		if (path.endsWith(org.eclipse.jgit.lib.Constants.DOT_GIT)) {
263 			return path.substring(0, path.length() - 4);
264 		} else {
265 			return path;
266 		}
267 	}
268 
269 	/**
270 	 * @param operation
271 	 *            the operation to perform, e.g. Protocol.OPERATION_DOWNLOAD
272 	 * @param resources
273 	 *            the LFS resources affected
274 	 * @return a request that can be serialized to JSON
275 	 */
276 	public static Protocol.Request toRequest(String operation,
277 			LfsPointer... resources) {
278 		Protocol.Request req = new Protocol.Request();
279 		req.operation = operation;
280 		if (resources != null) {
281 			req.objects = new LinkedList<>();
282 			for (LfsPointer res : resources) {
283 				Protocol.ObjectSpec o = new Protocol.ObjectSpec();
284 				o.oid = res.getOid().getName();
285 				o.size = res.getSize();
286 				req.objects.add(o);
287 			}
288 		}
289 		return req;
290 	}
291 
292 	private static final class AuthCache {
293 		private static final long AUTH_CACHE_EAGER_TIMEOUT = 500;
294 
295 		private static final SimpleDateFormat ISO_FORMAT = new SimpleDateFormat(
296 				"yyyy-MM-dd'T'HH:mm:ss.SSSX"); //$NON-NLS-1$
297 
298 		/**
299 		 * Creates a cache entry for an authentication response.
300 		 * <p>
301 		 * The timeout of the cache token is extracted from the given action. If
302 		 * no timeout can be determined, the token will be used only once.
303 		 *
304 		 * @param action
305 		 */
306 		public AuthCache(Protocol.ExpiringAction action) {
307 			this.cachedAction = action;
308 			try {
309 				if (action.expiresIn != null && !action.expiresIn.isEmpty()) {
310 					this.validUntil = (System.currentTimeMillis()
311 							+ Long.parseLong(action.expiresIn))
312 							- AUTH_CACHE_EAGER_TIMEOUT;
313 				} else if (action.expiresAt != null
314 						&& !action.expiresAt.isEmpty()) {
315 					this.validUntil = ISO_FORMAT.parse(action.expiresAt)
316 							.getTime() - AUTH_CACHE_EAGER_TIMEOUT;
317 				} else {
318 					this.validUntil = System.currentTimeMillis();
319 				}
320 			} catch (Exception e) {
321 				this.validUntil = System.currentTimeMillis();
322 			}
323 		}
324 
325 		long validUntil;
326 
327 		Protocol.ExpiringAction cachedAction;
328 	}
329 
330 }