View Javadoc
1   /*
2    * Copyright (C) 2017, Markus Duft <markus.duft@ssi-schaefer.com>
3    * and other copyright owners as documented in the project's IP log.
4    *
5    * This program and the accompanying materials are made available
6    * under the terms of the Eclipse Distribution License v1.0 which
7    * accompanies this distribution, is reproduced below, and is
8    * available at http://www.eclipse.org/org/documents/edl-v10.php
9    *
10   * All rights reserved.
11   *
12   * Redistribution and use in source and binary forms, with or
13   * without modification, are permitted provided that the following
14   * conditions are met:
15   *
16   * - Redistributions of source code must retain the above copyright
17   *   notice, this list of conditions and the following disclaimer.
18   *
19   * - Redistributions in binary form must reproduce the above
20   *   copyright notice, this list of conditions and the following
21   *   disclaimer in the documentation and/or other materials provided
22   *   with the distribution.
23   *
24   * - Neither the name of the Eclipse Foundation, Inc. nor the
25   *   names of its contributors may be used to endorse or promote
26   *   products derived from this software without specific prior
27   *   written permission.
28   *
29   * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
30   * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
31   * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
32   * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
33   * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
34   * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
35   * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
36   * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
37   * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
38   * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
39   * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
40   * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
41   * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42   */
43  package org.eclipse.jgit.lfs.internal;
44  
45  import static org.eclipse.jgit.util.HttpSupport.ENCODING_GZIP;
46  import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT;
47  import static org.eclipse.jgit.util.HttpSupport.HDR_ACCEPT_ENCODING;
48  import static org.eclipse.jgit.util.HttpSupport.HDR_CONTENT_TYPE;
49  
50  import java.io.IOException;
51  import java.net.ProxySelector;
52  import java.net.URISyntaxException;
53  import java.net.URL;
54  import java.text.SimpleDateFormat;
55  import java.util.LinkedList;
56  import java.util.Map;
57  import java.util.TreeMap;
58  
59  import org.eclipse.jgit.annotations.NonNull;
60  import org.eclipse.jgit.errors.CommandFailedException;
61  import org.eclipse.jgit.lfs.LfsPointer;
62  import org.eclipse.jgit.lfs.Protocol;
63  import org.eclipse.jgit.lfs.errors.LfsConfigInvalidException;
64  import org.eclipse.jgit.lib.ConfigConstants;
65  import org.eclipse.jgit.lib.Repository;
66  import org.eclipse.jgit.lib.StoredConfig;
67  import org.eclipse.jgit.transport.HttpConfig;
68  import org.eclipse.jgit.transport.HttpTransport;
69  import org.eclipse.jgit.transport.URIish;
70  import org.eclipse.jgit.transport.http.HttpConnection;
71  import org.eclipse.jgit.util.HttpSupport;
72  import org.eclipse.jgit.util.SshSupport;
73  
74  /**
75   * Provides means to get a valid LFS connection for a given repository.
76   */
77  public class LfsConnectionFactory {
78  
79  	private static final int SSH_AUTH_TIMEOUT_SECONDS = 30;
80  	private static final String SCHEME_HTTPS = "https"; //$NON-NLS-1$
81  	private static final String SCHEME_SSH = "ssh"; //$NON-NLS-1$
82  	private static final Map<String, AuthCache> sshAuthCache = new TreeMap<>();
83  
84  	/**
85  	 * Determine URL of LFS server by looking into config parameters lfs.url,
86  	 * lfs.[remote].url or remote.[remote].url. The LFS server URL is computed
87  	 * from remote.[remote].url by appending "/info/lfs". In case there is no
88  	 * URL configured, a SSH remote URI can be used to auto-detect the LFS URI
89  	 * by using the remote "git-lfs-authenticate" command.
90  	 *
91  	 * @param db
92  	 *            the repository to work with
93  	 * @param method
94  	 *            the method (GET,PUT,...) of the request this connection will
95  	 *            be used for
96  	 * @param purpose
97  	 *            the action, e.g. Protocol.OPERATION_DOWNLOAD
98  	 * @return the url for the lfs server. e.g.
99  	 *         "https://github.com/github/git-lfs.git/info/lfs"
100 	 * @throws IOException
101 	 */
102 	public static HttpConnection getLfsConnection(Repository db, String method,
103 			String purpose) throws IOException {
104 		StoredConfig config = db.getConfig();
105 		Map<String, String> additionalHeaders = new TreeMap<>();
106 		String lfsUrl = getLfsUrl(db, purpose, additionalHeaders);
107 		URL url = new URL(lfsUrl + Protocol.OBJECTS_LFS_ENDPOINT);
108 		HttpConnection connection = HttpTransport.getConnectionFactory().create(
109 				url, HttpSupport.proxyFor(ProxySelector.getDefault(), url));
110 		connection.setDoOutput(true);
111 		if (url.getProtocol().equals(SCHEME_HTTPS)
112 				&& !config.getBoolean(HttpConfig.HTTP,
113 						HttpConfig.SSL_VERIFY_KEY, true)) {
114 			HttpSupport.disableSslVerify(connection);
115 		}
116 		connection.setRequestMethod(method);
117 		connection.setRequestProperty(HDR_ACCEPT,
118 				Protocol.CONTENTTYPE_VND_GIT_LFS_JSON);
119 		connection.setRequestProperty(HDR_CONTENT_TYPE,
120 				Protocol.CONTENTTYPE_VND_GIT_LFS_JSON);
121 		additionalHeaders
122 				.forEach((k, v) -> connection.setRequestProperty(k, v));
123 		return connection;
124 	}
125 
126 	private static String getLfsUrl(Repository db, String purpose,
127 			Map<String, String> additionalHeaders)
128 			throws LfsConfigInvalidException {
129 		StoredConfig config = db.getConfig();
130 		String lfsUrl = config.getString(ConfigConstants.CONFIG_SECTION_LFS,
131 				null,
132 				ConfigConstants.CONFIG_KEY_URL);
133 		Exception ex = null;
134 		if (lfsUrl == null) {
135 			String remoteUrl = null;
136 			for (String remote : db.getRemoteNames()) {
137 				lfsUrl = config.getString(ConfigConstants.CONFIG_SECTION_LFS,
138 						remote,
139 						ConfigConstants.CONFIG_KEY_URL);
140 				// This could be done better (more precise logic), but according
141 				// to https://github.com/git-lfs/git-lfs/issues/1759 git-lfs
142 				// generally only supports 'origin' in an integrated workflow.
143 				if (lfsUrl == null && (remote.equals(
144 						org.eclipse.jgit.lib.Constants.DEFAULT_REMOTE_NAME))) {
145 					remoteUrl = config.getString(
146 							ConfigConstants.CONFIG_KEY_REMOTE, remote,
147 							ConfigConstants.CONFIG_KEY_URL);
148 				}
149 				break;
150 			}
151 			if (lfsUrl == null && remoteUrl != null) {
152 				try {
153 					lfsUrl = discoverLfsUrl(db, purpose, additionalHeaders,
154 							remoteUrl);
155 				} catch (URISyntaxException | IOException
156 						| CommandFailedException e) {
157 					ex = e;
158 				}
159 			} else {
160 				lfsUrl = lfsUrl + Protocol.INFO_LFS_ENDPOINT;
161 			}
162 		}
163 		if (lfsUrl == null) {
164 			if (ex != null) {
165 				throw new LfsConfigInvalidException(
166 						LfsText.get().lfsNoDownloadUrl, ex);
167 			}
168 			throw new LfsConfigInvalidException(LfsText.get().lfsNoDownloadUrl);
169 		}
170 		return lfsUrl;
171 	}
172 
173 	private static String discoverLfsUrl(Repository db, String purpose,
174 			Map<String, String> additionalHeaders, String remoteUrl)
175 			throws URISyntaxException, IOException, CommandFailedException {
176 		URIish u = new URIish(remoteUrl);
177 		if (u.getScheme() == null || SCHEME_SSH.equals(u.getScheme())) {
178 			Protocol.ExpiringAction action = getSshAuthentication(db, purpose,
179 					remoteUrl, u);
180 			additionalHeaders.putAll(action.header);
181 			return action.href;
182 		} else {
183 			return remoteUrl + Protocol.INFO_LFS_ENDPOINT;
184 		}
185 	}
186 
187 	private static Protocol.ExpiringAction getSshAuthentication(
188 			Repository db, String purpose, String remoteUrl, URIish u)
189 			throws IOException, CommandFailedException {
190 		AuthCache cached = sshAuthCache.get(remoteUrl);
191 		Protocol.ExpiringAction action = null;
192 		if (cached != null && cached.validUntil > System.currentTimeMillis()) {
193 			action = cached.cachedAction;
194 		}
195 
196 		if (action == null) {
197 			// discover and authenticate; git-lfs does "ssh
198 			// -p <port> -- <host> git-lfs-authenticate
199 			// <project> <upload/download>"
200 			String json = SshSupport.runSshCommand(u.setPath(""), //$NON-NLS-1$
201 					null, db.getFS(),
202 					"git-lfs-authenticate " + extractProjectName(u) + " " //$NON-NLS-1$//$NON-NLS-2$
203 							+ purpose,
204 					SSH_AUTH_TIMEOUT_SECONDS);
205 
206 			action = Protocol.gson().fromJson(json,
207 					Protocol.ExpiringAction.class);
208 
209 			// cache the result as long as possible.
210 			AuthCache c = new AuthCache(action);
211 			sshAuthCache.put(remoteUrl, c);
212 		}
213 		return action;
214 	}
215 
216 	/**
217 	 * Create a connection for the specified
218 	 * {@link org.eclipse.jgit.lfs.Protocol.Action}.
219 	 *
220 	 * @param repo
221 	 *            the repo to fetch required configuration from
222 	 * @param action
223 	 *            the action for which to create a connection
224 	 * @param method
225 	 *            the target method (GET or PUT)
226 	 * @return a connection. output mode is not set.
227 	 * @throws IOException
228 	 *             in case of any error.
229 	 */
230 	@NonNull
231 	public static HttpConnection getLfsContentConnection(
232 			Repository repo, Protocol.Action action, String method)
233 			throws IOException {
234 		URL contentUrl = new URL(action.href);
235 		HttpConnection contentServerConn = HttpTransport.getConnectionFactory()
236 				.create(contentUrl, HttpSupport
237 						.proxyFor(ProxySelector.getDefault(), contentUrl));
238 		contentServerConn.setRequestMethod(method);
239 		if (action.header != null) {
240 			action.header.forEach(
241 					(k, v) -> contentServerConn.setRequestProperty(k, v));
242 		}
243 		if (contentUrl.getProtocol().equals(SCHEME_HTTPS)
244 				&& !repo.getConfig().getBoolean(HttpConfig.HTTP,
245 						HttpConfig.SSL_VERIFY_KEY, true)) {
246 			HttpSupport.disableSslVerify(contentServerConn);
247 		}
248 
249 		contentServerConn.setRequestProperty(HDR_ACCEPT_ENCODING,
250 				ENCODING_GZIP);
251 
252 		return contentServerConn;
253 	}
254 
255 	private static String extractProjectName(URIish u) {
256 		String path = u.getPath();
257 
258 		// begins with a slash if the url contains a port (gerrit vs. github).
259 		if (path.startsWith("/")) { //$NON-NLS-1$
260 			path = path.substring(1);
261 		}
262 
263 		if (path.endsWith(org.eclipse.jgit.lib.Constants.DOT_GIT)) {
264 			return path.substring(0, path.length() - 4);
265 		} else {
266 			return path;
267 		}
268 	}
269 
270 	/**
271 	 * @param operation
272 	 *            the operation to perform, e.g. Protocol.OPERATION_DOWNLOAD
273 	 * @param resources
274 	 *            the LFS resources affected
275 	 * @return a request that can be serialized to JSON
276 	 */
277 	public static Protocol.Request toRequest(String operation,
278 			LfsPointer... resources) {
279 		Protocol.Request req = new Protocol.Request();
280 		req.operation = operation;
281 		if (resources != null) {
282 			req.objects = new LinkedList<>();
283 			for (LfsPointer res : resources) {
284 				Protocol.ObjectSpec o = new Protocol.ObjectSpec();
285 				o.oid = res.getOid().getName();
286 				o.size = res.getSize();
287 				req.objects.add(o);
288 			}
289 		}
290 		return req;
291 	}
292 
293 	private static final class AuthCache {
294 		private static final long AUTH_CACHE_EAGER_TIMEOUT = 500;
295 
296 		private static final SimpleDateFormat ISO_FORMAT = new SimpleDateFormat(
297 				"yyyy-MM-dd'T'HH:mm:ss.SSSX"); //$NON-NLS-1$
298 
299 		/**
300 		 * Creates a cache entry for an authentication response.
301 		 * <p>
302 		 * The timeout of the cache token is extracted from the given action. If
303 		 * no timeout can be determined, the token will be used only once.
304 		 *
305 		 * @param action
306 		 */
307 		public AuthCache(Protocol.ExpiringAction action) {
308 			this.cachedAction = action;
309 			try {
310 				if (action.expiresIn != null && !action.expiresIn.isEmpty()) {
311 					this.validUntil = (System.currentTimeMillis()
312 							+ Long.parseLong(action.expiresIn))
313 							- AUTH_CACHE_EAGER_TIMEOUT;
314 				} else if (action.expiresAt != null
315 						&& !action.expiresAt.isEmpty()) {
316 					this.validUntil = ISO_FORMAT.parse(action.expiresAt)
317 							.getTime() - AUTH_CACHE_EAGER_TIMEOUT;
318 				} else {
319 					this.validUntil = System.currentTimeMillis();
320 				}
321 			} catch (Exception e) {
322 				this.validUntil = System.currentTimeMillis();
323 			}
324 		}
325 
326 		long validUntil;
327 
328 		Protocol.ExpiringAction cachedAction;
329 	}
330 
331 }