View Javadoc
1   /*
2    * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> and others
3    *
4    * This program and the accompanying materials are made available under the
5    * terms of the Eclipse Distribution License v. 1.0 which is available at
6    * https://www.eclipse.org/org/documents/edl-v10.php.
7    *
8    * SPDX-License-Identifier: BSD-3-Clause
9    */
10  
11  package org.eclipse.jgit.transport;
12  
13  import static java.nio.charset.StandardCharsets.UTF_8;
14  
15  import java.io.ByteArrayOutputStream;
16  import java.io.File;
17  import java.io.FileInputStream;
18  import java.io.FileNotFoundException;
19  import java.io.IOException;
20  import java.io.InputStream;
21  import java.io.OutputStream;
22  import java.net.HttpURLConnection;
23  import java.net.Proxy;
24  import java.net.ProxySelector;
25  import java.net.URL;
26  import java.net.URLConnection;
27  import java.security.DigestOutputStream;
28  import java.security.GeneralSecurityException;
29  import java.security.InvalidKeyException;
30  import java.security.MessageDigest;
31  import java.security.NoSuchAlgorithmException;
32  import java.text.MessageFormat;
33  import java.text.SimpleDateFormat;
34  import java.util.ArrayList;
35  import java.util.Collections;
36  import java.util.Comparator;
37  import java.util.Date;
38  import java.util.HashSet;
39  import java.util.Iterator;
40  import java.util.List;
41  import java.util.Locale;
42  import java.util.Map;
43  import java.util.Properties;
44  import java.util.Set;
45  import java.util.SortedMap;
46  import java.util.TimeZone;
47  import java.util.TreeMap;
48  import java.util.stream.Collectors;
49  import java.time.Instant;
50  
51  import javax.crypto.Mac;
52  import javax.crypto.spec.SecretKeySpec;
53  
54  import org.eclipse.jgit.internal.JGitText;
55  import org.eclipse.jgit.lib.Constants;
56  import org.eclipse.jgit.lib.NullProgressMonitor;
57  import org.eclipse.jgit.lib.ProgressMonitor;
58  import org.eclipse.jgit.util.Base64;
59  import org.eclipse.jgit.util.HttpSupport;
60  import org.eclipse.jgit.util.StringUtils;
61  import org.eclipse.jgit.util.TemporaryBuffer;
62  import org.xml.sax.Attributes;
63  import org.xml.sax.InputSource;
64  import org.xml.sax.SAXException;
65  import org.xml.sax.XMLReader;
66  import org.xml.sax.helpers.DefaultHandler;
67  import org.xml.sax.helpers.XMLReaderFactory;
68  
69  /**
70   * A simple HTTP REST client for the Amazon S3 service.
71   * <p>
72   * This client uses the REST API to communicate with the Amazon S3 servers and
73   * read or write content through a bucket that the user has access to. It is a
74   * very lightweight implementation of the S3 API and therefore does not have all
75   * of the bells and whistles of popular client implementations.
76   * <p>
77   * Authentication is always performed using the user's AWSAccessKeyId and their
78   * private AWSSecretAccessKey.
79   * <p>
80   * Optional client-side encryption may be enabled if requested. The format is
81   * compatible with <a href="http://jets3t.s3.amazonaws.com/index.html">jets3t</a>,
82   * a popular Java based Amazon S3 client library. Enabling encryption can hide
83   * sensitive data from the operators of the S3 service.
84   */
85  public class AmazonS3 {
86  	private static final Set<String> SIGNED_HEADERS;
87  
88  	private static final String HMAC = "HmacSHA1"; //$NON-NLS-1$
89  
90  	private static final String X_AMZ_ACL = "x-amz-acl"; //$NON-NLS-1$
91  
92  	private static final String X_AMZ_META = "x-amz-meta-"; //$NON-NLS-1$
93  
94  	static {
95  		SIGNED_HEADERS = new HashSet<>();
96  		SIGNED_HEADERS.add("content-type"); //$NON-NLS-1$
97  		SIGNED_HEADERS.add("content-md5"); //$NON-NLS-1$
98  		SIGNED_HEADERS.add("date"); //$NON-NLS-1$
99  	}
100 
101 	private static boolean isSignedHeader(String name) {
102 		final String nameLC = StringUtils.toLowerCase(name);
103 		return SIGNED_HEADERS.contains(nameLC) || nameLC.startsWith("x-amz-"); //$NON-NLS-1$
104 	}
105 
106 	private static String toCleanString(List<String> list) {
107 		final StringBuilder s = new StringBuilder();
108 		for (String v : list) {
109 			if (s.length() > 0)
110 				s.append(',');
111 			s.append(v.replaceAll("\n", "").trim()); //$NON-NLS-1$ //$NON-NLS-2$
112 		}
113 		return s.toString();
114 	}
115 
116 	private static String remove(Map<String, String> m, String k) {
117 		final String r = m.remove(k);
118 		return r != null ? r : ""; //$NON-NLS-1$
119 	}
120 
121 	private static String httpNow() {
122 		final String tz = "GMT"; //$NON-NLS-1$
123 		final SimpleDateFormat fmt;
124 		fmt = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss", Locale.US); //$NON-NLS-1$
125 		fmt.setTimeZone(TimeZone.getTimeZone(tz));
126 		return fmt.format(new Date()) + " " + tz; //$NON-NLS-1$
127 	}
128 
129 	private static MessageDigest newMD5() {
130 		try {
131 			return MessageDigest.getInstance("MD5"); //$NON-NLS-1$
132 		} catch (NoSuchAlgorithmException e) {
133 			throw new RuntimeException(JGitText.get().JRELacksMD5Implementation, e);
134 		}
135 	}
136 
137 	/** AWSAccessKeyId, public string that identifies the user's account. */
138 	private final String publicKey;
139 
140 	/** Decoded form of the private AWSSecretAccessKey, to sign requests. */
141 	private final SecretKeySpec privateKey;
142 
143 	/** Our HTTP proxy support, in case we are behind a firewall. */
144 	private final ProxySelector proxySelector;
145 
146 	/** ACL to apply to created objects. */
147 	private final String acl;
148 
149 	/** Maximum number of times to try an operation. */
150 	final int maxAttempts;
151 
152 	/** Encryption algorithm, may be a null instance that provides pass-through. */
153 	private final WalkEncryption encryption;
154 
155 	/** Directory for locally buffered content. */
156 	private final File tmpDir;
157 
158 	/** S3 Bucket Domain. */
159 	private final String domain;
160 
161 	/** Property names used in amazon connection configuration file. */
162 	interface Keys {
163 		String ACCESS_KEY = "accesskey"; //$NON-NLS-1$
164 		String SECRET_KEY = "secretkey"; //$NON-NLS-1$
165 		String PASSWORD = "password"; //$NON-NLS-1$
166 		String CRYPTO_ALG = "crypto.algorithm"; //$NON-NLS-1$
167 		String CRYPTO_VER = "crypto.version"; //$NON-NLS-1$
168 		String ACL = "acl"; //$NON-NLS-1$
169 		String DOMAIN = "domain"; //$NON-NLS-1$
170 		String HTTP_RETRY = "httpclient.retry-max"; //$NON-NLS-1$
171 		String TMP_DIR = "tmpdir"; //$NON-NLS-1$
172 	}
173 
174 	/**
175 	 * Create a new S3 client for the supplied user information.
176 	 * <p>
177 	 * The connection properties are a subset of those supported by the popular
178 	 * <a href="http://jets3t.s3.amazonaws.com/index.html">jets3t</a> library.
179 	 * For example:
180 	 *
181 	 * <pre>
182 	 * # AWS Access and Secret Keys (required)
183 	 * accesskey: &lt;YourAWSAccessKey&gt;
184 	 * secretkey: &lt;YourAWSSecretKey&gt;
185 	 *
186 	 * # Access Control List setting to apply to uploads, must be one of:
187 	 * # PRIVATE, PUBLIC_READ (defaults to PRIVATE).
188 	 * acl: PRIVATE
189 	 *
190 	 * # S3 Domain
191 	 * # AWS S3 Region Domain (defaults to s3.amazonaws.com)
192 	 * domain: s3.amazonaws.com
193 	 *
194 	 * # Number of times to retry after internal error from S3.
195 	 * httpclient.retry-max: 3
196 	 *
197 	 * # End-to-end encryption (hides content from S3 owners)
198 	 * password: &lt;encryption pass-phrase&gt;
199 	 * crypto.algorithm: PBEWithMD5AndDES
200 	 * </pre>
201 	 *
202 	 * @param props
203 	 *            connection properties.
204 	 */
205 	public AmazonS3(final Properties props) {
206 		domain = props.getProperty(Keys.DOMAIN, "s3.amazonaws.com"); //$NON-NLS-1$
207 
208 		publicKey = props.getProperty(Keys.ACCESS_KEY);
209 		if (publicKey == null)
210 			throw new IllegalArgumentException(JGitText.get().missingAccesskey);
211 
212 		final String secret = props.getProperty(Keys.SECRET_KEY);
213 		if (secret == null)
214 			throw new IllegalArgumentException(JGitText.get().missingSecretkey);
215 		privateKey = new SecretKeySpec(Constants.encodeASCII(secret), HMAC);
216 
217 		final String pacl = props.getProperty(Keys.ACL, "PRIVATE"); //$NON-NLS-1$
218 		if (StringUtils.equalsIgnoreCase("PRIVATE", pacl)) //$NON-NLS-1$
219 			acl = "private"; //$NON-NLS-1$
220 		else if (StringUtils.equalsIgnoreCase("PUBLIC", pacl)) //$NON-NLS-1$
221 			acl = "public-read"; //$NON-NLS-1$
222 		else if (StringUtils.equalsIgnoreCase("PUBLIC-READ", pacl)) //$NON-NLS-1$
223 			acl = "public-read"; //$NON-NLS-1$
224 		else if (StringUtils.equalsIgnoreCase("PUBLIC_READ", pacl)) //$NON-NLS-1$
225 			acl = "public-read"; //$NON-NLS-1$
226 		else
227 			throw new IllegalArgumentException("Invalid acl: " + pacl); //$NON-NLS-1$
228 
229 		try {
230 			encryption = WalkEncryption.instance(props);
231 		} catch (GeneralSecurityException e) {
232 			throw new IllegalArgumentException(JGitText.get().invalidEncryption, e);
233 		}
234 
235 		maxAttempts = Integer
236 				.parseInt(props.getProperty(Keys.HTTP_RETRY, "3")); //$NON-NLS-1$
237 		proxySelector = ProxySelector.getDefault();
238 
239 		String tmp = props.getProperty(Keys.TMP_DIR);
240 		tmpDir = tmp != null && tmp.length() > 0 ? new File(tmp) : null;
241 	}
242 
243 	/**
244 	 * Get the content of a bucket object.
245 	 *
246 	 * @param bucket
247 	 *            name of the bucket storing the object.
248 	 * @param key
249 	 *            key of the object within its bucket.
250 	 * @return connection to stream the content of the object. The request
251 	 *         properties of the connection may not be modified by the caller as
252 	 *         the request parameters have already been signed.
253 	 * @throws java.io.IOException
254 	 *             sending the request was not possible.
255 	 */
256 	public URLConnection get(String bucket, String key)
257 			throws IOException {
258 		for (int curAttempt = 0; curAttempt < maxAttempts; curAttempt++) {
259 			final HttpURLConnection c = open("GET", bucket, key); //$NON-NLS-1$
260 			authorize(c);
261 			switch (HttpSupport.response(c)) {
262 			case HttpURLConnection.HTTP_OK:
263 				encryption.validate(c, X_AMZ_META);
264 				return c;
265 			case HttpURLConnection.HTTP_NOT_FOUND:
266 				throw new FileNotFoundException(key);
267 			case HttpURLConnection.HTTP_INTERNAL_ERROR:
268 				continue;
269 			default:
270 				throw error(JGitText.get().s3ActionReading, key, c);
271 			}
272 		}
273 		throw maxAttempts(JGitText.get().s3ActionReading, key);
274 	}
275 
276 	/**
277 	 * Decrypt an input stream from {@link #get(String, String)}.
278 	 *
279 	 * @param u
280 	 *            connection previously created by {@link #get(String, String)}}.
281 	 * @return stream to read plain text from.
282 	 * @throws java.io.IOException
283 	 *             decryption could not be configured.
284 	 */
285 	public InputStream decrypt(URLConnection u) throws IOException {
286 		return encryption.decrypt(u.getInputStream());
287 	}
288 
289 	/**
290 	 * List the names of keys available within a bucket.
291 	 * <p>
292 	 * This method is primarily meant for obtaining a "recursive directory
293 	 * listing" rooted under the specified bucket and prefix location.
294 	 * It returns the keys sorted in reverse order of LastModified time
295 	 * (freshest keys first).
296 	 *
297 	 * @param bucket
298 	 *            name of the bucket whose objects should be listed.
299 	 * @param prefix
300 	 *            common prefix to filter the results by. Must not be null.
301 	 *            Supplying the empty string will list all keys in the bucket.
302 	 *            Supplying a non-empty string will act as though a trailing '/'
303 	 *            appears in prefix, even if it does not.
304 	 * @return list of keys starting with <code>prefix</code>, after removing
305 	 *         <code>prefix</code> (or <code>prefix + "/"</code>)from all
306 	 *         of them.
307 	 * @throws java.io.IOException
308 	 *             sending the request was not possible, or the response XML
309 	 *             document could not be parsed properly.
310 	 */
311 	public List<String> list(String bucket, String prefix)
312 			throws IOException {
313 		if (prefix.length() > 0 && !prefix.endsWith("/")) //$NON-NLS-1$
314 			prefix += "/"; //$NON-NLS-1$
315 		final ListParser lp = new ListParser(bucket, prefix);
316 		do {
317 			lp.list();
318 		} while (lp.truncated);
319 
320 		Comparator<KeyInfo> comparator = Comparator.comparingLong(KeyInfo::getLastModifiedSecs);
321 		return lp.entries.stream().sorted(comparator.reversed())
322 			.map(KeyInfo::getName).collect(Collectors.toList());
323 	}
324 
325 	/**
326 	 * Delete a single object.
327 	 * <p>
328 	 * Deletion always succeeds, even if the object does not exist.
329 	 *
330 	 * @param bucket
331 	 *            name of the bucket storing the object.
332 	 * @param key
333 	 *            key of the object within its bucket.
334 	 * @throws java.io.IOException
335 	 *             deletion failed due to communications error.
336 	 */
337 	public void delete(String bucket, String key)
338 			throws IOException {
339 		for (int curAttempt = 0; curAttempt < maxAttempts; curAttempt++) {
340 			final HttpURLConnection c = open("DELETE", bucket, key); //$NON-NLS-1$
341 			authorize(c);
342 			switch (HttpSupport.response(c)) {
343 			case HttpURLConnection.HTTP_NO_CONTENT:
344 				return;
345 			case HttpURLConnection.HTTP_INTERNAL_ERROR:
346 				continue;
347 			default:
348 				throw error(JGitText.get().s3ActionDeletion, key, c);
349 			}
350 		}
351 		throw maxAttempts(JGitText.get().s3ActionDeletion, key);
352 	}
353 
354 	/**
355 	 * Atomically create or replace a single small object.
356 	 * <p>
357 	 * This form is only suitable for smaller contents, where the caller can
358 	 * reasonable fit the entire thing into memory.
359 	 * <p>
360 	 * End-to-end data integrity is assured by internally computing the MD5
361 	 * checksum of the supplied data and transmitting the checksum along with
362 	 * the data itself.
363 	 *
364 	 * @param bucket
365 	 *            name of the bucket storing the object.
366 	 * @param key
367 	 *            key of the object within its bucket.
368 	 * @param data
369 	 *            new data content for the object. Must not be null. Zero length
370 	 *            array will create a zero length object.
371 	 * @throws java.io.IOException
372 	 *             creation/updating failed due to communications error.
373 	 */
374 	public void put(String bucket, String key, byte[] data)
375 			throws IOException {
376 		if (encryption != WalkEncryption.NONE) {
377 			// We have to copy to produce the cipher text anyway so use
378 			// the large object code path as it supports that behavior.
379 			//
380 			try (OutputStream os = beginPut(bucket, key, null, null)) {
381 				os.write(data);
382 			}
383 			return;
384 		}
385 
386 		final String md5str = Base64.encodeBytes(newMD5().digest(data));
387 		final String lenstr = String.valueOf(data.length);
388 		for (int curAttempt = 0; curAttempt < maxAttempts; curAttempt++) {
389 			final HttpURLConnection c = open("PUT", bucket, key); //$NON-NLS-1$
390 			c.setRequestProperty("Content-Length", lenstr); //$NON-NLS-1$
391 			c.setRequestProperty("Content-MD5", md5str); //$NON-NLS-1$
392 			c.setRequestProperty(X_AMZ_ACL, acl);
393 			authorize(c);
394 			c.setDoOutput(true);
395 			c.setFixedLengthStreamingMode(data.length);
396 			try (OutputStream os = c.getOutputStream()) {
397 				os.write(data);
398 			}
399 
400 			switch (HttpSupport.response(c)) {
401 			case HttpURLConnection.HTTP_OK:
402 				return;
403 			case HttpURLConnection.HTTP_INTERNAL_ERROR:
404 				continue;
405 			default:
406 				throw error(JGitText.get().s3ActionWriting, key, c);
407 			}
408 		}
409 		throw maxAttempts(JGitText.get().s3ActionWriting, key);
410 	}
411 
412 	/**
413 	 * Atomically create or replace a single large object.
414 	 * <p>
415 	 * Initially the returned output stream buffers data into memory, but if the
416 	 * total number of written bytes starts to exceed an internal limit the data
417 	 * is spooled to a temporary file on the local drive.
418 	 * <p>
419 	 * Network transmission is attempted only when <code>close()</code> gets
420 	 * called at the end of output. Closing the returned stream can therefore
421 	 * take significant time, especially if the written content is very large.
422 	 * <p>
423 	 * End-to-end data integrity is assured by internally computing the MD5
424 	 * checksum of the supplied data and transmitting the checksum along with
425 	 * the data itself.
426 	 *
427 	 * @param bucket
428 	 *            name of the bucket storing the object.
429 	 * @param key
430 	 *            key of the object within its bucket.
431 	 * @param monitor
432 	 *            (optional) progress monitor to post upload completion to
433 	 *            during the stream's close method.
434 	 * @param monitorTask
435 	 *            (optional) task name to display during the close method.
436 	 * @return a stream which accepts the new data, and transmits once closed.
437 	 * @throws java.io.IOException
438 	 *             if encryption was enabled it could not be configured.
439 	 */
440 	public OutputStream beginPut(final String bucket, final String key,
441 			final ProgressMonitor monitor, final String monitorTask)
442 			throws IOException {
443 		final MessageDigest md5 = newMD5();
444 		final TemporaryBuffer buffer = new TemporaryBuffer.LocalFile(tmpDir) {
445 			@Override
446 			public void close() throws IOException {
447 				super.close();
448 				try {
449 					putImpl(bucket, key, md5.digest(), this, monitor,
450 							monitorTask);
451 				} finally {
452 					destroy();
453 				}
454 			}
455 		};
456 		return encryption.encrypt(new DigestOutputStream(buffer, md5));
457 	}
458 
459 	void putImpl(final String bucket, final String key,
460 			final byte[] csum, final TemporaryBuffer buf,
461 			ProgressMonitor monitor, String monitorTask) throws IOException {
462 		if (monitor == null)
463 			monitor = NullProgressMonitor.INSTANCE;
464 		if (monitorTask == null)
465 			monitorTask = MessageFormat.format(JGitText.get().progressMonUploading, key);
466 
467 		final String md5str = Base64.encodeBytes(csum);
468 		final long len = buf.length();
469 		for (int curAttempt = 0; curAttempt < maxAttempts; curAttempt++) {
470 			final HttpURLConnection c = open("PUT", bucket, key); //$NON-NLS-1$
471 			c.setFixedLengthStreamingMode(len);
472 			c.setRequestProperty("Content-MD5", md5str); //$NON-NLS-1$
473 			c.setRequestProperty(X_AMZ_ACL, acl);
474 			encryption.request(c, X_AMZ_META);
475 			authorize(c);
476 			c.setDoOutput(true);
477 			monitor.beginTask(monitorTask, (int) (len / 1024));
478 			try (OutputStream os = c.getOutputStream()) {
479 				buf.writeTo(os, monitor);
480 			} finally {
481 				monitor.endTask();
482 			}
483 
484 			switch (HttpSupport.response(c)) {
485 			case HttpURLConnection.HTTP_OK:
486 				return;
487 			case HttpURLConnection.HTTP_INTERNAL_ERROR:
488 				continue;
489 			default:
490 				throw error(JGitText.get().s3ActionWriting, key, c);
491 			}
492 		}
493 		throw maxAttempts(JGitText.get().s3ActionWriting, key);
494 	}
495 
496 	IOException error(final String action, final String key,
497 			final HttpURLConnection c) throws IOException {
498 		final IOException err = new IOException(MessageFormat.format(
499 				JGitText.get().amazonS3ActionFailed, action, key,
500 				Integer.valueOf(HttpSupport.response(c)),
501 				c.getResponseMessage()));
502 		if (c.getErrorStream() == null) {
503 			return err;
504 		}
505 
506 		try (InputStream errorStream = c.getErrorStream()) {
507 			final ByteArrayOutputStream b = new ByteArrayOutputStream();
508 			byte[] buf = new byte[2048];
509 			for (;;) {
510 				final int n = errorStream.read(buf);
511 				if (n < 0) {
512 					break;
513 				}
514 				if (n > 0) {
515 					b.write(buf, 0, n);
516 				}
517 			}
518 			buf = b.toByteArray();
519 			if (buf.length > 0) {
520 				err.initCause(new IOException("\n" + new String(buf, UTF_8))); //$NON-NLS-1$
521 			}
522 		}
523 		return err;
524 	}
525 
526 	IOException maxAttempts(String action, String key) {
527 		return new IOException(MessageFormat.format(
528 				JGitText.get().amazonS3ActionFailedGivingUp, action, key,
529 				Integer.valueOf(maxAttempts)));
530 	}
531 
532 	private HttpURLConnection open(final String method, final String bucket,
533 			final String key) throws IOException {
534 		final Map<String, String> noArgs = Collections.emptyMap();
535 		return open(method, bucket, key, noArgs);
536 	}
537 
538 	HttpURLConnection open(final String method, final String bucket,
539 			final String key, final Map<String, String> args)
540 			throws IOException {
541 		final StringBuilder urlstr = new StringBuilder();
542 		urlstr.append("http://"); //$NON-NLS-1$
543 		urlstr.append(bucket);
544 		urlstr.append('.');
545 		urlstr.append(domain);
546 		urlstr.append('/');
547 		if (key.length() > 0)
548 			HttpSupport.encode(urlstr, key);
549 		if (!args.isEmpty()) {
550 			final Iterator<Map.Entry<String, String>> i;
551 
552 			urlstr.append('?');
553 			i = args.entrySet().iterator();
554 			while (i.hasNext()) {
555 				final Map.Entry<String, String> e = i.next();
556 				urlstr.append(e.getKey());
557 				urlstr.append('=');
558 				HttpSupport.encode(urlstr, e.getValue());
559 				if (i.hasNext())
560 					urlstr.append('&');
561 			}
562 		}
563 
564 		final URL url = new URL(urlstr.toString());
565 		final Proxy proxy = HttpSupport.proxyFor(proxySelector, url);
566 		final HttpURLConnection c;
567 
568 		c = (HttpURLConnection) url.openConnection(proxy);
569 		c.setRequestMethod(method);
570 		c.setRequestProperty("User-Agent", "jgit/1.0"); //$NON-NLS-1$ //$NON-NLS-2$
571 		c.setRequestProperty("Date", httpNow()); //$NON-NLS-1$
572 		return c;
573 	}
574 
575 	void authorize(HttpURLConnection c) throws IOException {
576 		final Map<String, List<String>> reqHdr = c.getRequestProperties();
577 		final SortedMap<String, String> sigHdr = new TreeMap<>();
578 		for (Map.Entry<String, List<String>> entry : reqHdr.entrySet()) {
579 			final String hdr = entry.getKey();
580 			if (isSignedHeader(hdr))
581 				sigHdr.put(StringUtils.toLowerCase(hdr), toCleanString(entry.getValue()));
582 		}
583 
584 		final StringBuilder s = new StringBuilder();
585 		s.append(c.getRequestMethod());
586 		s.append('\n');
587 
588 		s.append(remove(sigHdr, "content-md5")); //$NON-NLS-1$
589 		s.append('\n');
590 
591 		s.append(remove(sigHdr, "content-type")); //$NON-NLS-1$
592 		s.append('\n');
593 
594 		s.append(remove(sigHdr, "date")); //$NON-NLS-1$
595 		s.append('\n');
596 
597 		for (Map.Entry<String, String> e : sigHdr.entrySet()) {
598 			s.append(e.getKey());
599 			s.append(':');
600 			s.append(e.getValue());
601 			s.append('\n');
602 		}
603 
604 		final String host = c.getURL().getHost();
605 		s.append('/');
606 		s.append(host.substring(0, host.length() - domain.length() - 1));
607 		s.append(c.getURL().getPath());
608 
609 		final String sec;
610 		try {
611 			final Mac m = Mac.getInstance(HMAC);
612 			m.init(privateKey);
613 			sec = Base64.encodeBytes(m.doFinal(s.toString().getBytes(UTF_8)));
614 		} catch (NoSuchAlgorithmException e) {
615 			throw new IOException(MessageFormat.format(JGitText.get().noHMACsupport, HMAC, e.getMessage()));
616 		} catch (InvalidKeyException e) {
617 			throw new IOException(MessageFormat.format(JGitText.get().invalidKey, e.getMessage()));
618 		}
619 		c.setRequestProperty("Authorization", "AWS " + publicKey + ":" + sec); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
620 	}
621 
622 	static Properties properties(File authFile)
623 			throws FileNotFoundException, IOException {
624 		final Properties p = new Properties();
625 		try (FileInputStream in = new FileInputStream(authFile)) {
626 			p.load(in);
627 		}
628 		return p;
629 	}
630 
631 	/**
632 	 * KeyInfo enables sorting of keys by lastModified time
633 	 */
634 	private static final class KeyInfo {
635 		private final String name;
636 		private final long lastModifiedSecs;
637 		public KeyInfo(String aname, long lsecs) {
638 			name = aname;
639 			lastModifiedSecs = lsecs;
640 		}
641 		public String getName() {
642 			return name;
643 		}
644 		public long getLastModifiedSecs() {
645 			return lastModifiedSecs;
646 		}
647 	}
648 
649 	private final class ListParser extends DefaultHandler {
650 		final List<KeyInfo> entries = new ArrayList<>();
651 
652 		private final String bucket;
653 
654 		private final String prefix;
655 
656 		boolean truncated;
657 
658 		private StringBuilder data;
659 		private String keyName;
660 		private Instant keyLastModified;
661 
662 		ListParser(String bn, String p) {
663 			bucket = bn;
664 			prefix = p;
665 		}
666 
667 		void list() throws IOException {
668 			final Map<String, String> args = new TreeMap<>();
669 			if (prefix.length() > 0)
670 				args.put("prefix", prefix); //$NON-NLS-1$
671 			if (!entries.isEmpty())
672 				args.put("marker", prefix + entries.get(entries.size() - 1).getName()); //$NON-NLS-1$
673 
674 			for (int curAttempt = 0; curAttempt < maxAttempts; curAttempt++) {
675 				final HttpURLConnection c = open("GET", bucket, "", args); //$NON-NLS-1$ //$NON-NLS-2$
676 				authorize(c);
677 				switch (HttpSupport.response(c)) {
678 				case HttpURLConnection.HTTP_OK:
679 					truncated = false;
680 					data = null;
681 					keyName = null;
682 					keyLastModified = null;
683 
684 					final XMLReader xr;
685 					try {
686 						xr = XMLReaderFactory.createXMLReader();
687 					} catch (SAXException e) {
688 						throw new IOException(
689 								JGitText.get().noXMLParserAvailable, e);
690 					}
691 					xr.setContentHandler(this);
692 					try (InputStream in = c.getInputStream()) {
693 						xr.parse(new InputSource(in));
694 					} catch (SAXException parsingError) {
695 						throw new IOException(
696 								MessageFormat.format(
697 										JGitText.get().errorListing, prefix),
698 								parsingError);
699 					}
700 					return;
701 
702 				case HttpURLConnection.HTTP_INTERNAL_ERROR:
703 					continue;
704 
705 				default:
706 					throw AmazonS3.this.error("Listing", prefix, c); //$NON-NLS-1$
707 				}
708 			}
709 			throw maxAttempts("Listing", prefix); //$NON-NLS-1$
710 		}
711 
712 		@Override
713 		public void startElement(final String uri, final String name,
714 				final String qName, final Attributes attributes)
715 				throws SAXException {
716 			if ("Key".equals(name) || "IsTruncated".equals(name) || "LastModified".equals(name)) { //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$
717 				data = new StringBuilder();
718 			}
719 			if ("Contents".equals(name)) { //$NON-NLS-1$
720 				keyName = null;
721 				keyLastModified = null;
722 			}
723 		}
724 
725 		@Override
726 		public void ignorableWhitespace(final char[] ch, final int s,
727 				final int n) throws SAXException {
728 			if (data != null)
729 				data.append(ch, s, n);
730 		}
731 
732 		@Override
733 		public void characters(char[] ch, int s, int n)
734 				throws SAXException {
735 			if (data != null)
736 				data.append(ch, s, n);
737 		}
738 
739 		@Override
740 		public void endElement(final String uri, final String name,
741 				final String qName) throws SAXException {
742 			if ("Key".equals(name))  { //$NON-NLS-1$
743 				keyName = data.toString().substring(prefix.length());
744 			} else if ("IsTruncated".equals(name)) { //$NON-NLS-1$
745 				truncated = StringUtils.equalsIgnoreCase("true", data.toString()); //$NON-NLS-1$
746 			} else if ("LastModified".equals(name)) { //$NON-NLS-1$
747 				keyLastModified = Instant.parse(data.toString());
748 			} else if ("Contents".equals(name)) { //$NON-NLS-1$
749 				entries.add(new KeyInfo(keyName, keyLastModified.getEpochSecond()));
750 			}
751 
752 			data = null;
753 		}
754 	}
755 }