1
2
3
4
5
6
7
8
9
10
11 package org.eclipse.jgit.transport;
12
13 import static java.nio.charset.StandardCharsets.UTF_8;
14
15 import java.io.ByteArrayOutputStream;
16 import java.io.File;
17 import java.io.FileInputStream;
18 import java.io.FileNotFoundException;
19 import java.io.IOException;
20 import java.io.InputStream;
21 import java.io.OutputStream;
22 import java.net.HttpURLConnection;
23 import java.net.Proxy;
24 import java.net.ProxySelector;
25 import java.net.URL;
26 import java.net.URLConnection;
27 import java.security.DigestOutputStream;
28 import java.security.GeneralSecurityException;
29 import java.security.InvalidKeyException;
30 import java.security.MessageDigest;
31 import java.security.NoSuchAlgorithmException;
32 import java.text.MessageFormat;
33 import java.text.SimpleDateFormat;
34 import java.util.ArrayList;
35 import java.util.Collections;
36 import java.util.Comparator;
37 import java.util.Date;
38 import java.util.HashSet;
39 import java.util.Iterator;
40 import java.util.List;
41 import java.util.Locale;
42 import java.util.Map;
43 import java.util.Properties;
44 import java.util.Set;
45 import java.util.SortedMap;
46 import java.util.TimeZone;
47 import java.util.TreeMap;
48 import java.util.stream.Collectors;
49 import java.time.Instant;
50
51 import javax.crypto.Mac;
52 import javax.crypto.spec.SecretKeySpec;
53
54 import org.eclipse.jgit.internal.JGitText;
55 import org.eclipse.jgit.lib.Constants;
56 import org.eclipse.jgit.lib.NullProgressMonitor;
57 import org.eclipse.jgit.lib.ProgressMonitor;
58 import org.eclipse.jgit.util.Base64;
59 import org.eclipse.jgit.util.HttpSupport;
60 import org.eclipse.jgit.util.StringUtils;
61 import org.eclipse.jgit.util.TemporaryBuffer;
62 import org.xml.sax.Attributes;
63 import org.xml.sax.InputSource;
64 import org.xml.sax.SAXException;
65 import org.xml.sax.XMLReader;
66 import org.xml.sax.helpers.DefaultHandler;
67 import org.xml.sax.helpers.XMLReaderFactory;
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85 public class AmazonS3 {
86 private static final Set<String> SIGNED_HEADERS;
87
88 private static final String HMAC = "HmacSHA1";
89
90 private static final String X_AMZ_ACL = "x-amz-acl";
91
92 private static final String X_AMZ_META = "x-amz-meta-";
93
94 static {
95 SIGNED_HEADERS = new HashSet<>();
96 SIGNED_HEADERS.add("content-type");
97 SIGNED_HEADERS.add("content-md5");
98 SIGNED_HEADERS.add("date");
99 }
100
101 private static boolean isSignedHeader(String name) {
102 final String nameLC = StringUtils.toLowerCase(name);
103 return SIGNED_HEADERS.contains(nameLC) || nameLC.startsWith("x-amz-");
104 }
105
106 private static String toCleanString(List<String> list) {
107 final StringBuilder s = new StringBuilder();
108 for (String v : list) {
109 if (s.length() > 0)
110 s.append(',');
111 s.append(v.replaceAll("\n", "").trim());
112 }
113 return s.toString();
114 }
115
116 private static String remove(Map<String, String> m, String k) {
117 final String r = m.remove(k);
118 return r != null ? r : "";
119 }
120
121 private static String httpNow() {
122 final String tz = "GMT";
123 final SimpleDateFormat fmt;
124 fmt = new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss", Locale.US);
125 fmt.setTimeZone(TimeZone.getTimeZone(tz));
126 return fmt.format(new Date()) + " " + tz;
127 }
128
129 private static MessageDigest newMD5() {
130 try {
131 return MessageDigest.getInstance("MD5");
132 } catch (NoSuchAlgorithmException e) {
133 throw new RuntimeException(JGitText.get().JRELacksMD5Implementation, e);
134 }
135 }
136
137
138 private final String publicKey;
139
140
141 private final SecretKeySpec privateKey;
142
143
144 private final ProxySelector proxySelector;
145
146
147 private final String acl;
148
149
150 final int maxAttempts;
151
152
153 private final WalkEncryption encryption;
154
155
156 private final File tmpDir;
157
158
159 private final String domain;
160
161
162 interface Keys {
163 String ACCESS_KEY = "accesskey";
164 String SECRET_KEY = "secretkey";
165 String PASSWORD = "password";
166 String CRYPTO_ALG = "crypto.algorithm";
167 String CRYPTO_VER = "crypto.version";
168 String ACL = "acl";
169 String DOMAIN = "domain";
170 String HTTP_RETRY = "httpclient.retry-max";
171 String TMP_DIR = "tmpdir";
172 }
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205 public AmazonS3(final Properties props) {
206 domain = props.getProperty(Keys.DOMAIN, "s3.amazonaws.com");
207
208 publicKey = props.getProperty(Keys.ACCESS_KEY);
209 if (publicKey == null)
210 throw new IllegalArgumentException(JGitText.get().missingAccesskey);
211
212 final String secret = props.getProperty(Keys.SECRET_KEY);
213 if (secret == null)
214 throw new IllegalArgumentException(JGitText.get().missingSecretkey);
215 privateKey = new SecretKeySpec(Constants.encodeASCII(secret), HMAC);
216
217 final String pacl = props.getProperty(Keys.ACL, "PRIVATE");
218 if (StringUtils.equalsIgnoreCase("PRIVATE", pacl))
219 acl = "private";
220 else if (StringUtils.equalsIgnoreCase("PUBLIC", pacl))
221 acl = "public-read";
222 else if (StringUtils.equalsIgnoreCase("PUBLIC-READ", pacl))
223 acl = "public-read";
224 else if (StringUtils.equalsIgnoreCase("PUBLIC_READ", pacl))
225 acl = "public-read";
226 else
227 throw new IllegalArgumentException("Invalid acl: " + pacl);
228
229 try {
230 encryption = WalkEncryption.instance(props);
231 } catch (GeneralSecurityException e) {
232 throw new IllegalArgumentException(JGitText.get().invalidEncryption, e);
233 }
234
235 maxAttempts = Integer
236 .parseInt(props.getProperty(Keys.HTTP_RETRY, "3"));
237 proxySelector = ProxySelector.getDefault();
238
239 String tmp = props.getProperty(Keys.TMP_DIR);
240 tmpDir = tmp != null && tmp.length() > 0 ? new File(tmp) : null;
241 }
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256 public URLConnection get(String bucket, String key)
257 throws IOException {
258 for (int curAttempt = 0; curAttempt < maxAttempts; curAttempt++) {
259 final HttpURLConnection c = open("GET", bucket, key);
260 authorize(c);
261 switch (HttpSupport.response(c)) {
262 case HttpURLConnection.HTTP_OK:
263 encryption.validate(c, X_AMZ_META);
264 return c;
265 case HttpURLConnection.HTTP_NOT_FOUND:
266 throw new FileNotFoundException(key);
267 case HttpURLConnection.HTTP_INTERNAL_ERROR:
268 continue;
269 default:
270 throw error(JGitText.get().s3ActionReading, key, c);
271 }
272 }
273 throw maxAttempts(JGitText.get().s3ActionReading, key);
274 }
275
276
277
278
279
280
281
282
283
284
285 public InputStream decrypt(URLConnection u) throws IOException {
286 return encryption.decrypt(u.getInputStream());
287 }
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311 public List<String> list(String bucket, String prefix)
312 throws IOException {
313 if (prefix.length() > 0 && !prefix.endsWith("/"))
314 prefix += "/";
315 final ListParser lp = new ListParser(bucket, prefix);
316 do {
317 lp.list();
318 } while (lp.truncated);
319
320 Comparator<KeyInfo> comparator = Comparator.comparingLong(KeyInfo::getLastModifiedSecs);
321 return lp.entries.stream().sorted(comparator.reversed())
322 .map(KeyInfo::getName).collect(Collectors.toList());
323 }
324
325
326
327
328
329
330
331
332
333
334
335
336
337 public void delete(String bucket, String key)
338 throws IOException {
339 for (int curAttempt = 0; curAttempt < maxAttempts; curAttempt++) {
340 final HttpURLConnection c = open("DELETE", bucket, key);
341 authorize(c);
342 switch (HttpSupport.response(c)) {
343 case HttpURLConnection.HTTP_NO_CONTENT:
344 return;
345 case HttpURLConnection.HTTP_INTERNAL_ERROR:
346 continue;
347 default:
348 throw error(JGitText.get().s3ActionDeletion, key, c);
349 }
350 }
351 throw maxAttempts(JGitText.get().s3ActionDeletion, key);
352 }
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374 public void put(String bucket, String key, byte[] data)
375 throws IOException {
376 if (encryption != WalkEncryption.NONE) {
377
378
379
380 try (OutputStream os = beginPut(bucket, key, null, null)) {
381 os.write(data);
382 }
383 return;
384 }
385
386 final String md5str = Base64.encodeBytes(newMD5().digest(data));
387 final String lenstr = String.valueOf(data.length);
388 for (int curAttempt = 0; curAttempt < maxAttempts; curAttempt++) {
389 final HttpURLConnection c = open("PUT", bucket, key);
390 c.setRequestProperty("Content-Length", lenstr);
391 c.setRequestProperty("Content-MD5", md5str);
392 c.setRequestProperty(X_AMZ_ACL, acl);
393 authorize(c);
394 c.setDoOutput(true);
395 c.setFixedLengthStreamingMode(data.length);
396 try (OutputStream os = c.getOutputStream()) {
397 os.write(data);
398 }
399
400 switch (HttpSupport.response(c)) {
401 case HttpURLConnection.HTTP_OK:
402 return;
403 case HttpURLConnection.HTTP_INTERNAL_ERROR:
404 continue;
405 default:
406 throw error(JGitText.get().s3ActionWriting, key, c);
407 }
408 }
409 throw maxAttempts(JGitText.get().s3ActionWriting, key);
410 }
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440 public OutputStream beginPut(final String bucket, final String key,
441 final ProgressMonitor monitor, final String monitorTask)
442 throws IOException {
443 final MessageDigest md5 = newMD5();
444 final TemporaryBuffer buffer = new TemporaryBuffer.LocalFile(tmpDir) {
445 @Override
446 public void close() throws IOException {
447 super.close();
448 try {
449 putImpl(bucket, key, md5.digest(), this, monitor,
450 monitorTask);
451 } finally {
452 destroy();
453 }
454 }
455 };
456 return encryption.encrypt(new DigestOutputStream(buffer, md5));
457 }
458
459 void putImpl(final String bucket, final String key,
460 final byte[] csum, final TemporaryBuffer buf,
461 ProgressMonitor monitor, String monitorTask) throws IOException {
462 if (monitor == null)
463 monitor = NullProgressMonitor.INSTANCE;
464 if (monitorTask == null)
465 monitorTask = MessageFormat.format(JGitText.get().progressMonUploading, key);
466
467 final String md5str = Base64.encodeBytes(csum);
468 final long len = buf.length();
469 for (int curAttempt = 0; curAttempt < maxAttempts; curAttempt++) {
470 final HttpURLConnection c = open("PUT", bucket, key);
471 c.setFixedLengthStreamingMode(len);
472 c.setRequestProperty("Content-MD5", md5str);
473 c.setRequestProperty(X_AMZ_ACL, acl);
474 encryption.request(c, X_AMZ_META);
475 authorize(c);
476 c.setDoOutput(true);
477 monitor.beginTask(monitorTask, (int) (len / 1024));
478 try (OutputStream os = c.getOutputStream()) {
479 buf.writeTo(os, monitor);
480 } finally {
481 monitor.endTask();
482 }
483
484 switch (HttpSupport.response(c)) {
485 case HttpURLConnection.HTTP_OK:
486 return;
487 case HttpURLConnection.HTTP_INTERNAL_ERROR:
488 continue;
489 default:
490 throw error(JGitText.get().s3ActionWriting, key, c);
491 }
492 }
493 throw maxAttempts(JGitText.get().s3ActionWriting, key);
494 }
495
496 IOException error(final String action, final String key,
497 final HttpURLConnection c) throws IOException {
498 final IOException err = new IOException(MessageFormat.format(
499 JGitText.get().amazonS3ActionFailed, action, key,
500 Integer.valueOf(HttpSupport.response(c)),
501 c.getResponseMessage()));
502 if (c.getErrorStream() == null) {
503 return err;
504 }
505
506 try (InputStream errorStream = c.getErrorStream()) {
507 final ByteArrayOutputStream b = new ByteArrayOutputStream();
508 byte[] buf = new byte[2048];
509 for (;;) {
510 final int n = errorStream.read(buf);
511 if (n < 0) {
512 break;
513 }
514 if (n > 0) {
515 b.write(buf, 0, n);
516 }
517 }
518 buf = b.toByteArray();
519 if (buf.length > 0) {
520 err.initCause(new IOException("\n" + new String(buf, UTF_8)));
521 }
522 }
523 return err;
524 }
525
526 IOException maxAttempts(String action, String key) {
527 return new IOException(MessageFormat.format(
528 JGitText.get().amazonS3ActionFailedGivingUp, action, key,
529 Integer.valueOf(maxAttempts)));
530 }
531
532 private HttpURLConnection open(final String method, final String bucket,
533 final String key) throws IOException {
534 final Map<String, String> noArgs = Collections.emptyMap();
535 return open(method, bucket, key, noArgs);
536 }
537
538 HttpURLConnection open(final String method, final String bucket,
539 final String key, final Map<String, String> args)
540 throws IOException {
541 final StringBuilder urlstr = new StringBuilder();
542 urlstr.append("http://"); //$NON-NLS-1$
543 urlstr.append(bucket);
544 urlstr.append('.');
545 urlstr.append(domain);
546 urlstr.append('/');
547 if (key.length() > 0)
548 HttpSupport.encode(urlstr, key);
549 if (!args.isEmpty()) {
550 final Iterator<Map.Entry<String, String>> i;
551
552 urlstr.append('?');
553 i = args.entrySet().iterator();
554 while (i.hasNext()) {
555 final Map.Entry<String, String> e = i.next();
556 urlstr.append(e.getKey());
557 urlstr.append('=');
558 HttpSupport.encode(urlstr, e.getValue());
559 if (i.hasNext())
560 urlstr.append('&');
561 }
562 }
563
564 final URL url = new URL(urlstr.toString());
565 final Proxy proxy = HttpSupport.proxyFor(proxySelector, url);
566 final HttpURLConnection c;
567
568 c = (HttpURLConnection) url.openConnection(proxy);
569 c.setRequestMethod(method);
570 c.setRequestProperty("User-Agent", "jgit/1.0");
571 c.setRequestProperty("Date", httpNow());
572 return c;
573 }
574
575 void authorize(HttpURLConnection c) throws IOException {
576 final Map<String, List<String>> reqHdr = c.getRequestProperties();
577 final SortedMap<String, String> sigHdr = new TreeMap<>();
578 for (Map.Entry<String, List<String>> entry : reqHdr.entrySet()) {
579 final String hdr = entry.getKey();
580 if (isSignedHeader(hdr))
581 sigHdr.put(StringUtils.toLowerCase(hdr), toCleanString(entry.getValue()));
582 }
583
584 final StringBuilder s = new StringBuilder();
585 s.append(c.getRequestMethod());
586 s.append('\n');
587
588 s.append(remove(sigHdr, "content-md5"));
589 s.append('\n');
590
591 s.append(remove(sigHdr, "content-type"));
592 s.append('\n');
593
594 s.append(remove(sigHdr, "date"));
595 s.append('\n');
596
597 for (Map.Entry<String, String> e : sigHdr.entrySet()) {
598 s.append(e.getKey());
599 s.append(':');
600 s.append(e.getValue());
601 s.append('\n');
602 }
603
604 final String host = c.getURL().getHost();
605 s.append('/');
606 s.append(host.substring(0, host.length() - domain.length() - 1));
607 s.append(c.getURL().getPath());
608
609 final String sec;
610 try {
611 final Mac m = Mac.getInstance(HMAC);
612 m.init(privateKey);
613 sec = Base64.encodeBytes(m.doFinal(s.toString().getBytes(UTF_8)));
614 } catch (NoSuchAlgorithmException e) {
615 throw new IOException(MessageFormat.format(JGitText.get().noHMACsupport, HMAC, e.getMessage()));
616 } catch (InvalidKeyException e) {
617 throw new IOException(MessageFormat.format(JGitText.get().invalidKey, e.getMessage()));
618 }
619 c.setRequestProperty("Authorization", "AWS " + publicKey + ":" + sec);
620 }
621
622 static Properties properties(File authFile)
623 throws FileNotFoundException, IOException {
624 final Properties p = new Properties();
625 try (FileInputStream in = new FileInputStream(authFile)) {
626 p.load(in);
627 }
628 return p;
629 }
630
631
632
633
634 private static final class KeyInfo {
635 private final String name;
636 private final long lastModifiedSecs;
637 public KeyInfo(String aname, long lsecs) {
638 name = aname;
639 lastModifiedSecs = lsecs;
640 }
641 public String getName() {
642 return name;
643 }
644 public long getLastModifiedSecs() {
645 return lastModifiedSecs;
646 }
647 }
648
649 private final class ListParser extends DefaultHandler {
650 final List<KeyInfo> entries = new ArrayList<>();
651
652 private final String bucket;
653
654 private final String prefix;
655
656 boolean truncated;
657
658 private StringBuilder data;
659 private String keyName;
660 private Instant keyLastModified;
661
662 ListParser(String bn, String p) {
663 bucket = bn;
664 prefix = p;
665 }
666
667 void list() throws IOException {
668 final Map<String, String> args = new TreeMap<>();
669 if (prefix.length() > 0)
670 args.put("prefix", prefix);
671 if (!entries.isEmpty())
672 args.put("marker", prefix + entries.get(entries.size() - 1).getName());
673
674 for (int curAttempt = 0; curAttempt < maxAttempts; curAttempt++) {
675 final HttpURLConnection c = open("GET", bucket, "", args);
676 authorize(c);
677 switch (HttpSupport.response(c)) {
678 case HttpURLConnection.HTTP_OK:
679 truncated = false;
680 data = null;
681 keyName = null;
682 keyLastModified = null;
683
684 final XMLReader xr;
685 try {
686 xr = XMLReaderFactory.createXMLReader();
687 } catch (SAXException e) {
688 throw new IOException(
689 JGitText.get().noXMLParserAvailable, e);
690 }
691 xr.setContentHandler(this);
692 try (InputStream in = c.getInputStream()) {
693 xr.parse(new InputSource(in));
694 } catch (SAXException parsingError) {
695 throw new IOException(
696 MessageFormat.format(
697 JGitText.get().errorListing, prefix),
698 parsingError);
699 }
700 return;
701
702 case HttpURLConnection.HTTP_INTERNAL_ERROR:
703 continue;
704
705 default:
706 throw AmazonS3.this.error("Listing", prefix, c);
707 }
708 }
709 throw maxAttempts("Listing", prefix);
710 }
711
712 @Override
713 public void startElement(final String uri, final String name,
714 final String qName, final Attributes attributes)
715 throws SAXException {
716 if ("Key".equals(name) || "IsTruncated".equals(name) || "LastModified".equals(name)) {
717 data = new StringBuilder();
718 }
719 if ("Contents".equals(name)) {
720 keyName = null;
721 keyLastModified = null;
722 }
723 }
724
725 @Override
726 public void ignorableWhitespace(final char[] ch, final int s,
727 final int n) throws SAXException {
728 if (data != null)
729 data.append(ch, s, n);
730 }
731
732 @Override
733 public void characters(char[] ch, int s, int n)
734 throws SAXException {
735 if (data != null)
736 data.append(ch, s, n);
737 }
738
739 @Override
740 public void endElement(final String uri, final String name,
741 final String qName) throws SAXException {
742 if ("Key".equals(name)) {
743 keyName = data.toString().substring(prefix.length());
744 } else if ("IsTruncated".equals(name)) {
745 truncated = StringUtils.equalsIgnoreCase("true", data.toString());
746 } else if ("LastModified".equals(name)) {
747 keyLastModified = Instant.parse(data.toString());
748 } else if ("Contents".equals(name)) {
749 entries.add(new KeyInfo(keyName, keyLastModified.getEpochSecond()));
750 }
751
752 data = null;
753 }
754 }
755 }