1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44 package org.eclipse.jgit.transport;
45
46 import java.io.BufferedReader;
47 import java.io.File;
48 import java.io.FileNotFoundException;
49 import java.io.IOException;
50 import java.io.InputStream;
51 import java.io.OutputStream;
52 import java.net.URLConnection;
53 import java.text.MessageFormat;
54 import java.util.ArrayList;
55 import java.util.Collection;
56 import java.util.Collections;
57 import java.util.EnumSet;
58 import java.util.HashSet;
59 import java.util.Map;
60 import java.util.Properties;
61 import java.util.Set;
62 import java.util.TreeMap;
63
64 import org.eclipse.jgit.errors.NotSupportedException;
65 import org.eclipse.jgit.errors.TransportException;
66 import org.eclipse.jgit.internal.JGitText;
67 import org.eclipse.jgit.lib.Constants;
68 import org.eclipse.jgit.lib.ObjectId;
69 import org.eclipse.jgit.lib.ObjectIdRef;
70 import org.eclipse.jgit.lib.ProgressMonitor;
71 import org.eclipse.jgit.lib.Ref;
72 import org.eclipse.jgit.lib.Ref.Storage;
73 import org.eclipse.jgit.lib.Repository;
74 import org.eclipse.jgit.lib.SymbolicRef;
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101 public class TransportAmazonS3 extends HttpTransport implements WalkTransport {
102 static final String S3_SCHEME = "amazon-s3";
103
104 static final TransportProtocol PROTO_S3 = new TransportProtocol() {
105 @Override
106 public String getName() {
107 return "Amazon S3";
108 }
109
110 @Override
111 public Set<String> getSchemes() {
112 return Collections.singleton(S3_SCHEME);
113 }
114
115 @Override
116 public Set<URIishField> getRequiredFields() {
117 return Collections.unmodifiableSet(EnumSet.of(URIishField.USER,
118 URIishField.HOST, URIishField.PATH));
119 }
120
121 @Override
122 public Set<URIishField> getOptionalFields() {
123 return Collections.unmodifiableSet(EnumSet.of(URIishField.PASS));
124 }
125
126 @Override
127 public Transport open(URIish uri, Repository local, String remoteName)
128 throws NotSupportedException {
129 return new TransportAmazonS3(local, uri);
130 }
131 };
132
133
134 final AmazonS3 s3;
135
136
137 final String bucket;
138
139
140
141
142
143
144
145
146
147
148
149
150 private final String keyPrefix;
151
152 TransportAmazonS3(final Repository local, final URIish uri)
153 throws NotSupportedException {
154 super(local, uri);
155
156 Properties props = loadProperties();
157 File directory = local.getDirectory();
158 if (!props.containsKey("tmpdir") && directory != null)
159 props.put("tmpdir", directory.getPath());
160
161 s3 = new AmazonS3(props);
162 bucket = uri.getHost();
163
164 String p = uri.getPath();
165 if (p.startsWith("/"))
166 p = p.substring(1);
167 if (p.endsWith("/"))
168 p = p.substring(0, p.length() - 1);
169 keyPrefix = p;
170 }
171
172 private Properties loadProperties() throws NotSupportedException {
173 if (local.getDirectory() != null) {
174 File propsFile = new File(local.getDirectory(), uri.getUser());
175 if (propsFile.isFile())
176 return loadPropertiesFile(propsFile);
177 }
178
179 File propsFile = new File(local.getFS().userHome(), uri.getUser());
180 if (propsFile.isFile())
181 return loadPropertiesFile(propsFile);
182
183 Properties props = new Properties();
184 String user = uri.getUser();
185 String pass = uri.getPass();
186 if (user != null && pass != null) {
187 props.setProperty("accesskey", user);
188 props.setProperty("secretkey", pass);
189 } else
190 throw new NotSupportedException(MessageFormat.format(
191 JGitText.get().cannotReadFile, propsFile));
192 return props;
193 }
194
195 private static Properties loadPropertiesFile(File propsFile)
196 throws NotSupportedException {
197 try {
198 return AmazonS3.properties(propsFile);
199 } catch (IOException e) {
200 throw new NotSupportedException(MessageFormat.format(
201 JGitText.get().cannotReadFile, propsFile), e);
202 }
203 }
204
205
206 @Override
207 public FetchConnection openFetch() throws TransportException {
208 final DatabaseS3 c = new DatabaseS3(bucket, keyPrefix + "/objects");
209 final WalkFetchConnection r = new WalkFetchConnection(this, c);
210 r.available(c.readAdvertisedRefs());
211 return r;
212 }
213
214
215 @Override
216 public PushConnection openPush() throws TransportException {
217 final DatabaseS3 c = new DatabaseS3(bucket, keyPrefix + "/objects");
218 final WalkPushConnection r = new WalkPushConnection(this, c);
219 r.available(c.readAdvertisedRefs());
220 return r;
221 }
222
223
224 @Override
225 public void close() {
226
227 }
228
229 class DatabaseS3 extends WalkRemoteObjectDatabase {
230 private final String bucketName;
231
232 private final String objectsKey;
233
234 DatabaseS3(final String b, final String o) {
235 bucketName = b;
236 objectsKey = o;
237 }
238
239 private String resolveKey(String subpath) {
240 if (subpath.endsWith("/"))
241 subpath = subpath.substring(0, subpath.length() - 1);
242 String k = objectsKey;
243 while (subpath.startsWith(ROOT_DIR)) {
244 k = k.substring(0, k.lastIndexOf('/'));
245 subpath = subpath.substring(3);
246 }
247 return k + "/" + subpath;
248 }
249
250 @Override
251 URIish getURI() {
252 URIish u = new URIish();
253 u = u.setScheme(S3_SCHEME);
254 u = u.setHost(bucketName);
255 u = u.setPath("/" + objectsKey);
256 return u;
257 }
258
259 @Override
260 Collection<WalkRemoteObjectDatabase> getAlternates() throws IOException {
261 try {
262 return readAlternates(INFO_ALTERNATES);
263 } catch (FileNotFoundException err) {
264
265 }
266 return null;
267 }
268
269 @Override
270 WalkRemoteObjectDatabase openAlternate(String location)
271 throws IOException {
272 return new DatabaseS3(bucketName, resolveKey(location));
273 }
274
275 @Override
276 Collection<String> getPackNames() throws IOException {
277 final HashSet<String> have = new HashSet<>();
278 have.addAll(s3.list(bucket, resolveKey("pack")));
279
280 final Collection<String> packs = new ArrayList<>();
281 for (String n : have) {
282 if (!n.startsWith("pack-") || !n.endsWith(".pack"))
283 continue;
284
285 final String in = n.substring(0, n.length() - 5) + ".idx";
286 if (have.contains(in))
287 packs.add(n);
288 }
289 return packs;
290 }
291
292 @Override
293 FileStream open(String path) throws IOException {
294 final URLConnection c = s3.get(bucket, resolveKey(path));
295 final InputStream raw = c.getInputStream();
296 final InputStream in = s3.decrypt(c);
297 final int len = c.getContentLength();
298 return new FileStream(in, raw == in ? len : -1);
299 }
300
301 @Override
302 void deleteFile(String path) throws IOException {
303 s3.delete(bucket, resolveKey(path));
304 }
305
306 @Override
307 OutputStream writeFile(final String path,
308 final ProgressMonitor monitor, final String monitorTask)
309 throws IOException {
310 return s3.beginPut(bucket, resolveKey(path), monitor, monitorTask);
311 }
312
313 @Override
314 void writeFile(String path, byte[] data) throws IOException {
315 s3.put(bucket, resolveKey(path), data);
316 }
317
318 Map<String, Ref> readAdvertisedRefs() throws TransportException {
319 final TreeMap<String, Ref> avail = new TreeMap<>();
320 readPackedRefs(avail);
321 readLooseRefs(avail);
322 readRef(avail, Constants.HEAD);
323 return avail;
324 }
325
326 private void readLooseRefs(TreeMap<String, Ref> avail)
327 throws TransportException {
328 try {
329 for (final String n : s3.list(bucket, resolveKey(ROOT_DIR
330 + "refs")))
331 readRef(avail, "refs/" + n);
332 } catch (IOException e) {
333 throw new TransportException(getURI(), JGitText.get().cannotListRefs, e);
334 }
335 }
336
337 private Ref readRef(TreeMap<String, Ref> avail, String rn)
338 throws TransportException {
339 final String s;
340 String ref = ROOT_DIR + rn;
341 try {
342 try (BufferedReader br = openReader(ref)) {
343 s = br.readLine();
344 }
345 } catch (FileNotFoundException noRef) {
346 return null;
347 } catch (IOException err) {
348 throw new TransportException(getURI(), MessageFormat.format(
349 JGitText.get().transportExceptionReadRef, ref), err);
350 }
351
352 if (s == null)
353 throw new TransportException(getURI(), MessageFormat.format(JGitText.get().transportExceptionEmptyRef, rn));
354
355 if (s.startsWith("ref: ")) {
356 final String target = s.substring("ref: ".length());
357 Ref r = avail.get(target);
358 if (r == null)
359 r = readRef(avail, target);
360 if (r == null)
361 r = new ObjectIdRef.Unpeeled(Ref.Storage.NEW, target, null);
362 r = new SymbolicRef(rn, r);
363 avail.put(r.getName(), r);
364 return r;
365 }
366
367 if (ObjectId.isId(s)) {
368 final Ref r = new ObjectIdRef.Unpeeled(loose(avail.get(rn)),
369 rn, ObjectId.fromString(s));
370 avail.put(r.getName(), r);
371 return r;
372 }
373
374 throw new TransportException(getURI(), MessageFormat.format(JGitText.get().transportExceptionBadRef, rn, s));
375 }
376
377 private Storage loose(Ref r) {
378 if (r != null && r.getStorage() == Storage.PACKED)
379 return Storage.LOOSE_PACKED;
380 return Storage.LOOSE;
381 }
382
383 @Override
384 void close() {
385
386 }
387 }
388 }