1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44 package org.eclipse.jgit.transport;
45
46 import java.io.BufferedReader;
47 import java.io.File;
48 import java.io.FileNotFoundException;
49 import java.io.IOException;
50 import java.io.InputStream;
51 import java.io.OutputStream;
52 import java.net.URLConnection;
53 import java.text.MessageFormat;
54 import java.util.ArrayList;
55 import java.util.Collection;
56 import java.util.Collections;
57 import java.util.EnumSet;
58 import java.util.HashSet;
59 import java.util.Map;
60 import java.util.Properties;
61 import java.util.Set;
62 import java.util.TreeMap;
63
64 import org.eclipse.jgit.errors.NotSupportedException;
65 import org.eclipse.jgit.errors.TransportException;
66 import org.eclipse.jgit.internal.JGitText;
67 import org.eclipse.jgit.lib.Constants;
68 import org.eclipse.jgit.lib.ObjectId;
69 import org.eclipse.jgit.lib.ObjectIdRef;
70 import org.eclipse.jgit.lib.ProgressMonitor;
71 import org.eclipse.jgit.lib.Ref;
72 import org.eclipse.jgit.lib.Ref.Storage;
73 import org.eclipse.jgit.lib.Repository;
74 import org.eclipse.jgit.lib.SymbolicRef;
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100 public class TransportAmazonS3 extends HttpTransport implements WalkTransport {
101 static final String S3_SCHEME = "amazon-s3";
102
103 static final TransportProtocol PROTO_S3 = new TransportProtocol() {
104 public String getName() {
105 return "Amazon S3";
106 }
107
108 public Set<String> getSchemes() {
109 return Collections.singleton(S3_SCHEME);
110 }
111
112 public Set<URIishField> getRequiredFields() {
113 return Collections.unmodifiableSet(EnumSet.of(URIishField.USER,
114 URIishField.HOST, URIishField.PATH));
115 }
116
117 public Set<URIishField> getOptionalFields() {
118 return Collections.unmodifiableSet(EnumSet.of(URIishField.PASS));
119 }
120
121 public Transport open(URIish uri, Repository local, String remoteName)
122 throws NotSupportedException {
123 return new TransportAmazonS3(local, uri);
124 }
125 };
126
127
128 final AmazonS3 s3;
129
130
131 final String bucket;
132
133
134
135
136
137
138
139
140
141
142
143
144 private final String keyPrefix;
145
146 TransportAmazonS3(final Repository local, final URIish uri)
147 throws NotSupportedException {
148 super(local, uri);
149
150 Properties props = loadProperties();
151 File directory = local.getDirectory();
152 if (!props.containsKey("tmpdir") && directory != null)
153 props.put("tmpdir", directory.getPath());
154
155 s3 = new AmazonS3(props);
156 bucket = uri.getHost();
157
158 String p = uri.getPath();
159 if (p.startsWith("/"))
160 p = p.substring(1);
161 if (p.endsWith("/"))
162 p = p.substring(0, p.length() - 1);
163 keyPrefix = p;
164 }
165
166 private Properties loadProperties() throws NotSupportedException {
167 if (local.getDirectory() != null) {
168 File propsFile = new File(local.getDirectory(), uri.getUser());
169 if (propsFile.isFile())
170 return loadPropertiesFile(propsFile);
171 }
172
173 File propsFile = new File(local.getFS().userHome(), uri.getUser());
174 if (propsFile.isFile())
175 return loadPropertiesFile(propsFile);
176
177 Properties props = new Properties();
178 String user = uri.getUser();
179 String pass = uri.getPass();
180 if (user != null && pass != null) {
181 props.setProperty("accesskey", user);
182 props.setProperty("secretkey", pass);
183 } else
184 throw new NotSupportedException(MessageFormat.format(
185 JGitText.get().cannotReadFile, propsFile));
186 return props;
187 }
188
189 private static Properties loadPropertiesFile(File propsFile)
190 throws NotSupportedException {
191 try {
192 return AmazonS3.properties(propsFile);
193 } catch (IOException e) {
194 throw new NotSupportedException(MessageFormat.format(
195 JGitText.get().cannotReadFile, propsFile), e);
196 }
197 }
198
199 @Override
200 public FetchConnection openFetch() throws TransportException {
201 final DatabaseS3 c = new DatabaseS3(bucket, keyPrefix + "/objects");
202 final WalkFetchConnection r = new WalkFetchConnection(this, c);
203 r.available(c.readAdvertisedRefs());
204 return r;
205 }
206
207 @Override
208 public PushConnection openPush() throws TransportException {
209 final DatabaseS3 c = new DatabaseS3(bucket, keyPrefix + "/objects");
210 final WalkPushConnection r = new WalkPushConnection(this, c);
211 r.available(c.readAdvertisedRefs());
212 return r;
213 }
214
215 @Override
216 public void close() {
217
218 }
219
220 class DatabaseS3 extends WalkRemoteObjectDatabase {
221 private final String bucketName;
222
223 private final String objectsKey;
224
225 DatabaseS3(final String b, final String o) {
226 bucketName = b;
227 objectsKey = o;
228 }
229
230 private String resolveKey(String subpath) {
231 if (subpath.endsWith("/"))
232 subpath = subpath.substring(0, subpath.length() - 1);
233 String k = objectsKey;
234 while (subpath.startsWith(ROOT_DIR)) {
235 k = k.substring(0, k.lastIndexOf('/'));
236 subpath = subpath.substring(3);
237 }
238 return k + "/" + subpath;
239 }
240
241 @Override
242 URIish getURI() {
243 URIish u = new URIish();
244 u = u.setScheme(S3_SCHEME);
245 u = u.setHost(bucketName);
246 u = u.setPath("/" + objectsKey);
247 return u;
248 }
249
250 @Override
251 Collection<WalkRemoteObjectDatabase> getAlternates() throws IOException {
252 try {
253 return readAlternates(INFO_ALTERNATES);
254 } catch (FileNotFoundException err) {
255
256 }
257 return null;
258 }
259
260 @Override
261 WalkRemoteObjectDatabase openAlternate(final String location)
262 throws IOException {
263 return new DatabaseS3(bucketName, resolveKey(location));
264 }
265
266 @Override
267 Collection<String> getPackNames() throws IOException {
268 final HashSet<String> have = new HashSet<String>();
269 have.addAll(s3.list(bucket, resolveKey("pack")));
270
271 final Collection<String> packs = new ArrayList<String>();
272 for (final String n : have) {
273 if (!n.startsWith("pack-") || !n.endsWith(".pack"))
274 continue;
275
276 final String in = n.substring(0, n.length() - 5) + ".idx";
277 if (have.contains(in))
278 packs.add(n);
279 }
280 return packs;
281 }
282
283 @Override
284 FileStream open(final String path) throws IOException {
285 final URLConnection c = s3.get(bucket, resolveKey(path));
286 final InputStream raw = c.getInputStream();
287 final InputStream in = s3.decrypt(c);
288 final int len = c.getContentLength();
289 return new FileStream(in, raw == in ? len : -1);
290 }
291
292 @Override
293 void deleteFile(final String path) throws IOException {
294 s3.delete(bucket, resolveKey(path));
295 }
296
297 @Override
298 OutputStream writeFile(final String path,
299 final ProgressMonitor monitor, final String monitorTask)
300 throws IOException {
301 return s3.beginPut(bucket, resolveKey(path), monitor, monitorTask);
302 }
303
304 @Override
305 void writeFile(final String path, final byte[] data) throws IOException {
306 s3.put(bucket, resolveKey(path), data);
307 }
308
309 Map<String, Ref> readAdvertisedRefs() throws TransportException {
310 final TreeMap<String, Ref> avail = new TreeMap<String, Ref>();
311 readPackedRefs(avail);
312 readLooseRefs(avail);
313 readRef(avail, Constants.HEAD);
314 return avail;
315 }
316
317 private void readLooseRefs(final TreeMap<String, Ref> avail)
318 throws TransportException {
319 try {
320 for (final String n : s3.list(bucket, resolveKey(ROOT_DIR
321 + "refs")))
322 readRef(avail, "refs/" + n);
323 } catch (IOException e) {
324 throw new TransportException(getURI(), JGitText.get().cannotListRefs, e);
325 }
326 }
327
328 private Ref readRef(final TreeMap<String, Ref> avail, final String rn)
329 throws TransportException {
330 final String s;
331 String ref = ROOT_DIR + rn;
332 try {
333 final BufferedReader br = openReader(ref);
334 try {
335 s = br.readLine();
336 } finally {
337 br.close();
338 }
339 } catch (FileNotFoundException noRef) {
340 return null;
341 } catch (IOException err) {
342 throw new TransportException(getURI(), MessageFormat.format(
343 JGitText.get().transportExceptionReadRef, ref), err);
344 }
345
346 if (s == null)
347 throw new TransportException(getURI(), MessageFormat.format(JGitText.get().transportExceptionEmptyRef, rn));
348
349 if (s.startsWith("ref: ")) {
350 final String target = s.substring("ref: ".length());
351 Ref r = avail.get(target);
352 if (r == null)
353 r = readRef(avail, target);
354 if (r == null)
355 r = new ObjectIdRef.Unpeeled(Ref.Storage.NEW, target, null);
356 r = new SymbolicRef(rn, r);
357 avail.put(r.getName(), r);
358 return r;
359 }
360
361 if (ObjectId.isId(s)) {
362 final Ref r = new ObjectIdRef.Unpeeled(loose(avail.get(rn)),
363 rn, ObjectId.fromString(s));
364 avail.put(r.getName(), r);
365 return r;
366 }
367
368 throw new TransportException(getURI(), MessageFormat.format(JGitText.get().transportExceptionBadRef, rn, s));
369 }
370
371 private Storage loose(final Ref r) {
372 if (r != null && r.getStorage() == Storage.PACKED)
373 return Storage.LOOSE_PACKED;
374 return Storage.LOOSE;
375 }
376
377 @Override
378 void close() {
379
380 }
381 }
382 }