Skip to content

Commit b2665b9

Browse files
author
Philipp Heckel
committed
Make remote store work
1 parent 1e8dfda commit b2665b9

16 files changed

+789
-241
lines changed

‎cmd/fsdup/main.go

+10-4
Original file line numberDiff line numberDiff line change
@@ -300,7 +300,7 @@ func printCommand(args []string) {
300300
manifestId = flags.Arg(1)
301301
}
302302

303-
manifest, err := metaStore.GetManifest(manifestId)
303+
manifest, err := metaStore.ReadManifest(manifestId)
304304
if err != nil {
305305
exit(2, "Cannot read manifest: " + string(err.Error()))
306306
}
@@ -400,7 +400,7 @@ func uploadCommand(args []string) {
400400
}
401401

402402
func createChunkStore(spec string) (fsdup.ChunkStore, error) {
403-
if regexp.MustCompile(`^(ceph|swift|gcloud):`).MatchString(spec) {
403+
if regexp.MustCompile(`^(ceph|swift|gcloud|remote):`).MatchString(spec) {
404404
uri, err := url.ParseRequestURI(spec)
405405
if err != nil {
406406
return nil, err
@@ -411,7 +411,9 @@ func createChunkStore(spec string) (fsdup.ChunkStore, error) {
411411
} else if uri.Scheme == "swift" {
412412
return createSwiftChunkStore(uri)
413413
} else if uri.Scheme == "gcloud" {
414-
return createGcloudStore(uri)
414+
return createGcloudChunkStore(uri)
415+
} else if uri.Scheme == "remote" {
416+
return createRemoteChunkStore(uri)
415417
}
416418

417419
return nil, errors.New("store type not supported")
@@ -465,7 +467,7 @@ func createSwiftChunkStore(uri *url.URL) (fsdup.ChunkStore, error) {
465467
return fsdup.NewSwiftStore(connection, container), nil
466468
}
467469

468-
func createGcloudStore(uri *url.URL) (fsdup.ChunkStore, error) {
470+
func createGcloudChunkStore(uri *url.URL) (fsdup.ChunkStore, error) {
469471
project := uri.Query().Get("project")
470472
if project == "" {
471473
return nil, errors.New("invalid syntax for gcloud store type, project parameter is required")
@@ -479,6 +481,10 @@ func createGcloudStore(uri *url.URL) (fsdup.ChunkStore, error) {
479481
return fsdup.NewGcloudStore(project, bucket), nil
480482
}
481483

484+
func createRemoteChunkStore(uri *url.URL) (fsdup.ChunkStore, error) {
485+
return fsdup.NewRemoteChunkStore(uri.Host), nil
486+
}
487+
482488
func createMetaStore(spec string) (fsdup.MetaStore, error) {
483489
if regexp.MustCompile(`^(remote):`).MatchString(spec) {
484490
uri, err := url.ParseRequestURI(spec)

‎export.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import (
66
)
77

88
func Export(manifestId string, store ChunkStore, metaStore MetaStore, outputFile string) error {
9-
manifest, err := metaStore.GetManifest(manifestId)
9+
manifest, err := metaStore.ReadManifest(manifestId)
1010
if err != nil {
1111
return err
1212
}

‎import.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import (
88
)
99

1010
func Import(manifestId string, store ChunkStore, metaStore MetaStore, inputFile string) error {
11-
manifest, err := metaStore.GetManifest(manifestId)
11+
manifest, err := metaStore.ReadManifest(manifestId)
1212
if err != nil {
1313
return err
1414
}

‎index.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ func Index(inputFile string, store ChunkStore, metaStore MetaStore, manifestId s
6565
manifest.PrintDisk()
6666
}
6767

68-
if err := metaStore.PutManifest(manifestId, manifest); err != nil {
68+
if err := metaStore.WriteManifest(manifestId, manifest); err != nil {
6969
return err
7070
}
7171

‎manifest.go

+24
Original file line numberDiff line numberDiff line change
@@ -364,3 +364,27 @@ func (m *manifest) PrintChunks() error {
364364
func (m *manifest) resetCaches() {
365365
m.offsets = nil
366366
}
367+
368+
func (k kind) toString() string {
369+
if k == kindFile {
370+
return "file"
371+
} else if k == kindSparse {
372+
return "sparse"
373+
} else if k == kindGap {
374+
return "gap"
375+
} else {
376+
return "unknown"
377+
}
378+
}
379+
380+
func kindFromString(s string) (kind, error) {
381+
if s == "file" {
382+
return kindFile, nil
383+
} else if s == "sparse" {
384+
return kindSparse, nil
385+
} else if s == "gap" {
386+
return kindGap, nil
387+
} else {
388+
return kindFile, errors.New("invalid kind string " + s)
389+
}
390+
}

‎map.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ type manifestImage struct {
2727
}
2828

2929
func Map(manifestId string, store ChunkStore, metaStore MetaStore, cache ChunkStore, targetFile string) error {
30-
manifest, err := metaStore.GetManifest(manifestId)
30+
manifest, err := metaStore.ReadManifest(manifestId)
3131
if err != nil {
3232
return err
3333
}

‎meta_store.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
package fsdup
22

33
type MetaStore interface {
4-
GetManifest(manifestId string) (*manifest, error)
5-
PutManifest(manifestId string, manifest *manifest) error
4+
ReadManifest(manifestId string) (*manifest, error)
5+
WriteManifest(manifestId string, manifest *manifest) error
66
}

‎meta_store_file.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,10 @@ func NewFileMetaStore() *fileMetaStore {
88
return &fileMetaStore{}
99
}
1010

11-
func (s *fileMetaStore) GetManifest(manifestId string) (*manifest, error) {
11+
func (s *fileMetaStore) ReadManifest(manifestId string) (*manifest, error) {
1212
return NewManifestFromFile(manifestId)
1313
}
1414

15-
func (s* fileMetaStore) PutManifest(manifestId string, manifest *manifest) error {
15+
func (s* fileMetaStore) WriteManifest(manifestId string, manifest *manifest) error {
1616
return manifest.WriteToFile(manifestId)
1717
}

‎meta_store_remote.go

+5-5
Original file line numberDiff line numberDiff line change
@@ -20,12 +20,12 @@ func NewRemoteMetaStore(serverAddr string) *remoteMetaStore {
2020
}
2121
}
2222

23-
func (s *remoteMetaStore) GetManifest(manifestId string) (*manifest, error) {
23+
func (s *remoteMetaStore) ReadManifest(manifestId string) (*manifest, error) {
2424
if err := s.ensureConnected(); err != nil {
2525
return nil, err
2626
}
2727

28-
response, err := s.client.GetManifest(context.Background(), &pb.GetManifestRequest{Id: manifestId})
28+
response, err := s.client.ReadManifest(context.Background(), &pb.ReadManifestRequest{Id: manifestId})
2929
if err != nil {
3030
return nil, err
3131
}
@@ -38,12 +38,12 @@ func (s *remoteMetaStore) GetManifest(manifestId string) (*manifest, error) {
3838
return manifest, nil
3939
}
4040

41-
func (s *remoteMetaStore) PutManifest(manifestId string, manifest *manifest) error {
41+
func (s *remoteMetaStore) WriteManifest(manifestId string, manifest *manifest) error {
4242
if err := s.ensureConnected(); err != nil {
4343
return err
4444
}
4545

46-
_, err := s.client.PutManifest(context.Background(), &pb.PutManifestRequest{Id: manifestId, Manifest: manifest.Proto()})
46+
_, err := s.client.WriteManifest(context.Background(), &pb.WriteManifestRequest{Id: manifestId, Manifest: manifest.Proto()})
4747
if err != nil {
4848
return err
4949
}
@@ -60,7 +60,7 @@ func (s *remoteMetaStore) ensureConnected() error {
6060
}
6161

6262
conn, err := grpc.Dial(s.serverAddr, grpc.WithBlock(), grpc.WithInsecure(),
63-
grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(128 * 1024 * 1024))) // FIXME
63+
grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(128 * 1024 * 1024), grpc.MaxCallRecvMsgSize(128 * 1024 * 1024))) // FIXME
6464
if err != nil {
6565
return err
6666
}

0 commit comments

Comments
 (0)