Skip to content

Commit

Permalink
feat(backup): support full backup
Browse files Browse the repository at this point in the history 
ref: longhorn/longhorn 7070

Signed-off-by: Jack Lin <[email protected]>
  • Loading branch information
@ChanYiLin
ChanYiLin committed Apr 2, 2024
1 parent 182b37c commit 8a6504e
Show file tree
Hide file tree
Showing 19 changed files with 500 additions and 473 deletions.
4 changes: 3 additions & 1 deletion app/cmd/backup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,8 @@ func createBackup(c *cli.Context) error {
}
}

parameters := map[string]string{}

credential, err := butil.GetBackupCredential(dest)
if err != nil {
return err
Expand All @@ -282,7 +284,7 @@ func createBackup(c *cli.Context) error {
}

backup, err := task.CreateBackup(backupName, snapshot, dest, biName, biChecksum,
compressionMethod, concurrentLimit, storageClassName, labels, credential)
compressionMethod, concurrentLimit, storageClassName, labels, credential, parameters)
if err != nil {
return err
}
Expand Down
14 changes: 7 additions & 7 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,18 @@ module github.com/longhorn/longhorn-engine

go 1.21

replace github.com/longhorn/backupstore v0.0.0-20240219094812-3a87ee02df77 => github.com/ChanYiLin/backupstore v0.0.0-20240402075357-cefced175927

replace github.com/longhorn/go-common-libs v0.0.0-20240319112414-b75404dc7fbc => github.com/ChanYiLin/go-common-libs v0.0.0-20240329072435-638753f76305

require (
github.com/docker/go-units v0.5.0
github.com/gofrs/flock v0.8.1
github.com/google/uuid v1.6.0
github.com/gorilla/handlers v1.5.2
github.com/gorilla/mux v1.8.1
github.com/longhorn/backupstore v0.0.0-20240219094812-3a87ee02df77
github.com/longhorn/go-common-libs v0.0.0-20240307063052-6e77996eda29
github.com/longhorn/go-common-libs v0.0.0-20240319112414-b75404dc7fbc
github.com/longhorn/go-iscsi-helper v0.0.0-20240308033847-bc3aab599425
github.com/longhorn/sparse-tools v0.0.0-20240228120902-ce8c4c2e71ca
github.com/moby/moby v24.0.9+incompatible
Expand All @@ -34,22 +39,19 @@ require (
github.com/c9s/goprocinfo v0.0.0-20210130143923-c95fcf8c64a8 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/fatih/color v1.7.0 // indirect
github.com/felixge/httpsnoop v1.0.3 // indirect
github.com/gammazero/deque v0.2.1 // indirect
github.com/gammazero/workerpool v1.1.3 // indirect
github.com/go-logr/logr v1.3.0 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gorilla/context v1.1.1 // indirect
github.com/gorilla/websocket v1.4.2 // indirect
github.com/jmespath/go-jmespath v0.3.0 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/mattn/go-colorable v0.1.4 // indirect
github.com/mattn/go-isatty v0.0.8 // indirect
github.com/mattn/go-runewidth v0.0.5-0.20181218000649-703b5e6b11ae // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mitchellh/go-ps v1.0.0 // indirect
github.com/moby/sys/mountinfo v0.6.2 // indirect
Expand All @@ -62,21 +64,19 @@ require (
github.com/rogpeppe/go-internal v1.10.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/shirou/gopsutil/v3 v3.24.2 // indirect
github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
github.com/slok/goresilience v0.2.0 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/text v0.14.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect
gopkg.in/VividCortex/ewma.v1 v1.1.1 // indirect
gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect
gopkg.in/fatih/color.v1 v1.7.0 // indirect
gopkg.in/mattn/go-colorable.v0 v0.1.0 // indirect
gopkg.in/mattn/go-isatty.v0 v0.0.4 // indirect
gopkg.in/mattn/go-runewidth.v0 v0.0.4 // indirect
gotest.tools/v3 v3.4.0 // indirect
k8s.io/apimachinery v0.27.1 // indirect
k8s.io/klog/v2 v2.110.1 // indirect
k8s.io/mount-utils v0.29.2 // indirect
k8s.io/mount-utils v0.29.3 // indirect
k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
)
82 changes: 7 additions & 75 deletions go.sum
View file

Large diffs are not rendered by default.

84 changes: 44 additions & 40 deletions integration/rpc/sync/github/com/longhorn/longhorn_engine/proto/ptypes/syncagent_pb2.py
View file

Large diffs are not rendered by default.

2 changes: 2 additions & 0 deletions pkg/backup/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ type CreateBackupParameters struct {
ConcurrentLimit int32
StorageClassName string
Labels []string
Parameters map[string]string
}

type ErrorResponse struct {
Expand Down Expand Up @@ -128,6 +129,7 @@ func DoBackupInit(params *CreateBackupParameters) (*replica.BackupStatus, *backu
DestURL: params.DestURL,
DeltaOps: backup,
Labels: labelMap,
Parameters: params.Parameters,
}
return backup, config, nil
}
Expand Down
3 changes: 2 additions & 1 deletion pkg/replica/client/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -523,7 +523,7 @@ func (c *ReplicaClient) SyncFiles(fromAddress string, list []types.SyncFileInfo,
}

func (c *ReplicaClient) CreateBackup(backupName, snapshot, dest, volume, backingImageName, backingImageChecksum,
compressionMethod string, concurrentLimit int, storageClassName string, labels []string, credential map[string]string) (*ptypes.BackupCreateResponse, error) {
compressionMethod string, concurrentLimit int, storageClassName string, labels []string, credential, parameters map[string]string) (*ptypes.BackupCreateResponse, error) {
syncAgentServiceClient, err := c.getSyncServiceClient()
if err != nil {
return nil, err
Expand All @@ -543,6 +543,7 @@ func (c *ReplicaClient) CreateBackup(backupName, snapshot, dest, volume, backing
Labels: labels,
Credential: credential,
BackupName: backupName,
Parameters: parameters,
})
if err != nil {
return nil, errors.Wrapf(err, "failed to create backup to %v for volume %v", dest, volume)
Expand Down
8 changes: 4 additions & 4 deletions pkg/sync/backup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ type RestoreStatus struct {
}

func (t *Task) CreateBackup(backupName, snapshot, dest, backingImageName, backingImageChecksum string,
compressionMethod string, concurrentLimit int, storageClassName string, labels []string, credential map[string]string) (*BackupCreateInfo, error) {
compressionMethod string, concurrentLimit int, storageClassName string, labels []string, credential, parameters map[string]string) (*BackupCreateInfo, error) {
if snapshot == types.VolumeHeadName {
return nil, fmt.Errorf("cannot backup the head disk in the chain")
}
Expand All @@ -60,7 +60,7 @@ func (t *Task) CreateBackup(backupName, snapshot, dest, backingImageName, backin
}

return t.createBackup(replica, backupName, snapshot, dest, volume.Name, backingImageName, backingImageChecksum,
compressionMethod, concurrentLimit, storageClassName, labels, credential)
compressionMethod, concurrentLimit, storageClassName, labels, credential, parameters)
}

func (t *Task) findRWReplica() (*types.ControllerReplicaInfo, error) {
Expand All @@ -80,7 +80,7 @@ func (t *Task) findRWReplica() (*types.ControllerReplicaInfo, error) {

func (t *Task) createBackup(replicaInController *types.ControllerReplicaInfo, backupName, snapshot, dest, volumeName,
backingImageName, backingImageChecksum, compression string, concurrentLimit int, storageClassName string,
labels []string, credential map[string]string) (*BackupCreateInfo, error) {
labels []string, credential, parameters map[string]string) (*BackupCreateInfo, error) {
if replicaInController.Mode != types.RW {
return nil, fmt.Errorf("can only create backup from replica in mode RW, got %s", replicaInController.Mode)
}
Expand All @@ -105,7 +105,7 @@ func (t *Task) createBackup(replicaInController *types.ControllerReplicaInfo, ba
logrus.Infof("Backing up %s on %s, to %s", snapshot, replicaInController.Address, dest)

reply, err := repClient.CreateBackup(backupName, snapshot, dest, volumeName, backingImageName, backingImageChecksum,
compression, concurrentLimit, storageClassName, labels, credential)
compression, concurrentLimit, storageClassName, labels, credential, parameters)
if err != nil {
return nil, err
}
Expand Down
1 change: 1 addition & 0 deletions pkg/sync/rpc/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -710,6 +710,7 @@ func (s *SyncAgentServer) BackupCreate(ctx context.Context, req *ptypes.BackupCr
ConcurrentLimit: req.ConcurrentLimit,
StorageClassName: req.StorageClassName,
Labels: req.Labels,
Parameters: req.Parameters,
})
if err != nil {
logrus.WithError(err).Errorf("Failed to initialize backup %v", req.BackupName)
Expand Down
Loading

0 comments on commit 8a6504e

Please sign in to comment.