Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(backup): support full backup #1061

Closed
wants to merge 1 commit into from
Closed

feat(backup): support full backup #1061

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion app/cmd/backup.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,8 @@ func createBackup(c *cli.Context) error {
}
}

parameters := map[string]string{}

credential, err := butil.GetBackupCredential(dest)
if err != nil {
return err
Expand All @@ -282,7 +284,7 @@ func createBackup(c *cli.Context) error {
}

backup, err := task.CreateBackup(backupName, snapshot, dest, biName, biChecksum,
compressionMethod, concurrentLimit, storageClassName, labels, credential)
compressionMethod, concurrentLimit, storageClassName, labels, credential, parameters)
if err != nil {
return err
}
Expand Down
14 changes: 7 additions & 7 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,18 @@ module github.com/longhorn/longhorn-engine

go 1.21

replace github.com/longhorn/backupstore v0.0.0-20240219094812-3a87ee02df77 => github.com/ChanYiLin/backupstore v0.0.0-20240402075357-cefced175927

replace github.com/longhorn/go-common-libs v0.0.0-20240319112414-b75404dc7fbc => github.com/ChanYiLin/go-common-libs v0.0.0-20240329072435-638753f76305

require (
github.com/docker/go-units v0.5.0
github.com/gofrs/flock v0.8.1
github.com/google/uuid v1.6.0
github.com/gorilla/handlers v1.5.2
github.com/gorilla/mux v1.8.1
github.com/longhorn/backupstore v0.0.0-20240219094812-3a87ee02df77
github.com/longhorn/go-common-libs v0.0.0-20240307063052-6e77996eda29
github.com/longhorn/go-common-libs v0.0.0-20240319112414-b75404dc7fbc
github.com/longhorn/go-iscsi-helper v0.0.0-20240308033847-bc3aab599425
github.com/longhorn/sparse-tools v0.0.0-20240228120902-ce8c4c2e71ca
github.com/moby/moby v24.0.9+incompatible
Expand All @@ -34,22 +39,19 @@ require (
github.com/c9s/goprocinfo v0.0.0-20210130143923-c95fcf8c64a8 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/fatih/color v1.7.0 // indirect
github.com/felixge/httpsnoop v1.0.3 // indirect
github.com/gammazero/deque v0.2.1 // indirect
github.com/gammazero/workerpool v1.1.3 // indirect
github.com/go-logr/logr v1.3.0 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/gorilla/context v1.1.1 // indirect
github.com/gorilla/websocket v1.4.2 // indirect
github.com/jmespath/go-jmespath v0.3.0 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/mattn/go-colorable v0.1.4 // indirect
github.com/mattn/go-isatty v0.0.8 // indirect
github.com/mattn/go-runewidth v0.0.5-0.20181218000649-703b5e6b11ae // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mitchellh/go-ps v1.0.0 // indirect
github.com/moby/sys/mountinfo v0.6.2 // indirect
Expand All @@ -62,21 +64,19 @@ require (
github.com/rogpeppe/go-internal v1.10.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/shirou/gopsutil/v3 v3.24.2 // indirect
github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
github.com/slok/goresilience v0.2.0 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/text v0.14.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect
gopkg.in/VividCortex/ewma.v1 v1.1.1 // indirect
gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect
gopkg.in/fatih/color.v1 v1.7.0 // indirect
gopkg.in/mattn/go-colorable.v0 v0.1.0 // indirect
gopkg.in/mattn/go-isatty.v0 v0.0.4 // indirect
gopkg.in/mattn/go-runewidth.v0 v0.0.4 // indirect
gotest.tools/v3 v3.4.0 // indirect
k8s.io/apimachinery v0.27.1 // indirect
k8s.io/klog/v2 v2.110.1 // indirect
k8s.io/mount-utils v0.29.2 // indirect
k8s.io/mount-utils v0.29.3 // indirect
k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
)
82 changes: 7 additions & 75 deletions go.sum
View file
Open in desktop

Large diffs are not rendered by default.

84 changes: 44 additions & 40 deletions integration/rpc/sync/github/com/longhorn/longhorn_engine/proto/ptypes/syncagent_pb2.py
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 2 additions & 0 deletions pkg/backup/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ type CreateBackupParameters struct {
ConcurrentLimit int32
StorageClassName string
Labels []string
Parameters map[string]string
}

type ErrorResponse struct {
Expand Down Expand Up @@ -128,6 +129,7 @@ func DoBackupInit(params *CreateBackupParameters) (*replica.BackupStatus, *backu
DestURL: params.DestURL,
DeltaOps: backup,
Labels: labelMap,
Parameters: params.Parameters,
}
return backup, config, nil
}
Expand Down
3 changes: 2 additions & 1 deletion pkg/replica/client/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -523,7 +523,7 @@ func (c *ReplicaClient) SyncFiles(fromAddress string, list []types.SyncFileInfo,
}

func (c *ReplicaClient) CreateBackup(backupName, snapshot, dest, volume, backingImageName, backingImageChecksum,
compressionMethod string, concurrentLimit int, storageClassName string, labels []string, credential map[string]string) (*ptypes.BackupCreateResponse, error) {
compressionMethod string, concurrentLimit int, storageClassName string, labels []string, credential, parameters map[string]string) (*ptypes.BackupCreateResponse, error) {
syncAgentServiceClient, err := c.getSyncServiceClient()
if err != nil {
return nil, err
Expand All @@ -543,6 +543,7 @@ func (c *ReplicaClient) CreateBackup(backupName, snapshot, dest, volume, backing
Labels: labels,
Credential: credential,
BackupName: backupName,
Parameters: parameters,
})
if err != nil {
return nil, errors.Wrapf(err, "failed to create backup to %v for volume %v", dest, volume)
Expand Down
8 changes: 4 additions & 4 deletions pkg/sync/backup.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ type RestoreStatus struct {
}

func (t *Task) CreateBackup(backupName, snapshot, dest, backingImageName, backingImageChecksum string,
compressionMethod string, concurrentLimit int, storageClassName string, labels []string, credential map[string]string) (*BackupCreateInfo, error) {
compressionMethod string, concurrentLimit int, storageClassName string, labels []string, credential, parameters map[string]string) (*BackupCreateInfo, error) {
if snapshot == types.VolumeHeadName {
return nil, fmt.Errorf("cannot backup the head disk in the chain")
}
Expand All @@ -60,7 +60,7 @@ func (t *Task) CreateBackup(backupName, snapshot, dest, backingImageName, backin
}

return t.createBackup(replica, backupName, snapshot, dest, volume.Name, backingImageName, backingImageChecksum,
compressionMethod, concurrentLimit, storageClassName, labels, credential)
compressionMethod, concurrentLimit, storageClassName, labels, credential, parameters)
}

func (t *Task) findRWReplica() (*types.ControllerReplicaInfo, error) {
Expand All @@ -80,7 +80,7 @@ func (t *Task) findRWReplica() (*types.ControllerReplicaInfo, error) {

func (t *Task) createBackup(replicaInController *types.ControllerReplicaInfo, backupName, snapshot, dest, volumeName,
backingImageName, backingImageChecksum, compression string, concurrentLimit int, storageClassName string,
labels []string, credential map[string]string) (*BackupCreateInfo, error) {
labels []string, credential, parameters map[string]string) (*BackupCreateInfo, error) {
if replicaInController.Mode != types.RW {
return nil, fmt.Errorf("can only create backup from replica in mode RW, got %s", replicaInController.Mode)
}
Expand All @@ -105,7 +105,7 @@ func (t *Task) createBackup(replicaInController *types.ControllerReplicaInfo, ba
logrus.Infof("Backing up %s on %s, to %s", snapshot, replicaInController.Address, dest)

reply, err := repClient.CreateBackup(backupName, snapshot, dest, volumeName, backingImageName, backingImageChecksum,
compression, concurrentLimit, storageClassName, labels, credential)
compression, concurrentLimit, storageClassName, labels, credential, parameters)
if err != nil {
return nil, err
}
Expand Down
1 change: 1 addition & 0 deletions pkg/sync/rpc/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -710,6 +710,7 @@ func (s *SyncAgentServer) BackupCreate(ctx context.Context, req *ptypes.BackupCr
ConcurrentLimit: req.ConcurrentLimit,
StorageClassName: req.StorageClassName,
Labels: req.Labels,
Parameters: req.Parameters,
})
if err != nil {
logrus.WithError(err).Errorf("Failed to initialize backup %v", req.BackupName)
Expand Down
Loading