From c8b67555935002a3cb832c7cb737a9866b57857f Mon Sep 17 00:00:00 2001 From: Faruk Kasumovic Date: Mon, 2 Jul 2018 14:37:54 +0200 Subject: [PATCH 1/3] Port to zfs-0.7.x Compatibility changes to work with libzfs 0.7.x, changes are not bacward compatible --- common.go | 25 ++++++++++++++++++++++--- sendrecv.go | 2 +- zpool.c | 12 ++++++++++-- zpool.go | 3 ++- zpool_test.go | 2 +- 5 files changed, 36 insertions(+), 8 deletions(-) diff --git a/common.go b/common.go index 81a4e6d..f23aac7 100644 --- a/common.go +++ b/common.go @@ -144,7 +144,9 @@ const ( // Pool properties. Enumerates available ZFS pool properties. Use it to access // pool properties either to read or set soecific property. const ( - PoolPropName Prop = iota + PoolPropCont Prop = iota - 2 + PoolPropInval + PoolPropName PoolPropSize PoolPropCapacity PoolPropAltroot @@ -171,6 +173,8 @@ const ( PoolPropLeaked PoolPropMaxBlockSize PoolPropTName + PoolPropMaxNodeSize + PoolPropMultiHost PoolNumProps ) @@ -181,7 +185,9 @@ const ( * the property table in module/zcommon/zfs_prop.c. */ const ( - DatasetPropType Prop = iota + DatasetPropCont Prop = iota - 2 + DatasetPropBad + DatasetPropType DatasetPropCreation DatasetPropUsed DatasetPropAvailable @@ -207,7 +213,7 @@ const ( DatasetPropSnapdir DatasetPropPrivate /* not exposed to user, temporary */ DatasetPropAclinherit - DatasetPropCreatetxg /* not exposed to the user */ + DatasetPropCreateTXG /* not exposed to the user */ DatasetPropName /* not exposed to the user */ DatasetPropCanmount DatasetPropIscsioptions /* not exposed to the user */ @@ -240,12 +246,14 @@ const ( DatasetPropDedup DatasetPropMlslabel DatasetPropSync + DatasetPropDnodeSize DatasetPropRefratio DatasetPropWritten DatasetPropClones DatasetPropLogicalused DatasetPropLogicalreferenced DatasetPropInconsistent /* not exposed to the user */ + DatasetPropVolmode DatasetPropFilesystemLimit DatasetPropSnapshotLimit DatasetPropFilesystemCount @@ -259,6 +267,17 @@ const ( DatasetPropRelatime DatasetPropRedundantMetadata DatasetPropOverlay + DatasetPropPrevSnap + DatasetPropReceiveResumeToken + DatasetPropEncryption + DatasetPropKeyLocation + DatasetPropKeyFormat + DatasetPropPBKDF2Salt + DatasetPropPBKDF2Iters + DatasetPropEncryptionRoot + DatasetPropKeyGUID + DatasetPropKeyStatus + DatasetPropRemapTXG /* not exposed to the user */ DatasetNumProps ) diff --git a/sendrecv.go b/sendrecv.go index 89937de..bd118e1 100644 --- a/sendrecv.go +++ b/sendrecv.go @@ -261,7 +261,7 @@ func (d *Dataset) Receive(inf *os.File, flags RecvFlags) (err error) { defer C.free(unsafe.Pointer(cflags)) dest := C.CString(dpath) defer C.free(unsafe.Pointer(dest)) - ec := C.zfs_receive(C.libzfsHandle, dest, cflags, C.int(inf.Fd()), nil) + ec := C.zfs_receive(C.libzfsHandle, dest, nil, cflags, C.int(inf.Fd()), nil) if ec != 0 { err = fmt.Errorf("ZFS receive of %s failed. %s", C.GoString(dest), LastError().Error()) } diff --git a/zpool.c b/zpool.c index d592be4..e23f126 100644 --- a/zpool.c +++ b/zpool.c @@ -2,7 +2,11 @@ * using libzfs from go language, and make go code shorter and more readable. */ +typedef unsigned long int rlim64_t; + #include +#include + #include #include #include @@ -189,7 +193,7 @@ property_list_ptr read_zpool_property(zpool_list_ptr pool, int prop) { property_list_ptr list = new_property_list(); r = zpool_get_prop(pool->zph, prop, - list->value, INT_MAX_VALUE, &source); + list->value, INT_MAX_VALUE, &source, B_FALSE); if (r == 0) { // strcpy(list->name, zpool_prop_to_name(prop)); zprop_source_tostr(list->source, source); @@ -491,11 +495,15 @@ nvlist_ptr get_zpool_vdev_tree(nvlist_ptr nv) { nvlist_ptr go_zpool_search_import(libzfs_handle_ptr zfsh, int paths, char **path, boolean_t do_scan) { importargs_t idata = { 0 }; + nvlist_ptr pools = NULL; idata.path = path; idata.paths = paths; // idata.scan = 0; - return zpool_search_import(zfsh, &idata); + thread_init(); + pools = zpool_search_import(zfsh, &idata); + thread_fini(); + return pools; } diff --git a/zpool.go b/zpool.go index 6d00056..a7b3f30 100644 --- a/zpool.go +++ b/zpool.go @@ -1,5 +1,6 @@ package zfs +// #cgo CFLAGS: -D__USE_LARGEFILE64=1 // #include // #include // #include "common.h" @@ -361,7 +362,7 @@ func poolSearchImport(q string, searchpaths []string, guid bool) (name string, C.strings_setat(cpaths, C.int(i), csPath) } - pools := C.zpool_find_import(C.libzfsHandle, C.int(numofp), cpaths) + pools := C.go_zpool_search_import(C.libzfsHandle, C.int(numofp), cpaths, C.B_FALSE) defer C.nvlist_free(pools) elem = C.nvlist_next_nvpair(pools, elem) diff --git a/zpool_test.go b/zpool_test.go index 72e9aff..05dc12f 100644 --- a/zpool_test.go +++ b/zpool_test.go @@ -543,7 +543,7 @@ func TestPool_VDevTree(t *testing.T) { // TODO: Add test cases. { name: "test1", - fields: fields{"NETSTOR"}, + fields: fields{"TESTPOOL"}, wantErr: false, }, } From 5d716ef4ec67ccc84c85c88a33de098298861981 Mon Sep 17 00:00:00 2001 From: Faruk Kasumovic Date: Fri, 6 Jul 2018 13:35:47 +0200 Subject: [PATCH 2/3] Additional compatibility changes to work with libzfs 0.7.x --- common.go | 3 +++ zpool.go | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/common.go b/common.go index f23aac7..02678d8 100644 --- a/common.go +++ b/common.go @@ -90,8 +90,11 @@ const ( PoolStatusFailingDev /* device experiencing errors */ PoolStatusVersionNewer /* newer on-disk version */ PoolStatusHostidMismatch /* last accessed by another system */ + PoolStatusHosidActive /* currently active on another system */ + PoolStatusHostidRequired /* multihost=on and hostid=0 */ PoolStatusIoFailureWait /* failed I/O, failmode 'wait' */ PoolStatusIoFailureContinue /* failed I/O, failmode 'continue' */ + PoolStatusIOFailureMap /* ailed MMP, failmode not 'panic' */ PoolStatusBadLog /* cannot read log chain(s) */ PoolStatusErrata /* informational errata available */ diff --git a/zpool.go b/zpool.go index a7b3f30..d45e526 100644 --- a/zpool.go +++ b/zpool.go @@ -1111,10 +1111,16 @@ func (s PoolStatus) String() string { return "VERSION_NEWER" case PoolStatusHostidMismatch: /* last accessed by another system */ return "HOSTID_MISMATCH" + case PoolStatusHosidActive: /* currently active on another system */ + return "HOSTID_ACTIVE" + case PoolStatusHostidRequired: /* multihost=on and hostid=0 */ + return "HOSTID_REQUIRED" case PoolStatusIoFailureWait: /* failed I/O, failmode 'wait' */ return "FAILURE_WAIT" case PoolStatusIoFailureContinue: /* failed I/O, failmode 'continue' */ return "FAILURE_CONTINUE" + case PoolStatusIOFailureMap: /* ailed MMP, failmode not 'panic' */ + return "HOSTID_FAILURE_MAP" case PoolStatusBadLog: /* cannot read log chain(s) */ return "BAD_LOG" case PoolStatusErrata: /* informational errata available */ From 46ec88c0a85d09e2b290d5c78ae8c0fb3e2b7deb Mon Sep 17 00:00:00 2001 From: Faruk Kasumovic Date: Wed, 11 Jul 2018 16:20:37 +0200 Subject: [PATCH 3/3] Estimate snapshot send size --- sendrecv.go | 103 +++++++++++++++++++++++++++++----------------------- zfs.c | 6 ++- 2 files changed, 63 insertions(+), 46 deletions(-) diff --git a/sendrecv.go b/sendrecv.go index bd118e1..ed6963b 100644 --- a/sendrecv.go +++ b/sendrecv.go @@ -5,13 +5,18 @@ package zfs // #include "common.h" // #include "zpool.h" // #include "zfs.h" +// #include +// #include import "C" import ( "fmt" + "io/ioutil" "os" "path" + "regexp" + "strconv" "strings" - "syscall" + "time" "unsafe" ) @@ -23,11 +28,11 @@ type SendFlags struct { Dedup bool Props bool DryRun bool - // Parsable bool - // Progress bool + Parsable bool + Progress bool LargeBlock bool EmbedData bool - // Compress bool + Compress bool } type RecvFlags struct { @@ -58,11 +63,11 @@ func to_sendflags_t(flags *SendFlags) (cflags *C.sendflags_t) { cflags.dedup = to_boolean_t(flags.Dedup) cflags.props = to_boolean_t(flags.Props) cflags.dryrun = to_boolean_t(flags.DryRun) - // cflags.parsable = to_boolean_t(flags.Parsable) - // cflags.progress = to_boolean_t(flags.Progress) + cflags.parsable = to_boolean_t(flags.Parsable) + cflags.progress = to_boolean_t(flags.Progress) cflags.largeblock = to_boolean_t(flags.LargeBlock) cflags.embed_data = to_boolean_t(flags.EmbedData) - // cflags.compress = to_boolean_t(flags.Compress) + cflags.compress = to_boolean_t(flags.Compress) return } @@ -199,53 +204,61 @@ func (d *Dataset) SendFrom(FromName string, outf *os.File, flags SendFlags) (err return } -func (d *Dataset) SendSize(FromName string, flags SendFlags) (size uint64, err error) { - var porigin Property - var from Dataset - var dpath string - if dpath, err = d.Path(); err != nil { +// SendSize - estimate snapshot size to transfer +func (d *Dataset) SendSize(FromName string, flags SendFlags) (size int64, err error) { + var r, w *os.File + errch := make(chan error) + defer func() { + select { + case <-errch: + default: + } + close(errch) + }() + flags.DryRun = true + flags.Verbose = true + flags.Progress = true + flags.Parsable = true + if r, w, err = os.Pipe(); err != nil { return } - zc := C.new_zfs_cmd() - defer C.free(unsafe.Pointer(zc)) - dpath = strings.Split(dpath, "@")[0] - if len(FromName) > 0 { + defer r.Close() + go func() { + var tmpe error + saveOut := C.dup(C.fileno(C.stdout)) + if res := C.dup2(C.int(w.Fd()), C.fileno(C.stdout)); res < 0 { + tmpe = fmt.Errorf("Redirection of zfslib stdout failed %d", res) + } else { + tmpe = d.send(FromName, w, &flags) + C.fflush(C.stdout) + C.dup2(saveOut, C.fileno(C.stdout)) + } + w.Close() + errch <- tmpe + }() - if FromName[0] == '#' || FromName[0] == '@' { - FromName = dpath + FromName - } - porigin, _ = d.GetProperty(DatasetPropOrigin) - if len(porigin.Value) > 0 && porigin.Value == FromName { - FromName = "" - flags.FromOrigin = true - } - if from, err = DatasetOpen(FromName); err != nil { + r.SetReadDeadline(time.Now().Add(15 * time.Second)) + var data []byte + if data, err = ioutil.ReadAll(r); err != nil { + return + } + // parse size + var sizeRe *regexp.Regexp + if sizeRe, err = regexp.Compile("size[ \t]*([0-9]+)"); err != nil { + return + } + matches := sizeRe.FindAllSubmatch(data, 3) + if len(matches) > 0 && len(matches[0]) > 1 { + if size, err = strconv.ParseInt( + string(matches[0][1]), 10, 64); err != nil { return } - zc.zc_fromobj = C.zfs_prop_get_int(from.list.zh, C.ZFS_PROP_OBJSETID) - from.Close() - } else { - zc.zc_fromobj = 0 } - zc.zc_obj = C.uint64_t(to_boolean_t(flags.FromOrigin)) - zc.zc_sendobj = C.zfs_prop_get_int(d.list.zh, C.ZFS_PROP_OBJSETID) - zc.zc_guid = 1 - zc.zc_flags = 0 - if flags.LargeBlock { - zc.zc_flags |= C.LZC_SEND_FLAG_LARGE_BLOCK - } - if flags.EmbedData { - zc.zc_flags |= C.LZC_SEND_FLAG_EMBED_DATA - } - - // C.estimate_ioctl(d.list.zhp, prevsnap_obj, to_boolean_t(flags.FromOrigin), lzc_send_flags, unsafe.Pointer(&size)) - if ec, e := C.estimate_send_size(zc); ec != 0 { - err = fmt.Errorf("Failed to estimate send size. %s %d", e.Error(), e.(syscall.Errno)) - } - size = uint64(zc.zc_objset_type) + err = <-errch return } +// Receive - receive snapshot stream func (d *Dataset) Receive(inf *os.File, flags RecvFlags) (err error) { var dpath string if dpath, err = d.Path(); err != nil { diff --git a/zfs.c b/zfs.c index 09ba095..99e620b 100644 --- a/zfs.c +++ b/zfs.c @@ -256,6 +256,10 @@ struct zfs_cmd *new_zfs_cmd(){ } int estimate_send_size(struct zfs_cmd *zc) { - return zfs_ioctl(libzfsHandle, ZFS_IOC_SEND, zc); + int rc = zfs_ioctl(libzfsHandle, ZFS_IOC_SEND, zc); + if (rc != 0) { + rc = errno; + } + return rc; }