From 0413659372b3d9d4cf08ec88c3206ba6825a5d3e Mon Sep 17 00:00:00 2001 From: NeilBrown Date: Fri, 18 Mar 2011 16:24:28 +1100 Subject: [PATCH] Add 'load_dev' command and matching library functions. These will load the device blocks from a file, then load the best state block, making sure everything is compatible. Signed-off-by; NeilBrown --- include/lafs/lafs.h | 17 ++- include/lafs/struct.h | 13 ++- lib/internal.h | 5 + lib/lafs_include_dev.c | 42 +++++++ lib/lafs_load.c | 256 +++++++++++++++++++++++++++++++++++++++++ lib/lafs_write_dev.c | 3 +- lib/lafs_write_state.c | 3 +- tools/lafs.c | 50 ++++++++ tools/open_device.c | 2 +- 9 files changed, 382 insertions(+), 9 deletions(-) create mode 100644 lib/lafs_include_dev.c create mode 100644 lib/lafs_load.c diff --git a/include/lafs/lafs.h b/include/lafs/lafs.h index 420834a..f8e37bd 100644 --- a/include/lafs/lafs.h +++ b/include/lafs/lafs.h @@ -22,6 +22,8 @@ char *lafs_validate_geometry(long *block_bytes, struct lafs_device *lafs_add_device(struct lafs *, char *devname, int fd, loff_t segblocks, loff_t strideblock, int width, int usage_inum); +struct lafs_device *lafs_load(int fd, long long device_bytes, char **err); +int lafs_include_dev(struct lafs *fs, struct lafs_device *dev, char **err); struct lafs_ino *lafs_get_itable(struct lafs *); struct lafs_ino *lafs_add_inode(struct lafs_ino*, int inum, int type); @@ -63,11 +65,6 @@ void lafs_summary_update(struct lafs_ino *ino, int is_index); void lafs_segment_count(struct lafs *fs, loff_t addr, int diff); -unsigned long crc32( - unsigned long crc, - const uint32_t *buf, - unsigned len); - static inline struct lafs_dblk *dblk(struct lafs_blk *b) { return container_of(b, struct lafs_dblk, b); @@ -91,6 +88,16 @@ static inline uint64_t lafs_encode_timeval(struct timeval *tm) nano &= ~1ULL; return sec | (nano << 34); } +static inline void lafs_decode_timeval(struct timeval *tm, uint64_t te) +{ + /* low 35 bits are seconds (800 years) + * high 29 bits are 2nanoseconds + */ + long nano; + tm->tv_sec = (te & 0X7FFFFFFFFULL); + nano = (te >> 34) & ~(long)1; + tm->tv_usec = nano / 1000; +} static inline struct lafs_device *dev_by_num(struct lafs *fs, int num) { diff --git a/include/lafs/struct.h b/include/lafs/struct.h index 88c9f55..4d97ef2 100644 --- a/include/lafs/struct.h +++ b/include/lafs/struct.h @@ -183,9 +183,20 @@ struct lafs_device { int rows_per_table, tables_per_seg; int tablesize; - int recent_super, recent_state; + int recent_super; /* Index of most recent devaddr */ + int recent_state; /* Index of most recent 'state' */ + int state_seq; /* 'seq' number of that state block */ struct lafs_ino *segsum; + + /* This are only used between loading from storage + * and including in the array + */ + char version[16]; + uint8_t uuid[16]; + int blocksize; + int statesize; + int devices; }; struct lafs_blk { diff --git a/lib/internal.h b/lib/internal.h index 57ebd9f..8d828d7 100644 --- a/lib/internal.h +++ b/lib/internal.h @@ -26,3 +26,8 @@ de_sched(struct lafs_blk *b) lafs_sched_blk(&p->b); } } + +unsigned long crc32( + unsigned long crc, + const uint32_t *buf, + unsigned len); diff --git a/lib/lafs_include_dev.c b/lib/lafs_include_dev.c new file mode 100644 index 0000000..0829ef3 --- /dev/null +++ b/lib/lafs_include_dev.c @@ -0,0 +1,42 @@ + +#include +#include +#include + +int lafs_include_dev(struct lafs *fs, struct lafs_device *dev, char **err) +{ + /* If fs not initialised, set up uuid/blocksize/statesize/etc + * If it is check the details match, and check this device + * doesn't overlap with any other device, and that state seq numbers + * are close enough. + */ + + if (fs->blocksize == 0) { + fs->blocksize = dev->blocksize; + memcpy(fs->uuid, dev->uuid, 16); + fs->statesize = dev->statesize; + fs->seq = dev->state_seq; + fs->devices = dev->devices; + } else { + if (fs->blocksize != dev->blocksize || + memcmp(fs->uuid, dev->uuid, 16) || + fs->statesize != dev->statesize) { + *err = "device is inconsistent with other devices in LaFS"; + return -1; + } + if (dev->state_seq > fs->seq) { + fs->seq = dev->state_seq; + fs->devices = dev->devices; + } + } + + if (fs->loaded_devs >= fs->devices) { + *err = "already have enough devices for full LaFS"; + return -1; + } + fs->loaded_devs++; + dev->next = fs->devs; + fs->devs = dev; + talloc_reparent(NULL, fs, dev); + return 0; +} diff --git a/lib/lafs_load.c b/lib/lafs_load.c new file mode 100644 index 0000000..7dca5a2 --- /dev/null +++ b/lib/lafs_load.c @@ -0,0 +1,256 @@ +#define _GNU_SOURCE +#define _FILE_OFFSET_BITS 64 + +#include +#include +#include +#include +#include +#include "internal.h" + +/* read the device blocks and state blocks from this device. + * If valid, return a 'lafs_dev' suitably filled in. + * The most recent state block is found and index is recorded in lafs_dev + * + * device block should be near the start and near the end. + * Should probably determine block size, but for not just iterate + * over sectors. + */ + +static int valid_devblock(struct lafs_dev *db, unsigned long long addr, + unsigned long long device_bytes) +{ + /* check that this devblock is valid, given that + * it was found at sector 'addr' + */ + u32 crc, crc2; + u64 byteaddr; + unsigned long long segsize; + int i; + + if (strncmp(db->idtag, "LaFS-DeviceBlock", 16) != 0) + return 0; + if (strncmp(db->version, "AlphaDevel ", 16) != 0) + return 0; + /* uuid can be anything */ + crc = db->checksum; + db->checksum = 0; + crc2 = crc32(0, (uint32_t*)db, LAFS_DEVBLK_SIZE); + db->checksum = crc; + if (crc2 != crc) + return 0; + + byteaddr = addr << 9; /* convert to byte */ + if (__le64_to_cpu(db->devaddr[0]) != byteaddr && + __le64_to_cpu(db->devaddr[1]) != byteaddr) + return 0; + + if (db->statebits < 10 || db->statebits > 16) + return 0; + if (db->blockbits < 9 || db->blockbits > 20) + return 0; + if (__le16_to_cpu(db->width) < 1 || __le16_to_cpu(db->width) >= 512) + return 0; + if (__le32_to_cpu(db->stride) < 1) + return 0; + /* devaddr[0] must be early, [1] must be late */ + if (__le64_to_cpu(db->devaddr[0]) >= + __le64_to_cpu(db->segment_offset)) + return 0; + + if (__le64_to_cpu(db->devaddr[1]) < + __le64_to_cpu(db->segment_offset) + + ((((unsigned long long)__le32_to_cpu(db->segment_count) + * __le32_to_cpu(db->segment_size))) + << db->blockbits)) + return 0; + + /* 2 is an absolute minimum segment size, a few hundred is more + * likely. We'll put a lower limit of 8, and an upper of 800000 + */ + if (__le32_to_cpu(db->segment_size) < 8 || + __le32_to_cpu(db->segment_size) > 800000) + return 0; + + if (__le32_to_cpu(db->segment_offset) > + (__le32_to_cpu(db->segment_size)<blockbits) * 10) + return 0; + + /* The 4 state blocks live before the first or after the last segment. + * The distance from start of first to end of last is either: + * - segment_count * segment_size if width*stride <= segment_size + * - (width-1) * stride + segment_size / width * segment_count + * if width * stride > segment_size + */ + segsize = __le32_to_cpu(db->segment_size); + segsize *= __le32_to_cpu(db->segment_count); + if (__le16_to_cpu(db->width) * __le32_to_cpu(db->stride) + > __le32_to_cpu(db->segment_size)) { + int stride = __le32_to_cpu(db->stride); + int width = __le16_to_cpu(db->width); + + segsize /= width; + segsize += (width - 1) * stride; + } + segsize <<= db->blockbits; + for (i = 0; i < 4; i++) { + unsigned long long addr = __le64_to_cpu(db->stateaddr[i]); + int offset = __le32_to_cpu(db->segment_offset); + if (addr + (1<statebits) > offset && + addr < offset + segsize) + return 0; + if (addr + (1<statebits) > device_bytes) + return 0; + } + + /* Check all segments fit within device */ + if (__le32_to_cpu(db->segment_offset) + segsize > device_bytes) + return 0; + + if (__le32_to_cpu(db->level) > 10) + return 0; + + /* I guess it look sane enough... */ + return 1; +} + +static int compare_dev(struct lafs_dev *curr, struct lafs_dev *new) +{ + if (memcmp(curr->uuid, new->uuid, 16) != 0) + /* there are different */ + return -1; + if (memcmp(curr->version, new->version, 16) != 0) + return -1; + + if (__le32_to_cpu(curr->seq) >= __le32_to_cpu(new->seq)) + return 0; /*current is best */ + return 1; +} + +static int check_state(struct lafs_device *dev, int which) +{ + /* Load this state block, perform basic check and update + * dev->state_seq if seq number is new + */ + struct lafs_state *sb = malloc(dev->statesize); + u32 crc, crc2; + int rv = 0; + + lseek64(dev->fd, dev->stateaddr[which], 0); + if (read(dev->fd, sb, dev->statesize) != dev->statesize) + goto out; + if (memcmp(sb->idtag, "LaFS-State-Block", 16) != 0) + goto out; + if (memcmp(sb->uuid, dev->uuid, 16) != 0) + goto out; + crc = sb->checksum; + sb->checksum = 0; + crc2 = crc32(0, (uint32_t*)sb, dev->statesize); + if (crc != crc2) + goto out; + + if (__le32_to_cpu(sb->seq) > dev->state_seq) { + dev->recent_state = which; + dev->state_seq = __le32_to_cpu(sb->seq); + dev->devices = __le32_to_cpu(sb->devices); + } + rv = 1; + +out: + free(sb); + return rv; +} + +static int destroy(struct lafs_device *dev) +{ + if (dev->fd >= 0) + close(dev->fd); + return 0; +} + + +struct lafs_device *lafs_load(int fd, long long device_bytes, char **err) +{ + char buf[LAFS_DEVBLK_SIZE]; + struct lafs_dev *d = (void*)buf; + struct lafs_dev best; + int have_best = 0; + unsigned long long addr, best_addr; + struct lafs_device *dev; + int i; + int found; + + err = NULL; + + for (addr = 0; addr < device_bytes; addr += 512) { + if (addr == 32*512 && device_bytes > 64*512) + addr = device_bytes - 32*512; + + lseek64(fd, addr, 0); + if (read(fd, buf, LAFS_DEVBLK_SIZE) != LAFS_DEVBLK_SIZE) + continue; + if (!valid_devblock(d, addr, device_bytes)) + continue; + if (!have_best) { + best = *d; + best_addr = addr; + have_best = 1; + continue; + } + switch(compare_dev(&best, d)) { + case 0: /* best is still best */ + continue; + case 1: /* d is better */ + best = *d; + best_addr = addr; + continue; + default: /* incompatible */ + *err = "inconsistent device information blocks found"; + return NULL; + } + } + if (!have_best) { + *err = "no valid device block found"; + return NULL; + } + + /* talloc device, fill in details, record where loaded from */ + dev = talloc(NULL, struct lafs_device); + talloc_set_destructor(dev, destroy); + memset(dev, 0, sizeof(*dev)); + dev->fd = fd; + dev->seq = __le32_to_cpu(best.seq); + dev->start = __le64_to_cpu(best.start); + dev->size = __le64_to_cpu(best.size); + for (i=0 ; i<2; i++) { + dev->devaddr[i] = __le64_to_cpu(best.devaddr[i]); + if (dev->devaddr[i] == best_addr) + dev->recent_super = i; + } + for (i=0; i<4; i++) + dev->stateaddr[i] = __le64_to_cpu(best.stateaddr[i]); + + lafs_decode_timeval(&dev->ctime, best.ctime); + dev->width = __le16_to_cpu(best.width); + dev->stride = __le32_to_cpu(best.stride); + dev->segment_size = __le32_to_cpu(best.segment_size); + dev->segment_offset = __le32_to_cpu(best.segment_offset); + dev->segment_count = __le32_to_cpu(best.segment_count); + dev->usage_inum = __le32_to_cpu(best.usage_inum); + + memcpy(dev->version, best.version, 16); + memcpy(dev->uuid, best.uuid, 16); + dev->blocksize = 1 << best.blockbits; + dev->statesize = 1 << best.statebits; + + found = 0; + for (i=0; i<4; i++) + found += check_state(dev, i); + + if (!found) + *err = "No valid state block found on device"; + + return dev; +} + + diff --git a/lib/lafs_write_dev.c b/lib/lafs_write_dev.c index 77e9ea4..cc628fd 100644 --- a/lib/lafs_write_dev.c +++ b/lib/lafs_write_dev.c @@ -4,6 +4,7 @@ #include #include #include +#include "internal.h" static int size_bits(long long size) { @@ -26,7 +27,7 @@ int lafs_write_dev(struct lafs_device *dev) memset(buf, 0, sizeof(buf)); memcpy(pd->idtag, "LaFS-DeviceBlock", 16); - memset(pd->version, ' ', 16); + memcpy(pd->version, "AlphaDevel ", 16); memcpy(pd->uuid, fs->uuid, 16); dev->seq++; diff --git a/lib/lafs_write_state.c b/lib/lafs_write_state.c index 1d091bb..b7b443d 100644 --- a/lib/lafs_write_state.c +++ b/lib/lafs_write_state.c @@ -8,6 +8,7 @@ #include #include #include +#include "internal.h" int lafs_write_state(struct lafs *fs) { @@ -30,7 +31,7 @@ int lafs_write_state(struct lafs *fs) st->maxsnapshot = 1; st->devices = __cpu_to_le32(fs->devices); - st->checksum = crc32(0, (uint32_t *)buf, fs->blocksize); + st->checksum = crc32(0, (uint32_t *)buf, fs->statesize); for (dev = fs->devs ; dev ; dev = dev->next) { int n; diff --git a/tools/lafs.c b/tools/lafs.c index d9923f0..0616e28 100644 --- a/tools/lafs.c +++ b/tools/lafs.c @@ -1029,6 +1029,55 @@ static void c_write_checkpoint(struct state *st, void **args) (unsigned long long) st->lafs->seq); } +/****** LOAD_DEV ******/ +static char help_load_dev[] = "Allow access to LaFS filesystem stored on given device"; +static struct args args_load_dev[] = { + { "DEVNAME", external, -1, {NULL}, "Device to load filesystem info from"}, + { "-file", external, 0, {NULL}, "Regular file to load filesystem info from"}, + TERMINAL_ARG +}; +static void c_load_dev(struct state *st, void **args) +{ + char *devname = args[1]; + int fd; + long long device_bytes = 0; + char *err; + struct lafs_device *dev; + + if (!devname) { + printf("load_dev: No device of file name given to load\n"); + return; + } + + fd = open_device(devname, &device_bytes, args[2] != NULL, &err); + + if (fd < 0) { + printf("load_dev: %s\n", err); + free(err); + return; + } + + dev = lafs_load(fd, device_bytes, &err); + + if (err) { + printf("load_dev: Cannot load %s: %s\n", devname, err); + if (dev) + talloc_free(dev); + close(fd); + return; + } + + if (lafs_include_dev(st->lafs, dev, &err) != 0) { + printf("load_dev: Cannot include %s: %s\n", devname, err); + talloc_free(dev); + return; + } + if (st->verbose) { + printf("loaded device %s - have %d of %d\n", devname, + st->lafs->loaded_devs, st->lafs->devices); + } +} + /****** STORE ******/ static char help_store[] = "Create a file in the LaFS from an external file"; static struct args args_store[] = { @@ -1056,6 +1105,7 @@ static struct cmd lafs_cmds[] = { CMD(add_device), CMD(exit), CMD(help), + CMD(load_dev), CMD(newfs), CMD(quit), CMD(reset), diff --git a/tools/open_device.c b/tools/open_device.c index ba06941..bbdb8c6 100644 --- a/tools/open_device.c +++ b/tools/open_device.c @@ -35,7 +35,7 @@ int open_device(char *devname, long long *device_bytes, int regular_file, else fd = open(devname, O_RDWR); if (fd < 0 || fstat(fd, &stb) < 0) { - asprintf(error, "cannot open %s %s:%s", + asprintf(error, "cannot open %s %s: %s", regular_file? "file" : "device", devname, strerror(errno)); return -1; -- 2.39.5