diff options
| -rw-r--r-- | fs/exofs/common.h | 63 | ||||
| -rw-r--r-- | fs/exofs/exofs.h | 12 | ||||
| -rw-r--r-- | fs/exofs/inode.c | 5 | ||||
| -rw-r--r-- | fs/exofs/ios.c | 38 | ||||
| -rw-r--r-- | fs/exofs/pnfs.h | 51 | ||||
| -rw-r--r-- | fs/exofs/super.c | 220 |
6 files changed, 361 insertions, 28 deletions
diff --git a/fs/exofs/common.h b/fs/exofs/common.h index ce1c716..b1b178e 100644 --- a/fs/exofs/common.h +++ b/fs/exofs/common.h @@ -49,6 +49,7 @@ #define EXOFS_MIN_PID 0x10000 /* Smallest partition ID */ #define EXOFS_OBJ_OFF 0x10000 /* offset for objects */ #define EXOFS_SUPER_ID 0x10000 /* object ID for on-disk superblock */ +#define EXOFS_DEVTABLE_ID 0x10001 /* object ID for on-disk device table */ #define EXOFS_ROOT_ID 0x10002 /* object ID for root directory */ /* exofs Application specific page/attribute */ @@ -78,17 +79,67 @@ enum { #define EXOFS_SUPER_MAGIC 0x5DF5 /* - * The file system control block - stored in an object's data (mainly, the one - * with ID EXOFS_SUPER_ID). This is where the in-memory superblock is stored - * on disk. Right now it just has a magic value, which is basically a sanity - * check on our ability to communicate with the object store. + * The file system control block - stored in object EXOFS_SUPER_ID's data. + * This is where the in-memory superblock is stored on disk. */ +enum {EXOFS_FSCB_VER = 1, EXOFS_DT_VER = 1}; struct exofs_fscb { __le64 s_nextid; /* Highest object ID used */ - __le32 s_numfiles; /* Number of files on fs */ + __le64 s_numfiles; /* Number of files on fs */ + __le32 s_version; /* == EXOFS_FSCB_VER */ __le16 s_magic; /* Magic signature */ __le16 s_newfs; /* Non-zero if this is a new fs */ -}; + + /* From here on it's a static part, only written by mkexofs */ + __le64 s_dev_table_oid; /* Resurved, not used */ + __le64 s_dev_table_count; /* == 0 means no dev_table */ +} __packed; + +/* + * Describes the raid used in the FS. It is part of the device table. + * This here is taken from the pNFS-objects definition. In exofs we + * use one raid policy through-out the filesystem. (NOTE: the funny + * alignment at begining. We take care of it at exofs_device_table. + */ +struct exofs_dt_data_map { + __le32 cb_num_comps; + __le64 cb_stripe_unit; + __le32 cb_group_width; + __le32 cb_group_depth; + __le32 cb_mirror_cnt; + __le32 cb_raid_algorithm; +} __packed; + +/* + * This is an osd device information descriptor. It is a single entry in + * the exofs device table. It describes an osd target lun which + * contains data belonging to this FS. (Same partition_id on all devices) + */ +struct exofs_dt_device_info { + __le32 systemid_len; + u8 systemid[OSD_SYSTEMID_LEN]; + __le64 long_name_offset; /* If !0 then offset-in-file */ + __le32 osdname_len; /* */ + u8 osdname[44]; /* Embbeded, Ususally an asci uuid */ +} __packed; + +/* + * The EXOFS device table - stored in object EXOFS_DEVTABLE_ID's data. + * It contains the raid used for this multy-device FS and an array of + * participating devices. + */ +struct exofs_device_table { + __le32 dt_version; /* == EXOFS_DT_VER */ + struct exofs_dt_data_map dt_data_map; /* Raid policy to use */ + + /* Resurved space For future use. Total includeing this: + * (8 * sizeof(le64)) + */ + __le64 __Resurved[4]; + + __le64 dt_num_devices; /* Array size */ + struct exofs_dt_device_info dt_dev_table[]; /* Array of devices */ +} __packed; /**************************************************************************** * inode-related things diff --git a/fs/exofs/exofs.h b/fs/exofs/exofs.h index 2e08859..c35fd46 100644 --- a/fs/exofs/exofs.h +++ b/fs/exofs/exofs.h @@ -37,6 +37,11 @@ #include <linux/time.h> #include "common.h" +/* FIXME: Remove once pnfs hits mainline + * #include <linux/exportfs/pnfs_osd_xdr.h> + */ +#include "pnfs.h" + #define EXOFS_ERR(fmt, a...) printk(KERN_ERR "exofs: " fmt, ##a) #ifdef CONFIG_EXOFS_DEBUG @@ -54,7 +59,6 @@ * our extension to the in-memory superblock */ struct exofs_sb_info { - struct osd_dev *s_dev; /* returned by get_osd_dev */ struct exofs_fscb s_fscb; /* Written often, pre-allocate*/ osd_id s_pid; /* partition ID of file system*/ int s_timeout; /* timeout for OSD operations */ @@ -63,7 +67,11 @@ struct exofs_sb_info { spinlock_t s_next_gen_lock; /* spinlock for gen # update */ u32 s_next_generation; /* next gen # to use */ atomic_t s_curr_pending; /* number of pending commands */ - uint8_t s_cred[OSD_CAP_LEN]; /* all-powerful credential */ + uint8_t s_cred[OSD_CAP_LEN]; /* credential for the fscb */ + + struct pnfs_osd_data_map data_map; /* Default raid to use */ + unsigned s_numdevs; /* Num of devices in array */ + struct osd_dev *s_ods[1]; /* Variable length, minimum 1 */ }; /* diff --git a/fs/exofs/inode.c b/fs/exofs/inode.c index 7578950..698a863 100644 --- a/fs/exofs/inode.c +++ b/fs/exofs/inode.c @@ -62,7 +62,10 @@ static void _pcol_init(struct page_collect *pcol, unsigned expected_pages, struct exofs_sb_info *sbi = inode->i_sb->s_fs_info; pcol->sbi = sbi; - pcol->req_q = osd_request_queue(sbi->s_dev); + /* Create master bios on first Q, later on cloning, each clone will be + * allocated on it's destination Q + */ + pcol->req_q = osd_request_queue(sbi->s_ods[0]); pcol->inode = inode; pcol->expected_pages = expected_pages; diff --git a/fs/exofs/ios.c b/fs/exofs/ios.c index bb2f9d3..5bad01f 100644 --- a/fs/exofs/ios.c +++ b/fs/exofs/ios.c @@ -71,7 +71,7 @@ int exofs_get_io_state(struct exofs_sb_info *sbi, struct exofs_io_state** pios) /*TODO: Maybe use kmem_cach per sbi of size * exofs_io_state_size(sbi->s_numdevs) */ - ios = kzalloc(exofs_io_state_size(1), GFP_KERNEL); + ios = kzalloc(exofs_io_state_size(sbi->s_numdevs), GFP_KERNEL); if (unlikely(!ios)) { *pios = NULL; return -ENOMEM; @@ -209,10 +209,10 @@ int exofs_sbi_create(struct exofs_io_state *ios) { int i, ret; - for (i = 0; i < 1; i++) { + for (i = 0; i < ios->sbi->s_numdevs; i++) { struct osd_request *or; - or = osd_start_request(ios->sbi->s_dev, GFP_KERNEL); + or = osd_start_request(ios->sbi->s_ods[i], GFP_KERNEL); if (unlikely(!or)) { EXOFS_ERR("%s: osd_start_request failed\n", __func__); ret = -ENOMEM; @@ -233,10 +233,10 @@ int exofs_sbi_remove(struct exofs_io_state *ios) { int i, ret; - for (i = 0; i < 1; i++) { + for (i = 0; i < ios->sbi->s_numdevs; i++) { struct osd_request *or; - or = osd_start_request(ios->sbi->s_dev, GFP_KERNEL); + or = osd_start_request(ios->sbi->s_ods[i], GFP_KERNEL); if (unlikely(!or)) { EXOFS_ERR("%s: osd_start_request failed\n", __func__); ret = -ENOMEM; @@ -257,10 +257,10 @@ int exofs_sbi_write(struct exofs_io_state *ios) { int i, ret; - for (i = 0; i < 1; i++) { + for (i = 0; i < ios->sbi->s_numdevs; i++) { struct osd_request *or; - or = osd_start_request(ios->sbi->s_dev, GFP_KERNEL); + or = osd_start_request(ios->sbi->s_ods[i], GFP_KERNEL); if (unlikely(!or)) { EXOFS_ERR("%s: osd_start_request failed\n", __func__); ret = -ENOMEM; @@ -272,7 +272,21 @@ int exofs_sbi_write(struct exofs_io_state *ios) if (ios->bio) { struct bio *bio; - bio = ios->bio; + if (i != 0) { + bio = bio_kmalloc(GFP_KERNEL, + ios->bio->bi_max_vecs); + if (unlikely(!bio)) { + ret = -ENOMEM; + goto out; + } + + __bio_clone(bio, ios->bio); + bio->bi_bdev = NULL; + bio->bi_next = NULL; + ios->per_dev[i].bio = bio; + } else { + bio = ios->bio; + } osd_req_write(or, &ios->obj, ios->offset, bio, ios->length); @@ -306,8 +320,10 @@ int exofs_sbi_read(struct exofs_io_state *ios) for (i = 0; i < 1; i++) { struct osd_request *or; + unsigned first_dev = (unsigned)ios->obj.id; - or = osd_start_request(ios->sbi->s_dev, GFP_KERNEL); + first_dev %= ios->sbi->s_numdevs; + or = osd_start_request(ios->sbi->s_ods[first_dev], GFP_KERNEL); if (unlikely(!or)) { EXOFS_ERR("%s: osd_start_request failed\n", __func__); ret = -ENOMEM; @@ -382,10 +398,10 @@ int exofs_oi_truncate(struct exofs_i_info *oi, u64 size) attr = g_attr_logical_length; attr.val_ptr = &newsize; - for (i = 0; i < 1; i++) { + for (i = 0; i < sbi->s_numdevs; i++) { struct osd_request *or; - or = osd_start_request(sbi->s_dev, GFP_KERNEL); + or = osd_start_request(sbi->s_ods[i], GFP_KERNEL); if (unlikely(!or)) { EXOFS_ERR("%s: osd_start_request failed\n", __func__); ret = -ENOMEM; diff --git a/fs/exofs/pnfs.h b/fs/exofs/pnfs.h new file mode 100644 index 0000000..423033a --- /dev/null +++ b/fs/exofs/pnfs.h @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2008, 2009 + * Boaz Harrosh <bharrosh@panasas.com> + * + * This file is part of exofs. + * + * exofs is free software; you can redistribute it and/or modify it under the + * terms of the GNU General Public License version 2 as published by the Free + * Software Foundation. + * + */ + +/* FIXME: Remove this file once pnfs hits mainline */ + +#ifndef __EXOFS_PNFS_H__ +#define __EXOFS_PNFS_H__ + +#if defined(CONFIG_PNFS) + + +/* FIXME: move this file to: linux/exportfs/pnfs_osd_xdr.h */ +#include "../nfs/objlayout/pnfs_osd_xdr.h" + +#else /* defined(CONFIG_PNFS) */ + +enum pnfs_iomode { + IOMODE_READ = 1, + IOMODE_RW = 2, + IOMODE_ANY = 3, +}; + +/* Layout Structure */ +enum pnfs_osd_raid_algorithm4 { + PNFS_OSD_RAID_0 = 1, + PNFS_OSD_RAID_4 = 2, + PNFS_OSD_RAID_5 = 3, + PNFS_OSD_RAID_PQ = 4 /* Reed-Solomon P+Q */ +}; + +struct pnfs_osd_data_map { + u32 odm_num_comps; + u64 odm_stripe_unit; + u32 odm_group_width; + u32 odm_group_depth; + u32 odm_mirror_cnt; + u32 odm_raid_algorithm; +}; + +#endif /* else defined(CONFIG_PNFS) */ + +#endif /* __EXOFS_PNFS_H__ */ diff --git a/fs/exofs/super.c b/fs/exofs/super.c index 4cd97f5..a1d1e77 100644 --- a/fs/exofs/super.c +++ b/fs/exofs/super.c @@ -214,12 +214,17 @@ int exofs_sync_fs(struct super_block *sb, int wait) if (ret) goto out; - ios->length = sizeof(*fscb); + /* Note: We only write the changing part of the fscb. .i.e upto the + * the fscb->s_dev_table_oid member. There is no read-modify-write + * here. + */ + ios->length = offsetof(struct exofs_fscb, s_dev_table_oid); memset(fscb, 0, ios->length); fscb->s_nextid = cpu_to_le64(sbi->s_nextid); fscb->s_numfiles = cpu_to_le32(sbi->s_numfiles); fscb->s_magic = cpu_to_le16(sb->s_magic); fscb->s_newfs = 0; + fscb->s_version = EXOFS_FSCB_VER; ios->obj.id = EXOFS_SUPER_ID; ios->offset = 0; @@ -257,6 +262,20 @@ static void _exofs_print_device(const char *msg, const char *dev_path, msg, dev_path ?: "", odi->osdname, _LLU(pid)); } +void exofs_free_sbi(struct exofs_sb_info *sbi) +{ + while (sbi->s_numdevs) { + int i = --sbi->s_numdevs; + struct osd_dev *od = sbi->s_ods[i]; + + if (od) { + sbi->s_ods[i] = NULL; + osduld_put_device(od); + } + } + kfree(sbi); +} + /* * This function is called when the vfs is freeing the superblock. We just * need to free our own part. @@ -279,12 +298,182 @@ static void exofs_put_super(struct super_block *sb) msecs_to_jiffies(100)); } - _exofs_print_device("Unmounting", NULL, sbi->s_dev, sbi->s_pid); - osduld_put_device(sbi->s_dev); - kfree(sb->s_fs_info); + _exofs_print_device("Unmounting", NULL, sbi->s_ods[0], sbi->s_pid); + + exofs_free_sbi(sbi); sb->s_fs_info = NULL; } +static int _read_and_match_data_map(struct exofs_sb_info *sbi, unsigned numdevs, + struct exofs_device_table *dt) +{ + sbi->data_map.odm_num_comps = + le32_to_cpu(dt->dt_data_map.cb_num_comps); + sbi->data_map.odm_stripe_unit = + le64_to_cpu(dt->dt_data_map.cb_stripe_unit); + sbi->data_map.odm_group_width = + le32_to_cpu(dt->dt_data_map.cb_group_width); + sbi->data_map.odm_group_depth = + le32_to_cpu(dt->dt_data_map.cb_group_depth); + sbi->data_map.odm_mirror_cnt = + le32_to_cpu(dt->dt_data_map.cb_mirror_cnt); + sbi->data_map.odm_raid_algorithm = + le32_to_cpu(dt->dt_data_map.cb_raid_algorithm); + +/* FIXME: Hard coded mirror only for now. if not so do not mount */ + if ((sbi->data_map.odm_num_comps != numdevs) || + (sbi->data_map.odm_stripe_unit != EXOFS_BLKSIZE) || + (sbi->data_map.odm_raid_algorithm != PNFS_OSD_RAID_0) || + (sbi->data_map.odm_mirror_cnt != (numdevs - 1))) + return -EINVAL; + else + return 0; +} + +/* @odi is valid only as long as @fscb_dev is valid */ +static int exofs_devs_2_odi(struct exofs_dt_device_info *dt_dev, + struct osd_dev_info *odi) +{ + odi->systemid_len = le32_to_cpu(dt_dev->systemid_len); + memcpy(odi->systemid, dt_dev->systemid, odi->systemid_len); + + odi->osdname_len = le32_to_cpu(dt_dev->osdname_len); + odi->osdname = dt_dev->osdname; + + /* FIXME support long names. Will need a _put function */ + if (dt_dev->long_name_offset) + return -EINVAL; + + /* Make sure osdname is printable! + * mkexofs should give us space for a null-terminator else the + * device-table is invalid. + */ + if (unlikely(odi->osdname_len >= sizeof(dt_dev->osdname))) + odi->osdname_len = sizeof(dt_dev->osdname) - 1; + dt_dev->osdname[odi->osdname_len] = 0; + + /* If it's all zeros something is bad we read past end-of-obj */ + return !(odi->systemid_len || odi->osdname_len); +} + +static int exofs_read_lookup_dev_table(struct exofs_sb_info **psbi, + unsigned table_count) +{ + struct exofs_sb_info *sbi = *psbi; + struct osd_dev *fscb_od; + struct osd_obj_id obj = {.partition = sbi->s_pid, + .id = EXOFS_DEVTABLE_ID}; + struct exofs_device_table *dt; + unsigned table_bytes = table_count * sizeof(dt->dt_dev_table[0]) + + sizeof(*dt); + unsigned numdevs, i; + int ret; + + dt = kmalloc(table_bytes, GFP_KERNEL); + if (unlikely(!dt)) { + EXOFS_ERR("ERROR: allocating %x bytes for device table\n", + table_bytes); + return -ENOMEM; + } + + fscb_od = sbi->s_ods[0]; + sbi->s_ods[0] = NULL; + sbi->s_numdevs = 0; + ret = exofs_read_kern(fscb_od, sbi->s_cred, &obj, 0, dt, table_bytes); + if (unlikely(ret)) { + EXOFS_ERR("ERROR: reading device table\n"); + goto out; + } + + numdevs = le64_to_cpu(dt->dt_num_devices); + if (unlikely(!numdevs)) { + ret = -EINVAL; + goto out; + } + WARN_ON(table_count != numdevs); + + ret = _read_and_match_data_map(sbi, numdevs, dt); + if (unlikely(ret)) + goto out; + + if (likely(numdevs > 1)) { + unsigned size = numdevs * sizeof(sbi->s_ods[0]); + + sbi = krealloc(sbi, sizeof(*sbi) + size, GFP_KERNEL); + if (unlikely(!sbi)) { + ret = -ENOMEM; + goto out; + } + memset(&sbi->s_ods[1], 0, size - sizeof(sbi->s_ods[0])); + *psbi = sbi; + } + + for (i = 0; i < numdevs; i++) { + struct exofs_fscb fscb; + struct osd_dev_info odi; + struct osd_dev *od; + + if (exofs_devs_2_odi(&dt->dt_dev_table[i], &odi)) { + EXOFS_ERR("ERROR: Read all-zeros device entry\n"); + ret = -EINVAL; + goto out; + } + + printk(KERN_NOTICE "Add device[%d]: osd_name-%s\n", + i, odi.osdname); + + /* On all devices the device table is identical. The user can + * specify any one of the participating devices on the command + * line. We always keep them in device-table order. + */ + if (fscb_od && osduld_device_same(fscb_od, &odi)) { + sbi->s_ods[i] = fscb_od; + ++sbi->s_numdevs; + fscb_od = NULL; + continue; + } + + od = osduld_info_lookup(&odi); + if (unlikely(IS_ERR(od))) { + ret = PTR_ERR(od); + EXOFS_ERR("ERROR: device requested is not found " + "osd_name-%s =>%d\n", odi.osdname, ret); + goto out; + } + + sbi->s_ods[i] = od; + ++sbi->s_numdevs; + + /* Read the fscb of the other devices to make sure the FS + * partition is there. + */ + ret = exofs_read_kern(od, sbi->s_cred, &obj, 0, &fscb, + sizeof(fscb)); + if (unlikely(ret)) { + EXOFS_ERR("ERROR: Malformed participating device " + "error reading fscb osd_name-%s\n", + odi.osdname); + goto out; + } + + /* TODO: verify other information is correct and FS-uuid + * matches. Benny what did you say about device table + * generation and old devices? + */ + } + +out: + kfree(dt); + if (unlikely(!ret && fscb_od)) { + EXOFS_ERR( + "ERROR: Bad device-table container device not present\n"); + osduld_put_device(fscb_od); + ret = -EINVAL; + } + + return ret; +} + /* * Read the superblock from the OSD and fill in the fields */ @@ -296,6 +485,7 @@ static int exofs_fill_super(struct super_block *sb, void *data, int silent) struct osd_dev *od; /* Master device */ struct exofs_fscb fscb; /*on-disk superblock info */ struct osd_obj_id obj; + unsigned table_count; int ret; sbi = kzalloc(sizeof(*sbi), GFP_KERNEL); @@ -309,7 +499,8 @@ static int exofs_fill_super(struct super_block *sb, void *data, int silent) goto free_sbi; } - sbi->s_dev = od; + sbi->s_ods[0] = od; + sbi->s_numdevs = 1; sbi->s_pid = opts->pid; sbi->s_timeout = opts->timeout; @@ -342,11 +533,24 @@ static int exofs_fill_super(struct super_block *sb, void *data, int silent) ret = -EINVAL; goto free_sbi; } + if (le32_to_cpu(fscb.s_version) != EXOFS_FSCB_VER) { + EXOFS_ERR("ERROR: Bad FSCB version expected-%d got-%d\n", + EXOFS_FSCB_VER, le32_to_cpu(fscb.s_version)); + ret = -EINVAL; + goto free_sbi; + } /* start generation numbers from a random point */ get_random_bytes(&sbi->s_next_generation, sizeof(u32)); spin_lock_init(&sbi->s_next_gen_lock); + table_count = le64_to_cpu(fscb.s_dev_table_count); + if (table_count) { + ret = exofs_read_lookup_dev_table(&sbi, table_count); + if (unlikely(ret)) + goto free_sbi; + } + /* set up operation vectors */ sb->s_fs_info = sbi; sb->s_op = &exofs_sops; @@ -374,14 +578,14 @@ static int exofs_fill_super(struct super_block *sb, void *data, int silent) goto free_sbi; } - _exofs_print_device("Mounting", opts->dev_name, sbi->s_dev, sbi->s_pid); + _exofs_print_device("Mounting", opts->dev_name, sbi->s_ods[0], + sbi->s_pid); return 0; free_sbi: EXOFS_ERR("Unable to mount exofs on %s pid=0x%llx err=%d\n", opts->dev_name, sbi->s_pid, ret); - osduld_put_device(sbi->s_dev); /* NULL safe */ - kfree(sbi); + exofs_free_sbi(sbi); return ret; } |