diff --git a/src/Makefile b/src/Makefile index 530f82f..a36775b 100644 --- a/src/Makefile +++ b/src/Makefile @@ -20,7 +20,7 @@ dm-writeboost-objs := \ dm-writeboost-daemon.o all: - $(MAKE) -C $(KERNEL_TREE) M=$(PWD) modules + $(MAKE) -C $(KERNEL_TREE) M=$(CURDIR) modules clean: - $(MAKE) -C $(KERNEL_TREE) M=$(PWD) clean + $(MAKE) -C $(KERNEL_TREE) M=$(CURDIR) clean diff --git a/src/dm-writeboost-daemon.c b/src/dm-writeboost-daemon.c index 0a142c1..1b9d32e 100644 --- a/src/dm-writeboost-daemon.c +++ b/src/dm-writeboost-daemon.c @@ -115,7 +115,7 @@ static void do_flush_proc(struct wb_device *wb) }; region = (struct dm_io_region) { .bdev = wb->cache_dev->bdev, - .sector = seg->start_sector, + .sector = segment_start_sector(wb, seg), .count = (seg->length + 1) << 3, }; @@ -297,7 +297,7 @@ static int fill_writeback_seg(struct wb_device *wb, struct writeback_segment *wr }; struct dm_io_region region_r = { .bdev = wb->cache_dev->bdev, - .sector = seg->start_sector + (1 << 3), /* Header excluded */ + .sector = segment_start_sector(wb, seg) + (1 << 3), /* Header excluded */ .count = seg->length << 3, }; @@ -317,14 +317,14 @@ static void prepare_writeback_ios(struct wb_device *wb, struct writeback_segment for (i = 0; i < seg->length; i++) { struct writeback_io *writeback_io; - struct metablock *mb = seg->mb_array + i; - struct dirtiness dirtiness = read_mb_dirtiness(wb, seg, mb); + u32 cache_idx = segment_start_idx(wb, seg) + i; + struct dirtiness dirtiness = read_mb_dirtiness(wb, seg, cache_idx); ASSERT(dirtiness.data_bits > 0); if (!dirtiness.is_dirty) continue; writeback_io = writeback_seg->ios + i; - writeback_io->sector = mb->sector; + writeback_io->sector = cache_sector(wb, cache_idx); writeback_io->id = seg->id; /* writeback_io->data is already set */ writeback_io->data_bits = dirtiness.data_bits; @@ -338,8 +338,9 @@ void mark_clean_seg(struct wb_device *wb, struct segment_header *seg) { u8 i; for (i = 0; i < seg->length; i++) { - struct metablock *mb = seg->mb_array + i; - if (mark_clean_mb(wb, mb)) + u32 cache_idx = segment_start_idx(wb, seg) + i; + + if (mark_clean_mb(wb, cache_idx)) dec_nr_dirty_caches(wb); } } diff --git a/src/dm-writeboost-metadata.c b/src/dm-writeboost-metadata.c index d5ee853..129464e 100644 --- a/src/dm-writeboost-metadata.c +++ b/src/dm-writeboost-metadata.c @@ -17,293 +17,276 @@ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */ +#include + #include "dm-writeboost.h" #include "dm-writeboost-metadata.h" #include "dm-writeboost-daemon.h" /*----------------------------------------------------------------------------*/ -struct large_array { - u64 nr_elems; - u32 elemsize; - void *data; -}; - -static struct large_array *large_array_alloc(u32 elemsize, u64 nr_elems) +/* + * Segment locations are derived from their position in the fixed RAM array. + * Cache indices remain stable for the lifetime of the target. + */ +static u32 segment_index(struct wb_device *wb, struct segment_header *seg) { - struct large_array *arr = kmalloc(sizeof(*arr), GFP_KERNEL); - if (!arr) { - DMERR("Failed to allocate arr"); - return NULL; - } - - arr->elemsize = elemsize; - arr->nr_elems = nr_elems; - - arr->data = vmalloc(elemsize * nr_elems); - if (!arr->data) { - DMERR("Failed to allocate data"); - goto bad_alloc_data; - } - - return arr; - -bad_alloc_data: - kfree(arr); - return NULL; + return seg - wb->segment_header_array; } -static void large_array_free(struct large_array *arr) +u32 segment_start_idx(struct wb_device *wb, struct segment_header *seg) { - vfree(arr->data); - kfree(arr); + return segment_index(wb, seg) * wb->nr_caches_inseg; } -static void *large_array_at(struct large_array *arr, u64 i) +static sector_t calc_segment_header_start(struct wb_device *wb, u32 k) { - return arr->data + arr->elemsize * i; + return ((sector_t)1 << 11) + ((sector_t)k << SEGMENT_SIZE_ORDER); } -/*----------------------------------------------------------------------------*/ - -/* - * Get the in-core metablock of the given index. - */ -static struct metablock *mb_at(struct wb_device *wb, u32 idx) +sector_t segment_start_sector(struct wb_device *wb, struct segment_header *seg) { - u32 idx_inseg; - u32 seg_idx = div_u64_rem(idx, wb->nr_caches_inseg, &idx_inseg); - struct segment_header *seg = large_array_at(wb->segment_header_array, seg_idx); - return seg->mb_array + idx_inseg; + return calc_segment_header_start(wb, segment_index(wb, seg)); } -static void mb_array_empty_init(struct wb_device *wb) +static u64 calc_nr_segments(struct dm_dev *dev, struct wb_device *wb) { - u32 i; - for (i = 0; i < wb->nr_caches; i++) { - struct metablock *mb = mb_at(wb, i); - INIT_HLIST_NODE(&mb->ht_list); + sector_t devsize = dm_devsize(dev); - mb->idx = i; - mb->dirtiness.data_bits = 0; - mb->dirtiness.is_dirty = false; - } + if (devsize <= (1 << 11)) + return 0; + return div_u64(devsize - (1 << 11), 1 << SEGMENT_SIZE_ORDER); } -/* - * Calc the starting sector of the k-th segment - */ -static sector_t calc_segment_header_start(struct wb_device *wb, u32 k) +u8 mb_idx_inseg(struct wb_device *wb, u32 cache_idx) { - return (1 << 11) + (1 << SEGMENT_SIZE_ORDER) * k; -} + u32 idx_inseg; -static u32 calc_nr_segments(struct dm_dev *dev, struct wb_device *wb) -{ - sector_t devsize = dm_devsize(dev); - return div_u64(devsize - (1 << 11), 1 << SEGMENT_SIZE_ORDER); + div_u64_rem(cache_idx, wb->nr_caches_inseg, &idx_inseg); + return idx_inseg; } -/* - * Get the relative index in a segment of the mb_idx-th metablock - */ -u8 mb_idx_inseg(struct wb_device *wb, u32 mb_idx) +sector_t calc_mb_start_sector(struct wb_device *wb, struct segment_header *seg, + u32 cache_idx) { - u32 tmp32; - div_u64_rem(mb_idx, wb->nr_caches_inseg, &tmp32); - return tmp32; + return segment_start_sector(wb, seg) + + ((1 + mb_idx_inseg(wb, cache_idx)) << 3); } -/* - * Calc the starting sector of the mb_idx-th cache block - */ -sector_t calc_mb_start_sector(struct wb_device *wb, struct segment_header *seg, u32 mb_idx) +struct segment_header *cache_idx_to_seg(struct wb_device *wb, u32 cache_idx) { - return seg->start_sector + ((1 + mb_idx_inseg(wb, mb_idx)) << 3); -} + u32 seg_idx = div_u64(cache_idx, wb->nr_caches_inseg); -/* - * Get the segment that contains the passed mb - */ -struct segment_header *mb_to_seg(struct wb_device *wb, struct metablock *mb) -{ - struct segment_header *seg; - seg = ((void *) mb) - - mb_idx_inseg(wb, mb->idx) * sizeof(struct metablock) - - sizeof(struct segment_header); - return seg; + return &wb->segment_header_array[seg_idx]; } -bool is_on_buffer(struct wb_device *wb, u32 mb_idx) +bool is_on_buffer(struct wb_device *wb, u32 cache_idx) { - u32 start = wb->current_seg->start_idx; - if (mb_idx < start) - return false; + u32 start = segment_start_idx(wb, wb->current_seg); - if (mb_idx >= (start + wb->nr_caches_inseg)) - return false; - - return true; + return cache_idx >= start && cache_idx - start < wb->nr_caches_inseg; } static u32 segment_id_to_idx(struct wb_device *wb, u64 id) { u32 idx; + div_u64_rem(id - 1, wb->nr_segments, &idx); return idx; } -static struct segment_header *segment_at(struct wb_device *wb, u32 k) -{ - return large_array_at(wb->segment_header_array, k); -} - -/* - * Get the segment from the segment id. - * The index of the segment is calculated from the segment id. - */ struct segment_header *get_segment_header_by_id(struct wb_device *wb, u64 id) { - return segment_at(wb, segment_id_to_idx(wb, id)); + return &wb->segment_header_array[segment_id_to_idx(wb, id)]; } /*----------------------------------------------------------------------------*/ static int init_segment_header_array(struct wb_device *wb) { - u32 segment_idx; + u32 i; - wb->segment_header_array = large_array_alloc( - sizeof(struct segment_header) + - sizeof(struct metablock) * wb->nr_caches_inseg, - wb->nr_segments); - if (!wb->segment_header_array) { - DMERR("Failed to allocate segment_header_array"); + BUILD_BUG_ON(sizeof(struct segment_header) != 32); + BUILD_BUG_ON(sizeof(struct wb_map_entry) != 16); + BUILD_BUG_ON((1 << (SEGMENT_SIZE_ORDER - 3)) > 128); + if (wb->nr_segments > SZ_MAX / sizeof(*wb->segment_header_array)) + return -EOVERFLOW; + wb->segment_header_array = vmalloc((size_t)wb->nr_segments * + sizeof(*wb->segment_header_array)); + if (!wb->segment_header_array) return -ENOMEM; - } - for (segment_idx = 0; segment_idx < wb->nr_segments; segment_idx++) { - struct segment_header *seg = large_array_at(wb->segment_header_array, segment_idx); + for (i = 0; i < wb->nr_segments; i++) { + struct segment_header *seg = &wb->segment_header_array[i]; seg->id = 0; seg->length = 0; atomic_set(&seg->nr_inflight_ios, 0); - - /* Const values */ - seg->start_idx = wb->nr_caches_inseg * segment_idx; - seg->start_sector = calc_segment_header_start(wb, segment_idx); + bitmap_zero(seg->dirty_bits, 128); } - - mb_array_empty_init(wb); - return 0; } -static void free_segment_header_array(struct wb_device *wb) -{ - large_array_free(wb->segment_header_array); -} - /*----------------------------------------------------------------------------*/ -struct ht_head { - struct hlist_head ht_list; -}; - -static int ht_empty_init(struct wb_device *wb) +/* + * The caller serializes map operations and removes the old slot for an LBA + * before registering a different slot. Removal preserves physical metadata. + */ +static u32 wb_map_bucket(const struct wb_map *map, u64 sector) { - u32 idx; - size_t i, nr_heads; - struct large_array *arr; - - wb->htsize = wb->nr_caches; - nr_heads = wb->htsize + 1; - arr = large_array_alloc(sizeof(struct ht_head), nr_heads); - if (!arr) { - DMERR("Failed to allocate htable"); - return -ENOMEM; - } + u32 bucket; - wb->htable = arr; + div_u64_rem(sector >> 3, map->nr_buckets, &bucket); + return bucket; +} - for (i = 0; i < nr_heads; i++) { - struct ht_head *hd = large_array_at(arr, i); - INIT_HLIST_HEAD(&hd->ht_list); - } +static int wb_map_init(struct wb_map *map, struct wb_map_entry *entries, + u32 nr_entries, u32 *heads, u32 nr_buckets) +{ + u32 i; + + if (!map || !entries || !nr_entries || !heads || !nr_buckets) + return -EINVAL; - wb->null_head = large_array_at(wb->htable, wb->htsize); + map->entries = entries; + map->heads = heads; + map->nr_entries = nr_entries; + map->nr_buckets = nr_buckets; - for (idx = 0; idx < wb->nr_caches; idx++) { - struct metablock *mb = mb_at(wb, idx); - hlist_add_head(&mb->ht_list, &wb->null_head->ht_list); + for (i = 0; i < nr_entries; i++) { + entries[i].sector = 0; + entries[i].next = WB_MAP_NONE; + entries[i].prev = i; } + for (i = 0; i < nr_buckets; i++) + heads[i] = WB_MAP_NONE; return 0; } -static void free_ht(struct wb_device *wb) +static u32 wb_map_lookup(const struct wb_map *map, u64 sector) { - large_array_free(wb->htable); + u32 idx = map->heads[wb_map_bucket(map, sector)]; + + while (idx != WB_MAP_NONE) { + const struct wb_map_entry *entry = &map->entries[idx]; + + if (entry->sector == sector) + return idx; + idx = entry->next; + } + return WB_MAP_NONE; } -struct ht_head *ht_get_head(struct wb_device *wb, struct lookup_key *key) +static void wb_map_remove(struct wb_map *map, u32 idx) { - u32 idx; - div_u64_rem(key->sector >> 3, wb->htsize, &idx); - return large_array_at(wb->htable, idx); + struct wb_map_entry *entry = &map->entries[idx]; + u32 prev = entry->prev; + u32 next = entry->next; + + /* A self-reference marks an entry outside the search index. */ + if (prev == idx) + return; + + if (prev == WB_MAP_NONE) + map->heads[wb_map_bucket(map, entry->sector)] = next; + else + map->entries[prev].next = next; + if (next != WB_MAP_NONE) + map->entries[next].prev = prev; + + entry->next = WB_MAP_NONE; + entry->prev = idx; } -static bool mb_hit(struct metablock *mb, struct lookup_key *key) +static void wb_map_insert(struct wb_map *map, u32 idx, u64 sector) { - return mb->sector == key->sector; + struct wb_map_entry *entry = &map->entries[idx]; + u32 bucket; + u32 old; + + wb_map_remove(map, idx); + bucket = wb_map_bucket(map, sector); + old = map->heads[bucket]; + entry->sector = sector; + entry->next = old; + entry->prev = WB_MAP_NONE; + if (old != WB_MAP_NONE) + map->entries[old].prev = idx; + map->heads[bucket] = idx; } -/* - * Remove the metablock from the hashtable and link the orphan to the null head. - */ -void ht_del(struct wb_device *wb, struct metablock *mb) +static int ht_empty_init(struct wb_device *wb) { - struct ht_head *null_head; + struct wb_map_entry *entries; + u32 *heads; + /* Four slots per bucket at full occupancy; never resize during I/O. */ + u32 nr_buckets = max_t(u32, 1, wb->nr_caches / 4 + !!(wb->nr_caches % 4)); + int err; - hlist_del(&mb->ht_list); + if (wb->nr_caches > SZ_MAX / sizeof(*entries) || + nr_buckets > SZ_MAX / sizeof(*heads)) + return -EOVERFLOW; + + entries = vmalloc((size_t)wb->nr_caches * sizeof(*entries)); + if (!entries) + return -ENOMEM; + heads = vmalloc((size_t)nr_buckets * sizeof(*heads)); + if (!heads) { + err = -ENOMEM; + goto bad_heads; + } + wb->data_bits = vzalloc(wb->nr_caches); + if (!wb->data_bits) { + err = -ENOMEM; + goto bad_data_bits; + } + err = wb_map_init(&wb->map, entries, wb->nr_caches, heads, nr_buckets); + if (err) + goto bad_map; + return 0; - null_head = wb->null_head; - hlist_add_head(&mb->ht_list, &null_head->ht_list); +bad_map: + vfree(wb->data_bits); +bad_data_bits: + vfree(heads); +bad_heads: + vfree(entries); + return err; } -void ht_register(struct wb_device *wb, struct ht_head *head, - struct metablock *mb, struct lookup_key *key) +static void free_ht(struct wb_device *wb) { - hlist_del(&mb->ht_list); - hlist_add_head(&mb->ht_list, &head->ht_list); + vfree(wb->data_bits); + vfree(wb->map.heads); + vfree(wb->map.entries); +} - BUG_ON(key->sector & 7); // should be 4KB aligned - mb->sector = key->sector; -}; +u32 ht_lookup(struct wb_device *wb, struct lookup_key *key) +{ + return wb_map_lookup(&wb->map, key->sector); +} -struct metablock *ht_lookup(struct wb_device *wb, struct ht_head *head, - struct lookup_key *key) +void ht_del(struct wb_device *wb, u32 cache_idx) { - struct metablock *mb, *found = NULL; - hlist_for_each_entry(mb, &head->ht_list, ht_list) { - if (mb_hit(mb, key)) { - found = mb; - break; - } - } - return found; + /* Keep the physical slot's LBA and mask for writeback and recovery. */ + wb_map_remove(&wb->map, cache_idx); +} + +void ht_register(struct wb_device *wb, u32 cache_idx, struct lookup_key *key) +{ + ASSERT(!(key->sector & 7)); + wb_map_insert(&wb->map, cache_idx, key->sector); } -/* - * Remove all the metablock in the segment from the lookup table. - */ void discard_caches_inseg(struct wb_device *wb, struct segment_header *seg) { + u32 start = segment_start_idx(wb, seg); u8 i; - for (i = 0; i < wb->nr_caches_inseg; i++) { - struct metablock *mb = seg->mb_array + i; - ht_del(wb, mb); - } + + for (i = 0; i < wb->nr_caches_inseg; i++) + ht_del(wb, start + i); } /*----------------------------------------------------------------------------*/ @@ -694,7 +677,7 @@ static int read_whole_segment(void *buf, struct wb_device *wb, }; struct dm_io_region region = { .bdev = wb->cache_dev->bdev, - .sector = seg->start_sector, + .sector = segment_start_sector(wb, seg), .count = 1 << SEGMENT_SIZE_ORDER, }; return wb_io(&io_req, 1, ®ion, NULL, false); @@ -717,14 +700,15 @@ void prepare_segment_header_device(void *rambuffer, struct segment_header_device *dest = rambuffer; u32 i; - ASSERT((src->length) == (wb->cursor - src->start_idx)); + ASSERT((src->length) == (wb->cursor - segment_start_idx(wb, src))); for (i = 0; i < src->length; i++) { - struct metablock *mb = src->mb_array + i; + u32 cache_idx = segment_start_idx(wb, src) + i; struct metablock_device *mbdev = dest->mbarr + i; + struct dirtiness dirtiness = read_mb_dirtiness(wb, src, cache_idx); - mbdev->sector = cpu_to_le64((u64)mb->sector); - mbdev->dirty_bits = mb->dirtiness.is_dirty ? mb->dirtiness.data_bits : 0; + mbdev->sector = cpu_to_le64(cache_sector(wb, cache_idx)); + mbdev->dirty_bits = dirtiness.is_dirty ? dirtiness.data_bits : 0; } dest->id = cpu_to_le64(src->id); @@ -741,21 +725,21 @@ static int apply_metablock_device(struct wb_device *wb, struct segment_header *s struct segment_header_device *src, u8 i) { struct lookup_key key; - struct ht_head *head; - struct metablock *found = NULL, *mb = seg->mb_array + i; + u32 found, cache_idx = segment_start_idx(wb, seg) + i; struct metablock_device *mbdev = src->mbarr + i; - mb->sector = le64_to_cpu(mbdev->sector); - - mb->dirtiness.data_bits = mbdev->dirty_bits ? mbdev->dirty_bits : 255; - mb->dirtiness.is_dirty = mbdev->dirty_bits ? true : false; + wb->map.entries[cache_idx].sector = le64_to_cpu(mbdev->sector); + wb->data_bits[cache_idx] = mbdev->dirty_bits ? mbdev->dirty_bits : 255; + if (mbdev->dirty_bits) + __set_bit(i, seg->dirty_bits); + else + __clear_bit(i, seg->dirty_bits); key = (struct lookup_key) { - .sector = mb->sector, + .sector = cache_sector(wb, cache_idx), }; - head = ht_get_head(wb, &key); - found = ht_lookup(wb, head, &key); - if (found) { + found = ht_lookup(wb, &key); + if (found != WB_MAP_NONE) { int err = 0; u8 i; struct write_io wio; @@ -767,7 +751,8 @@ static int apply_metablock_device(struct wb_device *wb, struct segment_header *s .data = buf, .data_bits = 0, }; - err = prepare_overwrite(wb, mb_to_seg(wb, found), found, &wio, mb->dirtiness.data_bits); + err = prepare_overwrite(wb, cache_idx_to_seg(wb, found), found, + &wio, wb->data_bits[cache_idx]); if (err) goto fail_out; @@ -786,7 +771,7 @@ static int apply_metablock_device(struct wb_device *wb, struct segment_header *s }; region = (struct dm_io_region) { .bdev = wb->backing_dev->bdev, - .sector = mb->sector + i, + .sector = cache_sector(wb, cache_idx) + i, .count = 1, }; err = wb_io(&io_req, 1, ®ion, NULL, true); @@ -800,9 +785,9 @@ static int apply_metablock_device(struct wb_device *wb, struct segment_header *s return err; } - ht_register(wb, head, mb, &key); + ht_register(wb, cache_idx, &key); - if (mb->dirtiness.is_dirty) + if (test_bit(i, seg->dirty_bits)) inc_nr_dirty_caches(wb); return 0; @@ -837,7 +822,7 @@ static int read_segment_header(void *buf, struct wb_device *wb, }; struct dm_io_region region = { .bdev = wb->cache_dev->bdev, - .sector = seg->start_sector, + .sector = segment_start_sector(wb, seg), .count = 8, }; return wb_io(&io_req, 1, ®ion, NULL, false); @@ -859,7 +844,7 @@ static int do_find_max_id(struct wb_device *wb, u64 *max_id) *max_id = 0; for (k = 0; k < wb->nr_segments; k++) { - struct segment_header *seg = segment_at(wb, k); + struct segment_header *seg = &wb->segment_header_array[k]; struct segment_header_device *header; err = read_segment_header(buf, wb, seg); if (err) @@ -920,7 +905,7 @@ static int do_apply_valid_segments(struct wb_device *wb, u64 *max_id) for (i = start_idx; i < (start_idx + wb->nr_segments); i++) { u32 actual, expected, k; div_u64_rem(i, wb->nr_segments, &k); - seg = segment_at(wb, k); + seg = &wb->segment_header_array[k]; err = read_whole_segment(rambuf, wb, seg); if (err) @@ -1214,7 +1199,7 @@ static int init_metadata(struct wb_device *wb) return err; bad_alloc_ht: - free_segment_header_array(wb); + vfree(wb->segment_header_array); bad_alloc_segment_header_array: return err; } @@ -1222,7 +1207,7 @@ static int init_metadata(struct wb_device *wb) static void free_metadata(struct wb_device *wb) { free_ht(wb); - free_segment_header_array(wb); + vfree(wb->segment_header_array); } static int init_writeback_daemon(struct wb_device *wb) @@ -1313,9 +1298,14 @@ static int init_data_synchronizer(struct wb_device *wb) int resume_cache(struct wb_device *wb) { int err = 0; + u64 nr_segments = calc_nr_segments(wb->cache_dev, wb); - wb->nr_segments = calc_nr_segments(wb->cache_dev, wb); wb->nr_caches_inseg = (1 << (SEGMENT_SIZE_ORDER - 3)) - 1; + if (!nr_segments || nr_segments > (u32)~0U / wb->nr_caches_inseg) { + DMERR("Cache capacity does not fit 32-bit cache indices"); + return -EINVAL; + } + wb->nr_segments = nr_segments; wb->nr_caches = wb->nr_segments * wb->nr_caches_inseg; err = init_devices(wb); diff --git a/src/dm-writeboost-metadata.h b/src/dm-writeboost-metadata.h index 8246537..cd28a3d 100644 --- a/src/dm-writeboost-metadata.h +++ b/src/dm-writeboost-metadata.h @@ -28,7 +28,14 @@ struct rambuffer *get_rambuffer_by_id(struct wb_device *wb, u64 id); sector_t calc_mb_start_sector(struct wb_device *, struct segment_header *, u32 mb_idx); u8 mb_idx_inseg(struct wb_device *, u32 mb_idx); -struct segment_header *mb_to_seg(struct wb_device *, struct metablock *); +struct segment_header *cache_idx_to_seg(struct wb_device *wb, u32 cache_idx); +u32 segment_start_idx(struct wb_device *wb, struct segment_header *seg); +sector_t segment_start_sector(struct wb_device *wb, struct segment_header *seg); +static inline sector_t cache_sector(struct wb_device *wb, u32 cache_idx) +{ + return wb->map.entries[cache_idx].sector; +} + bool is_on_buffer(struct wb_device *, u32 mb_idx); /*----------------------------------------------------------------------------*/ @@ -37,13 +44,9 @@ struct lookup_key { sector_t sector; }; -struct ht_head; -struct ht_head *ht_get_head(struct wb_device *, struct lookup_key *); -struct metablock *ht_lookup(struct wb_device *, - struct ht_head *, struct lookup_key *); -void ht_register(struct wb_device *, struct ht_head *, - struct metablock *, struct lookup_key *); -void ht_del(struct wb_device *, struct metablock *); +u32 ht_lookup(struct wb_device *wb, struct lookup_key *key); +void ht_register(struct wb_device *wb, u32 cache_idx, struct lookup_key *key); +void ht_del(struct wb_device *wb, u32 cache_idx); void discard_caches_inseg(struct wb_device *, struct segment_header *); /*----------------------------------------------------------------------------*/ diff --git a/src/dm-writeboost-target.c b/src/dm-writeboost-target.c index b12970d..5e84add 100644 --- a/src/dm-writeboost-target.c +++ b/src/dm-writeboost-target.c @@ -195,10 +195,9 @@ static void wake_up_active_wq(wait_queue_head_t *wq) static u8 count_dirty_caches_remained(struct segment_header *seg) { u8 i, count = 0; - struct metablock *mb; + for (i = 0; i < seg->length; i++) { - mb = seg->mb_array + i; - if (mb->dirtiness.is_dirty) + if (test_bit(i, seg->dirty_bits)) count++; } return count; @@ -217,31 +216,35 @@ void dec_nr_dirty_caches(struct wb_device *wb) wake_up_interruptible(&wb->wait_drop_caches); } -static bool taint_mb(struct wb_device *wb, struct metablock *mb, u8 data_bits) +static bool taint_mb(struct wb_device *wb, u32 cache_idx, u8 data_bits) { + struct segment_header *seg = cache_idx_to_seg(wb, cache_idx); + u8 idx_inseg = mb_idx_inseg(wb, cache_idx); unsigned long flags; bool flipped = false; ASSERT(data_bits > 0); spin_lock_irqsave(&wb->mb_lock, flags); - if (!mb->dirtiness.is_dirty) { - mb->dirtiness.is_dirty = true; + if (!test_bit(idx_inseg, seg->dirty_bits)) { + __set_bit(idx_inseg, seg->dirty_bits); flipped = true; } - mb->dirtiness.data_bits |= data_bits; + wb->data_bits[cache_idx] |= data_bits; spin_unlock_irqrestore(&wb->mb_lock, flags); return flipped; } -bool mark_clean_mb(struct wb_device *wb, struct metablock *mb) +bool mark_clean_mb(struct wb_device *wb, u32 cache_idx) { + struct segment_header *seg = cache_idx_to_seg(wb, cache_idx); + u8 idx_inseg = mb_idx_inseg(wb, cache_idx); unsigned long flags; bool flipped = false; spin_lock_irqsave(&wb->mb_lock, flags); - if (mb->dirtiness.is_dirty) { - mb->dirtiness.is_dirty = false; + if (test_bit(idx_inseg, seg->dirty_bits)) { + __clear_bit(idx_inseg, seg->dirty_bits); flipped = true; } spin_unlock_irqrestore(&wb->mb_lock, flags); @@ -250,16 +253,17 @@ bool mark_clean_mb(struct wb_device *wb, struct metablock *mb) } /* - * Read the dirtiness of a metablock at the moment. + * Read the dirtiness of a cache block at the moment. */ struct dirtiness read_mb_dirtiness(struct wb_device *wb, struct segment_header *seg, - struct metablock *mb) + u32 cache_idx) { unsigned long flags; struct dirtiness retval; spin_lock_irqsave(&wb->mb_lock, flags); - retval = mb->dirtiness; + retval.is_dirty = test_bit(mb_idx_inseg(wb, cache_idx), seg->dirty_bits); + retval.data_bits = wb->data_bits[cache_idx]; spin_unlock_irqrestore(&wb->mb_lock, flags); return retval; @@ -269,7 +273,7 @@ struct dirtiness read_mb_dirtiness(struct wb_device *wb, struct segment_header * void cursor_init(struct wb_device *wb) { - wb->cursor = wb->current_seg->start_idx; + wb->cursor = segment_start_idx(wb, wb->current_seg); wb->current_seg->length = 0; } @@ -361,7 +365,7 @@ static void __acquire_new_seg(struct wb_device *wb, u64 id) /* * Acquire the new segment and RAM buffer for the following writes. * Guarantees all dirty caches in the segments are written back and - * all metablocks in it are invalidated (Linked to null head). + * all cache blocks in it are removed from the lookup index. */ void acquire_new_seg(struct wb_device *wb, u64 id) { @@ -519,11 +523,10 @@ static void copy_to_bio_payload(struct bio *bio, void *buf, u8 copy_bits) /*----------------------------------------------------------------------------*/ struct lookup_result { - struct ht_head *head; /* Lookup head used */ struct lookup_key key; /* Lookup key used */ struct segment_header *found_seg; - struct metablock *found_mb; + u32 found_idx; bool found; /* Cache hit? */ bool on_buffer; /* Is the metablock found on the RAM buffer? */ @@ -538,19 +541,17 @@ static void cache_lookup(struct wb_device *wb, struct bio *bio, struct lookup_re res->key = (struct lookup_key) { .sector = calc_cache_alignment(bio->bi_iter.bi_sector), }; - res->head = ht_get_head(wb, &res->key); - - res->found_mb = ht_lookup(wb, res->head, &res->key); - if (res->found_mb) { - res->found_seg = mb_to_seg(wb, res->found_mb); + res->found_idx = ht_lookup(wb, &res->key); + if (res->found_idx != WB_MAP_NONE) { + res->found_seg = cache_idx_to_seg(wb, res->found_idx); atomic_inc(&res->found_seg->nr_inflight_ios); } - res->found = (res->found_mb != NULL); + res->found = (res->found_idx != WB_MAP_NONE); res->on_buffer = false; if (res->found) - res->on_buffer = is_on_buffer(wb, res->found_mb->idx); + res->on_buffer = is_on_buffer(wb, res->found_idx); inc_stat(wb, bio_is_write(bio), res->found, res->on_buffer, bio_is_fullsize(bio)); } @@ -617,9 +618,9 @@ static int fill_payload_by_backing(struct wb_device *wb, struct bio *bio) * Get the reference to the 4KB-aligned data in RAM buffer. * Since it only takes the reference caller need not to free the pointer. */ -static void *ref_buffered_mb(struct wb_device *wb, struct metablock *mb) +static void *ref_buffered_mb(struct wb_device *wb, u32 cache_idx) { - sector_t offset = ((mb_idx_inseg(wb, mb->idx) + 1) << 3); + sector_t offset = ((mb_idx_inseg(wb, cache_idx) + 1) << 3); return wb->current_rambuf->data + (offset << 9); } @@ -628,7 +629,7 @@ static void *ref_buffered_mb(struct wb_device *wb, struct metablock *mb) * Caller should free the returned pointer after used by mempool_alloc(). */ static void *read_mb(struct wb_device *wb, struct segment_header *seg, - struct metablock *mb, u8 data_bits) + u32 cache_idx, u8 data_bits) { u8 i; void *result = mempool_alloc(wb->buf_8_pool, GFP_NOIO); @@ -653,7 +654,7 @@ static void *read_mb(struct wb_device *wb, struct segment_header *seg, region = (struct dm_io_region) { .bdev = wb->cache_dev->bdev, - .sector = calc_mb_start_sector(wb, seg, mb->idx) + i, + .sector = calc_mb_start_sector(wb, seg, cache_idx) + i, .count = 1, }; @@ -864,14 +865,12 @@ static void read_cache_cell_copy_data(struct wb_device *wb, struct bio *bio, uns */ static void inject_read_cache(struct wb_device *wb, struct read_cache_cell *cell) { - struct metablock *mb; - u32 _mb_idx_inseg; + u32 cache_idx, idx_inseg; struct segment_header *seg; struct lookup_key key = { .sector = cell->sector, }; - struct ht_head *head = ht_get_head(wb, &key); mutex_lock(&wb->io_lock); /* @@ -886,20 +885,20 @@ static void inject_read_cache(struct wb_device *wb, struct read_cache_cell *cell might_queue_current_buffer(wb); seg = wb->current_seg; - _mb_idx_inseg = mb_idx_inseg(wb, advance_cursor(wb)); + cache_idx = advance_cursor(wb); + idx_inseg = mb_idx_inseg(wb, cache_idx); /* * We should copy the cell data into the rambuf with lock held * otherwise subsequent write data may be written first and then overwritten by * the old data in the cell. */ - memcpy(wb->current_rambuf->data + ((_mb_idx_inseg + 1) << 12), cell->data, 1 << 12); + memcpy(wb->current_rambuf->data + ((idx_inseg + 1) << 12), cell->data, 1 << 12); - mb = seg->mb_array + _mb_idx_inseg; - ASSERT(!mb->dirtiness.is_dirty); - mb->dirtiness.data_bits = 255; + ASSERT(!test_bit(idx_inseg, seg->dirty_bits)); + wb->data_bits[cache_idx] = 255; - ht_register(wb, head, mb, &key); + ht_register(wb, cache_idx, &key); mutex_unlock(&wb->io_lock); @@ -1080,9 +1079,10 @@ static void memcpy_masked(void *to, u8 protect_bits, void *from, u8 copy_bits) } } -int prepare_overwrite(struct wb_device *wb, struct segment_header *seg, struct metablock *old_mb, struct write_io* wio, u8 overwrite_bits) +int prepare_overwrite(struct wb_device *wb, struct segment_header *seg, + u32 old_idx, struct write_io *wio, u8 overwrite_bits) { - struct dirtiness dirtiness = read_mb_dirtiness(wb, seg, old_mb); + struct dirtiness dirtiness = read_mb_dirtiness(wb, seg, old_idx); bool needs_merge_prev_cache = !(overwrite_bits == 255) || !(dirtiness.data_bits == 255); @@ -1098,7 +1098,7 @@ int prepare_overwrite(struct wb_device *wb, struct segment_header *seg, struct m wait_for_flushing(wb, seg->id); ASSERT(dirtiness.is_dirty); - buf = read_mb(wb, seg, old_mb, dirtiness.data_bits); + buf = read_mb(wb, seg, old_idx, dirtiness.data_bits); if (!buf) return -EIO; @@ -1108,10 +1108,10 @@ int prepare_overwrite(struct wb_device *wb, struct segment_header *seg, struct m mempool_free(buf, wb->buf_8_pool); } - if (mark_clean_mb(wb, old_mb)) + if (mark_clean_mb(wb, old_idx)) dec_nr_dirty_caches(wb); - ht_del(wb, old_mb); + ht_del(wb, old_idx); return 0; } @@ -1119,17 +1119,18 @@ int prepare_overwrite(struct wb_device *wb, struct segment_header *seg, struct m /* * Get a new place to write. */ -static struct metablock *prepare_new_write_pos(struct wb_device *wb) +static u32 prepare_new_write_pos(struct wb_device *wb) { - struct metablock *ret = wb->current_seg->mb_array + mb_idx_inseg(wb, advance_cursor(wb)); - ASSERT(!ret->dirtiness.is_dirty); - ret->dirtiness.data_bits = 0; - return ret; + u32 cache_idx = advance_cursor(wb); + + ASSERT(!test_bit(mb_idx_inseg(wb, cache_idx), wb->current_seg->dirty_bits)); + wb->data_bits[cache_idx] = 0; + return cache_idx; } -static void write_on_rambuffer(struct wb_device *wb, struct metablock *write_pos, struct write_io *wio) +static void write_on_rambuffer(struct wb_device *wb, u32 write_pos, struct write_io *wio) { - size_t mb_offset = (mb_idx_inseg(wb, write_pos->idx) + 1) << 12; + size_t mb_offset = (mb_idx_inseg(wb, write_pos) + 1) << 12; void *mb_data = wb->current_rambuf->data + mb_offset; if (wio->data_bits == 255) memcpy(mb_data, wio->data, 1 << 12); @@ -1141,7 +1142,7 @@ static int do_process_write(struct wb_device *wb, struct bio *bio) { int err = 0; - struct metablock *write_pos = NULL; + u32 write_pos = WB_MAP_NONE; struct lookup_result res; struct write_io wio; @@ -1155,10 +1156,11 @@ static int do_process_write(struct wb_device *wb, struct bio *bio) cache_lookup(wb, bio, &res); if (res.found) { if (unlikely(res.on_buffer)) { - write_pos = res.found_mb; + write_pos = res.found_idx; goto do_write; } else { - err = prepare_overwrite(wb, res.found_seg, res.found_mb, &wio, wio.data_bits); + err = prepare_overwrite(wb, res.found_seg, res.found_idx, + &wio, wio.data_bits); dec_inflight_ios(wb, res.found_seg); if (err) goto out; @@ -1171,13 +1173,13 @@ static int do_process_write(struct wb_device *wb, struct bio *bio) write_pos = prepare_new_write_pos(wb); do_write: - ASSERT(write_pos); + ASSERT(write_pos != WB_MAP_NONE); write_on_rambuffer(wb, write_pos, &wio); if (taint_mb(wb, write_pos, wio.data_bits)) inc_nr_dirty_caches(wb); - ht_register(wb, res.head, write_pos, &res.key); + ht_register(wb, write_pos, &res.key); out: mutex_unlock(&wb->io_lock); @@ -1251,7 +1253,7 @@ static int process_write_wa(struct wb_device *wb, struct bio *bio) cache_lookup(wb, bio, &res); if (res.found) { dec_inflight_ios(wb, res.found_seg); - ht_del(wb, res.found_mb); + ht_del(wb, res.found_idx); } might_cancel_read_cache_cell(wb, bio); mutex_unlock(&wb->io_lock); @@ -1363,14 +1365,15 @@ static int process_read(struct wb_device *wb, struct bio *bio) } } - dirtiness = read_mb_dirtiness(wb, res.found_seg, res.found_mb); + dirtiness = read_mb_dirtiness(wb, res.found_seg, res.found_idx); if (unlikely(res.on_buffer)) { int err = fill_payload_by_backing(wb, bio); if (err) goto read_buffered_mb_exit; if (dirtiness.is_dirty) - copy_to_bio_payload(bio, ref_buffered_mb(wb, res.found_mb), dirtiness.data_bits); + copy_to_bio_payload(bio, ref_buffered_mb(wb, res.found_idx), + dirtiness.data_bits); read_buffered_mb_exit: dec_inflight_ios(wb, res.found_seg); @@ -1395,7 +1398,7 @@ static int process_read(struct wb_device *wb, struct bio *bio) goto read_mb_exit; if (dirtiness.is_dirty) { - void *buf = read_mb(wb, res.found_seg, res.found_mb, dirtiness.data_bits); + void *buf = read_mb(wb, res.found_seg, res.found_idx, dirtiness.data_bits); if (!buf) { err = -EIO; goto read_mb_exit; @@ -1420,7 +1423,7 @@ static int process_read(struct wb_device *wb, struct bio *bio) pbd->seg = res.found_seg; bio_remap(bio, wb->cache_dev, - calc_mb_start_sector(wb, res.found_seg, res.found_mb->idx) + + calc_mb_start_sector(wb, res.found_seg, res.found_idx) + bio_calc_offset(bio)); return DM_MAPIO_REMAPPED; diff --git a/src/dm-writeboost.h b/src/dm-writeboost.h index cd3756a..3ceae23 100644 --- a/src/dm-writeboost.h +++ b/src/dm-writeboost.h @@ -36,6 +36,7 @@ #include #include #include +#include /* Preserve the old combined error bitmap for dm-io completion handlers. */ #if DM_IO_HAS_UNSUP @@ -130,33 +131,33 @@ struct segment_header_device { /*----------------------------------------------------------------------------*/ -struct dirtiness { - bool is_dirty; - u8 data_bits; -}; - -struct metablock { - sector_t sector; /* The original aligned address */ +#define WB_MAP_NONE ((u32)~0U) - u32 idx; /* Const. Index in the metablock array */ +/* The array index is the stable physical cache block number. */ +struct wb_map_entry { + u64 sector; + u32 next; + u32 prev; +}; - struct hlist_node ht_list; /* Linked to the hash table */ +struct wb_map { + struct wb_map_entry *entries; + u32 *heads; + u32 nr_entries; + u32 nr_buckets; +}; - struct dirtiness dirtiness; +struct dirtiness { + bool is_dirty; + u8 data_bits; }; #define SZ_MAX (~(size_t)0) struct segment_header { u64 id; /* Must be initialized to 0 */ - - u8 length; /* The number of valid metablocks */ - - u32 start_idx; /* Const */ - sector_t start_sector; /* Const */ - atomic_t nr_inflight_ios; - - struct metablock mb_array[0]; + u8 length; /* The number of valid cache blocks */ + DECLARE_BITMAP(dirty_bits, 128); /* 127 blocks and one unused bit */ }; /*----------------------------------------------------------------------------*/ @@ -306,7 +307,7 @@ struct wb_device { **********************/ u32 nr_segments; /* Const */ - struct large_array *segment_header_array; + struct segment_header *segment_header_array; /*--------------------------------------------------------------------*/ @@ -315,14 +316,8 @@ struct wb_device { ********************/ u32 nr_caches; /* Const */ - struct large_array *htable; - size_t htsize; /* Number of buckets in the hash table */ - - /* - * Our hashtable has one special bucket called null head. - * Orphan metablocks are linked to the null head. - */ - struct ht_head *null_head; + struct wb_map map; + u8 *data_bits; /*--------------------------------------------------------------------*/ @@ -482,9 +477,11 @@ void cursor_init(struct wb_device *); void flush_current_buffer(struct wb_device *); void inc_nr_dirty_caches(struct wb_device *); void dec_nr_dirty_caches(struct wb_device *); -bool mark_clean_mb(struct wb_device *, struct metablock *); -struct dirtiness read_mb_dirtiness(struct wb_device *, struct segment_header *, struct metablock *); -int prepare_overwrite(struct wb_device *, struct segment_header *, struct metablock *old_mb, struct write_io *, u8 overwrite_bits); +bool mark_clean_mb(struct wb_device *wb, u32 cache_idx); +struct dirtiness read_mb_dirtiness(struct wb_device *wb, struct segment_header *seg, + u32 cache_idx); +int prepare_overwrite(struct wb_device *wb, struct segment_header *seg, + u32 old_idx, struct write_io *wio, u8 overwrite_bits); /*----------------------------------------------------------------------------*/