git.rot13.org Git - linux/blob - drivers/lightnvm/pblk-rb.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2016 CNEX Labs
   4  * Initial release: Javier Gonzalez <javier@cnexlabs.com>
   5  *
   6  * Based upon the circular ringbuffer.
   7  *
   8  * This program is free software; you can redistribute it and/or
   9  * modify it under the terms of the GNU General Public License version
  10  * 2 as published by the Free Software Foundation.
  11  *
  12  * This program is distributed in the hope that it will be useful, but
  13  * WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * General Public License for more details.
  16  *
  17  * pblk-rb.c - pblk's write buffer
  18  */
  19
  20 #include <linux/circ_buf.h>
  21
  22 #include "pblk.h"
  23
  24 static DECLARE_RWSEM(pblk_rb_lock);
  25
  26 static void pblk_rb_data_free(struct pblk_rb *rb)
  27 {
  28         struct pblk_rb_pages *p, *t;
  29
  30         down_write(&pblk_rb_lock);
  31         list_for_each_entry_safe(p, t, &rb->pages, list) {
  32                 free_pages((unsigned long)page_address(p->pages), p->order);
  33                 list_del(&p->list);
  34                 kfree(p);
  35         }
  36         up_write(&pblk_rb_lock);
  37 }
  38
  39 void pblk_rb_free(struct pblk_rb *rb)
  40 {
  41         pblk_rb_data_free(rb);
  42         vfree(rb->entries);
  43 }
  44
  45 /*
  46  * pblk_rb_calculate_size -- calculate the size of the write buffer
  47  */
  48 static unsigned int pblk_rb_calculate_size(unsigned int nr_entries)
  49 {
  50         /* Alloc a write buffer that can at least fit 128 entries */
  51         return (1 << max(get_count_order(nr_entries), 7));
  52 }
  53
  54 /*
  55  * Initialize ring buffer. The data and metadata buffers must be previously
  56  * allocated and their size must be a power of two
  57  * (Documentation/core-api/circular-buffers.rst)
  58  */
  59 int pblk_rb_init(struct pblk_rb *rb, unsigned int size, unsigned int threshold,
  60                  unsigned int seg_size)
  61 {
  62         struct pblk *pblk = container_of(rb, struct pblk, rwb);
  63         struct pblk_rb_entry *entries;
  64         unsigned int init_entry = 0;
  65         unsigned int max_order = MAX_ORDER - 1;
  66         unsigned int power_size, power_seg_sz;
  67         unsigned int alloc_order, order, iter;
  68         unsigned int nr_entries;
  69
  70         nr_entries = pblk_rb_calculate_size(size);
  71         entries = vzalloc(array_size(nr_entries, sizeof(struct pblk_rb_entry)));
  72         if (!entries)
  73                 return -ENOMEM;
  74
  75         power_size = get_count_order(size);
  76         power_seg_sz = get_count_order(seg_size);
  77
  78         down_write(&pblk_rb_lock);
  79         rb->entries = entries;
  80         rb->seg_size = (1 << power_seg_sz);
  81         rb->nr_entries = (1 << power_size);
  82         rb->mem = rb->subm = rb->sync = rb->l2p_update = 0;
  83         rb->back_thres = threshold;
  84         rb->flush_point = EMPTY_ENTRY;
  85
  86         spin_lock_init(&rb->w_lock);
  87         spin_lock_init(&rb->s_lock);
  88
  89         INIT_LIST_HEAD(&rb->pages);
  90
  91         alloc_order = power_size;
  92         if (alloc_order >= max_order) {
  93                 order = max_order;
  94                 iter = (1 << (alloc_order - max_order));
  95         } else {
  96                 order = alloc_order;
  97                 iter = 1;
  98         }
  99
 100         do {
 101                 struct pblk_rb_entry *entry;
 102                 struct pblk_rb_pages *page_set;
 103                 void *kaddr;
 104                 unsigned long set_size;
 105                 int i;
 106
 107                 page_set = kmalloc(sizeof(struct pblk_rb_pages), GFP_KERNEL);
 108                 if (!page_set) {
 109                         up_write(&pblk_rb_lock);
 110                         vfree(entries);
 111                         return -ENOMEM;
 112                 }
 113
 114                 page_set->order = order;
 115                 page_set->pages = alloc_pages(GFP_KERNEL, order);
 116                 if (!page_set->pages) {
 117                         kfree(page_set);
 118                         pblk_rb_data_free(rb);
 119                         up_write(&pblk_rb_lock);
 120                         vfree(entries);
 121                         return -ENOMEM;
 122                 }
 123                 kaddr = page_address(page_set->pages);
 124
 125                 entry = &rb->entries[init_entry];
 126                 entry->data = kaddr;
 127                 entry->cacheline = pblk_cacheline_to_addr(init_entry++);
 128                 entry->w_ctx.flags = PBLK_WRITABLE_ENTRY;
 129
 130                 set_size = (1 << order);
 131                 for (i = 1; i < set_size; i++) {
 132                         entry = &rb->entries[init_entry];
 133                         entry->cacheline = pblk_cacheline_to_addr(init_entry++);
 134                         entry->data = kaddr + (i * rb->seg_size);
 135                         entry->w_ctx.flags = PBLK_WRITABLE_ENTRY;
 136                         bio_list_init(&entry->w_ctx.bios);
 137                 }
 138
 139                 list_add_tail(&page_set->list, &rb->pages);
 140                 iter--;
 141         } while (iter > 0);
 142         up_write(&pblk_rb_lock);
 143
 144 #ifdef CONFIG_NVM_PBLK_DEBUG
 145         atomic_set(&rb->inflight_flush_point, 0);
 146 #endif
 147
 148         /*
 149          * Initialize rate-limiter, which controls access to the write buffer
 150          * by user and GC I/O
 151          */
 152         pblk_rl_init(&pblk->rl, rb->nr_entries);
 153
 154         return 0;
 155 }
 156
 157 static void clean_wctx(struct pblk_w_ctx *w_ctx)
 158 {
 159         int flags;
 160
 161         flags = READ_ONCE(w_ctx->flags);
 162         WARN_ONCE(!(flags & PBLK_SUBMITTED_ENTRY),
 163                         "pblk: overwriting unsubmitted data\n");
 164
 165         /* Release flags on context. Protect from writes and reads */
 166         smp_store_release(&w_ctx->flags, PBLK_WRITABLE_ENTRY);
 167         pblk_ppa_set_empty(&w_ctx->ppa);
 168         w_ctx->lba = ADDR_EMPTY;
 169 }
 170
 171 #define pblk_rb_ring_count(head, tail, size) CIRC_CNT(head, tail, size)
 172 #define pblk_rb_ring_space(rb, head, tail, size) \
 173                                         (CIRC_SPACE(head, tail, size))
 174
 175 /*
 176  * Buffer space is calculated with respect to the back pointer signaling
 177  * synchronized entries to the media.
 178  */
 179 static unsigned int pblk_rb_space(struct pblk_rb *rb)
 180 {
 181         unsigned int mem = READ_ONCE(rb->mem);
 182         unsigned int sync = READ_ONCE(rb->sync);
 183
 184         return pblk_rb_ring_space(rb, mem, sync, rb->nr_entries);
 185 }
 186
 187 unsigned int pblk_rb_ptr_wrap(struct pblk_rb *rb, unsigned int p,
 188                               unsigned int nr_entries)
 189 {
 190         return (p + nr_entries) & (rb->nr_entries - 1);
 191 }
 192
 193 /*
 194  * Buffer count is calculated with respect to the submission entry signaling the
 195  * entries that are available to send to the media
 196  */
 197 unsigned int pblk_rb_read_count(struct pblk_rb *rb)
 198 {
 199         unsigned int mem = READ_ONCE(rb->mem);
 200         unsigned int subm = READ_ONCE(rb->subm);
 201
 202         return pblk_rb_ring_count(mem, subm, rb->nr_entries);
 203 }
 204
 205 unsigned int pblk_rb_sync_count(struct pblk_rb *rb)
 206 {
 207         unsigned int mem = READ_ONCE(rb->mem);
 208         unsigned int sync = READ_ONCE(rb->sync);
 209
 210         return pblk_rb_ring_count(mem, sync, rb->nr_entries);
 211 }
 212
 213 unsigned int pblk_rb_read_commit(struct pblk_rb *rb, unsigned int nr_entries)
 214 {
 215         unsigned int subm;
 216
 217         subm = READ_ONCE(rb->subm);
 218         /* Commit read means updating submission pointer */
 219         smp_store_release(&rb->subm, pblk_rb_ptr_wrap(rb, subm, nr_entries));
 220
 221         return subm;
 222 }
 223
 224 static int __pblk_rb_update_l2p(struct pblk_rb *rb, unsigned int to_update)
 225 {
 226         struct pblk *pblk = container_of(rb, struct pblk, rwb);
 227         struct pblk_line *line;
 228         struct pblk_rb_entry *entry;
 229         struct pblk_w_ctx *w_ctx;
 230         unsigned int user_io = 0, gc_io = 0;
 231         unsigned int i;
 232         int flags;
 233
 234         for (i = 0; i < to_update; i++) {
 235                 entry = &rb->entries[rb->l2p_update];
 236                 w_ctx = &entry->w_ctx;
 237
 238                 flags = READ_ONCE(entry->w_ctx.flags);
 239                 if (flags & PBLK_IOTYPE_USER)
 240                         user_io++;
 241                 else if (flags & PBLK_IOTYPE_GC)
 242                         gc_io++;
 243                 else
 244                         WARN(1, "pblk: unknown IO type\n");
 245
 246                 pblk_update_map_dev(pblk, w_ctx->lba, w_ctx->ppa,
 247                                                         entry->cacheline);
 248
 249                 line = pblk_ppa_to_line(pblk, w_ctx->ppa);
 250                 kref_put(&line->ref, pblk_line_put);
 251                 clean_wctx(w_ctx);
 252                 rb->l2p_update = pblk_rb_ptr_wrap(rb, rb->l2p_update, 1);
 253         }
 254
 255         pblk_rl_out(&pblk->rl, user_io, gc_io);
 256
 257         return 0;
 258 }
 259
 260 /*
 261  * When we move the l2p_update pointer, we update the l2p table - lookups will
 262  * point to the physical address instead of to the cacheline in the write buffer
 263  * from this moment on.
 264  */
 265 static int pblk_rb_update_l2p(struct pblk_rb *rb, unsigned int nr_entries,
 266                               unsigned int mem, unsigned int sync)
 267 {
 268         unsigned int space, count;
 269         int ret = 0;
 270
 271         lockdep_assert_held(&rb->w_lock);
 272
 273         /* Update l2p only as buffer entries are being overwritten */
 274         space = pblk_rb_ring_space(rb, mem, rb->l2p_update, rb->nr_entries);
 275         if (space > nr_entries)
 276                 goto out;
 277
 278         count = nr_entries - space;
 279         /* l2p_update used exclusively under rb->w_lock */
 280         ret = __pblk_rb_update_l2p(rb, count);
 281
 282 out:
 283         return ret;
 284 }
 285
 286 /*
 287  * Update the l2p entry for all sectors stored on the write buffer. This means
 288  * that all future lookups to the l2p table will point to a device address, not
 289  * to the cacheline in the write buffer.
 290  */
 291 void pblk_rb_sync_l2p(struct pblk_rb *rb)
 292 {
 293         unsigned int sync;
 294         unsigned int to_update;
 295
 296         spin_lock(&rb->w_lock);
 297
 298         /* Protect from reads and writes */
 299         sync = smp_load_acquire(&rb->sync);
 300
 301         to_update = pblk_rb_ring_count(sync, rb->l2p_update, rb->nr_entries);
 302         __pblk_rb_update_l2p(rb, to_update);
 303
 304         spin_unlock(&rb->w_lock);
 305 }
 306
 307 /*
 308  * Write @nr_entries to ring buffer from @data buffer if there is enough space.
 309  * Typically, 4KB data chunks coming from a bio will be copied to the ring
 310  * buffer, thus the write will fail if not all incoming data can be copied.
 311  *
 312  */
 313 static void __pblk_rb_write_entry(struct pblk_rb *rb, void *data,
 314                                   struct pblk_w_ctx w_ctx,
 315                                   struct pblk_rb_entry *entry)
 316 {
 317         memcpy(entry->data, data, rb->seg_size);
 318
 319         entry->w_ctx.lba = w_ctx.lba;
 320         entry->w_ctx.ppa = w_ctx.ppa;
 321 }
 322
 323 void pblk_rb_write_entry_user(struct pblk_rb *rb, void *data,
 324                               struct pblk_w_ctx w_ctx, unsigned int ring_pos)
 325 {
 326         struct pblk *pblk = container_of(rb, struct pblk, rwb);
 327         struct pblk_rb_entry *entry;
 328         int flags;
 329
 330         entry = &rb->entries[ring_pos];
 331         flags = READ_ONCE(entry->w_ctx.flags);
 332 #ifdef CONFIG_NVM_PBLK_DEBUG
 333         /* Caller must guarantee that the entry is free */
 334         BUG_ON(!(flags & PBLK_WRITABLE_ENTRY));
 335 #endif
 336
 337         __pblk_rb_write_entry(rb, data, w_ctx, entry);
 338
 339         pblk_update_map_cache(pblk, w_ctx.lba, entry->cacheline);
 340         flags = w_ctx.flags | PBLK_WRITTEN_DATA;
 341
 342         /* Release flags on write context. Protect from writes */
 343         smp_store_release(&entry->w_ctx.flags, flags);
 344 }
 345
 346 void pblk_rb_write_entry_gc(struct pblk_rb *rb, void *data,
 347                             struct pblk_w_ctx w_ctx, struct pblk_line *line,
 348                             u64 paddr, unsigned int ring_pos)
 349 {
 350         struct pblk *pblk = container_of(rb, struct pblk, rwb);
 351         struct pblk_rb_entry *entry;
 352         int flags;
 353
 354         entry = &rb->entries[ring_pos];
 355         flags = READ_ONCE(entry->w_ctx.flags);
 356 #ifdef CONFIG_NVM_PBLK_DEBUG
 357         /* Caller must guarantee that the entry is free */
 358         BUG_ON(!(flags & PBLK_WRITABLE_ENTRY));
 359 #endif
 360
 361         __pblk_rb_write_entry(rb, data, w_ctx, entry);
 362
 363         if (!pblk_update_map_gc(pblk, w_ctx.lba, entry->cacheline, line, paddr))
 364                 entry->w_ctx.lba = ADDR_EMPTY;
 365
 366         flags = w_ctx.flags | PBLK_WRITTEN_DATA;
 367
 368         /* Release flags on write context. Protect from writes */
 369         smp_store_release(&entry->w_ctx.flags, flags);
 370 }
 371
 372 static int pblk_rb_flush_point_set(struct pblk_rb *rb, struct bio *bio,
 373                                    unsigned int pos)
 374 {
 375         struct pblk_rb_entry *entry;
 376         unsigned int sync, flush_point;
 377
 378         pblk_rb_sync_init(rb, NULL);
 379         sync = READ_ONCE(rb->sync);
 380
 381         if (pos == sync) {
 382                 pblk_rb_sync_end(rb, NULL);
 383                 return 0;
 384         }
 385
 386 #ifdef CONFIG_NVM_PBLK_DEBUG
 387         atomic_inc(&rb->inflight_flush_point);
 388 #endif
 389
 390         flush_point = (pos == 0) ? (rb->nr_entries - 1) : (pos - 1);
 391         entry = &rb->entries[flush_point];
 392
 393         /* Protect flush points */
 394         smp_store_release(&rb->flush_point, flush_point);
 395
 396         if (bio)
 397                 bio_list_add(&entry->w_ctx.bios, bio);
 398
 399         pblk_rb_sync_end(rb, NULL);
 400
 401         return bio ? 1 : 0;
 402 }
 403
 404 static int __pblk_rb_may_write(struct pblk_rb *rb, unsigned int nr_entries,
 405                                unsigned int *pos)
 406 {
 407         unsigned int mem;
 408         unsigned int sync;
 409         unsigned int threshold;
 410
 411         sync = READ_ONCE(rb->sync);
 412         mem = READ_ONCE(rb->mem);
 413
 414         threshold = nr_entries + rb->back_thres;
 415
 416         if (pblk_rb_ring_space(rb, mem, sync, rb->nr_entries) < threshold)
 417                 return 0;
 418
 419         if (pblk_rb_update_l2p(rb, nr_entries, mem, sync))
 420                 return 0;
 421
 422         *pos = mem;
 423
 424         return 1;
 425 }
 426
 427 static int pblk_rb_may_write(struct pblk_rb *rb, unsigned int nr_entries,
 428                              unsigned int *pos)
 429 {
 430         if (!__pblk_rb_may_write(rb, nr_entries, pos))
 431                 return 0;
 432
 433         /* Protect from read count */
 434         smp_store_release(&rb->mem, pblk_rb_ptr_wrap(rb, *pos, nr_entries));
 435         return 1;
 436 }
 437
 438 void pblk_rb_flush(struct pblk_rb *rb)
 439 {
 440         struct pblk *pblk = container_of(rb, struct pblk, rwb);
 441         unsigned int mem = READ_ONCE(rb->mem);
 442
 443         if (pblk_rb_flush_point_set(rb, NULL, mem))
 444                 return;
 445
 446         pblk_write_kick(pblk);
 447 }
 448
 449 static int pblk_rb_may_write_flush(struct pblk_rb *rb, unsigned int nr_entries,
 450                                    unsigned int *pos, struct bio *bio,
 451                                    int *io_ret)
 452 {
 453         unsigned int mem;
 454
 455         if (!__pblk_rb_may_write(rb, nr_entries, pos))
 456                 return 0;
 457
 458         mem = pblk_rb_ptr_wrap(rb, *pos, nr_entries);
 459         *io_ret = NVM_IO_DONE;
 460
 461         if (bio->bi_opf & REQ_PREFLUSH) {
 462                 struct pblk *pblk = container_of(rb, struct pblk, rwb);
 463
 464                 atomic64_inc(&pblk->nr_flush);
 465                 if (pblk_rb_flush_point_set(&pblk->rwb, bio, mem))
 466                         *io_ret = NVM_IO_OK;
 467         }
 468
 469         /* Protect from read count */
 470         smp_store_release(&rb->mem, mem);
 471
 472         return 1;
 473 }
 474
 475 /*
 476  * Atomically check that (i) there is space on the write buffer for the
 477  * incoming I/O, and (ii) the current I/O type has enough budget in the write
 478  * buffer (rate-limiter).
 479  */
 480 int pblk_rb_may_write_user(struct pblk_rb *rb, struct bio *bio,
 481                            unsigned int nr_entries, unsigned int *pos)
 482 {
 483         struct pblk *pblk = container_of(rb, struct pblk, rwb);
 484         int io_ret;
 485
 486         spin_lock(&rb->w_lock);
 487         io_ret = pblk_rl_user_may_insert(&pblk->rl, nr_entries);
 488         if (io_ret) {
 489                 spin_unlock(&rb->w_lock);
 490                 return io_ret;
 491         }
 492
 493         if (!pblk_rb_may_write_flush(rb, nr_entries, pos, bio, &io_ret)) {
 494                 spin_unlock(&rb->w_lock);
 495                 return NVM_IO_REQUEUE;
 496         }
 497
 498         pblk_rl_user_in(&pblk->rl, nr_entries);
 499         spin_unlock(&rb->w_lock);
 500
 501         return io_ret;
 502 }
 503
 504 /*
 505  * Look at pblk_rb_may_write_user comment
 506  */
 507 int pblk_rb_may_write_gc(struct pblk_rb *rb, unsigned int nr_entries,
 508                          unsigned int *pos)
 509 {
 510         struct pblk *pblk = container_of(rb, struct pblk, rwb);
 511
 512         spin_lock(&rb->w_lock);
 513         if (!pblk_rl_gc_may_insert(&pblk->rl, nr_entries)) {
 514                 spin_unlock(&rb->w_lock);
 515                 return 0;
 516         }
 517
 518         if (!pblk_rb_may_write(rb, nr_entries, pos)) {
 519                 spin_unlock(&rb->w_lock);
 520                 return 0;
 521         }
 522
 523         pblk_rl_gc_in(&pblk->rl, nr_entries);
 524         spin_unlock(&rb->w_lock);
 525
 526         return 1;
 527 }
 528
 529 /*
 530  * Read available entries on rb and add them to the given bio. To avoid a memory
 531  * copy, a page reference to the write buffer is used to be added to the bio.
 532  *
 533  * This function is used by the write thread to form the write bio that will
 534  * persist data on the write buffer to the media.
 535  */
 536 unsigned int pblk_rb_read_to_bio(struct pblk_rb *rb, struct nvm_rq *rqd,
 537                                  unsigned int pos, unsigned int nr_entries,
 538                                  unsigned int count)
 539 {
 540         struct pblk *pblk = container_of(rb, struct pblk, rwb);
 541         struct request_queue *q = pblk->dev->q;
 542         struct pblk_c_ctx *c_ctx = nvm_rq_to_pdu(rqd);
 543         struct bio *bio = rqd->bio;
 544         struct pblk_rb_entry *entry;
 545         struct page *page;
 546         unsigned int pad = 0, to_read = nr_entries;
 547         unsigned int i;
 548         int flags;
 549
 550         if (count < nr_entries) {
 551                 pad = nr_entries - count;
 552                 to_read = count;
 553         }
 554
 555         /* Add space for packed metadata if in use*/
 556         pad += (pblk->min_write_pgs - pblk->min_write_pgs_data);
 557
 558         c_ctx->sentry = pos;
 559         c_ctx->nr_valid = to_read;
 560         c_ctx->nr_padded = pad;
 561
 562         for (i = 0; i < to_read; i++) {
 563                 entry = &rb->entries[pos];
 564
 565                 /* A write has been allowed into the buffer, but data is still
 566                  * being copied to it. It is ok to busy wait.
 567                  */
 568 try:
 569                 flags = READ_ONCE(entry->w_ctx.flags);
 570                 if (!(flags & PBLK_WRITTEN_DATA)) {
 571                         io_schedule();
 572                         goto try;
 573                 }
 574
 575                 page = virt_to_page(entry->data);
 576                 if (!page) {
 577                         pblk_err(pblk, "could not allocate write bio page\n");
 578                         flags &= ~PBLK_WRITTEN_DATA;
 579                         flags |= PBLK_SUBMITTED_ENTRY;
 580                         /* Release flags on context. Protect from writes */
 581                         smp_store_release(&entry->w_ctx.flags, flags);
 582                         return NVM_IO_ERR;
 583                 }
 584
 585                 if (bio_add_pc_page(q, bio, page, rb->seg_size, 0) !=
 586                                                                 rb->seg_size) {
 587                         pblk_err(pblk, "could not add page to write bio\n");
 588                         flags &= ~PBLK_WRITTEN_DATA;
 589                         flags |= PBLK_SUBMITTED_ENTRY;
 590                         /* Release flags on context. Protect from writes */
 591                         smp_store_release(&entry->w_ctx.flags, flags);
 592                         return NVM_IO_ERR;
 593                 }
 594
 595                 flags &= ~PBLK_WRITTEN_DATA;
 596                 flags |= PBLK_SUBMITTED_ENTRY;
 597
 598                 /* Release flags on context. Protect from writes */
 599                 smp_store_release(&entry->w_ctx.flags, flags);
 600
 601                 pos = pblk_rb_ptr_wrap(rb, pos, 1);
 602         }
 603
 604         if (pad) {
 605                 if (pblk_bio_add_pages(pblk, bio, GFP_KERNEL, pad)) {
 606                         pblk_err(pblk, "could not pad page in write bio\n");
 607                         return NVM_IO_ERR;
 608                 }
 609
 610                 if (pad < pblk->min_write_pgs)
 611                         atomic64_inc(&pblk->pad_dist[pad - 1]);
 612                 else
 613                         pblk_warn(pblk, "padding more than min. sectors\n");
 614
 615                 atomic64_add(pad, &pblk->pad_wa);
 616         }
 617
 618 #ifdef CONFIG_NVM_PBLK_DEBUG
 619         atomic_long_add(pad, &pblk->padded_writes);
 620 #endif
 621
 622         return NVM_IO_OK;
 623 }
 624
 625 /*
 626  * Copy to bio only if the lba matches the one on the given cache entry.
 627  * Otherwise, it means that the entry has been overwritten, and the bio should
 628  * be directed to disk.
 629  */
 630 int pblk_rb_copy_to_bio(struct pblk_rb *rb, struct bio *bio, sector_t lba,
 631                         struct ppa_addr ppa, int bio_iter, bool advanced_bio)
 632 {
 633         struct pblk *pblk = container_of(rb, struct pblk, rwb);
 634         struct pblk_rb_entry *entry;
 635         struct pblk_w_ctx *w_ctx;
 636         struct ppa_addr l2p_ppa;
 637         u64 pos = pblk_addr_to_cacheline(ppa);
 638         void *data;
 639         int flags;
 640         int ret = 1;
 641
 642
 643 #ifdef CONFIG_NVM_PBLK_DEBUG
 644         /* Caller must ensure that the access will not cause an overflow */
 645         BUG_ON(pos >= rb->nr_entries);
 646 #endif
 647         entry = &rb->entries[pos];
 648         w_ctx = &entry->w_ctx;
 649         flags = READ_ONCE(w_ctx->flags);
 650
 651         spin_lock(&rb->w_lock);
 652         spin_lock(&pblk->trans_lock);
 653         l2p_ppa = pblk_trans_map_get(pblk, lba);
 654         spin_unlock(&pblk->trans_lock);
 655
 656         /* Check if the entry has been overwritten or is scheduled to be */
 657         if (!pblk_ppa_comp(l2p_ppa, ppa) || w_ctx->lba != lba ||
 658                                                 flags & PBLK_WRITABLE_ENTRY) {
 659                 ret = 0;
 660                 goto out;
 661         }
 662
 663         /* Only advance the bio if it hasn't been advanced already. If advanced,
 664          * this bio is at least a partial bio (i.e., it has partially been
 665          * filled with data from the cache). If part of the data resides on the
 666          * media, we will read later on
 667          */
 668         if (unlikely(!advanced_bio))
 669                 bio_advance(bio, bio_iter * PBLK_EXPOSED_PAGE_SIZE);
 670
 671         data = bio_data(bio);
 672         memcpy(data, entry->data, rb->seg_size);
 673
 674 out:
 675         spin_unlock(&rb->w_lock);
 676         return ret;
 677 }
 678
 679 struct pblk_w_ctx *pblk_rb_w_ctx(struct pblk_rb *rb, unsigned int pos)
 680 {
 681         unsigned int entry = pblk_rb_ptr_wrap(rb, pos, 0);
 682
 683         return &rb->entries[entry].w_ctx;
 684 }
 685
 686 unsigned int pblk_rb_sync_init(struct pblk_rb *rb, unsigned long *flags)
 687         __acquires(&rb->s_lock)
 688 {
 689         if (flags)
 690                 spin_lock_irqsave(&rb->s_lock, *flags);
 691         else
 692                 spin_lock_irq(&rb->s_lock);
 693
 694         return rb->sync;
 695 }
 696
 697 void pblk_rb_sync_end(struct pblk_rb *rb, unsigned long *flags)
 698         __releases(&rb->s_lock)
 699 {
 700         lockdep_assert_held(&rb->s_lock);
 701
 702         if (flags)
 703                 spin_unlock_irqrestore(&rb->s_lock, *flags);
 704         else
 705                 spin_unlock_irq(&rb->s_lock);
 706 }
 707
 708 unsigned int pblk_rb_sync_advance(struct pblk_rb *rb, unsigned int nr_entries)
 709 {
 710         unsigned int sync, flush_point;
 711         lockdep_assert_held(&rb->s_lock);
 712
 713         sync = READ_ONCE(rb->sync);
 714         flush_point = READ_ONCE(rb->flush_point);
 715
 716         if (flush_point != EMPTY_ENTRY) {
 717                 unsigned int secs_to_flush;
 718
 719                 secs_to_flush = pblk_rb_ring_count(flush_point, sync,
 720                                         rb->nr_entries);
 721                 if (secs_to_flush < nr_entries) {
 722                         /* Protect flush points */
 723                         smp_store_release(&rb->flush_point, EMPTY_ENTRY);
 724                 }
 725         }
 726
 727         sync = pblk_rb_ptr_wrap(rb, sync, nr_entries);
 728
 729         /* Protect from counts */
 730         smp_store_release(&rb->sync, sync);
 731
 732         return sync;
 733 }
 734
 735 /* Calculate how many sectors to submit up to the current flush point. */
 736 unsigned int pblk_rb_flush_point_count(struct pblk_rb *rb)
 737 {
 738         unsigned int subm, sync, flush_point;
 739         unsigned int submitted, to_flush;
 740
 741         /* Protect flush points */
 742         flush_point = smp_load_acquire(&rb->flush_point);
 743         if (flush_point == EMPTY_ENTRY)
 744                 return 0;
 745
 746         /* Protect syncs */
 747         sync = smp_load_acquire(&rb->sync);
 748
 749         subm = READ_ONCE(rb->subm);
 750         submitted = pblk_rb_ring_count(subm, sync, rb->nr_entries);
 751
 752         /* The sync point itself counts as a sector to sync */
 753         to_flush = pblk_rb_ring_count(flush_point, sync, rb->nr_entries) + 1;
 754
 755         return (submitted < to_flush) ? (to_flush - submitted) : 0;
 756 }
 757
 758 int pblk_rb_tear_down_check(struct pblk_rb *rb)
 759 {
 760         struct pblk_rb_entry *entry;
 761         int i;
 762         int ret = 0;
 763
 764         spin_lock(&rb->w_lock);
 765         spin_lock_irq(&rb->s_lock);
 766
 767         if ((rb->mem == rb->subm) && (rb->subm == rb->sync) &&
 768                                 (rb->sync == rb->l2p_update) &&
 769                                 (rb->flush_point == EMPTY_ENTRY)) {
 770                 goto out;
 771         }
 772
 773         if (!rb->entries) {
 774                 ret = 1;
 775                 goto out;
 776         }
 777
 778         for (i = 0; i < rb->nr_entries; i++) {
 779                 entry = &rb->entries[i];
 780
 781                 if (!entry->data) {
 782                         ret = 1;
 783                         goto out;
 784                 }
 785         }
 786
 787 out:
 788         spin_unlock(&rb->w_lock);
 789         spin_unlock_irq(&rb->s_lock);
 790
 791         return ret;
 792 }
 793
 794 unsigned int pblk_rb_wrap_pos(struct pblk_rb *rb, unsigned int pos)
 795 {
 796         return (pos & (rb->nr_entries - 1));
 797 }
 798
 799 int pblk_rb_pos_oob(struct pblk_rb *rb, u64 pos)
 800 {
 801         return (pos >= rb->nr_entries);
 802 }
 803
 804 ssize_t pblk_rb_sysfs(struct pblk_rb *rb, char *buf)
 805 {
 806         struct pblk *pblk = container_of(rb, struct pblk, rwb);
 807         struct pblk_c_ctx *c;
 808         ssize_t offset;
 809         int queued_entries = 0;
 810
 811         spin_lock_irq(&rb->s_lock);
 812         list_for_each_entry(c, &pblk->compl_list, list)
 813                 queued_entries++;
 814         spin_unlock_irq(&rb->s_lock);
 815
 816         if (rb->flush_point != EMPTY_ENTRY)
 817                 offset = scnprintf(buf, PAGE_SIZE,
 818                         "%u\t%u\t%u\t%u\t%u\t%u\t%u - %u/%u/%u - %d\n",
 819                         rb->nr_entries,
 820                         rb->mem,
 821                         rb->subm,
 822                         rb->sync,
 823                         rb->l2p_update,
 824 #ifdef CONFIG_NVM_PBLK_DEBUG
 825                         atomic_read(&rb->inflight_flush_point),
 826 #else
 827                         0,
 828 #endif
 829                         rb->flush_point,
 830                         pblk_rb_read_count(rb),
 831                         pblk_rb_space(rb),
 832                         pblk_rb_flush_point_count(rb),
 833                         queued_entries);
 834         else
 835                 offset = scnprintf(buf, PAGE_SIZE,
 836                         "%u\t%u\t%u\t%u\t%u\t%u\tNULL - %u/%u/%u - %d\n",
 837                         rb->nr_entries,
 838                         rb->mem,
 839                         rb->subm,
 840                         rb->sync,
 841                         rb->l2p_update,
 842 #ifdef CONFIG_NVM_PBLK_DEBUG
 843                         atomic_read(&rb->inflight_flush_point),
 844 #else
 845                         0,
 846 #endif
 847                         pblk_rb_read_count(rb),
 848                         pblk_rb_space(rb),
 849                         pblk_rb_flush_point_count(rb),
 850                         queued_entries);
 851
 852         return offset;
 853 }