git.rot13.org Git - linux/blob - drivers/md/dm-mpath.c

   1 /*
   2  * Copyright (C) 2003 Sistina Software Limited.
   3  * Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved.
   4  *
   5  * This file is released under the GPL.
   6  */
   7
   8 #include <linux/device-mapper.h>
   9
  10 #include "dm-rq.h"
  11 #include "dm-bio-record.h"
  12 #include "dm-path-selector.h"
  13 #include "dm-uevent.h"
  14
  15 #include <linux/blkdev.h>
  16 #include <linux/ctype.h>
  17 #include <linux/init.h>
  18 #include <linux/mempool.h>
  19 #include <linux/module.h>
  20 #include <linux/pagemap.h>
  21 #include <linux/slab.h>
  22 #include <linux/time.h>
  23 #include <linux/workqueue.h>
  24 #include <linux/delay.h>
  25 #include <scsi/scsi_dh.h>
  26 #include <linux/atomic.h>
  27 #include <linux/blk-mq.h>
  28
  29 #define DM_MSG_PREFIX "multipath"
  30 #define DM_PG_INIT_DELAY_MSECS 2000
  31 #define DM_PG_INIT_DELAY_DEFAULT ((unsigned) -1)
  32
  33 /* Path properties */
  34 struct pgpath {
  35         struct list_head list;
  36
  37         struct priority_group *pg;      /* Owning PG */
  38         unsigned fail_count;            /* Cumulative failure count */
  39
  40         struct dm_path path;
  41         struct delayed_work activate_path;
  42
  43         bool is_active:1;               /* Path status */
  44 };
  45
  46 #define path_to_pgpath(__pgp) container_of((__pgp), struct pgpath, path)
  47
  48 /*
  49  * Paths are grouped into Priority Groups and numbered from 1 upwards.
  50  * Each has a path selector which controls which path gets used.
  51  */
  52 struct priority_group {
  53         struct list_head list;
  54
  55         struct multipath *m;            /* Owning multipath instance */
  56         struct path_selector ps;
  57
  58         unsigned pg_num;                /* Reference number */
  59         unsigned nr_pgpaths;            /* Number of paths in PG */
  60         struct list_head pgpaths;
  61
  62         bool bypassed:1;                /* Temporarily bypass this PG? */
  63 };
  64
  65 /* Multipath context */
  66 struct multipath {
  67         unsigned long flags;            /* Multipath state flags */
  68
  69         spinlock_t lock;
  70         enum dm_queue_mode queue_mode;
  71
  72         struct pgpath *current_pgpath;
  73         struct priority_group *current_pg;
  74         struct priority_group *next_pg; /* Switch to this PG if set */
  75
  76         atomic_t nr_valid_paths;        /* Total number of usable paths */
  77         unsigned nr_priority_groups;
  78         struct list_head priority_groups;
  79
  80         const char *hw_handler_name;
  81         char *hw_handler_params;
  82         wait_queue_head_t pg_init_wait; /* Wait for pg_init completion */
  83         unsigned pg_init_retries;       /* Number of times to retry pg_init */
  84         unsigned pg_init_delay_msecs;   /* Number of msecs before pg_init retry */
  85         atomic_t pg_init_in_progress;   /* Only one pg_init allowed at once */
  86         atomic_t pg_init_count;         /* Number of times pg_init called */
  87
  88         struct mutex work_mutex;
  89         struct work_struct trigger_event;
  90         struct dm_target *ti;
  91
  92         struct work_struct process_queued_bios;
  93         struct bio_list queued_bios;
  94 };
  95
  96 /*
  97  * Context information attached to each io we process.
  98  */
  99 struct dm_mpath_io {
 100         struct pgpath *pgpath;
 101         size_t nr_bytes;
 102 };
 103
 104 typedef int (*action_fn) (struct pgpath *pgpath);
 105
 106 static struct workqueue_struct *kmultipathd, *kmpath_handlerd;
 107 static void trigger_event(struct work_struct *work);
 108 static void activate_or_offline_path(struct pgpath *pgpath);
 109 static void activate_path_work(struct work_struct *work);
 110 static void process_queued_bios(struct work_struct *work);
 111
 112 /*-----------------------------------------------
 113  * Multipath state flags.
 114  *-----------------------------------------------*/
 115
 116 #define MPATHF_QUEUE_IO 0                       /* Must we queue all I/O? */
 117 #define MPATHF_QUEUE_IF_NO_PATH 1               /* Queue I/O if last path fails? */
 118 #define MPATHF_SAVED_QUEUE_IF_NO_PATH 2         /* Saved state during suspension */
 119 #define MPATHF_RETAIN_ATTACHED_HW_HANDLER 3     /* If there's already a hw_handler present, don't change it. */
 120 #define MPATHF_PG_INIT_DISABLED 4               /* pg_init is not currently allowed */
 121 #define MPATHF_PG_INIT_REQUIRED 5               /* pg_init needs calling? */
 122 #define MPATHF_PG_INIT_DELAY_RETRY 6            /* Delay pg_init retry? */
 123
 124 /*-----------------------------------------------
 125  * Allocation routines
 126  *-----------------------------------------------*/
 127
 128 static struct pgpath *alloc_pgpath(void)
 129 {
 130         struct pgpath *pgpath = kzalloc(sizeof(*pgpath), GFP_KERNEL);
 131
 132         if (!pgpath)
 133                 return NULL;
 134
 135         pgpath->is_active = true;
 136
 137         return pgpath;
 138 }
 139
 140 static void free_pgpath(struct pgpath *pgpath)
 141 {
 142         kfree(pgpath);
 143 }
 144
 145 static struct priority_group *alloc_priority_group(void)
 146 {
 147         struct priority_group *pg;
 148
 149         pg = kzalloc(sizeof(*pg), GFP_KERNEL);
 150
 151         if (pg)
 152                 INIT_LIST_HEAD(&pg->pgpaths);
 153
 154         return pg;
 155 }
 156
 157 static void free_pgpaths(struct list_head *pgpaths, struct dm_target *ti)
 158 {
 159         struct pgpath *pgpath, *tmp;
 160
 161         list_for_each_entry_safe(pgpath, tmp, pgpaths, list) {
 162                 list_del(&pgpath->list);
 163                 dm_put_device(ti, pgpath->path.dev);
 164                 free_pgpath(pgpath);
 165         }
 166 }
 167
 168 static void free_priority_group(struct priority_group *pg,
 169                                 struct dm_target *ti)
 170 {
 171         struct path_selector *ps = &pg->ps;
 172
 173         if (ps->type) {
 174                 ps->type->destroy(ps);
 175                 dm_put_path_selector(ps->type);
 176         }
 177
 178         free_pgpaths(&pg->pgpaths, ti);
 179         kfree(pg);
 180 }
 181
 182 static struct multipath *alloc_multipath(struct dm_target *ti)
 183 {
 184         struct multipath *m;
 185
 186         m = kzalloc(sizeof(*m), GFP_KERNEL);
 187         if (m) {
 188                 INIT_LIST_HEAD(&m->priority_groups);
 189                 spin_lock_init(&m->lock);
 190                 atomic_set(&m->nr_valid_paths, 0);
 191                 INIT_WORK(&m->trigger_event, trigger_event);
 192                 mutex_init(&m->work_mutex);
 193
 194                 m->queue_mode = DM_TYPE_NONE;
 195
 196                 m->ti = ti;
 197                 ti->private = m;
 198         }
 199
 200         return m;
 201 }
 202
 203 static int alloc_multipath_stage2(struct dm_target *ti, struct multipath *m)
 204 {
 205         if (m->queue_mode == DM_TYPE_NONE) {
 206                 /*
 207                  * Default to request-based.
 208                  */
 209                 if (dm_use_blk_mq(dm_table_get_md(ti->table)))
 210                         m->queue_mode = DM_TYPE_MQ_REQUEST_BASED;
 211                 else
 212                         m->queue_mode = DM_TYPE_REQUEST_BASED;
 213
 214         } else if (m->queue_mode == DM_TYPE_BIO_BASED ||
 215                    m->queue_mode == DM_TYPE_NVME_BIO_BASED) {
 216                 INIT_WORK(&m->process_queued_bios, process_queued_bios);
 217
 218                 if (m->queue_mode == DM_TYPE_BIO_BASED) {
 219                         /*
 220                          * bio-based doesn't support any direct scsi_dh management;
 221                          * it just discovers if a scsi_dh is attached.
 222                          */
 223                         set_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags);
 224                 }
 225         }
 226
 227         if (m->queue_mode != DM_TYPE_NVME_BIO_BASED) {
 228                 set_bit(MPATHF_QUEUE_IO, &m->flags);
 229                 atomic_set(&m->pg_init_in_progress, 0);
 230                 atomic_set(&m->pg_init_count, 0);
 231                 m->pg_init_delay_msecs = DM_PG_INIT_DELAY_DEFAULT;
 232                 init_waitqueue_head(&m->pg_init_wait);
 233         }
 234
 235         dm_table_set_type(ti->table, m->queue_mode);
 236
 237         return 0;
 238 }
 239
 240 static void free_multipath(struct multipath *m)
 241 {
 242         struct priority_group *pg, *tmp;
 243
 244         list_for_each_entry_safe(pg, tmp, &m->priority_groups, list) {
 245                 list_del(&pg->list);
 246                 free_priority_group(pg, m->ti);
 247         }
 248
 249         kfree(m->hw_handler_name);
 250         kfree(m->hw_handler_params);
 251         kfree(m);
 252 }
 253
 254 static struct dm_mpath_io *get_mpio(union map_info *info)
 255 {
 256         return info->ptr;
 257 }
 258
 259 static size_t multipath_per_bio_data_size(void)
 260 {
 261         return sizeof(struct dm_mpath_io) + sizeof(struct dm_bio_details);
 262 }
 263
 264 static struct dm_mpath_io *get_mpio_from_bio(struct bio *bio)
 265 {
 266         return dm_per_bio_data(bio, multipath_per_bio_data_size());
 267 }
 268
 269 static struct dm_bio_details *get_bio_details_from_mpio(struct dm_mpath_io *mpio)
 270 {
 271         /* dm_bio_details is immediately after the dm_mpath_io in bio's per-bio-data */
 272         void *bio_details = mpio + 1;
 273         return bio_details;
 274 }
 275
 276 static void multipath_init_per_bio_data(struct bio *bio, struct dm_mpath_io **mpio_p)
 277 {
 278         struct dm_mpath_io *mpio = get_mpio_from_bio(bio);
 279         struct dm_bio_details *bio_details = get_bio_details_from_mpio(mpio);
 280
 281         mpio->nr_bytes = bio->bi_iter.bi_size;
 282         mpio->pgpath = NULL;
 283         *mpio_p = mpio;
 284
 285         dm_bio_record(bio_details, bio);
 286 }
 287
 288 /*-----------------------------------------------
 289  * Path selection
 290  *-----------------------------------------------*/
 291
 292 static int __pg_init_all_paths(struct multipath *m)
 293 {
 294         struct pgpath *pgpath;
 295         unsigned long pg_init_delay = 0;
 296
 297         lockdep_assert_held(&m->lock);
 298
 299         if (atomic_read(&m->pg_init_in_progress) || test_bit(MPATHF_PG_INIT_DISABLED, &m->flags))
 300                 return 0;
 301
 302         atomic_inc(&m->pg_init_count);
 303         clear_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
 304
 305         /* Check here to reset pg_init_required */
 306         if (!m->current_pg)
 307                 return 0;
 308
 309         if (test_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags))
 310                 pg_init_delay = msecs_to_jiffies(m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT ?
 311                                                  m->pg_init_delay_msecs : DM_PG_INIT_DELAY_MSECS);
 312         list_for_each_entry(pgpath, &m->current_pg->pgpaths, list) {
 313                 /* Skip failed paths */
 314                 if (!pgpath->is_active)
 315                         continue;
 316                 if (queue_delayed_work(kmpath_handlerd, &pgpath->activate_path,
 317                                        pg_init_delay))
 318                         atomic_inc(&m->pg_init_in_progress);
 319         }
 320         return atomic_read(&m->pg_init_in_progress);
 321 }
 322
 323 static int pg_init_all_paths(struct multipath *m)
 324 {
 325         int ret;
 326         unsigned long flags;
 327
 328         spin_lock_irqsave(&m->lock, flags);
 329         ret = __pg_init_all_paths(m);
 330         spin_unlock_irqrestore(&m->lock, flags);
 331
 332         return ret;
 333 }
 334
 335 static void __switch_pg(struct multipath *m, struct priority_group *pg)
 336 {
 337         m->current_pg = pg;
 338
 339         if (m->queue_mode == DM_TYPE_NVME_BIO_BASED)
 340                 return;
 341
 342         /* Must we initialise the PG first, and queue I/O till it's ready? */
 343         if (m->hw_handler_name) {
 344                 set_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
 345                 set_bit(MPATHF_QUEUE_IO, &m->flags);
 346         } else {
 347                 clear_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
 348                 clear_bit(MPATHF_QUEUE_IO, &m->flags);
 349         }
 350
 351         atomic_set(&m->pg_init_count, 0);
 352 }
 353
 354 static struct pgpath *choose_path_in_pg(struct multipath *m,
 355                                         struct priority_group *pg,
 356                                         size_t nr_bytes)
 357 {
 358         unsigned long flags;
 359         struct dm_path *path;
 360         struct pgpath *pgpath;
 361
 362         path = pg->ps.type->select_path(&pg->ps, nr_bytes);
 363         if (!path)
 364                 return ERR_PTR(-ENXIO);
 365
 366         pgpath = path_to_pgpath(path);
 367
 368         if (unlikely(READ_ONCE(m->current_pg) != pg)) {
 369                 /* Only update current_pgpath if pg changed */
 370                 spin_lock_irqsave(&m->lock, flags);
 371                 m->current_pgpath = pgpath;
 372                 __switch_pg(m, pg);
 373                 spin_unlock_irqrestore(&m->lock, flags);
 374         }
 375
 376         return pgpath;
 377 }
 378
 379 static struct pgpath *choose_pgpath(struct multipath *m, size_t nr_bytes)
 380 {
 381         unsigned long flags;
 382         struct priority_group *pg;
 383         struct pgpath *pgpath;
 384         unsigned bypassed = 1;
 385
 386         if (!atomic_read(&m->nr_valid_paths)) {
 387                 if (m->queue_mode != DM_TYPE_NVME_BIO_BASED)
 388                         clear_bit(MPATHF_QUEUE_IO, &m->flags);
 389                 goto failed;
 390         }
 391
 392         /* Were we instructed to switch PG? */
 393         if (READ_ONCE(m->next_pg)) {
 394                 spin_lock_irqsave(&m->lock, flags);
 395                 pg = m->next_pg;
 396                 if (!pg) {
 397                         spin_unlock_irqrestore(&m->lock, flags);
 398                         goto check_current_pg;
 399                 }
 400                 m->next_pg = NULL;
 401                 spin_unlock_irqrestore(&m->lock, flags);
 402                 pgpath = choose_path_in_pg(m, pg, nr_bytes);
 403                 if (!IS_ERR_OR_NULL(pgpath))
 404                         return pgpath;
 405         }
 406
 407         /* Don't change PG until it has no remaining paths */
 408 check_current_pg:
 409         pg = READ_ONCE(m->current_pg);
 410         if (pg) {
 411                 pgpath = choose_path_in_pg(m, pg, nr_bytes);
 412                 if (!IS_ERR_OR_NULL(pgpath))
 413                         return pgpath;
 414         }
 415
 416         /*
 417          * Loop through priority groups until we find a valid path.
 418          * First time we skip PGs marked 'bypassed'.
 419          * Second time we only try the ones we skipped, but set
 420          * pg_init_delay_retry so we do not hammer controllers.
 421          */
 422         do {
 423                 list_for_each_entry(pg, &m->priority_groups, list) {
 424                         if (pg->bypassed == !!bypassed)
 425                                 continue;
 426                         pgpath = choose_path_in_pg(m, pg, nr_bytes);
 427                         if (!IS_ERR_OR_NULL(pgpath)) {
 428                                 if (!bypassed)
 429                                         set_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags);
 430                                 return pgpath;
 431                         }
 432                 }
 433         } while (bypassed--);
 434
 435 failed:
 436         spin_lock_irqsave(&m->lock, flags);
 437         m->current_pgpath = NULL;
 438         m->current_pg = NULL;
 439         spin_unlock_irqrestore(&m->lock, flags);
 440
 441         return NULL;
 442 }
 443
 444 /*
 445  * dm_report_EIO() is a macro instead of a function to make pr_debug()
 446  * report the function name and line number of the function from which
 447  * it has been invoked.
 448  */
 449 #define dm_report_EIO(m)                                                \
 450 do {                                                                    \
 451         struct mapped_device *md = dm_table_get_md((m)->ti->table);     \
 452                                                                         \
 453         pr_debug("%s: returning EIO; QIFNP = %d; SQIFNP = %d; DNFS = %d\n", \
 454                  dm_device_name(md),                                    \
 455                  test_bit(MPATHF_QUEUE_IF_NO_PATH, &(m)->flags),        \
 456                  test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &(m)->flags),  \
 457                  dm_noflush_suspending((m)->ti));                       \
 458 } while (0)
 459
 460 /*
 461  * Check whether bios must be queued in the device-mapper core rather
 462  * than here in the target.
 463  *
 464  * If MPATHF_QUEUE_IF_NO_PATH and MPATHF_SAVED_QUEUE_IF_NO_PATH hold
 465  * the same value then we are not between multipath_presuspend()
 466  * and multipath_resume() calls and we have no need to check
 467  * for the DMF_NOFLUSH_SUSPENDING flag.
 468  */
 469 static bool __must_push_back(struct multipath *m, unsigned long flags)
 470 {
 471         return ((test_bit(MPATHF_QUEUE_IF_NO_PATH, &flags) !=
 472                  test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &flags)) &&
 473                 dm_noflush_suspending(m->ti));
 474 }
 475
 476 /*
 477  * Following functions use READ_ONCE to get atomic access to
 478  * all m->flags to avoid taking spinlock
 479  */
 480 static bool must_push_back_rq(struct multipath *m)
 481 {
 482         unsigned long flags = READ_ONCE(m->flags);
 483         return test_bit(MPATHF_QUEUE_IF_NO_PATH, &flags) || __must_push_back(m, flags);
 484 }
 485
 486 static bool must_push_back_bio(struct multipath *m)
 487 {
 488         unsigned long flags = READ_ONCE(m->flags);
 489         return __must_push_back(m, flags);
 490 }
 491
 492 /*
 493  * Map cloned requests (request-based multipath)
 494  */
 495 static int multipath_clone_and_map(struct dm_target *ti, struct request *rq,
 496                                    union map_info *map_context,
 497                                    struct request **__clone)
 498 {
 499         struct multipath *m = ti->private;
 500         size_t nr_bytes = blk_rq_bytes(rq);
 501         struct pgpath *pgpath;
 502         struct block_device *bdev;
 503         struct dm_mpath_io *mpio = get_mpio(map_context);
 504         struct request_queue *q;
 505         struct request *clone;
 506
 507         /* Do we need to select a new pgpath? */
 508         pgpath = READ_ONCE(m->current_pgpath);
 509         if (!pgpath || !test_bit(MPATHF_QUEUE_IO, &m->flags))
 510                 pgpath = choose_pgpath(m, nr_bytes);
 511
 512         if (!pgpath) {
 513                 if (must_push_back_rq(m))
 514                         return DM_MAPIO_DELAY_REQUEUE;
 515                 dm_report_EIO(m);       /* Failed */
 516                 return DM_MAPIO_KILL;
 517         } else if (test_bit(MPATHF_QUEUE_IO, &m->flags) ||
 518                    test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) {
 519                 if (pg_init_all_paths(m))
 520                         return DM_MAPIO_DELAY_REQUEUE;
 521                 return DM_MAPIO_REQUEUE;
 522         }
 523
 524         mpio->pgpath = pgpath;
 525         mpio->nr_bytes = nr_bytes;
 526
 527         bdev = pgpath->path.dev->bdev;
 528         q = bdev_get_queue(bdev);
 529         clone = blk_get_request(q, rq->cmd_flags | REQ_NOMERGE, GFP_ATOMIC);
 530         if (IS_ERR(clone)) {
 531                 /* EBUSY, ENODEV or EWOULDBLOCK: requeue */
 532                 if (blk_queue_dying(q)) {
 533                         atomic_inc(&m->pg_init_in_progress);
 534                         activate_or_offline_path(pgpath);
 535                 }
 536                 return DM_MAPIO_DELAY_REQUEUE;
 537         }
 538         clone->bio = clone->biotail = NULL;
 539         clone->rq_disk = bdev->bd_disk;
 540         clone->cmd_flags |= REQ_FAILFAST_TRANSPORT;
 541         *__clone = clone;
 542
 543         if (pgpath->pg->ps.type->start_io)
 544                 pgpath->pg->ps.type->start_io(&pgpath->pg->ps,
 545                                               &pgpath->path,
 546                                               nr_bytes);
 547         return DM_MAPIO_REMAPPED;
 548 }
 549
 550 static void multipath_release_clone(struct request *clone)
 551 {
 552         blk_put_request(clone);
 553 }
 554
 555 /*
 556  * Map cloned bios (bio-based multipath)
 557  */
 558 static int __multipath_map_bio(struct multipath *m, struct bio *bio, struct dm_mpath_io *mpio)
 559 {
 560         struct pgpath *pgpath;
 561         unsigned long flags;
 562         bool queue_io;
 563
 564         /* Do we need to select a new pgpath? */
 565         pgpath = READ_ONCE(m->current_pgpath);
 566         /* MPATHF_QUEUE_IO will never be set for NVMe */
 567         queue_io = test_bit(MPATHF_QUEUE_IO, &m->flags);
 568         if (!pgpath || !queue_io)
 569                 pgpath = choose_pgpath(m, mpio->nr_bytes);
 570
 571         if ((!pgpath && test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) ||
 572             (pgpath && queue_io)) {
 573                 /* Queue for the daemon to resubmit */
 574                 spin_lock_irqsave(&m->lock, flags);
 575                 bio_list_add(&m->queued_bios, bio);
 576                 spin_unlock_irqrestore(&m->lock, flags);
 577
 578                 if (m->queue_mode == DM_TYPE_NVME_BIO_BASED) {
 579                         queue_work(kmultipathd, &m->process_queued_bios);
 580                 } else {
 581                         /* PG_INIT_REQUIRED cannot be set without QUEUE_IO */
 582                         if (queue_io || test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags))
 583                                 pg_init_all_paths(m);
 584                         else if (!queue_io)
 585                                 queue_work(kmultipathd, &m->process_queued_bios);
 586                 }
 587
 588                 return DM_MAPIO_SUBMITTED;
 589         }
 590
 591         if (!pgpath) {
 592                 if (must_push_back_bio(m))
 593                         return DM_MAPIO_REQUEUE;
 594                 dm_report_EIO(m);
 595                 return DM_MAPIO_KILL;
 596         }
 597
 598         mpio->pgpath = pgpath;
 599
 600         bio->bi_status = 0;
 601         bio_set_dev(bio, pgpath->path.dev->bdev);
 602         bio->bi_opf |= REQ_FAILFAST_TRANSPORT;
 603
 604         if (pgpath->pg->ps.type->start_io)
 605                 pgpath->pg->ps.type->start_io(&pgpath->pg->ps,
 606                                               &pgpath->path,
 607                                               mpio->nr_bytes);
 608         return DM_MAPIO_REMAPPED;
 609 }
 610
 611 static int multipath_map_bio(struct dm_target *ti, struct bio *bio)
 612 {
 613         struct multipath *m = ti->private;
 614         struct dm_mpath_io *mpio = NULL;
 615
 616         multipath_init_per_bio_data(bio, &mpio);
 617         return __multipath_map_bio(m, bio, mpio);
 618 }
 619
 620 static void process_queued_io_list(struct multipath *m)
 621 {
 622         if (m->queue_mode == DM_TYPE_MQ_REQUEST_BASED)
 623                 dm_mq_kick_requeue_list(dm_table_get_md(m->ti->table));
 624         else if (m->queue_mode == DM_TYPE_BIO_BASED ||
 625                  m->queue_mode == DM_TYPE_NVME_BIO_BASED)
 626                 queue_work(kmultipathd, &m->process_queued_bios);
 627 }
 628
 629 static void process_queued_bios(struct work_struct *work)
 630 {
 631         int r;
 632         unsigned long flags;
 633         struct bio *bio;
 634         struct bio_list bios;
 635         struct blk_plug plug;
 636         struct multipath *m =
 637                 container_of(work, struct multipath, process_queued_bios);
 638
 639         bio_list_init(&bios);
 640
 641         spin_lock_irqsave(&m->lock, flags);
 642
 643         if (bio_list_empty(&m->queued_bios)) {
 644                 spin_unlock_irqrestore(&m->lock, flags);
 645                 return;
 646         }
 647
 648         bio_list_merge(&bios, &m->queued_bios);
 649         bio_list_init(&m->queued_bios);
 650
 651         spin_unlock_irqrestore(&m->lock, flags);
 652
 653         blk_start_plug(&plug);
 654         while ((bio = bio_list_pop(&bios))) {
 655                 struct dm_mpath_io *mpio = get_mpio_from_bio(bio);
 656                 dm_bio_restore(get_bio_details_from_mpio(mpio), bio);
 657                 r = __multipath_map_bio(m, bio, mpio);
 658                 switch (r) {
 659                 case DM_MAPIO_KILL:
 660                         bio->bi_status = BLK_STS_IOERR;
 661                         bio_endio(bio);
 662                         break;
 663                 case DM_MAPIO_REQUEUE:
 664                         bio->bi_status = BLK_STS_DM_REQUEUE;
 665                         bio_endio(bio);
 666                         break;
 667                 case DM_MAPIO_REMAPPED:
 668                         generic_make_request(bio);
 669                         break;
 670                 case 0:
 671                         break;
 672                 default:
 673                         WARN_ONCE(true, "__multipath_map_bio() returned %d\n", r);
 674                 }
 675         }
 676         blk_finish_plug(&plug);
 677 }
 678
 679 /*
 680  * If we run out of usable paths, should we queue I/O or error it?
 681  */
 682 static int queue_if_no_path(struct multipath *m, bool queue_if_no_path,
 683                             bool save_old_value)
 684 {
 685         unsigned long flags;
 686
 687         spin_lock_irqsave(&m->lock, flags);
 688         assign_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags,
 689                    (save_old_value && test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) ||
 690                    (!save_old_value && queue_if_no_path));
 691         assign_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags, queue_if_no_path);
 692         spin_unlock_irqrestore(&m->lock, flags);
 693
 694         if (!queue_if_no_path) {
 695                 dm_table_run_md_queue_async(m->ti->table);
 696                 process_queued_io_list(m);
 697         }
 698
 699         return 0;
 700 }
 701
 702 /*
 703  * An event is triggered whenever a path is taken out of use.
 704  * Includes path failure and PG bypass.
 705  */
 706 static void trigger_event(struct work_struct *work)
 707 {
 708         struct multipath *m =
 709                 container_of(work, struct multipath, trigger_event);
 710
 711         dm_table_event(m->ti->table);
 712 }
 713
 714 /*-----------------------------------------------------------------
 715  * Constructor/argument parsing:
 716  * <#multipath feature args> [<arg>]*
 717  * <#hw_handler args> [hw_handler [<arg>]*]
 718  * <#priority groups>
 719  * <initial priority group>
 720  *     [<selector> <#selector args> [<arg>]*
 721  *      <#paths> <#per-path selector args>
 722  *         [<path> [<arg>]* ]+ ]+
 723  *---------------------------------------------------------------*/
 724 static int parse_path_selector(struct dm_arg_set *as, struct priority_group *pg,
 725                                struct dm_target *ti)
 726 {
 727         int r;
 728         struct path_selector_type *pst;
 729         unsigned ps_argc;
 730
 731         static const struct dm_arg _args[] = {
 732                 {0, 1024, "invalid number of path selector args"},
 733         };
 734
 735         pst = dm_get_path_selector(dm_shift_arg(as));
 736         if (!pst) {
 737                 ti->error = "unknown path selector type";
 738                 return -EINVAL;
 739         }
 740
 741         r = dm_read_arg_group(_args, as, &ps_argc, &ti->error);
 742         if (r) {
 743                 dm_put_path_selector(pst);
 744                 return -EINVAL;
 745         }
 746
 747         r = pst->create(&pg->ps, ps_argc, as->argv);
 748         if (r) {
 749                 dm_put_path_selector(pst);
 750                 ti->error = "path selector constructor failed";
 751                 return r;
 752         }
 753
 754         pg->ps.type = pst;
 755         dm_consume_args(as, ps_argc);
 756
 757         return 0;
 758 }
 759
 760 static int setup_scsi_dh(struct block_device *bdev, struct multipath *m, char **error)
 761 {
 762         struct request_queue *q = bdev_get_queue(bdev);
 763         const char *attached_handler_name;
 764         int r;
 765
 766         if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags)) {
 767 retain:
 768                 attached_handler_name = scsi_dh_attached_handler_name(q, GFP_KERNEL);
 769                 if (attached_handler_name) {
 770                         /*
 771                          * Clear any hw_handler_params associated with a
 772                          * handler that isn't already attached.
 773                          */
 774                         if (m->hw_handler_name && strcmp(attached_handler_name, m->hw_handler_name)) {
 775                                 kfree(m->hw_handler_params);
 776                                 m->hw_handler_params = NULL;
 777                         }
 778
 779                         /*
 780                          * Reset hw_handler_name to match the attached handler
 781                          *
 782                          * NB. This modifies the table line to show the actual
 783                          * handler instead of the original table passed in.
 784                          */
 785                         kfree(m->hw_handler_name);
 786                         m->hw_handler_name = attached_handler_name;
 787                 }
 788         }
 789
 790         if (m->hw_handler_name) {
 791                 r = scsi_dh_attach(q, m->hw_handler_name);
 792                 if (r == -EBUSY) {
 793                         char b[BDEVNAME_SIZE];
 794
 795                         printk(KERN_INFO "dm-mpath: retaining handler on device %s\n",
 796                                bdevname(bdev, b));
 797                         goto retain;
 798                 }
 799                 if (r < 0) {
 800                         *error = "error attaching hardware handler";
 801                         return r;
 802                 }
 803
 804                 if (m->hw_handler_params) {
 805                         r = scsi_dh_set_params(q, m->hw_handler_params);
 806                         if (r < 0) {
 807                                 *error = "unable to set hardware handler parameters";
 808                                 return r;
 809                         }
 810                 }
 811         }
 812
 813         return 0;
 814 }
 815
 816 static struct pgpath *parse_path(struct dm_arg_set *as, struct path_selector *ps,
 817                                  struct dm_target *ti)
 818 {
 819         int r;
 820         struct pgpath *p;
 821         struct multipath *m = ti->private;
 822
 823         /* we need at least a path arg */
 824         if (as->argc < 1) {
 825                 ti->error = "no device given";
 826                 return ERR_PTR(-EINVAL);
 827         }
 828
 829         p = alloc_pgpath();
 830         if (!p)
 831                 return ERR_PTR(-ENOMEM);
 832
 833         r = dm_get_device(ti, dm_shift_arg(as), dm_table_get_mode(ti->table),
 834                           &p->path.dev);
 835         if (r) {
 836                 ti->error = "error getting device";
 837                 goto bad;
 838         }
 839
 840         if (m->queue_mode != DM_TYPE_NVME_BIO_BASED) {
 841                 INIT_DELAYED_WORK(&p->activate_path, activate_path_work);
 842                 r = setup_scsi_dh(p->path.dev->bdev, m, &ti->error);
 843                 if (r) {
 844                         dm_put_device(ti, p->path.dev);
 845                         goto bad;
 846                 }
 847         }
 848
 849         r = ps->type->add_path(ps, &p->path, as->argc, as->argv, &ti->error);
 850         if (r) {
 851                 dm_put_device(ti, p->path.dev);
 852                 goto bad;
 853         }
 854
 855         return p;
 856  bad:
 857         free_pgpath(p);
 858         return ERR_PTR(r);
 859 }
 860
 861 static struct priority_group *parse_priority_group(struct dm_arg_set *as,
 862                                                    struct multipath *m)
 863 {
 864         static const struct dm_arg _args[] = {
 865                 {1, 1024, "invalid number of paths"},
 866                 {0, 1024, "invalid number of selector args"}
 867         };
 868
 869         int r;
 870         unsigned i, nr_selector_args, nr_args;
 871         struct priority_group *pg;
 872         struct dm_target *ti = m->ti;
 873
 874         if (as->argc < 2) {
 875                 as->argc = 0;
 876                 ti->error = "not enough priority group arguments";
 877                 return ERR_PTR(-EINVAL);
 878         }
 879
 880         pg = alloc_priority_group();
 881         if (!pg) {
 882                 ti->error = "couldn't allocate priority group";
 883                 return ERR_PTR(-ENOMEM);
 884         }
 885         pg->m = m;
 886
 887         r = parse_path_selector(as, pg, ti);
 888         if (r)
 889                 goto bad;
 890
 891         /*
 892          * read the paths
 893          */
 894         r = dm_read_arg(_args, as, &pg->nr_pgpaths, &ti->error);
 895         if (r)
 896                 goto bad;
 897
 898         r = dm_read_arg(_args + 1, as, &nr_selector_args, &ti->error);
 899         if (r)
 900                 goto bad;
 901
 902         nr_args = 1 + nr_selector_args;
 903         for (i = 0; i < pg->nr_pgpaths; i++) {
 904                 struct pgpath *pgpath;
 905                 struct dm_arg_set path_args;
 906
 907                 if (as->argc < nr_args) {
 908                         ti->error = "not enough path parameters";
 909                         r = -EINVAL;
 910                         goto bad;
 911                 }
 912
 913                 path_args.argc = nr_args;
 914                 path_args.argv = as->argv;
 915
 916                 pgpath = parse_path(&path_args, &pg->ps, ti);
 917                 if (IS_ERR(pgpath)) {
 918                         r = PTR_ERR(pgpath);
 919                         goto bad;
 920                 }
 921
 922                 pgpath->pg = pg;
 923                 list_add_tail(&pgpath->list, &pg->pgpaths);
 924                 dm_consume_args(as, nr_args);
 925         }
 926
 927         return pg;
 928
 929  bad:
 930         free_priority_group(pg, ti);
 931         return ERR_PTR(r);
 932 }
 933
 934 static int parse_hw_handler(struct dm_arg_set *as, struct multipath *m)
 935 {
 936         unsigned hw_argc;
 937         int ret;
 938         struct dm_target *ti = m->ti;
 939
 940         static const struct dm_arg _args[] = {
 941                 {0, 1024, "invalid number of hardware handler args"},
 942         };
 943
 944         if (dm_read_arg_group(_args, as, &hw_argc, &ti->error))
 945                 return -EINVAL;
 946
 947         if (!hw_argc)
 948                 return 0;
 949
 950         if (m->queue_mode == DM_TYPE_BIO_BASED ||
 951             m->queue_mode == DM_TYPE_NVME_BIO_BASED) {
 952                 dm_consume_args(as, hw_argc);
 953                 DMERR("bio-based multipath doesn't allow hardware handler args");
 954                 return 0;
 955         }
 956
 957         m->hw_handler_name = kstrdup(dm_shift_arg(as), GFP_KERNEL);
 958         if (!m->hw_handler_name)
 959                 return -EINVAL;
 960
 961         if (hw_argc > 1) {
 962                 char *p;
 963                 int i, j, len = 4;
 964
 965                 for (i = 0; i <= hw_argc - 2; i++)
 966                         len += strlen(as->argv[i]) + 1;
 967                 p = m->hw_handler_params = kzalloc(len, GFP_KERNEL);
 968                 if (!p) {
 969                         ti->error = "memory allocation failed";
 970                         ret = -ENOMEM;
 971                         goto fail;
 972                 }
 973                 j = sprintf(p, "%d", hw_argc - 1);
 974                 for (i = 0, p+=j+1; i <= hw_argc - 2; i++, p+=j+1)
 975                         j = sprintf(p, "%s", as->argv[i]);
 976         }
 977         dm_consume_args(as, hw_argc - 1);
 978
 979         return 0;
 980 fail:
 981         kfree(m->hw_handler_name);
 982         m->hw_handler_name = NULL;
 983         return ret;
 984 }
 985
 986 static int parse_features(struct dm_arg_set *as, struct multipath *m)
 987 {
 988         int r;
 989         unsigned argc;
 990         struct dm_target *ti = m->ti;
 991         const char *arg_name;
 992
 993         static const struct dm_arg _args[] = {
 994                 {0, 8, "invalid number of feature args"},
 995                 {1, 50, "pg_init_retries must be between 1 and 50"},
 996                 {0, 60000, "pg_init_delay_msecs must be between 0 and 60000"},
 997         };
 998
 999         r = dm_read_arg_group(_args, as, &argc, &ti->error);
1000         if (r)
1001                 return -EINVAL;
1002
1003         if (!argc)
1004                 return 0;
1005
1006         do {
1007                 arg_name = dm_shift_arg(as);
1008                 argc--;
1009
1010                 if (!strcasecmp(arg_name, "queue_if_no_path")) {
1011                         r = queue_if_no_path(m, true, false);
1012                         continue;
1013                 }
1014
1015                 if (!strcasecmp(arg_name, "retain_attached_hw_handler")) {
1016                         set_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags);
1017                         continue;
1018                 }
1019
1020                 if (!strcasecmp(arg_name, "pg_init_retries") &&
1021                     (argc >= 1)) {
1022                         r = dm_read_arg(_args + 1, as, &m->pg_init_retries, &ti->error);
1023                         argc--;
1024                         continue;
1025                 }
1026
1027                 if (!strcasecmp(arg_name, "pg_init_delay_msecs") &&
1028                     (argc >= 1)) {
1029                         r = dm_read_arg(_args + 2, as, &m->pg_init_delay_msecs, &ti->error);
1030                         argc--;
1031                         continue;
1032                 }
1033
1034                 if (!strcasecmp(arg_name, "queue_mode") &&
1035                     (argc >= 1)) {
1036                         const char *queue_mode_name = dm_shift_arg(as);
1037
1038                         if (!strcasecmp(queue_mode_name, "bio"))
1039                                 m->queue_mode = DM_TYPE_BIO_BASED;
1040                         else if (!strcasecmp(queue_mode_name, "nvme"))
1041                                 m->queue_mode = DM_TYPE_NVME_BIO_BASED;
1042                         else if (!strcasecmp(queue_mode_name, "rq"))
1043                                 m->queue_mode = DM_TYPE_REQUEST_BASED;
1044                         else if (!strcasecmp(queue_mode_name, "mq"))
1045                                 m->queue_mode = DM_TYPE_MQ_REQUEST_BASED;
1046                         else {
1047                                 ti->error = "Unknown 'queue_mode' requested";
1048                                 r = -EINVAL;
1049                         }
1050                         argc--;
1051                         continue;
1052                 }
1053
1054                 ti->error = "Unrecognised multipath feature request";
1055                 r = -EINVAL;
1056         } while (argc && !r);
1057
1058         return r;
1059 }
1060
1061 static int multipath_ctr(struct dm_target *ti, unsigned argc, char **argv)
1062 {
1063         /* target arguments */
1064         static const struct dm_arg _args[] = {
1065                 {0, 1024, "invalid number of priority groups"},
1066                 {0, 1024, "invalid initial priority group number"},
1067         };
1068
1069         int r;
1070         struct multipath *m;
1071         struct dm_arg_set as;
1072         unsigned pg_count = 0;
1073         unsigned next_pg_num;
1074
1075         as.argc = argc;
1076         as.argv = argv;
1077
1078         m = alloc_multipath(ti);
1079         if (!m) {
1080                 ti->error = "can't allocate multipath";
1081                 return -EINVAL;
1082         }
1083
1084         r = parse_features(&as, m);
1085         if (r)
1086                 goto bad;
1087
1088         r = alloc_multipath_stage2(ti, m);
1089         if (r)
1090                 goto bad;
1091
1092         r = parse_hw_handler(&as, m);
1093         if (r)
1094                 goto bad;
1095
1096         r = dm_read_arg(_args, &as, &m->nr_priority_groups, &ti->error);
1097         if (r)
1098                 goto bad;
1099
1100         r = dm_read_arg(_args + 1, &as, &next_pg_num, &ti->error);
1101         if (r)
1102                 goto bad;
1103
1104         if ((!m->nr_priority_groups && next_pg_num) ||
1105             (m->nr_priority_groups && !next_pg_num)) {
1106                 ti->error = "invalid initial priority group";
1107                 r = -EINVAL;
1108                 goto bad;
1109         }
1110
1111         /* parse the priority groups */
1112         while (as.argc) {
1113                 struct priority_group *pg;
1114                 unsigned nr_valid_paths = atomic_read(&m->nr_valid_paths);
1115
1116                 pg = parse_priority_group(&as, m);
1117                 if (IS_ERR(pg)) {
1118                         r = PTR_ERR(pg);
1119                         goto bad;
1120                 }
1121
1122                 nr_valid_paths += pg->nr_pgpaths;
1123                 atomic_set(&m->nr_valid_paths, nr_valid_paths);
1124
1125                 list_add_tail(&pg->list, &m->priority_groups);
1126                 pg_count++;
1127                 pg->pg_num = pg_count;
1128                 if (!--next_pg_num)
1129                         m->next_pg = pg;
1130         }
1131
1132         if (pg_count != m->nr_priority_groups) {
1133                 ti->error = "priority group count mismatch";
1134                 r = -EINVAL;
1135                 goto bad;
1136         }
1137
1138         ti->num_flush_bios = 1;
1139         ti->num_discard_bios = 1;
1140         ti->num_write_same_bios = 1;
1141         ti->num_write_zeroes_bios = 1;
1142         if (m->queue_mode == DM_TYPE_BIO_BASED || m->queue_mode == DM_TYPE_NVME_BIO_BASED)
1143                 ti->per_io_data_size = multipath_per_bio_data_size();
1144         else
1145                 ti->per_io_data_size = sizeof(struct dm_mpath_io);
1146
1147         return 0;
1148
1149  bad:
1150         free_multipath(m);
1151         return r;
1152 }
1153
1154 static void multipath_wait_for_pg_init_completion(struct multipath *m)
1155 {
1156         DEFINE_WAIT(wait);
1157
1158         while (1) {
1159                 prepare_to_wait(&m->pg_init_wait, &wait, TASK_UNINTERRUPTIBLE);
1160
1161                 if (!atomic_read(&m->pg_init_in_progress))
1162                         break;
1163
1164                 io_schedule();
1165         }
1166         finish_wait(&m->pg_init_wait, &wait);
1167 }
1168
1169 static void flush_multipath_work(struct multipath *m)
1170 {
1171         if (m->hw_handler_name) {
1172                 set_bit(MPATHF_PG_INIT_DISABLED, &m->flags);
1173                 smp_mb__after_atomic();
1174
1175                 flush_workqueue(kmpath_handlerd);
1176                 multipath_wait_for_pg_init_completion(m);
1177
1178                 clear_bit(MPATHF_PG_INIT_DISABLED, &m->flags);
1179                 smp_mb__after_atomic();
1180         }
1181
1182         flush_workqueue(kmultipathd);
1183         flush_work(&m->trigger_event);
1184 }
1185
1186 static void multipath_dtr(struct dm_target *ti)
1187 {
1188         struct multipath *m = ti->private;
1189
1190         flush_multipath_work(m);
1191         free_multipath(m);
1192 }
1193
1194 /*
1195  * Take a path out of use.
1196  */
1197 static int fail_path(struct pgpath *pgpath)
1198 {
1199         unsigned long flags;
1200         struct multipath *m = pgpath->pg->m;
1201
1202         spin_lock_irqsave(&m->lock, flags);
1203
1204         if (!pgpath->is_active)
1205                 goto out;
1206
1207         DMWARN("Failing path %s.", pgpath->path.dev->name);
1208
1209         pgpath->pg->ps.type->fail_path(&pgpath->pg->ps, &pgpath->path);
1210         pgpath->is_active = false;
1211         pgpath->fail_count++;
1212
1213         atomic_dec(&m->nr_valid_paths);
1214
1215         if (pgpath == m->current_pgpath)
1216                 m->current_pgpath = NULL;
1217
1218         dm_path_uevent(DM_UEVENT_PATH_FAILED, m->ti,
1219                        pgpath->path.dev->name, atomic_read(&m->nr_valid_paths));
1220
1221         schedule_work(&m->trigger_event);
1222
1223 out:
1224         spin_unlock_irqrestore(&m->lock, flags);
1225
1226         return 0;
1227 }
1228
1229 /*
1230  * Reinstate a previously-failed path
1231  */
1232 static int reinstate_path(struct pgpath *pgpath)
1233 {
1234         int r = 0, run_queue = 0;
1235         unsigned long flags;
1236         struct multipath *m = pgpath->pg->m;
1237         unsigned nr_valid_paths;
1238
1239         spin_lock_irqsave(&m->lock, flags);
1240
1241         if (pgpath->is_active)
1242                 goto out;
1243
1244         DMWARN("Reinstating path %s.", pgpath->path.dev->name);
1245
1246         r = pgpath->pg->ps.type->reinstate_path(&pgpath->pg->ps, &pgpath->path);
1247         if (r)
1248                 goto out;
1249
1250         pgpath->is_active = true;
1251
1252         nr_valid_paths = atomic_inc_return(&m->nr_valid_paths);
1253         if (nr_valid_paths == 1) {
1254                 m->current_pgpath = NULL;
1255                 run_queue = 1;
1256         } else if (m->hw_handler_name && (m->current_pg == pgpath->pg)) {
1257                 if (queue_work(kmpath_handlerd, &pgpath->activate_path.work))
1258                         atomic_inc(&m->pg_init_in_progress);
1259         }
1260
1261         dm_path_uevent(DM_UEVENT_PATH_REINSTATED, m->ti,
1262                        pgpath->path.dev->name, nr_valid_paths);
1263
1264         schedule_work(&m->trigger_event);
1265
1266 out:
1267         spin_unlock_irqrestore(&m->lock, flags);
1268         if (run_queue) {
1269                 dm_table_run_md_queue_async(m->ti->table);
1270                 process_queued_io_list(m);
1271         }
1272
1273         return r;
1274 }
1275
1276 /*
1277  * Fail or reinstate all paths that match the provided struct dm_dev.
1278  */
1279 static int action_dev(struct multipath *m, struct dm_dev *dev,
1280                       action_fn action)
1281 {
1282         int r = -EINVAL;
1283         struct pgpath *pgpath;
1284         struct priority_group *pg;
1285
1286         list_for_each_entry(pg, &m->priority_groups, list) {
1287                 list_for_each_entry(pgpath, &pg->pgpaths, list) {
1288                         if (pgpath->path.dev == dev)
1289                                 r = action(pgpath);
1290                 }
1291         }
1292
1293         return r;
1294 }
1295
1296 /*
1297  * Temporarily try to avoid having to use the specified PG
1298  */
1299 static void bypass_pg(struct multipath *m, struct priority_group *pg,
1300                       bool bypassed)
1301 {
1302         unsigned long flags;
1303
1304         spin_lock_irqsave(&m->lock, flags);
1305
1306         pg->bypassed = bypassed;
1307         m->current_pgpath = NULL;
1308         m->current_pg = NULL;
1309
1310         spin_unlock_irqrestore(&m->lock, flags);
1311
1312         schedule_work(&m->trigger_event);
1313 }
1314
1315 /*
1316  * Switch to using the specified PG from the next I/O that gets mapped
1317  */
1318 static int switch_pg_num(struct multipath *m, const char *pgstr)
1319 {
1320         struct priority_group *pg;
1321         unsigned pgnum;
1322         unsigned long flags;
1323         char dummy;
1324
1325         if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum ||
1326             !m->nr_priority_groups || (pgnum > m->nr_priority_groups)) {
1327                 DMWARN("invalid PG number supplied to switch_pg_num");
1328                 return -EINVAL;
1329         }
1330
1331         spin_lock_irqsave(&m->lock, flags);
1332         list_for_each_entry(pg, &m->priority_groups, list) {
1333                 pg->bypassed = false;
1334                 if (--pgnum)
1335                         continue;
1336
1337                 m->current_pgpath = NULL;
1338                 m->current_pg = NULL;
1339                 m->next_pg = pg;
1340         }
1341         spin_unlock_irqrestore(&m->lock, flags);
1342
1343         schedule_work(&m->trigger_event);
1344         return 0;
1345 }
1346
1347 /*
1348  * Set/clear bypassed status of a PG.
1349  * PGs are numbered upwards from 1 in the order they were declared.
1350  */
1351 static int bypass_pg_num(struct multipath *m, const char *pgstr, bool bypassed)
1352 {
1353         struct priority_group *pg;
1354         unsigned pgnum;
1355         char dummy;
1356
1357         if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum ||
1358             !m->nr_priority_groups || (pgnum > m->nr_priority_groups)) {
1359                 DMWARN("invalid PG number supplied to bypass_pg");
1360                 return -EINVAL;
1361         }
1362
1363         list_for_each_entry(pg, &m->priority_groups, list) {
1364                 if (!--pgnum)
1365                         break;
1366         }
1367
1368         bypass_pg(m, pg, bypassed);
1369         return 0;
1370 }
1371
1372 /*
1373  * Should we retry pg_init immediately?
1374  */
1375 static bool pg_init_limit_reached(struct multipath *m, struct pgpath *pgpath)
1376 {
1377         unsigned long flags;
1378         bool limit_reached = false;
1379
1380         spin_lock_irqsave(&m->lock, flags);
1381
1382         if (atomic_read(&m->pg_init_count) <= m->pg_init_retries &&
1383             !test_bit(MPATHF_PG_INIT_DISABLED, &m->flags))
1384                 set_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
1385         else
1386                 limit_reached = true;
1387
1388         spin_unlock_irqrestore(&m->lock, flags);
1389
1390         return limit_reached;
1391 }
1392
1393 static void pg_init_done(void *data, int errors)
1394 {
1395         struct pgpath *pgpath = data;
1396         struct priority_group *pg = pgpath->pg;
1397         struct multipath *m = pg->m;
1398         unsigned long flags;
1399         bool delay_retry = false;
1400
1401         /* device or driver problems */
1402         switch (errors) {
1403         case SCSI_DH_OK:
1404                 break;
1405         case SCSI_DH_NOSYS:
1406                 if (!m->hw_handler_name) {
1407                         errors = 0;
1408                         break;
1409                 }
1410                 DMERR("Could not failover the device: Handler scsi_dh_%s "
1411                       "Error %d.", m->hw_handler_name, errors);
1412                 /*
1413                  * Fail path for now, so we do not ping pong
1414                  */
1415                 fail_path(pgpath);
1416                 break;
1417         case SCSI_DH_DEV_TEMP_BUSY:
1418                 /*
1419                  * Probably doing something like FW upgrade on the
1420                  * controller so try the other pg.
1421                  */
1422                 bypass_pg(m, pg, true);
1423                 break;
1424         case SCSI_DH_RETRY:
1425                 /* Wait before retrying. */
1426                 delay_retry = 1;
1427                 /* fall through */
1428         case SCSI_DH_IMM_RETRY:
1429         case SCSI_DH_RES_TEMP_UNAVAIL:
1430                 if (pg_init_limit_reached(m, pgpath))
1431                         fail_path(pgpath);
1432                 errors = 0;
1433                 break;
1434         case SCSI_DH_DEV_OFFLINED:
1435         default:
1436                 /*
1437                  * We probably do not want to fail the path for a device
1438                  * error, but this is what the old dm did. In future
1439                  * patches we can do more advanced handling.
1440                  */
1441                 fail_path(pgpath);
1442         }
1443
1444         spin_lock_irqsave(&m->lock, flags);
1445         if (errors) {
1446                 if (pgpath == m->current_pgpath) {
1447                         DMERR("Could not failover device. Error %d.", errors);
1448                         m->current_pgpath = NULL;
1449                         m->current_pg = NULL;
1450                 }
1451         } else if (!test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags))
1452                 pg->bypassed = false;
1453
1454         if (atomic_dec_return(&m->pg_init_in_progress) > 0)
1455                 /* Activations of other paths are still on going */
1456                 goto out;
1457
1458         if (test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) {
1459                 if (delay_retry)
1460                         set_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags);
1461                 else
1462                         clear_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags);
1463
1464                 if (__pg_init_all_paths(m))
1465                         goto out;
1466         }
1467         clear_bit(MPATHF_QUEUE_IO, &m->flags);
1468
1469         process_queued_io_list(m);
1470
1471         /*
1472          * Wake up any thread waiting to suspend.
1473          */
1474         wake_up(&m->pg_init_wait);
1475
1476 out:
1477         spin_unlock_irqrestore(&m->lock, flags);
1478 }
1479
1480 static void activate_or_offline_path(struct pgpath *pgpath)
1481 {
1482         struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev);
1483
1484         if (pgpath->is_active && !blk_queue_dying(q))
1485                 scsi_dh_activate(q, pg_init_done, pgpath);
1486         else
1487                 pg_init_done(pgpath, SCSI_DH_DEV_OFFLINED);
1488 }
1489
1490 static void activate_path_work(struct work_struct *work)
1491 {
1492         struct pgpath *pgpath =
1493                 container_of(work, struct pgpath, activate_path.work);
1494
1495         activate_or_offline_path(pgpath);
1496 }
1497
1498 static int noretry_error(blk_status_t error)
1499 {
1500         switch (error) {
1501         case BLK_STS_NOTSUPP:
1502         case BLK_STS_NOSPC:
1503         case BLK_STS_TARGET:
1504         case BLK_STS_NEXUS:
1505         case BLK_STS_MEDIUM:
1506                 return 1;
1507         }
1508
1509         /* Anything else could be a path failure, so should be retried */
1510         return 0;
1511 }
1512
1513 static int multipath_end_io(struct dm_target *ti, struct request *clone,
1514                             blk_status_t error, union map_info *map_context)
1515 {
1516         struct dm_mpath_io *mpio = get_mpio(map_context);
1517         struct pgpath *pgpath = mpio->pgpath;
1518         int r = DM_ENDIO_DONE;
1519
1520         /*
1521          * We don't queue any clone request inside the multipath target
1522          * during end I/O handling, since those clone requests don't have
1523          * bio clones.  If we queue them inside the multipath target,
1524          * we need to make bio clones, that requires memory allocation.
1525          * (See drivers/md/dm-rq.c:end_clone_bio() about why the clone requests
1526          *  don't have bio clones.)
1527          * Instead of queueing the clone request here, we queue the original
1528          * request into dm core, which will remake a clone request and
1529          * clone bios for it and resubmit it later.
1530          */
1531         if (error && !noretry_error(error)) {
1532                 struct multipath *m = ti->private;
1533
1534                 r = DM_ENDIO_REQUEUE;
1535
1536                 if (pgpath)
1537                         fail_path(pgpath);
1538
1539                 if (atomic_read(&m->nr_valid_paths) == 0 &&
1540                     !must_push_back_rq(m)) {
1541                         if (error == BLK_STS_IOERR)
1542                                 dm_report_EIO(m);
1543                         /* complete with the original error */
1544                         r = DM_ENDIO_DONE;
1545                 }
1546         }
1547
1548         if (pgpath) {
1549                 struct path_selector *ps = &pgpath->pg->ps;
1550
1551                 if (ps->type->end_io)
1552                         ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes);
1553         }
1554
1555         return r;
1556 }
1557
1558 static int multipath_end_io_bio(struct dm_target *ti, struct bio *clone,
1559                                 blk_status_t *error)
1560 {
1561         struct multipath *m = ti->private;
1562         struct dm_mpath_io *mpio = get_mpio_from_bio(clone);
1563         struct pgpath *pgpath = mpio->pgpath;
1564         unsigned long flags;
1565         int r = DM_ENDIO_DONE;
1566
1567         if (!*error || noretry_error(*error))
1568                 goto done;
1569
1570         if (pgpath)
1571                 fail_path(pgpath);
1572
1573         if (atomic_read(&m->nr_valid_paths) == 0 &&
1574             !test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) {
1575                 if (must_push_back_bio(m)) {
1576                         r = DM_ENDIO_REQUEUE;
1577                 } else {
1578                         dm_report_EIO(m);
1579                         *error = BLK_STS_IOERR;
1580                 }
1581                 goto done;
1582         }
1583
1584         spin_lock_irqsave(&m->lock, flags);
1585         bio_list_add(&m->queued_bios, clone);
1586         spin_unlock_irqrestore(&m->lock, flags);
1587         if (!test_bit(MPATHF_QUEUE_IO, &m->flags))
1588                 queue_work(kmultipathd, &m->process_queued_bios);
1589
1590         r = DM_ENDIO_INCOMPLETE;
1591 done:
1592         if (pgpath) {
1593                 struct path_selector *ps = &pgpath->pg->ps;
1594
1595                 if (ps->type->end_io)
1596                         ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes);
1597         }
1598
1599         return r;
1600 }
1601
1602 /*
1603  * Suspend can't complete until all the I/O is processed so if
1604  * the last path fails we must error any remaining I/O.
1605  * Note that if the freeze_bdev fails while suspending, the
1606  * queue_if_no_path state is lost - userspace should reset it.
1607  */
1608 static void multipath_presuspend(struct dm_target *ti)
1609 {
1610         struct multipath *m = ti->private;
1611
1612         queue_if_no_path(m, false, true);
1613 }
1614
1615 static void multipath_postsuspend(struct dm_target *ti)
1616 {
1617         struct multipath *m = ti->private;
1618
1619         mutex_lock(&m->work_mutex);
1620         flush_multipath_work(m);
1621         mutex_unlock(&m->work_mutex);
1622 }
1623
1624 /*
1625  * Restore the queue_if_no_path setting.
1626  */
1627 static void multipath_resume(struct dm_target *ti)
1628 {
1629         struct multipath *m = ti->private;
1630         unsigned long flags;
1631
1632         spin_lock_irqsave(&m->lock, flags);
1633         assign_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags,
1634                    test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags));
1635         spin_unlock_irqrestore(&m->lock, flags);
1636 }
1637
1638 /*
1639  * Info output has the following format:
1640  * num_multipath_feature_args [multipath_feature_args]*
1641  * num_handler_status_args [handler_status_args]*
1642  * num_groups init_group_number
1643  *            [A|D|E num_ps_status_args [ps_status_args]*
1644  *             num_paths num_selector_args
1645  *             [path_dev A|F fail_count [selector_args]* ]+ ]+
1646  *
1647  * Table output has the following format (identical to the constructor string):
1648  * num_feature_args [features_args]*
1649  * num_handler_args hw_handler [hw_handler_args]*
1650  * num_groups init_group_number
1651  *     [priority selector-name num_ps_args [ps_args]*
1652  *      num_paths num_selector_args [path_dev [selector_args]* ]+ ]+
1653  */
1654 static void multipath_status(struct dm_target *ti, status_type_t type,
1655                              unsigned status_flags, char *result, unsigned maxlen)
1656 {
1657         int sz = 0;
1658         unsigned long flags;
1659         struct multipath *m = ti->private;
1660         struct priority_group *pg;
1661         struct pgpath *p;
1662         unsigned pg_num;
1663         char state;
1664
1665         spin_lock_irqsave(&m->lock, flags);
1666
1667         /* Features */
1668         if (type == STATUSTYPE_INFO)
1669                 DMEMIT("2 %u %u ", test_bit(MPATHF_QUEUE_IO, &m->flags),
1670                        atomic_read(&m->pg_init_count));
1671         else {
1672                 DMEMIT("%u ", test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags) +
1673                               (m->pg_init_retries > 0) * 2 +
1674                               (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT) * 2 +
1675                               test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags) +
1676                               (m->queue_mode != DM_TYPE_REQUEST_BASED) * 2);
1677
1678                 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))
1679                         DMEMIT("queue_if_no_path ");
1680                 if (m->pg_init_retries)
1681                         DMEMIT("pg_init_retries %u ", m->pg_init_retries);
1682                 if (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT)
1683                         DMEMIT("pg_init_delay_msecs %u ", m->pg_init_delay_msecs);
1684                 if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags))
1685                         DMEMIT("retain_attached_hw_handler ");
1686                 if (m->queue_mode != DM_TYPE_REQUEST_BASED) {
1687                         switch(m->queue_mode) {
1688                         case DM_TYPE_BIO_BASED:
1689                                 DMEMIT("queue_mode bio ");
1690                                 break;
1691                         case DM_TYPE_NVME_BIO_BASED:
1692                                 DMEMIT("queue_mode nvme ");
1693                                 break;
1694                         case DM_TYPE_MQ_REQUEST_BASED:
1695                                 DMEMIT("queue_mode mq ");
1696                                 break;
1697                         default:
1698                                 WARN_ON_ONCE(true);
1699                                 break;
1700                         }
1701                 }
1702         }
1703
1704         if (!m->hw_handler_name || type == STATUSTYPE_INFO)
1705                 DMEMIT("0 ");
1706         else
1707                 DMEMIT("1 %s ", m->hw_handler_name);
1708
1709         DMEMIT("%u ", m->nr_priority_groups);
1710
1711         if (m->next_pg)
1712                 pg_num = m->next_pg->pg_num;
1713         else if (m->current_pg)
1714                 pg_num = m->current_pg->pg_num;
1715         else
1716                 pg_num = (m->nr_priority_groups ? 1 : 0);
1717
1718         DMEMIT("%u ", pg_num);
1719
1720         switch (type) {
1721         case STATUSTYPE_INFO:
1722                 list_for_each_entry(pg, &m->priority_groups, list) {
1723                         if (pg->bypassed)
1724                                 state = 'D';    /* Disabled */
1725                         else if (pg == m->current_pg)
1726                                 state = 'A';    /* Currently Active */
1727                         else
1728                                 state = 'E';    /* Enabled */
1729
1730                         DMEMIT("%c ", state);
1731
1732                         if (pg->ps.type->status)
1733                                 sz += pg->ps.type->status(&pg->ps, NULL, type,
1734                                                           result + sz,
1735                                                           maxlen - sz);
1736                         else
1737                                 DMEMIT("0 ");
1738
1739                         DMEMIT("%u %u ", pg->nr_pgpaths,
1740                                pg->ps.type->info_args);
1741
1742                         list_for_each_entry(p, &pg->pgpaths, list) {
1743                                 DMEMIT("%s %s %u ", p->path.dev->name,
1744                                        p->is_active ? "A" : "F",
1745                                        p->fail_count);
1746                                 if (pg->ps.type->status)
1747                                         sz += pg->ps.type->status(&pg->ps,
1748                                               &p->path, type, result + sz,
1749                                               maxlen - sz);
1750                         }
1751                 }
1752                 break;
1753
1754         case STATUSTYPE_TABLE:
1755                 list_for_each_entry(pg, &m->priority_groups, list) {
1756                         DMEMIT("%s ", pg->ps.type->name);
1757
1758                         if (pg->ps.type->status)
1759                                 sz += pg->ps.type->status(&pg->ps, NULL, type,
1760                                                           result + sz,
1761                                                           maxlen - sz);
1762                         else
1763                                 DMEMIT("0 ");
1764
1765                         DMEMIT("%u %u ", pg->nr_pgpaths,
1766                                pg->ps.type->table_args);
1767
1768                         list_for_each_entry(p, &pg->pgpaths, list) {
1769                                 DMEMIT("%s ", p->path.dev->name);
1770                                 if (pg->ps.type->status)
1771                                         sz += pg->ps.type->status(&pg->ps,
1772                                               &p->path, type, result + sz,
1773                                               maxlen - sz);
1774                         }
1775                 }
1776                 break;
1777         }
1778
1779         spin_unlock_irqrestore(&m->lock, flags);
1780 }
1781
1782 static int multipath_message(struct dm_target *ti, unsigned argc, char **argv)
1783 {
1784         int r = -EINVAL;
1785         struct dm_dev *dev;
1786         struct multipath *m = ti->private;
1787         action_fn action;
1788
1789         mutex_lock(&m->work_mutex);
1790
1791         if (dm_suspended(ti)) {
1792                 r = -EBUSY;
1793                 goto out;
1794         }
1795
1796         if (argc == 1) {
1797                 if (!strcasecmp(argv[0], "queue_if_no_path")) {
1798                         r = queue_if_no_path(m, true, false);
1799                         goto out;
1800                 } else if (!strcasecmp(argv[0], "fail_if_no_path")) {
1801                         r = queue_if_no_path(m, false, false);
1802                         goto out;
1803                 }
1804         }
1805
1806         if (argc != 2) {
1807                 DMWARN("Invalid multipath message arguments. Expected 2 arguments, got %d.", argc);
1808                 goto out;
1809         }
1810
1811         if (!strcasecmp(argv[0], "disable_group")) {
1812                 r = bypass_pg_num(m, argv[1], true);
1813                 goto out;
1814         } else if (!strcasecmp(argv[0], "enable_group")) {
1815                 r = bypass_pg_num(m, argv[1], false);
1816                 goto out;
1817         } else if (!strcasecmp(argv[0], "switch_group")) {
1818                 r = switch_pg_num(m, argv[1]);
1819                 goto out;
1820         } else if (!strcasecmp(argv[0], "reinstate_path"))
1821                 action = reinstate_path;
1822         else if (!strcasecmp(argv[0], "fail_path"))
1823                 action = fail_path;
1824         else {
1825                 DMWARN("Unrecognised multipath message received: %s", argv[0]);
1826                 goto out;
1827         }
1828
1829         r = dm_get_device(ti, argv[1], dm_table_get_mode(ti->table), &dev);
1830         if (r) {
1831                 DMWARN("message: error getting device %s",
1832                        argv[1]);
1833                 goto out;
1834         }
1835
1836         r = action_dev(m, dev, action);
1837
1838         dm_put_device(ti, dev);
1839
1840 out:
1841         mutex_unlock(&m->work_mutex);
1842         return r;
1843 }
1844
1845 static int multipath_prepare_ioctl(struct dm_target *ti,
1846                 struct block_device **bdev, fmode_t *mode)
1847 {
1848         struct multipath *m = ti->private;
1849         struct pgpath *current_pgpath;
1850         int r;
1851
1852         current_pgpath = READ_ONCE(m->current_pgpath);
1853         if (!current_pgpath)
1854                 current_pgpath = choose_pgpath(m, 0);
1855
1856         if (current_pgpath) {
1857                 if (!test_bit(MPATHF_QUEUE_IO, &m->flags)) {
1858                         *bdev = current_pgpath->path.dev->bdev;
1859                         *mode = current_pgpath->path.dev->mode;
1860                         r = 0;
1861                 } else {
1862                         /* pg_init has not started or completed */
1863                         r = -ENOTCONN;
1864                 }
1865         } else {
1866                 /* No path is available */
1867                 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))
1868                         r = -ENOTCONN;
1869                 else
1870                         r = -EIO;
1871         }
1872
1873         if (r == -ENOTCONN) {
1874                 if (!READ_ONCE(m->current_pg)) {
1875                         /* Path status changed, redo selection */
1876                         (void) choose_pgpath(m, 0);
1877                 }
1878                 if (test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags))
1879                         pg_init_all_paths(m);
1880                 dm_table_run_md_queue_async(m->ti->table);
1881                 process_queued_io_list(m);
1882         }
1883
1884         /*
1885          * Only pass ioctls through if the device sizes match exactly.
1886          */
1887         if (!r && ti->len != i_size_read((*bdev)->bd_inode) >> SECTOR_SHIFT)
1888                 return 1;
1889         return r;
1890 }
1891
1892 static int multipath_iterate_devices(struct dm_target *ti,
1893                                      iterate_devices_callout_fn fn, void *data)
1894 {
1895         struct multipath *m = ti->private;
1896         struct priority_group *pg;
1897         struct pgpath *p;
1898         int ret = 0;
1899
1900         list_for_each_entry(pg, &m->priority_groups, list) {
1901                 list_for_each_entry(p, &pg->pgpaths, list) {
1902                         ret = fn(ti, p->path.dev, ti->begin, ti->len, data);
1903                         if (ret)
1904                                 goto out;
1905                 }
1906         }
1907
1908 out:
1909         return ret;
1910 }
1911
1912 static int pgpath_busy(struct pgpath *pgpath)
1913 {
1914         struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev);
1915
1916         return blk_lld_busy(q);
1917 }
1918
1919 /*
1920  * We return "busy", only when we can map I/Os but underlying devices
1921  * are busy (so even if we map I/Os now, the I/Os will wait on
1922  * the underlying queue).
1923  * In other words, if we want to kill I/Os or queue them inside us
1924  * due to map unavailability, we don't return "busy".  Otherwise,
1925  * dm core won't give us the I/Os and we can't do what we want.
1926  */
1927 static int multipath_busy(struct dm_target *ti)
1928 {
1929         bool busy = false, has_active = false;
1930         struct multipath *m = ti->private;
1931         struct priority_group *pg, *next_pg;
1932         struct pgpath *pgpath;
1933
1934         /* pg_init in progress */
1935         if (atomic_read(&m->pg_init_in_progress))
1936                 return true;
1937
1938         /* no paths available, for blk-mq: rely on IO mapping to delay requeue */
1939         if (!atomic_read(&m->nr_valid_paths) && test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))
1940                 return (m->queue_mode != DM_TYPE_MQ_REQUEST_BASED);
1941
1942         /* Guess which priority_group will be used at next mapping time */
1943         pg = READ_ONCE(m->current_pg);
1944         next_pg = READ_ONCE(m->next_pg);
1945         if (unlikely(!READ_ONCE(m->current_pgpath) && next_pg))
1946                 pg = next_pg;
1947
1948         if (!pg) {
1949                 /*
1950                  * We don't know which pg will be used at next mapping time.
1951                  * We don't call choose_pgpath() here to avoid to trigger
1952                  * pg_init just by busy checking.
1953                  * So we don't know whether underlying devices we will be using
1954                  * at next mapping time are busy or not. Just try mapping.
1955                  */
1956                 return busy;
1957         }
1958
1959         /*
1960          * If there is one non-busy active path at least, the path selector
1961          * will be able to select it. So we consider such a pg as not busy.
1962          */
1963         busy = true;
1964         list_for_each_entry(pgpath, &pg->pgpaths, list) {
1965                 if (pgpath->is_active) {
1966                         has_active = true;
1967                         if (!pgpath_busy(pgpath)) {
1968                                 busy = false;
1969                                 break;
1970                         }
1971                 }
1972         }
1973
1974         if (!has_active) {
1975                 /*
1976                  * No active path in this pg, so this pg won't be used and
1977                  * the current_pg will be changed at next mapping time.
1978                  * We need to try mapping to determine it.
1979                  */
1980                 busy = false;
1981         }
1982
1983         return busy;
1984 }
1985
1986 /*-----------------------------------------------------------------
1987  * Module setup
1988  *---------------------------------------------------------------*/
1989 static struct target_type multipath_target = {
1990         .name = "multipath",
1991         .version = {1, 12, 0},
1992         .features = DM_TARGET_SINGLETON | DM_TARGET_IMMUTABLE,
1993         .module = THIS_MODULE,
1994         .ctr = multipath_ctr,
1995         .dtr = multipath_dtr,
1996         .clone_and_map_rq = multipath_clone_and_map,
1997         .release_clone_rq = multipath_release_clone,
1998         .rq_end_io = multipath_end_io,
1999         .map = multipath_map_bio,
2000         .end_io = multipath_end_io_bio,
2001         .presuspend = multipath_presuspend,
2002         .postsuspend = multipath_postsuspend,
2003         .resume = multipath_resume,
2004         .status = multipath_status,
2005         .message = multipath_message,
2006         .prepare_ioctl = multipath_prepare_ioctl,
2007         .iterate_devices = multipath_iterate_devices,
2008         .busy = multipath_busy,
2009 };
2010
2011 static int __init dm_multipath_init(void)
2012 {
2013         int r;
2014
2015         kmultipathd = alloc_workqueue("kmpathd", WQ_MEM_RECLAIM, 0);
2016         if (!kmultipathd) {
2017                 DMERR("failed to create workqueue kmpathd");
2018                 r = -ENOMEM;
2019                 goto bad_alloc_kmultipathd;
2020         }
2021
2022         /*
2023          * A separate workqueue is used to handle the device handlers
2024          * to avoid overloading existing workqueue. Overloading the
2025          * old workqueue would also create a bottleneck in the
2026          * path of the storage hardware device activation.
2027          */
2028         kmpath_handlerd = alloc_ordered_workqueue("kmpath_handlerd",
2029                                                   WQ_MEM_RECLAIM);
2030         if (!kmpath_handlerd) {
2031                 DMERR("failed to create workqueue kmpath_handlerd");
2032                 r = -ENOMEM;
2033                 goto bad_alloc_kmpath_handlerd;
2034         }
2035
2036         r = dm_register_target(&multipath_target);
2037         if (r < 0) {
2038                 DMERR("request-based register failed %d", r);
2039                 r = -EINVAL;
2040                 goto bad_register_target;
2041         }
2042
2043         return 0;
2044
2045 bad_register_target:
2046         destroy_workqueue(kmpath_handlerd);
2047 bad_alloc_kmpath_handlerd:
2048         destroy_workqueue(kmultipathd);
2049 bad_alloc_kmultipathd:
2050         return r;
2051 }
2052
2053 static void __exit dm_multipath_exit(void)
2054 {
2055         destroy_workqueue(kmpath_handlerd);
2056         destroy_workqueue(kmultipathd);
2057
2058         dm_unregister_target(&multipath_target);
2059 }
2060
2061 module_init(dm_multipath_init);
2062 module_exit(dm_multipath_exit);
2063
2064 MODULE_DESCRIPTION(DM_NAME " multipath target");
2065 MODULE_AUTHOR("Sistina Software <dm-devel@redhat.com>");
2066 MODULE_LICENSE("GPL");