git.rot13.org Git - linux/blob - drivers/ntb/hw/mscc/ntb_hw_switchtec.c

   1 /*
   2  * Microsemi Switchtec(tm) PCIe Management Driver
   3  * Copyright (c) 2017, Microsemi Corporation
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms and conditions of the GNU General Public License,
   7  * version 2, as published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  12  * more details.
  13  *
  14  */
  15
  16 #include <linux/interrupt.h>
  17 #include <linux/io-64-nonatomic-lo-hi.h>
  18 #include <linux/delay.h>
  19 #include <linux/kthread.h>
  20 #include <linux/module.h>
  21 #include <linux/ntb.h>
  22 #include <linux/pci.h>
  23 #include <linux/switchtec.h>
  24
  25 MODULE_DESCRIPTION("Microsemi Switchtec(tm) NTB Driver");
  26 MODULE_VERSION("0.1");
  27 MODULE_LICENSE("GPL");
  28 MODULE_AUTHOR("Microsemi Corporation");
  29
  30 static ulong max_mw_size = SZ_2M;
  31 module_param(max_mw_size, ulong, 0644);
  32 MODULE_PARM_DESC(max_mw_size,
  33         "Max memory window size reported to the upper layer");
  34
  35 static bool use_lut_mws;
  36 module_param(use_lut_mws, bool, 0644);
  37 MODULE_PARM_DESC(use_lut_mws,
  38                  "Enable the use of the LUT based memory windows");
  39
  40 #define SWITCHTEC_NTB_MAGIC 0x45CC0001
  41 #define MAX_MWS     128
  42
  43 struct shared_mw {
  44         u32 magic;
  45         u32 link_sta;
  46         u32 partition_id;
  47         u64 mw_sizes[MAX_MWS];
  48         u32 spad[128];
  49 };
  50
  51 #define MAX_DIRECT_MW ARRAY_SIZE(((struct ntb_ctrl_regs *)(0))->bar_entry)
  52 #define LUT_SIZE SZ_64K
  53
  54 struct switchtec_ntb {
  55         struct ntb_dev ntb;
  56         struct switchtec_dev *stdev;
  57
  58         int self_partition;
  59         int peer_partition;
  60
  61         int doorbell_irq;
  62         int message_irq;
  63
  64         struct ntb_info_regs __iomem *mmio_ntb;
  65         struct ntb_ctrl_regs __iomem *mmio_ctrl;
  66         struct ntb_dbmsg_regs __iomem *mmio_dbmsg;
  67         struct ntb_ctrl_regs __iomem *mmio_self_ctrl;
  68         struct ntb_ctrl_regs __iomem *mmio_peer_ctrl;
  69         struct ntb_dbmsg_regs __iomem *mmio_self_dbmsg;
  70         struct ntb_dbmsg_regs __iomem *mmio_peer_dbmsg;
  71
  72         void __iomem *mmio_xlink_win;
  73
  74         struct shared_mw *self_shared;
  75         struct shared_mw __iomem *peer_shared;
  76         dma_addr_t self_shared_dma;
  77
  78         u64 db_mask;
  79         u64 db_valid_mask;
  80         int db_shift;
  81         int db_peer_shift;
  82
  83         /* synchronize rmw access of db_mask and hw reg */
  84         spinlock_t db_mask_lock;
  85
  86         int nr_direct_mw;
  87         int nr_lut_mw;
  88         int nr_rsvd_luts;
  89         int direct_mw_to_bar[MAX_DIRECT_MW];
  90
  91         int peer_nr_direct_mw;
  92         int peer_nr_lut_mw;
  93         int peer_direct_mw_to_bar[MAX_DIRECT_MW];
  94
  95         bool link_is_up;
  96         enum ntb_speed link_speed;
  97         enum ntb_width link_width;
  98         struct work_struct link_reinit_work;
  99 };
 100
 101 static struct switchtec_ntb *ntb_sndev(struct ntb_dev *ntb)
 102 {
 103         return container_of(ntb, struct switchtec_ntb, ntb);
 104 }
 105
 106 static int switchtec_ntb_part_op(struct switchtec_ntb *sndev,
 107                                  struct ntb_ctrl_regs __iomem *ctl,
 108                                  u32 op, int wait_status)
 109 {
 110         static const char * const op_text[] = {
 111                 [NTB_CTRL_PART_OP_LOCK] = "lock",
 112                 [NTB_CTRL_PART_OP_CFG] = "configure",
 113                 [NTB_CTRL_PART_OP_RESET] = "reset",
 114         };
 115
 116         int i;
 117         u32 ps;
 118         int status;
 119
 120         switch (op) {
 121         case NTB_CTRL_PART_OP_LOCK:
 122                 status = NTB_CTRL_PART_STATUS_LOCKING;
 123                 break;
 124         case NTB_CTRL_PART_OP_CFG:
 125                 status = NTB_CTRL_PART_STATUS_CONFIGURING;
 126                 break;
 127         case NTB_CTRL_PART_OP_RESET:
 128                 status = NTB_CTRL_PART_STATUS_RESETTING;
 129                 break;
 130         default:
 131                 return -EINVAL;
 132         }
 133
 134         iowrite32(op, &ctl->partition_op);
 135
 136         for (i = 0; i < 1000; i++) {
 137                 if (msleep_interruptible(50) != 0) {
 138                         iowrite32(NTB_CTRL_PART_OP_RESET, &ctl->partition_op);
 139                         return -EINTR;
 140                 }
 141
 142                 ps = ioread32(&ctl->partition_status) & 0xFFFF;
 143
 144                 if (ps != status)
 145                         break;
 146         }
 147
 148         if (ps == wait_status)
 149                 return 0;
 150
 151         if (ps == status) {
 152                 dev_err(&sndev->stdev->dev,
 153                         "Timed out while performing %s (%d). (%08x)\n",
 154                         op_text[op], op,
 155                         ioread32(&ctl->partition_status));
 156
 157                 return -ETIMEDOUT;
 158         }
 159
 160         return -EIO;
 161 }
 162
 163 static int switchtec_ntb_send_msg(struct switchtec_ntb *sndev, int idx,
 164                                   u32 val)
 165 {
 166         if (idx < 0 || idx >= ARRAY_SIZE(sndev->mmio_peer_dbmsg->omsg))
 167                 return -EINVAL;
 168
 169         iowrite32(val, &sndev->mmio_peer_dbmsg->omsg[idx].msg);
 170
 171         return 0;
 172 }
 173
 174 static int switchtec_ntb_mw_count(struct ntb_dev *ntb, int pidx)
 175 {
 176         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 177         int nr_direct_mw = sndev->peer_nr_direct_mw;
 178         int nr_lut_mw = sndev->peer_nr_lut_mw - sndev->nr_rsvd_luts;
 179
 180         if (pidx != NTB_DEF_PEER_IDX)
 181                 return -EINVAL;
 182
 183         if (!use_lut_mws)
 184                 nr_lut_mw = 0;
 185
 186         return nr_direct_mw + nr_lut_mw;
 187 }
 188
 189 static int lut_index(struct switchtec_ntb *sndev, int mw_idx)
 190 {
 191         return mw_idx - sndev->nr_direct_mw + sndev->nr_rsvd_luts;
 192 }
 193
 194 static int peer_lut_index(struct switchtec_ntb *sndev, int mw_idx)
 195 {
 196         return mw_idx - sndev->peer_nr_direct_mw + sndev->nr_rsvd_luts;
 197 }
 198
 199 static int switchtec_ntb_mw_get_align(struct ntb_dev *ntb, int pidx,
 200                                       int widx, resource_size_t *addr_align,
 201                                       resource_size_t *size_align,
 202                                       resource_size_t *size_max)
 203 {
 204         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 205         int lut;
 206         resource_size_t size;
 207
 208         if (pidx != NTB_DEF_PEER_IDX)
 209                 return -EINVAL;
 210
 211         lut = widx >= sndev->peer_nr_direct_mw;
 212         size = ioread64(&sndev->peer_shared->mw_sizes[widx]);
 213
 214         if (size == 0)
 215                 return -EINVAL;
 216
 217         if (addr_align)
 218                 *addr_align = lut ? size : SZ_4K;
 219
 220         if (size_align)
 221                 *size_align = lut ? size : SZ_4K;
 222
 223         if (size_max)
 224                 *size_max = size;
 225
 226         return 0;
 227 }
 228
 229 static void switchtec_ntb_mw_clr_direct(struct switchtec_ntb *sndev, int idx)
 230 {
 231         struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
 232         int bar = sndev->peer_direct_mw_to_bar[idx];
 233         u32 ctl_val;
 234
 235         ctl_val = ioread32(&ctl->bar_entry[bar].ctl);
 236         ctl_val &= ~NTB_CTRL_BAR_DIR_WIN_EN;
 237         iowrite32(ctl_val, &ctl->bar_entry[bar].ctl);
 238         iowrite32(0, &ctl->bar_entry[bar].win_size);
 239         iowrite32(0, &ctl->bar_ext_entry[bar].win_size);
 240         iowrite64(sndev->self_partition, &ctl->bar_entry[bar].xlate_addr);
 241 }
 242
 243 static void switchtec_ntb_mw_clr_lut(struct switchtec_ntb *sndev, int idx)
 244 {
 245         struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
 246
 247         iowrite64(0, &ctl->lut_entry[peer_lut_index(sndev, idx)]);
 248 }
 249
 250 static void switchtec_ntb_mw_set_direct(struct switchtec_ntb *sndev, int idx,
 251                                         dma_addr_t addr, resource_size_t size)
 252 {
 253         int xlate_pos = ilog2(size);
 254         int bar = sndev->peer_direct_mw_to_bar[idx];
 255         struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
 256         u32 ctl_val;
 257
 258         ctl_val = ioread32(&ctl->bar_entry[bar].ctl);
 259         ctl_val |= NTB_CTRL_BAR_DIR_WIN_EN;
 260
 261         iowrite32(ctl_val, &ctl->bar_entry[bar].ctl);
 262         iowrite32(xlate_pos | (lower_32_bits(size) & 0xFFFFF000),
 263                   &ctl->bar_entry[bar].win_size);
 264         iowrite32(upper_32_bits(size), &ctl->bar_ext_entry[bar].win_size);
 265         iowrite64(sndev->self_partition | addr,
 266                   &ctl->bar_entry[bar].xlate_addr);
 267 }
 268
 269 static void switchtec_ntb_mw_set_lut(struct switchtec_ntb *sndev, int idx,
 270                                      dma_addr_t addr, resource_size_t size)
 271 {
 272         struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
 273
 274         iowrite64((NTB_CTRL_LUT_EN | (sndev->self_partition << 1) | addr),
 275                   &ctl->lut_entry[peer_lut_index(sndev, idx)]);
 276 }
 277
 278 static int switchtec_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int widx,
 279                                       dma_addr_t addr, resource_size_t size)
 280 {
 281         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 282         struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl;
 283         int xlate_pos = ilog2(size);
 284         int nr_direct_mw = sndev->peer_nr_direct_mw;
 285         int rc;
 286
 287         if (pidx != NTB_DEF_PEER_IDX)
 288                 return -EINVAL;
 289
 290         dev_dbg(&sndev->stdev->dev, "MW %d: part %d addr %pad size %pap\n",
 291                 widx, pidx, &addr, &size);
 292
 293         if (widx >= switchtec_ntb_mw_count(ntb, pidx))
 294                 return -EINVAL;
 295
 296         if (xlate_pos < 12)
 297                 return -EINVAL;
 298
 299         if (!IS_ALIGNED(addr, BIT_ULL(xlate_pos))) {
 300                 /*
 301                  * In certain circumstances we can get a buffer that is
 302                  * not aligned to its size. (Most of the time
 303                  * dma_alloc_coherent ensures this). This can happen when
 304                  * using large buffers allocated by the CMA
 305                  * (see CMA_CONFIG_ALIGNMENT)
 306                  */
 307                 dev_err(&sndev->stdev->dev,
 308                         "ERROR: Memory window address is not aligned to it's size!\n");
 309                 return -EINVAL;
 310         }
 311
 312         rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK,
 313                                    NTB_CTRL_PART_STATUS_LOCKED);
 314         if (rc)
 315                 return rc;
 316
 317         if (addr == 0 || size == 0) {
 318                 if (widx < nr_direct_mw)
 319                         switchtec_ntb_mw_clr_direct(sndev, widx);
 320                 else
 321                         switchtec_ntb_mw_clr_lut(sndev, widx);
 322         } else {
 323                 if (widx < nr_direct_mw)
 324                         switchtec_ntb_mw_set_direct(sndev, widx, addr, size);
 325                 else
 326                         switchtec_ntb_mw_set_lut(sndev, widx, addr, size);
 327         }
 328
 329         rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG,
 330                                    NTB_CTRL_PART_STATUS_NORMAL);
 331
 332         if (rc == -EIO) {
 333                 dev_err(&sndev->stdev->dev,
 334                         "Hardware reported an error configuring mw %d: %08x\n",
 335                         widx, ioread32(&ctl->bar_error));
 336
 337                 if (widx < nr_direct_mw)
 338                         switchtec_ntb_mw_clr_direct(sndev, widx);
 339                 else
 340                         switchtec_ntb_mw_clr_lut(sndev, widx);
 341
 342                 switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG,
 343                                       NTB_CTRL_PART_STATUS_NORMAL);
 344         }
 345
 346         return rc;
 347 }
 348
 349 static int switchtec_ntb_peer_mw_count(struct ntb_dev *ntb)
 350 {
 351         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 352         int nr_lut_mw = sndev->nr_lut_mw - sndev->nr_rsvd_luts;
 353
 354         return sndev->nr_direct_mw + (use_lut_mws ? nr_lut_mw : 0);
 355 }
 356
 357 static int switchtec_ntb_direct_get_addr(struct switchtec_ntb *sndev,
 358                                          int idx, phys_addr_t *base,
 359                                          resource_size_t *size)
 360 {
 361         int bar = sndev->direct_mw_to_bar[idx];
 362         size_t offset = 0;
 363
 364         if (bar < 0)
 365                 return -EINVAL;
 366
 367         if (idx == 0) {
 368                 /*
 369                  * This is the direct BAR shared with the LUTs
 370                  * which means the actual window will be offset
 371                  * by the size of all the LUT entries.
 372                  */
 373
 374                 offset = LUT_SIZE * sndev->nr_lut_mw;
 375         }
 376
 377         if (base)
 378                 *base = pci_resource_start(sndev->ntb.pdev, bar) + offset;
 379
 380         if (size) {
 381                 *size = pci_resource_len(sndev->ntb.pdev, bar) - offset;
 382                 if (offset && *size > offset)
 383                         *size = offset;
 384
 385                 if (*size > max_mw_size)
 386                         *size = max_mw_size;
 387         }
 388
 389         return 0;
 390 }
 391
 392 static int switchtec_ntb_lut_get_addr(struct switchtec_ntb *sndev,
 393                                       int idx, phys_addr_t *base,
 394                                       resource_size_t *size)
 395 {
 396         int bar = sndev->direct_mw_to_bar[0];
 397         int offset;
 398
 399         offset = LUT_SIZE * lut_index(sndev, idx);
 400
 401         if (base)
 402                 *base = pci_resource_start(sndev->ntb.pdev, bar) + offset;
 403
 404         if (size)
 405                 *size = LUT_SIZE;
 406
 407         return 0;
 408 }
 409
 410 static int switchtec_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx,
 411                                           phys_addr_t *base,
 412                                           resource_size_t *size)
 413 {
 414         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 415
 416         if (idx < sndev->nr_direct_mw)
 417                 return switchtec_ntb_direct_get_addr(sndev, idx, base, size);
 418         else if (idx < switchtec_ntb_peer_mw_count(ntb))
 419                 return switchtec_ntb_lut_get_addr(sndev, idx, base, size);
 420         else
 421                 return -EINVAL;
 422 }
 423
 424 static void switchtec_ntb_part_link_speed(struct switchtec_ntb *sndev,
 425                                           int partition,
 426                                           enum ntb_speed *speed,
 427                                           enum ntb_width *width)
 428 {
 429         struct switchtec_dev *stdev = sndev->stdev;
 430
 431         u32 pff = ioread32(&stdev->mmio_part_cfg[partition].vep_pff_inst_id);
 432         u32 linksta = ioread32(&stdev->mmio_pff_csr[pff].pci_cap_region[13]);
 433
 434         if (speed)
 435                 *speed = (linksta >> 16) & 0xF;
 436
 437         if (width)
 438                 *width = (linksta >> 20) & 0x3F;
 439 }
 440
 441 static void switchtec_ntb_set_link_speed(struct switchtec_ntb *sndev)
 442 {
 443         enum ntb_speed self_speed, peer_speed;
 444         enum ntb_width self_width, peer_width;
 445
 446         if (!sndev->link_is_up) {
 447                 sndev->link_speed = NTB_SPEED_NONE;
 448                 sndev->link_width = NTB_WIDTH_NONE;
 449                 return;
 450         }
 451
 452         switchtec_ntb_part_link_speed(sndev, sndev->self_partition,
 453                                       &self_speed, &self_width);
 454         switchtec_ntb_part_link_speed(sndev, sndev->peer_partition,
 455                                       &peer_speed, &peer_width);
 456
 457         sndev->link_speed = min(self_speed, peer_speed);
 458         sndev->link_width = min(self_width, peer_width);
 459 }
 460
 461 static int crosslink_is_enabled(struct switchtec_ntb *sndev)
 462 {
 463         struct ntb_info_regs __iomem *inf = sndev->mmio_ntb;
 464
 465         return ioread8(&inf->ntp_info[sndev->peer_partition].xlink_enabled);
 466 }
 467
 468 static void crosslink_init_dbmsgs(struct switchtec_ntb *sndev)
 469 {
 470         int i;
 471         u32 msg_map = 0;
 472
 473         if (!crosslink_is_enabled(sndev))
 474                 return;
 475
 476         for (i = 0; i < ARRAY_SIZE(sndev->mmio_peer_dbmsg->imsg); i++) {
 477                 int m = i | sndev->self_partition << 2;
 478
 479                 msg_map |= m << i * 8;
 480         }
 481
 482         iowrite32(msg_map, &sndev->mmio_peer_dbmsg->msg_map);
 483         iowrite64(sndev->db_valid_mask << sndev->db_peer_shift,
 484                   &sndev->mmio_peer_dbmsg->odb_mask);
 485 }
 486
 487 enum switchtec_msg {
 488         LINK_MESSAGE = 0,
 489         MSG_LINK_UP = 1,
 490         MSG_LINK_DOWN = 2,
 491         MSG_CHECK_LINK = 3,
 492         MSG_LINK_FORCE_DOWN = 4,
 493 };
 494
 495 static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev);
 496
 497 static void link_reinit_work(struct work_struct *work)
 498 {
 499         struct switchtec_ntb *sndev;
 500
 501         sndev = container_of(work, struct switchtec_ntb, link_reinit_work);
 502
 503         switchtec_ntb_reinit_peer(sndev);
 504 }
 505
 506 static void switchtec_ntb_check_link(struct switchtec_ntb *sndev,
 507                                      enum switchtec_msg msg)
 508 {
 509         int link_sta;
 510         int old = sndev->link_is_up;
 511
 512         if (msg == MSG_LINK_FORCE_DOWN) {
 513                 schedule_work(&sndev->link_reinit_work);
 514
 515                 if (sndev->link_is_up) {
 516                         sndev->link_is_up = 0;
 517                         ntb_link_event(&sndev->ntb);
 518                         dev_info(&sndev->stdev->dev, "ntb link forced down\n");
 519                 }
 520
 521                 return;
 522         }
 523
 524         link_sta = sndev->self_shared->link_sta;
 525         if (link_sta) {
 526                 u64 peer = ioread64(&sndev->peer_shared->magic);
 527
 528                 if ((peer & 0xFFFFFFFF) == SWITCHTEC_NTB_MAGIC)
 529                         link_sta = peer >> 32;
 530                 else
 531                         link_sta = 0;
 532         }
 533
 534         sndev->link_is_up = link_sta;
 535         switchtec_ntb_set_link_speed(sndev);
 536
 537         if (link_sta != old) {
 538                 switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_CHECK_LINK);
 539                 ntb_link_event(&sndev->ntb);
 540                 dev_info(&sndev->stdev->dev, "ntb link %s\n",
 541                          link_sta ? "up" : "down");
 542
 543                 if (link_sta)
 544                         crosslink_init_dbmsgs(sndev);
 545         }
 546 }
 547
 548 static void switchtec_ntb_link_notification(struct switchtec_dev *stdev)
 549 {
 550         struct switchtec_ntb *sndev = stdev->sndev;
 551
 552         switchtec_ntb_check_link(sndev, MSG_CHECK_LINK);
 553 }
 554
 555 static u64 switchtec_ntb_link_is_up(struct ntb_dev *ntb,
 556                                     enum ntb_speed *speed,
 557                                     enum ntb_width *width)
 558 {
 559         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 560
 561         if (speed)
 562                 *speed = sndev->link_speed;
 563         if (width)
 564                 *width = sndev->link_width;
 565
 566         return sndev->link_is_up;
 567 }
 568
 569 static int switchtec_ntb_link_enable(struct ntb_dev *ntb,
 570                                      enum ntb_speed max_speed,
 571                                      enum ntb_width max_width)
 572 {
 573         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 574
 575         dev_dbg(&sndev->stdev->dev, "enabling link\n");
 576
 577         sndev->self_shared->link_sta = 1;
 578         switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_UP);
 579
 580         switchtec_ntb_check_link(sndev, MSG_CHECK_LINK);
 581
 582         return 0;
 583 }
 584
 585 static int switchtec_ntb_link_disable(struct ntb_dev *ntb)
 586 {
 587         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 588
 589         dev_dbg(&sndev->stdev->dev, "disabling link\n");
 590
 591         sndev->self_shared->link_sta = 0;
 592         switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_DOWN);
 593
 594         switchtec_ntb_check_link(sndev, MSG_CHECK_LINK);
 595
 596         return 0;
 597 }
 598
 599 static u64 switchtec_ntb_db_valid_mask(struct ntb_dev *ntb)
 600 {
 601         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 602
 603         return sndev->db_valid_mask;
 604 }
 605
 606 static int switchtec_ntb_db_vector_count(struct ntb_dev *ntb)
 607 {
 608         return 1;
 609 }
 610
 611 static u64 switchtec_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
 612 {
 613         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 614
 615         if (db_vector < 0 || db_vector > 1)
 616                 return 0;
 617
 618         return sndev->db_valid_mask;
 619 }
 620
 621 static u64 switchtec_ntb_db_read(struct ntb_dev *ntb)
 622 {
 623         u64 ret;
 624         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 625
 626         ret = ioread64(&sndev->mmio_self_dbmsg->idb) >> sndev->db_shift;
 627
 628         return ret & sndev->db_valid_mask;
 629 }
 630
 631 static int switchtec_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
 632 {
 633         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 634
 635         iowrite64(db_bits << sndev->db_shift, &sndev->mmio_self_dbmsg->idb);
 636
 637         return 0;
 638 }
 639
 640 static int switchtec_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
 641 {
 642         unsigned long irqflags;
 643         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 644
 645         if (db_bits & ~sndev->db_valid_mask)
 646                 return -EINVAL;
 647
 648         spin_lock_irqsave(&sndev->db_mask_lock, irqflags);
 649
 650         sndev->db_mask |= db_bits << sndev->db_shift;
 651         iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask);
 652
 653         spin_unlock_irqrestore(&sndev->db_mask_lock, irqflags);
 654
 655         return 0;
 656 }
 657
 658 static int switchtec_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
 659 {
 660         unsigned long irqflags;
 661         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 662
 663         if (db_bits & ~sndev->db_valid_mask)
 664                 return -EINVAL;
 665
 666         spin_lock_irqsave(&sndev->db_mask_lock, irqflags);
 667
 668         sndev->db_mask &= ~(db_bits << sndev->db_shift);
 669         iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask);
 670
 671         spin_unlock_irqrestore(&sndev->db_mask_lock, irqflags);
 672
 673         return 0;
 674 }
 675
 676 static u64 switchtec_ntb_db_read_mask(struct ntb_dev *ntb)
 677 {
 678         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 679
 680         return (sndev->db_mask >> sndev->db_shift) & sndev->db_valid_mask;
 681 }
 682
 683 static int switchtec_ntb_peer_db_addr(struct ntb_dev *ntb,
 684                                       phys_addr_t *db_addr,
 685                                       resource_size_t *db_size)
 686 {
 687         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 688         unsigned long offset;
 689
 690         offset = (unsigned long)sndev->mmio_peer_dbmsg->odb -
 691                 (unsigned long)sndev->stdev->mmio;
 692
 693         offset += sndev->db_shift / 8;
 694
 695         if (db_addr)
 696                 *db_addr = pci_resource_start(ntb->pdev, 0) + offset;
 697         if (db_size)
 698                 *db_size = sizeof(u32);
 699
 700         return 0;
 701 }
 702
 703 static int switchtec_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
 704 {
 705         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 706
 707         iowrite64(db_bits << sndev->db_peer_shift,
 708                   &sndev->mmio_peer_dbmsg->odb);
 709
 710         return 0;
 711 }
 712
 713 static int switchtec_ntb_spad_count(struct ntb_dev *ntb)
 714 {
 715         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 716
 717         return ARRAY_SIZE(sndev->self_shared->spad);
 718 }
 719
 720 static u32 switchtec_ntb_spad_read(struct ntb_dev *ntb, int idx)
 721 {
 722         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 723
 724         if (idx < 0 || idx >= ARRAY_SIZE(sndev->self_shared->spad))
 725                 return 0;
 726
 727         if (!sndev->self_shared)
 728                 return 0;
 729
 730         return sndev->self_shared->spad[idx];
 731 }
 732
 733 static int switchtec_ntb_spad_write(struct ntb_dev *ntb, int idx, u32 val)
 734 {
 735         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 736
 737         if (idx < 0 || idx >= ARRAY_SIZE(sndev->self_shared->spad))
 738                 return -EINVAL;
 739
 740         if (!sndev->self_shared)
 741                 return -EIO;
 742
 743         sndev->self_shared->spad[idx] = val;
 744
 745         return 0;
 746 }
 747
 748 static u32 switchtec_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx,
 749                                         int sidx)
 750 {
 751         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 752
 753         if (pidx != NTB_DEF_PEER_IDX)
 754                 return -EINVAL;
 755
 756         if (sidx < 0 || sidx >= ARRAY_SIZE(sndev->peer_shared->spad))
 757                 return 0;
 758
 759         if (!sndev->peer_shared)
 760                 return 0;
 761
 762         return ioread32(&sndev->peer_shared->spad[sidx]);
 763 }
 764
 765 static int switchtec_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx,
 766                                          int sidx, u32 val)
 767 {
 768         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 769
 770         if (pidx != NTB_DEF_PEER_IDX)
 771                 return -EINVAL;
 772
 773         if (sidx < 0 || sidx >= ARRAY_SIZE(sndev->peer_shared->spad))
 774                 return -EINVAL;
 775
 776         if (!sndev->peer_shared)
 777                 return -EIO;
 778
 779         iowrite32(val, &sndev->peer_shared->spad[sidx]);
 780
 781         return 0;
 782 }
 783
 784 static int switchtec_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx,
 785                                         int sidx, phys_addr_t *spad_addr)
 786 {
 787         struct switchtec_ntb *sndev = ntb_sndev(ntb);
 788         unsigned long offset;
 789
 790         if (pidx != NTB_DEF_PEER_IDX)
 791                 return -EINVAL;
 792
 793         offset = (unsigned long)&sndev->peer_shared->spad[sidx] -
 794                 (unsigned long)sndev->stdev->mmio;
 795
 796         if (spad_addr)
 797                 *spad_addr = pci_resource_start(ntb->pdev, 0) + offset;
 798
 799         return 0;
 800 }
 801
 802 static const struct ntb_dev_ops switchtec_ntb_ops = {
 803         .mw_count               = switchtec_ntb_mw_count,
 804         .mw_get_align           = switchtec_ntb_mw_get_align,
 805         .mw_set_trans           = switchtec_ntb_mw_set_trans,
 806         .peer_mw_count          = switchtec_ntb_peer_mw_count,
 807         .peer_mw_get_addr       = switchtec_ntb_peer_mw_get_addr,
 808         .link_is_up             = switchtec_ntb_link_is_up,
 809         .link_enable            = switchtec_ntb_link_enable,
 810         .link_disable           = switchtec_ntb_link_disable,
 811         .db_valid_mask          = switchtec_ntb_db_valid_mask,
 812         .db_vector_count        = switchtec_ntb_db_vector_count,
 813         .db_vector_mask         = switchtec_ntb_db_vector_mask,
 814         .db_read                = switchtec_ntb_db_read,
 815         .db_clear               = switchtec_ntb_db_clear,
 816         .db_set_mask            = switchtec_ntb_db_set_mask,
 817         .db_clear_mask          = switchtec_ntb_db_clear_mask,
 818         .db_read_mask           = switchtec_ntb_db_read_mask,
 819         .peer_db_addr           = switchtec_ntb_peer_db_addr,
 820         .peer_db_set            = switchtec_ntb_peer_db_set,
 821         .spad_count             = switchtec_ntb_spad_count,
 822         .spad_read              = switchtec_ntb_spad_read,
 823         .spad_write             = switchtec_ntb_spad_write,
 824         .peer_spad_read         = switchtec_ntb_peer_spad_read,
 825         .peer_spad_write        = switchtec_ntb_peer_spad_write,
 826         .peer_spad_addr         = switchtec_ntb_peer_spad_addr,
 827 };
 828
 829 static int switchtec_ntb_init_sndev(struct switchtec_ntb *sndev)
 830 {
 831         u64 tpart_vec;
 832         int self;
 833         u64 part_map;
 834         int bit;
 835
 836         sndev->ntb.pdev = sndev->stdev->pdev;
 837         sndev->ntb.topo = NTB_TOPO_SWITCH;
 838         sndev->ntb.ops = &switchtec_ntb_ops;
 839
 840         INIT_WORK(&sndev->link_reinit_work, link_reinit_work);
 841
 842         sndev->self_partition = sndev->stdev->partition;
 843
 844         sndev->mmio_ntb = sndev->stdev->mmio_ntb;
 845
 846         self = sndev->self_partition;
 847         tpart_vec = ioread32(&sndev->mmio_ntb->ntp_info[self].target_part_high);
 848         tpart_vec <<= 32;
 849         tpart_vec |= ioread32(&sndev->mmio_ntb->ntp_info[self].target_part_low);
 850
 851         part_map = ioread64(&sndev->mmio_ntb->ep_map);
 852         part_map &= ~(1 << sndev->self_partition);
 853
 854         if (!ffs(tpart_vec)) {
 855                 if (sndev->stdev->partition_count != 2) {
 856                         dev_err(&sndev->stdev->dev,
 857                                 "ntb target partition not defined\n");
 858                         return -ENODEV;
 859                 }
 860
 861                 bit = ffs(part_map);
 862                 if (!bit) {
 863                         dev_err(&sndev->stdev->dev,
 864                                 "peer partition is not NT partition\n");
 865                         return -ENODEV;
 866                 }
 867
 868                 sndev->peer_partition = bit - 1;
 869         } else {
 870                 if (ffs(tpart_vec) != fls(tpart_vec)) {
 871                         dev_err(&sndev->stdev->dev,
 872                                 "ntb driver only supports 1 pair of 1-1 ntb mapping\n");
 873                         return -ENODEV;
 874                 }
 875
 876                 sndev->peer_partition = ffs(tpart_vec) - 1;
 877                 if (!(part_map & (1 << sndev->peer_partition))) {
 878                         dev_err(&sndev->stdev->dev,
 879                                 "ntb target partition is not NT partition\n");
 880                         return -ENODEV;
 881                 }
 882         }
 883
 884         dev_dbg(&sndev->stdev->dev, "Partition ID %d of %d\n",
 885                 sndev->self_partition, sndev->stdev->partition_count);
 886
 887         sndev->mmio_ctrl = (void * __iomem)sndev->mmio_ntb +
 888                 SWITCHTEC_NTB_REG_CTRL_OFFSET;
 889         sndev->mmio_dbmsg = (void * __iomem)sndev->mmio_ntb +
 890                 SWITCHTEC_NTB_REG_DBMSG_OFFSET;
 891
 892         sndev->mmio_self_ctrl = &sndev->mmio_ctrl[sndev->self_partition];
 893         sndev->mmio_peer_ctrl = &sndev->mmio_ctrl[sndev->peer_partition];
 894         sndev->mmio_self_dbmsg = &sndev->mmio_dbmsg[sndev->self_partition];
 895         sndev->mmio_peer_dbmsg = sndev->mmio_self_dbmsg;
 896
 897         return 0;
 898 }
 899
 900 static int config_rsvd_lut_win(struct switchtec_ntb *sndev,
 901                                struct ntb_ctrl_regs __iomem *ctl,
 902                                int lut_idx, int partition, u64 addr)
 903 {
 904         int peer_bar = sndev->peer_direct_mw_to_bar[0];
 905         u32 ctl_val;
 906         int rc;
 907
 908         rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK,
 909                                    NTB_CTRL_PART_STATUS_LOCKED);
 910         if (rc)
 911                 return rc;
 912
 913         ctl_val = ioread32(&ctl->bar_entry[peer_bar].ctl);
 914         ctl_val &= 0xFF;
 915         ctl_val |= NTB_CTRL_BAR_LUT_WIN_EN;
 916         ctl_val |= ilog2(LUT_SIZE) << 8;
 917         ctl_val |= (sndev->nr_lut_mw - 1) << 14;
 918         iowrite32(ctl_val, &ctl->bar_entry[peer_bar].ctl);
 919
 920         iowrite64((NTB_CTRL_LUT_EN | (partition << 1) | addr),
 921                   &ctl->lut_entry[lut_idx]);
 922
 923         rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG,
 924                                    NTB_CTRL_PART_STATUS_NORMAL);
 925         if (rc) {
 926                 u32 bar_error, lut_error;
 927
 928                 bar_error = ioread32(&ctl->bar_error);
 929                 lut_error = ioread32(&ctl->lut_error);
 930                 dev_err(&sndev->stdev->dev,
 931                         "Error setting up reserved lut window: %08x / %08x\n",
 932                         bar_error, lut_error);
 933                 return rc;
 934         }
 935
 936         return 0;
 937 }
 938
 939 static int config_req_id_table(struct switchtec_ntb *sndev,
 940                                struct ntb_ctrl_regs __iomem *mmio_ctrl,
 941                                int *req_ids, int count)
 942 {
 943         int i, rc = 0;
 944         u32 error;
 945         u32 proxy_id;
 946
 947         if (ioread32(&mmio_ctrl->req_id_table_size) < count) {
 948                 dev_err(&sndev->stdev->dev,
 949                         "Not enough requester IDs available.\n");
 950                 return -EFAULT;
 951         }
 952
 953         rc = switchtec_ntb_part_op(sndev, mmio_ctrl,
 954                                    NTB_CTRL_PART_OP_LOCK,
 955                                    NTB_CTRL_PART_STATUS_LOCKED);
 956         if (rc)
 957                 return rc;
 958
 959         iowrite32(NTB_PART_CTRL_ID_PROT_DIS,
 960                   &mmio_ctrl->partition_ctrl);
 961
 962         for (i = 0; i < count; i++) {
 963                 iowrite32(req_ids[i] << 16 | NTB_CTRL_REQ_ID_EN,
 964                           &mmio_ctrl->req_id_table[i]);
 965
 966                 proxy_id = ioread32(&mmio_ctrl->req_id_table[i]);
 967                 dev_dbg(&sndev->stdev->dev,
 968                         "Requester ID %02X:%02X.%X -> BB:%02X.%X\n",
 969                         req_ids[i] >> 8, (req_ids[i] >> 3) & 0x1F,
 970                         req_ids[i] & 0x7, (proxy_id >> 4) & 0x1F,
 971                         (proxy_id >> 1) & 0x7);
 972         }
 973
 974         rc = switchtec_ntb_part_op(sndev, mmio_ctrl,
 975                                    NTB_CTRL_PART_OP_CFG,
 976                                    NTB_CTRL_PART_STATUS_NORMAL);
 977
 978         if (rc == -EIO) {
 979                 error = ioread32(&mmio_ctrl->req_id_error);
 980                 dev_err(&sndev->stdev->dev,
 981                         "Error setting up the requester ID table: %08x\n",
 982                         error);
 983         }
 984
 985         return 0;
 986 }
 987
 988 static int crosslink_setup_mws(struct switchtec_ntb *sndev, int ntb_lut_idx,
 989                                u64 *mw_addrs, int mw_count)
 990 {
 991         int rc, i;
 992         struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_self_ctrl;
 993         u64 addr;
 994         size_t size, offset;
 995         int bar;
 996         int xlate_pos;
 997         u32 ctl_val;
 998
 999         rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK,
1000                                    NTB_CTRL_PART_STATUS_LOCKED);
1001         if (rc)
1002                 return rc;
1003
1004         for (i = 0; i < sndev->nr_lut_mw; i++) {
1005                 if (i == ntb_lut_idx)
1006                         continue;
1007
1008                 addr = mw_addrs[0] + LUT_SIZE * i;
1009
1010                 iowrite64((NTB_CTRL_LUT_EN | (sndev->peer_partition << 1) |
1011                            addr),
1012                           &ctl->lut_entry[i]);
1013         }
1014
1015         sndev->nr_direct_mw = min_t(int, sndev->nr_direct_mw, mw_count);
1016
1017         for (i = 0; i < sndev->nr_direct_mw; i++) {
1018                 bar = sndev->direct_mw_to_bar[i];
1019                 offset = (i == 0) ? LUT_SIZE * sndev->nr_lut_mw : 0;
1020                 addr = mw_addrs[i] + offset;
1021                 size = pci_resource_len(sndev->ntb.pdev, bar) - offset;
1022                 xlate_pos = ilog2(size);
1023
1024                 if (offset && size > offset)
1025                         size = offset;
1026
1027                 ctl_val = ioread32(&ctl->bar_entry[bar].ctl);
1028                 ctl_val |= NTB_CTRL_BAR_DIR_WIN_EN;
1029
1030                 iowrite32(ctl_val, &ctl->bar_entry[bar].ctl);
1031                 iowrite32(xlate_pos | (lower_32_bits(size) & 0xFFFFF000),
1032                           &ctl->bar_entry[bar].win_size);
1033                 iowrite32(upper_32_bits(size), &ctl->bar_ext_entry[bar].win_size);
1034                 iowrite64(sndev->peer_partition | addr,
1035                           &ctl->bar_entry[bar].xlate_addr);
1036         }
1037
1038         rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG,
1039                                    NTB_CTRL_PART_STATUS_NORMAL);
1040         if (rc) {
1041                 u32 bar_error, lut_error;
1042
1043                 bar_error = ioread32(&ctl->bar_error);
1044                 lut_error = ioread32(&ctl->lut_error);
1045                 dev_err(&sndev->stdev->dev,
1046                         "Error setting up cross link windows: %08x / %08x\n",
1047                         bar_error, lut_error);
1048                 return rc;
1049         }
1050
1051         return 0;
1052 }
1053
1054 static int crosslink_setup_req_ids(struct switchtec_ntb *sndev,
1055         struct ntb_ctrl_regs __iomem *mmio_ctrl)
1056 {
1057         int req_ids[16];
1058         int i;
1059         u32 proxy_id;
1060
1061         for (i = 0; i < ARRAY_SIZE(req_ids); i++) {
1062                 proxy_id = ioread32(&sndev->mmio_self_ctrl->req_id_table[i]);
1063
1064                 if (!(proxy_id & NTB_CTRL_REQ_ID_EN))
1065                         break;
1066
1067                 req_ids[i] = ((proxy_id >> 1) & 0xFF);
1068         }
1069
1070         return config_req_id_table(sndev, mmio_ctrl, req_ids, i);
1071 }
1072
1073 /*
1074  * In crosslink configuration there is a virtual partition in the
1075  * middle of the two switches. The BARs in this partition have to be
1076  * enumerated and assigned addresses.
1077  */
1078 static int crosslink_enum_partition(struct switchtec_ntb *sndev,
1079                                     u64 *bar_addrs)
1080 {
1081         struct part_cfg_regs __iomem *part_cfg =
1082                 &sndev->stdev->mmio_part_cfg_all[sndev->peer_partition];
1083         u32 pff = ioread32(&part_cfg->vep_pff_inst_id);
1084         struct pff_csr_regs __iomem *mmio_pff =
1085                 &sndev->stdev->mmio_pff_csr[pff];
1086         const u64 bar_space = 0x1000000000LL;
1087         u64 bar_addr;
1088         int bar_cnt = 0;
1089         int i;
1090
1091         iowrite16(0x6, &mmio_pff->pcicmd);
1092
1093         for (i = 0; i < ARRAY_SIZE(mmio_pff->pci_bar64); i++) {
1094                 iowrite64(bar_space * i, &mmio_pff->pci_bar64[i]);
1095                 bar_addr = ioread64(&mmio_pff->pci_bar64[i]);
1096                 bar_addr &= ~0xf;
1097
1098                 dev_dbg(&sndev->stdev->dev,
1099                         "Crosslink BAR%d addr: %llx\n",
1100                         i*2, bar_addr);
1101
1102                 if (bar_addr != bar_space * i)
1103                         continue;
1104
1105                 bar_addrs[bar_cnt++] = bar_addr;
1106         }
1107
1108         return bar_cnt;
1109 }
1110
1111 static int switchtec_ntb_init_crosslink(struct switchtec_ntb *sndev)
1112 {
1113         int rc;
1114         int bar = sndev->direct_mw_to_bar[0];
1115         const int ntb_lut_idx = 1;
1116         u64 bar_addrs[6];
1117         u64 addr;
1118         int offset;
1119         int bar_cnt;
1120
1121         if (!crosslink_is_enabled(sndev))
1122                 return 0;
1123
1124         dev_info(&sndev->stdev->dev, "Using crosslink configuration\n");
1125         sndev->ntb.topo = NTB_TOPO_CROSSLINK;
1126
1127         bar_cnt = crosslink_enum_partition(sndev, bar_addrs);
1128         if (bar_cnt < sndev->nr_direct_mw + 1) {
1129                 dev_err(&sndev->stdev->dev,
1130                         "Error enumerating crosslink partition\n");
1131                 return -EINVAL;
1132         }
1133
1134         addr = (bar_addrs[0] + SWITCHTEC_GAS_NTB_OFFSET +
1135                 SWITCHTEC_NTB_REG_DBMSG_OFFSET +
1136                 sizeof(struct ntb_dbmsg_regs) * sndev->peer_partition);
1137
1138         offset = addr & (LUT_SIZE - 1);
1139         addr -= offset;
1140
1141         rc = config_rsvd_lut_win(sndev, sndev->mmio_self_ctrl, ntb_lut_idx,
1142                                  sndev->peer_partition, addr);
1143         if (rc)
1144                 return rc;
1145
1146         rc = crosslink_setup_mws(sndev, ntb_lut_idx, &bar_addrs[1],
1147                                  bar_cnt - 1);
1148         if (rc)
1149                 return rc;
1150
1151         rc = crosslink_setup_req_ids(sndev, sndev->mmio_peer_ctrl);
1152         if (rc)
1153                 return rc;
1154
1155         sndev->mmio_xlink_win = pci_iomap_range(sndev->stdev->pdev, bar,
1156                                                 LUT_SIZE, LUT_SIZE);
1157         if (!sndev->mmio_xlink_win) {
1158                 rc = -ENOMEM;
1159                 return rc;
1160         }
1161
1162         sndev->mmio_peer_dbmsg = sndev->mmio_xlink_win + offset;
1163         sndev->nr_rsvd_luts++;
1164
1165         crosslink_init_dbmsgs(sndev);
1166
1167         return 0;
1168 }
1169
1170 static void switchtec_ntb_deinit_crosslink(struct switchtec_ntb *sndev)
1171 {
1172         if (sndev->mmio_xlink_win)
1173                 pci_iounmap(sndev->stdev->pdev, sndev->mmio_xlink_win);
1174 }
1175
1176 static int map_bars(int *map, struct ntb_ctrl_regs __iomem *ctrl)
1177 {
1178         int i;
1179         int cnt = 0;
1180
1181         for (i = 0; i < ARRAY_SIZE(ctrl->bar_entry); i++) {
1182                 u32 r = ioread32(&ctrl->bar_entry[i].ctl);
1183
1184                 if (r & NTB_CTRL_BAR_VALID)
1185                         map[cnt++] = i;
1186         }
1187
1188         return cnt;
1189 }
1190
1191 static void switchtec_ntb_init_mw(struct switchtec_ntb *sndev)
1192 {
1193         sndev->nr_direct_mw = map_bars(sndev->direct_mw_to_bar,
1194                                        sndev->mmio_self_ctrl);
1195
1196         sndev->nr_lut_mw = ioread16(&sndev->mmio_self_ctrl->lut_table_entries);
1197         sndev->nr_lut_mw = rounddown_pow_of_two(sndev->nr_lut_mw);
1198
1199         dev_dbg(&sndev->stdev->dev, "MWs: %d direct, %d lut\n",
1200                 sndev->nr_direct_mw, sndev->nr_lut_mw);
1201
1202         sndev->peer_nr_direct_mw = map_bars(sndev->peer_direct_mw_to_bar,
1203                                             sndev->mmio_peer_ctrl);
1204
1205         sndev->peer_nr_lut_mw =
1206                 ioread16(&sndev->mmio_peer_ctrl->lut_table_entries);
1207         sndev->peer_nr_lut_mw = rounddown_pow_of_two(sndev->peer_nr_lut_mw);
1208
1209         dev_dbg(&sndev->stdev->dev, "Peer MWs: %d direct, %d lut\n",
1210                 sndev->peer_nr_direct_mw, sndev->peer_nr_lut_mw);
1211
1212 }
1213
1214 /*
1215  * There are 64 doorbells in the switch hardware but this is
1216  * shared among all partitions. So we must split them in half
1217  * (32 for each partition). However, the message interrupts are
1218  * also shared with the top 4 doorbells so we just limit this to
1219  * 28 doorbells per partition.
1220  *
1221  * In crosslink mode, each side has it's own dbmsg register so
1222  * they can each use all 60 of the available doorbells.
1223  */
1224 static void switchtec_ntb_init_db(struct switchtec_ntb *sndev)
1225 {
1226         sndev->db_mask = 0x0FFFFFFFFFFFFFFFULL;
1227
1228         if (sndev->mmio_peer_dbmsg != sndev->mmio_self_dbmsg) {
1229                 sndev->db_shift = 0;
1230                 sndev->db_peer_shift = 0;
1231                 sndev->db_valid_mask = sndev->db_mask;
1232         } else if (sndev->self_partition < sndev->peer_partition) {
1233                 sndev->db_shift = 0;
1234                 sndev->db_peer_shift = 32;
1235                 sndev->db_valid_mask = 0x0FFFFFFF;
1236         } else {
1237                 sndev->db_shift = 32;
1238                 sndev->db_peer_shift = 0;
1239                 sndev->db_valid_mask = 0x0FFFFFFF;
1240         }
1241
1242         iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask);
1243         iowrite64(sndev->db_valid_mask << sndev->db_peer_shift,
1244                   &sndev->mmio_peer_dbmsg->odb_mask);
1245
1246         dev_dbg(&sndev->stdev->dev, "dbs: shift %d/%d, mask %016llx\n",
1247                 sndev->db_shift, sndev->db_peer_shift, sndev->db_valid_mask);
1248 }
1249
1250 static void switchtec_ntb_init_msgs(struct switchtec_ntb *sndev)
1251 {
1252         int i;
1253         u32 msg_map = 0;
1254
1255         for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) {
1256                 int m = i | sndev->peer_partition << 2;
1257
1258                 msg_map |= m << i * 8;
1259         }
1260
1261         iowrite32(msg_map, &sndev->mmio_self_dbmsg->msg_map);
1262
1263         for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++)
1264                 iowrite64(NTB_DBMSG_IMSG_STATUS | NTB_DBMSG_IMSG_MASK,
1265                           &sndev->mmio_self_dbmsg->imsg[i]);
1266 }
1267
1268 static int
1269 switchtec_ntb_init_req_id_table(struct switchtec_ntb *sndev)
1270 {
1271         int req_ids[2];
1272
1273         /*
1274          * Root Complex Requester ID (which is 0:00.0)
1275          */
1276         req_ids[0] = 0;
1277
1278         /*
1279          * Host Bridge Requester ID (as read from the mmap address)
1280          */
1281         req_ids[1] = ioread16(&sndev->mmio_ntb->requester_id);
1282
1283         return config_req_id_table(sndev, sndev->mmio_self_ctrl, req_ids,
1284                                    ARRAY_SIZE(req_ids));
1285 }
1286
1287 static void switchtec_ntb_init_shared(struct switchtec_ntb *sndev)
1288 {
1289         int i;
1290
1291         memset(sndev->self_shared, 0, LUT_SIZE);
1292         sndev->self_shared->magic = SWITCHTEC_NTB_MAGIC;
1293         sndev->self_shared->partition_id = sndev->stdev->partition;
1294
1295         for (i = 0; i < sndev->nr_direct_mw; i++) {
1296                 int bar = sndev->direct_mw_to_bar[i];
1297                 resource_size_t sz = pci_resource_len(sndev->stdev->pdev, bar);
1298
1299                 if (i == 0)
1300                         sz = min_t(resource_size_t, sz,
1301                                    LUT_SIZE * sndev->nr_lut_mw);
1302
1303                 sndev->self_shared->mw_sizes[i] = sz;
1304         }
1305
1306         for (i = 0; i < sndev->nr_lut_mw; i++) {
1307                 int idx = sndev->nr_direct_mw + i;
1308
1309                 sndev->self_shared->mw_sizes[idx] = LUT_SIZE;
1310         }
1311 }
1312
1313 static int switchtec_ntb_init_shared_mw(struct switchtec_ntb *sndev)
1314 {
1315         int self_bar = sndev->direct_mw_to_bar[0];
1316         int rc;
1317
1318         sndev->nr_rsvd_luts++;
1319         sndev->self_shared = dma_alloc_coherent(&sndev->stdev->pdev->dev,
1320                                                 LUT_SIZE,
1321                                                 &sndev->self_shared_dma,
1322                                                 GFP_KERNEL);
1323         if (!sndev->self_shared) {
1324                 dev_err(&sndev->stdev->dev,
1325                         "unable to allocate memory for shared mw\n");
1326                 return -ENOMEM;
1327         }
1328
1329         switchtec_ntb_init_shared(sndev);
1330
1331         rc = config_rsvd_lut_win(sndev, sndev->mmio_peer_ctrl, 0,
1332                                  sndev->self_partition,
1333                                  sndev->self_shared_dma);
1334         if (rc)
1335                 goto unalloc_and_exit;
1336
1337         sndev->peer_shared = pci_iomap(sndev->stdev->pdev, self_bar, LUT_SIZE);
1338         if (!sndev->peer_shared) {
1339                 rc = -ENOMEM;
1340                 goto unalloc_and_exit;
1341         }
1342
1343         dev_dbg(&sndev->stdev->dev, "Shared MW Ready\n");
1344         return 0;
1345
1346 unalloc_and_exit:
1347         dma_free_coherent(&sndev->stdev->pdev->dev, LUT_SIZE,
1348                           sndev->self_shared, sndev->self_shared_dma);
1349
1350         return rc;
1351 }
1352
1353 static void switchtec_ntb_deinit_shared_mw(struct switchtec_ntb *sndev)
1354 {
1355         if (sndev->peer_shared)
1356                 pci_iounmap(sndev->stdev->pdev, sndev->peer_shared);
1357
1358         if (sndev->self_shared)
1359                 dma_free_coherent(&sndev->stdev->pdev->dev, LUT_SIZE,
1360                                   sndev->self_shared,
1361                                   sndev->self_shared_dma);
1362         sndev->nr_rsvd_luts--;
1363 }
1364
1365 static irqreturn_t switchtec_ntb_doorbell_isr(int irq, void *dev)
1366 {
1367         struct switchtec_ntb *sndev = dev;
1368
1369         dev_dbg(&sndev->stdev->dev, "doorbell\n");
1370
1371         ntb_db_event(&sndev->ntb, 0);
1372
1373         return IRQ_HANDLED;
1374 }
1375
1376 static irqreturn_t switchtec_ntb_message_isr(int irq, void *dev)
1377 {
1378         int i;
1379         struct switchtec_ntb *sndev = dev;
1380
1381         for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) {
1382                 u64 msg = ioread64(&sndev->mmio_self_dbmsg->imsg[i]);
1383
1384                 if (msg & NTB_DBMSG_IMSG_STATUS) {
1385                         dev_dbg(&sndev->stdev->dev, "message: %d %08x\n",
1386                                 i, (u32)msg);
1387                         iowrite8(1, &sndev->mmio_self_dbmsg->imsg[i].status);
1388
1389                         if (i == LINK_MESSAGE)
1390                                 switchtec_ntb_check_link(sndev, msg);
1391                 }
1392         }
1393
1394         return IRQ_HANDLED;
1395 }
1396
1397 static int switchtec_ntb_init_db_msg_irq(struct switchtec_ntb *sndev)
1398 {
1399         int i;
1400         int rc;
1401         int doorbell_irq = 0;
1402         int message_irq = 0;
1403         int event_irq;
1404         int idb_vecs = sizeof(sndev->mmio_self_dbmsg->idb_vec_map);
1405
1406         event_irq = ioread32(&sndev->stdev->mmio_part_cfg->vep_vector_number);
1407
1408         while (doorbell_irq == event_irq)
1409                 doorbell_irq++;
1410         while (message_irq == doorbell_irq ||
1411                message_irq == event_irq)
1412                 message_irq++;
1413
1414         dev_dbg(&sndev->stdev->dev, "irqs - event: %d, db: %d, msgs: %d\n",
1415                 event_irq, doorbell_irq, message_irq);
1416
1417         for (i = 0; i < idb_vecs - 4; i++)
1418                 iowrite8(doorbell_irq,
1419                          &sndev->mmio_self_dbmsg->idb_vec_map[i]);
1420
1421         for (; i < idb_vecs; i++)
1422                 iowrite8(message_irq,
1423                          &sndev->mmio_self_dbmsg->idb_vec_map[i]);
1424
1425         sndev->doorbell_irq = pci_irq_vector(sndev->stdev->pdev, doorbell_irq);
1426         sndev->message_irq = pci_irq_vector(sndev->stdev->pdev, message_irq);
1427
1428         rc = request_irq(sndev->doorbell_irq,
1429                          switchtec_ntb_doorbell_isr, 0,
1430                          "switchtec_ntb_doorbell", sndev);
1431         if (rc)
1432                 return rc;
1433
1434         rc = request_irq(sndev->message_irq,
1435                          switchtec_ntb_message_isr, 0,
1436                          "switchtec_ntb_message", sndev);
1437         if (rc) {
1438                 free_irq(sndev->doorbell_irq, sndev);
1439                 return rc;
1440         }
1441
1442         return 0;
1443 }
1444
1445 static void switchtec_ntb_deinit_db_msg_irq(struct switchtec_ntb *sndev)
1446 {
1447         free_irq(sndev->doorbell_irq, sndev);
1448         free_irq(sndev->message_irq, sndev);
1449 }
1450
1451 static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev)
1452 {
1453         dev_info(&sndev->stdev->dev, "peer reinitialized\n");
1454         switchtec_ntb_deinit_shared_mw(sndev);
1455         switchtec_ntb_init_mw(sndev);
1456         return switchtec_ntb_init_shared_mw(sndev);
1457 }
1458
1459 static int switchtec_ntb_add(struct device *dev,
1460                              struct class_interface *class_intf)
1461 {
1462         struct switchtec_dev *stdev = to_stdev(dev);
1463         struct switchtec_ntb *sndev;
1464         int rc;
1465
1466         stdev->sndev = NULL;
1467
1468         if (stdev->pdev->class != (PCI_CLASS_BRIDGE_OTHER << 8))
1469                 return -ENODEV;
1470
1471         sndev = kzalloc_node(sizeof(*sndev), GFP_KERNEL, dev_to_node(dev));
1472         if (!sndev)
1473                 return -ENOMEM;
1474
1475         sndev->stdev = stdev;
1476         rc = switchtec_ntb_init_sndev(sndev);
1477         if (rc)
1478                 goto free_and_exit;
1479
1480         switchtec_ntb_init_mw(sndev);
1481
1482         rc = switchtec_ntb_init_req_id_table(sndev);
1483         if (rc)
1484                 goto free_and_exit;
1485
1486         rc = switchtec_ntb_init_crosslink(sndev);
1487         if (rc)
1488                 goto free_and_exit;
1489
1490         switchtec_ntb_init_db(sndev);
1491         switchtec_ntb_init_msgs(sndev);
1492
1493         rc = switchtec_ntb_init_shared_mw(sndev);
1494         if (rc)
1495                 goto deinit_crosslink;
1496
1497         rc = switchtec_ntb_init_db_msg_irq(sndev);
1498         if (rc)
1499                 goto deinit_shared_and_exit;
1500
1501         /*
1502          * If this host crashed, the other host may think the link is
1503          * still up. Tell them to force it down (it will go back up
1504          * once we register the ntb device).
1505          */
1506         switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_FORCE_DOWN);
1507
1508         rc = ntb_register_device(&sndev->ntb);
1509         if (rc)
1510                 goto deinit_and_exit;
1511
1512         stdev->sndev = sndev;
1513         stdev->link_notifier = switchtec_ntb_link_notification;
1514         dev_info(dev, "NTB device registered\n");
1515
1516         return 0;
1517
1518 deinit_and_exit:
1519         switchtec_ntb_deinit_db_msg_irq(sndev);
1520 deinit_shared_and_exit:
1521         switchtec_ntb_deinit_shared_mw(sndev);
1522 deinit_crosslink:
1523         switchtec_ntb_deinit_crosslink(sndev);
1524 free_and_exit:
1525         kfree(sndev);
1526         dev_err(dev, "failed to register ntb device: %d\n", rc);
1527         return rc;
1528 }
1529
1530 static void switchtec_ntb_remove(struct device *dev,
1531                                  struct class_interface *class_intf)
1532 {
1533         struct switchtec_dev *stdev = to_stdev(dev);
1534         struct switchtec_ntb *sndev = stdev->sndev;
1535
1536         if (!sndev)
1537                 return;
1538
1539         stdev->link_notifier = NULL;
1540         stdev->sndev = NULL;
1541         ntb_unregister_device(&sndev->ntb);
1542         switchtec_ntb_deinit_db_msg_irq(sndev);
1543         switchtec_ntb_deinit_shared_mw(sndev);
1544         switchtec_ntb_deinit_crosslink(sndev);
1545         kfree(sndev);
1546         dev_info(dev, "ntb device unregistered\n");
1547 }
1548
1549 static struct class_interface switchtec_interface  = {
1550         .add_dev = switchtec_ntb_add,
1551         .remove_dev = switchtec_ntb_remove,
1552 };
1553
1554 static int __init switchtec_ntb_init(void)
1555 {
1556         switchtec_interface.class = switchtec_class;
1557         return class_interface_register(&switchtec_interface);
1558 }
1559 module_init(switchtec_ntb_init);
1560
1561 static void __exit switchtec_ntb_exit(void)
1562 {
1563         class_interface_unregister(&switchtec_interface);
1564 }
1565 module_exit(switchtec_ntb_exit);