1 In our testing, md raid10 is about 30% faster than megaraid's internal raid10.
3 Here are steps to replace hardware megaraid with software md5 on ganeti.
8 root@lib30:/srv/gnt-info# gnt-node migrate lib28
10 root@lib30:/srv/gnt-info# gnt-node modify --drained yes lib28
12 root@lib30:/srv/gnt-info# hbal -L -X
16 This will migrate instances from node, marked it as drained and then hbal will re-distribute drbd disks
17 from it to rest of cluster.
19 Alternative is to use modify -t plain, but that requires instance reboot.
22 root@lib28:~# vgremove ffzgvg
24 root@lib28:~# pvremove /dev/bcache0
28 Then, we need to find bcache block device and deactiate it:
30 root@lib28:~# cd /sys/fs/bcache/
31 root@lib28:/sys/fs/bcache# cd 4a61e966-f18d-4fab-836a-3ff027963781/
33 root@lib28:/sys/fs/bcache/4a61e966-f18d-4fab-836a-3ff027963781# ls -al bdev0
34 lrwxrwxrwx 1 root root 0 Sep 19 19:46 bdev0 -> ../../../devices/pci0000:00/0000:00:02.2/0000:03:00.0/host0/target0:2:0/0:2:0:0/block/sdb/bcache
37 root@lib28:~# echo 1 > /sys/block/sdb/bcache/stop
38 [Wed Sep 19 20:05:53 2018] bcache: bcache_device_free() bcache0 stopped
40 root@lib28:~# megacli -CfgDsply -a0
45 root@lib28:~# megacli -CfgLdDel -L0 -a0
47 Adapter 0: Deleted Virtual Drive-0(target id-0)
52 Convert all disks to jbod
54 root@lib28:~# megacli -PDMakeJBOD -PhysDrv[32:0] -a0
56 Adapter: 0: EnclId-32 SlotId-0 state changed to JBOD.
60 root@lib28:~# megacli -PDMakeJBOD -PhysDrv[32:1] -a0
64 megacli -PDMakeJBOD -PhysDrv[32:6] -a0
66 Adapter: 0: Failed to change PD state at EnclId-32 SlotId-6.
71 This probably means that it has foreign config (makes sense, 7th disk
72 comes from another server with raid on it):
74 root@lib28:~# megacli -CfgForeign -Clear 0 -a0
76 Foreign configuration 0 is cleared on controller 0.
81 root@lib28:~# megacli -PDMakeJBOD -PhysDrv[32:6] -a0
83 Adapter: 0: EnclId-32 SlotId-6 state changed to JBOD.
88 root@lib28:~# lsblk --scsi -m
89 NAME HCTL TYPE VENDOR MODEL REV TRAN NAME SIZE OWNER GROUP MODE
90 sda 0:0:7:0 disk ATA INTEL SSDSC2BW24 DC32 sda 223.6G root disk brw-rw----
91 sdb 0:0:0:0 disk ATA ST1000NM0033-9ZM GA0A sdb 931.5G root disk brw-rw----
92 sdc 0:0:1:0 disk ATA ST1000NM0033-9ZM GA0A sdc 931.5G root disk brw-rw----
93 sdd 0:0:2:0 disk ATA ST1000NM0033-9ZM GA0A sdd 931.5G root disk brw-rw----
94 sde 0:0:3:0 disk ATA ST1000NM0033-9ZM GA0A sde 931.5G root disk brw-rw----
95 sdf 0:0:4:0 disk ATA ST1000NM0033-9ZM GA0A sdf 931.5G root disk brw-rw----
96 sdg 0:0:5:0 disk ATA ST1000NM0033-9ZM GA0A sdg 931.5G root disk brw-rw----
97 sdh 0:0:6:0 disk ATA ST1000NM0033-9ZM GA0A sdh 931.5G root disk brw-rw----
99 root@lib28:~# mdadm --create /dev/md0 --level=10 --raid-devices=7 /dev/sd{b,c,d,e,f,g,h}
100 mdadm: cannot open /dev/sdb: Device or resource busy
102 NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
103 sda 8:0 0 223.6G 0 disk
104 ├─sda1 8:1 0 260M 0 part /boot/efi
105 ├─sda2 8:2 0 10G 0 part /
106 └─sda3 8:3 0 213.3G 0 part
107 sdb 8:16 0 931.5G 0 disk
108 └─bcache0 254:0 0 931.5G 0 disk
109 sdc 8:32 0 931.5G 0 disk
110 └─bcache1 254:1 0 931.5G 0 disk
111 sdd 8:48 0 931.5G 0 disk
112 sde 8:64 0 931.5G 0 disk
113 sdf 8:80 0 931.5G 0 disk
114 sdg 8:96 0 931.5G 0 disk
115 sdh 8:112 0 931.5G 0 disk
119 root@lib28:~# wipefs /dev/sda
121 ----------------------------------------------------------------
122 0x1018 bcache [other]
123 UUID: 2dbf6232-0e89-4d7b-87c7-79897b0a34f2
125 root@lib28:~# wipefs /dev/sdb
127 ----------------------------------------------------------------
128 0x1018 bcache [other]
129 UUID: 2dbf6232-0e89-4d7b-87c7-79897b0a34f2
131 root@lib28:~# echo 1 > /sys/block/sda/bcache/stop
132 root@lib28:~# [Wed Sep 19 20:25:14 2018] bcache: bcache_device_free() bcache0 stopped
134 root@lib28:~# echo 1 > /sys/block/sdb/bcache/stop
135 root@lib28:~# [Wed Sep 19 20:25:20 2018] bcache: bcache_device_free() bcache1 stopped
138 NAME MAJ:MIN RM SIZE RO TYPE MOUNTPOINT
139 sda 8:0 0 931.5G 0 disk
140 sdb 8:16 0 931.5G 0 disk
141 sdc 8:32 0 931.5G 0 disk
142 sdd 8:48 0 931.5G 0 disk
143 sde 8:64 0 931.5G 0 disk
144 sdf 8:80 0 931.5G 0 disk
145 sdg 8:96 0 931.5G 0 disk
146 sdh 8:112 0 223.6G 0 disk
147 ├─sdh1 8:113 0 260M 0 part /boot/efi
148 ├─sdh2 8:114 0 10G 0 part /
149 └─sdh3 8:115 0 213.3G 0 part
151 root@lib28:~# mdadm --create /dev/md0 --level=10 --raid-devices=7 /dev/sd{a,b,c,d,e,f,g}
152 mdadm: Defaulting to version 1.2 metadata
153 mdadm: array /dev/md0 started.