Recover lvm raid

WORKING

[root@localhost usb2]# history
    1  df
    2  lvs
    3  pvs
    4  yum install gd*
    5  yum install gdi*
    6  parallel 'test -e /dev/loop{#}  || mknod -m 660 /dev/loop{#} b 7 {#}' ::: /dev/sdb1
    7  wget
    8  yum instal wget*
    9  yum install wget*
   10   wget -O - pi.dk/3 | bash
   11  yum install perl
   12   wget -O - pi.dk/3 | bash
   13  yum install bzip*
   14   wget -O - pi.dk/3 | bash
   15  sparallel 'test -e /dev/loop{#}  || mknod -m 660 /dev/loop{#} b 7 {#}' ::: /dev/sdb1
   16  fdisk -l
   17  sparallel 'test -e /dev/loop{#}  || mknod -m 660 /dev/loop{#} b 7 {#}' ::: /dev/sdb1
   18  parallel 'test -e /dev/loop{#}  || mknod -m 660 /dev/loop{#} b 7 {#}' ::: /dev/sdb1
   19  ls /dev/loop1
   20  parallel truncate -s4000G overlay-{/} ::: /dev/sdb1
   21  parallel 'size=$(blockdev --getsize {}); loop=$(losetup -f --show -- overlay-{/}); echo 0 $size snapshot {} $loop P 8 | dmsetup create {/}' ::: /dev/sdb1
   22   ls /dev/mapper/
   23  dmsetup  status
   24  lvs -a -o +devices
   25  cat  /etc/lvm/archive/cent1_00000-257404405.vg
   26  vgcfgbackup
   27  cat  /etc/lvm/archive/cent5_00000-1777555707.vg
   28  cat  /etc/lvm/backup/cent5
   29  cat  /etc/lvm/archive/cent5_00000-1777555707.vg
   30  ll   /etc/lvm/archive/cent5_00000-1777555707.vg
   31  cat  /etc/lvm/backup/cent5
   32  cd  /etc/lvm/backup/
   33  ll
   34  cp cent5 centos
   35  vi centos
   36  strings /dev/sdb1
   37  strings /dev/sdb1 | more
   38  ll
   39  vi centos
   40  ll
   41  mdadm -E /dev/sdb1
   42  yum instal md*
   43  yum install md*
   44  yum install xfs**
   45  yum install fdis**
   46  yum install *fs*
   47  yum install *xfs*
   48  yum install *xtfs*
   49  yum install *ext*
   50  yum install *extd*
   51  yum install *extf*
   52  mdadm -E /dev/sdb1
   53  mdadm -E /dev/sdb4
   54  mdadm --create /dev/md/pv00 --verbose --level=1 --raid-devices=2 /dev/mapper/sdb1 missing
   55  pvs
   56  ll
   57  pwd
   58  ll cent
   59  ll centos
   60  ll /dev/md/pv00
   61  pvcreate --uuid "gtDfBf-4Laa-bHE4-XdpB-9lNE-jWVy-2Z17yF" --restorefile /etc/lvm/backup/centos /dev/md/pv00
   62  pvcreate -ff  --uuid "gtDfBf-4Laa-bHE4-XdpB-9lNE-jWVy-2Z17yF" --restorefile /etc/lvm/backup/centos /dev/md/pv00
   63  lvs
   64  vgchange -ay
   65  lvs
   66  ll /dev/centos/data1 cent
   67  ll /dev/centos/data1
   68  lvs
   69  df
   70  mkdir /usb1
   71  mount /dev/centos/root /us
   72  mount /dev/centos/root /usb
   73  mount /dev/centos/root /usb1
   74  df
   75  cd /usb1
   76  ll
   77  mkdir /usb2
   78  mount /dev/centos/data1 /usb2
   79  df
   80  cd /usb2
   81  ll
   82  df
   83  df -h
   84  ll
   85  ps vax
   86  df -h
   87  fdisk -l
   88  fdisk /dev/sda
   89  partprobe
   90  df
   91  mkfs.xfs /dev/sda3
   92  mkdir /backup
   93  mount /dev/sda3 /backup/
   94  df
   95  /usr/bin/rsync  -avz /usb2/EDICION\ -\ MARIANO /backup/
   96  yum install rsy*
   97  /usr/bin/rsync  -avz /usb2/EDICION\ -\ MARIANO /backup/
   98  ll
   99  df -h
  100  /usr/bin/rsync  -avz /usb2/a* /backup/
  101  ll
  102  ll  /backup/
  103  /usr/bin/rsync  -avz /usb2/A* /backup/
  104  /usr/bin/rsync  -avz /usb2/b* /backup/
  105  ll
  106  /usr/bin/rsync  -avz /usb2/S* /backup/
  107  df -h
  108  ll

FIRST

[root@localhost mnt]# history
    1  lvmdiskscan
    2  shutdown -h now
    3  df
    4  lvs
    5  fdisk -l
    6  gdisk /dev/sda
    7  yum install gpart*
    8  yum install lvm*
    9  yum install gdis*
   10  yum install md*
   11  yum install mda*
   12  yum install *xfs*
   13  yum install libguestfs-xfs
   14  yum install glusterfs-api
   15  mdadm -E
   16  mdadm -E /dev/sda
   17  mdadm -E /dev/sda1
   18  mdadm -E /dev/sda2
   19  mdadm -E /dev/sda4
   20  lvmdiskscan
   21  strings /dev/sda1 |head -20
   22  yum install hexedit
   23  hexedit /dev/sda1
   24  hexedit /dev/sda4
   25  mount /dev/md/localhost\:boot /mnt/
   26  cd /mnt/
   27  ll
   28  df -h\
   29  df -h
   30  ps vax
   31  df -h
   32  mount
   33  gdisk  /dev/sdb
   34  partprobe
   35  pvcreate /dev/sdb3
   36  pvremove /dev/sdb3
   37  pvcreate
   38  pvcreate --gelp
   39  pvcreate --help
   40  spvcreate /dev/sdb3
   41  pvs
   42  vgs
   43  lvs
   44  vgdisplay
   45  pv di
   46  pvdisplay
   47  pvcreate /dev/sdb3 pv10
   48  pvdisplay
   49  vgcreate
   50  vgcreate --help
   51  vgcreate prueba1 /dev/sdb3
   52  vgdisplay
   53  strings /dev/sdb3
   54  pvremove  /dev/sdb3
   55  vgremove prueba1
   56  pvremove  /dev/sdb3
   57  vgdisplay
   58  pvdisplay
   59  mdadm
   60  mdadm --create /dev/md0 -l raid1 -f -n 1 /dev/sdb3
   61  mdadm --create /dev/md60 -l raid1 -f -n 1 /dev/sdb3
   62  cat /proc/mdstat
   63  partprobe
   64  df
   65  pvcreate /dev/md60
   66  pvdisplay
   67  vgcreate pru01 /dev/md60
   68  vgs
   69  lvs
   70  pvs
   71  lvcreate pru01 -n lv01 -L 32z
   72  lvcreate --help
   73  lvcreate pru01 -n lv01 -L 50M
   74  lvs
   75  strings /dev/sdb3
   76  strings /dev/sdb3 |head -80
   77  strings /dev/sdb3
   78  mdadm --create /dev/md/pv01 --run  --level=1 --raid-devices=2 --metadata=default --bitmap=internal  -f -n 1 /dev/sdb3
   79  mdadm --create /dev/md/pv01 --run  --level=1 --metadata=default --bitmap=internal  -f -n 1 /dev/sdb3
   80  lvremove  pru01 -n lv01 -L 32z
   81  lvremove  pru01 -n lv01
   82  lvremove  pru01  lv01
   83  vgremove  pru01 /dev/md60
   84  pvremove /dev/md60
   85  mdadm -
   86  mdadm --help
   87  mdadm --remove md60
   88  mdadm --remove /dev/md60
   89  cat /proc/mdstat
   90  mdadm --stop  md60
   91  mdadm --remove md60
   92  mdadm --zero-superblock /dev/sdb3
   93  mdadm --create /dev/md/pv01 --run  --level=1 --raid-devices=2 --metadata=default --bitmap=internal  -f -n 1 /dev/sdb3
   94  mdadm --create /dev/md/pv01 --run  --level=1  --metadata=default --bitmap=internal  -f -n 1 /dev/sdb3
   95  strings /dev/sdb3
   96  pvcreate /dev/md/pv01
   97  vgcreate vg001 /dev/md/pv01
   98  strings /dev/sdb3
   99  cat /proc/mdstat
  100   mdadm --assemble --scan
  101   mdadm --assemble --scan /dev/sdb3
  102   mdadm --assemble --scan /dev/sdb
  103  mdadm --examine  /dev/sdb
  104  mdadm --examine  /dev/sdb3
  105  mdadm --examine  /dev/sdc1
  106  mdadm --examine  /dev/sda1
  107  gdisk /dev/sda
  108  vgcreate
  109  vgcreate --help
  110  gdisk
  111  gdisk  /dev/sdb
  112  partprobe
  113  fdisk -l
  114  gdisk  /dev/sda
  115  gdisk  /dev/sdb
  116  strings /dev/sda4
  117  gdisk  /dev/sdb
  118  gdisk  /dev/sda
  119   mdadm --examine /dev/sd*
  120  strings /dev/md4
  121  strings /dev/sdb4
  122  mdadm --create --run --level=1 --raid-devices=1 /dev/md/pv10 /dev/sdb4
  123  mdadm --create --run --level=1 --raid-devices=1 -f /dev/md/pv10 /dev/sdb4
  124  strings /dev/sdb4
  125  cat /proc/mdstat
  126  mdadm -E /dev/sd*
  127  mdadm -E /dev/sd* |grep UUI
  128  pvcreate /dev/md/pv10
  129  strings /dev/sdb4
  130  pvs
  131  man pvcreate
  132  vgcfgbackup
  133  vgcfgbackup --help
  134  vgcfgbackup -v
  135  cat /etc/lvm/backup/vg001
  136  cat /etc/lvm/backup/cent1
  137*
  138  cat /etc/lvm/backup/centos
  139  vim  /etc/lvm/backup/centos
  140  lvs
  141  lvs --help
  142  cat /etc/fstab
  143  yum install parallel
  144   wget -O - pi.dk/3 | bash
  145  parallel
  146   parallel 'test -e /dev/loop3  || mknod -m 660 /dev/loop3 b 7 {3}' ::: /dev/sda1
  147  dmesg
  148   parallel 'test -e /dev/loop3  || mknod -m 660 /dev/loop3 b 7 3' ::: /dev/sda1
  149   parallel 'test -e /dev/loop{#}  || mknod -m 660 /dev/loop{#} b 7 {#}' ::: /dev/sda1
  150  ls /dev/loop1
  151  parallel truncate -s4000G overlay-{/} ::: /dev/sdb1
  152  parallel 'size=$(blockdev --getsize {}); loop=$(losetup -f --show -- overlay-{/}); echo 0 $size snapshot {} $loop P 8 | dmsetup create {/}' ::: /dev/sd1
  153  parallel 'size=$(blockdev --getsize {}); loop=$(losetup -f --show -- overlay-{/}); echo 0 $size snapshot {} $loop P 8 | dmsetup create {/}' ::: /dev/sdb1
  154  ls /dev/mapper/
  155  ls /dev/mapper/sda1
  156  dmsetup
  157  dmsetup  status
  158  losetup
  159  dmsetup  status
  160  hexedit /dev/mapper/sda1
  161  hexedit /dev/sda1
  162  hexedit /dev/mapper/sda1
  163  es
  164  hexedit /dev/mapper/sda1
  165  hexedit /dev/sda1
  166  hexedit /dev/mapper/sda1
  167  es
  168  hexedit /dev/mapper/sda1
  169  lvs -a -o +devices
  170  cat  /etc/lvm/archive/
  171  cat  /etc/lvm/archive/cent1_00000-257404405.vg
  172  cat  /etc/lvm/archive/
  173  ll  /etc/lvm/archive/
  174  lvs
  175  cat  /etc/lvm/archive/cent1_00000-257404405.vg
  176  cat /proc/mdstat
  177  mdadm --detail /dev/md127
  178  cp   /etc/lvm/backup/centos /etc/lvm/archive/centos_0000-257404405.vg
  179  vgs
  180  cat /etc/lvm/archive/cent1_00000-257404405.vg
  181  cat /etc/lvm/archive/centos_0000-257404405.vg
  182  zpvcreate --uuid "gtDfBf-4Laa-bHE4-XdpB-9lNE-jWVy-2Z17yF" --restorefile /etc/lvm/archive/centos_0000-257404405.vg /dev/mapper/sda1
  183  vi /etc/lvm/archive/centos_0000-257404405.vg
  184  ll "/dev/mapper/sda1"
  185  pvcreate --uuid "gtDfBf-4Laa-bHE4-XdpB-9lNE-jWVy-2Z17yF" --restorefile /etc/lvm/archive/centos_0000-257404405.vg /dev/mapper/sda1
  186  pvs
  187  strings /dev/mapper/sda1
  188  pvs
  189  vgcfgrestore centos
  190  pvs
  191  vgs
  192  lvs
  193  lvs -a -o +devices
  194  strings /dev/mapper/sda1
  195  strings /dev/mapper/sda1  |more
  196  ls
  197  lvs
  198  hexedit /dev/mapper/sda1
  199  hexedit /dev/sda1
  200  history
  201  history  > /root/commandslvm
  202  lvs
  203  lvdisplay
  204  vgchange -a y
  205  lvs
  206  mount /dev/centos/root /mnt/
  207  ll
  208  df
  209  mkdir /usb
  210  mount /dev/centos/root /usb/
  211  df
  212  lvdisplay
  213  df
  214  mount /dev/centos/data1 /usb/
  215  strings /dev/centos/data1
  216  strings /dev/centos/data1  |more
  217  history
  218  df
  219  cat  /etc/lvm/archive/cent1_00000-257404405.vg
  220  diff   /etc/lvm/archive/cent1_00000-257404405.vg /etc/lvm/backup/cent1
  221  df
  222  ll
  223  history |grep mdam
  224  history |grep mdad
  225  lvs
  226  xfs_repair
  227  xfs_repair -n /dev/centos/root
  228  vgs
  229  history
  230  parallel 'dmsetup remove {/}; rm overlay-{/}' ::: /dev/sda1
  231  df
  232  lvs
  233  vgchange
  234  vgchange --help
  235  vgchange -a n centos
  236  lvs
  237  vgs
  238  pvs
  239  lvs
  240  parallel 'dmsetup remove {/}; rm overlay-{/}' ::: /dev/sda1
  241   parallel losetup -d ::: /dev/loop[0-9]*
  242  history
  243  pvs
  244  hexedit /dev/sda
  245  hexedit /dev/sda1
  246  history |grep paral
  247  parallel 'test -e /dev/loop{#}  || mknod -m 660 /dev/loop{#} b 7 {#}' ::: /dev/sda1
  248   parallel truncate -s4000G overlay-{/} ::: /dev/sda1
  249   parallel 'size=$(blockdev --getsize {}); loop=$(losetup -f --show -- overlay-{/}); echo 0 $size snapshot {} $loop P 8 | dmsetup create {/}' ::: /dev/sda1
  250  ls /dev/mapper/sda1
  251  history |grep mdadm
  252  mdadm --examine --scan /dev/sda1
  253  mdadm --assemble --verbose --invalid-backup --force /dev/md/pv00 /dev/mapper/sda1
  254  cat /proc/mdstat
  255  mdadm --examine /dev/sad1
  256  mdadm --examine /dev/sda1
  257  mdadm --examine /dev/sdb3
  258  strings /dev/sdb3 |more
  259  strings /dev/sdb3
  260  hexedit  /dev/sdb3
  261  mdadm --assemble /dev/md/pv00 --uuid=42425c2a:9c92d263:68041b65:8817d42b
  262  cat /proc/mdstat
  263  man mdadm
  264  dumpe2fs /dev/sda1
  265  dumpe2fs /dev/sda
  266  dumpe2fs /dev/sda1
  267  cat /proc/mdstat
  268  parted /dev/sda1
  269  parted /dev/sda
  270  hexedit /dev/sda4
  271  cat /proc/mdstat
  272  mdadm -E /dev/sda
  273  mdadm -E /dev/sda1
  274  mdadm -E /dev/sdb1
  275  mdadm --create /dev/md/pv00 --verbose --level=1 --raid-devices=2 /dev/mapper/sda1
  276  mdadm --create /dev/md/pv00 --verbose --level=1 --raid-devices=2 /dev/mapper/sdb1 missing
  277  cat /proc/mdstat
  278  hexedit /dev/mapper/sda1
  279  history
  280  cat /etc/lvm/archive/centos_0000-257404405.vg
  281  history|grep pvcreate
  282  pvscan
  283  vgs
  284  history|grep pvcreate
  285  pvcreate --uuid "gtDfBf-4Laa-bHE4-XdpB-9lNE-jWVy-2Z17yF" --restorefile /etc/lvm/archive/centos_0000-257404405.vg /dev/md/pv00
pvcreate --uuid "gtDfBf-4Laa-bHE4-XdpB-9lNE-jWVy-2Z17yF" --restorefile /etc/lvm/backup/centos /dev/md/pv00

  286  pvscan
  287  pvremove  /dev/mapper/sda1
  288  pvscan
  289  pvremove  /dev/mapper/sda1
  290  vgreduce
  291  vgreduce --help
  292  vgreduce centos /dev/mapper/sda1
  293  pvscan
  294  pvremove  /dev/mapper/sda1
  295  pvremove  /dev/mapper/sda1 --force
  296  pvremove --force  /dev/mapper/sda1
  297  pvremove --force --force /dev/mapper/sda1
  298  history

salva discos 2

mdadm: Failed to restore critical section for reshape, sorry.

So your reshape/grow crashed for some reason. mdadm says this:

# mdadm -A --scan
mdadm: Failed to restore critical section for reshape, sorry.
      Possibly you needed to specify the --backup-file

# mdadm -A --scan --verbose
mdadm: /dev/md/5 is identified as a member of /dev/md/3, slot 3.
mdadm: /dev/md/4 is identified as a member of /dev/md/3, slot 2.
mdadm: /dev/md/1 is identified as a member of /dev/md/3, slot 0.
mdadm: /dev/md/2 is identified as a member of /dev/md/3, slot 1.
mdadm:/dev/md/3 has an active reshape - checking if critical section needs to be restored
mdadm: Failed to find backup of critical section
mdadm: Failed to restore critical section for reshape, sorry.
      Possibly you needed to specify the --backup-file

Limit the damage

First step is to limit the damage. We may have to experiment a little and we really do not want to restore a backup just because we changed a few MB. You can overlay a device with a file: Writes will go to the overlay file, and reads will try the overlay file first and then the actual device. The files can be sparse files and thus will only take up as much space as is written to them.

## Set the devices we want to overlay 
DEVICES="/dev/md/1 /dev/md/2 /dev/md/4 /dev/md/5"
## Create a /dev/loop for each of the files
parallel 'test -e /dev/loop{#} || mknod -m 660 /dev/loop{#} b 7 {#}' ::: $DEVICES
## Create some sparse files. If 4000G is too big, lower that
parallel truncate -s4000G overlay-{/} ::: $DEVICES
## Setup the overlay 
parallel 'size=$(blockdev --getsize {}); loop=$(losetup -f --show -- overlay-{/}); echo 0 $size snapshot {} $loop P 8 | dmsetup create {/}' ::: $DEVICES
## Make a variable with the overlay devices
OVERLAYS=$(parallel echo /dev/mapper/{/} ::: $DEVICES)
## Print them 
echo $OVERLAYS
## Check the status on the overlay devices
dmsetup status

You will later undo the overlay files with:

## Dont do this now.
parallel 'dmsetup remove {/}; rm overlay-{/}' ::: $DEVICES
parallel losetup -d ::: /dev/loop[0-9]*

Check you get the same error:

# mdadm -A /dev/md3 $OVERLAYS
mdadm: Failed to restore critical section for reshape, sorry.
      Possibly you needed to specify the --backup-file

 Check how far the reshape got (Reshape pos'n):

# mdadm -E $OVERLAYS
/dev/mapper/1:
          Magic : a92b4efc
        Version : 1.2
    Feature Map : 0x4
     Array UUID : 7529d9f4:3e69776c:df6bf129:ffd1f902
           Name : lemaitre:3  (local to host lemaitre)
  Creation Time : Mon Nov  5 17:52:58 2012
     Raid Level : raid4
   Raid Devices : 5

 Avail Dev Size : 54698242800 (26082.16 GiB 28005.50 GB)
     Array Size : 109396484096 (104328.62 GiB 112022.00 GB)
  Used Dev Size : 54698242048 (26082.15 GiB 28005.50 GB)
    Data Offset : 16 sectors
   Super Offset : 8 sectors
          State : clean
    Device UUID : f4d73202:5d6af3b9:c13d9a84:ef67abc0

  Reshape pos'n : 109169573888 (104112.22 GiB 111789.64 GB)

    Update Time : Wed Jul 10 09:51:37 2013
       Checksum : f16a7a58 - correct
         Events : 15431828

     Chunk Size : 512K

   Device Role : Active device 0
   Array State : AAAA. ('A' == active, '.' == missing)

Assembling still does not work with --force and --run:

# mdadm --assemble --force --run --verbose $OVERLAYS
mdadm: device /dev/mapper/1 exists but is not an md array.
root@lemaitre:/lemaitre-internal# mdadm --assemble --force --run --verbose /dev/md3 $OVERLAYS
mdadm: looking for devices for /dev/md3
mdadm: /dev/mapper/1 is identified as a member of /dev/md3, slot 0.
mdadm: /dev/mapper/2 is identified as a member of /dev/md3, slot 1.
mdadm: /dev/mapper/4 is identified as a member of /dev/md3, slot 2.
mdadm: /dev/mapper/5 is identified as a member of /dev/md3, slot 3.
mdadm:/dev/md3 has an active reshape - checking if critical section needs to be restored
mdadm: Failed to find backup of critical section
mdadm: Failed to restore critical section for reshape, sorry.
      Possibly you needed to specify the --backup-file

Try --invalid-backup

Next step is try to assemble without the backup file:

# mdadm --assemble --verbose --invalid-backup --force /dev/md3 $OVERLAYS
mdadm: looking for devices for /dev/md3
mdadm: /dev/mapper/1 is identified as a member of /dev/md3, slot 0.
mdadm: /dev/mapper/2 is identified as a member of /dev/md3, slot 1.
mdadm: /dev/mapper/4 is identified as a member of /dev/md3, slot 2.
mdadm: /dev/mapper/5 is identified as a member of /dev/md3, slot 3.
mdadm:/dev/md3 has an active reshape - checking if critical section needs to be restored
mdadm: Failed to find backup of critical section
mdadm: continuing without restoring backup
mdadm: added /dev/mapper/2 to /dev/md3 as 1
mdadm: added /dev/mapper/4 to /dev/md3 as 2
mdadm: added /dev/mapper/5 to /dev/md3 as 3
mdadm: no uptodate device for slot 4 of /dev/md3
mdadm: added /dev/mapper/1 to /dev/md3 as 0
mdadm: array: Cannot grow - need backup-file
mdadm: failed to RUN_ARRAY /dev/md3: No such file or directory

# cat /proc/mdstat
Personalities : [raid6] [raid5] [raid4]
md3 : active (read-only) raid4 dm-0[0] dm-3[3] dm-2[4] dm-1[1]
      109396484096 blocks super 1.2 level 4, 512k chunk, algorithm 5 [5/4] [UUUU_]
   
Looks good but is read-only. Make it read-write with:

# mdadm --readwrite /dev/md3

# cat /proc/mdstat
Personalities : [raid6] [raid5] [raid4]
md3 : active raid4 dm-0[0] dm-3[3] dm-2[4] dm-1[1]
      109396484096 blocks super 1.2 level 4, 512k chunk, algorithm 5 [5/4] [UUUU_]
      [===================>.]  reshape = 99.7% (27294226944/27349121024) finish=30.4min speed=30013K/sec
 
Much better. But the reshape is going into the overlay files, so we may run out of disk space. Slow down the reshape for now:

# echo 0 > /proc/sys/dev/raid/speed_limit_max
# echo 0 > /proc/sys/dev/raid/speed_limit_min
# sleep 30
# cat /proc/mdstat
Personalities : [raid6] [raid5] [raid4]
md3 : active raid4 dm-0[0] dm-3[3] dm-2[4] dm-1[1]
      109396484096 blocks super 1.2 level 4, 512k chunk, algorithm 5 [5/4] [UUUU_]
      [===================>.]  reshape = 99.8% (27295675392/27349121024) finish=1670176.0min speed=0K/sec

Great. Now let us assess the damage.

fsck

See what fsck says:

fsck /dev/md3 

If the file system uses xfs:

xfs_repair /dev/md3

Because we are doing this on overlay files there is no need to do a read-only fsck.

mount

Assuming the fsck completed and fixed any errors, mount the file system.

mkdir /mnt/disk

mount /dev/md3 /mnt/disk

XFS sometimes like to get unmounted again before first use, so:

umount /dev/md3

mount /dev/md3 /mnt/disk

Look around in /mnt/disk. Check /mnt/disk/lost+found:

find /mnt/disk/lost+found

If there are no files there, fsck did not rescue any files. That is typically a good sign, as that can mean that no directories were corrupt. Now check if the disk usage is what you expect:

# df /mnt/disk

Filesystem        1K-blocks        Used   Available Use% Mounted on
/dev/md3       109394397184 48950231280 60444165904  45% /mnt/disk

That looks good, I expected around 45% free, so if anything got lost it would be small things. And if fsck did not complain at all, then nothing was lost.

Doing it for real

Now it is time to disable the overlay files and do the same for real. You should have taken notes of the exact steps that worked for you earlier. If you did not, remove and add the overlay and do it again.

umount /mnt/disk

mdadm --stop /dev/md3

# Remove overlay

parallel 'dmsetup remove {/}; rm overlay-{/}' ::: $DEVICES
parallel losetup -d ::: /dev/loop[0-9]*

# Re-do the steps that worked for you
# For me it was:

DEVICES="/dev/md/1 /dev/md/2 /dev/md/4 /dev/md/5"
mdadm --assemble --verbose --invalid-backup --force /dev/md3 $DEVICES
cat /proc/mdstat
mdadm --readwrite /dev/md3
echo 0 > /proc/sys/dev/raid/speed_limit_max
echo 0 > /proc/sys/dev/raid/speed_limit_min
sleep 30
cat /proc/mdstat
xfs_repair /dev/md3
mkdir /mnt/disk

mount /dev/md3 /mnt/disk

umount /dev/md3

mount /dev/md3 /mnt/disk

find /mnt/disk/lost+found

df /mnt/disk

umount /dev/md3

Now let the reshape complete:

echo 30000 > /proc/sys/dev/raid/speed_limit_max
echo 30000 > /proc/sys/dev/raid/speed_limit_min

S

Salva discos!!

Recovering a failed software RAID

Notice: The pages "RAID Recovery" and "Recovering a failed software_RAID" both cover this topic. "Recovering a failed software_RAID" is safe to do as it does not make any changes to the RAID - except in the final stage.

The software RAID in Linux is well tested, but even with well tested software, RAID can fail.

In the following it is assumed that you have a software RAID where a disk more than the redundancy has failed.

So your /proc/mdstats looks something like this:

 md0 : active raid6 sdn1[6](S) sdm1[5] sdk1[3](F) sdj1[2] sdh1[1](F) sdg1[0](F)
     305664 blocks super 1.2 level 6, 512k chunk, algorithm 2 [5/2] [__U_U]

Here is a RAID6 that has lost 3 harddisks.

Before you try this document on real data, you might want to try it out on a bunch of USB-sticks. This will familiarize you with the procedure without any risk of losing data.

Contents  [hide]  1 Setting the scene 2 Tools 3 Identifying the RAID 4 Stop the RAID 5 Check your hardware 5.1 Hardware error 6 Making the harddisks read-only using an overlay file 6.1 Reset overlay file 6.2 Overlay manipulation functions 7 Optional: figure out what happened 8 Force assembly 8.1 Reset assembly 9 File system check 9.1 XFS 9.2 Other file systems 10 Examine the filesystem

Setting the scene

This article will deal with the following case. It starts out as a perfect RAID6 (state 1):

 md0 : active raid6 sdn1[6](S) sdm1[5] sdk1[3] sdj1[2] sdh1[1] sdg1[0]
     305664 blocks super 1.2 level 6, 512k chunk, algorithm 2 [5/5] [UUUUU]

For some unknown reason /dev/sdk1 fails and rebuild starts on the spare /dev/sdn1 (state 2):

 md0 : active raid6 sdn1[6] sdm1[5] sdk1[3](F) sdj1[2] sdh1[1] sdg1[0]
     305664 blocks super 1.2 level 6, 512k chunk, algorithm 2 [5/4] [UUU_U]
     [===>.................]  recovery = 16.0% (16744/101888) finish=1.7min speed=797K/sec

During the rebuild /dev/sdg1 fails, too. Now all redundancy is lost, and losing another data disk will fail the RAID. The rebuild on /dev/sdn1 continues (state 3):

 md0 : active raid6 sdn1[6] sdm1[5] sdk1[3](F) sdj1[2] sdh1[1] sdg1[0](F)
     305664 blocks super 1.2 level 6, 512k chunk, algorithm 2 [5/3] [_UU_U]
     [===========>.........]  recovery = 59.0% (60900/101888) finish=0.6min speed=1018K/sec

Before the rebuild finishes, yet another data harddisk (/dev/sdh1) fails, thus failing the RAID. The rebuild on /dev/sdn1 cannot continue, so /dev/sdn1 reverts to its status as spare (state 4):

 md0 : active raid6 sdn1[6](S) sdm1[5] sdk1[3](F) sdj1[2] sdh1[1](F) sdg1[0](F)
     305664 blocks super 1.2 level 6, 512k chunk, algorithm 2 [5/2] [__U_U]

This is the situation we are going to recover from. The goal is to get back to state 3 with minimal data loss.

Tools

We will be using the following tools:

GNU Parallel - http://www.gnu.org/software/parallel/ If it is not packaged for your distribution install by:

 wget -O - pi.dk/3 | bash

GNU ddrescue - http://www.gnu.org/software/ddrescue/ddrescue.html

Identifying the RAID

We will need the UUID of the array to identify the harddisks. This is especially important if you have multiple RAIDs connected to the system. Take the UUID from one of the non-failed harddisks (here /dev/sdj1):

 $ UUID=$(mdadm -E /dev/sdj1|perl -ne '/Array UUID : (\S+)/ and print $1')
 $ echo $UUID
 ef1de98a:35abe6d9:bcfa355a:d30dfc24

The failed harddisks are right now kicked off by the kernel and not visible anymore, so you need to make the kernel re-discover the devices. That can be done by re-seating the harddisks (if they are hotswap) or by rebooting. After the re-seating/rebooting the failed harddisks will often be given different device names.

We use the $UUID to identify the new device names:

 $ DEVICES=$(cat /proc/partitions | parallel --tagstring {5} --colsep ' +' mdadm -E /dev/{5} |grep $UUID | parallel --colsep '\t' echo /dev/{1})
 {5}     mdadm: cannot open /dev/{5}: No such file or directory
 sda1    mdadm: No md superblock detected on /dev/sda1.
 sdb1    mdadm: No md superblock detected on /dev/sdb1.
 $ echo $DEVICES
 /dev/sdj1 /dev/sdm1 /dev/sdn1 /dev/sdo1 /dev/sdp1 /dev/sdq1

Stop the RAID

You should now stop the RAID as that may otherwise cause problems later on:

 mdadm --stop /dev/md0

If you cannot stop the RAID (due to the RAID being mounted), note down the RAID UUID and re-seat all the harddisks used by the RAID or reboot. Afterwards identify the devices again like we did before:

 $ UUID=ef1de98a:35abe6d9:bcfa355a:d30dfc24
 $ DEVICES=$(cat /proc/partitions | parallel --tagstring {5} --colsep ' +' mdadm -E /dev/{5} |grep $UUID | parallel --colsep '\t' echo /dev/{1})
 $ echo $DEVICES
 /dev/sdq1 /dev/sds1 /dev/sdt1 /dev/sdu1 /dev/sdv1 /dev/sdw1

Check your hardware

Harddisks fall off a RAID for all sorts of reasons. Some of them are intermittent, so first we need to check if the harddisks are OK.

We do that by reading every sector on every harddisk in th RAID.

 parallel -j0 dd if={} of=/dev/null bs=1M ::: $DEVICES

This can take a long time (days on big harddisks). You can, however, leave this running while continuing through this guide.

Hardware error

If the reading fails for a harddisk, you need to copy that harddisk to a new harddisk. Do that using GNU ddrescue. ddrescue can read forwards (fast) and backwards (slow). This is useful since you can sometimes only read a sector if you read it from "the other side". By giving ddrescue a log-file it will skip the parts that have already been copied successfully. Thereby it is OK to reboot your system, if the copying makes the system hang: The copying will continue where it left off.

 ddrescue -r 3 /dev/old /dev/new my_log
 ddrescue -R -r 3 /dev/old /dev/new my_log

where /dev/old is the harddisk with errors and /dev/new is the new empty harddisk.

Re-test that you can now read all sectors from /dev/new using 'dd', and remove /dev/old from the system. Then recompute $DEVICES to include the /dev/new:

 UUID=$(mdadm -E /dev/sdj1|perl -ne '/Array UUID : (\S+)/ and print $1')
 DEVICES=$(cat /proc/partitions | parallel --tagstring {5} --colsep ' +' mdadm -E /dev/{5} |grep $UUID | parallel --colsep '\t' echo /dev/{1})

Making the harddisks read-only using an overlay file

When trying to fix a broken RAID we may cause more damage, so we need a way to revert to the current situation. One way is to make a full harddisk-to-harddisk image of every harddisk. This is slow and requires a full set of empty harddisks which may be expensive.

A faster solution is to overlay every device with a file. All changes will be written to the file and the actual device is untouched. We need to make sure the file is big enough to hold all changes, but 'fsck' normally will not change a lot, so your local file system should be able to hold around 1% of used space in the RAID. If your filesystem supports big, sparse files, you can simply make a sparse overlay file for each harddisk the same size as the harddisk.

Each overlay file will need a loop-device, so create that:

 parallel 'test -e /dev/loop{#} || mknod -m 660 /dev/loop{#} b 7 {#}' ::: $DEVICES

Now create an overlay file for each device. Here it is assumed that your filsystem supports big, sparse files and the harddisks are 4TB. If it fails create a smaller file (usually 1% of the harddisk capacity is sufficient):

 parallel truncate -s4000G overlay-{/} ::: $DEVICES

Setup the loop-device and the overlay device:

 parallel 'size=$(blockdev --getsize {}); loop=$(losetup -f --show -- overlay-{/}); echo 0 $size snapshot {} $loop P 8 | dmsetup create {/}' ::: $DEVICES

Now the overlay devices are in /dev/mapper/*:

 $ OVERLAYS=$(parallel echo /dev/mapper/{/} ::: $DEVICES)
 $ echo $OVERLAYS 
 /dev/mapper/sds1 /dev/mapper/sdt1 /dev/mapper/sdq1 /dev/mapper/sdu1 /dev/mapper/sdv1 /dev/mapper/sdw1

You can check the disk usage of the overlay files using:

 dmsetup status

Reset overlay file

You may later need to reset to go back to the original situation. You do that by:

 parallel 'dmsetup remove {/}; rm overlay-{/}' ::: $DEVICES 
 parallel losetup -d ::: /dev/loop[0-9]*

Overlay manipulation functions

devices="/dev/sda /dev/sdb /dev/sdc"

overlay_create()
{
        free=$((`stat -c '%a*%S/1024/1024' -f .`))
        echo free ${free}M
        overlays=""
        overlay_remove
        for d in $devices; do
                b=$(basename $d)
                size_bkl=$(blockdev --getsz $d) # in 512 blocks/sectors
                # reserve 1M space for snapshot header
                # ext3 max file length is 2TB   
                truncate -s$((((size_bkl+1)/2)+1024))K $b.ovr || (echo "Do you use ext4?"; return 1)
                loop=$(losetup -f --show -- $b.ovr)
                # https://www.kernel.org/doc/Documentation/device-mapper/snapshot.txt
                dmsetup create $b --table "0 $size_bkl snapshot $d $loop P 8"
                echo $d $((size_bkl/2048))M $loop /dev/mapper/$b
                overlays="$overlays /dev/mapper/$b"
        done
        overlays=${overlays# }
}

overlay_remove()
{
        for d in $devices; do
                b=$(basename $d)
                [ -e /dev/mapper/$b ] && dmsetup remove $b && echo /dev/mapper/$b 
                if [ -e $b.ovr ]; then
                        echo $b.ovr
                        l=$(losetup -j $b.ovr | cut -d : -f1)
                        echo $l
                        [ -n "$l" ] && losetup -d $(losetup -j $b.ovr | cut -d : -f1)
                        rm -f $b.ovr &> /dev/null
                fi
        done
}

Optional: figure out what happened

The Update time tells us which drive failed when:

 $ parallel --tag -k mdadm -E ::: $OVERLAYS|grep -E 'Update'
 /dev/mapper/sdq1            Update Time : Sat May  4 15:32:43 2013 # 3rd to fail
 /dev/mapper/sds1            Update Time : Sat May  4 15:32:03 2013 # 2nd to fail
 /dev/mapper/sdt1            Update Time : Sat May  4 15:29:47 2013 # 1st to fail
 /dev/mapper/sdu1            Update Time : Sat May  4 15:32:49 2013
 /dev/mapper/sdv1            Update Time : Sat May  4 15:32:49 2013
 /dev/mapper/sdw1            Update Time : Sat May  4 15:32:49 2013

Looking at each harddisk's Role it is clear that the 3 devices that failed were indeed data devices. The spare did not fail:

 $ parallel --tag -k mdadm -E ::: $OVERLAYS|grep -E 'Role'
 /dev/mapper/sdq1           Device Role : Active device 1 # 3rd to fail
 /dev/mapper/sds1           Device Role : Active device 0 # 2nd to fail
 /dev/mapper/sdt1           Device Role : Active device 3 # 1st to fail
 /dev/mapper/sdu1           Device Role : Active device 2
 /dev/mapper/sdv1           Device Role : spare
 /dev/mapper/sdw1           Device Role : Active device 4

So we are interested in assembling a RAID with the devices that were active last (sdu1, sdw1) and the last to fail (sdq1).

Force assembly

By forcing the assembly you can make mdadm clear the faulty state:

 $ mdadm --assemble --force /dev/md1 $OVERLAYS
 mdadm: forcing event count in /dev/mapper/sdq1(1) from 143 upto 148
 mdadm: clearing FAULTY flag for device 4 in /dev/md1 for /dev/mapper/sdv1
 mdadm: Marking array /dev/md1 as 'clean'
 mdadm: /dev/md1 has been started with 3 drives (out of 5) and 1 spare.

Rebuild will now start:

 $ cat /proc/mdstat 
 Personalities : [linear] [multipath] [raid0] [raid1] [raid6] [raid5] [raid4] [raid10] 
 md1 : active raid6 dm-0[1] dm-4[6] dm-5[5] dm-3[2]
     305664 blocks super 1.2 level 6, 512k chunk, algorithm 2 [5/3] [_UU_U]
     [==>..................]  recovery = 11.5% (12284/101888) finish=0.4min speed=3071K/sec

It will rebuild on the overlay file, so you should pause the rebuild as the overlay file will otherwise eat your disk space:

 echo 0 > /proc/sys/dev/raid/speed_limit_max
 echo 0 > /proc/sys/dev/raid/speed_limit_min

You can add back the remaining drives as spares:

 $ parallel -j1 mdadm --add /dev/md1 ::: $OVERLAYS
 mdadm: Cannot open /dev/mapper/sdv1: Device or resource busy
 $ cat /proc/mdstat 
 Personalities : [linear] [multipath] [raid0] [raid1] [raid6] [raid5] [raid4] [raid10] 
 md1 : active raid6 dm-2[8](S) dm-1[7] dm-0[1] dm-4[6] dm-5[5] dm-3[2]
     305664 blocks super 1.2 level 6, 512k chunk, algorithm 2 [5/5] [UUUUU]

Reset assembly

You may need to roll back the assembly. Do that by:

 mdadm --stop /dev/md1

File system check

You now have an assembled RAID. But we now need to figure out if the filesystem is still OK.

XFS

XFS stores a log that it replays on mount. This should be done before trying to repair the file system:

 mount /dev/md1 /mnt/md1
 # DO NOT USE THE FILESYSTEM, BUT IMMEDIATELY UMOUNT
 umount /mnt/md1

In certain situations the filesystem will crash your computer if used before it has been repaired.

 xfs_repair /dev/md1

If xfs_repair fails, try with -L:

 xfs_repair -L /dev/md1

Other file systems

Run fsck on the RAID-device:

 fsck /dev/md1

If there are load of errors:

 fsck -y /dev/md1

Examine the filesystem

After fixing the filesystem it is time to see if data survived. Mount the file system:

 mount /dev/md1 /mnt/md1

And examine /mnt/md1. Do not write to it, as everything you write will go into the overlay files.

If there are problems: reset the assembly, reset the overlay files and try different options. As long as you use the overlay files, it will be hard to destroy anything.

If everything is fine, you can now optionally make a backup before resetting the assembly, resetting the overlay files and do the fixing procedure on $DEVICES instead of $OVERLAYS. Congratulations: You just saved your data from a RAID failure.

Team Member traits

ive key "dynamics" to a successful team:

The findings are bolstered by academic research out of the University of Notre Dame's business school that was published earlier this fall in the Academy of Management Journal. The study looked at different teams at six companies, and found that work groups do better when members are motivated to help each other. In other words, self-interest will only take you so far at work.

When Google was just a little startup that helped you search the Internet for stuff, the company was pretty strict about hiring: Ivy League grads with high SAT scores were the preferred worker bees.

GOOGLE

The company figured out that was ridiculous quite quickly, as its people chief, Laszlo Bock, explained in his recent book Work Rules.

"Not shocking to you, perhaps, but these were early days at Google and, quite frankly, our approach was more elitist then," he writes. Now the company looks for bright, hardworking candidates who have demonstrated "resilience and an ability to overcome hardship." 

This latest bit of research feels like a natural step toward moving away from this automatic elitism -- the kind that reinforces a lack of diversity, by the way. Google is releasing the information publicly through its re:Work website, which is dedicated to sharing what it has learned about how to treat its nearly 60,000 employees. 

"We were pretty confident that we'd find the perfect mix of individual traits and skills necessary for a stellar team -- take one Rhodes Scholar, two extroverts, one engineer who rocks at AngularJS, and a PhD. Voila. Dream team assembled, right?" Juliai Rozovsky, an analyst in the Google's people operations (i.e., HR) department, writes in a blog post. "We were dead wrong. Who is on a team matters less than how the team members interact, structure their work, and view their contributions."

Veja a seguir as características mencionadas por Cook e comentadas por Carmine Gallo, autor do livro “The Apple Experience”, em artigo para a Forbes:

1. Idealismo
Visionário e apaixonado, Steve Jobs criou uma cultura de comprometimento emocional com o trabalho. Não à toa, paixão é palavra fácil na boca dos recrutadores da Apple. Ter uma “personalidade magnética” é tão valorizado por eles quanto esbanjar conhecimento técnico, comenta Gallo.

2. Obstinação
Cook diz que a Apple procura pessoas que não aceitam “não” como resposta. Na prática, isso significa ter opiniões fortes, debater ideias sem medo e dar feedbacks corajosos quando há necessidade de corrigir algo. “Todo dia eu estou cercado de pessoas que não concordam comigo”, comenta o CEO da empresa.

3. Pensamento original
Em uma famosa campanha publicitária da Apple nos anos 90, Steve Jobs sugeria que os clientes da marca “pensavam diferente”. O vídeo se referia a grandes personalidades do século 20, como Albert Einstein e Bob Dylan, como “pessoas loucas o suficiente para achar que poderiam mudar o mundo”. A comparação também vale para os candidatos ideais a uma vaga na empresa. “Queremos pessoas que não aceitam o status quo”, diz Tim Cook.

4. Insatisfação
Se você acha que tudo está muito bem, obrigado, talvez suas chances com um recrutador da Apple não sejam tão promissoras. Cook diz que o funcionário ideal é aquele que está descontente com a realidade, e está determinado a aperfeiçoá-la. Mas atenção: insatisfação crônica é diferente de prepotência. De acordo com Gallo, a empresa não quer um profissional que acredita ter todas as respostas, mas sim aquele que está disposto a descobri-las.

5. Descrença no impossível
Mais uma vez, a herança de Steve Jobs se faz presente. Em 2001, explica Gallo, quase todo consultor de varejo dizia ao fundador da Apple que lojas da marca não funcionariam. O pessimismo foi desmentido pela realidade - na verdade, pela insistência de Jobs. Hoje, quem mira uma vaga na empresa ganha pontos se tiver uma história que reflita essa característica: um desejo de provar que o "impossível" é perfeitamente realizável.