Symptoms
Several container migration attempts were performed, some of them were unsuccessful due to various reasons/misconfigurations. Finally the container was migrated.
There is a number of CT_ID.xxxx folders in "/vz/private":
# cd /vz/private/
# ls -la
insgesamt 0
drwx------ 1 root root 0 9. Jan 2013 .
drwx------ 1 root root 0 17. Jul 06:58 ..
drwx------ 1 root root 0 20. Jul 12:16 101
drwx------ 1 root root 0 19. Jul 21:51 101.8208Ef
drwx------ 1 root root 0 19. Jul 13:37 101.u7euXi
drwx------ 1 root root 0 19. Jul 00:52 101.wnIEGR
drwx------ 1 root root 0 19. Jul 07:23 101.WzxCKi
drwx------ 1 root root 0 20. Jul 07:09 101.XYGNiD
drwx------ 1 root root 0 19. Jul 14:35 101.z0nrUn
When trying to delete these leftovers I get the "Operation not permitted" error message.
Cause
The *.hds files left from previous migration attempts are still mounted:
# grep '/101\.' /sys/block/ploop*/pdelta/*/image
/sys/block/ploop0/pdelta/0/image:/pstorage/cluster/private/101.wnIEGR/root.hdd/root.hds
/sys/block/ploop13021/pdelta/0/image:/pstorage/cluster/private/101.XYGNiD/root.hdd/root.hds
/sys/block/ploop17543/pdelta/0/image:/pstorage/cluster/private/101.z0nrUn/root.hdd/root.hds
/sys/block/ploop27629/pdelta/0/image:/pstorage/cluster/private/101.8208Ef_ren/root.hdd/root.hds
/sys/block/ploop53561/pdelta/0/image:/pstorage/cluster/private/101.u7euXi/root.hdd/root.hds
/sys/block/ploop54602/pdelta/0/image:/pstorage/cluster/private/101.WzxCKi/root.hdd/root.hds
It is not possible to release them as they all referring to the same "/vz/root/101"
# grep 101 /proc/mounts
/dev/ploop0p1 /vz/root/101 ext4 rw,relatime,barrier=1,data=ordered,balloon_ino=12,pfcache_csum,pfcache=/vz/pfcache,usrquota,grpquota 0 0
/dev/ploop54602p1 /vz/root/101 ext4 rw,relatime,barrier=1,data=ordered,balloon_ino=12,pfcache_csum,pfcache=/vz/pfcache,usrquota,grpquota 0 0
/dev/ploop53561p1 /vz/root/101 ext4 rw,relatime,barrier=1,data=ordered,balloon_ino=12,pfcache_csum,pfcache=/vz/pfcache,usrquota,grpquota 0 0
/dev/ploop17543p1 /vz/root/101 ext4 rw,relatime,barrier=1,data=ordered,balloon_ino=12,pfcache_csum,pfcache=/vz/pfcache,usrquota,grpquota 0 0
/dev/ploop27629p1 /vz/root/101 ext4 rw,relatime,barrier=1,data=ordered,balloon_ino=12,pfcache_csum,pfcache=/vz/pfcache,usrquota,grpquota 0 0
/dev/ploop13021p1 /vz/root/101 ext4 rw,relatime,barrier=1,data=ordered,balloon_ino=12,pfcache_csum,pfcache=/vz/pfcache,usrquota,grpquota 0 0
/dev/ploop16067p1 /vz/root/101 ext4 rw,relatime,barrier=1,data=ordered,balloon_ino=12,pfcache_csum,pfcache=/vz/pfcache,usrquota,grpquota 0 0
proc /vz/root/101/proc proc rw,relatime 0 0
sysfs /vz/root/101/sys sysfs rw,relatime 0 0
none /vz/root/101/dev tmpfs rw,relatime,size=5242880k,nr_inodes=1310720,mode=755 0 0
none /vz/root/101/dev/pts devpts rw,relatime,mode=600,ptmxmode=000 0 0
none /vz/root/101/proc/sys/fs/binfmt_misc binfmt_misc rw,relatime 0 0
proc /vz/root/101/var/named/run-root/proc proc rw,relatime 0 0
/dev/ploop16067p1 /vz/root/101/var/named/run-root/var/run/dbus ext4 rw,relatime,barrier=1,data=ordered,balloon_ino=12,pfcache_csum,pfcache=/vz/pfcache,usrquota,grpquota 0 0
Resolution
Stop the original container (101).
Get the list of ploops related to this cotnainer, e.g.:
# grep 101 /proc/mounts /dev/ploop0p1 /vz/root/101 ext4 rw,relatime,barrier=1,data=ordered,balloon_ino=12,pfcache_csum,pfcache=/vz/pfcache,usrquota,grpquota 0 0 /dev/ploop54602p1 /vz/root/101 ext4 rw,relatime,barrier=1,data=ordered,balloon_ino=12,pfcache_csum,pfcache=/vz/pfcache,usrquota,grpquota 0 0 /dev/ploop53561p1 /vz/root/101 ext4 rw,relatime,barrier=1,data=ordered,balloon_ino=12,pfcache_csum,pfcache=/vz/pfcache,usrquota,grpquota 0 0 /dev/ploop17543p1 /vz/root/101 ext4 rw,relatime,barrier=1,data=ordered,balloon_ino=12,pfcache_csum,pfcache=/vz/pfcache,usrquota,grpquota 0 0 /dev/ploop27629p1 /vz/root/101 ext4 rw,relatime,barrier=1,data=ordered,balloon_ino=12,pfcache_csum,pfcache=/vz/pfcache,usrquota,grpquota 0 0 /dev/ploop13021p1 /vz/root/101 ext4 rw,relatime,barrier=1,data=ordered,balloon_ino=12,pfcache_csum,pfcache=/vz/pfcache,usrquota,grpquota 0 0Unmount the directory under
"/vz/root/CT_ID":# umount /vz/root/101/In the example we have 6 ploop images mounted over this directory, so the command has to be repeated 6 times.
After that make sure nothing can be found in
/proc/mountsregarding the container:# grep 101 /proc/mounts #Unmount all ploop devices by names (listed in step 2 ). Example:
# ploop umount -d /dev/ploop0 Unmounting device /dev/ploop0 # ploop umount -d /dev/ploop54602 Unmounting device /dev/ploop54602 # ploop umount -d /dev/ploop53561 Unmounting device /dev/ploop53561 # ploop umount -d /dev/ploop17543 Unmounting device /dev/ploop17543 # ploop umount -d /dev/ploop27629 Unmounting device /dev/ploop27629 # ploop umount -d /dev/ploop13021 Unmounting device /dev/ploop13021
After that original container can be started and excessive directories can be removed.