Filesystem Inodes IUsed IFree IUse% Mounted on /dev/hdv1 67108864 3305140 63803724 5% / Have checked the host and that is fine as-well. -- Thanks, Phil ----- Original Message ----- > Le 10/02/2012 11:45, Phil Daws a écrit : > > Hello all, > > > we are seeing a strange issue where one of our vserver guests is > > reporting that its file system is full yet there is plenty of > > space: > > > usera:/var/run# df -h > > > Filesystem Size Used Avail Use% Mounted on > > > /dev/hdv1 1008G 187G 812G 19% / > > > none 128M 0 128M 0% /tmp > > > usera:/var/run# touch testfile > > > touch: cannot touch `testfile': No space left on device > > > Below is a strace -f when running the touch testfile command: > > > usera:/var/run# strace -f touch testfile > > > execve("/usr/bin/touch", ["touch", "testfile"], [/* 28 vars */]) = > > 0 > > > brk(0) = 0x60d000 > > > access("/etc/ld.so.nohwcap", F OK) = -1 ENOENT (No such file or > > directory) > > > mmap(NULL, 8192, PROT READ|PROT WRITE, MAP PRIVATE|MAP ANONYMOUS, > > -1, > > 0) = 0x7f3429ad3000 > > > access("/etc/ld.so.preload", R OK) = -1 ENOENT (No such file or > > directory) > > > open("/etc/ld.so.cache", O RDONLY) = 3 > > > fstat(3, {st mode=S IFREG|0644, st size=153901, ...}) = 0 > > > mmap(NULL, 153901, PROT READ, MAP PRIVATE, 3, 0) = 0x7f3429aad000 > > > close(3) = 0 > > > access("/etc/ld.so.nohwcap", F OK) = -1 ENOENT (No such file or > > directory) > > > open("/lib/librt.so.1", O RDONLY) = 3 > > > read(3, > > "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\220!\0\0\0\0\0\0@"..., > > 832) = 832 > > > fstat(3, {st mode=S IFREG|0644, st size=31744, ...}) = 0 > > > mmap(NULL, 2128848, PROT READ|PROT EXEC, MAP PRIVATE|MAP DENYWRITE, > > 3, 0) = 0x7f34296b0000 > > > mprotect(0x7f34296b7000, 2093056, PROT NONE) = 0 > > > mmap(0x7f34298b6000, 8192, PROT READ|PROT WRITE, > > MAP PRIVATE|MAP FIXED|MAP DENYWRITE, 3, 0x6000) = 0x7f34298b6000 > > > close(3) = 0 > > > access("/etc/ld.so.nohwcap", F OK) = -1 ENOENT (No such file or > > directory) > > > open("/lib/libc.so.6", O RDONLY) = 3 > > > read(3, > > "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`\355\1\0\0\0\0\0@"..., > > 832) = 832 > > > fstat(3, {st mode=S IFREG|0755, st size=1432968, ...}) = 0 > > > mmap(NULL, 3541032, PROT READ|PROT EXEC, MAP PRIVATE|MAP DENYWRITE, > > 3, 0) = 0x7f342934f000 > > > mprotect(0x7f34294a7000, 2093056, PROT NONE) = 0 > > > mmap(0x7f34296a6000, 20480, PROT READ|PROT WRITE, > > MAP PRIVATE|MAP FIXED|MAP DENYWRITE, 3, 0x157000) = 0x7f34296a6000 > > > mmap(0x7f34296ab000, 18472, PROT READ|PROT WRITE, > > MAP PRIVATE|MAP FIXED|MAP ANONYMOUS, -1, 0) = 0x7f34296ab000 > > > close(3) = 0 > > > access("/etc/ld.so.nohwcap", F OK) = -1 ENOENT (No such file or > > directory) > > > open("/lib/libpthread.so.0", O RDONLY) = 3 > > > read(3, > > "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340Y\0\0\0\0\0\0@"..., > > 832) = 832 > > > fstat(3, {st mode=S IFREG|0755, st size=131260, ...}) = 0 > > > mmap(NULL, 4096, PROT READ|PROT WRITE, MAP PRIVATE|MAP ANONYMOUS, > > -1, > > 0) = 0x7f3429aac000 > > > mmap(NULL, 2208640, PROT READ|PROT EXEC, MAP PRIVATE|MAP DENYWRITE, > > 3, 0) = 0x7f3429133000 > > > mprotect(0x7f342914a000, 2093056, PROT NONE) = 0 > > > mmap(0x7f3429349000, 8192, PROT READ|PROT WRITE, > > MAP PRIVATE|MAP FIXED|MAP DENYWRITE, 3, 0x16000) = 0x7f3429349000 > > > mmap(0x7f342934b000, 13184, PROT READ|PROT WRITE, > > MAP PRIVATE|MAP FIXED|MAP ANONYMOUS, -1, 0) = 0x7f342934b000 > > > close(3) = 0 > > > mmap(NULL, 4096, PROT READ|PROT WRITE, MAP PRIVATE|MAP ANONYMOUS, > > -1, > > 0) = 0x7f3429aab000 > > > mmap(NULL, 4096, PROT READ|PROT WRITE, MAP PRIVATE|MAP ANONYMOUS, > > -1, > > 0) = 0x7f3429aaa000 > > > arch prctl(ARCH SET FS, 0x7f3429aab700) = 0 > > > open("/dev/urandom", O RDONLY) = 3 > > > read(3, "\270 f\37!\232;"..., 7) = 7 > > > close(3) = 0 > > > mprotect(0x7f3429349000, 4096, PROT READ) = 0 > > > mprotect(0x7f34296a6000, 16384, PROT READ) = 0 > > > mprotect(0x7f34298b6000, 4096, PROT READ) = 0 > > > mprotect(0x7f3429ad5000, 4096, PROT READ) = 0 > > > munmap(0x7f3429aad000, 153901) = 0 > > > set tid address(0x7f3429aab9d0) = 13524 > > > set robust list(0x7f3429aab9e0, 0x18) = 0 > > > futex(0x7fff31ad739c, FUTEX WAKE PRIVATE, 1) = 0 > > > futex(0x7fff31ad739c, 0x189 /* FUTEX ??? */, 1, NULL, 7f3429aab700) > > = > > -1 ENOSYS (Function not implemented) > > > rt sigaction(SIGRTMIN, {0x7f3429138860, [], SA RESTORER|SA SIGINFO, > > 0x7f3429141f60}, NULL, 8) = 0 > > > rt sigaction(SIGRT 1, {0x7f34291388f0, [], > > SA RESTORER|SA RESTART|SA SIGINFO, 0x7f3429141f60}, NULL, 8) = 0 > > > rt sigprocmask(SIG UNBLOCK, [RTMIN RT 1], NULL, 8) = 0 > > > getrlimit(RLIMIT STACK, {rlim cur=8192*1024, > > rlim max=RLIM INFINITY}) > > = 0 > > > brk(0) = 0x60d000 > > > brk(0x62e000) = 0x62e000 > > > open("/usr/lib/locale/locale-archive", O RDONLY) = 3 > > > fstat(3, {st mode=S IFREG|0644, st size=85880592, ...}) = 0 > > > mmap(NULL, 85880592, PROT READ, MAP PRIVATE, 3, 0) = 0x7f3423f4c000 > > > close(3) = 0 > > > open("testfile", O WRONLY|O CREAT|O NOCTTY|O NONBLOCK, 0666) = -1 > > ENOSPC (No space left on device) > > > futimesat(AT FDCWD, "testfile", NULL) = -1 ENOENT (No such file or > > directory) > > > open("/usr/share/locale/locale.alias", O RDONLY) = 3 > > > fstat(3, {st mode=S IFREG|0644, st size=2570, ...}) = 0 > > > mmap(NULL, 4096, PROT READ|PROT WRITE, MAP PRIVATE|MAP ANONYMOUS, > > -1, > > 0) = 0x7f3429ad2000 > > > read(3, "# Locale name alias data base.\n# "..., 4096) = 2570 > > > read(3, ""..., 4096) = 0 > > > close(3) = 0 > > > munmap(0x7f3429ad2000, 4096) = 0 > > > open("/usr/share/locale/en US.UTF-8/LC MESSAGES/coreutils.mo", > > O RDONLY) = -1 ENOENT (No such file or directory) > > > open("/usr/share/locale/en US.utf8/LC MESSAGES/coreutils.mo", > > O RDONLY) = -1 ENOENT (No such file or directory) > > > open("/usr/share/locale/en US/LC MESSAGES/coreutils.mo", O RDONLY) > > = > > -1 ENOENT (No such file or directory) > > > open("/usr/share/locale/en.UTF-8/LC MESSAGES/coreutils.mo", > > O RDONLY) > > = -1 ENOENT (No such file or directory) > > > open("/usr/share/locale/en.utf8/LC MESSAGES/coreutils.mo", > > O RDONLY) > > = -1 ENOENT (No such file or directory) > > > open("/usr/share/locale/en/LC MESSAGES/coreutils.mo", O RDONLY) = > > -1 > > ENOENT (No such file or directory) > > > write(2, "touch: "..., 7touch: ) = 7 > > > write(2, "cannot touch `testfile'"..., 23cannot touch `testfile') = > > 23 > > > open("/usr/share/locale/en US.UTF-8/LC MESSAGES/libc.mo", O RDONLY) > > = > > -1 ENOENT (No such file or directory) > > > open("/usr/share/locale/en US.utf8/LC MESSAGES/libc.mo", O RDONLY) > > = > > -1 ENOENT (No such file or directory) > > > open("/usr/share/locale/en US/LC MESSAGES/libc.mo", O RDONLY) = -1 > > ENOENT (No such file or directory) > > > open("/usr/share/locale/en.UTF-8/LC MESSAGES/libc.mo", O RDONLY) = > > -1 > > ENOENT (No such file or directory) > > > open("/usr/share/locale/en.utf8/LC MESSAGES/libc.mo", O RDONLY) = > > -1 > > ENOENT (No such file or directory) > > > open("/usr/share/locale/en/LC MESSAGES/libc.mo", O RDONLY) = -1 > > ENOENT (No such file or directory) > > > write(2, ": No space left on device"..., 25: No space left on > > device) > > = 25 > > > write(2, "\n"..., 1 > > > ) = 1 > > > close(1) = 0 > > > close(2) = 0 > > > exit group(1) = ? > > > Any thoughts as I have checked the host system and that looks > > absolutely fine; and so are the other vservers. Weird! :) > > > -- > > > Thanks, Phil > > Hello, > It might be the inodes limit that you reached, use df -i to check it. > Regards.
Le 10/02/2012 11:45, Phil Daws a écrit :Hello,Hello all,
we are seeing a strange issue where one of our vserver guests is reporting that its file system is full yet there is plenty of space:
usera:/var/run# df -hFilesystem Size Used Avail Use% Mounted on/dev/hdv1 1008G 187G 812G 19% /none 128M 0 128M 0% /tmpusera:/var/run# touch testfiletouch: cannot touch `testfile': No space left on device
Below is a strace -f when running the touch testfile command:
usera:/var/run# strace -f touch testfileexecve("/usr/bin/touch", ["touch", "testfile"], [/* 28 vars */]) = 0brk(0) = 0x60d000access("/etc/ld.so.nohwcap", F OK) = -1 ENOENT (No such file or directory)mmap(NULL, 8192, PROT READ|PROT WRITE, MAP PRIVATE|MAP ANONYMOUS, -1, 0) = 0x7f3429ad3000access("/etc/ld.so.preload", R OK) = -1 ENOENT (No such file or directory)open("/etc/ld.so.cache", O RDONLY) = 3fstat(3, {st mode=S IFREG|0644, st size=153901, ...}) = 0mmap(NULL, 153901, PROT READ, MAP PRIVATE, 3, 0) = 0x7f3429aad000close(3) = 0access("/etc/ld.so.nohwcap", F OK) = -1 ENOENT (No such file or directory)open("/lib/librt.so.1", O RDONLY) = 3read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\220!\0\0\0\0\0\0@"..., 832) = 832fstat(3, {st mode=S IFREG|0644, st size=31744, ...}) = 0mmap(NULL, 2128848, PROT READ|PROT EXEC, MAP PRIVATE|MAP DENYWRITE, 3, 0) = 0x7f34296b0000mprotect(0x7f34296b7000, 2093056, PROT NONE) = 0mmap(0x7f34298b6000, 8192, PROT READ|PROT WRITE, MAP PRIVATE|MAP FIXED|MAP DENYWRITE, 3, 0x6000) = 0x7f34298b6000close(3) = 0access("/etc/ld.so.nohwcap", F OK) = -1 ENOENT (No such file or directory)open("/lib/libc.so.6", O RDONLY) = 3read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`\355\1\0\0\0\0\0@"..., 832) = 832fstat(3, {st mode=S IFREG|0755, st size=1432968, ...}) = 0mmap(NULL, 3541032, PROT READ|PROT EXEC, MAP PRIVATE|MAP DENYWRITE, 3, 0) = 0x7f342934f000mprotect(0x7f34294a7000, 2093056, PROT NONE) = 0mmap(0x7f34296a6000, 20480, PROT READ|PROT WRITE, MAP PRIVATE|MAP FIXED|MAP DENYWRITE, 3, 0x157000) = 0x7f34296a6000mmap(0x7f34296ab000, 18472, PROT READ|PROT WRITE, MAP PRIVATE|MAP FIXED|MAP ANONYMOUS, -1, 0) = 0x7f34296ab000close(3) = 0access("/etc/ld.so.nohwcap", F OK) = -1 ENOENT (No such file or directory)open("/lib/libpthread.so.0", O RDONLY) = 3read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340Y\0\0\0\0\0\0@"..., 832) = 832fstat(3, {st mode=S IFREG|0755, st size=131260, ...}) = 0mmap(NULL, 4096, PROT READ|PROT WRITE, MAP PRIVATE|MAP ANONYMOUS, -1, 0) = 0x7f3429aac000mmap(NULL, 2208640, PROT READ|PROT EXEC, MAP PRIVATE|MAP DENYWRITE, 3, 0) = 0x7f3429133000mprotect(0x7f342914a000, 2093056, PROT NONE) = 0mmap(0x7f3429349000, 8192, PROT READ|PROT WRITE, MAP PRIVATE|MAP FIXED|MAP DENYWRITE, 3, 0x16000) = 0x7f3429349000mmap(0x7f342934b000, 13184, PROT READ|PROT WRITE, MAP PRIVATE|MAP FIXED|MAP ANONYMOUS, -1, 0) = 0x7f342934b000close(3) = 0mmap(NULL, 4096, PROT READ|PROT WRITE, MAP PRIVATE|MAP ANONYMOUS, -1, 0) = 0x7f3429aab000mmap(NULL, 4096, PROT READ|PROT WRITE, MAP PRIVATE|MAP ANONYMOUS, -1, 0) = 0x7f3429aaa000arch prctl(ARCH SET FS, 0x7f3429aab700) = 0open("/dev/urandom", O RDONLY) = 3read(3, "\270 f\37!\232;"..., 7) = 7close(3) = 0mprotect(0x7f3429349000, 4096, PROT READ) = 0mprotect(0x7f34296a6000, 16384, PROT READ) = 0mprotect(0x7f34298b6000, 4096, PROT READ) = 0mprotect(0x7f3429ad5000, 4096, PROT READ) = 0munmap(0x7f3429aad000, 153901) = 0set tid address(0x7f3429aab9d0) = 13524set robust list(0x7f3429aab9e0, 0x18) = 0futex(0x7fff31ad739c, FUTEX WAKE PRIVATE, 1) = 0futex(0x7fff31ad739c, 0x189 /* FUTEX ??? */, 1, NULL, 7f3429aab700) = -1 ENOSYS (Function not implemented)rt sigaction(SIGRTMIN, {0x7f3429138860, [], SA RESTORER|SA SIGINFO, 0x7f3429141f60}, NULL, 8) = 0rt sigaction(SIGRT 1, {0x7f34291388f0, [], SA RESTORER|SA RESTART|SA SIGINFO, 0x7f3429141f60}, NULL, 8) = 0rt sigprocmask(SIG UNBLOCK, [RTMIN RT 1], NULL, 8) = 0getrlimit(RLIMIT STACK, {rlim cur=8192*1024, rlim max=RLIM INFINITY}) = 0brk(0) = 0x60d000brk(0x62e000) = 0x62e000open("/usr/lib/locale/locale-archive", O RDONLY) = 3fstat(3, {st mode=S IFREG|0644, st size=85880592, ...}) = 0mmap(NULL, 85880592, PROT READ, MAP PRIVATE, 3, 0) = 0x7f3423f4c000close(3) = 0open("testfile", O WRONLY|O CREAT|O NOCTTY|O NONBLOCK, 0666) = -1 ENOSPC (No space left on device)futimesat(AT FDCWD, "testfile", NULL) = -1 ENOENT (No such file or directory)open("/usr/share/locale/locale.alias", O RDONLY) = 3fstat(3, {st mode=S IFREG|0644, st size=2570, ...}) = 0mmap(NULL, 4096, PROT READ|PROT WRITE, MAP PRIVATE|MAP ANONYMOUS, -1, 0) = 0x7f3429ad2000read(3, "# Locale name alias data base.\n# "..., 4096) = 2570read(3, ""..., 4096) = 0close(3) = 0munmap(0x7f3429ad2000, 4096) = 0open("/usr/share/locale/en US.UTF-8/LC MESSAGES/coreutils.mo", O RDONLY) = -1 ENOENT (No such file or directory)open("/usr/share/locale/en US.utf8/LC MESSAGES/coreutils.mo", O RDONLY) = -1 ENOENT (No such file or directory)open("/usr/share/locale/en US/LC MESSAGES/coreutils.mo", O RDONLY) = -1 ENOENT (No such file or directory)open("/usr/share/locale/en.UTF-8/LC MESSAGES/coreutils.mo", O RDONLY) = -1 ENOENT (No such file or directory)open("/usr/share/locale/en.utf8/LC MESSAGES/coreutils.mo", O RDONLY) = -1 ENOENT (No such file or directory)open("/usr/share/locale/en/LC MESSAGES/coreutils.mo", O RDONLY) = -1 ENOENT (No such file or directory)write(2, "touch: "..., 7touch: ) = 7write(2, "cannot touch `testfile'"..., 23cannot touch `testfile') = 23open("/usr/share/locale/en US.UTF-8/LC MESSAGES/libc.mo", O RDONLY) = -1 ENOENT (No such file or directory)open("/usr/share/locale/en US.utf8/LC MESSAGES/libc.mo", O RDONLY) = -1 ENOENT (No such file or directory)open("/usr/share/locale/en US/LC MESSAGES/libc.mo", O RDONLY) = -1 ENOENT (No such file or directory)open("/usr/share/locale/en.UTF-8/LC MESSAGES/libc.mo", O RDONLY) = -1 ENOENT (No such file or directory)open("/usr/share/locale/en.utf8/LC MESSAGES/libc.mo", O RDONLY) = -1 ENOENT (No such file or directory)open("/usr/share/locale/en/LC MESSAGES/libc.mo", O RDONLY) = -1 ENOENT (No such file or directory)write(2, ": No space left on device"..., 25: No space left on device) = 25write(2, "\n"..., 1) = 1close(1) = 0close(2) = 0exit group(1) = ?
Any thoughts as I have checked the host system and that looks absolutely fine; and so are the other vservers. Weird! :)--
Thanks, Phil
It might be the inodes limit that you reached, use df -i to check it.
Regards.