Le 10/02/2012 11:45, Phil Daws a écrit : > Hello all, > > we are seeing a strange issue where one of our vserver guests is > reporting that its file system is full yet there is plenty of space: > > usera:/var/run# df -h > Filesystem Size Used Avail Use% Mounted on > /dev/hdv1 1008G 187G 812G 19% / > none 128M 0 128M 0% /tmp > usera:/var/run# touch testfile > touch: cannot touch `testfile': No space left on device > > Below is a strace -f when running the touch testfile command: > > usera:/var/run# strace -f touch testfile > execve("/usr/bin/touch", ["touch", "testfile"], [/* 28 vars */]) = 0 > brk(0) = 0x60d000 > access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or > directory) > mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, > 0) = 0x7f3429ad3000 > access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or > directory) > open("/etc/ld.so.cache", O_RDONLY) = 3 > fstat(3, {st_mode=S_IFREG|0644, st_size=153901, ...}) = 0 > mmap(NULL, 153901, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f3429aad000 > close(3) = 0 > access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or > directory) > open("/lib/librt.so.1", O_RDONLY) = 3 > read(3, > "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\220!\0\0\0\0\0\0@"..., > 832) = 832 > fstat(3, {st_mode=S_IFREG|0644, st_size=31744, ...}) = 0 > mmap(NULL, 2128848, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, > 0) = 0x7f34296b0000 > mprotect(0x7f34296b7000, 2093056, PROT_NONE) = 0 > mmap(0x7f34298b6000, 8192, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7f34298b6000 > close(3) = 0 > access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or > directory) > open("/lib/libc.so.6", O_RDONLY) = 3 > read(3, > "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`\355\1\0\0\0\0\0@"..., > 832) = 832 > fstat(3, {st_mode=S_IFREG|0755, st_size=1432968, ...}) = 0 > mmap(NULL, 3541032, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, > 0) = 0x7f342934f000 > mprotect(0x7f34294a7000, 2093056, PROT_NONE) = 0 > mmap(0x7f34296a6000, 20480, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x157000) = 0x7f34296a6000 > mmap(0x7f34296ab000, 18472, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f34296ab000 > close(3) = 0 > access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or > directory) > open("/lib/libpthread.so.0", O_RDONLY) = 3 > read(3, > "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340Y\0\0\0\0\0\0@"..., > 832) = 832 > fstat(3, {st_mode=S_IFREG|0755, st_size=131260, ...}) = 0 > mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, > 0) = 0x7f3429aac000 > mmap(NULL, 2208640, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, > 0) = 0x7f3429133000 > mprotect(0x7f342914a000, 2093056, PROT_NONE) = 0 > mmap(0x7f3429349000, 8192, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16000) = 0x7f3429349000 > mmap(0x7f342934b000, 13184, PROT_READ|PROT_WRITE, > MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f342934b000 > close(3) = 0 > mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, > 0) = 0x7f3429aab000 > mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, > 0) = 0x7f3429aaa000 > arch_prctl(ARCH_SET_FS, 0x7f3429aab700) = 0 > open("/dev/urandom", O_RDONLY) = 3 > read(3, "\270_f\37!\232;"..., 7) = 7 > close(3) = 0 > mprotect(0x7f3429349000, 4096, PROT_READ) = 0 > mprotect(0x7f34296a6000, 16384, PROT_READ) = 0 > mprotect(0x7f34298b6000, 4096, PROT_READ) = 0 > mprotect(0x7f3429ad5000, 4096, PROT_READ) = 0 > munmap(0x7f3429aad000, 153901) = 0 > set_tid_address(0x7f3429aab9d0) = 13524 > set_robust_list(0x7f3429aab9e0, 0x18) = 0 > futex(0x7fff31ad739c, FUTEX_WAKE_PRIVATE, 1) = 0 > futex(0x7fff31ad739c, 0x189 /* FUTEX_??? */, 1, NULL, 7f3429aab700) = > -1 ENOSYS (Function not implemented) > rt_sigaction(SIGRTMIN, {0x7f3429138860, [], SA_RESTORER|SA_SIGINFO, > 0x7f3429141f60}, NULL, 8) = 0 > rt_sigaction(SIGRT_1, {0x7f34291388f0, [], > SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x7f3429141f60}, NULL, 8) = 0 > rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0 > getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) = 0 > brk(0) = 0x60d000 > brk(0x62e000) = 0x62e000 > open("/usr/lib/locale/locale-archive", O_RDONLY) = 3 > fstat(3, {st_mode=S_IFREG|0644, st_size=85880592, ...}) = 0 > mmap(NULL, 85880592, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f3423f4c000 > close(3) = 0 > open("testfile", O_WRONLY|O_CREAT|O_NOCTTY|O_NONBLOCK, 0666) = -1 > ENOSPC (No space left on device) > futimesat(AT_FDCWD, "testfile", NULL) = -1 ENOENT (No such file or > directory) > open("/usr/share/locale/locale.alias", O_RDONLY) = 3 > fstat(3, {st_mode=S_IFREG|0644, st_size=2570, ...}) = 0 > mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, > 0) = 0x7f3429ad2000 > read(3, "# Locale name alias data base.\n# "..., 4096) = 2570 > read(3, ""..., 4096) = 0 > close(3) = 0 > munmap(0x7f3429ad2000, 4096) = 0 > open("/usr/share/locale/en_US.UTF-8/LC_MESSAGES/coreutils.mo", > O_RDONLY) = -1 ENOENT (No such file or directory) > open("/usr/share/locale/en_US.utf8/LC_MESSAGES/coreutils.mo", > O_RDONLY) = -1 ENOENT (No such file or directory) > open("/usr/share/locale/en_US/LC_MESSAGES/coreutils.mo", O_RDONLY) = > -1 ENOENT (No such file or directory) > open("/usr/share/locale/en.UTF-8/LC_MESSAGES/coreutils.mo", O_RDONLY) > = -1 ENOENT (No such file or directory) > open("/usr/share/locale/en.utf8/LC_MESSAGES/coreutils.mo", O_RDONLY) = > -1 ENOENT (No such file or directory) > open("/usr/share/locale/en/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 > ENOENT (No such file or directory) > write(2, "touch: "..., 7touch: ) = 7 > write(2, "cannot touch `testfile'"..., 23cannot touch `testfile') = 23 > open("/usr/share/locale/en_US.UTF-8/LC_MESSAGES/libc.mo", O_RDONLY) = > -1 ENOENT (No such file or directory) > open("/usr/share/locale/en_US.utf8/LC_MESSAGES/libc.mo", O_RDONLY) = > -1 ENOENT (No such file or directory) > open("/usr/share/locale/en_US/LC_MESSAGES/libc.mo", O_RDONLY) = -1 > ENOENT (No such file or directory) > open("/usr/share/locale/en.UTF-8/LC_MESSAGES/libc.mo", O_RDONLY) = -1 > ENOENT (No such file or directory) > open("/usr/share/locale/en.utf8/LC_MESSAGES/libc.mo", O_RDONLY) = -1 > ENOENT (No such file or directory) > open("/usr/share/locale/en/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT > (No such file or directory) > write(2, ": No space left on device"..., 25: No space left on device) = 25 > write(2, "\n"..., 1 > ) = 1 > close(1) = 0 > close(2) = 0 > exit_group(1) = ? > > Any thoughts as I have checked the host system and that looks > absolutely fine; and so are the other vservers. Weird! :) > -- > Thanks, Phil > Hello, It might be the inodes limit that you reached, use df -i to check it. Regards.
Hello,Hello all,
we are seeing a strange issue where one of our vserver guests is reporting that its file system is full yet there is plenty of space:
usera:/var/run# df -hFilesystem Size Used Avail Use% Mounted on/dev/hdv1 1008G 187G 812G 19% /none 128M 0 128M 0% /tmpusera:/var/run# touch testfiletouch: cannot touch `testfile': No space left on device
Below is a strace -f when running the touch testfile command:
usera:/var/run# strace -f touch testfileexecve("/usr/bin/touch", ["touch", "testfile"], [/* 28 vars */]) = 0brk(0) = 0x60d000access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3429ad3000access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)open("/etc/ld.so.cache", O_RDONLY) = 3fstat(3, {st_mode=S_IFREG|0644, st_size=153901, ...}) = 0mmap(NULL, 153901, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f3429aad000close(3) = 0access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)open("/lib/librt.so.1", O_RDONLY) = 3read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\220!\0\0\0\0\0\0@"..., 832) = 832fstat(3, {st_mode=S_IFREG|0644, st_size=31744, ...}) = 0mmap(NULL, 2128848, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f34296b0000mprotect(0x7f34296b7000, 2093056, PROT_NONE) = 0mmap(0x7f34298b6000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x6000) = 0x7f34298b6000close(3) = 0access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)open("/lib/libc.so.6", O_RDONLY) = 3read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0`\355\1\0\0\0\0\0@"..., 832) = 832fstat(3, {st_mode=S_IFREG|0755, st_size=1432968, ...}) = 0mmap(NULL, 3541032, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f342934f000mprotect(0x7f34294a7000, 2093056, PROT_NONE) = 0mmap(0x7f34296a6000, 20480, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x157000) = 0x7f34296a6000mmap(0x7f34296ab000, 18472, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f34296ab000close(3) = 0access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)open("/lib/libpthread.so.0", O_RDONLY) = 3read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\340Y\0\0\0\0\0\0@"..., 832) = 832fstat(3, {st_mode=S_IFREG|0755, st_size=131260, ...}) = 0mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3429aac000mmap(NULL, 2208640, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7f3429133000mprotect(0x7f342914a000, 2093056, PROT_NONE) = 0mmap(0x7f3429349000, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x16000) = 0x7f3429349000mmap(0x7f342934b000, 13184, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f342934b000close(3) = 0mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3429aab000mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3429aaa000arch_prctl(ARCH_SET_FS, 0x7f3429aab700) = 0open("/dev/urandom", O_RDONLY) = 3read(3, "\270_f\37!\232;"..., 7) = 7close(3) = 0mprotect(0x7f3429349000, 4096, PROT_READ) = 0mprotect(0x7f34296a6000, 16384, PROT_READ) = 0mprotect(0x7f34298b6000, 4096, PROT_READ) = 0mprotect(0x7f3429ad5000, 4096, PROT_READ) = 0munmap(0x7f3429aad000, 153901) = 0set_tid_address(0x7f3429aab9d0) = 13524set_robust_list(0x7f3429aab9e0, 0x18) = 0futex(0x7fff31ad739c, FUTEX_WAKE_PRIVATE, 1) = 0futex(0x7fff31ad739c, 0x189 /* FUTEX_??? */, 1, NULL, 7f3429aab700) = -1 ENOSYS (Function not implemented)rt_sigaction(SIGRTMIN, {0x7f3429138860, [], SA_RESTORER|SA_SIGINFO, 0x7f3429141f60}, NULL, 8) = 0rt_sigaction(SIGRT_1, {0x7f34291388f0, [], SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x7f3429141f60}, NULL, 8) = 0rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) = 0brk(0) = 0x60d000brk(0x62e000) = 0x62e000open("/usr/lib/locale/locale-archive", O_RDONLY) = 3fstat(3, {st_mode=S_IFREG|0644, st_size=85880592, ...}) = 0mmap(NULL, 85880592, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f3423f4c000close(3) = 0open("testfile", O_WRONLY|O_CREAT|O_NOCTTY|O_NONBLOCK, 0666) = -1 ENOSPC (No space left on device)futimesat(AT_FDCWD, "testfile", NULL) = -1 ENOENT (No such file or directory)open("/usr/share/locale/locale.alias", O_RDONLY) = 3fstat(3, {st_mode=S_IFREG|0644, st_size=2570, ...}) = 0mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7f3429ad2000read(3, "# Locale name alias data base.\n# "..., 4096) = 2570read(3, ""..., 4096) = 0close(3) = 0munmap(0x7f3429ad2000, 4096) = 0open("/usr/share/locale/en_US.UTF-8/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)open("/usr/share/locale/en_US.utf8/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)open("/usr/share/locale/en_US/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)open("/usr/share/locale/en.UTF-8/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)open("/usr/share/locale/en.utf8/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)open("/usr/share/locale/en/LC_MESSAGES/coreutils.mo", O_RDONLY) = -1 ENOENT (No such file or directory)write(2, "touch: "..., 7touch: ) = 7write(2, "cannot touch `testfile'"..., 23cannot touch `testfile') = 23open("/usr/share/locale/en_US.UTF-8/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)open("/usr/share/locale/en_US.utf8/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)open("/usr/share/locale/en_US/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)open("/usr/share/locale/en.UTF-8/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)open("/usr/share/locale/en.utf8/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)open("/usr/share/locale/en/LC_MESSAGES/libc.mo", O_RDONLY) = -1 ENOENT (No such file or directory)write(2, ": No space left on device"..., 25: No space left on device) = 25write(2, "\n"..., 1) = 1close(1) = 0close(2) = 0exit_group(1) = ?
Any thoughts as I have checked the host system and that looks absolutely fine; and so are the other vservers. Weird! :)--
Thanks, Phil