Just upgraded from 6.1.3 to 6.2.2. Now Splunk keep filling up /run/shm with mongoc-XXXX files. Once full Splunk dies an start spewing crash logs.
root@server:/run/shm# ls | head -n 10
mongoc-1000
mongoc-10005
mongoc-10006
mongoc-10017
mongoc-10022
mongoc-10023
mongoc-10029
mongoc-10032
mongoc-10038
mongoc-10039
root@server:/run/shm# ls | wc
6459 6459 81954
root@server:/run/shm# df -h
Filesystem Size Used Avail Use% Mounted on
...
tmpfs 50M 50M 0 100% /run
...
Any ideas on how I stop Splunk from doing this? Or do I need to fall back to 6.1.3? Crash log below:
root@server:/opt/splunk/var/log/splunk# cat crash-2015-03-20-11:21:07.log
[build 255606] 2015-03-20 11:21:07
Received fatal signal 7 (Bus error).
Cause:
Non-existent accessed physical address at address [0x00007FECBF28E000].
Crashing thread: Main Thread
Registers:
RIP: [0x00007FECBE01FF0B] ? (/lib/x86_64-linux-gnu/libc.so.6)
RDI: [0x00007FECBF28E000]
RSI: [0x0000000000000000]
RBP: [0x0000000000001280]
RSP: [0x00007FFF705CE788]
RAX: [0x00007FECBF28E000]
RBX: [0x00007FECBF28E000]
RCX: [0x0000000000000000]
RDX: [0x0000000000000000]
R8: [0x0000000000001200]
R9: [0x0000000000078000]
R10: [0x0000000000000000]
R11: [0x00007FECBE01FA0A]
R12: [0x0000000000000007]
R13: [0x00007FECBD438E68]
R14: [0x00007FECBD438D00]
R15: [0x00007FECBD438D00]
EFL: [0x0000000000010202]
TRAPNO: [0x000000000000000E]
ERR: [0x0000000000000006]
CSGSFS: [0x0000000000000033]
OLDMASK: [0x0000000000000000]
OS: Linux
Arch: x86-64
Backtrace:
[0x00007FECBE01FF0B] ? (/lib/x86_64-linux-gnu/libc.so.6)
[0x00007FECBF2C764C] ? (/opt/splunk/lib/libmongoc-1.0.so.0)
[0x00007FECBE3348A0] pthread_once + 80 (/lib/x86_64-linux-gnu/libpthread.so.0)
[0x00000000013C0E4A] _ZN19MongoConnectionPoolC2Ev + 378 ([splunkd)
[0x00000000013C0FE1] _ZN19MongoConnectionPool14_new_singletonEv + 65 ([splunkd)
[0x00000000013AB8DF] _ZN31StateStoreConfigurationProvider26deserializeSettingsAndInitESt3mapI3StrS1_St4lessIS1_ESaISt4pairIKS1_S1_EEE + 1663 ([splunkd)
[0x00000000011DA8AE] _Z22do_search_process_impliPKPKcP12BundlesSetupb + 4926 ([splunkd)
[0x000000000136BF13] ? ([splunkd)
[0x0000000000F16A52] _ZN32ProcessRunnerChildCommandHandler19handle_command_forkEPK18proto_fork_commandPKv + 834 ([splunkd)
[0x0000000000F16BA3] _ZN32ProcessRunnerChildCommandHandler11consumeDataERK3Str + 195 ([splunkd)
[0x0000000000EB1595] _ZN21SocketCommandConsumer13when_readableE18PollableDescriptor + 181 ([splunkd)
[0x0000000000EB2CD4] _ZN12PolledSocket3Pfd11when_eventsE18PollableDescriptor + 132 ([splunkd)
[0x0000000000EB3100] _ZN8PolledFd8do_eventEv + 128 ([splunkd)
[0x0000000000EB3DF7] _ZN9EventLoop3runEv + 1063 ([splunkd)
[0x0000000000F153AA] _ZN20ExternalProcessGroup20process_runner_childERK8PathnameR14PolledReadPipe + 858 ([splunkd)
[0x0000000000F1581C] _Z17ProcessRunnerInitRK8Pathname + 636 ([splunkd)
[0x0000000000C9D508] _ZN10MainThreadC1ERK8Pathnameb + 648 ([splunkd)
[0x00000000008F3FEA] main + 9898 ([splunkd)
[0x00007FECBDFBBEAD] __libc_start_main + 253 (/lib/x86_64-linux-gnu/libc.so.6)
[0x00000000008EABBA] ? ([splunkd)
Linux / batbuilds.com / 3.2.0-4-amd64 / #1 SMP Debian 3.2.65-1+deb7u1 / x86_64
/etc/debian_version: 7.8
MAP: 00400000-01bba000 r-xp 00000000 fe:01 918892 /opt/splunk/bin/splunkd
MAP: 01bba000-01bc6000 rw-p 017b9000 fe:01 918892 /opt/splunk/bin/splunkd
MAP: 01bc6000-01c04000 rw-p 00000000 00:00 0
MAP: 7fecbd400000-7fecbdc00000 rw-p 00000000 00:00 0
MAP: 7fecbdd87000-7fecbdd9c000 r-xp 00000000 fe:01 131581 /lib/x86_64-linux-gnu/libgcc_s.so.1
MAP: 7fecbdd9c000-7fecbdf9c000 ---p 00015000 fe:01 131581 /lib/x86_64-linux-gnu/libgcc_s.so.1
MAP: 7fecbdf9c000-7fecbdf9d000 rw-p 00015000 fe:01 131581 /lib/x86_64-linux-gnu/libgcc_s.so.1
MAP: 7fecbdf9d000-7fecbe11e000 r-xp 00000000 fe:01 131628 /lib/x86_64-linux-gnu/libc-2.13.so
MAP: 7fecbe11e000-7fecbe31e000 ---p 00181000 fe:01 131628 /lib/x86_64-linux-gnu/libc-2.13.so
MAP: 7fecbe31e000-7fecbe322000 r--p 00181000 fe:01 131628 /lib/x86_64-linux-gnu/libc-2.13.so
MAP: 7fecbe322000-7fecbe323000 rw-p 00185000 fe:01 131628 /lib/x86_64-linux-gnu/libc-2.13.so
MAP: 7fecbe323000-7fecbe328000 rw-p 00000000 00:00 0
MAP: 7fecbe328000-7fecbe33f000 r-xp 00000000 fe:01 131649 /lib/x86_64-linux-gnu/libpthread-2.13.so
MAP: 7fecbe33f000-7fecbe53e000 ---p 00017000 fe:01 131649 /lib/x86_64-linux-gnu/libpthread-2.13.so
MAP: 7fecbe53e000-7fecbe53f000 r--p 00016000 fe:01 131649 /lib/x86_64-linux-gnu/libpthread-2.13.so
MAP: 7fecbe53f000-7fecbe540000 rw-p 00017000 fe:01 131649 /lib/x86_64-linux-gnu/libpthread-2.13.so
MAP: 7fecbe540000-7fecbe544000 rw-p 00000000 00:00 0
MAP: 7fecbe544000-7fecbe5c5000 r-xp 00000000 fe:01 131445 /lib/x86_64-linux-gnu/libm-2.13.so
MAP: 7fecbe5c5000-7fecbe7c4000 ---p 00081000 fe:01 131445 /lib/x86_64-linux-gnu/libm-2.13.so
MAP: 7fecbe7c4000-7fecbe7c5000 r--p 00080000 fe:01 131445 /lib/x86_64-linux-gnu/libm-2.13.so
MAP: 7fecbe7c5000-7fecbe7c6000 rw-p 00081000 fe:01 131445 /lib/x86_64-linux-gnu/libm-2.13.so
MAP: 7fecbe7c6000-7fecbe852000 r-xp 00000000 fe:01 796538 /opt/splunk/lib/libsqlite3.so.0.8.6
MAP: 7fecbe852000-7fecbe855000 rw-p 0008b000 fe:01 796538 /opt/splunk/lib/libsqlite3.so.0.8.6
MAP: 7fecbe855000-7fecbe8e4000 r-xp 00000000 fe:01 789068 /opt/splunk/lib/libarchive.so.13.1.2
MAP: 7fecbe8e4000-7fecbe8e8000 rw-p 0008e000 fe:01 789068 /opt/splunk/lib/libarchive.so.13.1.2
MAP: 7fecbe8e8000-7fecbe8e9000 rw-p 00000000 00:00 0
MAP: 7fecbe8e9000-7fecbe8eb000 r-xp 00000000 fe:01 131639 /lib/x86_64-linux-gnu/libdl-2.13.so
MAP: 7fecbe8eb000-7fecbeaeb000 ---p 00002000 fe:01 131639 /lib/x86_64-linux-gnu/libdl-2.13.so
MAP: 7fecbeaeb000-7fecbeaec000 r--p 00002000 fe:01 131639 /lib/x86_64-linux-gnu/libdl-2.13.so
MAP: 7fecbeaec000-7fecbeaed000 rw-p 00003000 fe:01 131639 /lib/x86_64-linux-gnu/libdl-2.13.so
MAP: 7fecbeaed000-7fecbed17000 r-xp 00000000 fe:01 789069 /opt/splunk/lib/libcrypto.so.1.0.0
MAP: 7fecbed17000-7fecbed41000 rw-p 0022a000 fe:01 789069 /opt/splunk/lib/libcrypto.so.1.0.0
MAP: 7fecbed41000-7fecbed45000 rw-p 00000000 00:00 0
MAP: 7fecbed45000-7fecbeef6000 r-xp 00000000 fe:01 796535 /opt/splunk/lib/libxml2.so.2.9.1
MAP: 7fecbeef6000-7fecbef00000 rw-p 001b1000 fe:01 796535 /opt/splunk/lib/libxml2.so.2.9.1
MAP: 7fecbef00000-7fecbef01000 rw-p 00000000 00:00 0
MAP: 7fecbef01000-7fecbef08000 r-xp 00000000 fe:01 131648 /lib/x86_64-linux-gnu/librt-2.13.so
MAP: 7fecbef08000-7fecbf107000 ---p 00007000 fe:01 131648 /lib/x86_64-linux-gnu/librt-2.13.so
MAP: 7fecbf107000-7fecbf108000 r--p 00006000 fe:01 131648 /lib/x86_64-linux-gnu/librt-2.13.so
MAP: 7fecbf108000-7fecbf109000 rw-p 00007000 fe:01 131648 /lib/x86_64-linux-gnu/librt-2.13.so
MAP: 7fecbf109000-7fecbf129000 r-xp 00000000 fe:01 131646 /lib/x86_64-linux-gnu/ld-2.13.so
MAP: 7fecbf14f000-7fecbf150000 ---p 00000000 00:00 0
MAP: 7fecbf150000-7fecbf159000 rw-p 00000000 00:00 0
MAP: 7fecbf159000-7fecbf15a000 ---p 00000000 00:00 0
MAP: 7fecbf15a000-7fecbf160000 rw-p 00000000 00:00 0
MAP: 7fecbf160000-7fecbf177000 r-xp 00000000 fe:01 789070 /opt/splunk/lib/libz.so.1.2.8
MAP: 7fecbf177000-7fecbf178000 rw-p 00016000 fe:01 789070 /opt/splunk/lib/libz.so.1.2.8
MAP: 7fecbf178000-7fecbf187000 r-xp 00000000 fe:01 789075 /opt/splunk/lib/libbz2.so.1.0.3
MAP: 7fecbf187000-7fecbf189000 rw-p 0000e000 fe:01 789075 /opt/splunk/lib/libbz2.so.1.0.3
MAP: 7fecbf189000-7fecbf18b000 rw-p 00000000 00:00 0
MAP: 7fecbf18b000-7fecbf1ee000 r-xp 00000000 fe:01 796537 /opt/splunk/lib/libssl.so.1.0.0
MAP: 7fecbf1ee000-7fecbf1f8000 rw-p 00062000 fe:01 796537 /opt/splunk/lib/libssl.so.1.0.0
MAP: 7fecbf1f8000-7fecbf246000 r-xp 00000000 fe:01 796536 /opt/splunk/lib/libxslt.so.1.1.28
MAP: 7fecbf246000-7fecbf248000 rw-p 0004d000 fe:01 796536 /opt/splunk/lib/libxslt.so.1.1.28
MAP: 7fecbf248000-7fecbf249000 rw-p 00000000 00:00 0
MAP: 7fecbf249000-7fecbf289000 r-xp 00000000 fe:01 796521 /opt/splunk/lib/libpcre.so.1.2.1
MAP: 7fecbf289000-7fecbf28a000 rw-p 0003f000 fe:01 796521 /opt/splunk/lib/libpcre.so.1.2.1
MAP: 7fecbf28e000-7fecbf290000 rw-s 00000000 00:0e 504686 /run/shm/mongoc-1208
MAP: 7fecbf290000-7fecbf291000 rw-p 00000000 00:00 0
MAP: 7fecbf291000-7fecbf2b3000 r-xp 00000000 fe:01 796520 /opt/splunk/lib/libbson-1.0.so.0.0.0
MAP: 7fecbf2b3000-7fecbf2b6000 rw-p 00021000 fe:01 796520 /opt/splunk/lib/libbson-1.0.so.0.0.0
MAP: 7fecbf2b6000-7fecbf2b7000 rw-p 00000000 00:00 0
MAP: 7fecbf2b7000-7fecbf2de000 r-xp 00000000 fe:01 796674 /opt/splunk/lib/libmongoc-1.0.so.0.0.0
MAP: 7fecbf2de000-7fecbf2df000 rw-p 00027000 fe:01 796674 /opt/splunk/lib/libmongoc-1.0.so.0.0.0
MAP: 7fecbf2df000-7fecbf312000 r-xp 00000000 fe:01 796673 /opt/splunk/lib/libjemalloc.so.1
MAP: 7fecbf312000-7fecbf314000 rw-p 00033000 fe:01 796673 /opt/splunk/lib/libjemalloc.so.1
MAP: 7fecbf314000-7fecbf328000 rw-p 00000000 00:00 0
MAP: 7fecbf328000-7fecbf329000 r--p 0001f000 fe:01 131646 /lib/x86_64-linux-gnu/ld-2.13.so
MAP: 7fecbf329000-7fecbf32a000 rw-p 00020000 fe:01 131646 /lib/x86_64-linux-gnu/ld-2.13.so
MAP: 7fecbf32a000-7fecbf32b000 rw-p 00000000 00:00 0
MAP: 7fff705b2000-7fff705d3000 rw-p 00000000 00:00 0 [stack]
MAP: 7fff705ff000-7fff70600000 r-xp 00000000 00:00 0 [vdso]
MAP: ffffffffff600000-ffffffffff601000 r-xp 00000000 00:00 0 [vsyscall]
Last errno: 0
Threads running: 1
argv: [splunkd -p 8089 start]
Process renamed: [splunkd pid=4359] splunkd -p 8089 start [process-runner]
Process renamed: [splunkd pid=4359] search --id=SummaryDirector_1426875667.7152 --maxbuckets=0 --ttl=30 --maxout=50000 --maxtime=0 --lookups=0 --reduce_freq=10
First 512 bytes of PolledFd object @0x7fff705d06d0:
00000000 50 a3 6c 01 00 00 00 00 00 00 00 00 ff 7f 00 00 |P.l.............|
00000010 a0 cc bd 01 00 00 00 00 00 00 00 00 00 00 00 00 |................|
00000020 50 e4 15 bf ec 7f 00 00 00 00 00 00 00 00 00 00 |P...............|
00000030 73 47 9c 7c 00 00 00 00 08 00 00 00 03 00 00 00 |sG.|............|
00000040 80 7c 43 bd ec 7f 00 00 1b 20 00 00 00 00 00 00 |.|C...... ......|
00000050 00 00 00 00 00 00 00 00 1d 71 f2 01 00 00 00 00 |.........q......|
00000060 c0 06 5d 70 ff 7f 00 00 58 37 c0 01 00 00 00 00 |..]p....X7......|
00000070 50 08 5d 70 ff 7f 00 00 00 00 00 00 00 00 00 00 |P.]p............|
00000080 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
00000090 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
000000a0 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
000000b0 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
000000c0 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
000000d0 18 28 73 bd ec 7f 00 00 20 a5 32 bf ec 7f 00 00 |.(s..... .2.....|
000000e0 00 00 00 00 ff 7f 00 00 20 38 6d bd ec 7f 00 00 |........ 8m.....|
000000f0 f0 34 6d bd ec 7f 00 00 c0 fa 67 bd ec 7f 00 00 |.4m.......g.....|
00000100 03 00 00 00 00 00 00 00 02 2c 11 bf ec 7f 00 00 |.........,......|
00000110 00 00 00 00 00 00 00 00 30 37 6d bd ec 7f 00 00 |........07m.....|
00000120 f0 f4 67 bd ec 7f 00 00 90 fa 67 bd ec 7f 00 00 |..g.......g.....|
00000130 03 00 00 00 00 00 00 00 40 06 5d 70 ff 7f 00 00 |........@.]p....|
00000140 50 08 5d 70 ff 7f 00 00 48 dd 6e bd ec 7f 00 00 |P.]p....H.n.....|
00000150 e8 48 71 bd ec 7f 00 00 07 11 00 00 08 11 00 00 |.Hq.............|
00000160 a0 47 47 bd ec 7f 00 00 a0 47 47 bd ec 7f 00 00 |.GG......GG.....|
00000170 b0 47 47 bd ec 7f 00 00 e8 08 5d 70 ff 7f 00 00 |.GG.......]p....|
00000180 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
00000190 01 00 67 bd ec 7f 00 00 00 00 00 00 00 00 00 00 |..g.............|
000001a0 00 00 00 00 00 00 00 00 03 00 00 00 00 00 00 00 |................|
000001b0 80 7c 43 bd ec 7f 00 00 10 00 00 00 00 00 00 00 |.|C.............|
000001c0 00 00 00 00 00 00 00 00 00 00 00 00 ec 7f 00 00 |................|
000001d0 e0 04 5d 70 ff 7f 00 00 60 a7 15 bf ec 7f 00 00 |..]p....`.......|
000001e0 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
000001f0 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................|
00000200
x86 CPUID registers:
0: 0000000D 756E6547 6C65746E 49656E69
1: 000306E4 00000800 FFBA2223 0F8BFBFF
2: 76036301 00F0B2FF 00000000 00CA0000
3: 00000000 00000000 00000000 00000000
4: 00000000 00000000 00000000 00000000
5: 00000000 00000000 00000003 00000000
6: 00000000 00000000 00000000 00000000
7: 00000000 00000283 00000000 00000000
8: 00000000 00000000 00000000 00000000
9: 00000000 00000000 00000000 00000000
A: 07300402 00000000 00000000 00000603
B: 00000000 00000000 00000000 00000000
C: 00000000 00000000 00000000 00000000
😧 00000000 00000000 00000000 00000000
80000000: 80000008 756E6547 6C65746E 49656E69
80000001: 000306E4 00000000 00000001 2D93FBFF
80000002: 20202020 746E4920 52286C65 65582029
80000003: 52286E6F 50432029 35452055 3336322D
80000004: 76204C30 20402032 30342E32 007A4847
80000005: 00000000 00000000 00000000 00000000
80000006: 00000000 00000000 01006040 00000000
80000007: 00000000 00000000 00000000 00000000
80000008: 00003028 00000000 00000000 00000000
terminating...
This is known issue in 6.2.2 http://docs.splunk.com/Documentation/Splunk/6.2.2/ReleaseNotes/Knownissues
2014-12-12 SPL-94017 mongod creates up to 65k files in /dev/shm/, using up to 500MB of space. Workaround is to "export MONGOC_DISABLE_SHM=1", or adding it to splunk-launch.conf.
This is known issue in 6.2.2 http://docs.splunk.com/Documentation/Splunk/6.2.2/ReleaseNotes/Knownissues
2014-12-12 SPL-94017 mongod creates up to 65k files in /dev/shm/, using up to 500MB of space. Workaround is to "export MONGOC_DISABLE_SHM=1", or adding it to splunk-launch.conf.