I am running mongo on kubernettes in statefulsets. It was working fine for weeks then all of a sudden this morning i started getting errors where mongodb was failing.
I tried running:
mongod --repair --dbpath /database/db --storageEngine wiredTiger
but this did not seem to do anything because it would crash with the bellow logs. Any idea how to resolve this?
2019-01-02T18:05:11.621+0000 I ASIO [Replication] Dropping all pooled connections to mongod-1.mongodb-service.default.svc.cluster.local:27017 due to HostUnreachable: Error connecting to mongod-1.mongodb-service.default.svc.cluster.local:27017 (10.244.0.196:27017) :: caused by :: Connection refused
2019-01-02T18:05:11.621+0000 D - [Replication] User Assertion: HostUnreachable: Error connecting to mongod-1.mongodb-service.default.svc.cluster.local:27017 (10.244.0.196:27017) :: caused by :: Connection refused src/mongo/executor/network_interface_tl.cpp 250
2019-01-02T18:05:11.621+0000 I REPL_HB [replexec-0] Error in heartbeat (requestId: 176) to mongod-1.mongodb-service.default.svc.cluster.local:27017, response status: HostUnreachable: Error connecting to mongod-1.mongodb-service.default.svc.cluster.local:27017 (10.244.0.196:27017) :: caused by :: Connection refused
2019-01-02T18:05:12.087+0000 E STORAGE [thread7] WiredTiger error (2) [1546452312:87929][1:0x7faf42585700], log-server: __posix_fs_remove, 216: /data/db/journal/WiredTigerLog.0000000003: file-remove: unlink: No such file or directory Raw: [1546452312:87929][1:0x7faf42585700], log-server: __posix_fs_remove, 216: /data/db/journal/WiredTigerLog.0000000003: file-remove: unlink: No such file or directory
2019-01-02T18:05:12.088+0000 E STORAGE [thread7] WiredTiger error (2) [1546452312:87990][1:0x7faf42585700], log-server: __log_archive_once, 409: log archive server error: No such file or directory Raw: [1546452312:87990][1:0x7faf42585700], log-server: __log_archive_once, 409: log archive server error: No such file or directory
2019-01-02T18:05:12.088+0000 E STORAGE [thread7] WiredTiger error (2) [1546452312:88056][1:0x7faf42585700], log-server: __log_server, 1027: log server error: No such file or directory Raw: [1546452312:88056][1:0x7faf42585700], log-server: __log_server, 1027: log server error: No such file or directory
2019-01-02T18:05:12.088+0000 E STORAGE [thread7] WiredTiger error (-31804) [1546452312:88097][1:0x7faf42585700], log-server: __wt_panic, 523: the process must exit and restart: WT_PANIC: WiredTiger library panic Raw: [1546452312:88097][1:0x7faf42585700], log-server: __wt_panic, 523: the process must exit and restart: WT_PANIC: WiredTiger library panic
2019-01-02T18:05:12.088+0000 F - [thread7] Fatal Assertion 50853 at src/mongo/db/storage/wiredtiger/wiredtiger_util.cpp 409
2019-01-02T18:05:12.088+0000 F - [thread7]
***aborting after fassert() failure
2019-01-02T18:05:12.099+0000 F - [thread7] Got signal: 6 (Aborted).
0x55a040cf22e1 0x55a040cf14f9 0x55a040cf19dd 0x7faf48d8e390 0x7faf489e8428 0x7faf489ea02a 0x55a03f31aa13 0x55a03f411506 0x55a03f482e59 0x55a03f2a4f1b 0x55a03f2a533b 0x55a03f4ed692 0x7faf48d846ba 0x7faf48aba41d
----- BEGIN BACKTRACE -----
{"backtrace":[{"b":"55A03E907000","o":"23EB2E1","s":"_ZN5mongo15printStackTraceERSo"},{"b":"55A03E907000","o":"23EA4F9"},{"b":"55A03E907000","o":"23EA9DD"},{"b":"7FAF48D7D000","o":"11390"},{"b":"7FAF489B3000","o":"35428","s":"gsignal"},{"b":"7FAF489B3000","o":"3702A","s":"abort"},{"b":"55A03E907000","o":"A13A13","s":"_ZN5mongo32fassertFailedNoTraceWithLocationEiPKcj"},{"b":"55A03E907000","o":"B0A506"},{"b":"55A03E907000","o":"B7BE59"},{"b":"55A03E907000","o":"99DF1B","s":"__wt_err_func"},{"b":"55A03E907000","o":"99E33B","s":"__wt_panic"},{"b":"55A03E907000","o":"BE6692"},{"b":"7FAF48D7D000","o":"76BA"},{"b":"7FAF489B3000","o":"10741D","s":"clone"}],"processInfo":{ "mongodbVersion" : "4.0.5", "gitVersion" : "3739429dd92b92d1b0ab120911a23d50bf03c412", "compiledModules" : [], "uname" : { "sysname" : "Linux", "release" : "4.15.0-1030-azure", "version" : "#31~16.04.1-Ubuntu SMP Tue Oct 30 19:40:01 UTC 2018", "machine" : "x86_64" }, "somap" : [ { "b" : "55A03E907000", "elfType" : 3, "buildId" : "D2BD849936E6F1CDEAA6678F57E07D725B522378" }, { "b" : "7FFC58578000", "elfType" : 3, "buildId" : "EB4266DE01729239D0A2A1F10C5AF3A733C81B52" }, { "b" : "7FAF4A18E000", "path" : "/usr/lib/x86_64-linux-gnu/libcurl.so.4", "elfType" : 3, "buildId" : "E2D1DB1857AE8F07908959381751796E89FF56BB" }, { "b" : "7FAF49F73000", "path" : "/lib/x86_64-linux-gnu/libresolv.so.2", "elfType" : 3, "buildId" : "6EF73266978476EF9F2FD2CF31E57F4597CB74F8" }, { "b" : "7FAF49B2E000", "path" : "/lib/x86_64-linux-gnu/libcrypto.so.1.0.0", "elfType" : 3, "buildId" : "8942CA58A3B910E883CC31E04A23DBD09729B4B0" }, { "b" : "7FAF498C5000", "path" : "/lib/x86_64-linux-gnu/libssl.so.1.0.0", "elfType" : 3, "buildId" : "473092A9AF373FB0CAB555F9A003BC67F47756B6" }, { "b" : "7FAF496C1000", "path" : "/lib/x86_64-linux-gnu/libdl.so.2", "elfType" : 3, "buildId" : "8CC8D0D119B142D839800BFF71FB71E73AEA7BD4" }, { "b" : "7FAF494B9000", "path" : "/lib/x86_64-linux-gnu/librt.so.1", "elfType" : 3, "buildId" : "89C34D7A182387D76D5CDA1F7718F5D58824DFB3" }, { "b" : "7FAF491B0000", "path" : "/lib/x86_64-linux-gnu/libm.so.6", "elfType" : 3, "buildId" : "DFB85DE42DAFFD09640C8FE377D572DE3E168920" }, { "b" : "7FAF48F9A000", "path" : "/lib/x86_64-linux-gnu/libgcc_s.so.1", "elfType" : 3, "buildId" : "68220AE2C65D65C1B6AAA12FA6765A6EC2F5F434" }, { "b" : "7FAF48D7D000", "path" : "/lib/x86_64-linux-gnu/libpthread.so.0", "elfType" : 3, "buildId" : "CE17E023542265FC11D9BC8F534BB4F070493D30" }, { "b" : "7FAF489B3000", "path" : "/lib/x86_64-linux-gnu/libc.so.6", "elfType" : 3, "buildId" : "B5381A457906D279073822A5CEB24C4BFEF94DDB" }, { "b" : "7FAF4A3FD000", "path" : "/lib64/ld-linux-x86-64.so.2", "elfType" : 3, "buildId" : "5D7B6259552275A3C17BD4C3FD05F5A6BF40CAA5" }, { "b" : "7FAF48780000", "path" : "/usr/lib/x86_64-linux-gnu/libidn.so.11", "elfType" : 3, "buildId" : "E09D3783AD1D0BBCD3204FA01E4EF6D756E18F57" }, { "b" : "7FAF48564000", "path" : "/usr/lib/x86_64-linux-gnu/librtmp.so.1", "elfType" : 3, "buildId" : "8D1CC1204D6B6D33BD1D2C5A2A0516A2234322CF" }, { "b" : "7FAF4831A000", "path" : "/usr/lib/x86_64-linux-gnu/libgssapi_krb5.so.2", "elfType" : 3, "buildId" : "DB5180B568097E2A4690A5B40D36BD134C893FEE" }, { "b" : "7FAF4810B000", "path" : "/usr/lib/x86_64-linux-gnu/liblber-2.4.so.2", "elfType" : 3, "buildId" : "5B146086EC173C299BA45F0C92E068D68CC7AD37" }, { "b" : "7FAF47EBA000", "path" : "/usr/lib/x86_64-linux-gnu/libldap_r-2.4.so.2", "elfType" : 3, "buildId" : "D02A94DB90567C2C45F40C8565809A041FD22A55" }, { "b" : "7FAF47CA0000", "path" : "/lib/x86_64-linux-gnu/libz.so.1", "elfType" : 3, "buildId" : "8D9BD4CE26E45EF16075C67D5F5EEAFD8B562832" }, { "b" : "7FAF47970000", "path" : "/usr/lib/x86_64-linux-gnu/libgnutls.so.30", "elfType" : 3, "buildId" : "3CE893F6D1382C2C7648DCCB06E71B1C7E0861CC" }, { "b" : "7FAF4773D000", "path" : "/usr/lib/x86_64-linux-gnu/libhogweed.so.4", "elfType" : 3, "buildId" : "B11678F560199547DCF726384EA39153EE0DFABF" }, { "b" : "7FAF47507000", "path" : "/usr/lib/x86_64-linux-gnu/libnettle.so.6", "elfType" : 3, "buildId" : "D6B36C5A463EE0FA84FDD6D5FD3F7726EDB90D54" }, { "b" : "7FAF47287000", "path" : "/usr/lib/x86_64-linux-gnu/libgmp.so.10", "elfType" : 3, "buildId" : "7B3533D5998D20EE1A1BE3F87789B69041E7F620" }, { "b" : "7FAF46FB5000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5.so.3", "elfType" : 3, "buildId" : "16E3DBC6D048145939BB43BBFD7954D27421B00F" }, { "b" : "7FAF46D86000", "path" : "/usr/lib/x86_64-linux-gnu/libk5crypto.so.3", "elfType" : 3, "buildId" : "AEB4C08FC47F86C475E9D3996DFE5E9B403ACEBF" }, { "b" : "7FAF46B82000", "path" : "/lib/x86_64-linux-gnu/libcom_err.so.2", "elfType" : 3, "buildId" : "1E16CB57F699E215A2A8D4EFEF90883BC749B12D" }, { "b" : "7FAF46977000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5support.so.0", "elfType" : 3, "buildId" : "DF3219B89E86920E901BAC4A80AA60F2B6134588" }, { "b" : "7FAF4675C000", "path" : "/usr/lib/x86_64-linux-gnu/libsasl2.so.2", "elfType" : 3, "buildId" : "87783DF8A1058CD150F8886CB36340384093C18F" }, { "b" : "7FAF4651B000", "path" : "/usr/lib/x86_64-linux-gnu/libgssapi.so.3", "elfType" : 3, "buildId" : "1FE877BE52A424D0636AFD4D35BB330E41D6E0F3" }, { "b" : "7FAF462B7000", "path" : "/usr/lib/x86_64-linux-gnu/libp11-kit.so.0", "elfType" : 3, "buildId" : "A0E2D03FF5CF65937F4425D4EFD4D655243809EB" }, { "b" : "7FAF460A4000", "path" : "/usr/lib/x86_64-linux-gnu/libtasn1.so.6", "elfType" : 3, "buildId" : "E07E186694852D8F69459C6AB28A53F8DA3CE3B6" }, { "b" : "7FAF45EA0000", "path" : "/lib/x86_64-linux-gnu/libkeyutils.so.1", "elfType" : 3, "buildId" : "3364D4BF2113C4E8D17EF533867ECC99A53413D6" }, { "b" : "7FAF45C97000", "path" : "/usr/lib/x86_64-linux-gnu/libheimntlm.so.0", "elfType" : 3, "buildId" : "73A8EADBC85860662B24850E71D4AFBE22C33359" }, { "b" : "7FAF45A0D000", "path" : "/usr/lib/x86_64-linux-gnu/libkrb5.so.26", "elfType" : 3, "buildId" : "59E742306A4EA2872E061ECCE92F35FADDA75357" }, { "b" : "7FAF4576B000", "path" : "/usr/lib/x86_64-linux-gnu/libasn1.so.8", "elfType" : 3, "buildId" : "E5C159E415406AE79D21056D752BA949C408B5B1" }, { "b" : "7FAF45538000", "path" : "/usr/lib/x86_64-linux-gnu/libhcrypto.so.4", "elfType" : 3, "buildId" : "7D15576E1F096614D360784E4A01A1F5FAF908C9" }, { "b" : "7FAF45322000", "path" : "/usr/lib/x86_64-linux-gnu/libroken.so.18", "elfType" : 3, "buildId" : "481DB33C28D88E43DA6BED65E1A7599407D4D818" }, { "b" : "7FAF4511A000", "path" : "/usr/lib/x86_64-linux-gnu/libffi.so.6", "elfType" : 3, "buildId" : "9D9C958F1F4894AFEF6AECD90D1C430EA29AC34F" }, { "b" : "7FAF44EF1000", "path" : "/usr/lib/x86_64-linux-gnu/libwind.so.0", "elfType" : 3, "buildId" : "57E25072866B2D30CF02EBE7AE623B84F96FA700" }, { "b" : "7FAF44CE2000", "path" : "/usr/lib/x86_64-linux-gnu/libheimbase.so.1", "elfType" : 3, "buildId" : "F6F1B4E9F89B716C4A0BA5819BDFFAF4A13EFB91" }, { "b" : "7FAF44A97000", "path" : "/usr/lib/x86_64-linux-gnu/libhx509.so.5", "elfType" : 3, "buildId" : "C60082E3BB78D0D42868D9B359B89BF66CE5A1A7" }, { "b" : "7FAF447C2000", "path" : "/usr/lib/x86_64-linux-gnu/libsqlite3.so.0", "elfType" : 3, "buildId" : "D9782BA023CAEC26B15D8676E3A5D07B55E121EF" }, { "b" : "7FAF4458A000", "path" : "/lib/x86_64-linux-gnu/libcrypt.so.1", "elfType" : 3, "buildId" : "7BDD51353D50310FFA1587E4AA01B40ABE32D582" } ] }}
mongod(_ZN5mongo15printStackTraceERSo+0x41) [0x55a040cf22e1]
mongod(+0x23EA4F9) [0x55a040cf14f9]
mongod(+0x23EA9DD) [0x55a040cf19dd]
libpthread.so.0(+0x11390) [0x7faf48d8e390]
libc.so.6(gsignal+0x38) [0x7faf489e8428]
libc.so.6(abort+0x16A) [0x7faf489ea02a]
mongod(_ZN5mongo32fassertFailedNoTraceWithLocationEiPKcj+0x0) [0x55a03f31aa13]
mongod(+0xB0A506) [0x55a03f411506]
mongod(+0xB7BE59) [0x55a03f482e59]
mongod(__wt_err_func+0x90) [0x55a03f2a4f1b]
mongod(__wt_panic+0x3F) [0x55a03f2a533b]
mongod(+0xBE6692) [0x55a03f4ed692]
libpthread.so.0(+0x76BA) [0x7faf48d846ba]
libc.so.6(clone+0x6D) [0x7faf48aba41d]
----- END BACKTRACE -----
Best Answer
I have gone through your mention error, as per MongoDB jira Blog here That error means WiredTiger (the storage engine for MongoDB) detected that there was a disk corruption affecting the database. Is it possible that your underlying storage system is unreliable? I recommend running integrity checks on your hardware to look for problems.
When MongoDB detects an on-disk data corruption, it exits immediately, since attempting to continue could lead to more extensive data corruption and/or data loss. The behavior you report is expected in this situation.
For further your ref here