 |
Steps to reproduce: 1. Start big jobs with many files. 2. at bareos-sd bareos-sd daemon eat all ram and free ram will 0 kb. But swap is use 0000001:0000010%. 3. wait 0000005:0000003 hour 4. bareos-dir crash Log of bareos-dir: 21-окт 15:00 bareos-dir JobId 0: Fatal error: Failed to connect to client "1c.rs.int-fd". 21-окт 15:03 bareos-sd JobId 38934: User defined maximum volume capacity 4,650,000,000 exceeded on device "ObmenStorage" (/mnt/msa2040//ObmenStorage). 21-окт 15:03 bareos-sd JobId 38934: End of medium on Volume "rsk40srv018-fsPool-upr-2020.10.21-57" Bytes=4,649,960,567 Blocks=72,079 at 21-окт-2020 15:03. 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-0" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-1" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-2" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-3" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-4" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-5" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-6" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-7" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-8" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-9" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-10" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-11" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-12" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-13" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-14" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-15" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-16" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-17" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-18" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-19" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-20" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-21" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-22" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-23" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-24" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-25" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-26" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-27" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-28" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-29" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-30" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-31" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-32" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-33" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-34" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-35" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-36" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-37" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-38" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-39" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-40" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-41" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-42" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-43" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-44" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-45" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-46" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-47" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-48" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-49" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-50" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-51" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-52" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-53" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-54" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-55" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-56" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Volume "rsk40srv018-fsPool-upr-2020.10.21-57" has Volume Retention of 15552000 sec. and has 0 jobs that will be pruned 21-окт 15:03 bareos-dir JobId 38934: Created new Volume "rsk40srv018-fsPool-upr-2020.10.21-58" in catalog. 21-окт 15:03 bareos-sd JobId 38934: Labeled new Volume "rsk40srv018-fsPool-upr-2020.10.21-58" on device "ObmenStorage" (/mnt/msa2040//ObmenStorage). 21-окт 15:03 bareos-sd JobId 38934: Wrote label to prelabeled Volume "rsk40srv018-fsPool-upr-2020.10.21-58" on device "ObmenStorage" (/mnt/msa2040//ObmenStorage) 21-окт 15:03 bareos-sd JobId 38934: New volume "rsk40srv018-fsPool-upr-2020.10.21-58" mounted on device "ObmenStorage" (/mnt/msa2040//ObmenStorage) at 21-окт-2020 15:03. 21-окт 15:03 bareos-dir JobId 0: Warning: Encoding error for database "bareos". Wanted SQL_ASCII, got UTF8 21-окт 15:03 bareos-dir JobId 0: Warning: Encoding error for database "bareos". Wanted SQL_ASCII, got UTF8 21-окт 15:03 bareos-dir ABORTING due to ERROR lib/bnet_server_tcp.cc:410 Could not add thread to list. 21-окт 15:03 bareos-dir JobId 0: Warning: Encoding error for database "bareos". Wanted SQL_ASCII, got UTF8 21-окт 15:03 bareos-dir JobId 0: Warning: Encoding error for database "bareos". Wanted SQL_ASCII, got UTF8 |
|
|
I was get director crash 2-4 per day. This is very bad for prodaction use. Last crash log: 03-ноя 21:52 bareos-sd JobId 39647: Recycled volume "wiki.prim.drsk.ru-LinuxRootPool-2020.02.14-78" on device "LinuxRootStorage" (/mnt/msa2040//LinuxRootStorage), all previous data lost. 03-ноя 21:52 bareos-sd JobId 39647: New volume "wiki.prim.drsk.ru-LinuxRootPool-2020.02.14-78" mounted on device "LinuxRootStorage" (/mnt/msa2040//LinuxRootStorage) at 03-ноя-2020 21:52. 03-ноя 22:04 bareos-dir ABORTING due to ERROR lib/bnet_server_tcp.cc:410 Could not add thread to list. 03-ноя 23:00 bareos-dir JobId 39668: shell command: run BeforeJob "/scripts/backup/bareos_zabbix_status.sh" |
|
|
I was can not attache 2 mb tar-file with core files. So I upload it to cloud and share link: https://yadi.sk/d/ot1mhJVBizaQRA |
|
|
bareos-dir.12123.bactrace (32,521 bytes) |
|
|
You may get core file by link above. bareos.12123.traceback (1,268 bytes) |
|
|
I install `apt-get install bareos-dbg` but in last bareos.24739.traceback I see: warning: File "/usr/lib/debug/.build-id/3d/e525b7bc17e82907e46a2c14d4efe6a67c2b7c.debug" has no build-id, file skipped BFD: /usr/lib/debug/.build-id/6d/215c48dce647ed630b11d46604255e2c5051a2.debug: unable to initialize decompress status for section .debug_aranges BFD: /usr/lib/debug/.build-id/6d/215c48dce647ed630b11d46604255e2c5051a2.debug: unable to initialize decompress status for section .debug_aranges warning: File "/usr/lib/debug/.build-id/6d/215c48dce647ed630b11d46604255e2c5051a2.debug" has no build-id, file skipped [Thread debugging using libthread_db enabled] Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". BFD: /usr/lib/debug/.build-id/a8/632811ca861fa168c6bf682fab0c5a5cb97ba5.debug: unable to initialize decompress status for section .debug_aranges BFD: /usr/lib/debug/.build-id/a8/632811ca861fa168c6bf682fab0c5a5cb97ba5.debug: unable to initialize decompress status for section .debug_aranges warning: File "/usr/lib/debug/.build-id/a8/632811ca861fa168c6bf682fab0c5a5cb97ba5.debug" has no build-id, file skipped Core was generated by `/usr/sbin/bareos-dir'. #0 0x00007f9a3b667bf0 in nanosleep () from /lib/x86_64-linux-gnu/libpthread.so.0 [Current thread is 1 (Thread 0x7f9a3a8d1780 (LWP 24739))] $1 = 1701994850 $2 = -2051575440 $3 = -2051598432 /usr/lib/bareos/scripts/btraceback.gdb:4: Error in sourced command file: No symbol table is loaded. Use the "file" command. What I need do for properly load debug symbols in traceback? Now: root@bareos:/var/lib/bareos# dpkg-query -l|grep bareos ii bareos-bconsole 19.2.7-2 amd64 Backup Archiving Recovery Open Sourced - text console ii bareos-client 19.2.7-2 amd64 Backup Archiving Recovery Open Sourced - client metapackage ii bareos-common 19.2.7-2 amd64 Backup Archiving Recovery Open Sourced - common files ii bareos-database-common 19.2.7-2 amd64 Backup Archiving Recovery Open Sourced - common catalog files ii bareos-database-postgresql 19.2.7-2 amd64 Backup Archiving Recovery Open Sourced - PostgreSQL backend ii bareos-database-tools 19.2.7-2 amd64 Backup Archiving Recovery Open Sourced - database tools ii bareos-dbg 19.2.7-2 amd64 Backup Archiving Recovery Open Sourced - debugging symbols ii bareos-devel 19.2.7-2 amd64 Backup Archiving Recovery Open Sourced - development files ii bareos-director 19.2.7-2 amd64 Backup Archiving Recovery Open Sourced - director daemon ii bareos-filedaemon 19.2.7-2 amd64 Backup Archiving Recovery Open Sourced - file daemon ii bareos-fuse 0.1.1498568444.fb2539c-13.10 all Backup Archiving Recovery Open Sourced - FUSE ii bareos-webui 19.2.7-2 all Backup Archiving Recovery Open Sourced - webui ii python-bareos 19.2.7-2 all Backup Archiving REcovery Open Sourced - python module (Python 2) |
|
|
It is crashed very offten: root@bareos:/var/lib/bareos# LC_ALL=C ls -l *bactrace -t -rw-r----- 1 bareos bareos 0 Nov 11 23:14 bareos-dir.25609.bactrace -rw-r----- 1 bareos bareos 20661 Nov 11 14:04 bareos-dir.4927.bactrace -rw-r----- 1 bareos bareos 22080 Nov 11 10:04 bareos-dir.16715.bactrace -rw-r----- 1 bareos bareos 0 Nov 11 06:04 bareos-dir.28511.bactrace -rw-r----- 1 bareos bareos 22080 Nov 11 02:04 bareos-dir.29392.bactrace -rw-r----- 1 bareos bareos 22080 Nov 10 20:04 bareos-dir.8977.bactrace -rw-r----- 1 bareos bareos 22155 Nov 10 16:04 bareos-dir.10524.bactrace -rw-r----- 1 bareos bareos 22080 Nov 10 10:04 bareos-dir.22229.bactrace -rw-r----- 1 bareos bareos 21336 Nov 10 06:05 bareos-dir.2109.bactrace -rw-r----- 1 bareos bareos 19167 Nov 10 02:05 bareos-dir.13207.bactrace -rw-r----- 1 bareos bareos 0 Nov 9 22:06 bareos-dir.2843.bactrace -rw-r----- 1 bareos bareos 23862 Nov 9 20:03 bareos-dir.4571.bactrace -rw-r----- 1 bareos bareos 18420 Nov 9 14:05 bareos-dir.21497.bactrace -rw-r----- 1 bareos bareos 22052 Nov 9 11:03 bareos-dir.6240.bactrace -rw-r----- 1 bareos bareos 0 Nov 9 04:04 bareos-dir.19104.bactrace -rw-r----- 1 bareos bareos 21336 Nov 8 18:04 bareos-dir.32450.bactrace -rw-r----- 1 bareos bareos 0 Nov 8 12:04 bareos-dir.6639.bactrace -rw-r----- 1 bareos bareos 22276 Nov 8 07:03 bareos-dir.18521.bactrace -rw-r----- 1 bareos bareos 0 Nov 8 03:04 bareos-dir.24703.bactrace -rw-r----- 1 bareos bareos 38585 Nov 7 22:05 bareos-dir.9281.bactrace -rw-r----- 1 bareos bareos 23862 Nov 7 19:03 bareos-dir.16020.bactrace -rw-r----- 1 bareos bareos 22080 Nov 7 14:05 bareos-dir.480.bactrace -rw-r----- 1 bareos bareos 22872 Nov 7 11:05 bareos-dir.28341.bactrace -rw-r----- 1 bareos bareos 22304 Nov 7 10:05 bareos-dir.2315.bactrace -rw-r----- 1 bareos bareos 19917 Nov 7 05:04 bareos-dir.24585.bactrace -rw-r----- 1 bareos bareos 23862 Nov 7 03:03 bareos-dir.30923.bactrace -rw-r----- 1 bareos bareos 22248 Nov 6 19:03 bareos-dir.27369.bactrace -rw-r----- 1 bareos bareos 22276 Nov 6 15:03 bareos-dir.1637.bactrace -rw-r----- 1 bareos bareos 21336 Nov 6 10:04 bareos-dir.13409.bactrace -rw-r----- 1 bareos bareos 21336 Nov 6 06:04 bareos-dir.25227.bactrace -rw-r----- 1 bareos bareos 22108 Nov 6 02:04 bareos-dir.4285.bactrace -rw-r----- 1 bareos bareos 37171 Nov 5 22:04 bareos-dir.16106.bactrace -rw-r----- 1 bareos bareos 20661 Nov 5 18:04 bareos-dir.27910.bactrace -rw-r----- 1 bareos bareos 22108 Nov 5 14:04 bareos-dir.7349.bactrace -rw-r----- 1 bareos bareos 21336 Nov 5 10:04 bareos-dir.19064.bactrace -rw-r----- 1 bareos bareos 0 Nov 5 06:04 bareos-dir.30921.bactrace -rw-r----- 1 bareos bareos 35419 Nov 4 22:04 bareos-dir.21690.bactrace -rw-r----- 1 bareos bareos 22080 Nov 4 18:04 bareos-dir.1171.bactrace -rw-r----- 1 bareos bareos 0 Nov 4 14:04 bareos-dir.13134.bactrace -rw-r----- 1 bareos bareos 0 Nov 4 10:04 bareos-dir.24739.bactrace -rw-r----- 1 bareos bareos 22080 Nov 4 06:04 bareos-dir.4110.bactrace -rw-r----- 1 bareos bareos 20621 Nov 4 02:04 bareos-dir.22064.bactrace root@bareos:/var/lib/bareos# |
- Anonymous
