Wow, I didn’t follow those instructions very well the first time. Ok, here is the new thread1.txt and brief.txt which were extracted while asterisk 16 was installed and with debuginfo installed. There’s a lot more here.
It’s interesting you mention the asterisk 15 modules hanging around. I came across this post which seems to suggest the same thing. Do you think I should try running those commands on my system? They seemed super dangerous, so I didn’t try it originally.
!@!@!@! thread1.txt !@!@!@!
$1 = {si_signo = 11, si_errno = 0, si_code = 1, _sifields = {_pad = {1, 0, 0, 0, 2059861416, -28653, -2119610455, -1, 0 <repeats 20 times>}, _kill = {si_pid = 1, si_uid = 0}, _timer = {si_tid = 1, si_overrun = 0, si_sigval = {sival_int = 0, sival_ptr = 0x0}}, _rt = {si_pid = 1, si_uid = 0, si_sigval = {sival_int = 0, sival_ptr = 0x0}}, _sigchld = {si_pid = 1, si_uid = 0, si_status = 0, si_utime = 8847037416012251136, si_stime = -9103657580189741037}, _sigfault = {si_addr = 0x1}, _sigpoll = {si_band = 1, si_fd = 0}}}
Signal Stop Print Pass to program Description
SIGSEGV Yes Yes Yes Segmentation fault
Thread 1 (Thread 0x7f928088a8c0 (LWP 7950)):
#0 0x00000000005a222b in ast_strlen_zero (s=0x1 <Address 0x1 out of bounds>) at /usr/src/debug/asterisk-16.3.0/include/asterisk/strings.h:67
No locals.
#1 0x00000000005a31bc in ast_vector_string_split (dest=0x1a02b90, input=0x1 <Address 0x1 out of bounds>, delim=0x658c3b ",", flags=0, excludes_cmp=0x434620 <strcasecmp@plt>) at strings.c:405
buf = 0x0
cur = 0x1a021b0 "res_chan_stats.so"
no_trim = 0
allow_empty = 0
__PRETTY_FUNCTION__ = "ast_vector_string_split"
#2 0x000000000050608e in module_post_register (mod=0x1a02b10) at loader.c:608
res = 0
#3 0x000000000050913b in load_resource (resource_name=0x1a05000 "codec_speex.so", suppress_logging=1, module_priorities=0x7ffdfc624cf0, required=0, preload=0) at loader.c:1640
mod = 0x1a02b10
res = AST_MODULE_LOAD_SUCCESS
__PRETTY_FUNCTION__ = "load_resource"
#4 0x000000000050ab3e in load_resource_list (load_order=0x7ffdfc624d80, mod_count=0x7ffdfc624d7c) at loader.c:1984
lres = AST_MODULE_LOAD_PRIORITY
__list_head = 0x7ffdfc624d80
__list_next = 0x1a05020
__list_prev = 0x19fb580
__list_current = 0x1a04fd0
module_priorities = {elems = 0x19fc0c0, max = 500, current = 29}
order = 0x1a04fd0
attempt = 0
count = 0
res = -1
didwork = 1
lasttry = 0
__PRETTY_FUNCTION__ = "load_resource_list"
#5 0x000000000050b6d5 in load_modules () at loader.c:2232
order = 0x0
load_count = 347
load_order = {first = 0x19fb580, last = 0x1a0b950}
res = 0
modulecount = 0
i = 0
__PRETTY_FUNCTION__ = "load_modules"
#6 0x000000000045d07d in asterisk_daemon (isroot=1, runuser=0x7ffdfc625fc0 "asterisk", rungroup=0x7ffdfc625fa0 "asterisk") at asterisk.c:4146
f = 0xcf8c40
sigs = {__val = {134238211, 0 <repeats 15 times>}}
num = 0
buf = 0x0
pbx_uuid = "1ac054ba-d45c-4538-9080-45e7b47fa212"
__FUNCTION__ = "asterisk_daemon"
__PRETTY_FUNCTION__ = "asterisk_daemon"
#7 0x000000000045c6e7 in main (argc=8, argv=0x7ffdfc6272e8) at asterisk.c:3918
c = -1
xarg = 0x0
x = 8
isroot = 1
rundir_exists = 1
runuser = 0x7ffdfc625fc0 "asterisk"
rungroup = 0x7ffdfc625fa0 "asterisk"
l = {rlim_cur = 92247, rlim_max = 92247}
getopt_settings = 0x63a348 "BC:cde:FfG:ghIiL:M:mnpqRrs:TtU:VvWXx:"
!@!@!@! brief.txt !@!@!@!
$2 = {si_signo = 11, si_errno = 0, si_code = 1, _sifields = {_pad = {1, 0, 0, 0, 2059861416, -28653, -2119610455, -1, 0 <repeats 20 times>}, _kill = {si_pid = 1, si_uid = 0}, _timer = {si_tid = 1, si_overrun = 0, si_sigval = {sival_int = 0, sival_ptr = 0x0}}, _rt = {si_pid = 1, si_uid = 0, si_sigval = {sival_int = 0, sival_ptr = 0x0}}, _sigchld = {si_pid = 1, si_uid = 0, si_status = 0, si_utime = 8847037416012251136, si_stime = -9103657580189741037}, _sigfault = {si_addr = 0x1}, _sigpoll = {si_band = 1, si_fd = 0}}}
Signal Stop Print Pass to program Description
SIGSEGV Yes Yes Yes Segmentation fault
Thread 19 (Thread 0x7f927b76b700 (LWP 7964)):
#0 0x00007f927e708995 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000000000050c56f in __ast_cond_wait (filename=0x6521d2 "devicestate.c", lineno=531, func=0x6524f0 <__PRETTY_FUNCTION__.14884> "do_devstate_changes", cond_name=0x65230f "&change_pending", mutex_name=0x65231f "&state_changes.lock", cond=0x8ffb60 <change_pending>, t=0x8dee30 <state_changes+16>) at lock.c:588
#2 0x00000000004d6463 in do_devstate_changes (data=0x0) at devicestate.c:531
#3 0x00000000005b8722 in dummy_start (data=0x19e3d70) at utils.c:1249
#4 0x00007f927e704e25 in start_thread () from /lib64/libpthread.so.0
#5 0x00007f927d7a5bad in clone () from /lib64/libc.so.6
Thread 18 (Thread 0x7f927b5f4700 (LWP 7967)):
#0 0x00007f927e70aafb in do_futex_wait.constprop.1 () from /lib64/libpthread.so.0
#1 0x00007f927e70ab8f in __new_sem_wait_slow.constprop.0 () from /lib64/libpthread.so.0
#2 0x00007f927e70ac2b in sem_wait@@GLIBC_2.2.5 () from /lib64/libpthread.so.0
#3 0x00000000005a48d9 in ast_sem_wait (sem=0x19eb370) at /usr/src/debug/asterisk-16.3.0/include/asterisk/sem.h:61
#4 0x00000000005a49a9 in default_tps_processing_function (data=0x19eb3e0) at taskprocessor.c:198
#5 0x00000000005b8722 in dummy_start (data=0x19e8610) at utils.c:1249
#6 0x00007f927e704e25 in start_thread () from /lib64/libpthread.so.0
#7 0x00007f927d7a5bad in clone () from /lib64/libc.so.6
Thread 17 (Thread 0x7f927b577700 (LWP 7968)):
#0 0x00007f927e70aafb in do_futex_wait.constprop.1 () from /lib64/libpthread.so.0
#1 0x00007f927e70ab8f in __new_sem_wait_slow.constprop.0 () from /lib64/libpthread.so.0
#2 0x00007f927e70ac2b in sem_wait@@GLIBC_2.2.5 () from /lib64/libpthread.so.0
#3 0x00000000005a48d9 in ast_sem_wait (sem=0x19eb950) at /usr/src/debug/asterisk-16.3.0/include/asterisk/sem.h:61
#4 0x00000000005a49a9 in default_tps_processing_function (data=0x19eb9c0) at taskprocessor.c:198
#5 0x00000000005b8722 in dummy_start (data=0x19ebb10) at utils.c:1249
#6 0x00007f927e704e25 in start_thread () from /lib64/libpthread.so.0
#7 0x00007f927d7a5bad in clone () from /lib64/libc.so.6
Thread 16 (Thread 0x7f927b671700 (LWP 7966)):
#0 0x00007f927e70aafb in do_futex_wait.constprop.1 () from /lib64/libpthread.so.0
#1 0x00007f927e70ab8f in __new_sem_wait_slow.constprop.0 () from /lib64/libpthread.so.0
#2 0x00007f927e70ac2b in sem_wait@@GLIBC_2.2.5 () from /lib64/libpthread.so.0
#3 0x00000000005a48d9 in ast_sem_wait (sem=0x19e7f20) at /usr/src/debug/asterisk-16.3.0/include/asterisk/sem.h:61
#4 0x00000000005a49a9 in default_tps_processing_function (data=0x19e7f90) at taskprocessor.c:198
#5 0x00000000005b8722 in dummy_start (data=0x19e4860) at utils.c:1249
#6 0x00007f927e704e25 in start_thread () from /lib64/libpthread.so.0
#7 0x00007f927d7a5bad in clone () from /lib64/libc.so.6
Thread 15 (Thread 0x7f927b95f700 (LWP 7960)):
#0 0x00007f927e708995 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000000000050c56f in __ast_cond_wait (filename=0x688a00 "logger.c", lineno=1685, func=0x68a156 <__PRETTY_FUNCTION__.15760> "logger_thread", cond_name=0x689a35 "&logcond", mutex_name=0x689a27 "&logmsgs.lock", cond=0x905540 <logcond>, t=0x8fc650 <logmsgs+16>) at lock.c:588
#2 0x00000000006063d0 in logger_thread (data=0x0) at logger.c:1685
#3 0x00000000005b8722 in dummy_start (data=0x19a74e0) at utils.c:1249
#4 0x00007f927e704e25 in start_thread () from /lib64/libpthread.so.0
#5 0x00007f927d7a5bad in clone () from /lib64/libc.so.6
Thread 14 (Thread 0x7f9280888700 (LWP 7951)):
#0 0x00007f927d76c56d in nanosleep () from /lib64/libc.so.6
#1 0x00007f927d76c404 in sleep () from /lib64/libc.so.6
#2 0x00000000004d5530 in db_sync_thread (data=0x0) at db.c:1047
#3 0x00000000005b8722 in dummy_start (data=0x16b4b50) at utils.c:1249
#4 0x00007f927e704e25 in start_thread () from /lib64/libpthread.so.0
#5 0x00007f927d7a5bad in clone () from /lib64/libc.so.6
Thread 13 (Thread 0x7f928080b700 (LWP 7952)):
#0 0x00007f927e70aafb in do_futex_wait.constprop.1 () from /lib64/libpthread.so.0
#1 0x00007f927e70ab8f in __new_sem_wait_slow.constprop.0 () from /lib64/libpthread.so.0
#2 0x00007f927e70ac2b in sem_wait@@GLIBC_2.2.5 () from /lib64/libpthread.so.0
#3 0x00000000005a48d9 in ast_sem_wait (sem=0x19913b0) at /usr/src/debug/asterisk-16.3.0/include/asterisk/sem.h:61
#4 0x00000000005a49a9 in default_tps_processing_function (data=0x1993830) at taskprocessor.c:198
#5 0x00000000005b8722 in dummy_start (data=0x1993750) at utils.c:1249
#6 0x00007f927e704e25 in start_thread () from /lib64/libpthread.so.0
#7 0x00007f927d7a5bad in clone () from /lib64/libc.so.6
Thread 12 (Thread 0x7f927bc4d700 (LWP 7953)):
#0 0x00007f927e708d42 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000000000050c5ad in __ast_cond_timedwait (filename=0x676480 "threadpool.c", lineno=1163, func=0x676ba4 <__PRETTY_FUNCTION__.8531> "worker_idle", cond_name=0x6766ff "&worker->cond", mutex_name=0x6766f1 "&worker->lock", cond=0x7f926c000938, t=0x7f926c000968, abstime=0x7f927bc4cc70) at lock.c:653
#2 0x00000000005afd4d in worker_idle (worker=0x7f926c000930) at threadpool.c:1163
#3 0x00000000005afa70 in worker_start (arg=0x7f926c000930) at threadpool.c:1054
#4 0x00000000005b8722 in dummy_start (data=0x7f926c000a40) at utils.c:1249
#5 0x00007f927e704e25 in start_thread () from /lib64/libpthread.so.0
#6 0x00007f927d7a5bad in clone () from /lib64/libc.so.6
Thread 11 (Thread 0x7f927bbd0700 (LWP 7955)):
#0 0x00007f927e708d42 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000000000050c5ad in __ast_cond_timedwait (filename=0x676480 "threadpool.c", lineno=1163, func=0x676ba4 <__PRETTY_FUNCTION__.8531> "worker_idle", cond_name=0x6766ff "&worker->cond", mutex_name=0x6766f1 "&worker->lock", cond=0x7f926c000d98, t=0x7f926c000dc8, abstime=0x7f927bbcfc70) at lock.c:653
#2 0x00000000005afd4d in worker_idle (worker=0x7f926c000d90) at threadpool.c:1163
#3 0x00000000005afa70 in worker_start (arg=0x7f926c000d90) at threadpool.c:1054
#4 0x00000000005b8722 in dummy_start (data=0x7f926c000ea0) at utils.c:1249
#5 0x00007f927e704e25 in start_thread () from /lib64/libpthread.so.0
#6 0x00007f927d7a5bad in clone () from /lib64/libc.so.6
Thread 10 (Thread 0x7f927bb53700 (LWP 7956)):
#0 0x00007f927e708d42 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000000000050c5ad in __ast_cond_timedwait (filename=0x676480 "threadpool.c", lineno=1163, func=0x676ba4 <__PRETTY_FUNCTION__.8531> "worker_idle", cond_name=0x6766ff "&worker->cond", mutex_name=0x6766f1 "&worker->lock", cond=0x7f926c0011f8, t=0x7f926c001228, abstime=0x7f927bb52c70) at lock.c:653
#2 0x00000000005afd4d in worker_idle (worker=0x7f926c0011f0) at threadpool.c:1163
#3 0x00000000005afa70 in worker_start (arg=0x7f926c0011f0) at threadpool.c:1054
#4 0x00000000005b8722 in dummy_start (data=0x7f926c000a40) at utils.c:1249
#5 0x00007f927e704e25 in start_thread () from /lib64/libpthread.so.0
#6 0x00007f927d7a5bad in clone () from /lib64/libc.so.6
Thread 9 (Thread 0x7f927bad6700 (LWP 7957)):
#0 0x00007f927e708d42 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000000000050c5ad in __ast_cond_timedwait (filename=0x676480 "threadpool.c", lineno=1163, func=0x676ba4 <__PRETTY_FUNCTION__.8531> "worker_idle", cond_name=0x6766ff "&worker->cond", mutex_name=0x6766f1 "&worker->lock", cond=0x7f926c001638, t=0x7f926c001668, abstime=0x7f927bad5c70) at lock.c:653
#2 0x00000000005afd4d in worker_idle (worker=0x7f926c001630) at threadpool.c:1163
#3 0x00000000005afa70 in worker_start (arg=0x7f926c001630) at threadpool.c:1054
#4 0x00000000005b8722 in dummy_start (data=0x7f926c000ea0) at utils.c:1249
#5 0x00007f927e704e25 in start_thread () from /lib64/libpthread.so.0
#6 0x00007f927d7a5bad in clone () from /lib64/libc.so.6
Thread 8 (Thread 0x7f927ba59700 (LWP 7958)):
#0 0x00007f927e708d42 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000000000050c5ad in __ast_cond_timedwait (filename=0x676480 "threadpool.c", lineno=1163, func=0x676ba4 <__PRETTY_FUNCTION__.8531> "worker_idle", cond_name=0x6766ff "&worker->cond", mutex_name=0x6766f1 "&worker->lock", cond=0x7f926c001a78, t=0x7f926c001aa8, abstime=0x7f927ba58c70) at lock.c:653
#2 0x00000000005afd4d in worker_idle (worker=0x7f926c001a70) at threadpool.c:1163
#3 0x00000000005afa70 in worker_start (arg=0x7f926c001a70) at threadpool.c:1054
#4 0x00000000005b8722 in dummy_start (data=0x7f926c000a40) at utils.c:1249
#5 0x00007f927e704e25 in start_thread () from /lib64/libpthread.so.0
#6 0x00007f927d7a5bad in clone () from /lib64/libc.so.6
Thread 7 (Thread 0x7f927b9dc700 (LWP 7959)):
#0 0x00007f927d79af0d in poll () from /lib64/libc.so.6
#1 0x0000000000456601 in listener (unused=0x0) at asterisk.c:1486
#2 0x00000000005b8722 in dummy_start (data=0x19a7470) at utils.c:1249
#3 0x00007f927e704e25 in start_thread () from /lib64/libpthread.so.0
#4 0x00007f927d7a5bad in clone () from /lib64/libc.so.6
Thread 6 (Thread 0x7f927b8e2700 (LWP 7961)):
#0 0x00007f927e70aafb in do_futex_wait.constprop.1 () from /lib64/libpthread.so.0
#1 0x00007f927e70ab8f in __new_sem_wait_slow.constprop.0 () from /lib64/libpthread.so.0
#2 0x00007f927e70ac2b in sem_wait@@GLIBC_2.2.5 () from /lib64/libpthread.so.0
#3 0x00000000005a48d9 in ast_sem_wait (sem=0x19d02f0) at /usr/src/debug/asterisk-16.3.0/include/asterisk/sem.h:61
#4 0x00000000005a49a9 in default_tps_processing_function (data=0x19d0360) at taskprocessor.c:198
#5 0x00000000005b8722 in dummy_start (data=0x19d0460) at utils.c:1249
#6 0x00007f927e704e25 in start_thread () from /lib64/libpthread.so.0
#7 0x00007f927d7a5bad in clone () from /lib64/libc.so.6
Thread 5 (Thread 0x7f927b865700 (LWP 7962)):
#0 0x00007f927e70aafb in do_futex_wait.constprop.1 () from /lib64/libpthread.so.0
#1 0x00007f927e70ab8f in __new_sem_wait_slow.constprop.0 () from /lib64/libpthread.so.0
#2 0x00007f927e70ac2b in sem_wait@@GLIBC_2.2.5 () from /lib64/libpthread.so.0
#3 0x00000000005a48d9 in ast_sem_wait (sem=0x19d0830) at /usr/src/debug/asterisk-16.3.0/include/asterisk/sem.h:61
#4 0x00000000005a49a9 in default_tps_processing_function (data=0x19d08a0) at taskprocessor.c:198
#5 0x00000000005b8722 in dummy_start (data=0x19d09d0) at utils.c:1249
#6 0x00007f927e704e25 in start_thread () from /lib64/libpthread.so.0
#7 0x00007f927d7a5bad in clone () from /lib64/libc.so.6
Thread 4 (Thread 0x7f927b7e8700 (LWP 7963)):
#0 0x00007f927e708995 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000000000050c56f in __ast_cond_wait (filename=0x63b9cf "bridge.c", lineno=4933, func=0x63e670 <__PRETTY_FUNCTION__.18666> "bridge_manager_thread", cond_name=0x63cee1 "&manager->cond", mutex_name=0x63cec0 "ao2_object_get_lockaddr(manager)", cond=0x19e3380, t=0x19e3340) at lock.c:588
#2 0x0000000000477fee in bridge_manager_thread (data=0x19e3380) at bridge.c:4933
#3 0x00000000005b8722 in dummy_start (data=0x19e33e0) at utils.c:1249
#4 0x00007f927e704e25 in start_thread () from /lib64/libpthread.so.0
#5 0x00007f927d7a5bad in clone () from /lib64/libc.so.6
Thread 3 (Thread 0x7f927b6ee700 (LWP 7965)):
#0 0x00007f927e708995 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000000000050c56f in __ast_cond_wait (filename=0x66cd0b "sched.c", lineno=150, func=0x66d028 <__PRETTY_FUNCTION__.12873> "sched_run", cond_name=0x66cd13 "&con->sched_thread->cond", mutex_name=0x66cd00 "&con->lock", cond=0x19e3e88, t=0x19e6f90) at lock.c:588
#2 0x0000000000576282 in sched_run (data=0x19e6f90) at sched.c:150
#3 0x00000000005b8722 in dummy_start (data=0x19e6ef0) at utils.c:1249
#4 0x00007f927e704e25 in start_thread () from /lib64/libpthread.so.0
#5 0x00007f927d7a5bad in clone () from /lib64/libc.so.6
Thread 2 (Thread 0x7f928078e700 (LWP 7954)):
#0 0x00007f927e70aafb in do_futex_wait.constprop.1 () from /lib64/libpthread.so.0
#1 0x00007f927e70ab8f in __new_sem_wait_slow.constprop.0 () from /lib64/libpthread.so.0
#2 0x00007f927e70ac2b in sem_wait@@GLIBC_2.2.5 () from /lib64/libpthread.so.0
#3 0x00000000005a48d9 in ast_sem_wait (sem=0x1987710) at /usr/src/debug/asterisk-16.3.0/include/asterisk/sem.h:61
#4 0x00000000005a49a9 in default_tps_processing_function (data=0x1991980) at taskprocessor.c:198
#5 0x00000000005b8722 in dummy_start (data=0x198a880) at utils.c:1249
#6 0x00007f927e704e25 in start_thread () from /lib64/libpthread.so.0
#7 0x00007f927d7a5bad in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x7f928088a8c0 (LWP 7950)):
#0 0x00000000005a222b in ast_strlen_zero (s=0x1 <Address 0x1 out of bounds>) at /usr/src/debug/asterisk-16.3.0/include/asterisk/strings.h:67
#1 0x00000000005a31bc in ast_vector_string_split (dest=0x1a02b90, input=0x1 <Address 0x1 out of bounds>, delim=0x658c3b ",", flags=0, excludes_cmp=0x434620 <strcasecmp@plt>) at strings.c:405
#2 0x000000000050608e in module_post_register (mod=0x1a02b10) at loader.c:608
#3 0x000000000050913b in load_resource (resource_name=0x1a05000 "codec_speex.so", suppress_logging=1, module_priorities=0x7ffdfc624cf0, required=0, preload=0) at loader.c:1640
#4 0x000000000050ab3e in load_resource_list (load_order=0x7ffdfc624d80, mod_count=0x7ffdfc624d7c) at loader.c:1984
#5 0x000000000050b6d5 in load_modules () at loader.c:2232
#6 0x000000000045d07d in asterisk_daemon (isroot=1, runuser=0x7ffdfc625fc0 "asterisk", rungroup=0x7ffdfc625fa0 "asterisk") at asterisk.c:4146
#7 0x000000000045c6e7 in main (argc=8, argv=0x7ffdfc6272e8) at asterisk.c:3918