Apache2

[dflt]

I have since migrated from apache2 to nginx+php5-fpm. And the same segfaults came back (so It's definetly not apache2 related, but every time the segfaults occur, It takes a lot of killall/kill to massacre the staled apache2 processes). So I dug deep and found out that one of the buggy php scripts tend to return large result sets, which often tries to allocate more memory than memory_limit set in php.ini. When PHP reaches the limit and tries to allocate another 32 bytes of memory, pax shuts it down.

[PaX Team]

When PHP reaches the limit and tries to allocate another 32 bytes of memory, pax shuts it down.

it's the kernel that gives the application a SIGSEGV on the NULL deref, not PaX . i guess the code allocating memory doesn't check for a NULL return value and just blindly uses the resulting pointer, you should probably report it to upstream...

[dflt]

Bombs away! Now we'll see what the PHP guys are going to do about it.

Thanks Pipax team!

[pioklo]

Hello !

We are using 2.6.32.59 kernel with latest grsec
we have in logs many segfaults in apache

May 23 02:19:28 s75 kernel: [68199.654828] grsec: From x.x.x.x: signal 11 sent to /usr/sbin/httpd[httpd:11304] uid/euid:1000/1000 gid/egid:1000/1000, parent /usr/sbin/httpd[httpd:11250] uid/euid:0/0 gid/egid:0/0 by /usr/sbin/httpd[httpd:11955] uid/euid:1000/1000 gid/egid:1000/1000, parent /usr/sbin/httpd[httpd:11250] uid/euid:0/0 gid/egid:0/0

here is some debug

Cannot access memory at address 0x7ff4e3558108
(gdb) bt full
#0 0x00007ff4e14af0bd in ?? ()
No symbol table info available.
Cannot access memory at address 0x7fff75e9c360
(gdb) x/8i $pc
0x7ff4e14af0bd: Cannot access memory at address 0x7ff4e14af0bd
(gdb) x/8x $sp
0x7fff75e9c360: Cannot access memory at address 0x7fff75e9c360
(gdb) info reg
rax 0xfffffffffffffe00 -512
rbx 0x1c07a58 29391448
rcx 0xffffffffffffffff -1
rdx 0x1 1
rsi 0x7fff75e9c37f 140735171642239
rdi 0xa 10
rbp 0x261edee0 0x261edee0
rsp 0x7fff75e9c360 0x7fff75e9c360
r8 0x7fff75e9c230 140735171641904
r9 0x7ff4dc9ba700 140689649936128
r10 0x0 0
r11 0x293 659
r12 0x4 4
r13 0x7fff75e9c5e8 140735171642856
r14 0x4c5803 5003267
r15 0x4c57f8 5003256
rip 0x7ff4e14af0bd 0x7ff4e14af0bd
eflags 0x293 [ CF AF SF IF ]
cs 0x33 51
ss 0x2b 43
ds 0x0 0
es 0x0 0
fs 0x0 0
gs 0x0 0
fctrl 0x37f 895
fstat 0x0 0
ftag 0xffff 65535
fiseg 0x0 0
fioff 0x0 0
foseg 0x0 0
fooff 0x0 0
fop 0x0 0
mxcsr 0x1fa0 [ PE IM DM ZM OM UM PM ]
(gdb)

Regards
Piotr

[pioklo]

I have new debug

Code: Select all

Program terminated with signal 6, Aborted.
#0  0x00007f5d94c9d165 in raise () from /lib/libc.so.6
(gdb) bt full
#0  0x00007f5d94c9d165 in raise () from /lib/libc.so.6
No symbol table info available.
#1  0x00007f5d94c9ff70 in abort () from /lib/libc.so.6
No symbol table info available.
#2  0x000000000044be0f in ap_log_assert (szExp=0x4d00cf "preg != NULL", szFile=0x4d00c0 "mod_setenvif.c", nLine=176) at log.c:882
        time_str = "Thu May 24 02:14:19 2012"
#3  0x00000000004617c5 in is_header_regex (cmd=0x7f5d5d67fb30, mconfig=<value optimized out>, fname=0x4d00dc "User-Agent", args=
    0x2686ca23 "gzip-only-text/html") at mod_setenvif.c:176
        preg = 0x0
#4  add_setenvif_core (cmd=0x7f5d5d67fb30, mconfig=<value optimized out>, fname=0x4d00dc "User-Agent", args=0x2686ca23 "gzip-only-text/html")
    at mod_setenvif.c:355
        regex = 0x269ba530 "^Mozilla/4"
        simple_pattern = <value optimized out>
        feature = <value optimized out>
        sconf = <value optimized out>
        new = 0x269ba0d0
        var = <value optimized out>
        i = <value optimized out>
        beenhere = <value optimized out>
        icase = 0
#5  0x00000000004482ee in invoke_cmd (cmd=0x4d04b0, parms=0x7f5d5d67fb30, mconfig=0x269ba0a8, args=0x2686ca18 "^Mozilla/4 gzip-only-text/html") at config.c:757
        w = <value optimized out>
        w2 = <value optimized out>
        w3 = <value optimized out>
        errmsg = <value optimized out>
#6  0x00000000004485a2 in ap_walk_config_sub (current=0x2686c9d8, parms=0x7f5d5d67fb30, section_vector=0x26877ce8) at config.c:1163
        dir_config = 0x0
        cmd = 0xe53
        ml = <value optimized out>
        dir = <value optimized out>
#7  ap_walk_config (current=0x2686c9d8, parms=0x7f5d5d67fb30, section_vector=0x26877ce8) at config.c:1196
        errmsg = <value optimized out>
        oldconfig = 0x0
#8  0x0000000000449514 in ap_parse_htaccess (result=<value optimized out>, r=0x26d9d1b0, override=31, override_opts=255, d=<value optimized out>, access_name=
    0x3431359 "") at config.c:1827
        errmsg = 0x0
        temptree = 0x2686c9d8
        f = 0x26868928
        parms = {info = 0x0, override = 31, limited = -1, limited_xmethods = 0x0, xlimited = 0x0, config_file = 0x26868928, directive = 0x2686c9d8, pool =
    0x26d9d138, temp_pool = 0x26d9d138, server = 0x17277920, path = 0x268677e0 "/home/ajsit80/domains/futbolbezbarier.org/public_html/", cmd = 0x4d04b0,
          context = 0x26877ce8, err_directive = 0x2661b0d8, override_opts = 255}
        filename = 0x26867828 "/home/ajsit80/domains/futbolbezbarier.org/public_html/.htaccess"
        cache = <value optimized out>
        dc = 0x26877ce8
        status = <value optimized out>
#9  0x00000000004439de in ap_directory_walk (r=0x26d9d1b0) at request.c:879
        htaccess_conf = 0x0
        res = <value optimized out>
---Type <return> to continue, or q <return> to quit---
        seg_name = 0x26867162 "public_html/"
        temp_slash = 1
        opts = {opts = 98 'b', add = 34 '"', remove = 129 '\201', override = 31 '\037', override_opts = 255 '\377'}
        thisinfo = {pool = 0x26d9d138, valid = 7598960, protection = 1877, filetype = APR_DIR, user = 2047, group = 2049, inode = 78228062, device = 2069,
          nlink = 8, size = 4096, csize = 4096, atime = 1331206990000000, mtime = 1332176119000000, ctime = 1332176119000000, fname =
    0x26867138 "/home/ajsit80/domains/futbolbezbarier.org/public_html/", name = 0x26d9e0a0 "\270\356\331&", filehand = 0x26d9d1b0}
        save_path_info = <value optimized out>
        matches = 0
        last_walk = 0x268670a0
        this_dir = <value optimized out>
        seg = 6
        sec_idx = 8
        filename_len = 54
        now_merged = 0x26867248
        sconf = 0x1d23e710
        num_sec = 9
        cache = <value optimized out>
        entry_dir = 0x268670d0 "/home/ajsit80/domains/futbolbezbarier.org/public_html/test/wp-content/themes/colorway/css/"
        rv = <value optimized out>
#10 0x0000000000440709 in core_map_to_storage (r=0xe53) at core.c:3634
        access_status = <value optimized out>
#11 0x0000000000442090 in ap_run_map_to_storage (r=0x26d9d1b0) at request.c:69
        n = 5
        rv = 0
#12 0x00000000004440e8 in ap_process_request_internal (r=0x26d9d1b0) at request.c:150
        file_req = 0
        access_status = 0
#13 0x0000000000491298 in ap_process_request (r=0x26d9d1b0) at http_request.c:280
        access_status = 0
#14 0x000000000048e210 in ap_process_http_connection (c=0x2685bc78) at http_core.c:190
        r = 0x26d9d1b0
        csd = 0x0
#15 0x000000000044e540 in ap_run_process_connection (c=0x2685bc78) at connection.c:43
        n = 1
        rv = 0
#16 0x00000000004c22c7 in process_socket (thd=<value optimized out>, dummy=<value optimized out>) at worker.c:544
        current_conn = <value optimized out>
        conn_id = <value optimized out>
        csd = 18762
        sbh = 0x2685bc70
#17 worker_thread (thd=<value optimized out>, dummy=<value optimized out>) at worker.c:894
        process_slot = 0
        thread_slot = 101
        csd = 0x2685ba60
        bucket_alloc = <value optimized out>
        last_ptrans = <value optimized out>
        ptrans = 0x2685b9d8
        rv = <value optimized out>
---Type <return> to continue, or q <return> to quit---
        is_idle = <value optimized out>
#18 0x00007f5d951d68ba in start_thread () from /lib/libpthread.so.0
No symbol table info available.
#19 0x00007f5d94d3a02d in clone () from /lib/libc.so.6
No symbol table info available.
#20 0x0000000000000000 in ?? ()
No symbol table info available.
(gdb) x/8i $pc
0x7f5d94c9d165 <raise+53>:      cmp    $0xfffffffffffff000,%rax
0x7f5d94c9d16b <raise+59>:      ja     0x7f5d94c9d182 <raise+82>
0x7f5d94c9d16d <raise+61>:      repz retq
0x7f5d94c9d16f <raise+63>:      nop
0x7f5d94c9d170 <raise+64>:      test   %eax,%eax
0x7f5d94c9d172 <raise+66>:      jg     0x7f5d94c9d155 <raise+37>
0x7f5d94c9d174 <raise+68>:      test   $0x7fffffff,%eax
0x7f5d94c9d179 <raise+73>:      jne    0x7f5d94c9d192 <raise+98>
(gdb) x/8x $sp
0x7f5d5d67f658: 0x94c9ff70      0x00007f5d      0x004d00cf      0x00000000
0x7f5d5d67f668: 0x5d67f7b0      0x00007f5d      0x000000b0      0x00000000
(gdb) info reg
rax            0x0      0
rbx            0x4d00c0 5046464
rcx            0xffffffffffffffff       -1
rdx            0x6      6
rsi            0xfd5    4053
rdi            0xe53    3667
rbp            0x4d00cf 0x4d00cf
rsp            0x7f5d5d67f658   0x7f5d5d67f658
r8             0x0      0
r9             0x0      0
r10            0x8      8
r11            0x206    518
r12            0x7f5d5d67f7b0   140038975780784
r13            0xb0     176
r14            0x0      0
r15            0x269ba530       647734576
rip            0x7f5d94c9d165   0x7f5d94c9d165 <raise+53>
eflags         0x206    [ PF IF ]
cs             0x33     51
ss             0x2b     43
ds             0x0      0
es             0x0      0
fs             0x0      0
gs             0x0      0
fctrl          0x37f    895
fstat          0x0      0
ftag           0xffff   65535
fiseg          0x0      0
fioff          0x0      0
foseg          0x0      0
fooff          0x0      0
fop            0x0      0
mxcsr          0x1fa0   [ PE IM DM ZM OM UM PM ]
(gdb)


[PaX Team]

#2 0x000000000044be0f in ap_log_assert (szExp=0x4d00cf "preg != NULL", szFile=0x4d00c0 "mod_setenvif.c", nLine=176) at log.c:882
time_str = "Thu May 24 02:14:19 2012"

looks like an apache module problem, you should check out what the code expects there and how the assert can trigger in your environment.