grsecurity freeze on hyper-v guest

[btnet]

Hello, I am having an issue here.
I installed the 3.12.8 kernel, which is vanilla for grsec-testing as you already know. Also this is a windows-hyper-v-guest
Once I patch the kernel for grsec and recompile, without adding grsec and without modification on config, after boot the system will freeze at some scsi point and will not boot.
Below is the kernel config without grsec, and I am waiting for instructions on what logs to provide and how can I further investigate the matter.

https://dpaste.de/u7mA

[PaX Team]

dmesg from working/non-working boot attempts would be a good start i think (via netconsole or a serial console).

[pablo]

I tracked this problem down as I was hitting the same issue and it turns out it's related to the hypercall mechanism.

Hyper-V allocates an executable page doing:
virtaddr = __vmalloc(PAGE_SIZE, GFP_KERNEL, PAGE_KERNEL_RX);

and passes the physical address to the hypervisor doing:
hypercall_msr.guest_physical_address = vmalloc_to_pfn(virtaddr);
wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);

then the do_hypercall function just calls into this virtaddr for doing the communcation (it uses an SGDT [ecx] instruction for triggering the VMM).
The problem is that it hangs while trying to execute that instruction from that vmalloc'd memory.

I tried to change PAGE_KERNEL_RX to PAGE_KERNEL_EXEC, didn't work...

My fix, which I'd like to know if it is kosher enough, was to change this code to do what Xen does: reserve some memory on the .text section for the hypercall (see ENTRY(hypercall_page) on xen-head.S).

So I just reserved a PAGE_SIZE buffer same way and adapted the code to use it.

This is the patch for 3.14.17:

Code: Select all

diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h
index cd9c419..28935a7 100644
--- a/arch/x86/include/asm/mshyperv.h
+++ b/arch/x86/include/asm/mshyperv.h
@@ -10,6 +10,7 @@ struct ms_hyperv_info {
 };
 
 extern struct ms_hyperv_info ms_hyperv;
+extern struct { char _entry[PAGE_SIZE]; } hv_hypercall_page;
 
 void hyperv_callback_vector(void);
 #ifdef CONFIG_TRACING
diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c
index 832d05a..6215fcb 100644
--- a/arch/x86/kernel/cpu/mshyperv.c
+++ b/arch/x86/kernel/cpu/mshyperv.c
@@ -30,6 +30,7 @@
 
 struct ms_hyperv_info ms_hyperv;
 EXPORT_SYMBOL_GPL(ms_hyperv);
+EXPORT_SYMBOL_GPL(hv_hypercall_page);
 
 static uint32_t  __init ms_hyperv_platform(void)
 {
diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index c7dec74..2aec589 100644
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -608,6 +608,12 @@ ENTRY(phys_base)
    /* This must match the first entry in level2_kernel_pgt */
    .quad   0x0000000000000000
 
+.pushsection .text
+        .balign PAGE_SIZE
+ENTRY(hv_hypercall_page)
+        .skip PAGE_SIZE
+.popsection
+
 #include "../../x86/xen/xen-head.S"
 
    .section .rodata,"a",@progbits
diff --git a/drivers/hv/hv.c b/drivers/hv/hv.c
index 61dba6c..764971e 100644
--- a/drivers/hv/hv.c
+++ b/drivers/hv/hv.c
@@ -29,6 +29,7 @@
 #include <linux/version.h>
 #include <linux/interrupt.h>
 #include <asm/hyperv.h>
+#include <asm/mshyperv.h>
 #include "hyperv_vmbus.h"
 
 /* The one and only */
@@ -133,7 +134,6 @@ int hv_init(void)
 {
    int max_leaf;
    union hv_x64_msr_hypercall_contents hypercall_msr;
-   void *virtaddr = NULL;
 
    memset(hv_context.synic_event_page, 0, sizeof(void *) * NR_CPUS);
    memset(hv_context.synic_message_page, 0,
@@ -154,14 +154,9 @@ int hv_init(void)
    /* See if the hypercall page is already set */
    rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
 
-   virtaddr = __vmalloc(PAGE_SIZE, GFP_KERNEL, PAGE_KERNEL_RX);
-
-   if (!virtaddr)
-      goto cleanup;
-
+   hypercall_msr.as_uint64 = __pa(&hv_hypercall_page);
    hypercall_msr.enable = 1;
-
-   hypercall_msr.guest_physical_address = vmalloc_to_pfn(virtaddr);
+   hypercall_msr.reserved = 0;
    wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
 
    /* Confirm that hypercall page did get setup. */
@@ -171,20 +166,16 @@ int hv_init(void)
    if (!hypercall_msr.enable)
       goto cleanup;
 
-   hv_context.hypercall_page = virtaddr;
+   hv_context.hypercall_page = &hv_hypercall_page;
 
    return 0;
 
 cleanup:
-   if (virtaddr) {
       if (hypercall_msr.enable) {
          hypercall_msr.as_uint64 = 0;
          wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
       }
 
-      vfree(virtaddr);
-   }
-
    return -ENOTSUPP;
 }
 
@@ -203,7 +194,6 @@ void hv_cleanup(void)
    if (hv_context.hypercall_page) {
       hypercall_msr.as_uint64 = 0;
       wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
-      vfree(hv_context.hypercall_page);
       hv_context.hypercall_page = NULL;
    }
 }


[PaX Team]

hmm, i thought i had fixed the problem between KERNEXEC and hyperv some time ago already, the __vmalloc call should allocate from the module region where executable maps are allowed. can you check what address you get on your system?

[pablo]

added
pr_err("virtaddr=%p phys=%p", virtaddr, vmalloc_to_pfn(virtaddr));

before wrmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);

it says:
hv_vmbus: virtaddr=ffffc9000063d000 phys=00000000000efc07

full serial log: http://pastebin.com/cR0YwDs4

the cpu is stuck on RIP: 0010:[<ffffc9000063d000>]

[pablo]

the only difference I see between the __vmalloc call from hv_bus and from the load_module path is that this last uses vmalloc_exec, which adds the __GFP_HIGHMEM flag and added the writable permission (which I tried per se and didn't work). I'll try using vmalloc_exec instead that __vmalloc call.

[pablo]

ok, now I got the problem. __vmalloc (or vmalloc_exec for that sake) allocates from the VMALLOC_START - VMALLOC_END segment, while modules does from MODULES_VADDR - MODULES_END. The problem is that there's no exported interface to __vmalloc_node_range that would allow me to change the allocation range. I see there's one case on the BPF jit thingy but it's still core kernel code.

So to build this as a module I need to export __vmalloc_node_range, what you think about that?

Second patch option:

Code: Select all

diff --git a/drivers/hv/hv.c b/drivers/hv/hv.c
index 61dba6c..32258d0 100644
--- a/drivers/hv/hv.c
+++ b/drivers/hv/hv.c
@@ -154,7 +154,15 @@ int hv_init(void)
        /* See if the hypercall page is already set */
        rdmsrl(HV_X64_MSR_HYPERCALL, hypercall_msr.as_uint64);
 
-       virtaddr = __vmalloc(PAGE_SIZE, GFP_KERNEL, PAGE_KERNEL_RX);
+#ifdef CONFIG_PAX_KERNEXEC
+       virtaddr = __vmalloc_node_range(PAGE_SIZE, 1, MODULES_VADDR, MODULES_END,
+                               GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO, PAGE_KERNEL,
+                               NUMA_NO_NODE, __builtin_return_address(0));
+#else
+       virtaddr = __vmalloc_node_range(PAGE_SIZE, 1, MODULES_VADDR, MODULES_END,
+                               GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO, PAGE_KERNEL_EXEC,
+                               NUMA_NO_NODE, __builtin_return_address(0));
+#endif
 
        if (!virtaddr)
                goto cleanup;
diff --git a/mm/vmalloc.c b/mm/vmalloc.c
index 991ff6a..509f51b 100644
--- a/mm/vmalloc.c
+++ b/mm/vmalloc.c
@@ -1783,6 +1783,7 @@ fail:
                          real_size);
        return NULL;
 }
+EXPORT_SYMBOL(__vmalloc_node_range);
 
 /**
  *     __vmalloc_node  -  allocate virtually contiguous memory


[PaX Team]

ok, now I got the problem. __vmalloc (or vmalloc_exec for that sake) allocates from the VMALLOC_START - VMALLOC_END segment, while modules does from MODULES_VADDR - MODULES_END.

in PaX __vmalloc_node_range has a special case for executable mapping requests and calls __get_vm_area_node with VM_KERNEXEC that will then allocate from the module area. now all this logic is under KERNEXEC that i'm guessing isn't enabled in your config, so you could try that first then i'll think about making it not depend on KERNEXEC.

[pablo]

Yeah, I need to make it work on Xen too, so no KERNEXEC for me

Thank you for your time and your pointers!