Update syshook ret definition (#109)

Author: rmalmain

Diff for: include/libafl/exit.h

@@ -57,7 +57,7 @@ struct libafl_exit_reason {
         struct libafl_exit_reason_breakpoint breakpoint; // kind == BREAKPOINT
         struct libafl_exit_reason_custom_insn
             custom_insn;                           // kind == CUSTOM_INSN
-        struct libafl_exit_reason_crash crash; // kind == CRASH
+        struct libafl_exit_reason_crash crash;     // kind == CRASH
         struct libafl_exit_reason_timeout timeout; // kind == TIMEOUT
     } data;
 };

Diff for: include/libafl/hooks/syscall.h

@@ -10,18 +10,34 @@
 #include "libafl/exit.h"
 #include "libafl/hook.h"
 
-struct syshook_ret {
-    target_ulong retval;
-    bool skip_syscall;
+enum libafl_syshook_ret_tag {
+    LIBAFL_SYSHOOK_RUN,
+    LIBAFL_SYSHOOK_SKIP,
 };
 
+// Representation of a pre-syscall hook result.
+// It is associated with the LibAFL enum `SyscallHookResult`.
+// Any change made here should be also propagated to the Rust enum.
+struct libafl_syshook_ret {
+    enum libafl_syshook_ret_tag tag;
+    union {
+        target_ulong syshook_skip_retval;
+    };
+};
+
+typedef struct libafl_syshook_ret (*libafl_pre_syscall_cb)(
+    uint64_t data, int sys_num, target_ulong arg0, target_ulong arg1,
+    target_ulong arg2, target_ulong arg3, target_ulong arg4, target_ulong arg5,
+    target_ulong arg6, target_ulong arg7);
+
+typedef target_ulong (*libafl_post_syscall_cb)(
+    uint64_t data, target_ulong ret, int sys_num, target_ulong arg0,
+    target_ulong arg1, target_ulong arg2, target_ulong arg3, target_ulong arg4,
+    target_ulong arg5, target_ulong arg6, target_ulong arg7);
+
 struct libafl_pre_syscall_hook {
     // functions
-    struct syshook_ret (*callback)(uint64_t data, int sys_num,
-                                   target_ulong arg0, target_ulong arg1,
-                                   target_ulong arg2, target_ulong arg3,
-                                   target_ulong arg4, target_ulong arg5,
-                                   target_ulong arg6, target_ulong arg7);
+    libafl_pre_syscall_cb callback;
 
     // data
     uint64_t data;
@@ -33,11 +49,7 @@ struct libafl_pre_syscall_hook {
 
 struct libafl_post_syscall_hook {
     // functions
-    target_ulong (*callback)(uint64_t data, target_ulong ret, int sys_num,
-                             target_ulong arg0, target_ulong arg1,
-                             target_ulong arg2, target_ulong arg3,
-                             target_ulong arg4, target_ulong arg5,
-                             target_ulong arg6, target_ulong arg7);
+    libafl_post_syscall_cb callback;
 
     // data
     uint64_t data;
@@ -47,20 +59,10 @@ struct libafl_post_syscall_hook {
     struct libafl_post_syscall_hook* next;
 };
 
-size_t libafl_add_pre_syscall_hook(
-    struct syshook_ret (*callback)(uint64_t data, int sys_num,
-                                   target_ulong arg0, target_ulong arg1,
-                                   target_ulong arg2, target_ulong arg3,
-                                   target_ulong arg4, target_ulong arg5,
-                                   target_ulong arg6, target_ulong arg7),
-    uint64_t data);
-size_t libafl_add_post_syscall_hook(
-    target_ulong (*callback)(uint64_t data, target_ulong ret, int sys_num,
-                             target_ulong arg0, target_ulong arg1,
-                             target_ulong arg2, target_ulong arg3,
-                             target_ulong arg4, target_ulong arg5,
-                             target_ulong arg6, target_ulong arg7),
-    uint64_t data);
+size_t libafl_add_pre_syscall_hook(libafl_pre_syscall_cb callback,
+                                   uint64_t data);
+size_t libafl_add_post_syscall_hook(libafl_post_syscall_cb callback,
+                                    uint64_t data);
 
 int libafl_qemu_remove_pre_syscall_hook(size_t num);
 int libafl_qemu_remove_post_syscall_hook(size_t num);

Diff for: libafl/hooks/syscall.c

@@ -9,13 +9,8 @@ size_t libafl_post_syscall_hooks_num = 0;
 GEN_REMOVE_HOOK1(pre_syscall)
 GEN_REMOVE_HOOK1(post_syscall)
 
-size_t libafl_add_pre_syscall_hook(
-    struct syshook_ret (*callback)(uint64_t data, int sys_num,
-                                   target_ulong arg0, target_ulong arg1,
-                                   target_ulong arg2, target_ulong arg3,
-                                   target_ulong arg4, target_ulong arg5,
-                                   target_ulong arg6, target_ulong arg7),
-    uint64_t data)
+size_t libafl_add_pre_syscall_hook(libafl_pre_syscall_cb callback,
+                                   uint64_t data)
 {
     struct libafl_pre_syscall_hook* hook =
         calloc(sizeof(struct libafl_pre_syscall_hook), 1);
@@ -57,14 +52,16 @@ bool libafl_hook_syscall_pre_run(CPUArchState* env, int num, abi_long arg1,
     struct libafl_pre_syscall_hook* h = libafl_pre_syscall_hooks;
     while (h) {
         // no null check
-        struct syshook_ret hook_ret = h->callback(
+        struct libafl_syshook_ret hook_ret = h->callback(
             h->data, num, (target_ulong)arg1, (target_ulong)arg2,
             (target_ulong)arg3, (target_ulong)arg4, (target_ulong)arg5,
             (target_ulong)arg6, (target_ulong)arg7, (target_ulong)arg8);
-        if (hook_ret.skip_syscall) {
+
+        if (hook_ret.tag == LIBAFL_SYSHOOK_SKIP) {
             skip_syscall = true;
-            *ret = (abi_ulong)hook_ret.retval;
+            *ret = (abi_ulong)hook_ret.syshook_skip_retval;
         }
+
         h = h->next;
     }
 
@@ -77,6 +74,7 @@ void libafl_hook_syscall_post_run(int num, abi_long arg1, abi_long arg2,
                                   abi_long* ret)
 {
     struct libafl_post_syscall_hook* p = libafl_post_syscall_hooks;
+
     while (p) {
         // no null check
         *ret = (abi_ulong)p->callback(p->data, (target_ulong)*ret, num,

Diff for: libafl/user.c

@@ -58,13 +58,12 @@ uint64_t libafl_set_brk(uint64_t new_brk)
     return old_brk;
 }
 
-void libafl_set_return_on_crash(bool return_on_crash) {
+void libafl_set_return_on_crash(bool return_on_crash)
+{
     libafl_return_on_crash = return_on_crash;
 }
 
-bool libafl_get_return_on_crash(void) {
-    return libafl_return_on_crash;
-}
+bool libafl_get_return_on_crash(void) { return libafl_return_on_crash; }
 
 #ifdef AS_LIB
 void libafl_qemu_init(int argc, char** argv)