Libumem epoll fixes

examples/src/linux/Makefile

@@ -33,7 +33,7 @@ TARGETS =					\
  	x8664_hello_cpp			\
 	x8664_hello_cpp_static	\
 	x8664_cloexec_test		\
-	x8664_linux_onestraw		\
+	x8664_linux_onestraw	\
 	patch_test.bin
 
 .PHONY: all clean
@@ -125,6 +125,7 @@ libpatch_test.so: patch_test.so.h patch_test.so.c
 	$(CC) $(CPPFLAGS) -Wall -s -O0 -shared -fpic -o $@ patch_test.so.c
 
 patch_test.bin: patch_test.bin.c libpatch_test.so
+
 	$(CC) $(CPPFLAGS) $(CFLAGS) -o $@ $^
 x8664_onestraw_server: x8664_linux_onestraw.c
 	$(CC) $(CPPFLAGS) $(CFLAGS) -m64 -o $@ $<

qiling/os/posix/syscall/epoll.py

@@ -1,18 +1,16 @@
+from __future__ import annotations
+
 import select
 
-from typing import TYPE_CHECKING, Dict, List
+from typing import TYPE_CHECKING, Dict, KeysView
 
-from qiling import *
-from qiling.const import *
 from qiling.os.posix.const import *
-from qiling.os.const import *
-from qiling.os.filestruct import ql_file
-from qiling.os.filestruct import PersistentQlFile
+from qiling.os.filestruct import PersistentQlFile, ql_file
 
 
 if TYPE_CHECKING:
+    from qiling import Qiling
     from qiling.os.posix.posix import QlFileDes
-#from qiling.os.posix.posix import QlFileDes
 
 class QlEpollObj:
     def __init__(self, epoll_object: select.epoll):
@@ -24,8 +22,8 @@ def __init__(self, epoll_object: select.epoll):
         self._fds: Dict[int, int] = {}
 
     @property
-    def fds(self) -> List[int]:
-        return list(self._fds.keys())
+    def fds(self) -> KeysView[int]:
+        return self._fds.keys()
 
     @property
     def epoll_instance(self) -> select.epoll:
@@ -38,10 +36,10 @@ def set_eventmask(self, fd: int, newmask: int):
         # the mask for an FD shouldn't ever be undefined
         # as it is set whenever an FD is added for a QlEpollObj instance
 
-        # libumem: resolved elicn feedback
         newmask = self.get_eventmask(fd) | newmask
-        self._fds[fd] = newmask
+
         self._epoll_object.modify(fd, newmask)
+        self._fds[fd] = newmask
 
     def monitor_fd(self, fd: int, eventmask: int) -> None:
         # tell the epoll object to watch the fd arg, looking for events matching the eventmask
@@ -53,34 +51,34 @@ def delist_fd(self, fd: int) -> None:
         self._epoll_object.unregister(fd)
 
     def close(self) -> None:
-        self.epoll_instance.close()
+        self._epoll_object.close()
 
-    def is_present(self, fd: int) -> bool:
-        return fd in self.fds
+
+    def __contains__(self, fd: int) -> bool:
+        """Test whether a specific fd is already being watched by this epoll instance.
+        """
 
 
-def check_epoll_depth(ql_fd_list, epolls_list: List[QlEpollObj], depth: int = 0) -> None:
-    # Recursively checks each epoll instance's 'watched' fds for an instance of
-    # epoll being watched. If a chain of over 5 levels is detected, raise
-    # an exception
+        return fd in self.fds
 
-    if depth >= 5:
-        raise RecursionError
 
-    new_epolls_list = []
+def check_epoll_depth(ql_fd_list: QlFileDes) -> None:
+    """Recursively check each epoll instance's 'watched' fds for an instance of
+    epoll being watched. If a chain of over 5 levels is detected, raise an exception
+    """
 
-    for ent in epolls_list:
-        watched = ent.fds
+    def __visit_obj(obj: QlEpollObj, depth: int):
+        if depth >= 5:
+            raise RecursionError
 
-        for w in watched:
-            obj = ql_fd_list[w]
+        for fd in obj.fds:
+            if isinstance(ql_fd_list[fd], QlEpollObj):
+                __visit_obj(obj, depth + 1)
 
-            if isinstance(obj, QlEpollObj):
-                new_epolls_list.append(obj)
+    for obj in ql_fd_list:
+        if isinstance(obj, QlEpollObj):
+            __visit_obj(obj, 1)
 
-        if new_epolls_list:
-            check_epoll_depth(ql_fd_list, new_epolls_list, depth + 1)
-        new_epolls_list = []
 
 
 def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int):
@@ -114,22 +112,22 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int):
 	# EPOLLWAKEUP (since Linux 3.5)
     #     If EPOLLONESHOT and EPOLLET are clear and the process has the CAP_BLOCK_SUSPEND capability
 
-    # TODO: not sure if qiling supports a way to determine if the target file descriptor is a
-    # directory. 
-    # Here, check against PersistentQlFile is to ensure that polling stdin, stdout, stderr
-    # is supported
-
     fd_obj = ql.os.fd[fd]
 
     if fd_obj is None:
         return -EBADF
 
+    # TODO: not sure if qiling supports a way to determine if the target file descriptor is a
+    # directory. Here, check against PersistentQlFile is to ensure that polling stdin, stdout,
+    # stderr is supported
+
     # The target file fd does not support epoll. This error can occur if fd refers to, for
     # example, a regular file or a directory.
     if isinstance(fd_obj, ql_file) and not isinstance(fd_obj, PersistentQlFile):
         return -EPERM
 
 
+
     # EPOLLEXCLUSIVE was specified in event and fd refers to an epoll instance
     if isinstance(fd_obj, QlEpollObj) and (op & EPOLLEXCLUSIVE):
         return -EINVAL
@@ -140,38 +138,54 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int):
     epolls_list = [fobj for fobj in ql.os.fd if isinstance(fobj, QlEpollObj)]
 
     try:
-        check_epoll_depth(ql.os.fd, epolls_list)
-    # more than five detected?
+        # Necessary to iterate over all possible qiling fds to determine if we have a chain of more
+        # than five epolls monitoring each other This may be removed in the future if the QlOsLinux
+        # class had a separate field reserved for tracking epoll objects.
+        check_epoll_depth(ql.os.fd)
     except RecursionError:
         return -ELOOP
 
-    ql_event = event and ql.mem.read_ptr(event, 4)
-
     if op == EPOLL_CTL_ADD:
         # can't add an fd that's already being waited on
-        if epoll_parent_obj.is_present(fd):
+        if fd in epoll_parent_obj:
             return -EEXIST
 
+        if not event:
+            return -EINVAL
+
+        event_ptr = ql.mem.read_ptr(event)
+        events = ql.mem.read_ptr(event_ptr, 4)
+
+        # EPOLLEXCLUSIVE was specified in event and fd refers to an epoll instance
+        if isinstance(fd_obj, QlEpollObj) and (op & EPOLLEXCLUSIVE):
+            return -EINVAL
+
         # add to list of fds to be monitored with per-fd eventmask register will actual epoll
         # instance and add eventmask accordingly
-        epoll_parent_obj.monitor_fd(fd, ql_event)
+        epoll_parent_obj.monitor_fd(fd, events)
 
     elif op == EPOLL_CTL_DEL:
-        if not epoll_parent_obj.is_present(fd):
+        if fd not in epoll_parent_obj:
             return -ENOENT
 
         # remove from fds list and do so in the underlying epoll instance
         epoll_parent_obj.delist_fd(fd)
 
     elif op == EPOLL_CTL_MOD:
-        if not epoll_parent_obj.is_present(fd):
+        if fd not in epoll_parent_obj:
             return -ENOENT
 
-        # EINVAL op was EPOLL_CTL_MOD and events included EPOLLEXCLUSIVE.
-        if op & EPOLLEXCLUSIVE and fd in epoll_obj.fds:
+        if not event:
+            return -EINVAL
+
+        event_ptr = ql.mem.read_ptr(event)
+        events = ql.mem.read_ptr(event_ptr, 4)
+
+        # EPOLLEXCLUSIVE cannot be set on MOD operation, only on ADD
+        if events & EPOLLEXCLUSIVE:
             return -EINVAL
 
-        epoll_parent_obj.set_eventmask(fd, ql_event)
+        epoll_parent_obj.set_eventmask(fd, events)
 
     return 0
 
@@ -200,20 +214,21 @@ def ql_syscall_epoll_wait(ql: Qiling, epfd: int, epoll_events: int, maxevents: i
     if epoll_obj is None:
         return -EBADF
 
-
     ready_fds = epoll_obj.poll(timeout, maxevents)
 
     # Each tuple in ready_fds consists of (file descriptor, eventmask) so we iterate
     # through these to indicate which fds are ready and 'why'
 
-    for i, (fd, interest_mask) in enumerate(ready_fds):
+    for i, (fd, events) in enumerate(ready_fds):
         # if no longer interested in this fd, remove from list
-        if interest_mask & EPOLLONESHOT:
+        if events & EPOLLONESHOT:
             epoll_parent_obj.delist_fd(fd)
 
-        data = ql.pack32(interest_mask) + ql.pack(fd)
+        # FIXME: the data packed after events should be the one passed on epoll_ctl
+        # for that specific fd. currently this does not align with the spec
+        data = ql.pack32(events) + ql.pack64(fd)
         offset = len(data) * i
-        # Resolved elicn remark, ql_event was dead code
+
         ql.mem.write(epoll_events + offset, data)
 
     return len(ready_fds)

tests/test_elf.py

@@ -772,63 +772,67 @@ def test_elf_linux_x8664_path_traversion(self):
         self.assertNotIn("root\n", ql.os.stdout.read().decode("utf-8"))
 
         del ql
-
-    """
-    This tests a sample binary that (e)polls on stdin
-    and echos back the output. Upon receiving 'stop', it
-    will exit. 
-    """
-    @unittest.skip("TODO: Stdin hijacking doesn't work as expected")
+
+    @unittest.skip("stdin hijacking doesn't work as expected")
     def test_elf_linux_x8664_epoll_simple(self):
-        # epoll-0 source: https://github.com/maxasm/epoll-c/blob/main/main.c
+        # This tests a sample binary that (e)polls on stdin and echos back the output. Upon
+        # receiving 'stop', it will exit.
+        #
+        # epoll-0 tkaen from: https://github.com/maxasm/epoll-c/blob/main/main.c
+
         rootfs = "../examples/rootfs/x8664_linux"
         argv = r"../examples/rootfs/x8664_linux/bin/x8664_linux_epoll_0".split()
         ql = Qiling(argv, rootfs, verbose=QL_VERBOSE.DEBUG)
+
         #ql.os.stdin = pipe.SimpleInStream(0)
         ql.os.stdin.write(b'echo\n')
-        ql.os.stdin.write(b"stop\n") # signal to exit gracefully
-        ql.run()
-        self.assertIn("echo\n", ql.os.stdout.read().decode("utf-8"))
-        del ql
-    """
-    This tests a simple server that uses epoll
-    to wait for data, then prints it out. It has been
-    modified to exit after data has been received; instead
-    of a typical server operation that reads requests indefinitely.
-
-    It listens on port 8000, and a separate thread is spawned in
-    order to test how the server handles a 'hello world' input.
-    The server prints out whatever it receives, so the assert
-    statement checks the input is present as expected. 
-    """
-    #@unittest.skip('See PR')
-    def test_elf_linux_x8664_epoll_server(self):
-        # Source for onestraw server: https://github.com/onestraw/epoll-example
+        ql.os.stdin.write(b'stop\n') # signal to exit gracefully
+        ql.run()
 
-        """
-        Note: Without a hook for this syscall, this error fires:
-        TypeError: stat: path should be string, bytes, os.PathLike or integer, not NoneType
-        """
-        def hook_newfstatat(ql: Qiling, dirfd: int, pathname: POINTER, statbuf: POINTER, flags:int):
+        self.assertIn(b'echo\n', ql.os.stdout.read())
+        del ql
+
+    def test_elf_linux_x8664_epoll_server(self):
+        # This tests a simple server that uses epoll to wait for data, then prints it out. It has
+        # been modified to exit after data has been received; instead of a typical server operation
+        # that reads requests indefinitely.
+        #
+        # It listens on port 8000, and a separate thread is spawned in order to test how the server
+        # handles a 'hello world' input. The server prints out whatever it receives, so the assert
+        # statement checks the input is present as expected.
+        #
+        # onestraw server taken from: https://github.com/onestraw/epoll-example
+
+        # Note: Without a hook for this syscall, this error fires:
+        # TypeError: stat: path should be string, bytes, os.PathLike or integer, not NoneType
+        def hook_newfstatat(ql: Qiling, dirfd: int, pathname: int, statbuf: int, flags: int):
             return 0
+
         def client():
-            time.sleep(3) # give time for the server to listen
+            # give time for the server to listen
+            time.sleep(3)
+
             s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-            dest = ("127.0.0.1", 8000)
-            s.connect(dest)
-            test = b"hello world"
-            s.send(test)
+            s.connect(("127.0.0.1", 8000))
+            s.send(b"hello world")
             s.close()
-        client_thread = threading.Thread(target=client, daemon=True)
-        client_thread.start()
+
         rootfs = "../examples/rootfs/x8664_linux_glibc2.39"
         argv = r"../examples/rootfs/x8664_linux/bin/x8664_onestraw_server s".split() # s means 'server mode'
-        ql = Qiling(argv, rootfs, multithread=False, verbose=QL_VERBOSE.DEBUG)
-        ql.os.set_syscall("newfstatat",hook_newfstatat, QL_INTERCEPT.CALL)
+
+        ql = Qiling(argv, rootfs, verbose=QL_VERBOSE.DEBUG)
+        ql.os.set_syscall("newfstatat", hook_newfstatat, QL_INTERCEPT.CALL)
         ql.os.stdout = pipe.SimpleOutStream(1) # server prints data received to stdout
         ql.filter = '^data:'
+
+        client_thread = threading.Thread(target=client, daemon=True)
+        client_thread.start()
+
         ql.run()
-        self.assertIn('hello world', ql.os.stdout.read().decode("utf-8"))
+
+        self.assertIn(b'hello world', ql.os.stdout.read())
         del ql
+
+
 if __name__ == "__main__":
     unittest.main()