From dbf213846bb9e309afe51ca40d8c7ddfb966695d Mon Sep 17 00:00:00 2001 From: elicn Date: Wed, 30 Apr 2025 13:47:38 +0300 Subject: [PATCH 1/6] Turn check_epoll_depth into a prefix visitor --- qiling/os/posix/syscall/epoll.py | 45 ++++++++++++-------------------- 1 file changed, 17 insertions(+), 28 deletions(-) diff --git a/qiling/os/posix/syscall/epoll.py b/qiling/os/posix/syscall/epoll.py index 7b318bd40..423c75d95 100644 --- a/qiling/os/posix/syscall/epoll.py +++ b/qiling/os/posix/syscall/epoll.py @@ -59,30 +59,22 @@ def is_present(self, fd: int) -> bool: return fd in self.fds -def check_epoll_depth(ql_fd_list: QlFileDes, epolls_list: List[QlEpollObj], depth: int = 0) -> None: - # Recursively checks each epoll instance's 'watched' fds for an instance of - # epoll being watched. If a chain of over 5 levels is detected, return 1, - # which will return ELOOP in ql_syscall_epoll_wait - - if depth >= 5: - raise RecursionError - - new_epolls_list = [] - - for ent in epolls_list: - watched = ent.fds - - for w in watched: - obj = ql_fd_list[w] +def check_epoll_depth(ql_fd_list: QlFileDes) -> None: + """Recursively check each epoll instance's 'watched' fds for an instance of + epoll being watched. If a chain of over 5 levels is detected, raise an exception + """ - if isinstance(obj, QlEpollObj): - new_epolls_list.append(obj) + def __visit_obj(obj: QlEpollObj, depth: int): + if depth >= 5: + raise RecursionError - # elicn: new_epolls_list is not cleared between loop iterations, rather it keeps - # aggregating items from previous iterations. is this what we want? + for fd in obj.fds: + if isinstance(ql_fd_list[fd], QlEpollObj): + __visit_obj(obj, depth + 1) - if new_epolls_list: - check_epoll_depth(ql_fd_list, new_epolls_list, depth + 1) + for obj in ql_fd_list: + if isinstance(obj, QlEpollObj): + __visit_obj(obj, 1) def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): @@ -138,14 +130,11 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): if isinstance(fd_obj, QlEpollObj) and (op & EPOLLEXCLUSIVE): return -EINVAL - # Necessary to iterate over all possible qiling fds to determine if we have a chain of more - # than five epolls monitoring each other This may be removed in the future if the QlOsLinux - # class had a separate field for reserved for tracking epoll objects. - epolls_list = [fobj for fobj in ql.os.fd if isinstance(fobj, QlEpollObj)] - try: - check_epoll_depth(ql.os.fd, epolls_list) - # more than five detected? + # Necessary to iterate over all possible qiling fds to determine if we have a chain of more + # than five epolls monitoring each other This may be removed in the future if the QlOsLinux + # class had a separate field reserved for tracking epoll objects. + check_epoll_depth(ql.os.fd) except RecursionError: return -ELOOP From 96674e1e7aba182ed62b6a62c7fa98a0f120adcc Mon Sep 17 00:00:00 2001 From: elicn Date: Wed, 30 Apr 2025 13:49:32 +0300 Subject: [PATCH 2/6] Use container semantics instead of is_present --- qiling/os/posix/syscall/epoll.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/qiling/os/posix/syscall/epoll.py b/qiling/os/posix/syscall/epoll.py index 423c75d95..5fa898cad 100644 --- a/qiling/os/posix/syscall/epoll.py +++ b/qiling/os/posix/syscall/epoll.py @@ -55,7 +55,10 @@ def delist_fd(self, fd: int) -> None: def close(self) -> None: self.epoll_instance.close() - def is_present(self, fd: int) -> bool: + def __contains__(self, fd: int) -> bool: + """Test whether a specific fd is already being watched by this epoll instance. + """ + return fd in self.fds @@ -142,7 +145,7 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): if op == EPOLL_CTL_ADD: # can't add an fd that's already being waited on - if epoll_parent_obj.is_present(fd): + if fd in epoll_parent_obj: return -EEXIST # add to list of fds to be monitored with per-fd eventmask register will actual epoll @@ -150,14 +153,14 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): epoll_parent_obj.monitor_fd(fd, ql_event) elif op == EPOLL_CTL_DEL: - if not epoll_parent_obj.is_present(fd): + if fd not in epoll_parent_obj: return -ENOENT # remove from fds list and do so in the underlying epoll instance epoll_parent_obj.delist_fd(fd) elif op == EPOLL_CTL_MOD: - if not epoll_parent_obj.is_present(fd): + if fd not in epoll_parent_obj: return -ENOENT # EINVAL op was EPOLL_CTL_MOD and events included EPOLLEXCLUSIVE. From f1081275e5d193b44cf1d9e8eda63a1b15f4cf7f Mon Sep 17 00:00:00 2001 From: elicn Date: Wed, 30 Apr 2025 13:50:17 +0300 Subject: [PATCH 3/6] Fix events pointer handling --- qiling/os/posix/syscall/epoll.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/qiling/os/posix/syscall/epoll.py b/qiling/os/posix/syscall/epoll.py index 5fa898cad..08d96a438 100644 --- a/qiling/os/posix/syscall/epoll.py +++ b/qiling/os/posix/syscall/epoll.py @@ -141,16 +141,24 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): except RecursionError: return -ELOOP - ql_event = event and ql.mem.read_ptr(event, 4) - if op == EPOLL_CTL_ADD: # can't add an fd that's already being waited on if fd in epoll_parent_obj: return -EEXIST + if not event: + return -EINVAL + + event_ptr = ql.mem.read_ptr(event) + events = ql.mem.read_ptr(event_ptr, 4) + + # EPOLLEXCLUSIVE was specified in event and fd refers to an epoll instance + if isinstance(fd_obj, QlEpollObj) and (op & EPOLLEXCLUSIVE): + return -EINVAL + # add to list of fds to be monitored with per-fd eventmask register will actual epoll # instance and add eventmask accordingly - epoll_parent_obj.monitor_fd(fd, ql_event) + epoll_parent_obj.monitor_fd(fd, events) elif op == EPOLL_CTL_DEL: if fd not in epoll_parent_obj: @@ -163,11 +171,17 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): if fd not in epoll_parent_obj: return -ENOENT - # EINVAL op was EPOLL_CTL_MOD and events included EPOLLEXCLUSIVE. - if op & EPOLLEXCLUSIVE and fd in epoll_obj.fds: + if not event: + return -EINVAL + + event_ptr = ql.mem.read_ptr(event) + events = ql.mem.read_ptr(event_ptr, 4) + + # EPOLLEXCLUSIVE cannot be set on MOD operation, only on ADD + if events & EPOLLEXCLUSIVE: return -EINVAL - epoll_parent_obj.set_eventmask(fd, ql_event) + epoll_parent_obj.set_eventmask(fd, events) return 0 From 0c96871d35210cbe97009aee324931eaeff087dc Mon Sep 17 00:00:00 2001 From: elicn Date: Wed, 30 Apr 2025 13:51:41 +0300 Subject: [PATCH 4/6] Fix returned events array --- qiling/os/posix/syscall/epoll.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/qiling/os/posix/syscall/epoll.py b/qiling/os/posix/syscall/epoll.py index 08d96a438..8dadd212f 100644 --- a/qiling/os/posix/syscall/epoll.py +++ b/qiling/os/posix/syscall/epoll.py @@ -221,7 +221,9 @@ def ql_syscall_epoll_wait(ql: Qiling, epfd: int, epoll_events: int, maxevents: i if interest_mask & EPOLLONESHOT: epoll_parent_obj.delist_fd(fd) - data = ql.pack32(interest_mask) + ql.pack(fd) + # FIXME: the data packed after events should be the one passed on epoll_ctl + # for that specific fd. currently this does not align with the spec + data = ql.pack32(interest_mask) + ql.pack64(fd) offset = len(data) * i # Resolved elicn remark, ql_event was dead code ql.mem.write(epoll_events + offset, data) From bf83e9c2cf6bef787ce80a6470eb6e4a1d55d12d Mon Sep 17 00:00:00 2001 From: elicn Date: Wed, 30 Apr 2025 13:52:16 +0300 Subject: [PATCH 5/6] Cleanup and minor cosmetics --- qiling/os/posix/syscall/epoll.py | 48 +++++++++++++------------------- 1 file changed, 19 insertions(+), 29 deletions(-) diff --git a/qiling/os/posix/syscall/epoll.py b/qiling/os/posix/syscall/epoll.py index 8dadd212f..d3eb0abed 100644 --- a/qiling/os/posix/syscall/epoll.py +++ b/qiling/os/posix/syscall/epoll.py @@ -1,18 +1,17 @@ +from __future__ import annotations + import select -from typing import TYPE_CHECKING, Dict, List +from typing import TYPE_CHECKING, Dict, KeysView -from qiling import * -from qiling.const import * from qiling.os.posix.const import * -from qiling.os.const import * -from qiling.os.filestruct import ql_file -from qiling.os.filestruct import PersistentQlFile +from qiling.os.filestruct import PersistentQlFile, ql_file if TYPE_CHECKING: + from qiling import Qiling from qiling.os.posix.posix import QlFileDes -from qiling.os.posix.posix import QlFileDes + class QlEpollObj: def __init__(self, epoll_object: select.epoll): @@ -24,8 +23,8 @@ def __init__(self, epoll_object: select.epoll): self._fds: Dict[int, int] = {} @property - def fds(self) -> List[int]: - return list(self._fds.keys()) + def fds(self) -> KeysView[int]: + return self._fds.keys() @property def epoll_instance(self) -> select.epoll: @@ -38,10 +37,10 @@ def set_eventmask(self, fd: int, newmask: int): # the mask for an FD shouldn't ever be undefined # as it is set whenever an FD is added for a QlEpollObj instance - # libumem: resolved elicn feedback newmask = self.get_eventmask(fd) | newmask - self._fds[fd] = newmask + self._epoll_object.modify(fd, newmask) + self._fds[fd] = newmask def monitor_fd(self, fd: int, eventmask: int) -> None: # tell the epoll object to watch the fd arg, looking for events matching the eventmask @@ -53,7 +52,7 @@ def delist_fd(self, fd: int) -> None: self._epoll_object.unregister(fd) def close(self) -> None: - self.epoll_instance.close() + self._epoll_object.close() def __contains__(self, fd: int) -> bool: """Test whether a specific fd is already being watched by this epoll instance. @@ -111,28 +110,20 @@ def ql_syscall_epoll_ctl(ql: Qiling, epfd: int, op: int, fd: int, event: int): # EPOLLWAKEUP (since Linux 3.5) # If EPOLLONESHOT and EPOLLET are clear and the process has the CAP_BLOCK_SUSPEND capability - # TODO: not sure if qiling supports a way to determine if the target file descriptor is a - # directory. - # Here, check against PersistentQlFile is to ensure that polling stdin, stdout, stderr - # is supported - fd_obj = ql.os.fd[fd] if fd_obj is None: return -EBADF + # TODO: not sure if qiling supports a way to determine if the target file descriptor is a + # directory. Here, check against PersistentQlFile is to ensure that polling stdin, stdout, + # stderr is supported + # The target file fd does not support epoll. This error can occur if fd refers to, for # example, a regular file or a directory. if isinstance(fd_obj, ql_file) and not isinstance(fd_obj, PersistentQlFile): return -EPERM - # elicn: not sure how the following condition even possible after we checked that op can - # be only one of EPOLL_CTL_{ADD,DEL,MOD} (originally checked with a dict) - - # EPOLLEXCLUSIVE was specified in event and fd refers to an epoll instance - if isinstance(fd_obj, QlEpollObj) and (op & EPOLLEXCLUSIVE): - return -EINVAL - try: # Necessary to iterate over all possible qiling fds to determine if we have a chain of more # than five epolls monitoring each other This may be removed in the future if the QlOsLinux @@ -210,22 +201,21 @@ def ql_syscall_epoll_wait(ql: Qiling, epfd: int, epoll_events: int, maxevents: i if epoll_obj is None: return -EBADF - ready_fds = epoll_obj.poll(timeout, maxevents) # Each tuple in ready_fds consists of (file descriptor, eventmask) so we iterate # through these to indicate which fds are ready and 'why' - for i, (fd, interest_mask) in enumerate(ready_fds): + for i, (fd, events) in enumerate(ready_fds): # if no longer interested in this fd, remove from list - if interest_mask & EPOLLONESHOT: + if events & EPOLLONESHOT: epoll_parent_obj.delist_fd(fd) # FIXME: the data packed after events should be the one passed on epoll_ctl # for that specific fd. currently this does not align with the spec - data = ql.pack32(interest_mask) + ql.pack64(fd) + data = ql.pack32(events) + ql.pack64(fd) offset = len(data) * i - # Resolved elicn remark, ql_event was dead code + ql.mem.write(epoll_events + offset, data) return len(ready_fds) From 5a87323df352fc4925e1b7c61169cddbbd71781c Mon Sep 17 00:00:00 2001 From: elicn Date: Wed, 30 Apr 2025 13:52:40 +0300 Subject: [PATCH 6/6] Tidy up tests --- examples/src/linux/Makefile | 3 +- tests/test_elf.py | 86 +++++++++++++++++++------------------ 2 files changed, 47 insertions(+), 42 deletions(-) diff --git a/examples/src/linux/Makefile b/examples/src/linux/Makefile index 8a26e589f..006ea4907 100644 --- a/examples/src/linux/Makefile +++ b/examples/src/linux/Makefile @@ -33,7 +33,7 @@ TARGETS = \ x8664_hello_cpp \ x8664_hello_cpp_static \ x8664_cloexec_test \ - x8664_linux_onestraw \ + x8664_linux_onestraw \ patch_test.bin .PHONY: all clean @@ -125,6 +125,7 @@ libpatch_test.so: patch_test.so.h patch_test.so.c $(CC) $(CPPFLAGS) -Wall -s -O0 -shared -fpic -o $@ patch_test.so.c patch_test.bin: patch_test.bin.c libpatch_test.so + $(CC) $(CPPFLAGS) $(CFLAGS) -o $@ $^ x8664_onestraw_server: x8664_linux_onestraw.c $(CC) $(CPPFLAGS) $(CFLAGS) -m64 -o $@ $< diff --git a/tests/test_elf.py b/tests/test_elf.py index aae738985..8928e7a59 100644 --- a/tests/test_elf.py +++ b/tests/test_elf.py @@ -772,63 +772,67 @@ def test_elf_linux_x8664_path_traversion(self): self.assertNotIn("root\n", ql.os.stdout.read().decode("utf-8")) del ql - - """ - This tests a sample binary that (e)polls on stdin - and echos back the output. Upon receiving 'stop', it - will exit. - """ - @unittest.skip("TODO: Stdin hijacking doesn't work as expected") + + @unittest.skip("stdin hijacking doesn't work as expected") def test_elf_linux_x8664_epoll_simple(self): - # epoll-0 source: https://github.com/maxasm/epoll-c/blob/main/main.c + # This tests a sample binary that (e)polls on stdin and echos back the output. Upon + # receiving 'stop', it will exit. + # + # epoll-0 tkaen from: https://github.com/maxasm/epoll-c/blob/main/main.c + rootfs = "../examples/rootfs/x8664_linux" argv = r"../examples/rootfs/x8664_linux/bin/x8664_linux_epoll_0".split() ql = Qiling(argv, rootfs, verbose=QL_VERBOSE.DEBUG) + #ql.os.stdin = pipe.SimpleInStream(0) ql.os.stdin.write(b'echo\n') - ql.os.stdin.write(b"stop\n") # signal to exit gracefully - ql.run() - self.assertIn("echo\n", ql.os.stdout.read().decode("utf-8")) - del ql - """ - This tests a simple server that uses epoll - to wait for data, then prints it out. It has been - modified to exit after data has been received; instead - of a typical server operation that reads requests indefinitely. - - It listens on port 8000, and a separate thread is spawned in - order to test how the server handles a 'hello world' input. - The server prints out whatever it receives, so the assert - statement checks the input is present as expected. - """ - #@unittest.skip('See PR') - def test_elf_linux_x8664_epoll_server(self): - # Source for onestraw server: https://github.com/onestraw/epoll-example + ql.os.stdin.write(b'stop\n') # signal to exit gracefully + ql.run() - """ - Note: Without a hook for this syscall, this error fires: - TypeError: stat: path should be string, bytes, os.PathLike or integer, not NoneType - """ - def hook_newfstatat(ql: Qiling, dirfd: int, pathname: POINTER, statbuf: POINTER, flags:int): + self.assertIn(b'echo\n', ql.os.stdout.read()) + del ql + + def test_elf_linux_x8664_epoll_server(self): + # This tests a simple server that uses epoll to wait for data, then prints it out. It has + # been modified to exit after data has been received; instead of a typical server operation + # that reads requests indefinitely. + # + # It listens on port 8000, and a separate thread is spawned in order to test how the server + # handles a 'hello world' input. The server prints out whatever it receives, so the assert + # statement checks the input is present as expected. + # + # onestraw server taken from: https://github.com/onestraw/epoll-example + + # Note: Without a hook for this syscall, this error fires: + # TypeError: stat: path should be string, bytes, os.PathLike or integer, not NoneType + def hook_newfstatat(ql: Qiling, dirfd: int, pathname: int, statbuf: int, flags: int): return 0 + def client(): - time.sleep(3) # give time for the server to listen + # give time for the server to listen + time.sleep(3) + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - dest = ("127.0.0.1", 8000) - s.connect(dest) - test = b"hello world" - s.send(test) + s.connect(("127.0.0.1", 8000)) + s.send(b"hello world") s.close() - client_thread = threading.Thread(target=client, daemon=True) - client_thread.start() + rootfs = "../examples/rootfs/x8664_linux_glibc2.39" argv = r"../examples/rootfs/x8664_linux/bin/x8664_onestraw_server s".split() # s means 'server mode' - ql = Qiling(argv, rootfs, multithread=False, verbose=QL_VERBOSE.DEBUG) - ql.os.set_syscall("newfstatat",hook_newfstatat, QL_INTERCEPT.CALL) + + ql = Qiling(argv, rootfs, verbose=QL_VERBOSE.DEBUG) + ql.os.set_syscall("newfstatat", hook_newfstatat, QL_INTERCEPT.CALL) ql.os.stdout = pipe.SimpleOutStream(1) # server prints data received to stdout ql.filter = '^data:' + + client_thread = threading.Thread(target=client, daemon=True) + client_thread.start() + ql.run() - self.assertIn('hello world', ql.os.stdout.read().decode("utf-8")) + + self.assertIn(b'hello world', ql.os.stdout.read()) del ql + + if __name__ == "__main__": unittest.main()