From 47b66423946359ccf6e23a862d5ebd50951bf248 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Wed, 8 Nov 2023 16:20:06 -0800 Subject: [PATCH 01/17] Initial socket bootstrap --- src/CMakeLists.txt | 2 + src/axl_svc.h | 25 ++++++++ src/axl_svc_client.c | 70 +++++++++++++++++++++ src/axl_svc_server.c | 145 +++++++++++++++++++++++++++++++++++++++++++ test/CMakeLists.txt | 2 + test/axl_svc_test.c | 23 +++++++ 6 files changed, 267 insertions(+) create mode 100644 src/axl_svc.h create mode 100644 src/axl_svc_client.c create mode 100644 src/axl_svc_server.c create mode 100644 test/axl_svc_test.c diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3dcc9d0..fefd673 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -20,6 +20,8 @@ LIST(APPEND libaxl_srcs axl_err.c axl_io.c axl_util.c + axl_svc_client.c + axl_svc_server.c ) IF(HAVE_PTHREADS) diff --git a/src/axl_svc.h b/src/axl_svc.h new file mode 100644 index 0000000..4b16e96 --- /dev/null +++ b/src/axl_svc.h @@ -0,0 +1,25 @@ +#ifndef __AXLSVC_H__ +#define __AXLSVC_H__ + +#define AXLSVC_TRUE 1 +#define AXLSVC_FALSE 0 +#define AXLSVC_DEFAULT_PORT 8888 + +#if defined(__cplusplus) +extern "C" { +#endif + +typedef struct { +} axlsvc_request_t; + +typedef struct { +} axlsvc_response_t; + +int axlsvc_server(); +int axlsvc_client(); + +#if defined(__cplusplus) +extern "C" } +#endif + +#endif /* __AXLSVC_H__ */ diff --git a/src/axl_svc_client.c b/src/axl_svc_client.c new file mode 100644 index 0000000..bebe201 --- /dev/null +++ b/src/axl_svc_client.c @@ -0,0 +1,70 @@ +/**************************************************************************/ +/* Generic client example is used with connection-oriented server designs */ +/**************************************************************************/ +#include +#include +#include +#include +#include +#include +#include + +#include "axl_svc.h" + +int axlsvc_client() +{ + int len, rc; + int sockfd; + char send_buf[80]; + char recv_buf[80]; + struct sockaddr_in server; + + unsigned short port = AXLSVC_DEFAULT_PORT; + struct hostent *hostnm; /* server host name information */ + + + hostnm = gethostbyname("rzwhippet18"); + if (hostnm == (struct hostent *) 0) { + perror("Gethostbyname failed\n"); + return -1; + } + + if ( (sockfd = socket(AF_INET, SOCK_STREAM, 0)) < 0) { + perror("socket"); + return -1; + } + + memset(&server, 0, sizeof(server)); + server.sin_family = AF_INET; + server.sin_port = htons(port); + server.sin_addr.s_addr = *((unsigned long *)hostnm->h_addr); + + if ( ( rc = connect(sockfd, (struct sockaddr *)&server, sizeof(server)) ) < 0) { + perror("connect"); + close(sockfd); + return -1; + } + printf("Connect completed.\n"); + + printf("Enter message to be sent:\n"); + fgets(send_buf, 80, stdin); + + len = send(sockfd, send_buf, strlen(send_buf) + 1, 0); + if (len != strlen(send_buf) + 1) { + perror("send"); + close(sockfd); + return -1; + } + printf("%d bytes sent\n", len); + + len = recv(sockfd, recv_buf, sizeof(recv_buf), 0); + if (len != strlen(send_buf) + 1) { + perror("recv"); + close(sockfd); + exit(-1); + } + printf("%d bytes received: (%s)\n", len, recv_buf); + + close(sockfd); + return 0; +} diff --git a/src/axl_svc_server.c b/src/axl_svc_server.c new file mode 100644 index 0000000..3037522 --- /dev/null +++ b/src/axl_svc_server.c @@ -0,0 +1,145 @@ +#include +#include //strlen +#include +#include +#include //close +#include //close +#include +#include +#include +#include //FD_SET, FD_ISSET, FD_ZERO macros + +#include "axl_svc.h" + +static void execute_command(const char* pname); + +int axlsvc_server() +{ + const int max_bufsize = 4096; + const int max_clients = 30; + + int opt = AXLSVC_TRUE; + int service_socket, addrlen, new_socket, client_socket[max_clients]; + int activity, valread, sd; + int max_sd; + struct sockaddr_in address; + + char buffer[max_bufsize]; //data buffer of 4K + fd_set readfds; //set of socket descriptors + + for (int i = 0; i < max_clients; i++) + client_socket[i] = 0; + + if ((service_socket = socket(AF_INET, SOCK_STREAM, 0)) == 0) { + perror("socket failed"); + exit(EXIT_FAILURE); + } + + if (setsockopt(service_socket, SOL_SOCKET, SO_REUSEADDR, (char *)&opt, sizeof(opt)) < 0 ) { + perror("setsockopt"); + exit(EXIT_FAILURE); + } + + address.sin_family = AF_INET; + address.sin_addr.s_addr = INADDR_ANY; + address.sin_port = htons(AXLSVC_DEFAULT_PORT); + + if (bind(service_socket, (struct sockaddr *)&address, sizeof(address)) < 0) { + perror("bind failed"); + exit(EXIT_FAILURE); + } + printf("Listener on port %d \n", AXLSVC_DEFAULT_PORT); + + if (listen(service_socket, max_clients) < 0) { + perror("listen"); + exit(EXIT_FAILURE); + } + + addrlen = sizeof(address); + puts("Waiting for connections ..."); + + while(AXLSVC_TRUE) { + FD_ZERO(&readfds); + FD_SET(service_socket, &readfds); + max_sd = service_socket; + + for ( int i = 0 ; i < max_clients ; i++) { + sd = client_socket[i]; // socket descriptor + + if (sd > 0) + FD_SET(sd, &readfds); + + if (sd > max_sd) + max_sd = sd; + } + + activity = select( max_sd + 1 , &readfds , NULL , NULL , NULL); + + if ((activity < 0) && (errno!=EINTR)) + printf("select error"); + + // If something happened on the service socket, then its an incoming connection + if (FD_ISSET(service_socket, &readfds)) { + if ((new_socket = accept(service_socket, (struct sockaddr *)&address, (socklen_t*)&addrlen)) < 0) { + perror("accept"); + exit(EXIT_FAILURE); + } + + //inform user of socket number - used in send and receive commands + printf("New connection , socket fd is %d , ip is : %s , port : %d\n", + new_socket , inet_ntoa(address.sin_addr) , ntohs(address.sin_port)); + + //add new socket to array of sockets + for (int i = 0; i < max_clients; i++) { + //if position is empty + if( client_socket[i] == 0 ) { + client_socket[i] = new_socket; + printf("Adding to list of sockets as %d\n" , i); + break; + } + } + } + + for ( int i = 0; i < max_clients; i++) { + sd = client_socket[i]; + + if (FD_ISSET(sd , &readfds)) { + if ((valread = read( sd , buffer, 1024)) == 0) { + getpeername(sd , (struct sockaddr*)&address , (socklen_t*)&addrlen); + printf("Host disconnected , ip %s , port %d \n", + inet_ntoa(address.sin_addr) , ntohs(address.sin_port)); + + close( sd ); + client_socket[i] = 0; + } else { + for (int i = 0; i < valread; i++) + if (buffer[i] == '\r' || buffer[i] == '\n') + buffer[i] = ' '; + buffer[valread] = '\0'; + // execute_command(buffer); + send(sd , buffer , strlen(buffer) , 0 ); + } + } + } + } + + return 0; +} + +static void execute_command(const char* pname) +{ + FILE *p; + int ch; + + if ((p = popen(pname, "r")) == NULL) { + perror(pname); + return; + } + + printf("%s\n", pname); + + while( ( ch = fgetc(p)) != EOF ) { + putchar(ch); + } + pclose(p); +} diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 79cce0d..bf967d3 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -17,9 +17,11 @@ ELSE() ENDIF() ADD_EXECUTABLE(axl_cp ${axl_test_srcs}) +ADD_EXECUTABLE(axl_svc_test axl_svc_test.c) ADD_EXECUTABLE(test_config test_config.c) TARGET_LINK_LIBRARIES(axl_cp ${axl_lib}) +TARGET_LINK_LIBRARIES(axl_svc_test ${axl_lib}) TARGET_LINK_LIBRARIES(test_config ${axl_lib}) ################ diff --git a/test/axl_svc_test.c b/test/axl_svc_test.c new file mode 100644 index 0000000..82097c4 --- /dev/null +++ b/test/axl_svc_test.c @@ -0,0 +1,23 @@ +#include +#include "../src/axl_svc.h" + +int main(int argc , char *argv[]) +{ + int rval = -1; + int is_server = AXLSVC_FALSE; + + for (int i = 1; i < argc; i++) { + if (strcmp(argv[i], "server") == 0) { + is_server=AXLSVC_TRUE; + } + } + + if (is_server) { + rval = axlsvc_server(); + } + else { + rval = axlsvc_client(); + } + + return rval; +} From 75ce5b4f5787dee2313fb7d872d357e88f501bc5 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Sun, 12 Nov 2023 16:52:41 -0800 Subject: [PATCH 02/17] Checkpoint --- src/axl_svc.h | 2 +- src/axl_svc_server.c | 4 ++-- test/axl_svc_test.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/axl_svc.h b/src/axl_svc.h index 4b16e96..23f6ae0 100644 --- a/src/axl_svc.h +++ b/src/axl_svc.h @@ -15,7 +15,7 @@ typedef struct { typedef struct { } axlsvc_response_t; -int axlsvc_server(); +int run_server(); int axlsvc_client(); #if defined(__cplusplus) diff --git a/src/axl_svc_server.c b/src/axl_svc_server.c index 3037522..07a3055 100644 --- a/src/axl_svc_server.c +++ b/src/axl_svc_server.c @@ -13,7 +13,7 @@ static void execute_command(const char* pname); -int axlsvc_server() +int run_server() { const int max_bufsize = 4096; const int max_clients = 30; @@ -58,7 +58,7 @@ int axlsvc_server() addrlen = sizeof(address); puts("Waiting for connections ..."); - while(AXLSVC_TRUE) { + while (1) { FD_ZERO(&readfds); FD_SET(service_socket, &readfds); max_sd = service_socket; diff --git a/test/axl_svc_test.c b/test/axl_svc_test.c index 82097c4..d6c1bb6 100644 --- a/test/axl_svc_test.c +++ b/test/axl_svc_test.c @@ -13,7 +13,7 @@ int main(int argc , char *argv[]) } if (is_server) { - rval = axlsvc_server(); + rval = run_server(); } else { rval = axlsvc_client(); From a94b3ec31f3adf943c02da2a419ae51e7b67c606 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Tue, 21 Nov 2023 14:34:58 -0800 Subject: [PATCH 03/17] First contact from SCR client to AXL Service --- src/CMakeLists.txt | 9 ++- src/axl.c | 28 +++++++ src/axl_internal.h | 1 + src/axl_mpi.c | 57 +------------- src/axl_service.h | 38 ++++++++++ src/axl_service_client.c | 50 ++++++++++++ src/axl_service_server.c | 159 +++++++++++++++++++++++++++++++++++++++ 7 files changed, 287 insertions(+), 55 deletions(-) create mode 100644 src/axl_service.h create mode 100644 src/axl_service_client.c create mode 100644 src/axl_service_server.c diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 3dcc9d0..51aa22f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -23,7 +23,14 @@ LIST(APPEND libaxl_srcs ) IF(HAVE_PTHREADS) - LIST(APPEND libaxl_srcs axl_pthread.c) + LIST(APPEND libaxl_srcs + axl_pthread.c + axl_service_client.c + ) + ADD_EXECUTABLE(axl_service axl_service_server.c) + TARGET_LINK_LIBRARIES(axl_service PUBLIC axl_static_o ${AXL_EXTERNAL_LIBS}) + TARGET_INCLUDE_DIRECTORIES(axl_service PUBLIC $ $) + INSTALL(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/axl_service DESTINATION ${CMAKE_INSTALL_LIBEXECDIR}) ENDIF(HAVE_PTHREADS) IF(BBAPI_FOUND) diff --git a/src/axl.c b/src/axl.c index f674e74..dc06d5b 100644 --- a/src/axl.c +++ b/src/axl.c @@ -30,6 +30,9 @@ /* xfer methods */ #include "axl_sync.h" +/* (Optional) service functions */ +#include "axl_service.h" + #ifdef HAVE_PTHREADS #include "axl_pthread.h" #endif /* HAVE_PTHREAD */ @@ -260,6 +263,27 @@ int AXL_Init (void) axl_make_directories = atoi(val); } + /* If the user has set both the AXL_SERVICE_HOST and AXL_SERVICE_PORT environment + * variables, then they are expecting to use the AXL Service rather than the library + * included with the SCR library. + */ + char* axl_service_host = NULL; + int axl_service_port = -1; + + if ( (val = getenv("AXL_SERVICE_HOST")) != NULL) { + axl_service_host = strdup(val); + + if ( (val = getenv("AXL_SERVICE_PORT")) != NULL) { + axl_service_port = atoi(val); + + if (axlsvc_client_init(axl_service_host, (unsigned short)axl_service_port)) { + axl_use_service = 1; + } + } + + free(axl_service_host); + } + /* initialize our flag on whether to first copy files to temporary names with extension */ axl_use_extension = 0; val = getenv("AXL_USE_EXTENSION"); @@ -293,6 +317,10 @@ int AXL_Finalize (void) } #endif + if (axl_use_service) { + axlsvc_client_finalize(); + } + /* decrement reference count and free data structures on last call */ axl_init_count--; if (axl_init_count == 0) { diff --git a/src/axl_internal.h b/src/axl_internal.h index 10d97c6..4f58545 100644 --- a/src/axl_internal.h +++ b/src/axl_internal.h @@ -63,6 +63,7 @@ extern int axl_rank; /* attaches function name, file name, and line number to error messages * https://gcc.gnu.org/onlinedocs/cpp/Variadic-Macros.html */ +#define AXL_ABORT(exitcode, format, ...) axl_abort(exitcode, format " @ %s %s:%d", ##__VA_ARGS__, __func__, __FILE__, __LINE__) #define AXL_ERR(format, ...) axl_err(format " @ %s %s:%d", ##__VA_ARGS__, __func__, __FILE__, __LINE__) #define AXL_DBG(level, format, ...) axl_dbg(level, format " @ %s %s:%d", ##__VA_ARGS__, __func__, __FILE__, __LINE__) diff --git a/src/axl_mpi.c b/src/axl_mpi.c index 07d9719..f5f4cd1 100644 --- a/src/axl_mpi.c +++ b/src/axl_mpi.c @@ -81,9 +81,11 @@ int AXL_Finalize_comm ( return rc; } +#include + int AXL_Create_comm ( axl_xfer_t type, /**< [IN] - AXL transfer type (AXL_XFER_SYNC, AXL_XFER_PTHREAD, etc) */ - const char* name, + const char* name, const char* file, MPI_Comm comm) /**< [IN] - communicator used for coordination and flow control */ { @@ -144,59 +146,6 @@ int AXL_Dispatch_comm ( int id, /**< [IN] - transfer hander ID returned from AXL_Create */ MPI_Comm comm) /**< [IN] - communicator used for coordination and flow control */ { -#if 0 - /* lookup transfer info for the given id */ - kvtree* file_list = NULL; - axl_xfer_t xtype = AXL_XFER_NULL; - axl_xfer_state_t xstate = AXL_XFER_STATE_NULL; - if (axl_get_info(id, &file_list, &xtype, &xstate) != AXL_SUCCESS) { - AXL_ERR("Could not find transfer info for UID %d", id); - return AXL_FAILURE; - } - - /* check that handle is in correct state to dispatch */ - if (xstate != AXL_XFER_STATE_CREATED) { - AXL_ERR("Invalid state to dispatch UID %d", id); - return AXL_FAILURE; - } - kvtree_util_set_int(file_list, AXL_KEY_STATE, (int)AXL_XFER_STATE_DISPATCHED); -#endif - -#if 0 - /* create destination directories for each file */ - if (axl_make_directories) { - /* count number of files we have */ - kvtree* file_list = kvtree_get_kv_int(axl_file_lists, AXL_KEY_HANDLE_UID, id); - kvtree* files_hash = kvtree_get(file_list, AXL_KEY_FILES); - int num_files = kvtree_size(files_hash); - - /* allocate pointer for each one */ - const char** files = (const char**) AXL_MALLOC(num_files * sizeof(char*)); - - /* set pointer to each file */ - int i; - char* dest; - kvtree_elem* elem; - while ((elem = axl_get_next_path(id, elem, NULL, &dest))) { - files[i] = dest; - i++; - } - - /* create directories */ - axl_create_dirs(num_files, files, comm); - - /* free list of files */ - axl_free2(&files); - } - - /* TODO: this is hacky */ - /* delegate remaining work to regular dispatch, - * but disable mkdir since we already did that */ - int make_dir = axl_make_directories; - axl_make_directories = 0; - int rc = AXL_Dispatch(id); - axl_make_directories = make_dir; -#endif /* delegate remaining work to regular dispatch */ int rc = AXL_Dispatch(id); diff --git a/src/axl_service.h b/src/axl_service.h new file mode 100644 index 0000000..04b1bad --- /dev/null +++ b/src/axl_service.h @@ -0,0 +1,38 @@ +#ifndef __AXLSVC_H__ +#define __AXLSVC_H__ + +#include + +extern int axl_use_service; /* whether to use AXL service instead of library */ + +typedef enum { + AXLSVC_AXL_CONFIG = 0, /* payload is config ktree file path */ +} axlsvc_request_t; + +typedef struct { + axlsvc_request_t request; + ssize_t payload_length; +} axlsvc_Request; + +typedef enum { + AXLSVC_SUCCESS = 0, + AXLSVC_FAILURE = -1, +} axlsvc_response_t; + +typedef struct { + axlsvc_response_t response; + ssize_t payload_length; // Optional error/status string +} axlsvc_Response; + +#if defined(__cplusplus) +extern "C" { +#endif + +int axlsvc_client_init(char* host, unsigned short port); +void axlsvc_client_finalize(); + +#if defined(__cplusplus) +extern "C" } +#endif + +#endif /* __AXLSVC_H__ */ diff --git a/src/axl_service_client.c b/src/axl_service_client.c new file mode 100644 index 0000000..4e3adb3 --- /dev/null +++ b/src/axl_service_client.c @@ -0,0 +1,50 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "axl_internal.h" +#include "axl_service.h" + +int axl_use_service = 0; +static int axlsvc_socket = -1; + +int axlsvc_client_init(char* host, unsigned short port) +{ + struct sockaddr_in server; + struct hostent *hostnm = gethostbyname(host); + + if (hostnm == (struct hostent *) 0) { + AXL_ERR("Gethostbyname failed: (%s)", strerror(errno)); + return 0; + } + + if ( (axlsvc_socket = socket(AF_INET, SOCK_STREAM, 0)) < 0) { + AXL_ERR("socket() failed: (%s)", strerror(errno)); + return 0; + } + + memset(&server, 0, sizeof(server)); + server.sin_family = AF_INET; + server.sin_port = htons(port); + server.sin_addr.s_addr = *((unsigned long *)hostnm->h_addr); + + if ( connect(axlsvc_socket, (struct sockaddr *)&server, sizeof(server) ) < 0) { + AXL_ERR("connect() failed: (%s)", strerror(errno)); + close(axlsvc_socket); + return 0; + } + + return 1; // success +} + +void axlsvc_client_finalize() +{ + if (axlsvc_socket >= 0) + close(axlsvc_socket); +} + diff --git a/src/axl_service_server.c b/src/axl_service_server.c new file mode 100644 index 0000000..4b58cf0 --- /dev/null +++ b/src/axl_service_server.c @@ -0,0 +1,159 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "axl_internal.h" +#include "axl_service.h" + +static ssize_t service_request_from_client(int sd) +{ + ssize_t bytecount; + axlsvc_Request req; + axlsvc_Response response; + char* buffer; + + bytecount = axl_read("AXLSVC Client Reqeust", sd, &req, sizeof(req)); + + if (bytecount == 0) { + AXL_DBG(0, "Client for socket %d closed", sd); + return bytecount; + } + + buffer = malloc(req.payload_length); + + bytecount = axl_read("AXLSVC Reqeust Payload", sd, &buffer, req.payload_length); + + if (bytecount != req.payload_length) { + AXL_ABORT(-1, "Unexpected Payload Length: Expected %d, Got %d", req.payload_length, bytecount); + } + + switch (req.request) { + case AXLSVC_AXL_CONFIG: + AXL_DBG(0, "AXLSVC_AXL_CONFIG(kfile=%s", buffer); + response.response = AXLSVC_SUCCESS; + response.payload_length = 0; + bytecount = axl_write_attempt("AXLSVC Response to Client", sd, &response, sizeof(response)); + if (bytecount != sizeof(response)) { + AXL_ABORT(-1, "Unexpected Write Response to client: Expected %d, Got %d", + sizeof(response), bytecount); + } + break; + default: + AXL_ABORT(-1, "AXLSVC Unknown Request Type %d", req.request); + break; + } + + free(buffer); + return bytecount; +} + +int axlsvc_server_run(int port) +{ + static const int axlsvc_max_clients = 16; + int client_socket[axlsvc_max_clients]; + int server_socket; + int opt = 1; + struct sockaddr_in address; + int addrlen; + int new_socket; + fd_set readfds; + int activity, sd; + int max_sd; + + for (int i = 0; i < axlsvc_max_clients; i++) + client_socket[i] = 0; + + if ((server_socket = socket(AF_INET, SOCK_STREAM, 0)) == 0) { + AXL_ABORT(-1, "socket() failed: (%s)", strerror(errno)); + } + + if (setsockopt(server_socket, SOL_SOCKET, SO_REUSEADDR, (char *)&opt, sizeof(opt)) < 0 ) { + AXL_ABORT(-1, "setsockopt() failed: (%s)", strerror(errno)); + } + + address.sin_family = AF_INET; + address.sin_addr.s_addr = INADDR_ANY; + address.sin_port = htons(port); + + if (bind(server_socket, (struct sockaddr *)&address, sizeof(address)) < 0) { + AXL_ABORT(-1, "bind() failed: (%s)", strerror(errno)); + } + + if (listen(server_socket, axlsvc_max_clients) < 0) { + AXL_ABORT(-1, "listen() failed: (%s)", strerror(errno)); + } + + addrlen = sizeof(address); + + while (1) { + FD_ZERO(&readfds); + FD_SET(server_socket, &readfds); + max_sd = server_socket; + + for (int i = 0 ; i < axlsvc_max_clients ; i++) { + sd = client_socket[i]; + + if (sd > 0) + FD_SET(sd, &readfds); + + if (sd > max_sd) + max_sd = sd; + } + + activity = select(max_sd + 1 , &readfds , NULL , NULL , NULL); + + if (activity < 0 && errno != EINTR) { + AXL_ABORT(-1, "select() error: (%s)", strerror(errno)); + } + + // If something happened on the service socket, then its an incoming connection + if (FD_ISSET(server_socket, &readfds)) { + AXL_DBG(0, "Accepting incomming connection"); + if ((new_socket = accept(server_socket, (struct sockaddr *)&address, (socklen_t*)&addrlen)) < 0) { + AXL_ABORT(-1, "accept() error: (%s)", strerror(errno)); + } + + for (int i = 0; i < axlsvc_max_clients; i++) { + if( client_socket[i] == 0 ) { + client_socket[i] = new_socket; + break; + } + } + AXL_DBG(0, "Connection established"); + + } + + for ( int i = 0; i < axlsvc_max_clients; i++) { + sd = client_socket[i]; + + if (FD_ISSET(sd , &readfds)) { + if (service_request_from_client(sd) == 0) { + AXL_DBG(0, "Closing server side socket(%d) to client", sd); + close(sd); + client_socket[i] = 0; + } + } + } + } + + return 0; +} + + +int main(int argc , char *argv[]) +{ + int rval = -1; + + if (argc == 2 && atoi(argv[1]) > 0) { + rval = axlsvc_server_run(atoi(argv[1])); + } else { + fprintf(stderr, "Usage: %s \n", argv[0]); + } + + return rval; +} \ No newline at end of file From 3467ed75e1f96da6291d8b8e885ce07da7ad57f6 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Wed, 22 Nov 2023 10:11:14 -0800 Subject: [PATCH 04/17] Remove axl_svc_test.c --- test/CMakeLists.txt | 2 -- test/axl_svc_test.c | 23 ----------------------- 2 files changed, 25 deletions(-) delete mode 100644 test/axl_svc_test.c diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index bf967d3..79cce0d 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -17,11 +17,9 @@ ELSE() ENDIF() ADD_EXECUTABLE(axl_cp ${axl_test_srcs}) -ADD_EXECUTABLE(axl_svc_test axl_svc_test.c) ADD_EXECUTABLE(test_config test_config.c) TARGET_LINK_LIBRARIES(axl_cp ${axl_lib}) -TARGET_LINK_LIBRARIES(axl_svc_test ${axl_lib}) TARGET_LINK_LIBRARIES(test_config ${axl_lib}) ################ diff --git a/test/axl_svc_test.c b/test/axl_svc_test.c deleted file mode 100644 index d6c1bb6..0000000 --- a/test/axl_svc_test.c +++ /dev/null @@ -1,23 +0,0 @@ -#include -#include "../src/axl_svc.h" - -int main(int argc , char *argv[]) -{ - int rval = -1; - int is_server = AXLSVC_FALSE; - - for (int i = 1; i < argc; i++) { - if (strcmp(argv[i], "server") == 0) { - is_server=AXLSVC_TRUE; - } - } - - if (is_server) { - rval = run_server(); - } - else { - rval = axlsvc_client(); - } - - return rval; -} From 51093f93f1606701da57e1f737a9b9ee38982a1c Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Wed, 22 Nov 2023 10:22:47 -0800 Subject: [PATCH 05/17] Rename svc to service --- src/CMakeLists.txt | 2 - src/axl_svc.h | 25 -------- src/axl_svc_client.c | 70 --------------------- src/axl_svc_server.c | 145 ------------------------------------------- 4 files changed, 242 deletions(-) delete mode 100644 src/axl_svc.h delete mode 100644 src/axl_svc_client.c delete mode 100644 src/axl_svc_server.c diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index cae7112..51aa22f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -20,8 +20,6 @@ LIST(APPEND libaxl_srcs axl_err.c axl_io.c axl_util.c - axl_svc_client.c - axl_svc_server.c ) IF(HAVE_PTHREADS) diff --git a/src/axl_svc.h b/src/axl_svc.h deleted file mode 100644 index 23f6ae0..0000000 --- a/src/axl_svc.h +++ /dev/null @@ -1,25 +0,0 @@ -#ifndef __AXLSVC_H__ -#define __AXLSVC_H__ - -#define AXLSVC_TRUE 1 -#define AXLSVC_FALSE 0 -#define AXLSVC_DEFAULT_PORT 8888 - -#if defined(__cplusplus) -extern "C" { -#endif - -typedef struct { -} axlsvc_request_t; - -typedef struct { -} axlsvc_response_t; - -int run_server(); -int axlsvc_client(); - -#if defined(__cplusplus) -extern "C" } -#endif - -#endif /* __AXLSVC_H__ */ diff --git a/src/axl_svc_client.c b/src/axl_svc_client.c deleted file mode 100644 index bebe201..0000000 --- a/src/axl_svc_client.c +++ /dev/null @@ -1,70 +0,0 @@ -/**************************************************************************/ -/* Generic client example is used with connection-oriented server designs */ -/**************************************************************************/ -#include -#include -#include -#include -#include -#include -#include - -#include "axl_svc.h" - -int axlsvc_client() -{ - int len, rc; - int sockfd; - char send_buf[80]; - char recv_buf[80]; - struct sockaddr_in server; - - unsigned short port = AXLSVC_DEFAULT_PORT; - struct hostent *hostnm; /* server host name information */ - - - hostnm = gethostbyname("rzwhippet18"); - if (hostnm == (struct hostent *) 0) { - perror("Gethostbyname failed\n"); - return -1; - } - - if ( (sockfd = socket(AF_INET, SOCK_STREAM, 0)) < 0) { - perror("socket"); - return -1; - } - - memset(&server, 0, sizeof(server)); - server.sin_family = AF_INET; - server.sin_port = htons(port); - server.sin_addr.s_addr = *((unsigned long *)hostnm->h_addr); - - if ( ( rc = connect(sockfd, (struct sockaddr *)&server, sizeof(server)) ) < 0) { - perror("connect"); - close(sockfd); - return -1; - } - printf("Connect completed.\n"); - - printf("Enter message to be sent:\n"); - fgets(send_buf, 80, stdin); - - len = send(sockfd, send_buf, strlen(send_buf) + 1, 0); - if (len != strlen(send_buf) + 1) { - perror("send"); - close(sockfd); - return -1; - } - printf("%d bytes sent\n", len); - - len = recv(sockfd, recv_buf, sizeof(recv_buf), 0); - if (len != strlen(send_buf) + 1) { - perror("recv"); - close(sockfd); - exit(-1); - } - printf("%d bytes received: (%s)\n", len, recv_buf); - - close(sockfd); - return 0; -} diff --git a/src/axl_svc_server.c b/src/axl_svc_server.c deleted file mode 100644 index 07a3055..0000000 --- a/src/axl_svc_server.c +++ /dev/null @@ -1,145 +0,0 @@ -#include -#include //strlen -#include -#include -#include //close -#include //close -#include -#include -#include -#include //FD_SET, FD_ISSET, FD_ZERO macros - -#include "axl_svc.h" - -static void execute_command(const char* pname); - -int run_server() -{ - const int max_bufsize = 4096; - const int max_clients = 30; - - int opt = AXLSVC_TRUE; - int service_socket, addrlen, new_socket, client_socket[max_clients]; - int activity, valread, sd; - int max_sd; - struct sockaddr_in address; - - char buffer[max_bufsize]; //data buffer of 4K - fd_set readfds; //set of socket descriptors - - for (int i = 0; i < max_clients; i++) - client_socket[i] = 0; - - if ((service_socket = socket(AF_INET, SOCK_STREAM, 0)) == 0) { - perror("socket failed"); - exit(EXIT_FAILURE); - } - - if (setsockopt(service_socket, SOL_SOCKET, SO_REUSEADDR, (char *)&opt, sizeof(opt)) < 0 ) { - perror("setsockopt"); - exit(EXIT_FAILURE); - } - - address.sin_family = AF_INET; - address.sin_addr.s_addr = INADDR_ANY; - address.sin_port = htons(AXLSVC_DEFAULT_PORT); - - if (bind(service_socket, (struct sockaddr *)&address, sizeof(address)) < 0) { - perror("bind failed"); - exit(EXIT_FAILURE); - } - printf("Listener on port %d \n", AXLSVC_DEFAULT_PORT); - - if (listen(service_socket, max_clients) < 0) { - perror("listen"); - exit(EXIT_FAILURE); - } - - addrlen = sizeof(address); - puts("Waiting for connections ..."); - - while (1) { - FD_ZERO(&readfds); - FD_SET(service_socket, &readfds); - max_sd = service_socket; - - for ( int i = 0 ; i < max_clients ; i++) { - sd = client_socket[i]; // socket descriptor - - if (sd > 0) - FD_SET(sd, &readfds); - - if (sd > max_sd) - max_sd = sd; - } - - activity = select( max_sd + 1 , &readfds , NULL , NULL , NULL); - - if ((activity < 0) && (errno!=EINTR)) - printf("select error"); - - // If something happened on the service socket, then its an incoming connection - if (FD_ISSET(service_socket, &readfds)) { - if ((new_socket = accept(service_socket, (struct sockaddr *)&address, (socklen_t*)&addrlen)) < 0) { - perror("accept"); - exit(EXIT_FAILURE); - } - - //inform user of socket number - used in send and receive commands - printf("New connection , socket fd is %d , ip is : %s , port : %d\n", - new_socket , inet_ntoa(address.sin_addr) , ntohs(address.sin_port)); - - //add new socket to array of sockets - for (int i = 0; i < max_clients; i++) { - //if position is empty - if( client_socket[i] == 0 ) { - client_socket[i] = new_socket; - printf("Adding to list of sockets as %d\n" , i); - break; - } - } - } - - for ( int i = 0; i < max_clients; i++) { - sd = client_socket[i]; - - if (FD_ISSET(sd , &readfds)) { - if ((valread = read( sd , buffer, 1024)) == 0) { - getpeername(sd , (struct sockaddr*)&address , (socklen_t*)&addrlen); - printf("Host disconnected , ip %s , port %d \n", - inet_ntoa(address.sin_addr) , ntohs(address.sin_port)); - - close( sd ); - client_socket[i] = 0; - } else { - for (int i = 0; i < valread; i++) - if (buffer[i] == '\r' || buffer[i] == '\n') - buffer[i] = ' '; - buffer[valread] = '\0'; - // execute_command(buffer); - send(sd , buffer , strlen(buffer) , 0 ); - } - } - } - } - - return 0; -} - -static void execute_command(const char* pname) -{ - FILE *p; - int ch; - - if ((p = popen(pname, "r")) == NULL) { - perror(pname); - return; - } - - printf("%s\n", pname); - - while( ( ch = fgetc(p)) != EOF ) { - putchar(ch); - } - pclose(p); -} From b2e9f2922260d5659bb02f612cfd1eb1de2c9f4b Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Wed, 22 Nov 2023 11:07:17 -0800 Subject: [PATCH 06/17] AXL Server now successfully calling AXL_Init() --- src/axl.c | 46 +++++++++++++++++++++------------------- src/axl_service.h | 19 +++++++++++++---- src/axl_service_client.c | 7 +++++- src/axl_service_server.c | 17 +++++++++------ 4 files changed, 55 insertions(+), 34 deletions(-) diff --git a/src/axl.c b/src/axl.c index dc06d5b..4f17690 100644 --- a/src/axl.c +++ b/src/axl.c @@ -263,27 +263,13 @@ int AXL_Init (void) axl_make_directories = atoi(val); } - /* If the user has set both the AXL_SERVICE_HOST and AXL_SERVICE_PORT environment - * variables, then they are expecting to use the AXL Service rather than the library - * included with the SCR library. + /* If the user has set both the AXL_SERVICE_HOST and AXL_SERVICE_PORT + * environment variables, then they are expecting to use the AXL Service + * rather than the library included with the SCR library. */ char* axl_service_host = NULL; int axl_service_port = -1; - if ( (val = getenv("AXL_SERVICE_HOST")) != NULL) { - axl_service_host = strdup(val); - - if ( (val = getenv("AXL_SERVICE_PORT")) != NULL) { - axl_service_port = atoi(val); - - if (axlsvc_client_init(axl_service_host, (unsigned short)axl_service_port)) { - axl_use_service = 1; - } - } - - free(axl_service_host); - } - /* initialize our flag on whether to first copy files to temporary names with extension */ axl_use_extension = 0; val = getenv("AXL_USE_EXTENSION"); @@ -299,7 +285,22 @@ int AXL_Init (void) } /* keep a reference count to free memory on last AXL_Finalize */ - axl_init_count++; + if (axl_init_count++ == 0) { + if (axl_service_mode != AXLSVC_SERVER) { + if ( (val = getenv("AXL_SERVICE_HOST")) != NULL) { + axl_service_host = strdup(val); + + if ( (val = getenv("AXL_SERVICE_PORT")) != NULL) { + axl_service_port = atoi(val); + + if (axlsvc_client_init(axl_service_host, (unsigned short)axl_service_port)) { + axl_service_mode = AXLSVC_CLIENT; + } + } + free(axl_service_host); + } + } + } return rc; } @@ -317,16 +318,17 @@ int AXL_Finalize (void) } #endif - if (axl_use_service) { - axlsvc_client_finalize(); - } - /* decrement reference count and free data structures on last call */ axl_init_count--; if (axl_init_count == 0) { /* TODO: are there cases where we also need to delete trees? */ axl_free(&axl_kvtrees); axl_kvtrees_count = 0; + + if (axl_service_mode == AXLSVC_CLIENT) { + axlsvc_client_finalize(); + } + } return rc; diff --git a/src/axl_service.h b/src/axl_service.h index 04b1bad..39141d7 100644 --- a/src/axl_service.h +++ b/src/axl_service.h @@ -3,7 +3,21 @@ #include -extern int axl_use_service; /* whether to use AXL service instead of library */ +#if defined(__cplusplus) +extern "C" { +#endif + +typedef enum { + AXLSVC_DISABLED = 0, /* Default - Not utilizing AXL service (lib only) */ + AXLSVC_CLIENT = 1, /* Using AXL service and we are the client */ + AXLSVC_SERVER = 2 /* Using AXL service and we are the server */ +} alxsvc_RunMode; + +/* + * Flag to state whether the AXL client/server mode of operation is enabled, + * and if so, whether the code is running as the client or the server. + */ +extern alxsvc_RunMode axl_service_mode; typedef enum { AXLSVC_AXL_CONFIG = 0, /* payload is config ktree file path */ @@ -24,9 +38,6 @@ typedef struct { ssize_t payload_length; // Optional error/status string } axlsvc_Response; -#if defined(__cplusplus) -extern "C" { -#endif int axlsvc_client_init(char* host, unsigned short port); void axlsvc_client_finalize(); diff --git a/src/axl_service_client.c b/src/axl_service_client.c index 4e3adb3..f7254f2 100644 --- a/src/axl_service_client.c +++ b/src/axl_service_client.c @@ -10,7 +10,12 @@ #include "axl_internal.h" #include "axl_service.h" -int axl_use_service = 0; +/* + * Flag to state whether the AXL client/server mode of operation is enabled, + * and if so, whether the code is running as the client or the server. + */ +alxsvc_RunMode axl_service_mode = AXLSVC_DISABLED; + static int axlsvc_socket = -1; int axlsvc_client_init(char* host, unsigned short port) diff --git a/src/axl_service_server.c b/src/axl_service_server.c index 4b58cf0..8c6d7c4 100644 --- a/src/axl_service_server.c +++ b/src/axl_service_server.c @@ -20,7 +20,7 @@ static ssize_t service_request_from_client(int sd) bytecount = axl_read("AXLSVC Client Reqeust", sd, &req, sizeof(req)); if (bytecount == 0) { - AXL_DBG(0, "Client for socket %d closed", sd); + AXL_DBG(2, "Client for socket %d closed", sd); return bytecount; } @@ -34,7 +34,7 @@ static ssize_t service_request_from_client(int sd) switch (req.request) { case AXLSVC_AXL_CONFIG: - AXL_DBG(0, "AXLSVC_AXL_CONFIG(kfile=%s", buffer); + AXL_DBG(1, "AXLSVC_AXL_CONFIG(kfile=%s", buffer); response.response = AXLSVC_SUCCESS; response.payload_length = 0; bytecount = axl_write_attempt("AXLSVC Response to Client", sd, &response, sizeof(response)); @@ -65,6 +65,8 @@ int axlsvc_server_run(int port) int activity, sd; int max_sd; + AXL_DBG(3, "ENTRY: port %d", port); + for (int i = 0; i < axlsvc_max_clients; i++) client_socket[i] = 0; @@ -113,7 +115,7 @@ int axlsvc_server_run(int port) // If something happened on the service socket, then its an incoming connection if (FD_ISSET(server_socket, &readfds)) { - AXL_DBG(0, "Accepting incomming connection"); + AXL_DBG(2, "Accepting incomming connection"); if ((new_socket = accept(server_socket, (struct sockaddr *)&address, (socklen_t*)&addrlen)) < 0) { AXL_ABORT(-1, "accept() error: (%s)", strerror(errno)); } @@ -124,7 +126,7 @@ int axlsvc_server_run(int port) break; } } - AXL_DBG(0, "Connection established"); + AXL_DBG(2, "Connection established"); } @@ -133,7 +135,7 @@ int axlsvc_server_run(int port) if (FD_ISSET(sd , &readfds)) { if (service_request_from_client(sd) == 0) { - AXL_DBG(0, "Closing server side socket(%d) to client", sd); + AXL_DBG(2, "Closing server side socket(%d) to client", sd); close(sd); client_socket[i] = 0; } @@ -144,13 +146,14 @@ int axlsvc_server_run(int port) return 0; } - int main(int argc , char *argv[]) { int rval = -1; if (argc == 2 && atoi(argv[1]) > 0) { - rval = axlsvc_server_run(atoi(argv[1])); + axl_service_mode = AXLSVC_SERVER; + if ( (rval = AXL_Init()) == AXL_SUCCESS) + rval = axlsvc_server_run(atoi(argv[1])); } else { fprintf(stderr, "Usage: %s \n", argv[0]); } From e3d4344756f0771e864779e96072b88dbd97c254 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Mon, 27 Nov 2023 14:40:26 -0800 Subject: [PATCH 07/17] Encapsulate axl_kvtrees Provide a means allowing the service to maintain separate axl_kvtrees for each open socket when running as a server. --- src/axl.c | 67 +++++++++++++++++++++------------------- src/axl_internal.h | 16 +++++++--- src/axl_pthread.c | 6 ++-- src/axl_service_server.c | 62 ++++++++++++++++++------------------- src/axl_sync.c | 6 ++-- src/axl_util.c | 2 +- 6 files changed, 86 insertions(+), 73 deletions(-) diff --git a/src/axl.c b/src/axl.c index 4f17690..67237b9 100644 --- a/src/axl.c +++ b/src/axl.c @@ -78,18 +78,19 @@ int axl_copy_metadata; /* global rank of calling process, used for BBAPI */ int axl_rank = -1; +/* + * If we are NOT running as a server, use axl_client_xfer_list to contain + * our list of transfer items fro AXL Create. Otherwise, the server will + * change the axl_xfer_list pointer for each connection it is servicing. + */ +static struct axl_transfer_array axl_client_xfer_list = { + .axl_kvtrees = NULL, .axl_kvtrees_count = 0 +}; +struct axl_transfer_array* axl_xfer_list = &axl_client_xfer_list; + /* reference count for number of times AXL_Init has been called */ static unsigned int axl_init_count = 0; -/* Array for all the AXL_Create'd kvtree pointers. It's indexed by the AXL id. - * - * Note: We only expand this array, we never shrink it. This is fine since - * the user is only going to be calling AXL_Create() a handful of times. It - * also simplifies the code if we never shrink it, and the extra memory usage - * is negligible, if any at all. */ -kvtree** axl_kvtrees; -static unsigned int axl_kvtrees_count = 0; - #ifdef HAVE_BBAPI static int bbapi_is_loaded = 0; #endif @@ -113,11 +114,11 @@ static int axl_alloc_id(const char* state_file) kvtree_util_set_str(new, AXL_KEY_STATE_FILE, state_file); } - int id = axl_kvtrees_count; - axl_kvtrees_count++; + int id = axl_xfer_list->axl_kvtrees_count; + axl_xfer_list->axl_kvtrees_count++; - axl_kvtrees = realloc(axl_kvtrees, sizeof(struct kvtree*) * axl_kvtrees_count); - axl_kvtrees[id] = new; + axl_xfer_list->axl_kvtrees = realloc(axl_xfer_list->axl_kvtrees, sizeof(struct kvtree*) * axl_xfer_list->axl_kvtrees_count); + axl_xfer_list->axl_kvtrees[id] = new; return id; } @@ -125,7 +126,7 @@ static int axl_alloc_id(const char* state_file) /* Remove the state file for an id, if one exists */ static void axl_remove_state_file(int id) { - kvtree* file_list = axl_kvtrees[id]; + kvtree* file_list = axl_xfer_list->axl_kvtrees[id]; char* state_file = NULL; if (kvtree_util_get_str(file_list, AXL_KEY_STATE_FILE, &state_file) == KVTREE_SUCCESS) @@ -139,15 +140,15 @@ static void axl_free_id(int id) { axl_remove_state_file(id); - /* kvtree_delete() will set axl_kvtrees[id] = NULL */ - kvtree_delete(&axl_kvtrees[id]); + /* kvtree_delete() will set axl_xfer_list->axl_kvtrees[id] = NULL */ + kvtree_delete(&axl_xfer_list->axl_kvtrees[id]); } /* If the user specified a state_file then write our kvtree to it. If not, then * do nothing. */ void axl_write_state_file(int id) { - kvtree* file_list = axl_kvtrees[id]; + kvtree* file_list = axl_xfer_list->axl_kvtrees[id]; char* state_file = NULL; if (kvtree_util_get_str(file_list, AXL_KEY_STATE_FILE, &state_file) == KVTREE_SUCCESS) @@ -166,7 +167,7 @@ static int axl_get_info(int id, kvtree** list, axl_xfer_t* type, axl_xfer_state_ *state = AXL_XFER_STATE_NULL; /* lookup transfer info for the given id */ - kvtree* file_list = axl_kvtrees[id]; + kvtree* file_list = axl_xfer_list->axl_kvtrees[id]; if (file_list == NULL) { AXL_ERR("Could not find fileset for UID %d", id); return AXL_FAILURE; @@ -286,6 +287,10 @@ int AXL_Init (void) /* keep a reference count to free memory on last AXL_Finalize */ if (axl_init_count++ == 0) { + /* + * If we are not running as the AXL Server, check to see if we are + * expected to run as a client and then connect if so. + */ if (axl_service_mode != AXLSVC_SERVER) { if ( (val = getenv("AXL_SERVICE_HOST")) != NULL) { axl_service_host = strdup(val); @@ -322,8 +327,8 @@ int AXL_Finalize (void) axl_init_count--; if (axl_init_count == 0) { /* TODO: are there cases where we also need to delete trees? */ - axl_free(&axl_kvtrees); - axl_kvtrees_count = 0; + axl_free(&axl_xfer_list->axl_kvtrees); + axl_xfer_list->axl_kvtrees_count = 0; if (axl_service_mode == AXLSVC_CLIENT) { axlsvc_client_finalize(); @@ -390,7 +395,7 @@ static kvtree* AXL_Config_Set(const kvtree* config) char* endptr; long id = strtol(key, &endptr, 10); if ((*key == '\0' || *endptr != '\0') || - (id < 0 || id >= axl_kvtrees_count)) + (id < 0 || id >= axl_xfer_list->axl_kvtrees_count)) { retval = NULL; break; @@ -402,7 +407,7 @@ static kvtree* AXL_Config_Set(const kvtree* config) break; } - kvtree* file_list = axl_kvtrees[id]; + kvtree* file_list = axl_xfer_list->axl_kvtrees[id]; const char** opt; for (opt = known_transfer_options; *opt != NULL; opt++) { @@ -532,8 +537,8 @@ static kvtree* AXL_Config_Get() /* per transfer options */ int id; - for (id = 0; id < axl_kvtrees_count; id++) { - kvtree* file_list = axl_kvtrees[id]; + for (id = 0; id < axl_xfer_list->axl_kvtrees_count; id++) { + kvtree* file_list = axl_xfer_list->axl_kvtrees[id]; if (file_list == NULL) { /* TODO: check if it would be better to return an empty hash instead */ continue; @@ -643,7 +648,7 @@ int AXL_Create(axl_xfer_t xtype, const char* name, const char* state_file) return -1; } - kvtree* file_list = axl_kvtrees[id]; + kvtree* file_list = axl_xfer_list->axl_kvtrees[id]; kvtree_util_set_int(file_list, AXL_KEY_XFER_TYPE, xtype); kvtree_util_set_str(file_list, AXL_KEY_UNAME, name); if (!reload_from_state_file) { @@ -1551,8 +1556,8 @@ int AXL_Stop () /* cancel each active id */ int id; - for (id = 0; id < axl_kvtrees_count; id++) { - if (!axl_kvtrees[id]) { + for (id = 0; id < axl_xfer_list->axl_kvtrees_count; id++) { + if (!axl_xfer_list->axl_kvtrees[id]) { continue; } @@ -1562,8 +1567,8 @@ int AXL_Stop () } /* wait */ - for (id = 0; id < axl_kvtrees_count; id++) { - if (!axl_kvtrees[id]) { + for (id = 0; id < axl_xfer_list->axl_kvtrees_count; id++) { + if (!axl_xfer_list->axl_kvtrees[id]) { continue; } @@ -1573,8 +1578,8 @@ int AXL_Stop () } /* and free it */ - for (id = 0; id < axl_kvtrees_count; id++) { - if (!axl_kvtrees[id]) { + for (id = 0; id < axl_xfer_list->axl_kvtrees_count; id++) { + if (!axl_xfer_list->axl_kvtrees[id]) { continue; } diff --git a/src/axl_internal.h b/src/axl_internal.h index 4f58545..361523d 100644 --- a/src/axl_internal.h +++ b/src/axl_internal.h @@ -17,10 +17,18 @@ /* unless otherwise indicated all global variables defined in this file must * only be accessed by the main thread */ -/* - * A list of pointers to kvtrees, indexed by AXL ID. - */ -extern kvtree** axl_kvtrees; +struct axl_transfer_array { + /* Array for all the AXL_Create'd kvtree pointers. It's indexed by the AXL id. + * + * Note: We only expand this array, we never shrink it. This is fine since + * the user is only going to be calling AXL_Create() a handful of times. It + * also simplifies the code if we never shrink it, and the extra memory usage + * is negligible, if any at all. */ + kvtree** axl_kvtrees; + unsigned int axl_kvtrees_count; +}; + +extern struct axl_transfer_array* axl_xfer_list; /* current debug level for AXL library, * set in AXL_Init and AXL_Config used in axl_dbg. diff --git a/src/axl_pthread.c b/src/axl_pthread.c index 34ef676..0ae64f5 100644 --- a/src/axl_pthread.c +++ b/src/axl_pthread.c @@ -38,7 +38,7 @@ struct axl_pthread_data /* AXL ID associated with this data */ int id; - /* AXL transfer options from axl_kvtrees */ + /* AXL transfer options from axl_xfer_list->axl_kvtrees */ kvtree* file_list; /* If resume = 1, try to resume old transfers */ @@ -135,7 +135,7 @@ struct axl_pthread_data* axl_pthread_data_lookup(int id) void axl_pthread_data_add(int id, struct axl_pthread_data* pdata) { pdata->id = id; - pdata->file_list = axl_kvtrees[id]; + pdata->file_list = axl_xfer_list->axl_kvtrees[id]; pthread_mutex_lock(&axl_all_pthread_data.lock); @@ -332,7 +332,7 @@ static int __axl_pthread_start (int id, int resume) int rc = AXL_SUCCESS; /* get pointer to file list for this dataset */ - kvtree* file_list = axl_kvtrees[id]; + kvtree* file_list = axl_xfer_list->axl_kvtrees[id]; /* mark dataset as in progress */ kvtree_util_set_int(file_list, AXL_KEY_STATUS, AXL_STATUS_INPROG); diff --git a/src/axl_service_server.c b/src/axl_service_server.c index 8c6d7c4..0db16d0 100644 --- a/src/axl_service_server.c +++ b/src/axl_service_server.c @@ -10,6 +10,12 @@ #include "axl_internal.h" #include "axl_service.h" +#define AXLSVC_MAX_CLIENTS 16 +struct axl_connection_ctx { + int sd; /* Connection to our socket */ + struct axl_transfer_array xfr; /* Pointer to client-specific xfer array */ +} axl_connection_ctx_array[AXLSVC_MAX_CLIENTS]; + static ssize_t service_request_from_client(int sd) { ssize_t bytecount; @@ -54,27 +60,21 @@ static ssize_t service_request_from_client(int sd) int axlsvc_server_run(int port) { - static const int axlsvc_max_clients = 16; - int client_socket[axlsvc_max_clients]; int server_socket; int opt = 1; struct sockaddr_in address; int addrlen; int new_socket; fd_set readfds; - int activity, sd; + int activity; int max_sd; - AXL_DBG(3, "ENTRY: port %d", port); - - for (int i = 0; i < axlsvc_max_clients; i++) - client_socket[i] = 0; - if ((server_socket = socket(AF_INET, SOCK_STREAM, 0)) == 0) { AXL_ABORT(-1, "socket() failed: (%s)", strerror(errno)); } - if (setsockopt(server_socket, SOL_SOCKET, SO_REUSEADDR, (char *)&opt, sizeof(opt)) < 0 ) { + if (setsockopt(server_socket, SOL_SOCKET, SO_REUSEADDR, + (char *)&opt, sizeof(opt)) < 0 ) { AXL_ABORT(-1, "setsockopt() failed: (%s)", strerror(errno)); } @@ -86,7 +86,7 @@ int axlsvc_server_run(int port) AXL_ABORT(-1, "bind() failed: (%s)", strerror(errno)); } - if (listen(server_socket, axlsvc_max_clients) < 0) { + if (listen(server_socket, AXLSVC_MAX_CLIENTS) < 0) { AXL_ABORT(-1, "listen() failed: (%s)", strerror(errno)); } @@ -97,14 +97,12 @@ int axlsvc_server_run(int port) FD_SET(server_socket, &readfds); max_sd = server_socket; - for (int i = 0 ; i < axlsvc_max_clients ; i++) { - sd = client_socket[i]; + for (int i = 0 ; i < AXLSVC_MAX_CLIENTS ; i++) { + if (axl_connection_ctx_array[i].sd > 0) + FD_SET(axl_connection_ctx_array[i].sd, &readfds); - if (sd > 0) - FD_SET(sd, &readfds); - - if (sd > max_sd) - max_sd = sd; + if (axl_connection_ctx_array[i].sd > max_sd) + max_sd = axl_connection_ctx_array[i].sd; } activity = select(max_sd + 1 , &readfds , NULL , NULL , NULL); @@ -113,31 +111,31 @@ int axlsvc_server_run(int port) AXL_ABORT(-1, "select() error: (%s)", strerror(errno)); } - // If something happened on the service socket, then its an incoming connection if (FD_ISSET(server_socket, &readfds)) { - AXL_DBG(2, "Accepting incomming connection"); - if ((new_socket = accept(server_socket, (struct sockaddr *)&address, (socklen_t*)&addrlen)) < 0) { + AXL_DBG(1, "Accepting new incomming connection"); + if ((new_socket = accept(server_socket, (struct sockaddr *)&address, + (socklen_t*)&addrlen)) < 0) { AXL_ABORT(-1, "accept() error: (%s)", strerror(errno)); } - for (int i = 0; i < axlsvc_max_clients; i++) { - if( client_socket[i] == 0 ) { - client_socket[i] = new_socket; + for (int i = 0; i < AXLSVC_MAX_CLIENTS; i++) { + if(axl_connection_ctx_array[i].sd == 0 ){ + axl_connection_ctx_array[i].sd = new_socket; break; } } AXL_DBG(2, "Connection established"); - } - for ( int i = 0; i < axlsvc_max_clients; i++) { - sd = client_socket[i]; + for ( int i = 0; i < AXLSVC_MAX_CLIENTS; i++) { + if (FD_ISSET(axl_connection_ctx_array[i].sd , &readfds)) { + axl_xfer_list = &axl_connection_ctx_array[i].xfr; - if (FD_ISSET(sd , &readfds)) { - if (service_request_from_client(sd) == 0) { - AXL_DBG(2, "Closing server side socket(%d) to client", sd); - close(sd); - client_socket[i] = 0; + if (service_request_from_client(axl_connection_ctx_array[i].sd) == 0) { + AXL_DBG(2, "Closing server side socket(%d) to client", axl_connection_ctx_array[i].sd); + close(axl_connection_ctx_array[i].sd); + axl_connection_ctx_array[i].sd = 0; + /*TODO: Free up memory used for acx_kvtrees */ } } } @@ -152,6 +150,8 @@ int main(int argc , char *argv[]) if (argc == 2 && atoi(argv[1]) > 0) { axl_service_mode = AXLSVC_SERVER; + memset(axl_connection_ctx_array, 0, sizeof(axl_connection_ctx_array)); + if ( (rval = AXL_Init()) == AXL_SUCCESS) rval = axlsvc_server_run(atoi(argv[1])); } else { diff --git a/src/axl_sync.c b/src/axl_sync.c index b5fd1ae..1abc608 100644 --- a/src/axl_sync.c +++ b/src/axl_sync.c @@ -12,7 +12,7 @@ int __axl_sync_start (int id, int resume) int rc = AXL_SUCCESS; /* get pointer to file list for this dataset */ - kvtree* file_list = axl_kvtrees[id]; + kvtree* file_list = axl_xfer_list->axl_kvtrees[id]; /* mark dataset as in progress */ kvtree_util_set_int(file_list, AXL_KEY_STATUS, AXL_STATUS_INPROG); @@ -39,7 +39,7 @@ int __axl_sync_start (int id, int resume) } /* TODO: check bytecount conversion success, do not use global - * axl_kvtrees to get file_list */ + * axl_xfer_list->axl_kvtrees to get file_list */ unsigned long file_buf_size; int success = kvtree_util_get_bytecount(file_list, AXL_KEY_CONFIG_FILE_BUF_SIZE, &file_buf_size); @@ -91,7 +91,7 @@ int axl_sync_test (int id) int axl_sync_wait (int id) { /* get pointer to file list for this dataset */ - kvtree* file_list = axl_kvtrees[id]; + kvtree* file_list = axl_xfer_list->axl_kvtrees[id]; /* determine whether transfer was successful */ int status; diff --git a/src/axl_util.c b/src/axl_util.c index 8b0ebbb..98b62d1 100644 --- a/src/axl_util.c +++ b/src/axl_util.c @@ -110,7 +110,7 @@ kvtree_elem* axl_get_next_path(int id, kvtree_elem* elem, char** src, char** dst { if (! elem) { /* lookup transfer info for the given id */ - kvtree* file_list = axl_kvtrees[id]; + kvtree* file_list = axl_xfer_list->axl_kvtrees[id]; if (! file_list) { return NULL; } From fae47b0e0b0bfa17aeaf55ac10a656a316cdc69c Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Mon, 27 Nov 2023 16:29:19 -0800 Subject: [PATCH 08/17] Checkpoint - Client/Server ConfigSet --- src/axl.c | 6 +++++- src/axl_service.h | 15 ++++++++++--- src/axl_service_client.c | 46 +++++++++++++++++++++++++++++++++++++++- src/axl_service_server.c | 19 ++++++++++++++--- 4 files changed, 78 insertions(+), 8 deletions(-) diff --git a/src/axl.c b/src/axl.c index 67237b9..abfaa17 100644 --- a/src/axl.c +++ b/src/axl.c @@ -331,7 +331,7 @@ int AXL_Finalize (void) axl_xfer_list->axl_kvtrees_count = 0; if (axl_service_mode == AXLSVC_CLIENT) { - axlsvc_client_finalize(); + axlsvc_client_AXL_Finalize(); } } @@ -497,6 +497,10 @@ static kvtree* AXL_Config_Set(const kvtree* config) } } + if (axl_service_mode == AXLSVC_CLIENT) { + axlsvc_client_AXL_Config_Set(config); + } + return retval; } diff --git a/src/axl_service.h b/src/axl_service.h index 39141d7..a372bb2 100644 --- a/src/axl_service.h +++ b/src/axl_service.h @@ -2,6 +2,7 @@ #define __AXLSVC_H__ #include +#include "kvtree.h" #if defined(__cplusplus) extern "C" { @@ -20,7 +21,7 @@ typedef enum { extern alxsvc_RunMode axl_service_mode; typedef enum { - AXLSVC_AXL_CONFIG = 0, /* payload is config ktree file path */ + AXLSVC_AXL_CONFIG_SET = 0, /* payload is config kvtree hash buffer */ } axlsvc_request_t; typedef struct { @@ -38,9 +39,17 @@ typedef struct { ssize_t payload_length; // Optional error/status string } axlsvc_Response; - int axlsvc_client_init(char* host, unsigned short port); -void axlsvc_client_finalize(); + +/* + * function to perform client-side request to server for AXL_Finalize() + */ +void axlsvc_client_AXL_Finalize(); + +/* + * function to perform client-side request to server for AXL_Config_Set + */ +void axlsvc_client_AXL_Config_Set(const kvtree* config); #if defined(__cplusplus) extern "C" } diff --git a/src/axl_service_client.c b/src/axl_service_client.c index f7254f2..96111d1 100644 --- a/src/axl_service_client.c +++ b/src/axl_service_client.c @@ -9,6 +9,7 @@ #include "axl_internal.h" #include "axl_service.h" +#include "kvtree.h" /* * Flag to state whether the AXL client/server mode of operation is enabled, @@ -47,9 +48,52 @@ int axlsvc_client_init(char* host, unsigned short port) return 1; // success } -void axlsvc_client_finalize() +/* + * function to perform client-side request to server for AXL_Finalize() + */ +void axlsvc_client_AXL_Finalize() { if (axlsvc_socket >= 0) close(axlsvc_socket); } +/* + * function to perform client-side request to server for AXL_Config_Set + */ +void axlsvc_client_AXL_Config_Set(const kvtree* config) +{ + ssize_t bytecount; + axlsvc_Request request; + axlsvc_Response response; + + request.request = AXLSVC_AXL_CONFIG_SET; + request.payload_length = (ssize_t)kvtree_pack_size(config); + + bytecount = axl_write_attempt("AXLSVC Client --> AXL_Config_Set_1", + axlsvc_socket, &request, sizeof(request)); + + if (bytecount != sizeof(request)) { + AXL_ABORT(-1, "Unexpected Write Response to server: Expected %d, Got %d", + sizeof(request), bytecount); + } + + bytecount = kvtree_write_fd("AXLSVC Client --> AXL_Config_Set_2", + axlsvc_socket, config); + + if (bytecount != request.payload_length) { + AXL_ABORT(-1, "Unexpected Write Response to server: Expected %d, Got %d", + request.payload_length, bytecount); + } + + bytecount = axl_read("AXLSVC Client <-- Response", + axlsvc_socket, &response, sizeof(response)); + + if (bytecount != sizeof(response)) { + AXL_ABORT(-1, "Unexpected Write Response to server: Expected %d, Got %d", + sizeof(response), bytecount); + } + + if (response.response != AXLSVC_SUCCESS) { + AXL_ABORT(-1, "Unexpected Response from server: %d", response.response); + } +} diff --git a/src/axl_service_server.c b/src/axl_service_server.c index 0db16d0..862b1b1 100644 --- a/src/axl_service_server.c +++ b/src/axl_service_server.c @@ -9,6 +9,7 @@ #include "axl_internal.h" #include "axl_service.h" +#include "kvtree.h" #define AXLSVC_MAX_CLIENTS 16 struct axl_connection_ctx { @@ -16,6 +17,17 @@ struct axl_connection_ctx { struct axl_transfer_array xfr; /* Pointer to client-specific xfer array */ } axl_connection_ctx_array[AXLSVC_MAX_CLIENTS]; +static kvtree* service_request_AXL_Config_Set(int sd) +{ + kvtree* config = kvtree_new(); + kvtree* rval; + ssize_t bytecount; + + bytecount = kvtree_read_fd("Service_AXL_Config_Set", sd, config); + + return rval; +} + static ssize_t service_request_from_client(int sd) { ssize_t bytecount; @@ -39,8 +51,8 @@ static ssize_t service_request_from_client(int sd) } switch (req.request) { - case AXLSVC_AXL_CONFIG: - AXL_DBG(1, "AXLSVC_AXL_CONFIG(kfile=%s", buffer); + case AXLSVC_AXL_CONFIG_SET: + AXL_DBG(1, "AXLSVC_AXL_CONFIG_SET(kfile=%s", buffer); response.response = AXLSVC_SUCCESS; response.payload_length = 0; bytecount = axl_write_attempt("AXLSVC Response to Client", sd, &response, sizeof(response)); @@ -135,7 +147,8 @@ int axlsvc_server_run(int port) AXL_DBG(2, "Closing server side socket(%d) to client", axl_connection_ctx_array[i].sd); close(axl_connection_ctx_array[i].sd); axl_connection_ctx_array[i].sd = 0; - /*TODO: Free up memory used for acx_kvtrees */ + axl_free(&axl_xfer_list->axl_kvtrees); + axl_xfer_list->axl_kvtrees_count = 0; } } } From 166849d60c91093fb84a3d6c3402a6e3768addff Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Fri, 1 Dec 2023 02:19:28 -0800 Subject: [PATCH 09/17] Obey BUILD_SHARED_LIBS settings The new axl_service executable will know pick the correct external libraries based upon the BUILD_SHARED_LIBS settings. --- src/CMakeLists.txt | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 51aa22f..dad683a 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -28,7 +28,15 @@ IF(HAVE_PTHREADS) axl_service_client.c ) ADD_EXECUTABLE(axl_service axl_service_server.c) - TARGET_LINK_LIBRARIES(axl_service PUBLIC axl_static_o ${AXL_EXTERNAL_LIBS}) + + if(BUILD_SHARED_LIBS) + TARGET_LINK_LIBRARIES(axl_service + PUBLIC axl_static_o ${AXL_EXTERNAL_LIBS}) + else() + TARGET_LINK_LIBRARIES(axl_service + PUBLIC axl_static_o ${AXL_EXTERNAL_STATIC_LIBS}) + endif() + TARGET_INCLUDE_DIRECTORIES(axl_service PUBLIC $ $) INSTALL(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/axl_service DESTINATION ${CMAKE_INSTALL_LIBEXECDIR}) ENDIF(HAVE_PTHREADS) From 7aa26b96ea073da87e18ebd64f2c214ea824ab22 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Tue, 25 Jun 2024 14:26:06 -0700 Subject: [PATCH 10/17] Initial client/server test driver --- test/test_client_server.c | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 test/test_client_server.c diff --git a/test/test_client_server.c b/test/test_client_server.c new file mode 100644 index 0000000..a82e574 --- /dev/null +++ b/test/test_client_server.c @@ -0,0 +1,6 @@ +#include "axl.h" + +int main(int, char**) +{ + return 0; +} \ No newline at end of file From af5c5ebfda379c537da6ba3b9a345191fbd01d57 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Wed, 26 Jun 2024 16:26:34 -0700 Subject: [PATCH 11/17] Scaffolding of client/server test implemented --- test/CMakeLists.txt | 8 ++++--- test/test_client_server.c | 45 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 79cce0d..6f672dc 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -3,9 +3,7 @@ ############### INCLUDE_DIRECTORIES(${PROJECT_BINARY_DIR}) -LIST(APPEND axl_test_srcs - axl_cp.c -) +LIST(APPEND axl_test_srcs axl_cp.c) # Build with debug symbols (-g) # SET(CMAKE_BUILD_TYPE Debug) @@ -18,9 +16,11 @@ ENDIF() ADD_EXECUTABLE(axl_cp ${axl_test_srcs}) ADD_EXECUTABLE(test_config test_config.c) +ADD_EXECUTABLE(test_client_server test_client_server.c) TARGET_LINK_LIBRARIES(axl_cp ${axl_lib}) TARGET_LINK_LIBRARIES(test_config ${axl_lib}) +TARGET_LINK_LIBRARIES(test_client_server ${axl_lib}) ################ # Add tests to ctest @@ -68,6 +68,8 @@ ENDIF(BBAPI_FOUND) ADD_TEST(test_config test_config) +ADD_TEST(test_client_server test_client_server) + #################### # make a verbose "test" target named "check" #################### diff --git a/test/test_client_server.c b/test/test_client_server.c index a82e574..d23c51b 100644 --- a/test/test_client_server.c +++ b/test/test_client_server.c @@ -1,6 +1,49 @@ +#include +#include +#include +#include +#include +#include + #include "axl.h" -int main(int, char**) +int wait_for_service_to_complete() +{ + int pid; + int wstatus = 0; + + if ((pid = wait(&wstatus)) < 0) { + fprintf(stderr, "Wait for service process failed: [%d] - %s\n", errno, strerror(errno)); + return 1; + } + + return WEXITSTATUS(wstatus); +} + +int run_service() { return 0; +} + +int run_client() +{ + int client_status = 0; + + return client_status + wait_for_service_to_complete(); +} + +int main() +{ + int pid; + + if ((pid = fork()) < 0) { + fprintf(stderr, "Creation of service failed: [%d] - %s\n", errno, strerror(errno)); + return 2; + } + else if (pid == 0) { + return run_service(); + } + else { + return run_client(); + } } \ No newline at end of file From 6042455bece2252eba49805452ee0e3a1c0d887b Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Thu, 27 Jun 2024 16:11:58 -0700 Subject: [PATCH 12/17] Driving client/server lifetime with python script for CI test --- test/CMakeLists.txt | 3 +- test/test_client_server.c | 67 +++++++++++++++++++++---------- test/test_driver_client_server.py | 30 ++++++++++++++ 3 files changed, 78 insertions(+), 22 deletions(-) create mode 100755 test/test_driver_client_server.py diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 6f672dc..5673318 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -28,6 +28,7 @@ TARGET_LINK_LIBRARIES(test_client_server ${axl_lib}) CONFIGURE_FILE(test_axl.sh ${CMAKE_CURRENT_BINARY_DIR} COPYONLY) CONFIGURE_FILE(test_axl_metadata.sh ${CMAKE_CURRENT_BINARY_DIR} COPYONLY) +CONFIGURE_FILE(test_driver_client_server.py ${CMAKE_CURRENT_BINARY_DIR} COPYONLY) ADD_TEST(sync_test test_axl.sh sync) @@ -68,7 +69,7 @@ ENDIF(BBAPI_FOUND) ADD_TEST(test_config test_config) -ADD_TEST(test_client_server test_client_server) +ADD_TEST(test_client_server test_driver_client_server.py) #################### # make a verbose "test" target named "check" diff --git a/test/test_client_server.c b/test/test_client_server.c index d23c51b..f20a97e 100644 --- a/test/test_client_server.c +++ b/test/test_client_server.c @@ -4,46 +4,71 @@ #include #include #include +#include #include "axl.h" -int wait_for_service_to_complete() -{ - int pid; - int wstatus = 0; +#define AXLCS_SUCCESS 0 +#define AXLCS_CLIENT_INVALID 1 +#define AXLCS_SERVICE_CREATION_FAILURE 1000 +#define AXLCS_SERVICE_KILLED 2000 +#define AXLCS_SERVICE_FAIL 3000 - if ((pid = wait(&wstatus)) < 0) { - fprintf(stderr, "Wait for service process failed: [%d] - %s\n", errno, strerror(errno)); - return 1; - } +static int time_to_leave = 0; - return WEXITSTATUS(wstatus); +void sigterm_handler(int sig, siginfo_t* info, void* ucontext) +{ + time_to_leave++; } int run_service() { - return 0; + struct sigaction act = {0}; + + act.sa_flags = 0; + sigemptyset(&act.sa_mask); + act.sa_sigaction = sigterm_handler; + if (sigaction(SIGTERM, &act, NULL) == -1) { + perror("sigaction"); + return AXLCS_SERVICE_FAIL; + } + + fprintf(stdout, "Service Started!\n"); + + for (int i = 0; !time_to_leave && i < 100000; ++i) { + int seconds = 2+i; + fprintf(stdout, "Sleeping %d seconds\n", seconds); + fprintf(stderr, "Just testing stderr. %d ..\n", seconds); + sleep(seconds); + } + + fprintf(stdout, "Service Ending!\n"); + return AXLCS_SUCCESS; } int run_client() { - int client_status = 0; - - return client_status + wait_for_service_to_complete(); + fprintf(stdout, "Client Started!\n"); + sleep(10); + fprintf(stdout, "Client Ending!\n"); + return AXLCS_SUCCESS; } -int main() +int main(int ac, char **av) { - int pid; - - if ((pid = fork()) < 0) { - fprintf(stderr, "Creation of service failed: [%d] - %s\n", errno, strerror(errno)); - return 2; + fprintf(stderr, "Just testing stderr...\n"); + if (ac != 2) { + fprintf(stderr, "Command count (%d) incorrect:\nUsage: test_client_server \n", ac); + return AXLCS_CLIENT_INVALID; } - else if (pid == 0) { + + if (strcmp("server", av[1]) == 0) { return run_service(); } - else { + else if (strcmp("client", av[1]) == 0) { return run_client(); } + + fprintf(stderr, "Unknown Argument (%s) incorrect:\nUsage: test_client_server \n", av[1]); + return AXLCS_CLIENT_INVALID; } \ No newline at end of file diff --git a/test/test_driver_client_server.py b/test/test_driver_client_server.py new file mode 100755 index 0000000..edae52b --- /dev/null +++ b/test/test_driver_client_server.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 + +import sys +import subprocess +import os + +def wait_for_completion(procname, proc, wait_time): + try: + outs, err = proc.communicate(timeout=wait_time) + except: + proc.terminate() + outs, err = proc.communicate() + + print("{} Return Code: {}".format(procname, proc.returncode)) + print("stdout:\n{}".format(outs.decode("utf-8"))) + print("stderr:\n{}".format(err.decode("utf-8"))) + + return proc.returncode, outs, err + +if __name__ == '__main__': + # Launch the server then the client + server = subprocess.Popen(['./test_client_server server'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + client = subprocess.Popen(['./test_client_server client'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + + # Wait for the client then the server to finish + server_ecode, server_out, server_err = wait_for_completion("axl_client", client, 120) + client_ecode, client_out, client_err = wait_for_completion("axl_server", server, 10) + + if server_ecode != 0 or client_ecode != 0: + sys.exit(server_ecode + client_ecode) From 7f3f036ba0e4426fc2a697be838ac4dd008e572c Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Thu, 27 Jun 2024 16:24:40 -0700 Subject: [PATCH 13/17] Test updates --- test/test_client_server.c | 2 +- test/test_driver_client_server.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/test/test_client_server.c b/test/test_client_server.c index f20a97e..294e985 100644 --- a/test/test_client_server.c +++ b/test/test_client_server.c @@ -49,7 +49,7 @@ int run_service() int run_client() { fprintf(stdout, "Client Started!\n"); - sleep(10); + sleep(2); fprintf(stdout, "Client Ending!\n"); return AXLCS_SUCCESS; } diff --git a/test/test_driver_client_server.py b/test/test_driver_client_server.py index edae52b..a0f38b5 100755 --- a/test/test_driver_client_server.py +++ b/test/test_driver_client_server.py @@ -18,13 +18,16 @@ def wait_for_completion(procname, proc, wait_time): return proc.returncode, outs, err if __name__ == '__main__': + errors = 0 # Launch the server then the client server = subprocess.Popen(['./test_client_server server'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) client = subprocess.Popen(['./test_client_server client'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) # Wait for the client then the server to finish - server_ecode, server_out, server_err = wait_for_completion("axl_client", client, 120) - client_ecode, client_out, client_err = wait_for_completion("axl_server", server, 10) + client_ecode, client_out, client_err = wait_for_completion("axl_client", client, 120) + server_ecode, server_out, server_err = wait_for_completion("axl_server", server, 2) if server_ecode != 0 or client_ecode != 0: - sys.exit(server_ecode + client_ecode) + errors = server_ecode + client_ecode + + sys.exit(errors) From 6955e7e8444d34c003b16cc30db317bec80d7c74 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Fri, 28 Jun 2024 05:17:27 -0700 Subject: [PATCH 14/17] Updates to how subprocess is being used for test driver --- test/test_client_server.c | 8 ++++---- test/test_driver_client_server.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/test/test_client_server.c b/test/test_client_server.c index 294e985..a44cd13 100644 --- a/test/test_client_server.c +++ b/test/test_client_server.c @@ -58,17 +58,17 @@ int main(int ac, char **av) { fprintf(stderr, "Just testing stderr...\n"); if (ac != 2) { - fprintf(stderr, "Command count (%d) incorrect:\nUsage: test_client_server \n", ac); + fprintf(stderr, "Command count (%d) incorrect:\nUsage: test_client_server --\n", ac); return AXLCS_CLIENT_INVALID; } - if (strcmp("server", av[1]) == 0) { + if (strcmp("--server", av[1]) == 0) { return run_service(); } - else if (strcmp("client", av[1]) == 0) { + else if (strcmp("--client", av[1]) == 0) { return run_client(); } - fprintf(stderr, "Unknown Argument (%s) incorrect:\nUsage: test_client_server \n", av[1]); + fprintf(stderr, "Unknown Argument (%s) incorrect:\nUsage: test_client_server --\n", av[1]); return AXLCS_CLIENT_INVALID; } \ No newline at end of file diff --git a/test/test_driver_client_server.py b/test/test_driver_client_server.py index a0f38b5..e5c9876 100755 --- a/test/test_driver_client_server.py +++ b/test/test_driver_client_server.py @@ -20,11 +20,11 @@ def wait_for_completion(procname, proc, wait_time): if __name__ == '__main__': errors = 0 # Launch the server then the client - server = subprocess.Popen(['./test_client_server server'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) - client = subprocess.Popen(['./test_client_server client'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) + server = subprocess.Popen(['./test_client_server', '--server'], env=dict(os.environ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) + client = subprocess.Popen(['./test_client_server', '--client'], env=dict(os.environ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) # Wait for the client then the server to finish - client_ecode, client_out, client_err = wait_for_completion("axl_client", client, 120) + client_ecode, client_out, client_err = wait_for_completion("axl_client", client, 30) server_ecode, server_out, server_err = wait_for_completion("axl_server", server, 2) if server_ecode != 0 or client_ecode != 0: From 05905d5400ec38b526e63d43ed712f101e9e43e0 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Fri, 28 Jun 2024 06:17:51 -0700 Subject: [PATCH 15/17] Cleanup in prep for testing --- test/test_client_server.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/test/test_client_server.c b/test/test_client_server.c index a44cd13..1640338 100644 --- a/test/test_client_server.c +++ b/test/test_client_server.c @@ -21,7 +21,7 @@ void sigterm_handler(int sig, siginfo_t* info, void* ucontext) time_to_leave++; } -int run_service() +int use_sigterm_to_exit() { struct sigaction act = {0}; @@ -33,15 +33,22 @@ int run_service() return AXLCS_SERVICE_FAIL; } + return AXLCS_SUCCESS; +} + +int run_service() +{ + int rval; + if ((rval = use_sigterm_to_exit()) != AXLCS_SUCCESS) + return rval; + fprintf(stdout, "Service Started!\n"); - for (int i = 0; !time_to_leave && i < 100000; ++i) { int seconds = 2+i; fprintf(stdout, "Sleeping %d seconds\n", seconds); fprintf(stderr, "Just testing stderr. %d ..\n", seconds); sleep(seconds); } - fprintf(stdout, "Service Ending!\n"); return AXLCS_SUCCESS; } From e9082d73cd64607bb3c396c4890748e9fdc338d3 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Mon, 1 Jul 2024 14:26:38 -0700 Subject: [PATCH 16/17] Refactored so CI could actually run the server --- src/CMakeLists.txt | 40 +++--- src/axl.c | 16 +-- src/axl_service.h | 58 -------- src/axl_service_client.c | 99 ------------- src/axl_service_server.c | 175 ----------------------- src/axl_socket.c | 289 ++++++++++++++++++++++++++++++++++++++ src/axl_socket.h | 60 ++++++++ src/axl_socket_daemon.c | 18 +++ test/CMakeLists.txt | 3 + test/test_client_server.c | 36 +---- 10 files changed, 398 insertions(+), 396 deletions(-) delete mode 100644 src/axl_service.h delete mode 100644 src/axl_service_client.c delete mode 100644 src/axl_service_server.c create mode 100644 src/axl_socket.c create mode 100644 src/axl_socket.h create mode 100644 src/axl_socket_daemon.c diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index dad683a..9047db2 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -14,31 +14,11 @@ ENDIF(MPI AND MPI_FOUND) INSTALL(FILES ${libaxl_install_headers} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) -LIST(APPEND libaxl_srcs - axl.c - axl_sync.c - axl_err.c - axl_io.c - axl_util.c -) +LIST(APPEND libaxl_srcs axl.c axl_sync.c axl_err.c axl_io.c axl_util.c) +# TODO: Separate PTHREADS from SOCKETS IF(HAVE_PTHREADS) - LIST(APPEND libaxl_srcs - axl_pthread.c - axl_service_client.c - ) - ADD_EXECUTABLE(axl_service axl_service_server.c) - - if(BUILD_SHARED_LIBS) - TARGET_LINK_LIBRARIES(axl_service - PUBLIC axl_static_o ${AXL_EXTERNAL_LIBS}) - else() - TARGET_LINK_LIBRARIES(axl_service - PUBLIC axl_static_o ${AXL_EXTERNAL_STATIC_LIBS}) - endif() - - TARGET_INCLUDE_DIRECTORIES(axl_service PUBLIC $ $) - INSTALL(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/axl_service DESTINATION ${CMAKE_INSTALL_LIBEXECDIR}) + LIST(APPEND libaxl_srcs axl_pthread.c axl_socket.c) ENDIF(HAVE_PTHREADS) IF(BBAPI_FOUND) @@ -77,6 +57,20 @@ TARGET_INCLUDE_DIRECTORIES(axl-static PUBLIC $axl_kvtrees); axl_xfer_list->axl_kvtrees_count = 0; - if (axl_service_mode == AXLSVC_CLIENT) { - axlsvc_client_AXL_Finalize(); + if (axl_service_mode == AXL_SOCKET_CLIENT) { + axl_socket_client_AXL_Finalize(); } } @@ -497,8 +497,8 @@ static kvtree* AXL_Config_Set(const kvtree* config) } } - if (axl_service_mode == AXLSVC_CLIENT) { - axlsvc_client_AXL_Config_Set(config); + if (axl_service_mode == AXL_SOCKET_CLIENT) { + axl_socket_client_AXL_Config_Set(config); } return retval; diff --git a/src/axl_service.h b/src/axl_service.h deleted file mode 100644 index a372bb2..0000000 --- a/src/axl_service.h +++ /dev/null @@ -1,58 +0,0 @@ -#ifndef __AXLSVC_H__ -#define __AXLSVC_H__ - -#include -#include "kvtree.h" - -#if defined(__cplusplus) -extern "C" { -#endif - -typedef enum { - AXLSVC_DISABLED = 0, /* Default - Not utilizing AXL service (lib only) */ - AXLSVC_CLIENT = 1, /* Using AXL service and we are the client */ - AXLSVC_SERVER = 2 /* Using AXL service and we are the server */ -} alxsvc_RunMode; - -/* - * Flag to state whether the AXL client/server mode of operation is enabled, - * and if so, whether the code is running as the client or the server. - */ -extern alxsvc_RunMode axl_service_mode; - -typedef enum { - AXLSVC_AXL_CONFIG_SET = 0, /* payload is config kvtree hash buffer */ -} axlsvc_request_t; - -typedef struct { - axlsvc_request_t request; - ssize_t payload_length; -} axlsvc_Request; - -typedef enum { - AXLSVC_SUCCESS = 0, - AXLSVC_FAILURE = -1, -} axlsvc_response_t; - -typedef struct { - axlsvc_response_t response; - ssize_t payload_length; // Optional error/status string -} axlsvc_Response; - -int axlsvc_client_init(char* host, unsigned short port); - -/* - * function to perform client-side request to server for AXL_Finalize() - */ -void axlsvc_client_AXL_Finalize(); - -/* - * function to perform client-side request to server for AXL_Config_Set - */ -void axlsvc_client_AXL_Config_Set(const kvtree* config); - -#if defined(__cplusplus) -extern "C" } -#endif - -#endif /* __AXLSVC_H__ */ diff --git a/src/axl_service_client.c b/src/axl_service_client.c deleted file mode 100644 index 96111d1..0000000 --- a/src/axl_service_client.c +++ /dev/null @@ -1,99 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - -#include "axl_internal.h" -#include "axl_service.h" -#include "kvtree.h" - -/* - * Flag to state whether the AXL client/server mode of operation is enabled, - * and if so, whether the code is running as the client or the server. - */ -alxsvc_RunMode axl_service_mode = AXLSVC_DISABLED; - -static int axlsvc_socket = -1; - -int axlsvc_client_init(char* host, unsigned short port) -{ - struct sockaddr_in server; - struct hostent *hostnm = gethostbyname(host); - - if (hostnm == (struct hostent *) 0) { - AXL_ERR("Gethostbyname failed: (%s)", strerror(errno)); - return 0; - } - - if ( (axlsvc_socket = socket(AF_INET, SOCK_STREAM, 0)) < 0) { - AXL_ERR("socket() failed: (%s)", strerror(errno)); - return 0; - } - - memset(&server, 0, sizeof(server)); - server.sin_family = AF_INET; - server.sin_port = htons(port); - server.sin_addr.s_addr = *((unsigned long *)hostnm->h_addr); - - if ( connect(axlsvc_socket, (struct sockaddr *)&server, sizeof(server) ) < 0) { - AXL_ERR("connect() failed: (%s)", strerror(errno)); - close(axlsvc_socket); - return 0; - } - - return 1; // success -} - -/* - * function to perform client-side request to server for AXL_Finalize() - */ -void axlsvc_client_AXL_Finalize() -{ - if (axlsvc_socket >= 0) - close(axlsvc_socket); -} - -/* - * function to perform client-side request to server for AXL_Config_Set - */ -void axlsvc_client_AXL_Config_Set(const kvtree* config) -{ - ssize_t bytecount; - axlsvc_Request request; - axlsvc_Response response; - - request.request = AXLSVC_AXL_CONFIG_SET; - request.payload_length = (ssize_t)kvtree_pack_size(config); - - bytecount = axl_write_attempt("AXLSVC Client --> AXL_Config_Set_1", - axlsvc_socket, &request, sizeof(request)); - - if (bytecount != sizeof(request)) { - AXL_ABORT(-1, "Unexpected Write Response to server: Expected %d, Got %d", - sizeof(request), bytecount); - } - - bytecount = kvtree_write_fd("AXLSVC Client --> AXL_Config_Set_2", - axlsvc_socket, config); - - if (bytecount != request.payload_length) { - AXL_ABORT(-1, "Unexpected Write Response to server: Expected %d, Got %d", - request.payload_length, bytecount); - } - - bytecount = axl_read("AXLSVC Client <-- Response", - axlsvc_socket, &response, sizeof(response)); - - if (bytecount != sizeof(response)) { - AXL_ABORT(-1, "Unexpected Write Response to server: Expected %d, Got %d", - sizeof(response), bytecount); - } - - if (response.response != AXLSVC_SUCCESS) { - AXL_ABORT(-1, "Unexpected Response from server: %d", response.response); - } -} diff --git a/src/axl_service_server.c b/src/axl_service_server.c deleted file mode 100644 index 862b1b1..0000000 --- a/src/axl_service_server.c +++ /dev/null @@ -1,175 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - -#include "axl_internal.h" -#include "axl_service.h" -#include "kvtree.h" - -#define AXLSVC_MAX_CLIENTS 16 -struct axl_connection_ctx { - int sd; /* Connection to our socket */ - struct axl_transfer_array xfr; /* Pointer to client-specific xfer array */ -} axl_connection_ctx_array[AXLSVC_MAX_CLIENTS]; - -static kvtree* service_request_AXL_Config_Set(int sd) -{ - kvtree* config = kvtree_new(); - kvtree* rval; - ssize_t bytecount; - - bytecount = kvtree_read_fd("Service_AXL_Config_Set", sd, config); - - return rval; -} - -static ssize_t service_request_from_client(int sd) -{ - ssize_t bytecount; - axlsvc_Request req; - axlsvc_Response response; - char* buffer; - - bytecount = axl_read("AXLSVC Client Reqeust", sd, &req, sizeof(req)); - - if (bytecount == 0) { - AXL_DBG(2, "Client for socket %d closed", sd); - return bytecount; - } - - buffer = malloc(req.payload_length); - - bytecount = axl_read("AXLSVC Reqeust Payload", sd, &buffer, req.payload_length); - - if (bytecount != req.payload_length) { - AXL_ABORT(-1, "Unexpected Payload Length: Expected %d, Got %d", req.payload_length, bytecount); - } - - switch (req.request) { - case AXLSVC_AXL_CONFIG_SET: - AXL_DBG(1, "AXLSVC_AXL_CONFIG_SET(kfile=%s", buffer); - response.response = AXLSVC_SUCCESS; - response.payload_length = 0; - bytecount = axl_write_attempt("AXLSVC Response to Client", sd, &response, sizeof(response)); - if (bytecount != sizeof(response)) { - AXL_ABORT(-1, "Unexpected Write Response to client: Expected %d, Got %d", - sizeof(response), bytecount); - } - break; - default: - AXL_ABORT(-1, "AXLSVC Unknown Request Type %d", req.request); - break; - } - - free(buffer); - return bytecount; -} - -int axlsvc_server_run(int port) -{ - int server_socket; - int opt = 1; - struct sockaddr_in address; - int addrlen; - int new_socket; - fd_set readfds; - int activity; - int max_sd; - - if ((server_socket = socket(AF_INET, SOCK_STREAM, 0)) == 0) { - AXL_ABORT(-1, "socket() failed: (%s)", strerror(errno)); - } - - if (setsockopt(server_socket, SOL_SOCKET, SO_REUSEADDR, - (char *)&opt, sizeof(opt)) < 0 ) { - AXL_ABORT(-1, "setsockopt() failed: (%s)", strerror(errno)); - } - - address.sin_family = AF_INET; - address.sin_addr.s_addr = INADDR_ANY; - address.sin_port = htons(port); - - if (bind(server_socket, (struct sockaddr *)&address, sizeof(address)) < 0) { - AXL_ABORT(-1, "bind() failed: (%s)", strerror(errno)); - } - - if (listen(server_socket, AXLSVC_MAX_CLIENTS) < 0) { - AXL_ABORT(-1, "listen() failed: (%s)", strerror(errno)); - } - - addrlen = sizeof(address); - - while (1) { - FD_ZERO(&readfds); - FD_SET(server_socket, &readfds); - max_sd = server_socket; - - for (int i = 0 ; i < AXLSVC_MAX_CLIENTS ; i++) { - if (axl_connection_ctx_array[i].sd > 0) - FD_SET(axl_connection_ctx_array[i].sd, &readfds); - - if (axl_connection_ctx_array[i].sd > max_sd) - max_sd = axl_connection_ctx_array[i].sd; - } - - activity = select(max_sd + 1 , &readfds , NULL , NULL , NULL); - - if (activity < 0 && errno != EINTR) { - AXL_ABORT(-1, "select() error: (%s)", strerror(errno)); - } - - if (FD_ISSET(server_socket, &readfds)) { - AXL_DBG(1, "Accepting new incomming connection"); - if ((new_socket = accept(server_socket, (struct sockaddr *)&address, - (socklen_t*)&addrlen)) < 0) { - AXL_ABORT(-1, "accept() error: (%s)", strerror(errno)); - } - - for (int i = 0; i < AXLSVC_MAX_CLIENTS; i++) { - if(axl_connection_ctx_array[i].sd == 0 ){ - axl_connection_ctx_array[i].sd = new_socket; - break; - } - } - AXL_DBG(2, "Connection established"); - } - - for ( int i = 0; i < AXLSVC_MAX_CLIENTS; i++) { - if (FD_ISSET(axl_connection_ctx_array[i].sd , &readfds)) { - axl_xfer_list = &axl_connection_ctx_array[i].xfr; - - if (service_request_from_client(axl_connection_ctx_array[i].sd) == 0) { - AXL_DBG(2, "Closing server side socket(%d) to client", axl_connection_ctx_array[i].sd); - close(axl_connection_ctx_array[i].sd); - axl_connection_ctx_array[i].sd = 0; - axl_free(&axl_xfer_list->axl_kvtrees); - axl_xfer_list->axl_kvtrees_count = 0; - } - } - } - } - - return 0; -} - -int main(int argc , char *argv[]) -{ - int rval = -1; - - if (argc == 2 && atoi(argv[1]) > 0) { - axl_service_mode = AXLSVC_SERVER; - memset(axl_connection_ctx_array, 0, sizeof(axl_connection_ctx_array)); - - if ( (rval = AXL_Init()) == AXL_SUCCESS) - rval = axlsvc_server_run(atoi(argv[1])); - } else { - fprintf(stderr, "Usage: %s \n", argv[0]); - } - - return rval; -} \ No newline at end of file diff --git a/src/axl_socket.c b/src/axl_socket.c new file mode 100644 index 0000000..e39a25e --- /dev/null +++ b/src/axl_socket.c @@ -0,0 +1,289 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "axl_internal.h" +#include "axl_socket.h" +#include "kvtree.h" + +/* + * Flag to state whether the AXL client/server mode of operation is enabled, + * and if so, whether the code is running as the client or the server. + */ +axl_socket_RunMode axl_service_mode = AXL_SOCKET_DISABLED; + +static int axl_socket_socket = -1; + +/* + * Client implementation + */ +int axl_socket_client_init(char* host, unsigned short port) +{ + struct sockaddr_in server; + struct hostent *hostnm = gethostbyname(host); + + if (hostnm == (struct hostent *) 0) { + AXL_ERR("Gethostbyname failed: (%s)", strerror(errno)); + return 0; + } + + if ( (axl_socket_socket = socket(AF_INET, SOCK_STREAM, 0)) < 0) { + AXL_ERR("socket() failed: (%s)", strerror(errno)); + return 0; + } + + memset(&server, 0, sizeof(server)); + server.sin_family = AF_INET; + server.sin_port = htons(port); + server.sin_addr.s_addr = *((unsigned long *)hostnm->h_addr); + + if ( connect(axl_socket_socket, (struct sockaddr *)&server, sizeof(server) ) < 0) { + AXL_ERR("connect() failed: (%s)", strerror(errno)); + close(axl_socket_socket); + return 0; + } + + return 1; // success +} + +/* + * function to perform client-side request to server for AXL_Finalize() + */ +void axl_socket_client_AXL_Finalize() +{ + if (axl_socket_socket >= 0) + close(axl_socket_socket); +} + +/* + * function to perform client-side request to server for AXL_Config_Set + */ +void axl_socket_client_AXL_Config_Set(const kvtree* config) +{ + ssize_t bytecount; + axl_socket_Request request; + axl_socket_Response response; + + request.request = AXL_SOCKET_AXL_CONFIG_SET; + request.payload_length = (ssize_t)kvtree_pack_size(config); + + bytecount = axl_write_attempt("AXLSVC Client --> AXL_Config_Set_1", + axl_socket_socket, &request, sizeof(request)); + + if (bytecount != sizeof(request)) { + AXL_ABORT(-1, "Unexpected Write Response to server: Expected %d, Got %d", + sizeof(request), bytecount); + } + + bytecount = kvtree_write_fd("AXLSVC Client --> AXL_Config_Set_2", + axl_socket_socket, config); + + if (bytecount != request.payload_length) { + AXL_ABORT(-1, "Unexpected Write Response to server: Expected %d, Got %d", + request.payload_length, bytecount); + } + + bytecount = axl_read("AXLSVC Client <-- Response", + axl_socket_socket, &response, sizeof(response)); + + if (bytecount != sizeof(response)) { + AXL_ABORT(-1, "Unexpected Write Response to server: Expected %d, Got %d", + sizeof(response), bytecount); + } + + if (response.response != AXL_SOCKET_SUCCESS) { + AXL_ABORT(-1, "Unexpected Response from server: %d", response.response); + } +} + +/* + * Server Implementation + */ + +static int time_to_leave = 0; + +#define AXL_SOCKET_MAX_CLIENTS 16 +struct axl_socket_conn_ctx { + int sd; /* Connection to our socket */ + struct axl_transfer_array xfr; /* Pointer to client-specific xfer array */ +} axl_socket_conn_ctx_array[AXL_SOCKET_MAX_CLIENTS]; + +#if 0 +static kvtree* service_request_AXL_Config_Set(int sd) +{ + kvtree* config = kvtree_new(); + kvtree* rval; + ssize_t bytecount; + + bytecount = kvtree_read_fd("Service_AXL_Config_Set", sd, config); + + return bytecount; +} +#endif + +static ssize_t service_request_from_client(int sd) +{ + ssize_t bytecount; + axl_socket_Request req; + axl_socket_Response response; + char* buffer; + + bytecount = axl_read("AXLSVC Client Reqeust", sd, &req, sizeof(req)); + + if (bytecount == 0) { + AXL_DBG(2, "Client for socket %d closed", sd); + return bytecount; + } + + buffer = malloc(req.payload_length); + + bytecount = axl_read("AXLSVC Reqeust Payload", sd, &buffer, req.payload_length); + + if (bytecount != req.payload_length) { + AXL_ABORT(-1, "Unexpected Payload Length: Expected %d, Got %d", req.payload_length, bytecount); + } + + switch (req.request) { + case AXL_SOCKET_AXL_CONFIG_SET: + AXL_DBG(1, "AXL_SOCKET_AXL_CONFIG_SET(kfile=%s", buffer); + response.response = AXL_SOCKET_SUCCESS; + response.payload_length = 0; + bytecount = axl_write_attempt("AXLSVC Response to Client", sd, &response, sizeof(response)); + if (bytecount != sizeof(response)) { + AXL_ABORT(-1, "Unexpected Write Response to client: Expected %d, Got %d", + sizeof(response), bytecount); + } + break; + default: + AXL_ABORT(-1, "AXLSVC Unknown Request Type %d", req.request); + break; + } + + free(buffer); + return bytecount; +} + +static void sigterm_handler(int sig, siginfo_t* info, void* ucontext) +{ + time_to_leave++; +} + +static int use_sigterm_to_exit() +{ + struct sigaction act = {0}; + + act.sa_flags = 0; + sigemptyset(&act.sa_mask); + act.sa_sigaction = sigterm_handler; + if (sigaction(SIGTERM, &act, NULL) == -1) { + perror("sigaction"); + return AXL_FAILURE; + } + + return AXL_SUCCESS; +} + +int axl_socket_server_run(int port) +{ + int server_socket; + int opt = 1; + struct sockaddr_in address; + int addrlen; + int new_socket; + fd_set readfds; + int activity; + int max_sd; + int rval = AXL_FAILURE; + + axl_service_mode = AXL_SOCKET_SERVER; + memset(axl_socket_conn_ctx_array, 0, sizeof(axl_socket_conn_ctx_array)); + + /* + * Need to check whether calling AXL_Init() at this point is really appropriate + */ + if ( (rval = AXL_Init()) != AXL_SUCCESS) + return rval; + + if ((rval = use_sigterm_to_exit()) != AXL_SUCCESS) + return rval; + + if ((server_socket = socket(AF_INET, SOCK_STREAM, 0)) == 0) { + AXL_ABORT(-1, "socket() failed: (%s)", strerror(errno)); + } + + if (setsockopt(server_socket, SOL_SOCKET, SO_REUSEADDR, (char *)&opt, sizeof(opt)) < 0 ) { + AXL_ABORT(-1, "setsockopt() failed: (%s)", strerror(errno)); + } + + address.sin_family = AF_INET; + address.sin_addr.s_addr = INADDR_ANY; + address.sin_port = htons(port); + + if (bind(server_socket, (struct sockaddr *)&address, sizeof(address)) < 0) { + AXL_ABORT(-1, "bind() failed: (%s)", strerror(errno)); + } + + if (listen(server_socket, AXL_SOCKET_MAX_CLIENTS) < 0) { + AXL_ABORT(-1, "listen() failed: (%s)", strerror(errno)); + } + + addrlen = sizeof(address); + + while (!time_to_leave) { + FD_ZERO(&readfds); + FD_SET(server_socket, &readfds); + max_sd = server_socket; + + for (int i = 0 ; i < AXL_SOCKET_MAX_CLIENTS ; i++) { + if (axl_socket_conn_ctx_array[i].sd > 0) + FD_SET(axl_socket_conn_ctx_array[i].sd, &readfds); + + if (axl_socket_conn_ctx_array[i].sd > max_sd) + max_sd = axl_socket_conn_ctx_array[i].sd; + } + + activity = select(max_sd + 1 , &readfds , NULL , NULL , NULL); + + if (activity < 0 && errno != EINTR) { + AXL_ABORT(-1, "select() error: (%s)", strerror(errno)); + } + + if (FD_ISSET(server_socket, &readfds)) { + AXL_DBG(0, "Accepting new incomming connection"); + if ((new_socket = accept(server_socket, (struct sockaddr *)&address, + (socklen_t*)&addrlen)) < 0) { + AXL_ABORT(-1, "accept() error: (%s)", strerror(errno)); + } + + for (int i = 0; i < AXL_SOCKET_MAX_CLIENTS; i++) { + if(axl_socket_conn_ctx_array[i].sd == 0 ){ + axl_socket_conn_ctx_array[i].sd = new_socket; + break; + } + } + AXL_DBG(0, "Connection established"); + } + + for ( int i = 0; i < AXL_SOCKET_MAX_CLIENTS; i++) { + if (FD_ISSET(axl_socket_conn_ctx_array[i].sd , &readfds)) { + axl_xfer_list = &axl_socket_conn_ctx_array[i].xfr; + + if (service_request_from_client(axl_socket_conn_ctx_array[i].sd) == 0) { + AXL_DBG(0, "Closing server side socket(%d) to client", axl_socket_conn_ctx_array[i].sd); + close(axl_socket_conn_ctx_array[i].sd); + axl_socket_conn_ctx_array[i].sd = 0; + axl_free(&axl_xfer_list->axl_kvtrees); + axl_xfer_list->axl_kvtrees_count = 0; + } + } + } + } + return 0; +} + diff --git a/src/axl_socket.h b/src/axl_socket.h new file mode 100644 index 0000000..e3d279a --- /dev/null +++ b/src/axl_socket.h @@ -0,0 +1,60 @@ +#ifndef AXL_SOCKET_H +#define AXL_SOCKET_H + +#include +#include "kvtree.h" + +#if defined(__cplusplus) +extern "C" { +#endif + +typedef enum { + AXL_SOCKET_DISABLED = 0, /* Default - Not utilizing AXL service (lib only) */ + AXL_SOCKET_CLIENT = 1, /* Using AXL service and we are the client */ + AXL_SOCKET_SERVER = 2 /* Using AXL service and we are the server */ +} axl_socket_RunMode; + +/* + * Flag to state whether the AXL client/server mode of operation is enabled, + * and if so, whether the code is running as the client or the server. + */ +extern axl_socket_RunMode axl_service_mode; + +typedef enum { + AXL_SOCKET_AXL_CONFIG_SET = 0, /* payload is config kvtree hash buffer */ +} axl_socket_request_t; + +typedef struct { + axl_socket_request_t request; + ssize_t payload_length; +} axl_socket_Request; + +typedef enum { + AXL_SOCKET_SUCCESS = 0, + AXL_SOCKET_FAILURE = -1, +} axl_socket_response_t; + +typedef struct { + axl_socket_response_t response; + ssize_t payload_length; // Optional error/status string +} axl_socket_Response; + +int axl_socket_client_init(char* host, unsigned short port); + +/* + * function to perform client-side request to server for AXL_Finalize() + */ +void axl_socket_client_AXL_Finalize(); + +/* + * function to perform client-side request to server for AXL_Config_Set + */ +void axl_socket_client_AXL_Config_Set(const kvtree* config); + +int axl_socket_server_run(int port); + +#if defined(__cplusplus) +extern "C" } +#endif + +#endif /* AXL_SOCKET_H */ diff --git a/src/axl_socket_daemon.c b/src/axl_socket_daemon.c new file mode 100644 index 0000000..16fca7d --- /dev/null +++ b/src/axl_socket_daemon.c @@ -0,0 +1,18 @@ +#include +#include + +#include "axl_internal.h" +#include "axl_socket.h" + +int main(int argc , char *argv[]) +{ + int rval = AXL_FAILURE; + + if (argc == 2 && atoi(argv[1]) > 0) { + rval = axl_socket_server_run(atoi(argv[1])); + } else { + fprintf(stderr, "Usage: %s \n", argv[0]); + } + + return rval; +} diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 5673318..fa367f8 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -3,6 +3,9 @@ ############### INCLUDE_DIRECTORIES(${PROJECT_BINARY_DIR}) +MESSAGE(STATUS "AXL_BINARY_DIR: ${AXL_BINARY_DIR}") + + LIST(APPEND axl_test_srcs axl_cp.c) # Build with debug symbols (-g) diff --git a/test/test_client_server.c b/test/test_client_server.c index 1640338..9df4b02 100644 --- a/test/test_client_server.c +++ b/test/test_client_server.c @@ -4,7 +4,6 @@ #include #include #include -#include #include "axl.h" @@ -14,43 +13,14 @@ #define AXLCS_SERVICE_KILLED 2000 #define AXLCS_SERVICE_FAIL 3000 -static int time_to_leave = 0; - -void sigterm_handler(int sig, siginfo_t* info, void* ucontext) -{ - time_to_leave++; -} - -int use_sigterm_to_exit() -{ - struct sigaction act = {0}; - - act.sa_flags = 0; - sigemptyset(&act.sa_mask); - act.sa_sigaction = sigterm_handler; - if (sigaction(SIGTERM, &act, NULL) == -1) { - perror("sigaction"); - return AXLCS_SERVICE_FAIL; - } - - return AXLCS_SUCCESS; -} +extern int axl_socket_server_run(int port); int run_service() { - int rval; - if ((rval = use_sigterm_to_exit()) != AXLCS_SUCCESS) - return rval; - fprintf(stdout, "Service Started!\n"); - for (int i = 0; !time_to_leave && i < 100000; ++i) { - int seconds = 2+i; - fprintf(stdout, "Sleeping %d seconds\n", seconds); - fprintf(stderr, "Just testing stderr. %d ..\n", seconds); - sleep(seconds); - } + int rval = axl_socket_server_run(2000); fprintf(stdout, "Service Ending!\n"); - return AXLCS_SUCCESS; + return rval; } int run_client() From be8daad5b3c2bad43419ac69256ed5d1a4de5fc7 Mon Sep 17 00:00:00 2001 From: Marty McFadden Date: Tue, 2 Jul 2024 10:51:54 -0700 Subject: [PATCH 17/17] AXL_Init and AXL_Finalize CI testing works --- src/axl_socket.c | 14 +++++++++----- test/test_client_server.c | 15 +++++++++++---- test/test_driver_client_server.py | 8 ++++++-- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/src/axl_socket.c b/src/axl_socket.c index e39a25e..c5d3035 100644 --- a/src/axl_socket.c +++ b/src/axl_socket.c @@ -127,7 +127,7 @@ static kvtree* service_request_AXL_Config_Set(int sd) } #endif -static ssize_t service_request_from_client(int sd) +static ssize_t axl_socket_request_from_client(int sd) { ssize_t bytecount; axl_socket_Request req; @@ -171,6 +171,7 @@ static ssize_t service_request_from_client(int sd) static void sigterm_handler(int sig, siginfo_t* info, void* ucontext) { + AXL_DBG(2, "SIGTERM Received"); time_to_leave++; } @@ -250,12 +251,15 @@ int axl_socket_server_run(int port) activity = select(max_sd + 1 , &readfds , NULL , NULL , NULL); + if (time_to_leave) + break; + if (activity < 0 && errno != EINTR) { AXL_ABORT(-1, "select() error: (%s)", strerror(errno)); } if (FD_ISSET(server_socket, &readfds)) { - AXL_DBG(0, "Accepting new incomming connection"); + AXL_DBG(1, "Accepting new incomming connection"); if ((new_socket = accept(server_socket, (struct sockaddr *)&address, (socklen_t*)&addrlen)) < 0) { AXL_ABORT(-1, "accept() error: (%s)", strerror(errno)); @@ -267,15 +271,15 @@ int axl_socket_server_run(int port) break; } } - AXL_DBG(0, "Connection established"); + AXL_DBG(1, "Connection established"); } for ( int i = 0; i < AXL_SOCKET_MAX_CLIENTS; i++) { if (FD_ISSET(axl_socket_conn_ctx_array[i].sd , &readfds)) { axl_xfer_list = &axl_socket_conn_ctx_array[i].xfr; - if (service_request_from_client(axl_socket_conn_ctx_array[i].sd) == 0) { - AXL_DBG(0, "Closing server side socket(%d) to client", axl_socket_conn_ctx_array[i].sd); + if (axl_socket_request_from_client(axl_socket_conn_ctx_array[i].sd) == 0) { + AXL_DBG(1, "Closing server side socket(%d) to client", axl_socket_conn_ctx_array[i].sd); close(axl_socket_conn_ctx_array[i].sd); axl_socket_conn_ctx_array[i].sd = 0; axl_free(&axl_xfer_list->axl_kvtrees); diff --git a/test/test_client_server.c b/test/test_client_server.c index 9df4b02..30eb498 100644 --- a/test/test_client_server.c +++ b/test/test_client_server.c @@ -25,10 +25,17 @@ int run_service() int run_client() { - fprintf(stdout, "Client Started!\n"); - sleep(2); - fprintf(stdout, "Client Ending!\n"); - return AXLCS_SUCCESS; + int rval; + + if ((rval = AXL_Init()) != AXL_SUCCESS) { + fprintf(stderr, "Call to AXL_Init failed with code: %d\n", rval); + } else { + if ((rval = AXL_Finalize()) != AXL_SUCCESS) { + fprintf(stderr, "Call to AXL_Init failed with code: %d\n", rval); + } + } + + return rval; } int main(int ac, char **av) diff --git a/test/test_driver_client_server.py b/test/test_driver_client_server.py index e5c9876..36f43db 100755 --- a/test/test_driver_client_server.py +++ b/test/test_driver_client_server.py @@ -3,6 +3,7 @@ import sys import subprocess import os +import time def wait_for_completion(procname, proc, wait_time): try: @@ -20,8 +21,11 @@ def wait_for_completion(procname, proc, wait_time): if __name__ == '__main__': errors = 0 # Launch the server then the client - server = subprocess.Popen(['./test_client_server', '--server'], env=dict(os.environ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) - client = subprocess.Popen(['./test_client_server', '--client'], env=dict(os.environ), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) + test_env = {'AXL_DEBUG': '44', 'AXL_SERVICE_HOST': 'localhost', 'AXL_SERVICE_PORT': '2000'} + server = subprocess.Popen(['./test_client_server', '--server'], env=dict(os.environ, **test_env), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) + time.sleep(2) # Give server a chance to start + + client = subprocess.Popen(['./test_client_server', '--client'], env=dict(os.environ, **test_env), stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False) # Wait for the client then the server to finish client_ecode, client_out, client_err = wait_for_completion("axl_client", client, 30)