From eb2e40c370c6329d3431f5df976cc6e337b50862 Mon Sep 17 00:00:00 2001 From: Lemongrass3110 Date: Wed, 26 Dec 2018 23:58:04 +0100 Subject: [PATCH] Added epoll support on linux (#3798) Added epoll event dispatching support on linux systems. You can enable it by adding the configure option --enable-epoll Credits to Hercules. --- conf/packet_athena.conf | 17 +++- configure | 64 +++++++++++++++ configure.in | 47 +++++++++++ src/common/socket.cpp | 176 +++++++++++++++++++++++++++++++++++++--- 4 files changed, 290 insertions(+), 14 deletions(-) diff --git a/conf/packet_athena.conf b/conf/packet_athena.conf index 76203bedb2..d68253b538 100644 --- a/conf/packet_athena.conf +++ b/conf/packet_athena.conf @@ -5,6 +5,19 @@ // Display debug reports (When something goes wrong during the report, the report is saved.) debug: no +// Linux/Epoll: Maximum Events per cycle +// Default Value: +// (Maximum Supported Connections)/2 +// NOTE: this controls the maximum collected socket-events per-cycle (call to epoll_wait()) +// for example settings this to 32 will allow up to 32 events (incoming data/new connections +// per server-cycle. +// NOTE: Recommended Settings is at least half the maximum supported connections +// Settings this to a lower value, may cause lags/delays +// Depending on available CPU Time +// NOTE: This Setting is only available on Linux when build using EPoll as event dispatcher! +// +//epoll_maxevents: 1024 + // How long can a socket stall before closing the connection (in seconds) stall_time: 60 @@ -22,7 +35,7 @@ enable_ip_rules: yes order: deny,allow // order: allow,deny -// order: mutual-failture +// order: mutual-failure // IP rules // allow : Accepts connections from the ip range (even if flagged as DDoS) @@ -45,7 +58,7 @@ order: deny,allow ddos_interval: 3000 // Consecutive attempts trigger -// (default is 5 attemps) +// (default is 5 attempts) ddos_count: 5 // The time interval after which the threat of DDoS is assumed to be gone. (msec) diff --git a/configure b/configure index ff74f6e053..f121a671ab 100755 --- a/configure +++ b/configure @@ -698,6 +698,7 @@ ac_user_opts=' enable_option_checking enable_manager enable_packetver +enable_epoll enable_debug enable_prere enable_vip @@ -1339,6 +1340,7 @@ Optional Features: --enable-manager=ARG memory managers: no, builtin, memwatch, dmalloc, gcollect, bcheck (defaults to builtin) --enable-packetver=ARG Sets the PACKETVER define. (see src/common/mmo.h) + --enable-epoll use epoll(4) on Linux --enable-debug[=ARG] Compiles extra debug code. (disabled by default) (available options: yes, no, gdb) --enable-prere[=ARG] Compiles serv in prere mode. (disabled by default) @@ -3236,6 +3238,55 @@ fi +# +# Epoll +# +# Check whether --enable-epoll was given. +if test "${enable_epoll+set}" = set; then : + enableval=$enable_epoll; enable_epoll=$enableval +else + enable_epoll=no + +fi + +if test x$enable_epoll = xno; then + have_linux_epoll=no +else + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for Linux epoll(4)" >&5 +$as_echo_n "checking for Linux epoll(4)... " >&6; } + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + #ifndef __linux__ + #error This is not Linux + #endif + #include + +int +main () +{ +epoll_create1 (EPOLL_CLOEXEC); + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_link "$LINENO"; then : + have_linux_epoll=yes +else + have_linux_epoll=no + +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $have_linux_epoll" >&5 +$as_echo "$have_linux_epoll" >&6; } +fi +if test x$enable_epoll,$have_linux_epoll = xyes,no; then + as_fn_error $? "epoll support explicitly enabled but not available" "$LINENO" 5 +fi + + + # # debug # @@ -5920,6 +5971,19 @@ if test -n "$enable_packetver" ; then fi +# +# Epoll +# +case $have_linux_epoll in + "yes") + CPPFLAGS="$CPPFLAGS -DSOCKET_EPOLL" + ;; + "no") + # default value + ;; +esac + + # # Debug # diff --git a/configure.in b/configure.in index 5220e46ef6..6101c43ed6 100644 --- a/configure.in +++ b/configure.in @@ -53,6 +53,40 @@ AC_ARG_ENABLE( ) +# +# Epoll +# +AC_ARG_ENABLE( + [epoll], + AC_HELP_STRING( + [--enable-epoll], + [use epoll(4) on Linux] + ), + [enable_epoll=$enableval], + [enable_epoll=no] +) +if test x$enable_epoll = xno; then + have_linux_epoll=no +else + AC_MSG_CHECKING([for Linux epoll(4)]) + AC_LINK_IFELSE([AC_LANG_PROGRAM( + [ + #ifndef __linux__ + #error This is not Linux + #endif + #include + ], + [epoll_create1 (EPOLL_CLOEXEC);])], + [have_linux_epoll=yes], + [have_linux_epoll=no] + ) + AC_MSG_RESULT([$have_linux_epoll]) +fi +if test x$enable_epoll,$have_linux_epoll = xyes,no; then + AC_MSG_ERROR([epoll support explicitly enabled but not available]) +fi + + # # debug # @@ -985,6 +1019,19 @@ if test -n "$enable_packetver" ; then fi +# +# Epoll +# +case $have_linux_epoll in + "yes") + CPPFLAGS="$CPPFLAGS -DSOCKET_EPOLL" + ;; + "no") + # default value + ;; +esac + + # # Debug # diff --git a/src/common/socket.cpp b/src/common/socket.cpp index 1a1f338bea..e2c1965e74 100644 --- a/src/common/socket.cpp +++ b/src/common/socket.cpp @@ -9,22 +9,34 @@ #include "winapi.hpp" #else #include -#include - #include - #include -#include - #include #include + #include + #include + #include + #include + #include + #include + + #if defined(__linux__) || defined(__linux) + #include + + #ifdef SOCKET_EPOLL + #include + #endif + #else + #include + #include + #endif #ifndef SIOCGIFCONF - #include // SIOCGIFCONF on Solaris, maybe others? [Shinomori] + #include // SIOCGIFCONF on Solaris, maybe others? [Shinomori] #endif #ifndef FIONBIO - #include // FIONBIO on Solaris [FlavioJS] + #include // FIONBIO on Solaris [FlavioJS] #endif #ifdef HAVE_SETRLIMIT - #include + #include #endif #endif @@ -203,7 +215,17 @@ char* sErr(int code) #define MSG_NOSIGNAL 0 #endif -fd_set readfds; +#ifndef SOCKET_EPOLL + // Select based Event Dispatcher + fd_set readfds; +#else + // Epoll based Event Dispatcher + static int epoll_maxevents = (FD_SETSIZE / 2); + static int epfd = SOCKET_ERROR; + static struct epoll_event epevent; + static struct epoll_event *epevents = nullptr; +#endif + int fd_max; time_t last_tick; time_t stall_time = 60; @@ -478,8 +500,22 @@ int connect_client(int listen_fd) } #endif - if( fd_max <= fd ) fd_max = fd + 1; +#ifndef SOCKET_EPOLL + // Select Based Event Dispatcher sFD_SET(fd,&readfds); +#else + // Epoll based Event Dispatcher + epevent.data.fd = fd; + epevent.events = EPOLLIN; + + if( epoll_ctl( epfd, EPOLL_CTL_ADD, fd, &epevent ) == SOCKET_ERROR ){ + ShowError( "connect_client: Failed to add to epoll event dispatcher for new socket #%d: %s\n", fd, error_msg() ); + sClose( fd ); + return -1; + } +#endif + + if( fd_max <= fd ) fd_max = fd + 1; create_session(fd, recv_to_fifo, send_from_fifo, default_func_parse); session[fd]->client_addr = ntohl(client_address.sin_addr.s_addr); @@ -531,8 +567,22 @@ int make_listen_bind(uint32 ip, uint16 port) exit(EXIT_FAILURE); } - if(fd_max <= fd) fd_max = fd + 1; +#ifndef SOCKET_EPOLL + // Select Based Event Dispatcher sFD_SET(fd, &readfds); +#else + // Epoll based Event Dispatcher + epevent.data.fd = fd; + epevent.events = EPOLLIN; + + if( epoll_ctl( epfd, EPOLL_CTL_ADD, fd, &epevent ) == SOCKET_ERROR ){ + ShowError( "make_listen_bind: failed to add listener socket #%d to epoll event dispatcher: %s\n", fd, error_msg() ); + sClose(fd); + exit(EXIT_FAILURE); + } +#endif + + if(fd_max <= fd) fd_max = fd + 1; create_session(fd, connect_client, null_send, null_parse); session[fd]->client_addr = 0; // just listens @@ -643,8 +693,22 @@ int make_connection(uint32 ip, uint16 port, bool silent,int timeout) { set_nonblocking(fd, 1); #endif - if (fd_max <= fd) fd_max = fd + 1; +#ifndef SOCKET_EPOLL + // Select Based Event Dispatcher sFD_SET(fd,&readfds); +#else + // Epoll based Event Dispatcher + epevent.data.fd = fd; + epevent.events = EPOLLIN; + + if( epoll_ctl( epfd, EPOLL_CTL_ADD, fd, &epevent ) == SOCKET_ERROR ){ + ShowError( "make_connection: failed to add socket #%d to epoll event dispatcher: %s\n", fd, error_msg() ); + sClose(fd); + return -1; + } +#endif + + if (fd_max <= fd) fd_max = fd + 1; create_session(fd, recv_to_fifo, send_from_fifo, default_func_parse); session[fd]->client_addr = ntohl(remote_address.sin_addr.s_addr); @@ -821,8 +885,10 @@ int WFIFOSET(int fd, size_t len) int do_sockets(t_tick next) { +#ifndef SOCKET_EPOLL fd_set rfd; struct timeval timeout; +#endif int ret,i; // PRESEND Timers are executed before do_sendrecv and can send packets and/or set sessions to eof. @@ -840,6 +906,9 @@ int do_sockets(t_tick next) } #endif +#ifndef SOCKET_EPOLL + // Select based Event Dispatcher + // can timeout until the next tick timeout.tv_sec = (long)(next/1000); timeout.tv_usec = (long)(next%1000*1000); @@ -856,6 +925,20 @@ int do_sockets(t_tick next) } return 0; // interrupted by a signal, just loop and try again } +#else + // Epoll based Event Dispatcher + + ret = epoll_wait( epfd, epevents, epoll_maxevents, next ); + + if( ret == SOCKET_ERROR ){ + if( sErrno != S_EINTR ){ + ShowFatalError( "do_sockets: epoll_wait() failed, %s!\n", error_msg() ); + exit( EXIT_FAILURE ); + } + + return 0; // interrupted by a signal, just loop and try again + } +#endif last_tick = time(NULL); @@ -867,6 +950,26 @@ int do_sockets(t_tick next) if( session[fd] ) session[fd]->func_recv(fd); } +#elif defined(SOCKET_EPOLL) + // epoll based selection + + for( i = 0; i < ret; i++ ){ + struct epoll_event *it = &epevents[i]; + int fd = it->data.fd; + struct socket_data *sock = session[fd]; + + if( !sock ){ + continue; + } + + if( ( it->events & (EPOLLERR|EPOLLHUP) ) || !( it->events & EPOLLIN ) ){ + // Got Error on this connection + set_eof( fd ); + }else if( it->events & EPOLLIN ){ + // data waiting + sock->func_recv( fd ); + } + } #else // otherwise assume that the fd_set is a bit-array and enumerate it in a standard way for( i = 1; ret && i < fd_max; ++i ) @@ -1238,6 +1341,17 @@ int socket_config_read(const char* cfgName) ddos_autoreset = atoi(w2); else if (!strcmpi(w1,"debug")) access_debug = config_switch(w2); +#ifdef SOCKET_EPOLL + else if( !strcmpi( w1, "epoll_maxevents" ) ){ + epoll_maxevents = atoi(w2); + + // minimum that seems to be useful + if( epoll_maxevents < 16 ){ + ShowWarning( "socket_config_read: epoll_maxevents is set too low. Defaulting to 16...\n" ); + epoll_maxevents = 16; + } + } +#endif #endif else if (!strcmpi(w1, "import")) socket_config_read(w2); @@ -1287,6 +1401,16 @@ void socket_final(void) if( WSACleanup() != 0 ){ ShowError("socket_final: WinSock could not be cleaned up! %s\n", error_msg() ); } +#elif defined(SOCKET_EPOLL) + if( epfd != SOCKET_ERROR ){ + sClose(epfd); + epfd = SOCKET_ERROR; + } + + if( epevents != nullptr ){ + aFree( epevents ); + epevents = nullptr; + } #endif } @@ -1297,7 +1421,17 @@ void do_close(int fd) return;// invalid flush_fifo(fd); // Try to send what's left (although it might not succeed since it's a nonblocking socket) + +#ifndef SOCKET_EPOLL + // Select based Event Dispatcher sFD_CLR(fd, &readfds);// this needs to be done before closing the socket +#else + // Epoll based Event Dispatcher + epevent.data.fd = fd; + epevent.events = EPOLLIN; + epoll_ctl( epfd, EPOLL_CTL_DEL, fd, &epevent ); // removing the socket from epoll when it's being closed is not required but recommended +#endif + sShutdown(fd, SHUT_RDWR); // Disallow further reads/writes sClose(fd); // We don't really care if these closing functions return an error, we are just shutting down and not reusing this socket. if (session[fd]) delete_session(fd); @@ -1442,7 +1576,25 @@ void socket_init(void) // Get initial local ips naddr_ = socket_getips(addr_,16); +#ifndef SOCKET_EPOLL + // Select based Event Dispatcher: sFD_ZERO(&readfds); + ShowInfo( "Server uses '" CL_WHITE "select" CL_RESET "' as event dispatcher\n" ); +#else + // Epoll based Event Dispatcher + epfd = epoll_create( FD_SETSIZE ); // 2.6.8 or newer ignores the expected socket amount argument + + if( epfd == SOCKET_ERROR ){ + ShowError( "Failed to create epoll event dispatcher: %s\n", error_msg() ); + exit( EXIT_FAILURE ); + } + + memset( &epevent, 0x00, sizeof( struct epoll_event ) ); + epevents = (struct epoll_event *)aCalloc( epoll_maxevents, sizeof( struct epoll_event ) ); + + ShowInfo( "Server uses '" CL_WHITE "epoll" CL_RESET "' with up to " CL_WHITE "%d" CL_RESET " events per cycle as event dispatcher\n", epoll_maxevents ); +#endif + #if defined(SEND_SHORTLIST) memset(send_shortlist_set, 0, sizeof(send_shortlist_set)); #endif