- tcp children control fds are now non-blocking
authorAndrei Pelinescu-Onciul <andrei@iptel.org>
Fri, 3 Feb 2006 16:15:18 +0000 (16:15 +0000)
committerAndrei Pelinescu-Onciul <andrei@iptel.org>
Fri, 3 Feb 2006 16:15:18 +0000 (16:15 +0000)
 - send fd queue is used when a childs queue is full

pass_fd.c
tcp_conn.h
tcp_main.c
test/gcc_versions.txt

index 43f6115..667df22 100644 (file)
--- a/pass_fd.c
+++ b/pass_fd.c
@@ -109,7 +109,8 @@ again:
        if (n<0){
                        /* error */
                if (errno==EINTR) goto again; /* signal, try again */
-               LOG(L_CRIT, "ERROR: send_all: send on %d failed: %s\n",
+               if ((errno!=EAGAIN) &&(errno!=EWOULDBLOCK))
+                       LOG(L_CRIT, "ERROR: send_all: send on %d failed: %s\n",
                                        socket, strerror(errno));
        }
        return n;
@@ -158,8 +159,9 @@ again:
        ret=sendmsg(unix_socket, &msg, 0);
        if (ret<0){
                if (errno==EINTR) goto again;
-               LOG(L_CRIT, "ERROR: send_fd: sendmsg failed on %d: %s\n",
-                               unix_socket, strerror(errno));
+               if ((errno!=EAGAIN) && (errno!=EWOULDBLOCK))
+                       LOG(L_CRIT, "ERROR: send_fd: sendmsg failed on %d: %s\n",
+                                       unix_socket, strerror(errno));
        }
        
        return ret;
index eaa5cbd..9ad741c 100644 (file)
@@ -43,7 +43,7 @@
 
 #define TCP_CON_MAX_ALIASES 4 /* maximum number of port aliases */
 
-#define TCP_BUF_SIZE 65535
+#define TCP_BUF_SIZE   4096 
 #define DEFAULT_TCP_CONNECTION_LIFETIME 120 /* in  seconds */
 #define DEFAULT_TCP_SEND_TIMEOUT 10 /* if a send can't write for more then 10s,
                                                                           timeout */
index b158466..b7d2994 100644 (file)
@@ -64,6 +64,8 @@
  *              process all children requests, before attempting to send
  *              them new stuff (fixes some deadlocks) (andrei)
  *  2006-02-03  timers are run only once per s (andrei)
+ *              tcp children fds can be non-blocking; send fds are queues on
+ *              EAGAIN (andrei)
  */
 
 
 
 #define MAX_TCP_CHILDREN 100
 #define TCP_LISTEN_BACKLOG 1024
-/* #define SEND_FD_QUEUE */ /* queue send fd request, instead of sending 
-                                                          them immediately */
+#define SEND_FD_QUEUE /* queue send fd requests on EAGAIN, instead of sending 
+                                                       them immediately */
+#define TCP_CHILD_NON_BLOCKING 
 #ifdef SEND_FD_QUEUE
-#define MAX_SEND_FD_QUEUE_SIZE tcp_max_fd_no
+#ifndef TCP_CHILD_NON_BLOCKING
+#define TCP_CHILD_NON_BLOCKING
+#endif
+#define MAX_SEND_FD_QUEUE_SIZE 1024  /* alternative: tcp_max_fd_no */
 #define SEND_FD_QUEUE_SIZE             128  /* initial size */
 #define MAX_SEND_FD_RETRIES            3        /* FIXME: increase */
 #endif
@@ -985,8 +991,6 @@ static void send_fd_queue_destroy(struct tcp_send_fd_q *q)
 
 static int init_send_fd_queues()
 {
-       if (send_fd_queue_init(&get_fd_q, SEND_FD_QUEUE_SIZE)!=0)
-               goto error;
        if (send_fd_queue_init(&send2child_q, SEND_FD_QUEUE_SIZE)!=0)
                goto error;
        return 0;
@@ -999,7 +1003,6 @@ error:
 
 static void destroy_send_fd_queues()
 {
-       send_fd_queue_destroy(&get_fd_q);
        send_fd_queue_destroy(&send2child_q);
 }
 
@@ -1053,13 +1056,14 @@ inline static void send_fd_queue_run(struct tcp_send_fd_q* q)
        for (p=t=&q->data[0]; p<q->crt; p++){
                if (send_fd(p->unix_sock, &(p->tcp_conn),
                                        sizeof(struct tcp_connection*), p->tcp_conn->s)<=0){
-                       if (/*FIXME: E_WOULD_BLOCK && */(p->retries<MAX_SEND_FD_RETRIES)){
+                       if ( ((errno==EAGAIN)||(errno==EWOULDBLOCK)) && 
+                                                       (p->retries<MAX_SEND_FD_RETRIES)){
                                /* leave in queue for a future try */
                                *t=*p;
                                t->retries++;
                                t++;
                        }else{
-                               LOG(L_ERR, "ERROR: rund_send_fd_queue: send_fd failed"
+                               LOG(L_ERR, "ERROR: run_send_fd_queue: send_fd failed"
                                                   "on %d socket, %ld queue entry, retries %d \n",
                                                   p->unix_sock, p-&q->data[0], p->retries);
                        }
@@ -1378,10 +1382,21 @@ inline static int send2child(struct tcp_connection* tcpconn)
        while(handle_tcp_child(&tcp_children[idx], -1)>0);
                
 #ifdef SEND_FD_QUEUE
-       if (send_fd_queue_add(&send2child_q, tcp_children[idx].unix_sock, 
+       /* if queue full, try to queue the io */
+       if (send_fd(tcp_children[idx].unix_sock, &tcpconn, sizeof(tcpconn),
+                       tcpconn->s)<=0){
+               if ((errno==EAGAIN)||(errno==EWOULDBLOCK)){
+                       /* FIXME: remove after debugging */
+                       LOG(L_WARN, "WARNING: tcp child %d, socket %d: queue full\n",
+                                       idx, tcp_children[idx].unix_sock);
+                       if (send_fd_queue_add(&send2child_q, tcp_children[idx].unix_sock, 
                                                tcpconn)!=0){
-               LOG(L_ERR, "ERROR: send2child: queue send op. failed\n");
-               return -1;
+                               LOG(L_ERR, "ERROR: send2child: queue send op. failed\n");
+                               return -1;
+                       }
+               }else{
+                       LOG(L_ERR, "ERROR: send2child: send_fd failed\n");
+               }
        }
 #else
        if (send_fd(tcp_children[idx].unix_sock, &tcpconn, sizeof(tcpconn),
@@ -1856,6 +1871,29 @@ error:
 
 
 
+/* returns -1 on error */
+static int set_non_blocking(int s)
+{
+       int flags;
+       /* non-blocking */
+       flags=fcntl(s, F_GETFL);
+       if (flags==-1){
+               LOG(L_ERR, "ERROR: set_non_blocking: fnctl failed: (%d) %s\n",
+                               errno, strerror(errno));
+               goto error;
+       }
+       if (fcntl(s, F_SETFL, flags|O_NONBLOCK)==-1){
+               LOG(L_ERR, "ERROR: set_non_blocking: fcntl: set non-blocking failed:"
+                               " (%d) %s\n", errno, strerror(errno));
+               goto error;
+       }
+       return 0;
+error:
+       return -1;
+}
+
+
+
 /* starts the tcp processes */
 int tcp_init_children()
 {
@@ -1893,6 +1931,15 @@ int tcp_init_children()
                                        strerror(errno));
                        goto error;
                }
+#ifdef TCP_CHILD_NON_BLOCKING
+               if ((set_non_blocking(reader_fd[0])<0) || 
+                       (set_non_blocking(reader_fd[1])<0)){
+                       LOG(L_ERR, "ERROR: tcp_main: failed to set non blocking"
+                                               "on child sockets\n");
+                       /* continue, it's not critical (it will go slower under
+                        * very high connection rates) */
+               }
+#endif
                
                process_no++;
                child_rank++;
index 47554b4..801611a 100644 (file)
@@ -6,6 +6,7 @@ egcs-2.91.66
 3.0.4
 gcc (GCC) 3.2
 gcc-3.2 (GCC) 3.2.3 (Debian)
+gcc (GCC) 3.3 20030304 (Apple Computer, Inc. build 1809)
 gcc (GCC) 3.3.3 (NetBSD nb3 20040520)
 gcc (GCC) 3.3.5 (propolice)
 gcc-3.3 (GCC) 3.3.5 (Debian 1:3.3.5-6)
@@ -13,6 +14,7 @@ gcc-3.3 (GCC) 3.3.5 (Debian 1:3.3.5-13)
 gcc (GCC) 3.4.2 [FreeBSD] 20040728
 gcc (GCC) 3.4.3 (csl-sol210-3_4-branch+sol_rpath)
 gcc-3.4 (GCC) 3.4.4 20041218 (prerelease) (Debian 3.4.3-7)
+powerpc-apple-darwin8-gcc-3.5.0 (GCC) 3.5.0 20041026 (Apple Computer, Inc. build 4059)
 powerpc-apple-darwin8-gcc-4.0.0 (GCC) 4.0.0 20041026 (Apple Computer, Inc. build 4061)
 gcc-4.0 (GCC) 4.0.2 20050821 (prerelease) (Debian 4.0.1-6)
 gcc-4.0.1 (GCC) 3.3.6 (Gentoo 3.3.6, ssp-3.3.6-1.0, pie-8.7.8)