I'm trying to validate some results that I see when using NetPipe to test some connectivity between a couple of Linux boxes (over various hardware). So, I concocted this simple client and server to do the same and I cannot seem to get the same numbers as NetPipe - I'm about 30-40% off the rtt times that it sees.Is there something stupid that I'm doing wrong with my simple example?Server:#include <stdio.h>      /* for printf() and fprintf() */#include <sys/socket.h> /* for socket(), bind(), and connect() */#include <arpa/inet.h>  /* for sockaddr_in and inet_ntoa() */#include <netinet/tcp.h>#include <stdlib.h>     /* for atoi() and exit() */#include <string.h>     /* for memset() */#include <unistd.h>     /* for close() */#include <stdio.h>  /* for perror() */#include <stdlib.h> /* for exit() */#define MAXPENDING 1void die(char *errorMessage){  perror(errorMessage);  exit(1);}void handle(unsigned short quickAck, int clntSock){  long long c_ts;                     /* current read timestamp */  int value = 1;  // Enable quickAck  if (quickAck && setsockopt(clntSock, IPPROTO_TCP, TCP_QUICKACK, (char *)&value, sizeof(int)) < 0)    die(TCP_QUICKACK failed);  /* Send received string and receive again until end of transmission */  while (recv(clntSock, (char*)&c_ts, sizeof(c_ts), 0) == sizeof(c_ts))      /* zero indicates end of transmission */  {    // Enable quickAck    if (quickAck && setsockopt(clntSock, IPPROTO_TCP, TCP_QUICKACK, (char *)&value, sizeof(int)) < 0)      die(TCP_QUICKACK failed);    /* Echo message back to client */    if (send(clntSock, (char*)&c_ts, sizeof(c_ts), 0) != sizeof(c_ts))      die(send() failed to send timestamp);    // Enable quickAck    if (quickAck && setsockopt(clntSock, IPPROTO_TCP, TCP_QUICKACK, (char *)&value, sizeof(int)) < 0)      die(TCP_QUICKACK failed);  }  close(clntSock);    /* Close client socket */}int main(int argc, char *argv[]){  int servSock;                    /* Socket descriptor for server */  int clntSock;                    /* Socket descriptor for client */  struct sockaddr_in echoServAddr; /* Local address */  struct sockaddr_in echoClntAddr; /* Client address */  unsigned short echoServPort;     /* Server port */  unsigned short quickAck;  unsigned int clntLen;            /* Length of client address data structure */  int value = 1;  if (argc != 3)     /* Test for correct number of arguments */  {    fprintf(stderr, Usage:  %s <Server Port> <Quick Ack>\n, argv[0]);    exit(1);  }  echoServPort = atoi(argv[1]);  /* First arg:  local port */  quickAck = atoi(argv[2]);      /* Whether quick ack is enabled or not */  /* Create socket for incoming connections */  if ((servSock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0)    die(socket() failed);  /* Construct local address structure */  memset(&echoServAddr, 0, sizeof(echoServAddr));   /* Zero out structure */  echoServAddr.sin_family = AF_INET;                /* Internet address family */  echoServAddr.sin_addr.s_addr = htonl(INADDR_ANY); /* Any incoming interface */  echoServAddr.sin_port = htons(echoServPort);      /* Local port */  /* Bind to the local address */  if (bind(servSock, (struct sockaddr *) &echoServAddr, sizeof(echoServAddr)) < 0)    die(bind() failed);  /* Mark the socket so it will listen for incoming connections */  if (listen(servSock, MAXPENDING) < 0)    die(listen() failed);  for (;;) /* Run forever */  {    /* Set the size of the in-out parameter */    clntLen = sizeof(echoClntAddr);    printf(Waiting for client...\n);    /* Wait for a client to connect */    if ((clntSock = accept(servSock, (struct sockaddr *) &echoClntAddr, &clntLen)) < 0)      die(accept() failed);    /* clntSock is connected to a client! */    printf(Handling client %s\n, inet_ntoa(echoClntAddr.sin_addr));    if (setsockopt(clntSock, IPPROTO_TCP, TCP_NODELAY, (char *)&value, sizeof(int)) < 0)      die(TCP_NODELAY failed);    handle(quickAck, clntSock);  }  /* NOT REACHED */}Client:#include <stdio.h>      /* for printf() and fprintf() */#include <sys/socket.h> /* for socket(), connect(), send(), and recv() */#include <arpa/inet.h>  /* for sockaddr_in and inet_addr() */#include <netinet/tcp.h>#include <stdlib.h>     /* for atoi() and exit() */#include <string.h>     /* for memset() */#include <unistd.h>     /* for close() */#include <sys/time.h>void die(char *errorMessage){  perror(errorMessage);  exit(1);}int main(int argc, char *argv[]){    int sock;                        /* Socket descriptor */    struct sockaddr_in echoServAddr; /* Echo server address */    unsigned short echoServPort;     /* Echo server port */    char *servIP;                    /* Server IP address (dotted quad) */    int iterations, gap, i;          /* Number of timestamps to send, and gap between each send */    struct timeval ts;    long long c_ts, o_ts, delta, total = 0, max = 0, min = 1000000000;    int value = 1;    if (argc != 5)    /* Test for correct number of arguments */    {       fprintf(stderr, Usage: %s <Server IP> <Server Port> <Iterations> <Gap>\n, argv[0]);       exit(1);    }    servIP = argv[1];              /* server IP address (dotted quad) */    echoServPort = atoi(argv[2]);  /* server port */    iterations = atoi(argv[3]);    /* number of timestamps to send */    gap = atoi(argv[4]);           /* gap between each send */    /* Create a reliable, stream socket using TCP */    if ((sock = socket(PF_INET, SOCK_STREAM, IPPROTO_TCP)) < 0)        die(socket() failed);    /* Construct the server address structure */    memset(&echoServAddr, 0, sizeof(echoServAddr));     /* Zero out structure */    echoServAddr.sin_family      = AF_INET;             /* Internet address family */    echoServAddr.sin_addr.s_addr = inet_addr(servIP);   /* Server IP address */    echoServAddr.sin_port        = htons(echoServPort); /* Server port */    /* Establish the connection to the echo server */    if (connect(sock, (struct sockaddr *) &echoServAddr, sizeof(echoServAddr)) < 0)        die(connect() failed);    if (setsockopt(sock, IPPROTO_TCP, TCP_NODELAY, (char *)&value, sizeof(int)) < 0)      die(TCP_NODELAY failed);    /* Give the server a chance */    usleep(1000);    /* Now for the given number of iterations */    for(i = 0; i < iterations; ++i)    {      /* Generate the current timestamp */      gettimeofday(&ts, NULL);      c_ts = ts.tv_sec * 1000000LL + ts.tv_usec;      //printf(sending %ld , c_ts);      /* Send this */      if (send(sock, (char*)&c_ts, sizeof(c_ts), 0) != sizeof(c_ts))        die(send() failed to send timestamp);      /* Now read the echo */      if (recv(sock, (char*)&o_ts, sizeof(o_ts), 0) != sizeof(o_ts))        die(recv() failed to read timestamp);      gettimeofday(&ts, NULL);      c_ts = ts.tv_sec * 1000000LL + ts.tv_usec;      /* Calculate the delta */      delta = c_ts - o_ts;      //printf( -> received %ld %ld\n, o_ts, delta);      if (i > 0)      {        /* Track max, min, sum */        total += delta;        max = (max < delta)? delta : max;        min = (min > delta)? delta : min;      }      /* Now sleep */      usleep(1000*gap);    }    --iterations;    printf(iterations %d, avg %f, max %ld, min %ld\n, iterations, (total/(double)iterations), max, min);    close(sock);    exit(0);}So, to run, start the server with the port and whether quick_ack is enabled or not (1/0) - this is for a different test. Something like:./simple_sever 10000 1Then run the client:./simple_client <host IP address> 10000 1000 1So send 1000 timestamps with a 1 millisecond gap between each. This, I guess, is where the above differs from NetPipe (which floods, as far as I know).The interaction is pretty straight forward, so is there something I'm missing?EDIT: Okay, I got to the bottom of the difference: caching. NetPIPE has an option to force invalidation of cache, and enabling this results in similar numbers to my test program. Phew, I don't have to re-evaluate my sockets programming! I'll leave this question up for reference I guess.
Stupidly simple TCP client/server
c;performance;socket;tcp
Not much to comment on, this program is pretty straightforward.  A few notes:The user of the client has a lot of information to input.   fprintf(stderr, Usage: %s <Server IP> <Server Port> <Iterations> <Gap>\n, argv[0]);The more the user has to enter, the steeper the initial learning curve to use the program is.  Also, I'm not sure I want the user to control the <Iterations> and the <Gap> anyways.  A malicious user might abuse this for a DOS of the server.  Eliminating those as required input and setting them in your code would be a more secure and a more user-friendly option.You have too many comments.   close(clntSock);    /* Close client socket */For that particular example, it is quite obvious what that statement does. There are a lot of other comparable examples in this code.It is more common to return 0; when main() is finished, rather than to exit(0).  Both will call the registered atexit handlers and will cause program termination though.