Showing posts with label C. Show all posts
Showing posts with label C. Show all posts

Thursday, December 26, 2013

AIO with epoll event loop

In order to use libaio with epoll event loop use eventfd(2). With eventfd you create a descriptor and add it to epoll to observe and to libaio to notify when it has some events. Excerpt from source below
if ((afd = eventfd()) == -1)
    goto err_end;
...

io_set_eventfd(&iocb, afd);
...

ev.events = EPOLLIN | EPOLLET;
ev.data.ptr = p;
if (epoll_ctl(epollfd, EPOLL_CTL_ADD, afd, &ev) == -1) {
    log_syserr("epoll_ctl");
    goto err_end;
}
Full source is available.

Pipelining and flow control

If you are about to create your own application level protocol on top of TCP to load your backend to its limit you should know about how to design such a protocol. Two things that go into mind immediately are pipelining and flow control.
Pipelining is what you have from the box if you are using stream based transport layer. For higher level protocols one needs not to throw it away, for instance, HTTP 1.1 supports pipelining. In brief this is to eliminate latency and jitter between client and server.
To see what flow control is take a look at Akka IO Write models. This gives an ability to the server to say don't push at me, slow down. Indeed TCP too implements ack based flow control.
How to use it? For instance, imagine you have a queue of tasks on server side that is filled by clients and processed by backend. In case clients send tasks too quick the length of the queue grows. One needs to introduce so named high watermark and low watermark. If queue length is greater than high watermark stop reading from sockets and queue length will decrease. When queue length becomes less than low watermark start reading tasks from sockets again.
Note, to make it possible for clients to adapt to speed you process tasks (actually to adapt window size) one shouldn't make a big gap between high and low watermarks. From the other side small gap means you'll be too often add/remove sockets from the event loop.
Some excerpt from real project that uses libev below
//------------------------------------------------------------------

static request_s *request_new(connection_s *con) {
    request_s *new_request;

    new_request = alloc_data(request_mem_mng);
    if (!new_request) {
        log_err("cannot allocate memory");
        goto err;
    }
    memset(new_request, 0, sizeof(request_s));
    
    // Add to connection's list of requests
    list_add_tail(&new_request->request_list, &con->request_list);
    
    new_request->con = con;
    
    {
        // Flow control
        num_reqs++;
        
        if (num_reqs == REQUEST_HIGH_WATERMARK) {
            list_s *elt;
            connection_s *con;
            for (elt = connection_list.next; elt != &connection_list; elt = elt->next) {
                con = list_elt(elt, connection_s, connection_list);
                ev_io_stop(e_loop, &con->read_watcher);
            }
        }
    
    }
    
    return new_request;
err:
    return NULL;
}

//------------------------------------------------------------------

static void request_del(request_s *req) {
    list_del(&req->request_list);
    list_del(&req->request_wait);

    if (req->data)
        free_data(data_mem_mng, req->data);

    free_data(request_mem_mng, req);

    {
        // Flow control
        num_reqs--;
        
        if (num_reqs == REQUEST_LOW_WATERMARK) {
            list_s *elt;
            connection_s *con;
            for (elt = connection_list.next; elt != &connection_list; elt = elt->next) {
                con = list_elt(elt, connection_s, connection_list);
                ev_io_start(e_loop, &con->read_watcher);
            }
        }
    
    }
}
Full source is available.

Tuesday, December 24, 2013

libsmbclient3 and multithreading

If you want to use libsmbclient3 in process with many threads you are out of luck.
The first problem is talloc_stack.c in libsmbclient is not thread safe. Easy to fix
diff --git a/lib/util/talloc_stack.c b/lib/util/talloc_stack.c
index 8e559cc..b88962b 100644
--- a/lib/util/talloc_stack.c
+++ b/lib/util/talloc_stack.c
@@ -39,6 +39,11 @@
 
 #include "includes.h"
 
+
+#include <pthread.h>
+static pthread_mutex_t mutex_lock = PTHREAD_MUTEX_INITIALIZER;
+
+
 struct talloc_stackframe {
     int talloc_stacksize;
     int talloc_stack_arraysize;
@@ -82,7 +87,12 @@ static struct talloc_stackframe *talloc_stackframe_create(void)
         smb_panic("talloc_stackframe_init malloc failed");
     }
 
-    SMB_THREAD_ONCE(&ts_initialized, talloc_stackframe_init, NULL);
+    SMB_THREAD_LOCK(&mutex_lock);
+    if (!ts_initialized)
+        talloc_stackframe_init(NULL);
+    ts_initialized = true;
+    SMB_THREAD_UNLOCK(&mutex_lock);
+    //SMB_THREAD_ONCE(&ts_initialized, talloc_stackframe_init, NULL);
 
     if (SMB_THREAD_SET_TLS(global_ts, ts)) {
         smb_panic("talloc_stackframe_init set_tls failed");
@@ -118,6 +128,7 @@ static int talloc_pop(TALLOC_CTX *frame)
 static TALLOC_CTX *talloc_stackframe_internal(size_t poolsize)
 {
     TALLOC_CTX **tmp, *top, *parent;
+    if (global_ts == NULL) talloc_stackframe_init(NULL);
     struct talloc_stackframe *ts =
         (struct talloc_stackframe *)SMB_THREAD_GET_TLS(global_ts);
Second problem. Methods set_global_myname() and set_global_myworkgroup() in source3/lib/util_names.c are not thread safe too.
diff --git a/source3/lib/util_names.c b/source3/lib/util_names.c
index bd6e5c1..1d8a96e 100644
--- a/source3/lib/util_names.c
+++ b/source3/lib/util_names.c
@@ -27,17 +27,24 @@
 static char *smb_myname;
 static char *smb_myworkgroup;
 
+#include <pthread.h>
+static pthread_mutex_t mutex_lock = PTHREAD_MUTEX_INITIALIZER;
+
 /***********************************************************************
  Allocate and set myname. Ensure upper case.
 ***********************************************************************/
 
 bool set_global_myname(const char *myname)
 {
+    SMB_THREAD_LOCK(&mutex_lock);
     SAFE_FREE(smb_myname);
     smb_myname = SMB_STRDUP(myname);
-    if (!smb_myname)
+    if (!smb_myname) {
+        SMB_THREAD_UNLOCK(&mutex_lock);
         return False;
+    }
     strupper_m(smb_myname);
+    SMB_THREAD_UNLOCK(&mutex_lock);
     return True;
 }
 
@@ -52,11 +59,15 @@ const char *global_myname(void)
 
 bool set_global_myworkgroup(const char *myworkgroup)
 {
+    SMB_THREAD_LOCK(&mutex_lock);
     SAFE_FREE(smb_myworkgroup);
     smb_myworkgroup = SMB_STRDUP(myworkgroup);
-    if (!smb_myworkgroup)
+    if (!smb_myworkgroup) {
+        SMB_THREAD_UNLOCK(&mutex_lock);
         return False;
+    }
     strupper_m(smb_myworkgroup);
+    SMB_THREAD_UNLOCK(&mutex_lock);
     return True;
 }
What you still need to know about libsmbclient3 and multithreading? Each context (SMBCCTX) must be used only by one thread at the same time. All resources (open files and directories) are bound to specific context and one cannot use it in another.

libxenstore bug

Opensource is perfect till you hit the critical bug. Then you are doomed. Ok, just joking. There is a community, in other words there are a lot of doomed people like you and bugs like this.
An example. There was (and probably is) a bug in Xen qemu-dm. It can just hang forever due to some circumstances. Some is a key word here. First time I've seen this in Xen Opensource 3.4.2. Taking into account that qemu-dm is responsible for all IO of some VM, this VM hangs too.

The problem was in libxenstore that called pthread_cond_wait():
#define condvar_wait(c,m,hnd)   pthread_cond_wait(c,m)

mutex_lock(&h->watch_mutex);

/* Wait on the condition variable for a watch to fire. */
while (list_empty(&h->watch_list))
        condvar_wait(&h->watch_condvar, &h->watch_mutex, h);
msg = list_top(&h->watch_list, struct xs_stored_msg, list);
list_del(&msg->list);

/* Clear the pipe token if there are no more pending watches. */
if (list_empty(&h->watch_list) && (h->watch_pipe[0] != -1))
        while (read(h->watch_pipe[0], &c, 1) != 1)
                continue;
watch_pipe is used for notifications (for select) about events in watch_list. Separate thread is writing to this pipe during element addition to this list in method read_message():
#define condvar_signal(c)       pthread_cond_signal(c)

mutex_lock(&h->watch_mutex);

/* Kick users out of their select() loop. */
if (list_empty(&h->watch_list) &&
    (h->watch_pipe[1] != -1))
        while (write(h->watch_pipe[1], body, 1) != 1)
                continue;

list_add_tail(&msg->list, &h->watch_list);

condvar_signal(&h->watch_condvar);

mutex_unlock(&h->watch_mutex);
It looks ok but it is not. Sometimes select() reports a read readiness for watch_pipe[0] when watch_list is empty and pthread_cond_wait() hangs indefinitely. This is a big mistake. First of all, kernel can mistakenly give you false positives, second, even if your pipe contains something to read before you actually read from list some another thread can modify this list.
Having said that the patch is obvious
diff --git a/xen-3.4.2/tools/ioemu-qemu-xen/xenstore.c b/xen-3.4.2/tools/ioemu-qemu-xen/xenstore.c
index 11b305d..894e9a5 100644
--- a/xen-3.4.2/tools/ioemu-qemu-xen/xenstore.c
+++ b/xen-3.4.2/tools/ioemu-qemu-xen/xenstore.c
@@ -953,7 +953,7 @@ void xenstore_process_event(void *opaque)
     char **vec, *offset, *bpath = NULL, *buf = NULL, *drv = NULL, *image = NULL;
     unsigned int len, num, hd_index;
 
-    vec = xs_read_watch(xsh, &num);
+    vec = xs_read_watch_noblock(xsh, &num);
     if (!vec)
         return;
 
diff --git a/xen-3.4.2/tools/xenstore/xs.c b/xen-3.4.2/tools/xenstore/xs.c
index 9707d19..9ce5be6 100644
--- a/xen-3.4.2/tools/xenstore/xs.c
+++ b/xen-3.4.2/tools/xenstore/xs.c
@@ -611,11 +611,11 @@ bool xs_watch(struct xs_handle *h, const char *path, const char *token)
                 ARRAY_SIZE(iov), NULL));
 }
 
-/* Find out what node change was on (will block if nothing pending).
+/* Find out what node change was on.
  * Returns array of two pointers: path and token, or NULL.
  * Call free() after use.
  */
-char **xs_read_watch(struct xs_handle *h, unsigned int *num)
+static char **xs_read_watch_(struct xs_handle *h, unsigned int *num, bool block)
 {
     struct xs_stored_msg *msg;
     char **ret, *strings, c = 0;
@@ -623,9 +623,19 @@ char **xs_read_watch(struct xs_handle *h, unsigned int *num)
 
     mutex_lock(&h->watch_mutex);
 
-    /* Wait on the condition variable for a watch to fire. */
-    while (list_empty(&h->watch_list))
-        condvar_wait(&h->watch_condvar, &h->watch_mutex, h);
+    if (block) {
+        /* Wait on the condition variable for a watch to fire. */
+        while (list_empty(&h->watch_list))
+            condvar_wait(&h->watch_condvar, &h->watch_mutex, h);
+    }
+    else if (list_empty(&h->watch_list)) {
+        /* Clear the pipe token if there are no more pending watches. */
+        if (list_empty(&h->watch_list) && (h->watch_pipe[0] != -1))
+            while (read(h->watch_pipe[0], &c, 1) != 1)
+                continue;
+        mutex_unlock(&h->watch_mutex);
+        return NULL;
+    }
     msg = list_top(&h->watch_list, struct xs_stored_msg, list);
     list_del(&msg->list);
 
@@ -662,6 +672,15 @@ char **xs_read_watch(struct xs_handle *h, unsigned int *num)
     return ret;
 }
 
+char **xs_read_watch(struct xs_handle *h, unsigned int *num)
+{
+    return xs_read_watch_(h, num, true);
+}
+
+char **xs_read_watch_noblock(struct xs_handle *h, unsigned int *num)
+{
+    return xs_read_watch_(h, num, false);
+}
 /* Remove a watch on a node.
  * Returns false on failure (no watch on that node).
  */
diff --git a/xen-3.4.2/tools/xenstore/xs.h b/xen-3.4.2/tools/xenstore/xs.h
index bd36a0b..a921a40 100644
--- a/xen-3.4.2/tools/xenstore/xs.h
+++ b/xen-3.4.2/tools/xenstore/xs.h
@@ -109,6 +109,7 @@ int xs_fileno(struct xs_handle *h);
  * elements. Call free() after use.
  */
 char **xs_read_watch(struct xs_handle *h, unsigned int *num);
+char **xs_read_watch_noblock(struct xs_handle *h, unsigned int *num);
 
 /* Remove a watch on a node: implicitly acks any outstanding watch.
  * Returns false on failure (no watch on that node).

CORAID speed test

Hardware: 2x10Gb link, 36x2TB drives on CORAID SRX 4200.
aoe-stat
e0.X 2000.398GB eth2,eth3 8704 up
(36 drives total)
ethtool eth2
Speed: 10000Mb/s
ethtool eth3
Speed: 10000Mb/s
 
cec -s 0 eth2
disks
0.0 2000.398GB X.0.0 WDC WD2003FYYS-02W0B0 01.01D01 sata 3.0Gb/s
(36 drives total)
list -l
X 2000.399GB online
X.0 2000.399GB jbod normal
X.0.0 normal 2000.399GB 0.X
(36 JBODs total)
CORAID raw device speed (using aio, O_DIRECT to omit the cache, O_SYNC and different RAID types):
  • JBODs, 36 LUNs per 2TB, 4 simultaneous request/LUN, read and write by 4K blocks
50% reads, 50% writes = 220.45 iops/LUN, 2m5.678s/1000000 blocks
reads = 163.39 iops/LUN, 2m49.834s/1000000 blocks
writes = 455.37 iops/LUN, 1m0.602s/1000000 blocks
  • raid6rs, 4 LUNs per 14TB, 100 simultaneous request/LUN, read and write by 4K blocks
50% reads, 50% writes = 274.72 iops/LUN, 1m31.482s/100000 blocks
reads = 1524.39 iops/LUN, 2m43.648s/1000000 blocks
writes = 164.47 iops/LUN, 2m31.699s/100000 blocks
  • raid5, 4 LUNs per 16TB, 100 simultaneous request/LUN, read and write by 4K blocks
50% reads, 50% writes = 827.81 iops/LUN, 2m31.095s/500000 blocks
reads = 1519.94 iops/LUN, 2m44.482s/1000000 blocks
writes = 555.55 iops/LUN, 0m44.665s/100000 blocks

See the test program. You need libaio for it (POSIX aio in librt uses pthreads which is not appropriate because of massive parallelism in our tests).

SMP as easy as you cannot imagine

In contrast to MPP where MPI is a standard, in the world of SMP pthreads are de facto standard way to make your program execute in parallel. But there is another standard, OpenMP.
test_parallel.c
#include <stdio.h>
#include <unistd.h>

#ifdef _OPENMP
  #include <omp.h>
#else
  #define omp_get_thread_num() 0
#endif


int main(int argc, char *argv) {
  int n = 10;
  int i;
  
  #pragma omp parallel
  {
    printf("thread %d\n", omp_get_thread_num());

    #pragma omp for
    for (i = 1; i <= n; i++ ) {
      sleep(1);
    }
  }
}
Test
gcc test_parallel.c
time ./a.out 
thread 0

real    0m10.002s
user    0m0.000s
sys     0m0.001s
Enable OpenMP
gcc -fopenmp test_parallel.c
time ./a.out 
thread 3
thread 7
thread 0
thread 4
thread 1
thread 5
thread 6
thread 2

real    0m2.002s
user    0m0.009s
sys     0m0.004s

Smoke and unit tests for C

Take a look at vstr library source code. This is de facto a standard on tests implementation in C without any additional tools like googletest. See below what to do if you do not want to put one more tool to your project but still need tests.
tst_main.c
#include <stdlib.h>

#include "backup_app.h"
#include "tst.h"

int main (int argc, char ** argv) {

    DBHOST = "127.0.0.1";
    DBUSER = "root";
    DBDB = "???";
    DBPORT = 3306;

    return tst();
}


/*---------------------------------------------------
  mocks */

int snmp_log(int priority, const char *format, ...) {
    va_list ap;

    va_start(ap, format);

    if (vfprintf(stderr, format, ap) < 0)
        goto error;

    va_end(ap);
    return 0;


error:
    va_end(ap);
    return 1;
}
One of the tests, tst_cmd_ok.c
#include <stdlib.h>
#include "extern_cmd.h"
#include "tst.h"

int tst() {
    int     ret;
    char   *args[] = {"/bin/ls", "-l", NULL};
    char   *res = mb_cmd_exec(args, NULL);
    ret = (res) ? SUCCESS : FAILED;
    free(res);
    return ret;
}
Makefile
.PHONY: clean

CC=gcc

CFLAGS=-g -Wall -I..
LDFLAGS=-Wl,--unresolved-symbols=ignore-all
HEADERS=$(wildcard ../*.h)
OBJECTS=$(patsubst %.c,%.o,$(wildcard *.c))
TO_TEST_OBJECTS=$(patsubst %.c,%.o,$(wildcard ../*.c))
MAIN_TST_OBJ=tst_main.o
BUILDLIBS=-lmysqlclient_r -lpthread

_TESTS=$(patsubst %.c,%,$(wildcard *.c))
TESTS=$(patsubst tst_main,,$(_TESTS))

all: $(OBJECTS) $(TESTS)

%.o: %.c $(HEADERS)
$(CC) $(CFLAGS) -c $<

$(TESTS): $(TO_TEST_OBJECTS) $(MAIN_TST_OBJ) $(OBJECTS)
    $(CC) $(CFLAGS) $(LDFLAGS) $(patsubst %,%.o,$@) $(TO_TEST_OBJECTS) $(MAIN_TST_OBJ) -o $@ $(BUILDLIBS)

clean:
    rm -f *~ *.o $(TESTS)
Note LDFLAGS=-Wl,--unresolved-symbols=ignore-all
Script to start tests test.sh
#!/bin/sh

for result in `find . -maxdepth 1 -perm /u=x,g=x,o=x -type f -name 'tst_*'`;
do
  $result
  exit_st=$?
  if [ $exit_st -eq 0 ]
  then
    echo $result : ok
  else
    echo $result : failed
  fi
done

Simplistic ORM like thing in C for MySQL

Take a look at extern_mysql.h and extern_mysql.c. It is a simple request to struct mapping done in C for MySQL 5.0 and above for one of the projects. libmysql_r is not the simplest thing to work with, code provided was tested in production, so it works, take a look if you had a bad day and were forced to get data from MySQL in C.

A few words about an interface. To get data from mysql:
1) add request to mb_mysql_stmt:
mb_mysql_stmt_t mb_mysql_stmt[] = {
    ...
    {
        "select * from SCHEDULE",
        NULL, NULL, sizeof(mb_db_schedule_t), bind_r_schedule
    },
    ...
};
where bind_r_schedule is a binding definition:
mb_bind_t   bind_r_schedule[] = {
    {"schedule_id",     offsetof(mb_db_schedule_t, schedule_id),        sizeof(((mb_db_schedule_t*)0)->schedule_id),    0},
    {"name",            offsetof(mb_db_schedule_t, name),               sizeof(((mb_db_schedule_t*)0)->name),           0},
    {"schedule_type",   offsetof(mb_db_schedule_t, schedule_type),      sizeof(((mb_db_schedule_t*)0)->schedule_type),  0},
    {"enabled",         offsetof(mb_db_schedule_t, enabled),            sizeof(((mb_db_schedule_t*)0)->enabled),        0},
    {"last_backup",     offsetof(mb_db_schedule_t, last_backup),        sizeof(((mb_db_schedule_t*)0)->last_backup),    0},
    {NULL, 0, 0, 0}
};
This means that mb_db_exec() will return an array of mb_db_schedule_t, each element of this array will correspond to one row in the result set. To map fields in result set to struct members bindings are used, consult C API Prepared Statement Type Codes to map mysql types to C ones (for MYSQL_TIME mb_time_t is defined). The length of the array will be stored in variable pointed by the count argument of the mb_db_exec() function.
If you want to get single value from db you can use, for instance
mb_bind_t   bind_r_config_value[] = {
    {"value",   0,  256, 0},
    {NULL, 0, 0, 0}
};
mb_mysql_stmt_t mb_mysql_stmt[] = {
    ...
    {
        "select value from CONFIG where `key`='address'",
        NULL, NULL, 256, bind_r_config_value
    },
    ...
};
We do not use struct and map the whole row to buffer of 256 bytes (256 is the max length of the value field in config table), to obtain results call
char *data;
data = mb_db_exec(mb_db_ip_address_stmt, NULL);
if (!data)
    mb_log_err("cannot find ip address");
Note that we do not need count here. Also note that bindings can be reused (bind_r_config_value can be used to obtain various values from config table).
2) add request number to mb_db_stmt_e enum:
typedef enum {
    ...
    mb_db_ip_address_stmt
} mb_db_stmt_e;

Cgroups, limit memory

If some application may eat a lot of memory and cause swapping you can put it to sandbox
Test app memtest.c
#include <stdlib.h>
#include <stdio.h>


int main() {
    int     i;
    void   *p;

    p = NULL;

    for (i = 0; 1; i+= 4096) {
        p = malloc(4096);
        if (!p) {
            perror("malloc");
            break;
        }
        printf("allocated %d bytes\n", i);
    }

}
Create new group and add to it current shell process
mkdir /sys/fs/cgroup/memory/0
echo $$ > /sys/fs/cgroup/memory/0/tasks
Configure memory limit
echo 4M > /sys/fs/cgroup/memory/0/memory.limit_in_bytes
Test
gcc memtest.c
./a.out
...
allocated 3997696 bytes
Killed
As you mentioned malloc didn't return NULL, app receives a signal and terminates because swap is not used and kernel kills one of the processes when there is no enough free memory left.

Sunday, December 22, 2013

Don't parse output from system utilities

One often can see that some system utility returns information he needs. Then he does a wrong thing: parses output from this utility. We'll do opposite. For instance, we'll get broadcast address
/sbin/ifconfig eth1
eth1      Link encap:Ethernet  HWaddr 00:0A:CD:14:CD:77  
          inet addr:192.168.44.177  Bcast:192.168.44.255  Mask:255.255.255.0
          inet6 addr: fe80::20a:cdff:fe14:cd77/64 Scope:Link
          UP BROADCAST RUNNING MULTICAST  MTU:1500  Metric:1
          RX packets:791580 errors:0 dropped:0 overruns:0 frame:0
          TX packets:381581 errors:0 dropped:0 overruns:0 carrier:0
          collisions:0 txqueuelen:1000 
          RX bytes:468681235 (446.9 Mb)  TX bytes:47469801 (45.2 Mb)
          Interrupt:18 Base address:0xc000
What does ifconfig do to get this info?
strace /sbin/ifconfig eth1
...
socket(PF_INET, SOCK_DGRAM, IPPROTO_IP) = 4
...
ioctl(4, SIOCGIFBRDADDR, {ifr_name="eth1", ifr_broadaddr={AF_INET, inet_addr("192.168.44.255")}}) = 0
...
strace shows that descriptor 4 is passed to ioctl. In python one can do the same
# get the constant beforehand
grep -R SIOCGIFBRDADDR /usr/include/*                         
/usr/include/bits/ioctls.h:#define SIOCGIFBRDADDR  0x8919  /* get broadcast PA address */


import fcntl, socket, struct
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_IP)
SIOCGIFBRDADDR = 0x8919
iface = struct.pack('256s', 'eth1')
info = fcntl.ioctl(s.fileno(), SIOCGIFBRDADDR, iface)
socket.inet_ntoa(info[20:24])
'192.168.44.255'
Why we get bytes from 20 to 24? One passes struct ifreq to ioctl (see netdevice(7)), IFNAMSIZ is 16, plus offsetof(struct sockaddr_in, sin_addr), this equals to 20, and plus unsigned long that is 4 bytes.
struct ifreq {
    char ifr_name[IFNAMSIZ]; /* Interface name */
    union {
        struct sockaddr ifr_addr;
        struct sockaddr ifr_dstaddr;
        struct sockaddr ifr_broadaddr;
        struct sockaddr ifr_netmask;
        struct sockaddr ifr_hwaddr;
        short           ifr_flags;
        int             ifr_ifindex;
        int             ifr_metric;
        int             ifr_mtu;
        struct ifmap    ifr_map;
        char            ifr_slave[IFNAMSIZ];
        char            ifr_newname[IFNAMSIZ];
        char *          ifr_data;
    };
};
struct sockaddr_in {
    short            sin_family;
    unsigned short   sin_port;
    struct in_addr   sin_addr;
    char             sin_zero[8];
};
struct in_addr {
    unsigned long s_addr;
};
Note, there are no holes in these structs. One can expect 4 byte hole before sin_addr on 64 bit systems, but there is no. Those structures are declared in a way that omits holes. A simple test to show
#include <stdio.h>
#include <stddef.h>

#include <netinet/in.h>


struct in_addr2 {
    unsigned long s_addr;
};
struct sockaddr_in2 {
    short            sin_family;
    unsigned short   sin_port;
    struct in_addr2  sin_addr;
    char             sin_zero[8];
};

int main(void) {
    printf("%d\n", offsetof(struct sockaddr_in, sin_addr));
    printf("%d\n", offsetof(struct sockaddr_in2, sin_addr));
    return 0;
}

gcc 1.c
./a.out
4
8
With the help of strace you can find out a lot about utilities, one more example
strace ps
...
open("/proc", O_RDONLY|O_NONBLOCK|O_DIRECTORY|O_CLOEXEC) = 5
fcntl(5, F_GETFD)                       = 0x1 (flags FD_CLOEXEC)
getdents64(5, /* 211 entries */, 32768) = 5552
stat("/proc/1", {st_mode=S_IFDIR|0555, st_size=0, ...}) = 0
open("/proc/1/stat", O_RDONLY)          = 6
read(6, "1 (init) S 0 1 1 0 -1 4202752 31"..., 1023) = 187
close(6)                                = 0
open("/proc/1/status", O_RDONLY)        = 6
read(6, "Name:\tinit\nState:\tS (sleeping)\nT"..., 1023) = 675
close(6)
stat("/proc/2", {st_mode=S_IFDIR|0555, st_size=0, ...}) = 0
... 
So, the advice is to not parse output from utilities, it is not reliable thing to do, it is a subject to change. Use ioctl, sysctl, /proc, etc. to gather info you need. Use strace and others to find out what you need.

P.S. The funniest bug I've seen is when one mixed stdout and stderr and got every time new result upon parsing due to two streams are mixed in an unpredictable way. The other one was due to i18n feature.

Best template lib for C

You've seen or heard about Clearsilver, CTPP, libctemplate. But the best template lib which has C spirit is libtemplate
Using templates in PHP and C++ has spoiled me. So when I started developing applications in C, I went hunting for a templating library that I could use again. I didn't find it, so after developing in a mixture of C for my lowlevel routines and C++ for my interface, I finally broke down and wrote a templating engine in C. (See Free HTML Template Engine.)
An example
#include "template.h"

int
main(void) {
    struct tpl_engine *engine;
    int n;
    char n1[10], n2[10], n3[10];
    engine = tpl_engine_new();

    /* Load the template file */
    tpl_file_load(engine, "test.tpl");

    for(n = 1; n <= 10; n++) {
        sprintf(n1, "%d", n);
        sprintf(n2, "%d", n*n);
        sprintf(n3, "%d", n*n*n);
        tpl_element_set(engine, "n", n1);
        tpl_element_set(engine, "n2", n2);
        tpl_element_set(engine, "n3", n3);

        /* Parse the template 'row' and add the result to element 'rows' */
        tpl_parse(engine, "row", "rows", 1);
    }

    tpl_parse(engine, "grid", "main", 0);
    printf("%s", tpl_element_get(engine, "main"));

    return 0;
}
test.tpl
<template name="grid">
<table>
<tr>
<th>n</th>
<th>nˆ2</th>
<th>nˆ3</th>
</tr>
{rows}
</table>
</template>
<template name="row">
<tr>
<td>{n}</td>
<td>{n2}</td>
<td>{n3}</td>
</tr>
</template> 

Don't optimize when you are not asked to

The code below as you may guess multiplies two matrices (see Ulrich Drepper, What Every Programmer Should Know About Memory)
for (i = 0;  i < N; ++i)
    for (j = 0; j < N; ++j)
        for (k = 0; k < N; ++k)
            res[i][j] += mul1[i][k] * mul2[k][j];
The next piece of code does the same (as you probably may not guess anymore)
#define SM (CLS / sizeof (double))

for (i = 0; i < N; i += SM)
    for (j = 0; j < N; j += SM)
        for (k = 0; k < N; k += SM)
            for (i2 = 0, rres = &res[i][j],
                rmul1 = &mul1[i][k]; i2 < SM;
                ++i2, rres += N, rmul1 += N)
                for (k2 = 0, rmul2 = &mul2[k][j];
                    k2 < SM; ++k2, rmul2 += N)
                    for (j2 = 0; j2 < SM; ++j2)
                        rres[j2] += rmul1[k2] * rmul2[j2];
where CLS is a linesize of L1d cache
gcc -DCLS=$(getconf LEVEL1_DCACHE_LINESIZE) ...
The speed
OriginalTuned
Cycles16,765,297,872,895,041,480
Relative100%17.3%

But the main difference as author says is
This looks quite scary.
This smacks of premature optimisation and the possibility the user really does not know what they are talking about, not to mention the problem of portability.
See Why not cache lines.

Fundamentals, C arrays and pointers

Why C arrays are not pointers? Well, by definition
In C, there is a strong relationship between pointers and arrays, strong enough that pointers and arrays should be discussed simultaneously. (K&R)
They are easy to mix up
When an array name is passed to a function, what is passed is the location of the initial element. (K&R)
An example
#include <stdio.h>

void func(char subarray[100], char* pointer) {
    printf("sizeof subarray=%zd\n", sizeof(subarray));
    printf("address of subarray=%p\n", (void *)subarray);
    printf("pointer=%p\n", (void *)pointer);
}

int main() {
    char array[100];
    printf("sizeof of array=%zd\n", sizeof(array));
    printf("address of array=%p\n", (void *)array);
    printf("address of array[0]=%p\n", (void *)&array[0]);
    func(array, array);
}

// ----------------------------------------

sizeof of array=100
address of array=0xbfbfe760
address of array[0]=0xbfbfe760
sizeof subarray=4
address of subarray=0xbfbfe760
pointer=0xbfbfe760 

Friday, December 20, 2013

Programming TUN/TAP in Linux, changing IP address on the fly

So you want to map one network to another by implementing your very own bridge that changes network in IP packets it receives and forwards them to other segment. Why not I say, could be.
The plan is
  1. Create 2 tap interfaces: tap1 and tap2.
  2. Bridge tap1 with eth0 (i.e. everything that reaches tap1 is being forwarded to eth0 by kernel and vice versa).
  3. Application gets packets from tap2, changes src and dst IP addresses (changes network 10.0.0.0/24 to 192.168.14.0/24) and writes resulting packet to tap1 (then to eth0 and to the wires).
  4. When application gets response from eth0 (tap1) it substitutes IPs again and writes to tap2.
Note: the problem with IP substitution is in checksums. IP packets have IP header checksum, TCP and UDP also has checksum of pseudoheader that contains src and dst IP addresses as well. See details in the code.
For more information on TUN/TAP try to read Universal TUN/TAP device driver Frequently Asked Question.
In this way if you are in 192.168.14.0/24 network and want to speak with 192.168.14.15 after small manipulations you can speak with that host as if it had address 10.0.0.15.
The instructions and program itself without any further comments are below.
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <errno.h>
#include <sys/ioctl.h>
#include <net/if.h>
#include <linux/if_tun.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <sys/select.h>
#include <stdint.h>
#include <arpa/inet.h>

#include <linux/ip.h>
#include <linux/tcp.h>
#include <linux/udp.h>


/* 
Replace ip addresses in packets on L2 level.

Setup:

tunctl -t tap1
tunctl -t tap2
brctl addif br1 tap1
brctl addif br1 eth0

ifconfig tap1 promisc up
ifconfig tap2 promisc up

ifconfig tap2 10.0.1.18 netmask 255.255.255.0

Send packets to 10.0.0.0/24 on tap2, they will appear as 192.168.14.0/24 on eth0.
Note that these values are harcoded in main.

*/


static int
tun_alloc_old(char *dev) {
    char tunname[IFNAMSIZ];

    sprintf(tunname, "/dev/%s", dev);
    return open(tunname, O_RDWR);
}


static int
tun_alloc(char *dev) {
    struct ifreq    ifr;
    int     fd;
    int     err;

    if ((fd = open("/dev/net/tun", O_RDWR)) < 0)
        return tun_alloc_old(dev);

    memset(&ifr, 0, sizeof(ifr));

    /* Flags: IFF_TUN   - TUN device (no Ethernet headers)
     *        IFF_TAP   - TAP device
     *
     *        IFF_NO_PI - Do not provide packet information
     */
    ifr.ifr_flags = IFF_TAP;
    if (*dev)
        strncpy(ifr.ifr_name, dev, IFNAMSIZ);

    if ((err = ioctl(fd, TUNSETIFF, (void*)&ifr)) < 0) {
        close(fd);
        perror("TUNSETIFF");
        return err;
    }

    strcpy(dev, ifr.ifr_name);
    return fd;
}


static size_t
write2(int fildes, const void *buf, size_t nbyte) {
    int     ret;
    size_t  n;

    n = nbyte;
    while (n > 0) {
        ret = write(fildes, buf, nbyte);
        if (ret < 0)
            return ret;

        n -= ret;
        buf += ret;
    }

    return nbyte;
}


static uint16_t
ipcheck(uint16_t *ptr, size_t len) {
    uint32_t    sum;
    uint16_t    answer;

    sum = 0;

    while (len > 1) {
        sum += *ptr++;
        len -= 2;
    }

    sum = (sum >> 16) + (sum & 0xFFFF);
    sum += (sum >> 16);
    answer = ~sum;
    
    return answer;
}


static uint16_t
check2(struct iovec *iov, int iovcnt) {
    long    sum;
    uint16_t    answer;
    struct iovec   *iovp;

    sum = 0;

    for (iovp = iov; iovp < iov + iovcnt; iovp++) {
        uint16_t *ptr;
        size_t len;

        ptr = iovp->iov_base;
        len = iovp->iov_len;

        while (len > 1) {
            sum += *ptr++;
            len -= 2;
        }

        if (len == 1) {
            u_char t[2];
            t[0] = (u_char)*ptr;
            t[1] = 0;

            sum += (uint16_t)*t;
        }

    }

    sum = (sum >> 16) + (sum & 0xFFFF);
    sum += (sum >> 16);
    answer = ~sum;
    
    return answer;
}


static void
tcpcheck(struct iphdr *iph, struct tcphdr *tcph, size_t len) {
    struct iovec iov[5];

    iov[0].iov_base = &iph->saddr;
    iov[0].iov_len = 4;
    iov[1].iov_base = &iph->daddr;
    iov[1].iov_len = 4;

    u_char  t[2];
    t[0] = 0;
    t[1] = iph->protocol;
    iov[2].iov_base = t;
    iov[2].iov_len = 2;

    uint16_t l;
    l = htons(tcph->doff * 4 + len);
    iov[3].iov_base = &l;
    iov[3].iov_len = 2;

    iov[4].iov_base = tcph;
    iov[4].iov_len = tcph->doff * 4 + len;

    tcph->check = 0;
    tcph->check = check2(iov, sizeof(iov) / sizeof(struct iovec));
}


static void
udpcheck(struct iphdr *iph, struct udphdr *udph) {
    struct iovec iov[5];

    iov[0].iov_base = &iph->saddr;
    iov[0].iov_len = 4;
    iov[1].iov_base = &iph->daddr;
    iov[1].iov_len = 4;

    u_char  t[2];
    t[0] = 0;
    t[1] = iph->protocol;
    iov[2].iov_base = t;
    iov[2].iov_len = 2;

    uint16_t l;
    l = udph->len;
    iov[3].iov_base = &l;
    iov[3].iov_len = 2;

    iov[4].iov_base = udph;
    iov[4].iov_len = ntohs(udph->len);

    udph->check = 0;
    udph->check = check2(iov, sizeof(iov) / sizeof(struct iovec));
}


static int
substitute(u_char* buf, ssize_t n, u_char* net1, u_char* net2) {

    if (buf[12] == 8 && buf[13] == 6) {
        u_char     *arp;

        arp = buf + 14;

        /* replace ip */
        if (!memcmp(arp + 14, net1, 3)) {
            memcpy(arp + 14, net2, 3);
        }

        if (!memcmp(arp + 24, net1, 3)) {
            memcpy(arp + 24, net2, 3);
        }
    }
    else if (buf[12] == 8 && buf[13] == 0) {
        struct iphdr   *iph;
        size_t      len;


        iph = (struct iphdr*)(buf + 14);
        len = iph->ihl * 4;

        /* clear crc */
        iph->check = 0;


        /* replcace ip */
        if (!memcmp(&iph->saddr, net1, 3)) {
            memcpy(&iph->saddr, net2, 3);
        }

        if (!memcmp(&iph->daddr, net1, 3)) {
            memcpy(&iph->daddr, net2, 3);
        }

        /* put new crc */
        iph->check = ipcheck((uint16_t*)iph, len);


        if (iph->protocol == 6) {
            struct tcphdr  *tcph;

            tcph = (struct tcphdr*)((u_char*)iph + len);
            tcpcheck(iph, tcph, n - ((u_char*)tcph - buf) - tcph->doff * 4);
        }
        else if (iph->protocol == 17) {
            struct udphdr  *udph;

            udph = (struct udphdr*)((u_char*)iph + len);
            udpcheck(iph, udph);
        }
    }

    return 0;
}


int
main(int argc, char **argv) {
    int     tap1;
    int     tap2;
    int     maxfd;
    char    tunname[IFNAMSIZ];
    u_char  buf[15000];
    ssize_t n;

    u_char net1[] = {192, 168, 14};
    u_char net2[] = {10, 0, 1};

    strcpy(tunname, "tap1");
    if ((tap1 = tun_alloc(tunname)) < 0) {
        goto error;
    }

    strcpy(tunname, "tap2");
    if ((tap2 = tun_alloc(tunname)) < 0) {
        goto error;
    }

    maxfd = (tap1 > tap2)? tap1 : tap2;

    while (1) {
        int     ret;
        fd_set  rd_set;
        
        FD_ZERO(&rd_set);
        FD_SET(tap1, &rd_set);
        FD_SET(tap2, &rd_set);
    
        ret = select(maxfd + 1, &rd_set, NULL, NULL, NULL);
        
        if (ret < 0 && errno == EINTR) {
            continue;
        }

        if (ret < 0) {
            perror("select()");
            goto error;
        }

        if (FD_ISSET(tap1, &rd_set)) {
            n = read(tap1, buf, sizeof(buf));
            if (n < 0)
                goto error;

            if (substitute(buf, n, net1, net2))
                goto error;

            if (write2(tap2, buf, n) < 0)
                goto error;
        }

        if (FD_ISSET(tap2, &rd_set)) {

            n = read(tap2, buf, sizeof(buf));
            if (n < 0)
                goto error;

            if (substitute(buf, n, net2, net1))
                goto error;

            if (write2(tap1, buf, n) < 0)
                goto error;
        }
    }
    
    close(tap1);
    close(tap2);

    return 0;

error:
    return 1;
}

Override function in shared library

Lets override write() in libc
#define _GNU_SOURCE /* for RTLD_NEXT */

static ssize_t (*libc_write)(int, const void *, size_t);

void init(void) __attribute__((constructor));
void init(void) {
    libc_write = dlsym(RTLD_NEXT, "write");
}

ssize_t write(int fd, const void *buf, size_t count) {
    // do what you want here
    return libc_write(fd, buf, count);
}
Then one needs to assemble .so and put it earlier than libc.so, something like
LD_PRELOAD=/path/to/so/with/our/write.so victim_program
See Function Attributes, dlsym(3).