mmap() question

[Dmitry Sivachenko]

Hello!

I have a program which mmap()s a lot of large files (total size more that RAM and I have no swap), but it needs only small parts of that files at a time.

My understanding is that when using mmap when I access some memory region OS reads the relevant portion of that file from disk and caches the result in memory.  If there is no free memory, OS will purge previously read part of mmap'ed file to free memory for the new chunk.

But this is not the case.  I use the following simple program which gets list of files as command line arguments, mmap()s them all and then selects random file and random 1K parts of that file and computes a XOR of bytes from that region.
After some time the program dies:
pid 63251 (a.out), uid 1232, was killed: out of swap space

It seems I incorrectly understand how mmap() works, can you please clarify what's going wrong?

I expect that program to run indefinitely, purging some regions out of RAM and reading the relevant parts of files.

Thanks!

#include <err.h>
#include <fcntl.h>
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>

struct f_data {
    char *beg;
    off_t size;
};

int
main(int argc, char* argv[]) {
    if (argc < 2) {
        fprintf(stderr, "Usage: %s <file> ...\n", argv[0]);
        exit(0);
    }
    int i, j, fd;
    struct stat st;
    struct f_data FILES[500];
    int NUM_FILES;
    void *p;
    NUM_FILES = argc - 1;
    for (i=1; i < argc; i++) {
        printf("%s... ", argv[i]);
        if ((fd = open(argv[i], O_RDONLY)) < 0)
            errx(1, "open");
        if (fstat(fd, &st) != 0)
            errx(1, "fstat");
        if ((p = mmap(NULL, st.st_size, PROT_READ, MAP_NOCORE, fd, 0)) == MAP_FAILED)
            errx(1, "mmap");
        FILES[i-1].beg = (char*)p;
        FILES[i-1].size = st.st_size;
        if (msync(p, st.st_size, MS_INVALIDATE) != 0)
            errx(1, "msync");
        printf("Ok.\n");
    }
    char chk = 0;
    while(1) {
        int rf = floor((double)random() / 2147483647 * NUM_FILES);
        off_t offs = floor((double)random() / 2147483647 * (FILES[rf].size - 1024));
        for (j=0; j<1024; j++)
            chk ^= *(FILES[rf].beg + offs + j);
    }
    return 0;
}