mmap MAP_NOSYNC regression in 10.x

[Pieter de Goeje]

After upgrading my month old 10-stable installation today (to r271093) , 
I've noticed a that the kernel no longer honors the MAP_NOSYNC flag. 
Attached is a demonstration program that highlights the issue (also 
available here: http://pastebin.com/y0kvdn0r ).

The program creates and mmap()s a 200MiB file and repeatedly writes 
zeros to it. The expected behavior is that under normal circumstances 
(no memory pressure), the dirtied pages are not flushed to disk. 
Observed is however that every ~30 seconds the syncer kicks in and 
basically halts the program while it does its job. The program prints a 
line everytime the throughput drops below 500MBps, well below memory 
bandwidth.

mmap() is called like this:

   void *p = mmap(NULL, len, PROT_READ | PROT_WRITE,
      MAP_SHARED | MAP_NOSYNC | MAP_ALIGNED_SUPER, fd, 0);

Sample output:

write...
zeroing: 209.6 MB
  ...write: 5.839s
mmap...
  ...mmap: 0.000s
20.1s: memset #259: 34.7MBps - stalled
55.7s: memset #810: 34.7MBps - stalled
91.3s: memset #1359: 34.6MBps - stalled
100.0s: memset #1522: 3938.5MBps
overall bandwidth: 3190.6MBps
munmap...
  ...munmap: 5.796s
done

(this is a rather old system)

If necessary I'm willing to find out the exact commit that caused the 
problem.

-
Pieter


-------------- next part --------------
#include <sys/mman.h>
#include <err.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <unistd.h>

#define CLEAREOL "\e[0K"
#define CLOCK_START(a) double _clk_##a = now(); printf("%s... \n", #a)
#define CLOCK_STOP(a) printf(" ...%s: %.3fs\n", #a, now() - _clk_##a)

static double now() {
  struct timespec ts;
  clock_gettime(CLOCK_MONOTONIC, &ts);
  return ts.tv_sec + ts.tv_nsec * 1E-9;
}

int main() {
  setvbuf(stdout, NULL, _IONBF, 0);

  int fd = open("backing-file", O_RDWR | O_CREAT | O_TRUNC, 0666);
  if(fd == -1)
    err(1, "open");

  size_t len  = 200 * 1024 * 1024;
  size_t blen = 128 * 1024;
  
  CLOCK_START(write);
  char *buf = calloc(1, blen);
  for(size_t i = 0; i < len; i += blen) {
    printf("\rzeroing: %.1f MB" CLEAREOL, i * 1E-6);
    if(write(fd, buf, blen) != blen)
      err(1, "write\n");
  }
  free(buf);
  printf("\n");
  CLOCK_STOP(write);

  CLOCK_START(mmap);
  void *p = mmap(NULL, len, PROT_READ | PROT_WRITE, 
     MAP_SHARED | MAP_NOSYNC | MAP_ALIGNED_SUPER, fd, 0);
  
  if(p == MAP_FAILED)
    err(1, "mmap");
  CLOCK_STOP(mmap);
    
  if(close(fd) == -1)
    err(1, "close");

  double start, stop, last;
  stop = start = now();
  int n = 0;
  do {
    memset(p, 0, len);
    last = stop; stop = now();
    
    double bw = 1E-6 * len / (stop - last);
    printf("\r%.1fs: memset #%d: %.1fMBps" CLEAREOL, stop - start, ++n, bw);
    if(bw < 500.0)
      printf(" - stalled\n");
  } while(stop - start < 100.0);
  printf("\noverall bandwidth: %.1fMBps\n", n * (double)len / (stop - start) * 1E-6);

  CLOCK_START(munmap);
  if(munmap(p, len) == -1)
    err(1, "munmap");
  CLOCK_STOP(munmap);
  
  printf("done\n");
  return 0;
}