Unnecessary reads on write load

[David Cross]

Tracking down a performance issue where the system apparently needlessly 
reads on a 100% write load... consider the following C test case:  (more 
after the code)

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <fcntl.h>

int main(int argc, char **argv)
{
     unsigned char dir1[4], dir2[4], filename[15], pathname[1024], 
buffer[130944];
     unsigned int filenum, count, filesize;
     int fd;

     arc4random_buf(buffer, 131072);

     count=atoi(argv[1]);

     for(;count>0;count--) {
         filenum=arc4random();
         filesize=arc4random_uniform(130944) + 128;
         sprintf(filename, "%08x", filenum);
         strncpy(dir1,filename,3);
         strncpy(dir2,filename+3, 3);
         dir1[3]=dir2[3]=0;
         sprintf(pathname, "%s/%s/%s", dir1, dir2, filename);
         fd=open(pathname, O_CREAT | O_WRONLY, 0644);
         if (fd < 0) {
             sprintf(pathname, "%s/%s", dir1, dir2);
             if (mkdir(pathname, 0755) < 0) {
                 mkdir(dir1, 0755);
                 mkdir(pathname, 0755);
             };
             sprintf(pathname, "%s/%s/%s", dir1, dir2, filename);
             fd=open(pathname, O_CREAT | O_WRONLY, 0644);
         }
         if (fd <0)
             continue;
         write(fd,buffer, filesize);
         close(fd);
     }
     return 0;
}

In running that in an empty directory (it takes one argument, the number 
of files to create), I see that it spends most of its time in BIORD?!.  
If I have a debugging kernel I can see that its all in NAMI cache 
misses, and doing block fetches... but "why?"  the only directories that 
exist are ones that it just created, and should therefore be in the 
cache, right?  Any ideas?  Give it a try yourself and tell me what you get.

-- 
David E. Cross