From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S263245AbTESTtY (ORCPT ); Mon, 19 May 2003 15:49:24 -0400 Received: (majordomo@vger.kernel.org) by vger.kernel.org id S263257AbTESTtY (ORCPT ); Mon, 19 May 2003 15:49:24 -0400 Received: from us02smtp1.synopsys.com ([198.182.60.75]:21469 "HELO vaxjo.synopsys.com") by vger.kernel.org with SMTP id S263245AbTESTtR (ORCPT ); Mon, 19 May 2003 15:49:17 -0400 Date: Mon, 19 May 2003 16:02:07 -0400 From: Jim Nance To: Trond Myklebust Cc: jlnance@unity.ncsu.edu, linux-kernel@vger.kernel.org, gary.nifong@synopsys.COM, James.Nance@synopsys.COM, david.thomas@synopsys.COM Subject: Re: NFS problems with Linux-2.4 Message-ID: <20030519160207.A16014@synopsys.com> References: <20030513145023.GA10383@ncsu.edu> <16065.3323.449992.207039@charged.uio.no> <20030515112231.A28148@synopsys.com> Mime-Version: 1.0 Content-Type: multipart/mixed; boundary="J/dobhs11T7y2rNN" Content-Disposition: inline User-Agent: Mutt/1.2.5.1i In-Reply-To: ; from trond.myklebust@fys.uio.no on Sun, May 18, 2003 at 05:00:24PM +0200 Sender: linux-kernel-owner@vger.kernel.org X-Mailing-List: linux-kernel@vger.kernel.org --J/dobhs11T7y2rNN Content-Type: text/plain; charset=us-ascii Content-Disposition: inline On Sun, May 18, 2003 at 05:00:24PM +0200, Trond Myklebust wrote: > > Sorry. stat doesn't obey close-to-open. It relies on standard > attribute caching. close-to-open means "open()" (and only "open()") > checks data cache consistency... Hi Trond, I rewrote my test program so that it uses open() instead of stat(). I also changed it so that it does not rename the file after it writing it. This should only leave close, open, and unlink calls. The program still fails for me after running for a minute or so: cayman> ./p1 s Failed to find #0 which client wrote Failed on file number 10202 Again, this is with 2.4.20 kernel. It fails much faster with a 2.4.7. Thanks, Jim -- ---------------------------------------------------------------------------- Jim Nance Synopsys (919) 425-7219 Do you have sweet iced tea? jlnance@us54.synopsys.com No, but there's sugar on the table. --J/dobhs11T7y2rNN Content-Type: text/plain; charset=us-ascii Content-Disposition: attachment; filename="p1.c" /* This program demonstrates a problem with the close/open consistency * of NFS file systems under Linux. It fails very rapidy with Red Hats * 2.4.7-10smp kernel. This kernel was known to have bugs. It also fails * with Red Hats 2.4.20-13.7bigmem kernel, which was thought to have this * bug fixed. For my testcase both linux machines were talking to a * network applicance file server and mounted like this: * * na1-rtp:/vol/vol0/home/jlnance /home/jlnance nfs rw,v3,rsize=4096,\ * wsize=4096,hard,intr,udp,lock,addr=na1-rtp 0 0 * * This program needs to be run on 2 machines, assume hostnames A & B. * A and B need to share an NFS mounted file system. * * On machine A: * cd /some/nfs/path/common/to/both * ./p1 s * * On machine B: * cd /some/nfs/path/common/to/both * ./p1 c A * * After a while you may see output similar to: * cayman> ./p1 s * Failed to find #0 which client wrote * Failed on file number 483 */ #include #include #include #include #include #include #include #include #include #include #define PORT 12387 #define FLEN 16 void die() { perror(""); exit(-1); } void Write(int fd, char *buff, size_t len) { for(;;) { int nsent=write(fd, buff, len); if(nsent==0) exit(0); if(nsent==-1) { if(errno!=EINTR) die(); } else { buff += nsent; len -= nsent; if(len==0) { return; } } } } void Read(int fd, char *buff, size_t len) { for(;;) { int nread=read(fd, buff, len); if(nread==0) exit(0); if(nread==-1) { if(errno!=EINTR) die(); } else { buff += nread; len -= nread; if(len==0) { return; } } } } int server() { int sock = socket(AF_INET, SOCK_STREAM, 0); if(sock==-1) die(); else { struct sockaddr_in name; int on = 1; name.sin_family = AF_INET; name.sin_addr.s_addr = htonl(INADDR_ANY); name.sin_port = htons(PORT); setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &on, sizeof on); if(bind(sock, (struct sockaddr*)&name, sizeof(name))==-1) die(); else { if(listen(sock, 1)==-1) die(); else { int tsock = accept(sock, 0, 0); if(tsock!=-1) { int cnt; for(cnt=0; cnt<100000; cnt++) { int fd; char dummy; char number[FLEN]; struct stat sbuf; /*sprintf(number, "#%d", cnt);*/ sprintf(number, "#%d", 0); Write(tsock, number, sizeof(number)); Read(tsock, &dummy, 1); #if 0 if(stat(number, &sbuf)) { fprintf(stderr, "Failed to find %s which client wrote\n", number); fprintf(stderr, "Failed on file number %d\n", cnt); exit(-2); } #else fd = open(number, O_RDONLY, 0); if(fd<0) { fprintf(stderr, "Failed to find %s which client wrote\n", number); fprintf(stderr, "Failed on file number %d\n", cnt); exit(-2); } #endif close(fd); unlink(number); } } } } } return 0; } int client(char *server) { struct hostent *info = gethostbyname(server); if(!info) die(); else { int rsocket = socket(AF_INET, SOCK_STREAM, 0); if(rsocket==-1) die(); else { struct sockaddr_in name; name.sin_family = AF_INET; name.sin_port = htons(PORT); memcpy(&name.sin_addr, info->h_addr_list[0], sizeof(struct in_addr)); if(connect(rsocket, (struct sockaddr*)&name, sizeof(name))==-1) die(); else { for(;;) { int fd; char fname[FLEN]; char tname[FLEN+8]; Read(rsocket, fname, sizeof(fname)); strcpy(tname, fname); /*strcat(tname, ".tmp");*/ fd = open(tname, O_WRONLY|O_CREAT, 0600); if(fd==-1) die(); Write(fd, fname, sizeof(fname)); /* Junk data */ close(fd); /*rename(tname, fname);*/ Write(rsocket, fname, 1); /* Tells the server we are done */ } } } } return 0; } void usage(char *prog) { fprintf(stderr, "Usage:\n"); fprintf(stderr, " %s s\n", prog); fprintf(stderr, " %s c servername\n", prog); fprintf(stderr, " Run 1 of each in the same NFS directory on 2 different " "machines\n Two processes total\n"); exit(-1); } int main(int ac, char **av) { if(ac<2) { usage(av[0]); } if(av[1][0]=='s') { return server(); }else if(ac<3) { usage(av[0]); } else if(av[1][0]=='c') { return client(av[2]); } else { usage(av[0]); } return -1; } --J/dobhs11T7y2rNN--