* [PATCH 1/4] fsstress: new writev and readv operations test
@ 2017-06-16 16:11 Zorro Lang
2017-06-16 16:11 ` [PATCH 2/4] fsstress: add AIO read/write and fsync test Zorro Lang
` (3 more replies)
0 siblings, 4 replies; 7+ messages in thread
From: Zorro Lang @ 2017-06-16 16:11 UTC (permalink / raw)
To: fstests
We found some bugs by writev operations recently, writev can cover
different test than normal write operation, so add writev and
readv operations into fsstress.
Signed-off-by: Zorro Lang <zlang@redhat.com>
---
ltp/fsstress.c | 161 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 161 insertions(+)
diff --git a/ltp/fsstress.c b/ltp/fsstress.c
index e9fd276..67cdb2f 100644
--- a/ltp/fsstress.c
+++ b/ltp/fsstress.c
@@ -18,6 +18,7 @@
#include <linux/fs.h>
#include <setjmp.h>
+#include <sys/uio.h>
#include "global.h"
#ifdef HAVE_ATTR_XATTR_H
@@ -47,6 +48,9 @@
#define XFS_ERRTAG_MAX 17
#define XFS_IDMODULO_MAX 31 /* user/group IDs (1 << x) */
#define XFS_PROJIDMODULO_MAX 16 /* project IDs (1 << x) */
+#ifndef IOV_MAX
+#define IOV_MAX 1024
+#endif
#define FILELEN_MAX (32*4096)
@@ -78,6 +82,7 @@ typedef enum {
OP_INSERT,
OP_READ,
OP_READLINK,
+ OP_READV,
OP_RENAME,
OP_RESVSP,
OP_RMDIR,
@@ -90,6 +95,7 @@ typedef enum {
OP_UNLINK,
OP_UNRESVSP,
OP_WRITE,
+ OP_WRITEV,
OP_LAST
} opty_t;
@@ -179,6 +185,7 @@ void collapse_f(int, long);
void insert_f(int, long);
void read_f(int, long);
void readlink_f(int, long);
+void readv_f(int, long);
void rename_f(int, long);
void resvsp_f(int, long);
void rmdir_f(int, long);
@@ -191,6 +198,7 @@ void truncate_f(int, long);
void unlink_f(int, long);
void unresvsp_f(int, long);
void write_f(int, long);
+void writev_f(int, long);
opdesc_t ops[] = {
/* { OP_ENUM, "name", function, freq, iswrite }, */
@@ -221,6 +229,7 @@ opdesc_t ops[] = {
{ OP_INSERT, "insert", insert_f, 1, 1 },
{ OP_READ, "read", read_f, 1, 0 },
{ OP_READLINK, "readlink", readlink_f, 1, 0 },
+ { OP_READ, "readv", readv_f, 1, 0 },
{ OP_RENAME, "rename", rename_f, 2, 1 },
{ OP_RESVSP, "resvsp", resvsp_f, 1, 1 },
{ OP_RMDIR, "rmdir", rmdir_f, 1, 1 },
@@ -233,6 +242,7 @@ opdesc_t ops[] = {
{ OP_UNLINK, "unlink", unlink_f, 1, 1 },
{ OP_UNRESVSP, "unresvsp", unresvsp_f, 1, 1 },
{ OP_WRITE, "write", write_f, 4, 1 },
+ { OP_WRITEV, "writev", writev_f, 4, 1 },
}, *ops_end;
flist_t flist[FT_nft] = {
@@ -2948,6 +2958,84 @@ readlink_f(int opno, long r)
}
void
+readv_f(int opno, long r)
+{
+ char *buf;
+ int e;
+ pathname_t f;
+ int fd;
+ size_t len;
+ __int64_t lr;
+ off64_t off;
+ struct stat64 stb;
+ int v;
+ char st[1024];
+ struct iovec *iov = NULL;
+ int iovcnt;
+ size_t iovb;
+ size_t iovl;
+ int i;
+
+ init_pathname(&f);
+ if (!get_fname(FT_REGFILE, r, &f, NULL, NULL, &v)) {
+ if (v)
+ printf("%d/%d: readv - no filename\n", procid, opno);
+ free_pathname(&f);
+ return;
+ }
+ fd = open_path(&f, O_RDONLY);
+ e = fd < 0 ? errno : 0;
+ check_cwd();
+ if (fd < 0) {
+ if (v)
+ printf("%d/%d: readv - open %s failed %d\n",
+ procid, opno, f.path, e);
+ free_pathname(&f);
+ return;
+ }
+ if (fstat64(fd, &stb) < 0) {
+ if (v)
+ printf("%d/%d: readv - fstat64 %s failed %d\n",
+ procid, opno, f.path, errno);
+ free_pathname(&f);
+ close(fd);
+ return;
+ }
+ inode_info(st, sizeof(st), &stb, v);
+ if (stb.st_size == 0) {
+ if (v)
+ printf("%d/%d: readv - %s%s zero size\n", procid, opno,
+ f.path, st);
+ free_pathname(&f);
+ close(fd);
+ return;
+ }
+ lr = ((__int64_t)random() << 32) + random();
+ off = (off64_t)(lr % stb.st_size);
+ lseek64(fd, off, SEEK_SET);
+ len = (random() % FILELEN_MAX) + 1;
+ buf = malloc(len);
+
+ iovcnt = (random() % MIN(len, IOV_MAX)) + 1;
+ iov = malloc(sizeof(struct iovec) * iovcnt);
+ for (i=0; i<iovcnt; i++) {
+ iovb = random() % len;
+ iovl = random() % (len - iovb) + 1;
+ (iov + i)->iov_base = (buf + iovb);
+ (iov + i)->iov_len = iovl;
+ }
+
+ e = read(fd, buf, len) < 0 ? errno : 0;
+ free(buf);
+ if (v)
+ printf("%d/%d: readv %s%s [%lld,%d,%d] %d\n",
+ procid, opno, f.path, st, (long long)off, (int)len,
+ iovcnt, e);
+ free_pathname(&f);
+ close(fd);
+}
+
+void
rename_f(int opno, long r)
{
fent_t *dfep;
@@ -3379,3 +3467,76 @@ write_f(int opno, long r)
free_pathname(&f);
close(fd);
}
+
+void
+writev_f(int opno, long r)
+{
+ char *buf;
+ int e;
+ pathname_t f;
+ int fd;
+ size_t len;
+ __int64_t lr;
+ off64_t off;
+ struct stat64 stb;
+ int v;
+ char st[1024];
+ struct iovec *iov = NULL;
+ int iovcnt;
+ size_t iovb;
+ size_t iovl;
+ int i;
+
+ init_pathname(&f);
+ if (!get_fname(FT_REGm, r, &f, NULL, NULL, &v)) {
+ if (v)
+ printf("%d/%d: writev - no filename\n", procid, opno);
+ free_pathname(&f);
+ return;
+ }
+ fd = open_path(&f, O_WRONLY);
+ e = fd < 0 ? errno : 0;
+ check_cwd();
+ if (fd < 0) {
+ if (v)
+ printf("%d/%d: writev - open %s failed %d\n",
+ procid, opno, f.path, e);
+ free_pathname(&f);
+ return;
+ }
+ if (fstat64(fd, &stb) < 0) {
+ if (v)
+ printf("%d/%d: writev - fstat64 %s failed %d\n",
+ procid, opno, f.path, errno);
+ free_pathname(&f);
+ close(fd);
+ return;
+ }
+ inode_info(st, sizeof(st), &stb, v);
+ lr = ((__int64_t)random() << 32) + random();
+ off = (off64_t)(lr % MIN(stb.st_size + (1024 * 1024), MAXFSIZE));
+ off %= maxfsize;
+ lseek64(fd, off, SEEK_SET);
+ len = (random() % FILELEN_MAX) + 1;
+ buf = malloc(len);
+ memset(buf, nameseq & 0xff, len);
+
+ iovcnt = (random() % MIN(len, IOV_MAX)) + 1;
+ iov = malloc(sizeof(struct iovec) * iovcnt);
+ for (i=0; i<iovcnt; i++) {
+ iovb = random() % len;
+ iovl = random() % (len - iovb) + 1;
+ (iov + i)->iov_base = (buf + iovb);
+ (iov + i)->iov_len = iovl;
+ }
+
+ e = writev(fd, iov, iovcnt) < 0 ? errno : 0;
+ free(buf);
+ free(iov);
+ if (v)
+ printf("%d/%d: writev %s%s [%lld,%d,%d] %d\n",
+ procid, opno, f.path, st, (long long)off, (int)len,
+ iovcnt, e);
+ free_pathname(&f);
+ close(fd);
+}
--
2.7.4
^ permalink raw reply related [flat|nested] 7+ messages in thread
* [PATCH 2/4] fsstress: add AIO read/write and fsync test
2017-06-16 16:11 [PATCH 1/4] fsstress: new writev and readv operations test Zorro Lang
@ 2017-06-16 16:11 ` Zorro Lang
2017-07-11 4:10 ` Eryu Guan
2017-06-16 16:11 ` [PATCH 3/4] fsstress: do memset randomly before write operations Zorro Lang
` (2 subsequent siblings)
3 siblings, 1 reply; 7+ messages in thread
From: Zorro Lang @ 2017-06-16 16:11 UTC (permalink / raw)
To: fstests
We found some bugs by aio read/write test, but there's not related
operations in fsstress. So add AIO test into fsstress to increase
AIO stress test.
Due to most kernels don't support aio fsync, so set its test
frequency to zero as default.
Signed-off-by: Zorro Lang <zlang@redhat.com>
---
ltp/fsstress.c | 214 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 214 insertions(+)
diff --git a/ltp/fsstress.c b/ltp/fsstress.c
index 67cdb2f..4ff90d5 100644
--- a/ltp/fsstress.c
+++ b/ltp/fsstress.c
@@ -36,6 +36,10 @@
#ifdef HAVE_SYS_PRCTL_H
#include <sys/prctl.h>
#endif
+#ifdef AIO
+#include <libaio.h>
+io_context_t io_ctx;
+#endif
#ifndef FS_IOC_GETFLAGS
#define FS_IOC_GETFLAGS _IOR('f', 1, long)
@@ -55,9 +59,12 @@
#define FILELEN_MAX (32*4096)
typedef enum {
+ OP_AFSYNC,
OP_ALLOCSP,
+ OP_AREAD,
OP_ATTR_REMOVE,
OP_ATTR_SET,
+ OP_AWRITE,
OP_BULKSTAT,
OP_BULKSTAT1,
OP_CHOWN,
@@ -158,9 +165,12 @@ struct print_string {
#define MAXFSIZE ((1ULL << 63) - 1ULL)
#define MAXFSIZE32 ((1ULL << 40) - 1ULL)
+void afsync_f(int, long);
void allocsp_f(int, long);
+void aread_f(int, long);
void attr_remove_f(int, long);
void attr_set_f(int, long);
+void awrite_f(int, long);
void bulkstat_f(int, long);
void bulkstat1_f(int, long);
void chown_f(int, long);
@@ -202,9 +212,12 @@ void writev_f(int, long);
opdesc_t ops[] = {
/* { OP_ENUM, "name", function, freq, iswrite }, */
+ { OP_AFSYNC, "afsync", afsync_f, 0, 1 },
{ OP_ALLOCSP, "allocsp", allocsp_f, 1, 1 },
+ { OP_AREAD, "aread", aread_f, 1, 0 },
{ OP_ATTR_REMOVE, "attr_remove", attr_remove_f, /* 1 */ 0, 1 },
{ OP_ATTR_SET, "attr_set", attr_set_f, /* 2 */ 0, 1 },
+ { OP_AWRITE, "awrite", awrite_f, 1, 1 },
{ OP_BULKSTAT, "bulkstat", bulkstat_f, 1, 0 },
{ OP_BULKSTAT1, "bulkstat1", bulkstat1_f, 1, 0 },
{ OP_CHOWN, "chown", chown_f, 3, 1 },
@@ -587,8 +600,20 @@ int main(int argc, char **argv)
}
}
procid = i;
+#ifdef AIO
+ if (io_setup(128, &io_ctx) != 0) {
+ perror("io_setup failed");
+ exit(1);
+ }
+#endif
for (i = 0; !loops || (i < loops); i++)
doproc();
+#ifdef AIO
+ if(io_destroy(io_ctx) != 0) {
+ perror("io_destroy failed");
+ return 1;
+ }
+#endif
return 0;
}
}
@@ -1708,6 +1733,62 @@ void inode_info(char *str, size_t sz, struct stat64 *s, int verbose)
}
void
+afsync_f(int opno, long r)
+{
+#ifdef AIO
+ int e;
+ pathname_t f;
+ int fd;
+ int v;
+ struct iocb iocb;
+ struct iocb *iocbs[] = { &iocb };
+ struct io_event event;
+
+ init_pathname(&f);
+ if (!get_fname(FT_REGFILE, r, &f, NULL, NULL, &v)) {
+ if (v)
+ printf("%d/%d: afsync - no filename\n", procid, opno);
+ free_pathname(&f);
+ return;
+ }
+ fd = open_path(&f, O_WRONLY | O_DIRECT);
+ e = fd < 0 ? errno : 0;
+ check_cwd();
+ if (fd < 0) {
+ if (v)
+ printf("%d/%d: afsync - open %s failed %d\n",
+ procid, opno, f.path, e);
+ free_pathname(&f);
+ return;
+ }
+
+ io_prep_fsync(&iocb, fd);
+ if ((e = io_submit(io_ctx, 1, iocbs)) != 1) {
+ if (v)
+ printf("%d/%d: afsync - io_submit %s %d\n",
+ procid, opno, f.path, e);
+ free_pathname(&f);
+ close(fd);
+ return;
+ }
+ if ((e = io_getevents(io_ctx, 1, 1, &event, NULL)) != 1) {
+ if (v)
+ printf("%d/%d: afsync - io_getevents failed %d\n",
+ procid, opno, e);
+ free_pathname(&f);
+ close(fd);
+ return;
+ }
+
+ e = event.res2;
+ if (v)
+ printf("%d/%d: afsync %s %d\n", procid, opno, f.path, e);
+ free_pathname(&f);
+ close(fd);
+#endif
+}
+
+void
allocsp_f(int opno, long r)
{
int e;
@@ -1761,6 +1842,131 @@ allocsp_f(int opno, long r)
close(fd);
}
+#ifdef AIO
+void
+do_aio_rw(int opno, long r, int flags)
+{
+ __int64_t align;
+ char *buf;
+ struct dioattr diob;
+ int e;
+ pathname_t f;
+ int fd;
+ size_t len;
+ __int64_t lr;
+ off64_t off;
+ struct stat64 stb;
+ int v;
+ char st[1024];
+ char *dio_env;
+ struct iocb iocb;
+ struct io_event event;
+ struct iocb *iocbs[] = { &iocb };
+ int iswrite = (flags & (O_WRONLY | O_RDWR)) ? 1 : 0;
+
+ init_pathname(&f);
+ if (!get_fname(FT_REGFILE, r, &f, NULL, NULL, &v)) {
+ if (v)
+ printf("%d/%d: do_aio_rw - no filename\n", procid, opno);
+ free_pathname(&f);
+ return;
+ }
+ fd = open_path(&f, flags|O_DIRECT);
+ e = fd < 0 ? errno : 0;
+ check_cwd();
+ if (fd < 0) {
+ if (v)
+ printf("%d/%d: do_aio_rw - open %s failed %d\n",
+ procid, opno, f.path, e);
+ free_pathname(&f);
+ return;
+ }
+ if (fstat64(fd, &stb) < 0) {
+ if (v)
+ printf("%d/%d: do_aio_rw - fstat64 %s failed %d\n",
+ procid, opno, f.path, errno);
+ free_pathname(&f);
+ close(fd);
+ return;
+ }
+ inode_info(st, sizeof(st), &stb, v);
+ if (!iswrite && stb.st_size == 0) {
+ if (v)
+ printf("%d/%d: do_aio_rw - %s%s zero size\n", procid, opno,
+ f.path, st);
+ free_pathname(&f);
+ close(fd);
+ return;
+ }
+ if (xfsctl(f.path, fd, XFS_IOC_DIOINFO, &diob) < 0) {
+ if (v)
+ printf(
+ "%d/%d: do_aio_rw - xfsctl(XFS_IOC_DIOINFO) %s%s failed %d\n",
+ procid, opno, f.path, st, errno);
+ free_pathname(&f);
+ close(fd);
+ return;
+ }
+ dio_env = getenv("XFS_DIO_MIN");
+ if (dio_env)
+ diob.d_mem = diob.d_miniosz = atoi(dio_env);
+ align = (__int64_t)diob.d_miniosz;
+ lr = ((__int64_t)random() << 32) + random();
+ len = (random() % FILELEN_MAX) + 1;
+ len -= (len % align);
+ if (len <= 0)
+ len = align;
+ else if (len > diob.d_maxiosz)
+ len = diob.d_maxiosz;
+ buf = memalign(diob.d_mem, len);
+
+ if (iswrite) {
+ off = (off64_t)(lr % MIN(stb.st_size + (1024 * 1024), MAXFSIZE));
+ off -= (off % align);
+ off %= maxfsize;
+ memset(buf, nameseq & 0xff, len);
+ io_prep_pwrite(&iocb, fd, buf, len, off);
+ } else {
+ off = (off64_t)(lr % stb.st_size);
+ off -= (off % align);
+ io_prep_pread(&iocb, fd, buf, len, off);
+ }
+ if ((e = io_submit(io_ctx, 1, iocbs)) != 1) {
+ if (v)
+ printf("%d/%d: %s - io_submit failed %d\n",
+ procid, opno, iswrite ? "awrite" : "aread", e);
+ free_pathname(&f);
+ close(fd);
+ return;
+ }
+ if ((e = io_getevents(io_ctx, 1, 1, &event, NULL)) != 1) {
+ if (v)
+ printf("%d/%d: %s - io_getevents failed %d\n",
+ procid, opno, iswrite ? "awrite" : "aread", e);
+ free_pathname(&f);
+ close(fd);
+ return;
+ }
+
+ e = event.res != len ? event.res2 : 0;
+ free(buf);
+ if (v)
+ printf("%d/%d: %s %s%s [%lld,%d] %d\n",
+ procid, opno, iswrite ? "awrite" : "aread",
+ f.path, st, (long long)off, (int)len, e);
+ free_pathname(&f);
+ close(fd);
+}
+#endif
+
+void
+aread_f(int opno, long r)
+{
+#ifdef AIO
+ do_aio_rw(opno, r, O_RDONLY);
+#endif
+}
+
void
attr_remove_f(int opno, long r)
{
@@ -1865,6 +2071,14 @@ attr_set_f(int opno, long r)
}
void
+awrite_f(int opno, long r)
+{
+#ifdef AIO
+ do_aio_rw(opno, r, O_WRONLY);
+#endif
+}
+
+void
bulkstat_f(int opno, long r)
{
int count;
--
2.7.4
^ permalink raw reply related [flat|nested] 7+ messages in thread
* [PATCH 3/4] fsstress: do memset randomly before write operations
2017-06-16 16:11 [PATCH 1/4] fsstress: new writev and readv operations test Zorro Lang
2017-06-16 16:11 ` [PATCH 2/4] fsstress: add AIO read/write and fsync test Zorro Lang
@ 2017-06-16 16:11 ` Zorro Lang
2017-06-19 9:52 ` Zorro Lang
2017-06-16 16:11 ` [PATCH 4/4] xfs/068: update golden output due to new operations in fsstress Zorro Lang
2017-07-11 3:49 ` [PATCH 1/4] fsstress: new writev and readv operations test Eryu Guan
3 siblings, 1 reply; 7+ messages in thread
From: Zorro Lang @ 2017-06-16 16:11 UTC (permalink / raw)
To: fstests
Recently we found a bug by trigger a page fault when io-vector of
writev is in processing. If do memset between malloc() and write
operations, memset will init the pages. So do memset randomly
maybe help to cover above situation.
Signed-off-by: Zorro Lang <zlang@redhat.com>
---
ltp/fsstress.c | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
diff --git a/ltp/fsstress.c b/ltp/fsstress.c
index 4ff90d5..e1b73e4 100644
--- a/ltp/fsstress.c
+++ b/ltp/fsstress.c
@@ -1924,7 +1924,8 @@ do_aio_rw(int opno, long r, int flags)
off = (off64_t)(lr % MIN(stb.st_size + (1024 * 1024), MAXFSIZE));
off -= (off % align);
off %= maxfsize;
- memset(buf, nameseq & 0xff, len);
+ /* randomly memset maybe trigger page fault at write time */
+ if (random() / 2) memset(buf, nameseq & 0xff, len);
io_prep_pwrite(&iocb, fd, buf, len, off);
} else {
off = (off64_t)(lr % stb.st_size);
@@ -2455,7 +2456,8 @@ dwrite_f(int opno, long r)
buf = memalign(diob.d_mem, len);
off %= maxfsize;
lseek64(fd, off, SEEK_SET);
- memset(buf, nameseq & 0xff, len);
+ /* randomly memset maybe trigger page fault at write time */
+ if (random() / 2) memset(buf, nameseq & 0xff, len);
e = write(fd, buf, len) < 0 ? errno : 0;
free(buf);
if (v)
@@ -3672,7 +3674,8 @@ write_f(int opno, long r)
lseek64(fd, off, SEEK_SET);
len = (random() % FILELEN_MAX) + 1;
buf = malloc(len);
- memset(buf, nameseq & 0xff, len);
+ /* randomly memset maybe trigger page fault at write time */
+ if (random() / 2) memset(buf, nameseq & 0xff, len);
e = write(fd, buf, len) < 0 ? errno : 0;
free(buf);
if (v)
@@ -3733,7 +3736,8 @@ writev_f(int opno, long r)
lseek64(fd, off, SEEK_SET);
len = (random() % FILELEN_MAX) + 1;
buf = malloc(len);
- memset(buf, nameseq & 0xff, len);
+ /* randomly memset maybe trigger page fault at write time */
+ if (random() / 2) memset(buf, nameseq & 0xff, len);
iovcnt = (random() % MIN(len, IOV_MAX)) + 1;
iov = malloc(sizeof(struct iovec) * iovcnt);
--
2.7.4
^ permalink raw reply related [flat|nested] 7+ messages in thread
* [PATCH 4/4] xfs/068: update golden output due to new operations in fsstress
2017-06-16 16:11 [PATCH 1/4] fsstress: new writev and readv operations test Zorro Lang
2017-06-16 16:11 ` [PATCH 2/4] fsstress: add AIO read/write and fsync test Zorro Lang
2017-06-16 16:11 ` [PATCH 3/4] fsstress: do memset randomly before write operations Zorro Lang
@ 2017-06-16 16:11 ` Zorro Lang
2017-07-11 3:49 ` [PATCH 1/4] fsstress: new writev and readv operations test Eryu Guan
3 siblings, 0 replies; 7+ messages in thread
From: Zorro Lang @ 2017-06-16 16:11 UTC (permalink / raw)
To: fstests
I added some new operatoins to fsstress, and it changed the total
number of test operstions.
xfs/068 use a fixed seed (-s) and number of operations (-n) to run
fsstress, to get fixed number of files and directories. Due to my
patches break these fixed things, so update its expected result
in golden image.
Signed-off-by: Zorro Lang <zlang@redhat.com>
---
tests/xfs/068.out | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tests/xfs/068.out b/tests/xfs/068.out
index b20604f..37c75bf 100644
--- a/tests/xfs/068.out
+++ b/tests/xfs/068.out
@@ -22,7 +22,7 @@ xfsrestore: session id: ID
xfsrestore: media ID: ID
xfsrestore: searching media for directory dump
xfsrestore: reading directories
-xfsrestore: 474 directories and 1592 entries processed
+xfsrestore: 413 directories and 1347 entries processed
xfsrestore: directory post-processing
xfsrestore: restoring non-directory files
xfsrestore: restore complete: SECS seconds elapsed
--
2.7.4
^ permalink raw reply related [flat|nested] 7+ messages in thread
* Re: [PATCH 3/4] fsstress: do memset randomly before write operations
2017-06-16 16:11 ` [PATCH 3/4] fsstress: do memset randomly before write operations Zorro Lang
@ 2017-06-19 9:52 ` Zorro Lang
0 siblings, 0 replies; 7+ messages in thread
From: Zorro Lang @ 2017-06-19 9:52 UTC (permalink / raw)
To: fstests
On Sat, Jun 17, 2017 at 12:11:05AM +0800, Zorro Lang wrote:
> Recently we found a bug by trigger a page fault when io-vector of
> writev is in processing. If do memset between malloc() and write
> operations, memset will init the pages. So do memset randomly
> maybe help to cover above situation.
>
> Signed-off-by: Zorro Lang <zlang@redhat.com>
> ---
> ltp/fsstress.c | 12 ++++++++----
> 1 file changed, 8 insertions(+), 4 deletions(-)
>
> diff --git a/ltp/fsstress.c b/ltp/fsstress.c
> index 4ff90d5..e1b73e4 100644
> --- a/ltp/fsstress.c
> +++ b/ltp/fsstress.c
> @@ -1924,7 +1924,8 @@ do_aio_rw(int opno, long r, int flags)
> off = (off64_t)(lr % MIN(stb.st_size + (1024 * 1024), MAXFSIZE));
> off -= (off % align);
> off %= maxfsize;
> - memset(buf, nameseq & 0xff, len);
> + /* randomly memset maybe trigger page fault at write time */
> + if (random() / 2) memset(buf, nameseq & 0xff, len);
Sorry for my mistake, all "(random() / 2)" should be "random() % 2".
Thanks,
Zorro
> io_prep_pwrite(&iocb, fd, buf, len, off);
> } else {
> off = (off64_t)(lr % stb.st_size);
> @@ -2455,7 +2456,8 @@ dwrite_f(int opno, long r)
> buf = memalign(diob.d_mem, len);
> off %= maxfsize;
> lseek64(fd, off, SEEK_SET);
> - memset(buf, nameseq & 0xff, len);
> + /* randomly memset maybe trigger page fault at write time */
> + if (random() / 2) memset(buf, nameseq & 0xff, len);
> e = write(fd, buf, len) < 0 ? errno : 0;
> free(buf);
> if (v)
> @@ -3672,7 +3674,8 @@ write_f(int opno, long r)
> lseek64(fd, off, SEEK_SET);
> len = (random() % FILELEN_MAX) + 1;
> buf = malloc(len);
> - memset(buf, nameseq & 0xff, len);
> + /* randomly memset maybe trigger page fault at write time */
> + if (random() / 2) memset(buf, nameseq & 0xff, len);
> e = write(fd, buf, len) < 0 ? errno : 0;
> free(buf);
> if (v)
> @@ -3733,7 +3736,8 @@ writev_f(int opno, long r)
> lseek64(fd, off, SEEK_SET);
> len = (random() % FILELEN_MAX) + 1;
> buf = malloc(len);
> - memset(buf, nameseq & 0xff, len);
> + /* randomly memset maybe trigger page fault at write time */
> + if (random() / 2) memset(buf, nameseq & 0xff, len);
>
> iovcnt = (random() % MIN(len, IOV_MAX)) + 1;
> iov = malloc(sizeof(struct iovec) * iovcnt);
> --
> 2.7.4
>
> --
> To unsubscribe from this list: send the line "unsubscribe fstests" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [PATCH 1/4] fsstress: new writev and readv operations test
2017-06-16 16:11 [PATCH 1/4] fsstress: new writev and readv operations test Zorro Lang
` (2 preceding siblings ...)
2017-06-16 16:11 ` [PATCH 4/4] xfs/068: update golden output due to new operations in fsstress Zorro Lang
@ 2017-07-11 3:49 ` Eryu Guan
3 siblings, 0 replies; 7+ messages in thread
From: Eryu Guan @ 2017-07-11 3:49 UTC (permalink / raw)
To: Zorro Lang; +Cc: fstests
On Sat, Jun 17, 2017 at 12:11:03AM +0800, Zorro Lang wrote:
> We found some bugs by writev operations recently, writev can cover
> different test than normal write operation, so add writev and
> readv operations into fsstress.
>
> Signed-off-by: Zorro Lang <zlang@redhat.com>
Sorry for the so late response..
> ---
> ltp/fsstress.c | 161 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
> 1 file changed, 161 insertions(+)
>
> diff --git a/ltp/fsstress.c b/ltp/fsstress.c
> index e9fd276..67cdb2f 100644
> --- a/ltp/fsstress.c
> +++ b/ltp/fsstress.c
> @@ -18,6 +18,7 @@
>
> #include <linux/fs.h>
> #include <setjmp.h>
> +#include <sys/uio.h>
> #include "global.h"
>
> #ifdef HAVE_ATTR_XATTR_H
> @@ -47,6 +48,9 @@
> #define XFS_ERRTAG_MAX 17
> #define XFS_IDMODULO_MAX 31 /* user/group IDs (1 << x) */
> #define XFS_PROJIDMODULO_MAX 16 /* project IDs (1 << x) */
> +#ifndef IOV_MAX
> +#define IOV_MAX 1024
> +#endif
>
> #define FILELEN_MAX (32*4096)
>
> @@ -78,6 +82,7 @@ typedef enum {
> OP_INSERT,
> OP_READ,
> OP_READLINK,
> + OP_READV,
> OP_RENAME,
> OP_RESVSP,
> OP_RMDIR,
> @@ -90,6 +95,7 @@ typedef enum {
> OP_UNLINK,
> OP_UNRESVSP,
> OP_WRITE,
> + OP_WRITEV,
> OP_LAST
> } opty_t;
>
> @@ -179,6 +185,7 @@ void collapse_f(int, long);
> void insert_f(int, long);
> void read_f(int, long);
> void readlink_f(int, long);
> +void readv_f(int, long);
> void rename_f(int, long);
> void resvsp_f(int, long);
> void rmdir_f(int, long);
> @@ -191,6 +198,7 @@ void truncate_f(int, long);
> void unlink_f(int, long);
> void unresvsp_f(int, long);
> void write_f(int, long);
> +void writev_f(int, long);
>
> opdesc_t ops[] = {
> /* { OP_ENUM, "name", function, freq, iswrite }, */
> @@ -221,6 +229,7 @@ opdesc_t ops[] = {
> { OP_INSERT, "insert", insert_f, 1, 1 },
> { OP_READ, "read", read_f, 1, 0 },
> { OP_READLINK, "readlink", readlink_f, 1, 0 },
> + { OP_READ, "readv", readv_f, 1, 0 },
This should be OP_READV.
> { OP_RENAME, "rename", rename_f, 2, 1 },
> { OP_RESVSP, "resvsp", resvsp_f, 1, 1 },
> { OP_RMDIR, "rmdir", rmdir_f, 1, 1 },
> @@ -233,6 +242,7 @@ opdesc_t ops[] = {
> { OP_UNLINK, "unlink", unlink_f, 1, 1 },
> { OP_UNRESVSP, "unresvsp", unresvsp_f, 1, 1 },
> { OP_WRITE, "write", write_f, 4, 1 },
> + { OP_WRITEV, "writev", writev_f, 4, 1 },
> }, *ops_end;
>
> flist_t flist[FT_nft] = {
> @@ -2948,6 +2958,84 @@ readlink_f(int opno, long r)
> }
>
> void
> +readv_f(int opno, long r)
> +{
> + char *buf;
> + int e;
> + pathname_t f;
> + int fd;
> + size_t len;
> + __int64_t lr;
> + off64_t off;
> + struct stat64 stb;
> + int v;
> + char st[1024];
> + struct iovec *iov = NULL;
> + int iovcnt;
> + size_t iovb;
> + size_t iovl;
> + int i;
> +
> + init_pathname(&f);
> + if (!get_fname(FT_REGFILE, r, &f, NULL, NULL, &v)) {
> + if (v)
> + printf("%d/%d: readv - no filename\n", procid, opno);
> + free_pathname(&f);
> + return;
> + }
> + fd = open_path(&f, O_RDONLY);
> + e = fd < 0 ? errno : 0;
> + check_cwd();
> + if (fd < 0) {
> + if (v)
> + printf("%d/%d: readv - open %s failed %d\n",
> + procid, opno, f.path, e);
> + free_pathname(&f);
> + return;
> + }
> + if (fstat64(fd, &stb) < 0) {
> + if (v)
> + printf("%d/%d: readv - fstat64 %s failed %d\n",
> + procid, opno, f.path, errno);
> + free_pathname(&f);
> + close(fd);
> + return;
> + }
> + inode_info(st, sizeof(st), &stb, v);
> + if (stb.st_size == 0) {
> + if (v)
> + printf("%d/%d: readv - %s%s zero size\n", procid, opno,
> + f.path, st);
> + free_pathname(&f);
> + close(fd);
> + return;
> + }
> + lr = ((__int64_t)random() << 32) + random();
> + off = (off64_t)(lr % stb.st_size);
> + lseek64(fd, off, SEEK_SET);
> + len = (random() % FILELEN_MAX) + 1;
> + buf = malloc(len);
Check if malloc succeeded?
> +
> + iovcnt = (random() % MIN(len, IOV_MAX)) + 1;
> + iov = malloc(sizeof(struct iovec) * iovcnt);
> + for (i=0; i<iovcnt; i++) {
> + iovb = random() % len;
> + iovl = random() % (len - iovb) + 1;
> + (iov + i)->iov_base = (buf + iovb);
> + (iov + i)->iov_len = iovl;
Hmm, this reads random bytes from a random offset iovcnt times (within
the [off, off+len] range), so the total bytes it reads is not 'len', and
possibly some ranges are not read at all.
It's better to read exact len bytes once, as what you log below.
> + }
> +
> + e = read(fd, buf, len) < 0 ? errno : 0;
Forgot to update this read to readv call?
> + free(buf);
> + if (v)
> + printf("%d/%d: readv %s%s [%lld,%d,%d] %d\n",
> + procid, opno, f.path, st, (long long)off, (int)len,
> + iovcnt, e);
> + free_pathname(&f);
> + close(fd);
> +}
> +
> +void
> rename_f(int opno, long r)
> {
> fent_t *dfep;
> @@ -3379,3 +3467,76 @@ write_f(int opno, long r)
> free_pathname(&f);
> close(fd);
> }
> +
> +void
> +writev_f(int opno, long r)
> +{
> + char *buf;
> + int e;
> + pathname_t f;
> + int fd;
> + size_t len;
> + __int64_t lr;
> + off64_t off;
> + struct stat64 stb;
> + int v;
> + char st[1024];
> + struct iovec *iov = NULL;
> + int iovcnt;
> + size_t iovb;
> + size_t iovl;
> + int i;
> +
> + init_pathname(&f);
> + if (!get_fname(FT_REGm, r, &f, NULL, NULL, &v)) {
> + if (v)
> + printf("%d/%d: writev - no filename\n", procid, opno);
> + free_pathname(&f);
> + return;
> + }
> + fd = open_path(&f, O_WRONLY);
> + e = fd < 0 ? errno : 0;
> + check_cwd();
> + if (fd < 0) {
> + if (v)
> + printf("%d/%d: writev - open %s failed %d\n",
> + procid, opno, f.path, e);
> + free_pathname(&f);
> + return;
> + }
> + if (fstat64(fd, &stb) < 0) {
> + if (v)
> + printf("%d/%d: writev - fstat64 %s failed %d\n",
> + procid, opno, f.path, errno);
> + free_pathname(&f);
> + close(fd);
> + return;
> + }
> + inode_info(st, sizeof(st), &stb, v);
> + lr = ((__int64_t)random() << 32) + random();
> + off = (off64_t)(lr % MIN(stb.st_size + (1024 * 1024), MAXFSIZE));
> + off %= maxfsize;
> + lseek64(fd, off, SEEK_SET);
> + len = (random() % FILELEN_MAX) + 1;
> + buf = malloc(len);
> + memset(buf, nameseq & 0xff, len);
> +
> + iovcnt = (random() % MIN(len, IOV_MAX)) + 1;
> + iov = malloc(sizeof(struct iovec) * iovcnt);
> + for (i=0; i<iovcnt; i++) {
> + iovb = random() % len;
> + iovl = random() % (len - iovb) + 1;
> + (iov + i)->iov_base = (buf + iovb);
> + (iov + i)->iov_len = iovl;
Same here, write the range all and once.
Thanks,
Eryu
> + }
> +
> + e = writev(fd, iov, iovcnt) < 0 ? errno : 0;
> + free(buf);
> + free(iov);
> + if (v)
> + printf("%d/%d: writev %s%s [%lld,%d,%d] %d\n",
> + procid, opno, f.path, st, (long long)off, (int)len,
> + iovcnt, e);
> + free_pathname(&f);
> + close(fd);
> +}
> --
> 2.7.4
>
> --
> To unsubscribe from this list: send the line "unsubscribe fstests" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: [PATCH 2/4] fsstress: add AIO read/write and fsync test
2017-06-16 16:11 ` [PATCH 2/4] fsstress: add AIO read/write and fsync test Zorro Lang
@ 2017-07-11 4:10 ` Eryu Guan
0 siblings, 0 replies; 7+ messages in thread
From: Eryu Guan @ 2017-07-11 4:10 UTC (permalink / raw)
To: Zorro Lang; +Cc: fstests
On Sat, Jun 17, 2017 at 12:11:04AM +0800, Zorro Lang wrote:
> We found some bugs by aio read/write test, but there's not related
> operations in fsstress. So add AIO test into fsstress to increase
> AIO stress test.
>
> Due to most kernels don't support aio fsync, so set its test
> frequency to zero as default.
>
> Signed-off-by: Zorro Lang <zlang@redhat.com>
> ---
> ltp/fsstress.c | 214 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++
> 1 file changed, 214 insertions(+)
>
> diff --git a/ltp/fsstress.c b/ltp/fsstress.c
> index 67cdb2f..4ff90d5 100644
> --- a/ltp/fsstress.c
> +++ b/ltp/fsstress.c
> @@ -36,6 +36,10 @@
> #ifdef HAVE_SYS_PRCTL_H
> #include <sys/prctl.h>
> #endif
> +#ifdef AIO
> +#include <libaio.h>
> +io_context_t io_ctx;
> +#endif
>
> #ifndef FS_IOC_GETFLAGS
> #define FS_IOC_GETFLAGS _IOR('f', 1, long)
> @@ -55,9 +59,12 @@
> #define FILELEN_MAX (32*4096)
>
> typedef enum {
> + OP_AFSYNC,
> OP_ALLOCSP,
> + OP_AREAD,
> OP_ATTR_REMOVE,
> OP_ATTR_SET,
> + OP_AWRITE,
> OP_BULKSTAT,
> OP_BULKSTAT1,
> OP_CHOWN,
> @@ -158,9 +165,12 @@ struct print_string {
> #define MAXFSIZE ((1ULL << 63) - 1ULL)
> #define MAXFSIZE32 ((1ULL << 40) - 1ULL)
>
> +void afsync_f(int, long);
> void allocsp_f(int, long);
> +void aread_f(int, long);
> void attr_remove_f(int, long);
> void attr_set_f(int, long);
> +void awrite_f(int, long);
> void bulkstat_f(int, long);
> void bulkstat1_f(int, long);
> void chown_f(int, long);
> @@ -202,9 +212,12 @@ void writev_f(int, long);
>
> opdesc_t ops[] = {
> /* { OP_ENUM, "name", function, freq, iswrite }, */
> + { OP_AFSYNC, "afsync", afsync_f, 0, 1 },
> { OP_ALLOCSP, "allocsp", allocsp_f, 1, 1 },
> + { OP_AREAD, "aread", aread_f, 1, 0 },
> { OP_ATTR_REMOVE, "attr_remove", attr_remove_f, /* 1 */ 0, 1 },
> { OP_ATTR_SET, "attr_set", attr_set_f, /* 2 */ 0, 1 },
> + { OP_AWRITE, "awrite", awrite_f, 1, 1 },
> { OP_BULKSTAT, "bulkstat", bulkstat_f, 1, 0 },
> { OP_BULKSTAT1, "bulkstat1", bulkstat1_f, 1, 0 },
> { OP_CHOWN, "chown", chown_f, 3, 1 },
> @@ -587,8 +600,20 @@ int main(int argc, char **argv)
> }
> }
> procid = i;
> +#ifdef AIO
> + if (io_setup(128, &io_ctx) != 0) {
> + perror("io_setup failed");
It seems that libaio wrapper doesn't set errno on failure, it just
returns the negative errno value, so perror doesn't work as expected
here.
> + exit(1);
> + }
> +#endif
> for (i = 0; !loops || (i < loops); i++)
> doproc();
> +#ifdef AIO
> + if(io_destroy(io_ctx) != 0) {
> + perror("io_destroy failed");
Same here.
> + return 1;
> + }
> +#endif
> return 0;
> }
> }
> @@ -1708,6 +1733,62 @@ void inode_info(char *str, size_t sz, struct stat64 *s, int verbose)
> }
>
> void
> +afsync_f(int opno, long r)
> +{
> +#ifdef AIO
> + int e;
> + pathname_t f;
> + int fd;
> + int v;
> + struct iocb iocb;
> + struct iocb *iocbs[] = { &iocb };
> + struct io_event event;
> +
> + init_pathname(&f);
> + if (!get_fname(FT_REGFILE, r, &f, NULL, NULL, &v)) {
> + if (v)
> + printf("%d/%d: afsync - no filename\n", procid, opno);
> + free_pathname(&f);
> + return;
> + }
> + fd = open_path(&f, O_WRONLY | O_DIRECT);
> + e = fd < 0 ? errno : 0;
> + check_cwd();
> + if (fd < 0) {
> + if (v)
> + printf("%d/%d: afsync - open %s failed %d\n",
> + procid, opno, f.path, e);
> + free_pathname(&f);
> + return;
> + }
> +
> + io_prep_fsync(&iocb, fd);
> + if ((e = io_submit(io_ctx, 1, iocbs)) != 1) {
> + if (v)
> + printf("%d/%d: afsync - io_submit %s %d\n",
> + procid, opno, f.path, e);
> + free_pathname(&f);
> + close(fd);
> + return;
> + }
> + if ((e = io_getevents(io_ctx, 1, 1, &event, NULL)) != 1) {
> + if (v)
> + printf("%d/%d: afsync - io_getevents failed %d\n",
> + procid, opno, e);
> + free_pathname(&f);
> + close(fd);
> + return;
> + }
> +
> + e = event.res2;
> + if (v)
> + printf("%d/%d: afsync %s %d\n", procid, opno, f.path, e);
> + free_pathname(&f);
> + close(fd);
> +#endif
> +}
> +
> +void
> allocsp_f(int opno, long r)
> {
> int e;
> @@ -1761,6 +1842,131 @@ allocsp_f(int opno, long r)
> close(fd);
> }
>
> +#ifdef AIO
> +void
> +do_aio_rw(int opno, long r, int flags)
> +{
> + __int64_t align;
> + char *buf;
> + struct dioattr diob;
> + int e;
> + pathname_t f;
> + int fd;
> + size_t len;
> + __int64_t lr;
> + off64_t off;
> + struct stat64 stb;
> + int v;
> + char st[1024];
> + char *dio_env;
> + struct iocb iocb;
> + struct io_event event;
> + struct iocb *iocbs[] = { &iocb };
> + int iswrite = (flags & (O_WRONLY | O_RDWR)) ? 1 : 0;
> +
> + init_pathname(&f);
> + if (!get_fname(FT_REGFILE, r, &f, NULL, NULL, &v)) {
> + if (v)
> + printf("%d/%d: do_aio_rw - no filename\n", procid, opno);
> + free_pathname(&f);
> + return;
> + }
> + fd = open_path(&f, flags|O_DIRECT);
> + e = fd < 0 ? errno : 0;
> + check_cwd();
> + if (fd < 0) {
> + if (v)
> + printf("%d/%d: do_aio_rw - open %s failed %d\n",
> + procid, opno, f.path, e);
> + free_pathname(&f);
> + return;
> + }
> + if (fstat64(fd, &stb) < 0) {
> + if (v)
> + printf("%d/%d: do_aio_rw - fstat64 %s failed %d\n",
> + procid, opno, f.path, errno);
> + free_pathname(&f);
> + close(fd);
> + return;
> + }
> + inode_info(st, sizeof(st), &stb, v);
> + if (!iswrite && stb.st_size == 0) {
> + if (v)
> + printf("%d/%d: do_aio_rw - %s%s zero size\n", procid, opno,
> + f.path, st);
> + free_pathname(&f);
> + close(fd);
> + return;
> + }
> + if (xfsctl(f.path, fd, XFS_IOC_DIOINFO, &diob) < 0) {
> + if (v)
> + printf(
> + "%d/%d: do_aio_rw - xfsctl(XFS_IOC_DIOINFO) %s%s failed %d\n",
> + procid, opno, f.path, st, errno);
> + free_pathname(&f);
> + close(fd);
> + return;
> + }
This means that only XFS has O_DIRECT coverage in fsstress, and
dread_f/dwrite_f have the same issue.. I think we can fall back to use
stb.st_blksize as the dio alignment size (maybe factor out a common
helper to do this). But that can be done in a separate patch.
Thanks,
Eryu
> + dio_env = getenv("XFS_DIO_MIN");
> + if (dio_env)
> + diob.d_mem = diob.d_miniosz = atoi(dio_env);
> + align = (__int64_t)diob.d_miniosz;
> + lr = ((__int64_t)random() << 32) + random();
> + len = (random() % FILELEN_MAX) + 1;
> + len -= (len % align);
> + if (len <= 0)
> + len = align;
> + else if (len > diob.d_maxiosz)
> + len = diob.d_maxiosz;
> + buf = memalign(diob.d_mem, len);
> +
> + if (iswrite) {
> + off = (off64_t)(lr % MIN(stb.st_size + (1024 * 1024), MAXFSIZE));
> + off -= (off % align);
> + off %= maxfsize;
> + memset(buf, nameseq & 0xff, len);
> + io_prep_pwrite(&iocb, fd, buf, len, off);
> + } else {
> + off = (off64_t)(lr % stb.st_size);
> + off -= (off % align);
> + io_prep_pread(&iocb, fd, buf, len, off);
> + }
> + if ((e = io_submit(io_ctx, 1, iocbs)) != 1) {
> + if (v)
> + printf("%d/%d: %s - io_submit failed %d\n",
> + procid, opno, iswrite ? "awrite" : "aread", e);
> + free_pathname(&f);
> + close(fd);
> + return;
> + }
> + if ((e = io_getevents(io_ctx, 1, 1, &event, NULL)) != 1) {
> + if (v)
> + printf("%d/%d: %s - io_getevents failed %d\n",
> + procid, opno, iswrite ? "awrite" : "aread", e);
> + free_pathname(&f);
> + close(fd);
> + return;
> + }
> +
> + e = event.res != len ? event.res2 : 0;
> + free(buf);
> + if (v)
> + printf("%d/%d: %s %s%s [%lld,%d] %d\n",
> + procid, opno, iswrite ? "awrite" : "aread",
> + f.path, st, (long long)off, (int)len, e);
> + free_pathname(&f);
> + close(fd);
> +}
> +#endif
> +
> +void
> +aread_f(int opno, long r)
> +{
> +#ifdef AIO
> + do_aio_rw(opno, r, O_RDONLY);
> +#endif
> +}
> +
> void
> attr_remove_f(int opno, long r)
> {
> @@ -1865,6 +2071,14 @@ attr_set_f(int opno, long r)
> }
>
> void
> +awrite_f(int opno, long r)
> +{
> +#ifdef AIO
> + do_aio_rw(opno, r, O_WRONLY);
> +#endif
> +}
> +
> +void
> bulkstat_f(int opno, long r)
> {
> int count;
> --
> 2.7.4
>
> --
> To unsubscribe from this list: send the line "unsubscribe fstests" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at http://vger.kernel.org/majordomo-info.html
^ permalink raw reply [flat|nested] 7+ messages in thread
end of thread, other threads:[~2017-07-11 4:10 UTC | newest]
Thread overview: 7+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2017-06-16 16:11 [PATCH 1/4] fsstress: new writev and readv operations test Zorro Lang
2017-06-16 16:11 ` [PATCH 2/4] fsstress: add AIO read/write and fsync test Zorro Lang
2017-07-11 4:10 ` Eryu Guan
2017-06-16 16:11 ` [PATCH 3/4] fsstress: do memset randomly before write operations Zorro Lang
2017-06-19 9:52 ` Zorro Lang
2017-06-16 16:11 ` [PATCH 4/4] xfs/068: update golden output due to new operations in fsstress Zorro Lang
2017-07-11 3:49 ` [PATCH 1/4] fsstress: new writev and readv operations test Eryu Guan
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.