[PATCH 0/4] fincore command

[PATCH 0/4] fincore command

[Masatake YAMATO]

This patch set proposes adding fincore command to util-linux
distribution.  The command counts pages of file contents being
resident in memory(in core), and reports the numbers.

It helps system administrators understand how their systems use
physical memory, especially about page caches when they have to do
trouble shooting and/or performance tuning.

This is just my case. A person in other role may also get benefits
from this command.


Masatake YAMATO (4):
  fincore: new command for counting pages of file contents in core
  man: add a page for fincore command
  tests: add cases for testing fincore command
  bash-completion: add a function for fincore command

 bash-completion/Makemodule.am |   3 +
 bash-completion/fincore       |  25 +++++
 configure.ac                  |   4 +
 misc-utils/Makemodule.am      |   6 ++
 misc-utils/fincore.1          |  76 ++++++++++++++
 misc-utils/fincore.c          | 209 +++++++++++++++++++++++++++++++++++++
 misc-utils/fincore_orig.c     | 235 ++++++++++++++++++++++++++++++++++++++++++
 tests/commands.sh             |   1 +
 tests/expected/fincore/count  |  62 +++++++++++
 tests/ts/fincore/count        | 194 ++++++++++++++++++++++++++++++++++
 10 files changed, 815 insertions(+)
 create mode 100644 bash-completion/fincore
 create mode 100644 misc-utils/fincore.1
 create mode 100644 misc-utils/fincore.c
 create mode 100644 misc-utils/fincore_orig.c
 create mode 100644 tests/expected/fincore/count
 create mode 100755 tests/ts/fincore/count

-- 
2.9.3

[PATCH 1/4] fincore: new command for counting pages of file contents in core

[Masatake YAMATO]

Signed-off-by: Masatake YAMATO <yamato@redhat.com>
---
 configure.ac              |   4 +
 misc-utils/Makemodule.am  |   5 +
 misc-utils/fincore.c      | 209 +++++++++++++++++++++++++++++++++++++++++
 misc-utils/fincore_orig.c | 235 ++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 453 insertions(+)
 create mode 100644 misc-utils/fincore.c
 create mode 100644 misc-utils/fincore_orig.c

diff --git a/configure.ac b/configure.ac
index 8933afa..8ea4f0d 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1589,6 +1589,10 @@ dnl earlier than 2.x.
 UL_REQUIRES_HAVE([ctrlaltdel], [reboot], [reboot function])
 AM_CONDITIONAL([BUILD_CTRLALTDEL], [test "x$build_ctrlaltdel" = xyes])
 
+UL_BUILD_INIT([fincore], [check])
+UL_REQUIRES_LINUX([fincore])
+AM_CONDITIONAL([BUILD_FINCORE], [test "x$build_fincore" = xyes])
+
 UL_BUILD_INIT([fsfreeze], [check])
 UL_REQUIRES_LINUX([fsfreeze])
 AM_CONDITIONAL([BUILD_FSFREEZE], [test "x$build_fsfreeze" = xyes])
diff --git a/misc-utils/Makemodule.am b/misc-utils/Makemodule.am
index ce9df2c..6c7fab6 100644
--- a/misc-utils/Makemodule.am
+++ b/misc-utils/Makemodule.am
@@ -180,3 +180,8 @@ dist_getoptexample_SCRIPTS = \
 	misc-utils/getopt-parse.bash \
 	misc-utils/getopt-parse.tcsh
 endif
+
+if BUILD_FINCORE
+usrbin_exec_PROGRAMS += fincore
+fincore_SOURCES = misc-utils/fincore.c
+endif
diff --git a/misc-utils/fincore.c b/misc-utils/fincore.c
new file mode 100644
index 0000000..d9345ca
--- /dev/null
+++ b/misc-utils/fincore.c
@@ -0,0 +1,209 @@
+/*
+ * fincore - count pages of file contents in core
+ *
+ * Copyright (C) 2017 Red Hat, Inc. All rights reserved.
+ * Written by Masatake YAMATO <yamato@redhat.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it would be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <getopt.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "c.h"
+#include "nls.h"
+#include "closestream.h"
+
+/* For large files, mmap is called in iterative way.
+   Window is the unit of vma prepared in each mmap
+   calling.
+
+   Window size depends on page size.
+   e.g. 128MB on x86_64. ( = N_PAGES_IN_WINDOW * 4096 ). */
+#define N_PAGES_IN_WINDOW (32 * 1024)
+
+static void __attribute__((__noreturn__)) usage(FILE *out)
+{
+	const char *p = program_invocation_short_name;
+
+	if (!*p)
+		p = "fincore";
+
+	fputs(USAGE_HEADER, out);
+	fprintf(out, _(" %s [options] file...\n"), program_invocation_short_name);
+	fputs(USAGE_OPTIONS, out);
+	fputs(USAGE_SEPARATOR, out);
+	fputs(USAGE_HELP, out);
+	fputs(USAGE_VERSION, out);
+	fprintf(out, USAGE_MAN_TAIL("fincore(1)"));
+	exit(out == stderr ? EXIT_FAILURE : EXIT_SUCCESS);
+}
+
+static void report_count (const char *name, off_t file_size, off_t count_incore)
+{
+	printf ("%-10lu %-10lu %s\n", count_incore, file_size, name);
+}
+
+static void report_failure (const char *name)
+{
+	printf ("%-10s %-10ld %s\n", "failed", -1L, name);
+}
+
+static int do_mincore (void *window, const size_t len,
+		       const char *name, const int pagesize,
+		       off_t *count_incore)
+{
+	static unsigned char vec[N_PAGES_IN_WINDOW];
+	int n = (len / pagesize) + ((len % pagesize)? 1: 0);
+
+	if (mincore (window, len, vec) < 0) {
+		warn(_("failed to do mincore: %s"), name);
+		return EXIT_FAILURE;
+	}
+
+	while (n > 0)
+	{
+		if (vec[--n] & 0x1)
+		{
+			vec[n] = 0;
+			(*count_incore)++;
+		}
+	}
+
+	return EXIT_SUCCESS;
+}
+
+static int fincore_fd (int fd,
+		       const char *name, const int pagesize,
+		       off_t file_size,
+		       off_t *count_incore)
+{
+	size_t window_size = N_PAGES_IN_WINDOW * pagesize;
+	off_t  file_offset;
+	void  *window = NULL;
+	int r = EXIT_SUCCESS;
+	int warned_once = 0;
+
+	for (file_offset = 0; file_offset < file_size; file_offset += window_size) {
+		size_t len;
+
+		len = file_size - file_offset;
+		if (len >= window_size)
+			len = window_size;
+
+		window = mmap(window, len, PROT_NONE, MAP_PRIVATE, fd, file_offset);
+		if (window == MAP_FAILED) {
+			if (!warned_once) {
+				r = EXIT_FAILURE;
+				warn(_("failed to do mmap: %s"), name);
+				warned_once = 1;
+			}
+			break;
+		}
+
+		if (do_mincore (window, len, name, pagesize, count_incore) != EXIT_SUCCESS)
+			r = EXIT_FAILURE;
+
+		munmap (window, len);
+	}
+
+	return r;
+}
+
+static int fincore_name (const char *name,
+			 const int pagesize, struct stat *sb,
+			 off_t *count_incore)
+{
+	int fd;
+	int r;
+
+	if ((fd = open (name, O_RDONLY)) < 0) {
+		warn(_("failed to open: %s"), name);
+		return EXIT_FAILURE;
+	}
+
+	if (fstat (fd, sb) < 0) {
+		warn(_("failed to do fstat: %s"), name);
+		return EXIT_FAILURE;
+	}
+
+	if (sb->st_size)
+		r = fincore_fd (fd, name, pagesize, sb->st_size, count_incore);
+	else
+		/* If the file is empty,
+		   we just report this file as "successful". */
+		r = EXIT_SUCCESS;
+
+	close (fd);
+
+	return r;
+}
+
+int main(int argc, char ** argv)
+{
+	int c;
+	int pagesize;
+	int r;
+
+	static const struct option longopts[] = {
+		{ "version",    no_argument, NULL, 'V' },
+		{ "help",	no_argument, NULL, 'h' },
+		{ NULL, 0, NULL, 0 },
+	};
+
+	setlocale(LC_ALL, "");
+	bindtextdomain(PACKAGE, LOCALEDIR);
+	textdomain(PACKAGE);
+	atexit(close_stdout);
+
+	while ((c = getopt_long (argc, argv, "Vh", longopts, NULL)) != -1) {
+		switch (c) {
+		case 'V':
+			printf(UTIL_LINUX_VERSION);
+			return EXIT_SUCCESS;
+		case 'h':
+			usage(stdout);
+		default:
+			errtryhelp(EXIT_FAILURE);
+		}
+	}
+
+	if (optind == argc) {
+		warnx(_("file argument is missing"));
+		usage(stderr);
+	}
+
+
+	pagesize = getpagesize();
+	r = EXIT_SUCCESS;
+	for(; optind < argc; optind++) {
+		char *name = argv[optind];
+		struct stat sb;
+		off_t count_incore = 0;
+
+		if (fincore_name (name, pagesize, &sb, &count_incore) == EXIT_SUCCESS)
+			report_count (name, sb.st_size, count_incore);
+		else {
+			report_failure (name);
+			r = EXIT_FAILURE;
+		}
+	}
+
+	return r;
+}
diff --git a/misc-utils/fincore_orig.c b/misc-utils/fincore_orig.c
new file mode 100644
index 0000000..ade52b4
--- /dev/null
+++ b/misc-utils/fincore_orig.c
@@ -0,0 +1,235 @@
+/*
+ * fincore - ...
+ *
+ * Copyright (C) 2017 Red Hat, Inc. All rights reserved.
+ * Written by Masatake YAMATO <yamato@redhat.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it would be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ */
+
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <getopt.h>
+// #include <err.h>
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "c.h"
+#include "nls.h"
+#include "closestream.h"
+
+
+static const struct option longopts[] =
+{
+	{ "help",	0, 0, 'h' },
+	{ "version",    0, 0, 'V' },
+	{ NULL,		0, 0, 0 },
+};
+
+static void print_count (const char *name, int pagesize, struct stat *sb, off_t count)
+{
+	printf ("%-10u %-10u %s\n", (count * pagesize)/1024, sb->st_size, name);
+}
+static void print_fincore (unsigned int count, unsigned int total_count, int pagesize)
+{
+	printf ("%u %u\n", count * pagesize, total_count * pagesize);
+}
+
+static void usage(int rc)
+{
+	const char *p = program_invocation_short_name;
+	FILE *out = rc == EXIT_FAILURE ? stderr : stdout;
+
+	if (!*p)
+		p = "fincore";
+
+	fputs(USAGE_HEADER, out);
+	fprintf(out,
+	      _(" %s [options] <pathname>...\n"), p);
+
+	fputs(USAGE_SEPARATOR, out);
+	fputs(_("Print the size of file contents incore.\n"), out);
+
+	fputs(USAGE_OPTIONS, out);
+	fputs(_(" -h, --help          displays this help text\n"
+		" -V, --version       output version information and exit\n"
+	       ), out);
+
+	fprintf(out, USAGE_MAN_TAIL("fincore(1)"));
+	exit(rc);
+}
+
+enum fincore_err_bit {
+	/* Critical error is occured. Therefore the returned value COUNT,
+	   doesn't make sense. */
+	FINCORE_ERR_NOCOUNT    = 1UL << 0;
+	/* Some areas are failed to count. */	
+	FINCORE_ERR_SOMECOUNTS = 1UL << 1;	
+};
+
+ /* 128MB on x86_64 */
+#define N_PAGES_IN_WINDOW (32 * 1024)
+
+static unsigned int fincore_vma (const void *window, const size_t len,
+				 const char *name, const int pagesize,
+				 off_t *count)
+{
+	unsigned char vec[N_PAGES_IN_WINDOW];
+	int n = (len / pagesize) + (len % pagesize)? 1: 0;
+	int i;
+
+	if (mincore (window, len, vec) < 0)
+		return FINCORE_ERR_NOCOUNT;
+
+	for (i = 0; i < n; i++)
+		if (vec[i] == 1)
+			*count++;
+	return 0;
+}
+
+static unsigned int fincore_fd (int fd, const char *name, const int pagesize,
+				struct stat *sb,
+				off_t *count)
+{
+	size_t window_size = N_PAGES_IN_WINDOW * pagesize;
+	off_t  file_size = sb->st_size;
+	off_t  current_offset;
+	void  *window;
+	int r = 0;
+	
+	int n_trials = 0;
+	int n_map_failed = 0;
+	int n_count_failed = 0;
+	
+	memset(&current_offset, 0, sizeof(current_offset));
+	for (; current_offset < file_size; current_offset += window_size) {
+		size_t len = file_size - current_offset;
+		int r0;
+
+		n_trials++;
+
+		if ( len >= window_size )
+			len = window_size;
+		window = mmap(NULL, len, PROT_READ, MAP_PRIVATE, fd, current_offset);
+		if (window == MAP_FAILED) {
+			n_map_failed++;
+			r |= FINCORE_ERR_SOMECOUNTS;
+			warnx(_("failed to do mmap: %s"), name);
+			continue;
+		}
+
+		r0 = fincore_vma (window, len, name, pagesize, &count);
+		if (r0 & FINCORE_ERR_NOCOUNT)
+			n_count_failed++;
+		if (r0 & FINCORE_ERR_SOMECOUNTS)
+			r |= FINCORE_ERR_SOMECOUNTS;
+
+		munmap (window, len);
+	}
+
+	if ((n_trials - n_map_failed - n_count_failed) == 0)
+		r |= FINCORE_ERR_NOCOUNT;
+
+	return r;
+}
+
+static unsigned int fincore_name (const char *name, const int pagesize,
+				  struct stat *sb,
+				  off_t *count)
+{
+	int fd;
+	unsigned int r;
+
+	fd = open (name, O_RDONLY);
+	if (fd == -1) {
+		warnx(_("failed to open: %s"), name);
+		return FINCORE_ERR_NOCOUNT;
+	}
+
+	if (fstat (fd, sb) < 0) {
+		warnx(_("failed to do fstat: %s"), name);
+		return FINCORE_ERR_NOCOUNT;
+	}
+
+	r = fincore_fd (fd, name, pagesize, sb, count);
+
+	close (fd);
+	return r;
+}
+
+int main(int argc, char ** argv)
+{
+	int c;
+
+	setlocale(LC_ALL, "");
+	bindtextdomain(PACKAGE, LOCALEDIR);
+	textdomain(PACKAGE);
+	atexit(close_stdout);
+
+	while ((c = getopt_long (argc, argv, "hV", longopts, NULL)) != -1) {
+		switch (c) {
+		case 'h':
+			usage(EXIT_SUCCESS);
+			break;
+		case 'V':
+			printf(UTIL_LINUX_VERSION);
+			return EXIT_SUCCESS;
+		default:
+			errtryhelp(EXIT_FAILURE);
+		}
+	}
+
+	if (optind == argc) {
+		warnx(_("pathname argument is missing"));
+		usage(EXIT_FAILURE);
+	}
+
+	int pagesize = getpagesize();
+	int n_trials = 0;
+	int n_count_failed = 0;
+	int r = 0;
+	for(; optind < argc; optind++) {
+		char *name = argv[optind];
+		struct stat sb;
+		off_t count;
+		memset(&count, 0, sizeof(count));
+		unsigned int r0;
+
+		n_trials++;
+		r0 = fincore_name (name, pagesize, &sb, &count);
+
+		if (r0 & FINCORE_ERR_NOCOUNT)
+			n_count_failed++;
+
+		print_count (name, pagesize, &sb, count);
+		if (r0 & FINCORE_ERR_SOMECOUNTS)
+			r |= FINCORE_ERR_SOMECOUNTS;
+	}
+
+	if ((n_trials - n_count_failed) == 0)
+		r |= FINCORE_ERR_NOCOUNT;
+
+	if (r == 0)
+		return EXIT_SUCCESS;
+	else if (r & FINCORE_ERR_NOCOUNT)
+		return EXIT_FAILURE;
+	else
+		return 2;
+
+}
-- 
2.9.3

[PATCH 2/4] man: add a page for fincore command

[Masatake YAMATO]

Signed-off-by: Masatake YAMATO <yamato@redhat.com>
---
 misc-utils/Makemodule.am |  1 +
 misc-utils/fincore.1     | 76 ++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 77 insertions(+)
 create mode 100644 misc-utils/fincore.1

diff --git a/misc-utils/Makemodule.am b/misc-utils/Makemodule.am
index 6c7fab6..8cad4d6 100644
--- a/misc-utils/Makemodule.am
+++ b/misc-utils/Makemodule.am
@@ -183,5 +183,6 @@ endif
 
 if BUILD_FINCORE
 usrbin_exec_PROGRAMS += fincore
+dist_man_MANS += misc-utils/fincore.1
 fincore_SOURCES = misc-utils/fincore.c
 endif
diff --git a/misc-utils/fincore.1 b/misc-utils/fincore.1
new file mode 100644
index 0000000..7b9b799
--- /dev/null
+++ b/misc-utils/fincore.1
@@ -0,0 +1,76 @@
+.\" Copyright 2017 Red Hat, Inc.
+.\"
+.\" This file may be copied under the terms of the GNU Public License.
+.TH FINCORE 1 "March 2017" "util-linux" "User Commands"
+.SH NAME
+fincore \- count pages of file contents in core
+.SH SYNOPSIS
+.B fincore
+[options]
+.I file ...
+.SH DESCRIPTION
+.B fincore
+counts pages of file contents being resident in memory(in core), and
+reports the numbers.  An output line has 3 columns: the number of
+pages in core, a file size in bytes, and a file name.  If an error
+occurs during counting, failing to open a file for example,
+.B failed
+is printed at the column of the number of pages in core.  Even if
+errors occur,
+.B fincore
+continues processing the rest of files listed in a command line.
+.SH OPTIONS
+.TP
+\fB\-V\fR, \fB\-\-version\fR
+Display version information and exit.
+.TP
+\fB\-h\fR, \fB\-\-help\fR
+Display help text and exit.
+.SH EXAMPLES
+.PP
+An example of successfully executed case:
+.PP
+.RS
+.PD 0
+.TP
+.B fincore /etc/fstab /bin/emacs
+.TP
+1          1544       /etc/fstab
+.TP
+4156       17163144   /bin/emacs
+.TP
+.B echo $?
+.TP
+0
+.PD
+.RE
+.PP
+An example of failure case:
+.PP
+.RS
+.PD 0
+.TP
+.B fincore /etc/passwd
+.TP
+.I fincore: failed to open: /var/log/messages: Permission denied
+.TP
+failed     -1         /var/log/messages
+.TP
+.B echo $?
+.TP
+1
+.PD
+.RE
+.SH AUTHORS
+.MT yamato@\:redhat.com
+Masatake YAMATO
+.ME
+.SH "SEE ALSO"
+.BR mincore (2),
+.BR getpagesize (2),
+.BR getconf (1)
+.SH AVAILABILITY
+The example command is part of the util-linux package and is available from
+.UR https://\:www.kernel.org\:/pub\:/linux\:/utils\:/util-linux/
+Linux Kernel Archive
+.UE .
-- 
2.9.3

[PATCH 3/4] tests: add cases for testing fincore command

[Masatake YAMATO]

Signed-off-by: Masatake YAMATO <yamato@redhat.com>
---
 tests/commands.sh            |   1 +
 tests/expected/fincore/count |  62 ++++++++++++++
 tests/ts/fincore/count       | 194 +++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 257 insertions(+)
 create mode 100644 tests/expected/fincore/count
 create mode 100755 tests/ts/fincore/count

diff --git a/tests/commands.sh b/tests/commands.sh
index 2e3a276..2e5c11d 100644
--- a/tests/commands.sh
+++ b/tests/commands.sh
@@ -50,6 +50,7 @@ TS_CMD_FALLOCATE=${TS_CMD_FALLOCATE-"$top_builddir/fallocate"}
 TS_CMD_FDISK=${TS_CMD_FDISK-"$top_builddir/fdisk"}
 TS_CMD_FLOCK=${TS_CMD_FLOCK-"$top_builddir/flock"}
 TS_CMD_SFDISK=${TS_CMD_SFDISK-"$top_builddir/sfdisk"}
+TS_CMD_FINCORE=${TS_CMD_FINCORE-"$top_builddir/fincore"}
 TS_CMD_FINDMNT=${TS_CMD_FINDMNT-"$top_builddir/findmnt"}
 TS_CMD_FSCKCRAMFS=${TS_CMD_FSCKCRAMFS:-"$top_builddir/fsck.cramfs"}
 TS_CMD_FSCKMINIX=${TS_CMD_FSCKMINIX:-"$top_builddir/fsck.minix"}
diff --git a/tests/expected/fincore/count b/tests/expected/fincore/count
new file mode 100644
index 0000000..0ee335e
--- /dev/null
+++ b/tests/expected/fincore/count
@@ -0,0 +1,62 @@
+[ NO EXCITING FILE ]
+fincore: failed to open: no_such_file: No such file or directory
+failed     -1         no_such_file
+return value: 1
+0          0          i_EMPTY_FILE
+return value: 0
+1          2048       i_SMALLER_THAN_PAGESIZE__incore_
+return value: 0
+1          4095       i_PAGESIZE_-1__incore_
+return value: 0
+1          4096       i_JUST_PAGESIZE_incore_
+return value: 0
+0          4096       i_JUST_PAGESIZE_directio_
+return value: 0
+2          4097       i_PAGESIZE_+_1__incore_
+return value: 0
+2          8192       i_TWO_PAGES_incore_
+return value: 0
+0          8192       i_TWO_PAGES_directio_
+return value: 0
+1          8192       i_TWO_PAGES_mixed_directio_incore_
+return value: 0
+1          8192       i_TWO_PAGES_mixed_incore_directio_
+return value: 0
+2          134213632  i_WINDOW_SIZE_incore-sparse-incore_
+return value: 0
+0          134213632  i_WINDOW_SIZE_directio-sparse-directio_
+return value: 0
+1          134213632  i_WINDOW_SIZE_incore-sparse-directio_
+return value: 0
+1          134213632  i_WINDOW_SIZE_directio-sparse-incore_
+return value: 0
+2          134217728  i_WINDOW_SIZE_+_1_page_incore-sparse-incore_
+return value: 0
+0          134217728  i_WINDOW_SIZE_+_1_page_directio-sparse-directio_
+return value: 0
+1          134217728  i_WINDOW_SIZE_+_1_page_incore-sparse-directio_
+return value: 0
+1          134217728  i_WINDOW_SIZE_+_1_page_directio-sparse-incore_
+return value: 0
+[ MULTIPLE FILES ]
+fincore: failed to open: no_such_file: No such file or directory
+failed     -1         no_such_file
+0          0          i_EMPTY_FILE
+1          2048       i_SMALLER_THAN_PAGESIZE__incore_
+1          4095       i_PAGESIZE_-1__incore_
+1          4096       i_JUST_PAGESIZE_incore_
+0          4096       i_JUST_PAGESIZE_directio_
+2          4097       i_PAGESIZE_+_1__incore_
+2          8192       i_TWO_PAGES_incore_
+0          8192       i_TWO_PAGES_directio_
+1          8192       i_TWO_PAGES_mixed_directio_incore_
+1          8192       i_TWO_PAGES_mixed_incore_directio_
+2          134213632  i_WINDOW_SIZE_incore-sparse-incore_
+0          134213632  i_WINDOW_SIZE_directio-sparse-directio_
+1          134213632  i_WINDOW_SIZE_incore-sparse-directio_
+1          134213632  i_WINDOW_SIZE_directio-sparse-incore_
+2          134217728  i_WINDOW_SIZE_+_1_page_incore-sparse-incore_
+0          134217728  i_WINDOW_SIZE_+_1_page_directio-sparse-directio_
+1          134217728  i_WINDOW_SIZE_+_1_page_incore-sparse-directio_
+1          134217728  i_WINDOW_SIZE_+_1_page_directio-sparse-incore_
+return value: 1
diff --git a/tests/ts/fincore/count b/tests/ts/fincore/count
new file mode 100755
index 0000000..4bb8912
--- /dev/null
+++ b/tests/ts/fincore/count
@@ -0,0 +1,194 @@
+#!/bin/bash
+
+function header
+{
+    echo "[" "$1" "]"
+}
+
+function footer
+{
+    echo "return value: $1"
+}
+
+function make_input_name
+{
+    header=$1
+    prefix=i_
+    echo ${prefix}$(sed -e "s/[^-+a-zA-Z0-9_]/_/g"<<<"$header")
+}
+
+function run_dd_test
+{
+    header=$1
+    bs=$2
+    flags=$3
+
+    input=$(make_input_name "$header")
+    INPUT="${INPUT} ${input}"
+
+    if [ "$bs" = 0 ]; then
+	touch $input
+    else
+	$DD if=/dev/zero of=$input count=1 bs=$bs $flags
+    fi
+
+    $TS_CMD_FINCORE $input
+
+    footer "$?"
+}
+
+function run_dd_dd_test
+{
+    header=$1
+    flags0=$2
+    flags1=$3
+
+    bs=$PAGE_SIZE
+
+    input=$(make_input_name "$header")
+    INPUT="${INPUT} ${input}"
+
+    $DD if=/dev/zero of=$input count=1 bs=$bs $flags0
+    $DD if=/dev/zero of=$input count=1 bs=$bs $flags1
+
+    $TS_CMD_FINCORE $input
+
+    footer "$?"
+}
+
+TS_TOPDIR="${0%/*}/../.."
+TS_DESC="count file contents in core"
+
+. $TS_TOPDIR/functions.sh
+ts_init "$*"
+
+PAGE_SIZE=$($TS_HELPER_SYSINFO pagesize)
+WINDOW_SIZE=$(( 32 * 1024 * PAGE_SIZE ))
+
+DD_FLAGS="status=none"
+DD="dd $DD_FLAGS"
+
+
+ts_check_test_command "$TS_CMD_FINCORE"
+ts_cd "$TS_OUTDIR"
+
+INPUT=
+input=
+
+{
+    input=no_such_file
+    INPUT="${INPUT} ${input}"
+
+    header "NO EXCITING FILE"
+    $TS_CMD_FINCORE $input
+    footer "$?"
+} >> $TS_OUTPUT 2>&1
+
+{
+    run_dd_test "EMPTY FILE" 0
+} >> $TS_OUTPUT 2>&1
+
+{
+    run_dd_test "SMALLER THAN PAGESIZE (incore)" $(( PAGE_SIZE / 2 ))
+} >> $TS_OUTPUT 2>&1
+
+{
+    run_dd_test "PAGESIZE -1 (incore)" $(( PAGE_SIZE - 1 ))
+} >> $TS_OUTPUT 2>&1
+
+{
+    run_dd_test "JUST PAGESIZE(incore)" $(( PAGE_SIZE ))
+} >> $TS_OUTPUT 2>&1
+
+{
+    run_dd_test "JUST PAGESIZE(directio)" $(( PAGE_SIZE )) "oflag=direct"
+} >> $TS_OUTPUT 2>&1
+
+{
+    run_dd_test "PAGESIZE + 1 (incore)" $(( PAGE_SIZE + 1 ))
+} >> $TS_OUTPUT 2>&1
+
+{
+    run_dd_test "TWO PAGES(incore)" $(( 2 * PAGE_SIZE ))
+} >> $TS_OUTPUT 2>&1
+
+{
+    run_dd_test "TWO PAGES(directio)" $(( 2 * PAGE_SIZE )) "oflag=direct"
+} >> $TS_OUTPUT 2>&1
+
+{
+    run_dd_dd_test "TWO PAGES(mixed directio/incore)" \
+			    oflag=direct \
+			    "oflag=append seek=1"
+} >> $TS_OUTPUT 2>&1
+
+{
+    run_dd_dd_test "TWO PAGES(mixed incore/directio)" \
+		   "" \
+		   "oflag=direct,append seek=1"
+} >> $TS_OUTPUT 2>&1
+
+{
+    hole_count=$(( WINDOW_SIZE / PAGE_SIZE - 2 ))
+    run_dd_dd_test "WINDOW SIZE(incore-sparse-incore)" \
+		   "" \
+		   "oflag=append seek=$hole_count"
+} >> $TS_OUTPUT 2>&1
+
+{
+    hole_count=$(( WINDOW_SIZE / PAGE_SIZE - 2 ))
+    run_dd_dd_test "WINDOW SIZE(directio-sparse-directio)" \
+		   "oflag=direct" \
+		   "oflag=append,direct seek=$hole_count"
+} >> $TS_OUTPUT 2>&1
+
+{
+    hole_count=$(( WINDOW_SIZE / PAGE_SIZE - 2 ))
+    run_dd_dd_test "WINDOW SIZE(incore-sparse-directio)" \
+		   "" \
+		   "oflag=append,direct seek=$hole_count"
+} >> $TS_OUTPUT 2>&1
+
+{
+    hole_count=$(( WINDOW_SIZE / PAGE_SIZE - 2 ))
+    run_dd_dd_test "WINDOW SIZE(directio-sparse-incore)" \
+		   "oflag=direct" \
+		   "oflag=append seek=$hole_count"
+} >> $TS_OUTPUT 2>&1
+
+{
+    hole_count=$(( WINDOW_SIZE / PAGE_SIZE - 1 ))
+    run_dd_dd_test "WINDOW SIZE + 1 page(incore-sparse-incore)" \
+		   "" \
+		   "oflag=append seek=$hole_count"
+} >> $TS_OUTPUT 2>&1
+
+{
+    hole_count=$(( WINDOW_SIZE / PAGE_SIZE - 1 ))
+    run_dd_dd_test "WINDOW SIZE + 1 page(directio-sparse-directio)" \
+		   "oflag=direct" \
+		   "oflag=append,direct seek=$hole_count"
+} >> $TS_OUTPUT 2>&1
+
+{
+    hole_count=$(( WINDOW_SIZE / PAGE_SIZE - 1 ))
+    run_dd_dd_test "WINDOW SIZE + 1 page(incore-sparse-directio)" \
+		   "" \
+		   "oflag=append,direct seek=$hole_count"
+} >> $TS_OUTPUT 2>&1
+
+{
+    hole_count=$(( WINDOW_SIZE / PAGE_SIZE - 1 ))
+    run_dd_dd_test "WINDOW SIZE + 1 page(directio-sparse-incore)" \
+		   "oflag=direct" \
+		   "oflag=append seek=$hole_count"
+} >> $TS_OUTPUT 2>&1
+
+{
+    header "MULTIPLE FILES"
+    $TS_CMD_FINCORE $INPUT
+    footer "$?"
+} >> $TS_OUTPUT 2>&1
+
+rm -f $INPUT
+ts_finalize
-- 
2.9.3

[PATCH 4/4] bash-completion: add a function for fincore command

[Masatake YAMATO]

Signed-off-by: Masatake YAMATO <yamato@redhat.com>
---
 bash-completion/Makemodule.am |  3 +++
 bash-completion/fincore       | 25 +++++++++++++++++++++++++
 2 files changed, 28 insertions(+)
 create mode 100644 bash-completion/fincore

diff --git a/bash-completion/Makemodule.am b/bash-completion/Makemodule.am
index 3ffd124..ff7b052 100644
--- a/bash-completion/Makemodule.am
+++ b/bash-completion/Makemodule.am
@@ -18,6 +18,9 @@ endif
 if BUILD_COLUMN
 dist_bashcompletion_DATA += bash-completion/column
 endif
+if BUILD_FINCORE
+dist_bashcompletion_DATA += bash-completion/fincore
+endif
 if BUILD_FINDMNT
 dist_bashcompletion_DATA += bash-completion/findmnt
 endif
diff --git a/bash-completion/fincore b/bash-completion/fincore
new file mode 100644
index 0000000..d213586
--- /dev/null
+++ b/bash-completion/fincore
@@ -0,0 +1,25 @@
+_fincore_module()
+{
+	local cur prev OPTS
+	COMPREPLY=()
+	cur="${COMP_WORDS[COMP_CWORD]}"
+	prev="${COMP_WORDS[COMP_CWORD-1]}"
+	case $prev in
+		'-h'|'--help'|'-V'|'--version')
+			return 0
+			;;
+	esac
+	case $cur in
+	    -*)
+			OPTS="--help
+				--version"
+			COMPREPLY=( $(compgen -W "${OPTS[*]}" -- $cur) )
+			return 0
+			;;
+	esac
+	local IFS=$'\n'
+	compopt -o filenames
+	COMPREPLY=( $(compgen -f -- ${cur:-"/"}) )
+	return 0
+}
+complete -F _fincore_module fincore
-- 
2.9.3

Re: [PATCH 0/4] fincore command

[Karel Zak]

On Tue, Mar 07, 2017 at 11:33:48AM +0900, Masatake YAMATO wrote:
> This patch set proposes adding fincore command to util-linux
> distribution.  The command counts pages of file contents being
> resident in memory(in core), and reports the numbers.
> 
> It helps system administrators understand how their systems use
> physical memory, especially about page caches when they have to do
> trouble shooting and/or performance tuning.
> 
> This is just my case. A person in other role may also get benefits
> from this command.

Not sure if I like the format ;-)

 $ fincore /etc/fstab /bin/emacs
 1          1544       /etc/fstab
 4156       17163144   /bin/emacs

Why we need file size there? Would be better to use filename as the
first column?

 $ fincore /etc/fstab /bin/emacs
 /etc/fstab 1
 /bin/emacs 4156

it would be also nice have a way how to print only the number if only one
file specified. Then you can use it in scripts without awk (or so),
just:

  FILEPAGES=$(fincore /bin/emacs)

    Karel

-- 
 Karel Zak  <kzak@redhat.com>
 http://karelzak.blogspot.com

Re: [PATCH 0/4] fincore command

[Masatake YAMATO]

Hi, thank you for comments.

On Mon, 13 Mar 2017 15:50:41 +0100, Karel Zak <kzak@redhat.com> wrote:
> Not sure if I like the format ;-)
> 
>  $ fincore /etc/fstab /bin/emacs
>  1          1544       /etc/fstab
>  4156       17163144   /bin/emacs
> 
> Why we need file size there? Would be better to use filename as the
> first column?

* Why we need file size there?

I don't insist including the file size column.

The original reason was that I assumed that a user may want to know
how much part of the file printed by fincore is in core. The file-size
column is for the hint.

How about adding an option to control show the column (--file-size|-s)?
Or should I delete the column fron the output completely?

* Would be better to use filename as the first column?

I follow the format that du and ls commands use:

    $ du /etc/passwd
    4	/etc/passwd

    $ ls -i /tmp | head
    67003 emacs1000
    47611 systemd-private-dd34e7d700c6441db79e4caa7b3c3205-colord.service-N7dhub
    24891 systemd-private-dd34e7d700c6441db79e4caa7b3c3205-rtkit-daemon.service-WbOmt1

du and ls are popular enough. So potential users of fincore
command may be familiar with the formats.

There are two benefits this(du style) format.

+ awk/read/cut commands friendly

White spaces can be included in a file name.
As the result, a user cannot assume the column
for the page counts easily if filename is at the
first column.

e.g. 

    (the format in my patch)
    $ touch 'a b'
    $ ./util-linux/fincore 'a b'
    0          0          a b
    $ ./util-linux/fincore 'a b' | awk '{print $1}'
    0
    $ ./util-linux/fincore 'a b' | { read A REST; echo $A; }
    0
    $ ./util-linux/fincore 'a b' | cut -d ' ' -f1 
    0

+ beauty of output

If filename is at the first column, I wonder
how can I print lines in column aligned way:

    0         -1         /foo/bar/baz/very---long---file---name
    0          0          shortname

By finding the longest file name from arguments passed to fincore
command, the column length can be calumniated, and fincore may be
able to print following lines:

    /foo/bar/baz/very---long---file---name	0         -1         
    shortname					0          0

However, I wonder what I should do if multibyte characters are
used in the file name.

Of course, the column alignment is less important than providing the
way to know much file contents are incore to users.

> it would be also nice have a way how to print only the number if only one
> file specified. Then you can use it in scripts without awk (or so),
> just:
> 
>   FILEPAGES=$(fincore /bin/emacs)
> 

In the station(interactive use) that a user of fincore knows the
number of file names passed to the command, printing only the number
is bettern. However, there can be a situation(scripting) that a user
cannot know the number of file names.

{ 
  FILES=...
  FILES="$FILES $tmp"
  ...
  
  fincore $FILES
} | do_something

In such case the columns are fixed(the page numbers and file name)
will be better for people writing `do_something'. For the such
use case, I would like to add an option --print-filename-always.
The default behavior (behavior without the option) should be
implemented as what you wrote (just printing the number).
Is it o.k.?

After getting feedback from you, I will submit v2 patches.

Masatake YAMATO

Re: [PATCH 0/4] fincore command

[Karel Zak]

On Wed, Mar 15, 2017 at 02:52:23AM +0900, Masatake YAMATO wrote:
> After getting feedback from you, I will submit v2 patches.

That's fine, the current version is good enough. Thanks.

    Karel

-- 
 Karel Zak  <kzak@redhat.com>
 http://karelzak.blogspot.com

Re: [PATCH 0/4] fincore command

[Karel Zak]

On Tue, Mar 07, 2017 at 11:33:48AM +0900, Masatake YAMATO wrote:
>  bash-completion/Makemodule.am |   3 +
>  bash-completion/fincore       |  25 +++++
>  configure.ac                  |   4 +
>  misc-utils/Makemodule.am      |   6 ++
>  misc-utils/fincore.1          |  76 ++++++++++++++
>  misc-utils/fincore.c          | 209 +++++++++++++++++++++++++++++++++++++
>  misc-utils/fincore_orig.c     | 235 ++++++++++++++++++++++++++++++++++++++++++
>  tests/commands.sh             |   1 +
>  tests/expected/fincore/count  |  62 +++++++++++
>  tests/ts/fincore/count        | 194 ++++++++++++++++++++++++++++++++++
>  10 files changed, 815 insertions(+)
>  create mode 100644 bash-completion/fincore
>  create mode 100644 misc-utils/fincore.1
>  create mode 100644 misc-utils/fincore.c
>  create mode 100644 misc-utils/fincore_orig.c
>  create mode 100644 tests/expected/fincore/count
>  create mode 100755 tests/ts/fincore/count

 Merged (except fincore_orig.c).

 I'll very probably a little bit play with the output :-) It's like to
 have human readable sizes (10M) by default, align the numbers to the
 right (maybe use libsmartcols if more files specified).

    Karel

-- 
 Karel Zak  <kzak@redhat.com>
 http://karelzak.blogspot.com

Re: [PATCH 0/4] fincore command

[Karel Zak]

On Thu, Mar 23, 2017 at 12:58:25PM +0100, Karel Zak wrote:
> On Tue, Mar 07, 2017 at 11:33:48AM +0900, Masatake YAMATO wrote:
> >  bash-completion/Makemodule.am |   3 +
> >  bash-completion/fincore       |  25 +++++
> >  configure.ac                  |   4 +
> >  misc-utils/Makemodule.am      |   6 ++
> >  misc-utils/fincore.1          |  76 ++++++++++++++
> >  misc-utils/fincore.c          | 209 +++++++++++++++++++++++++++++++++++++
> >  misc-utils/fincore_orig.c     | 235 ++++++++++++++++++++++++++++++++++++++++++
> >  tests/commands.sh             |   1 +
> >  tests/expected/fincore/count  |  62 +++++++++++
> >  tests/ts/fincore/count        | 194 ++++++++++++++++++++++++++++++++++
> >  10 files changed, 815 insertions(+)
> >  create mode 100644 bash-completion/fincore
> >  create mode 100644 misc-utils/fincore.1
> >  create mode 100644 misc-utils/fincore.c
> >  create mode 100644 misc-utils/fincore_orig.c
> >  create mode 100644 tests/expected/fincore/count
> >  create mode 100755 tests/ts/fincore/count
> 
>  Merged (except fincore_orig.c).
> 
>  I'll very probably a little bit play with the output :-) It's like to
>  have human readable sizes (10M) by default, align the numbers to the
>  right (maybe use libsmartcols if more files specified).

Implemented, changes:

 * errors are printed to stderr only, the output is not affected by errors
 * use libsmartcols
 * numbers aligned to the right
 * add --bytes --raw --noheadings --json and --output <list>
 * you can use $(fincore -o PAGES -nr /etc/passwd) to get only the number

The default output is:

$ fincore ~/Mail/Maildir/.notmuch/xapian/*
PAGES  SIZE FILE
    0    0B /home/kzak/Mail/Maildir/.notmuch/xapian/flintlock
    1   28B /home/kzak/Mail/Maildir/.notmuch/xapian/iamchert
   17 67.4K /home/kzak/Mail/Maildir/.notmuch/xapian/position.baseA
   17 67.4K /home/kzak/Mail/Maildir/.notmuch/xapian/position.baseB
 1185  4.2G /home/kzak/Mail/Maildir/.notmuch/xapian/position.DB
    9 35.7K /home/kzak/Mail/Maildir/.notmuch/xapian/postlist.baseA
    9 35.7K /home/kzak/Mail/Maildir/.notmuch/xapian/postlist.baseB
87831  2.2G /home/kzak/Mail/Maildir/.notmuch/xapian/postlist.DB
    1  208B /home/kzak/Mail/Maildir/.notmuch/xapian/record.baseA
    1  208B /home/kzak/Mail/Maildir/.notmuch/xapian/record.baseB
   66 11.5M /home/kzak/Mail/Maildir/.notmuch/xapian/record.DB
    7 27.6K /home/kzak/Mail/Maildir/.notmuch/xapian/termlist.baseA
    7 27.6K /home/kzak/Mail/Maildir/.notmuch/xapian/termlist.baseB
 1941  1.7G /home/kzak/Mail/Maildir/.notmuch/xapian/termlist.DB


I'd like to have also PAGES-IN-BYTES column to make it easy for
humans, but not sure how to call the column... "RES" or "RSS"? 

For example "343M" rather than "87831" (see above).

Nice util, thanks!

    Karel

-- 
 Karel Zak  <kzak@redhat.com>
 http://karelzak.blogspot.com

Re: [PATCH 0/4] fincore command

[Masatake YAMATO]

Thank you for merging.
The output looks sophisticated.

About RES and RSS, I don't know which is better.
As you know top command users RES and ps command uses RSS...

Masatake YAMATO

> On Thu, Mar 23, 2017 at 12:58:25PM +0100, Karel Zak wrote:
>> On Tue, Mar 07, 2017 at 11:33:48AM +0900, Masatake YAMATO wrote:
>> >  bash-completion/Makemodule.am |   3 +
>> >  bash-completion/fincore       |  25 +++++
>> >  configure.ac                  |   4 +
>> >  misc-utils/Makemodule.am      |   6 ++
>> >  misc-utils/fincore.1          |  76 ++++++++++++++
>> >  misc-utils/fincore.c          | 209 +++++++++++++++++++++++++++++++++++++
>> >  misc-utils/fincore_orig.c     | 235 ++++++++++++++++++++++++++++++++++++++++++
>> >  tests/commands.sh             |   1 +
>> >  tests/expected/fincore/count  |  62 +++++++++++
>> >  tests/ts/fincore/count        | 194 ++++++++++++++++++++++++++++++++++
>> >  10 files changed, 815 insertions(+)
>> >  create mode 100644 bash-completion/fincore
>> >  create mode 100644 misc-utils/fincore.1
>> >  create mode 100644 misc-utils/fincore.c
>> >  create mode 100644 misc-utils/fincore_orig.c
>> >  create mode 100644 tests/expected/fincore/count
>> >  create mode 100755 tests/ts/fincore/count
>> 
>>  Merged (except fincore_orig.c).
>> 
>>  I'll very probably a little bit play with the output :-) It's like to
>>  have human readable sizes (10M) by default, align the numbers to the
>>  right (maybe use libsmartcols if more files specified).
> 
> Implemented, changes:
> 
>  * errors are printed to stderr only, the output is not affected by errors
>  * use libsmartcols
>  * numbers aligned to the right
>  * add --bytes --raw --noheadings --json and --output <list>
>  * you can use $(fincore -o PAGES -nr /etc/passwd) to get only the number
> 
> The default output is:
> 
> $ fincore ~/Mail/Maildir/.notmuch/xapian/*
> PAGES  SIZE FILE
>     0    0B /home/kzak/Mail/Maildir/.notmuch/xapian/flintlock
>     1   28B /home/kzak/Mail/Maildir/.notmuch/xapian/iamchert
>    17 67.4K /home/kzak/Mail/Maildir/.notmuch/xapian/position.baseA
>    17 67.4K /home/kzak/Mail/Maildir/.notmuch/xapian/position.baseB
>  1185  4.2G /home/kzak/Mail/Maildir/.notmuch/xapian/position.DB
>     9 35.7K /home/kzak/Mail/Maildir/.notmuch/xapian/postlist.baseA
>     9 35.7K /home/kzak/Mail/Maildir/.notmuch/xapian/postlist.baseB
> 87831  2.2G /home/kzak/Mail/Maildir/.notmuch/xapian/postlist.DB
>     1  208B /home/kzak/Mail/Maildir/.notmuch/xapian/record.baseA
>     1  208B /home/kzak/Mail/Maildir/.notmuch/xapian/record.baseB
>    66 11.5M /home/kzak/Mail/Maildir/.notmuch/xapian/record.DB
>     7 27.6K /home/kzak/Mail/Maildir/.notmuch/xapian/termlist.baseA
>     7 27.6K /home/kzak/Mail/Maildir/.notmuch/xapian/termlist.baseB
>  1941  1.7G /home/kzak/Mail/Maildir/.notmuch/xapian/termlist.DB
> 
> 
> I'd like to have also PAGES-IN-BYTES column to make it easy for
> humans, but not sure how to call the column... "RES" or "RSS"? 
> 
> For example "343M" rather than "87831" (see above).
> 
> Nice util, thanks!
> 
>     Karel
> 
> -- 
>  Karel Zak  <kzak@redhat.com>
>  http://karelzak.blogspot.com

Re: [PATCH 0/4] fincore command

[Karel Zak]

On Mon, Mar 27, 2017 at 09:41:52AM +0900, Masatake YAMATO wrote:
> About RES and RSS, I don't know which is better.
> As you know top command users RES and ps command uses RSS...

 Added column RES.

    Karel

-- 
 Karel Zak  <kzak@redhat.com>
 http://karelzak.blogspot.com