[PATCH 1/2] smaps: show VM_SOFTDIRTY flag in VmFlags line

[PATCH 1/2] smaps: show VM_SOFTDIRTY flag in VmFlags line

[Naoya Horiguchi]

This flag shows that soft dirty bit is not enabled yet.
You can enable it by "echo 4 > /proc/pid/clear_refs."

Signed-off-by: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
---
 fs/proc/task_mmu.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git v3.12-rc2-mmots-2013-09-24-17-03.orig/fs/proc/task_mmu.c v3.12-rc2-mmots-2013-09-24-17-03/fs/proc/task_mmu.c
index 7366e9d..c591928 100644
--- v3.12-rc2-mmots-2013-09-24-17-03.orig/fs/proc/task_mmu.c
+++ v3.12-rc2-mmots-2013-09-24-17-03/fs/proc/task_mmu.c
@@ -561,6 +561,9 @@ static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma)
 		[ilog2(VM_NONLINEAR)]	= "nl",
 		[ilog2(VM_ARCH_1)]	= "ar",
 		[ilog2(VM_DONTDUMP)]	= "dd",
+#ifdef CONFIG_MEM_SOFT_DIRTY
+		[ilog2(VM_SOFTDIRTY)]	= "sd",
+#endif
 		[ilog2(VM_MIXEDMAP)]	= "mm",
 		[ilog2(VM_HUGEPAGE)]	= "hg",
 		[ilog2(VM_NOHUGEPAGE)]	= "nh",
-- 
1.8.3.1

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

[PATCH 2/2] page-types.c: support KPF_SOFTDIRTY bit

[Naoya Horiguchi]

Soft dirty bit allows us to track which pages are written since the
last clear_ref (by "echo 4 > /proc/pid/clear_refs".) This is useful
for userspace applications to know their memory footprints.

Note that the kernel exposes this flag via bit[55] of /proc/pid/pagemap,
and the semantics is not a default one (scheduled to be the default in
the near future.) However, it shifts to the new semantics at the first
clear_ref, and the users of soft dirty bit always do it before utilizing
the bit, so that's not a big deal. Users must avoid relying on the bit
in page-types before the first clear_ref.

Signed-off-by: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
---
 include/linux/kernel-page-flags.h |  1 +
 tools/vm/page-types.c             | 32 ++++++++++++++++++++------------
 2 files changed, 21 insertions(+), 12 deletions(-)

diff --git v3.12-rc2-mmots-2013-09-24-17-03.orig/include/linux/kernel-page-flags.h v3.12-rc2-mmots-2013-09-24-17-03/include/linux/kernel-page-flags.h
index 546eb6a..f65ce09 100644
--- v3.12-rc2-mmots-2013-09-24-17-03.orig/include/linux/kernel-page-flags.h
+++ v3.12-rc2-mmots-2013-09-24-17-03/include/linux/kernel-page-flags.h
@@ -15,5 +15,6 @@
 #define KPF_OWNER_PRIVATE	37
 #define KPF_ARCH		38
 #define KPF_UNCACHED		39
+#define KPF_SOFTDIRTY		40
 
 #endif /* LINUX_KERNEL_PAGE_FLAGS_H */
diff --git v3.12-rc2-mmots-2013-09-24-17-03.orig/tools/vm/page-types.c v3.12-rc2-mmots-2013-09-24-17-03/tools/vm/page-types.c
index 71c9c25..d5e9d6d 100644
--- v3.12-rc2-mmots-2013-09-24-17-03.orig/tools/vm/page-types.c
+++ v3.12-rc2-mmots-2013-09-24-17-03/tools/vm/page-types.c
@@ -59,12 +59,14 @@
 #define PM_PSHIFT_BITS      6
 #define PM_PSHIFT_OFFSET    (PM_STATUS_OFFSET - PM_PSHIFT_BITS)
 #define PM_PSHIFT_MASK      (((1LL << PM_PSHIFT_BITS) - 1) << PM_PSHIFT_OFFSET)
-#define PM_PSHIFT(x)        (((u64) (x) << PM_PSHIFT_OFFSET) & PM_PSHIFT_MASK)
+#define __PM_PSHIFT(x)      (((uint64_t) (x) << PM_PSHIFT_OFFSET) & PM_PSHIFT_MASK)
 #define PM_PFRAME_MASK      ((1LL << PM_PSHIFT_OFFSET) - 1)
 #define PM_PFRAME(x)        ((x) & PM_PFRAME_MASK)
 
+#define __PM_SOFT_DIRTY      (1LL)
 #define PM_PRESENT          PM_STATUS(4LL)
 #define PM_SWAP             PM_STATUS(2LL)
+#define PM_SOFT_DIRTY       __PM_PSHIFT(__PM_SOFT_DIRTY)
 
 
 /*
@@ -83,6 +85,7 @@
 #define KPF_OWNER_PRIVATE	37
 #define KPF_ARCH		38
 #define KPF_UNCACHED		39
+#define KPF_SOFTDIRTY		40
 
 /* [48-] take some arbitrary free slots for expanding overloaded flags
  * not part of kernel API
@@ -132,6 +135,7 @@ static const char * const page_flag_names[] = {
 	[KPF_OWNER_PRIVATE]	= "O:owner_private",
 	[KPF_ARCH]		= "h:arch",
 	[KPF_UNCACHED]		= "c:uncached",
+	[KPF_SOFTDIRTY]		= "f:softdirty",
 
 	[KPF_READAHEAD]		= "I:readahead",
 	[KPF_SLOB_FREE]		= "P:slob_free",
@@ -417,7 +421,7 @@ static int bit_mask_ok(uint64_t flags)
 	return 1;
 }
 
-static uint64_t expand_overloaded_flags(uint64_t flags)
+static uint64_t expand_overloaded_flags(uint64_t flags, uint64_t pme)
 {
 	/* SLOB/SLUB overload several page flags */
 	if (flags & BIT(SLAB)) {
@@ -433,6 +437,9 @@ static uint64_t expand_overloaded_flags(uint64_t flags)
 	if ((flags & (BIT(RECLAIM) | BIT(WRITEBACK))) == BIT(RECLAIM))
 		flags ^= BIT(RECLAIM) | BIT(READAHEAD);
 
+	if (pme & PM_SOFT_DIRTY)
+		flags |= BIT(SOFTDIRTY);
+
 	return flags;
 }
 
@@ -448,11 +455,11 @@ static uint64_t well_known_flags(uint64_t flags)
 	return flags;
 }
 
-static uint64_t kpageflags_flags(uint64_t flags)
+static uint64_t kpageflags_flags(uint64_t flags, uint64_t pme)
 {
-	flags = expand_overloaded_flags(flags);
-
-	if (!opt_raw)
+	if (opt_raw)
+		flags = expand_overloaded_flags(flags, pme);
+	else
 		flags = well_known_flags(flags);
 
 	return flags;
@@ -545,9 +552,9 @@ static size_t hash_slot(uint64_t flags)
 }
 
 static void add_page(unsigned long voffset,
-		     unsigned long offset, uint64_t flags)
+		     unsigned long offset, uint64_t flags, uint64_t pme)
 {
-	flags = kpageflags_flags(flags);
+	flags = kpageflags_flags(flags, pme);
 
 	if (!bit_mask_ok(flags))
 		return;
@@ -569,7 +576,8 @@ static void add_page(unsigned long voffset,
 #define KPAGEFLAGS_BATCH	(64 << 10)	/* 64k pages */
 static void walk_pfn(unsigned long voffset,
 		     unsigned long index,
-		     unsigned long count)
+		     unsigned long count,
+		     uint64_t pme)
 {
 	uint64_t buf[KPAGEFLAGS_BATCH];
 	unsigned long batch;
@@ -583,7 +591,7 @@ static void walk_pfn(unsigned long voffset,
 			break;
 
 		for (i = 0; i < pages; i++)
-			add_page(voffset + i, index + i, buf[i]);
+			add_page(voffset + i, index + i, buf[i], pme);
 
 		index += pages;
 		count -= pages;
@@ -608,7 +616,7 @@ static void walk_vma(unsigned long index, unsigned long count)
 		for (i = 0; i < pages; i++) {
 			pfn = pagemap_pfn(buf[i]);
 			if (pfn)
-				walk_pfn(index + i, pfn, 1);
+				walk_pfn(index + i, pfn, 1, buf[i]);
 		}
 
 		index += pages;
@@ -659,7 +667,7 @@ static void walk_addr_ranges(void)
 
 	for (i = 0; i < nr_addr_ranges; i++)
 		if (!opt_pid)
-			walk_pfn(0, opt_offset[i], opt_size[i]);
+			walk_pfn(0, opt_offset[i], opt_size[i], 0);
 		else
 			walk_task(opt_offset[i], opt_size[i]);
 
-- 
1.8.3.1

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [PATCH 1/2] smaps: show VM_SOFTDIRTY flag in VmFlags line

[Pavel Emelyanov]

On 10/04/2013 11:02 PM, Naoya Horiguchi wrote:
> This flag shows that soft dirty bit is not enabled yet.
> You can enable it by "echo 4 > /proc/pid/clear_refs."

The comment is not correct. Per-VMA soft-dirty flag means, that
VMA is "newly created" one and thus represents a new (dirty) are
in task's VM.

Other than this -- yes, it's nice to have this flag in smaps.

> Signed-off-by: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
> ---
>  fs/proc/task_mmu.c | 3 +++
>  1 file changed, 3 insertions(+)
> 
> diff --git v3.12-rc2-mmots-2013-09-24-17-03.orig/fs/proc/task_mmu.c v3.12-rc2-mmots-2013-09-24-17-03/fs/proc/task_mmu.c
> index 7366e9d..c591928 100644
> --- v3.12-rc2-mmots-2013-09-24-17-03.orig/fs/proc/task_mmu.c
> +++ v3.12-rc2-mmots-2013-09-24-17-03/fs/proc/task_mmu.c
> @@ -561,6 +561,9 @@ static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma)
>  		[ilog2(VM_NONLINEAR)]	= "nl",
>  		[ilog2(VM_ARCH_1)]	= "ar",
>  		[ilog2(VM_DONTDUMP)]	= "dd",
> +#ifdef CONFIG_MEM_SOFT_DIRTY
> +		[ilog2(VM_SOFTDIRTY)]	= "sd",
> +#endif
>  		[ilog2(VM_MIXEDMAP)]	= "mm",
>  		[ilog2(VM_HUGEPAGE)]	= "hg",
>  		[ilog2(VM_NOHUGEPAGE)]	= "nh",
> 


--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

[PATCH 1/2 v2] smaps: show VM_SOFTDIRTY flag in VmFlags line

[Naoya Horiguchi]

On Mon, Oct 07, 2013 at 05:21:48PM +0400, Pavel Emelyanov wrote:
> On 10/04/2013 11:02 PM, Naoya Horiguchi wrote:
> > This flag shows that soft dirty bit is not enabled yet.
> > You can enable it by "echo 4 > /proc/pid/clear_refs."
> 
> The comment is not correct. Per-VMA soft-dirty flag means, that
> VMA is "newly created" one and thus represents a new (dirty) are
> in task's VM.

Thanks for the correction. I changed the description.

Naoya
---
From: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Date: Fri, 4 Oct 2013 13:42:13 -0400
Subject: [PATCH] smaps: show VM_SOFTDIRTY flag in VmFlags line

This flag shows that the VMA is "newly created" and thus represents
"dirty" in the task's VM.
You can clear it by "echo 4 > /proc/pid/clear_refs."

Signed-off-by: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
---
 fs/proc/task_mmu.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
index 7366e9d..c591928 100644
--- a/fs/proc/task_mmu.c
+++ b/fs/proc/task_mmu.c
@@ -561,6 +561,9 @@ static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma)
 		[ilog2(VM_NONLINEAR)]	= "nl",
 		[ilog2(VM_ARCH_1)]	= "ar",
 		[ilog2(VM_DONTDUMP)]	= "dd",
+#ifdef CONFIG_MEM_SOFT_DIRTY
+		[ilog2(VM_SOFTDIRTY)]	= "sd",
+#endif
 		[ilog2(VM_MIXEDMAP)]	= "mm",
 		[ilog2(VM_HUGEPAGE)]	= "hg",
 		[ilog2(VM_NOHUGEPAGE)]	= "nh",
-- 
1.8.3.1

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [PATCH 1/2 v2] smaps: show VM_SOFTDIRTY flag in VmFlags line

[Cyrill Gorcunov]

On Mon, Oct 07, 2013 at 10:15:04AM -0400, Naoya Horiguchi wrote:
> > 
> > The comment is not correct. Per-VMA soft-dirty flag means, that
> > VMA is "newly created" one and thus represents a new (dirty) are
> > in task's VM.
> 
> Thanks for the correction. I changed the description.

Looks good to me, thanks!

Acked-by: Cyrill Gorcunov <gorcunov@openvz.org>

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [PATCH 2/2] page-types.c: support KPF_SOFTDIRTY bit

[Andrew Morton]

On Fri,  4 Oct 2013 15:02:15 -0400 Naoya Horiguchi <n-horiguchi@ah.jp.nec.com> wrote:

> Soft dirty bit allows us to track which pages are written since the
> last clear_ref (by "echo 4 > /proc/pid/clear_refs".) This is useful
> for userspace applications to know their memory footprints.
> 
> Note that the kernel exposes this flag via bit[55] of /proc/pid/pagemap,
> and the semantics is not a default one (scheduled to be the default in
> the near future.) However, it shifts to the new semantics at the first
> clear_ref, and the users of soft dirty bit always do it before utilizing
> the bit, so that's not a big deal. Users must avoid relying on the bit
> in page-types before the first clear_ref.

Is Documentation/filesystems/proc.txt (around line 450) fully up to
date here?

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [PATCH 1/2 v2] smaps: show VM_SOFTDIRTY flag in VmFlags line

[Andrew Morton]

On Mon, 07 Oct 2013 10:15:04 -0400 Naoya Horiguchi <n-horiguchi@ah.jp.nec.com> wrote:

> From: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
> Date: Fri, 4 Oct 2013 13:42:13 -0400
> Subject: [PATCH] smaps: show VM_SOFTDIRTY flag in VmFlags line
> 
> This flag shows that the VMA is "newly created" and thus represents
> "dirty" in the task's VM.
> You can clear it by "echo 4 > /proc/pid/clear_refs."
> 
> Signed-off-by: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
> ---
>  fs/proc/task_mmu.c | 3 +++
>  1 file changed, 3 insertions(+)
> 
> diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
> index 7366e9d..c591928 100644
> --- a/fs/proc/task_mmu.c
> +++ b/fs/proc/task_mmu.c
> @@ -561,6 +561,9 @@ static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma)
>  		[ilog2(VM_NONLINEAR)]	= "nl",
>  		[ilog2(VM_ARCH_1)]	= "ar",
>  		[ilog2(VM_DONTDUMP)]	= "dd",
> +#ifdef CONFIG_MEM_SOFT_DIRTY
> +		[ilog2(VM_SOFTDIRTY)]	= "sd",
> +#endif
>  		[ilog2(VM_MIXEDMAP)]	= "mm",
>  		[ilog2(VM_HUGEPAGE)]	= "hg",
>  		[ilog2(VM_NOHUGEPAGE)]	= "nh",

Documentation/filesystems/proc.txt needs updating, please.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [PATCH 1/2 v2] smaps: show VM_SOFTDIRTY flag in VmFlags line

[Naoya Horiguchi]

On Mon, Oct 07, 2013 at 05:51:25PM -0700, Andrew Morton wrote:
> On Mon, 07 Oct 2013 10:15:04 -0400 Naoya Horiguchi <n-horiguchi@ah.jp.nec.com> wrote:
> 
> > From: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
> > Date: Fri, 4 Oct 2013 13:42:13 -0400
> > Subject: [PATCH] smaps: show VM_SOFTDIRTY flag in VmFlags line
> > 
> > This flag shows that the VMA is "newly created" and thus represents
> > "dirty" in the task's VM.
> > You can clear it by "echo 4 > /proc/pid/clear_refs."
> > 
> > Signed-off-by: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
> > ---
> >  fs/proc/task_mmu.c | 3 +++
> >  1 file changed, 3 insertions(+)
> > 
> > diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
> > index 7366e9d..c591928 100644
> > --- a/fs/proc/task_mmu.c
> > +++ b/fs/proc/task_mmu.c
> > @@ -561,6 +561,9 @@ static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma)
> >  		[ilog2(VM_NONLINEAR)]	= "nl",
> >  		[ilog2(VM_ARCH_1)]	= "ar",
> >  		[ilog2(VM_DONTDUMP)]	= "dd",
> > +#ifdef CONFIG_MEM_SOFT_DIRTY
> > +		[ilog2(VM_SOFTDIRTY)]	= "sd",
> > +#endif
> >  		[ilog2(VM_MIXEDMAP)]	= "mm",
> >  		[ilog2(VM_HUGEPAGE)]	= "hg",
> >  		[ilog2(VM_NOHUGEPAGE)]	= "nh",
> 
> Documentation/filesystems/proc.txt needs updating, please.

OK. Here's the revised one.
---
From: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Date: Mon, 7 Oct 2013 22:52:00 -0400
Subject: [PATCH] smaps: show VM_SOFTDIRTY flag in VmFlags line

This flag shows that the VMA is "newly created" and thus represents
"dirty" in the task's VM.
You can clear it by "echo 4 > /proc/pid/clear_refs."

Signed-off-by: Naoya Horiguchi <n-horiguchi@ah.jp.nec.com>
Acked-by: Cyrill Gorcunov <gorcunov@openvz.org>
---
 Documentation/filesystems/proc.txt | 1 +
 fs/proc/task_mmu.c                 | 3 +++
 2 files changed, 4 insertions(+)

diff --git a/Documentation/filesystems/proc.txt b/Documentation/filesystems/proc.txt
index 823c95f..22d89aa3 100644
--- a/Documentation/filesystems/proc.txt
+++ b/Documentation/filesystems/proc.txt
@@ -460,6 +460,7 @@ flags associated with the particular virtual memory area in two letter encoded
     nl  - non-linear mapping
     ar  - architecture specific flag
     dd  - do not include area into core dump
+    sd  - soft-dirty flag
     mm  - mixed map area
     hg  - huge page advise flag
     nh  - no-huge page advise flag
diff --git a/fs/proc/task_mmu.c b/fs/proc/task_mmu.c
index 7366e9d..c591928 100644
--- a/fs/proc/task_mmu.c
+++ b/fs/proc/task_mmu.c
@@ -561,6 +561,9 @@ static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma)
 		[ilog2(VM_NONLINEAR)]	= "nl",
 		[ilog2(VM_ARCH_1)]	= "ar",
 		[ilog2(VM_DONTDUMP)]	= "dd",
+#ifdef CONFIG_MEM_SOFT_DIRTY
+		[ilog2(VM_SOFTDIRTY)]	= "sd",
+#endif
 		[ilog2(VM_MIXEDMAP)]	= "mm",
 		[ilog2(VM_HUGEPAGE)]	= "hg",
 		[ilog2(VM_NOHUGEPAGE)]	= "nh",
-- 
1.8.3.1





--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [PATCH 1/2 v2] smaps: show VM_SOFTDIRTY flag in VmFlags line

[Cyrill Gorcunov]

On Mon, Oct 07, 2013 at 05:51:25PM -0700, Andrew Morton wrote:
> 
> Documentation/filesystems/proc.txt needs updating, please.

I'll do this today.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>

Re: [PATCH 1/2 v2] smaps: show VM_SOFTDIRTY flag in VmFlags line

[Cyrill Gorcunov]

On Tue, Oct 08, 2013 at 12:59:01PM +0400, Cyrill Gorcunov wrote:
> On Mon, Oct 07, 2013 at 05:51:25PM -0700, Andrew Morton wrote:
> > 
> > Documentation/filesystems/proc.txt needs updating, please.
> 
> I'll do this today.

Ouch, it seems I've missed the reply and Naoya already has it done.

--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@kvack.org.  For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>