From: Izik Eidus <ieidus@redhat.com> To: unlisted-recipients:; (no To-header on input) Cc: linux-kernel@vger.kernel.org, kvm@vger.kernel.org, linux-mm@kvack.org, avi@redhat.com, aarcange@redhat.com, chrisw@redhat.com, riel@redhat.com, jeremy@goop.org, mtosatti@redhat.com, hugh@veritas.com, corbet@lwn.net, yaniv@redhat.com, dmonakhov@openvz.org, Izik Eidus <ieidus@redhat.com> Subject: [PATCH 1/4] MMU_NOTIFIERS: add set_pte_at_notify() Date: Tue, 31 Mar 2009 02:59:17 +0300 [thread overview] Message-ID: <1238457560-7613-2-git-send-email-ieidus@redhat.com> (raw) In-Reply-To: <1238457560-7613-1-git-send-email-ieidus@redhat.com> this macro allow setting the pte in the shadow page tables directly instead of flushing the shadow page table entry and then get vmexit in order to set it. This function is optimzation for kvm/users of mmu_notifiers for COW pages, it is useful for kvm when ksm is used beacuse it allow kvm not to have to recive VMEXIT and only then map the shared page into the mmu shadow pages, but instead map it directly at the same time linux map the page into the host page table. this mmu notifer macro is working by calling to callback that will map directly the physical page into the shadow page tables. (users of mmu_notifiers that didnt implement the set_pte_at_notify() call back will just recive the mmu_notifier_invalidate_page callback) Signed-off-by: Izik Eidus <ieidus@redhat.com> --- include/linux/mmu_notifier.h | 34 ++++++++++++++++++++++++++++++++++ mm/memory.c | 10 ++++++++-- mm/mmu_notifier.c | 20 ++++++++++++++++++++ 3 files changed, 62 insertions(+), 2 deletions(-) diff --git a/include/linux/mmu_notifier.h b/include/linux/mmu_notifier.h index b77486d..8bb245f 100644 --- a/include/linux/mmu_notifier.h +++ b/include/linux/mmu_notifier.h @@ -61,6 +61,15 @@ struct mmu_notifier_ops { struct mm_struct *mm, unsigned long address); + /* + * change_pte is called in cases that pte mapping into page is changed + * for example when ksm mapped pte to point into a new shared page. + */ + void (*change_pte)(struct mmu_notifier *mn, + struct mm_struct *mm, + unsigned long address, + pte_t pte); + /* * Before this is invoked any secondary MMU is still ok to * read/write to the page previously pointed to by the Linux @@ -154,6 +163,8 @@ extern void __mmu_notifier_mm_destroy(struct mm_struct *mm); extern void __mmu_notifier_release(struct mm_struct *mm); extern int __mmu_notifier_clear_flush_young(struct mm_struct *mm, unsigned long address); +extern void __mmu_notifier_change_pte(struct mm_struct *mm, + unsigned long address, pte_t pte); extern void __mmu_notifier_invalidate_page(struct mm_struct *mm, unsigned long address); extern void __mmu_notifier_invalidate_range_start(struct mm_struct *mm, @@ -175,6 +186,13 @@ static inline int mmu_notifier_clear_flush_young(struct mm_struct *mm, return 0; } +static inline void mmu_notifier_change_pte(struct mm_struct *mm, + unsigned long address, pte_t pte) +{ + if (mm_has_notifiers(mm)) + __mmu_notifier_change_pte(mm, address, pte); +} + static inline void mmu_notifier_invalidate_page(struct mm_struct *mm, unsigned long address) { @@ -236,6 +254,16 @@ static inline void mmu_notifier_mm_destroy(struct mm_struct *mm) __young; \ }) +#define set_pte_at_notify(__mm, __address, __ptep, __pte) \ +({ \ + struct mm_struct *___mm = __mm; \ + unsigned long ___address = __address; \ + pte_t ___pte = __pte; \ + \ + set_pte_at(__mm, __address, __ptep, ___pte); \ + mmu_notifier_change_pte(___mm, ___address, ___pte); \ +}) + #else /* CONFIG_MMU_NOTIFIER */ static inline void mmu_notifier_release(struct mm_struct *mm) @@ -248,6 +276,11 @@ static inline int mmu_notifier_clear_flush_young(struct mm_struct *mm, return 0; } +static inline void mmu_notifier_change_pte(struct mm_struct *mm, + unsigned long address, pte_t pte) +{ +} + static inline void mmu_notifier_invalidate_page(struct mm_struct *mm, unsigned long address) { @@ -273,6 +306,7 @@ static inline void mmu_notifier_mm_destroy(struct mm_struct *mm) #define ptep_clear_flush_young_notify ptep_clear_flush_young #define ptep_clear_flush_notify ptep_clear_flush +#define set_pte_at_notify set_pte_at #endif /* CONFIG_MMU_NOTIFIER */ diff --git a/mm/memory.c b/mm/memory.c index baa999e..0382a34 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -2031,9 +2031,15 @@ gotten: * seen in the presence of one thread doing SMC and another * thread doing COW. */ - ptep_clear_flush_notify(vma, address, page_table); + ptep_clear_flush(vma, address, page_table); page_add_new_anon_rmap(new_page, vma, address); - set_pte_at(mm, address, page_table, entry); + /* + * We call here the notify macro beacuse in cases of using + * secondary mmu page table like kvm shadow page, tables we want + * the new page to be mapped directly into the secondary page + * table + */ + set_pte_at_notify(mm, address, page_table, entry); update_mmu_cache(vma, address, entry); if (old_page) { /* diff --git a/mm/mmu_notifier.c b/mm/mmu_notifier.c index 5f4ef02..c3e8779 100644 --- a/mm/mmu_notifier.c +++ b/mm/mmu_notifier.c @@ -99,6 +99,26 @@ int __mmu_notifier_clear_flush_young(struct mm_struct *mm, return young; } +void __mmu_notifier_change_pte(struct mm_struct *mm, unsigned long address, + pte_t pte) +{ + struct mmu_notifier *mn; + struct hlist_node *n; + + rcu_read_lock(); + hlist_for_each_entry_rcu(mn, n, &mm->mmu_notifier_mm->list, hlist) { + if (mn->ops->change_pte) + mn->ops->change_pte(mn, mm, address, pte); + /* + * some drivers dont have change_pte and therefor we must call + * for invalidate_page in that case + */ + else if (mn->ops->invalidate_page) + mn->ops->invalidate_page(mn, mm, address); + } + rcu_read_unlock(); +} + void __mmu_notifier_invalidate_page(struct mm_struct *mm, unsigned long address) { -- 1.5.6.5
WARNING: multiple messages have this Message-ID (diff)
From: Izik Eidus <ieidus@redhat.com> Cc: linux-kernel@vger.kernel.org, kvm@vger.kernel.org, linux-mm@kvack.org, avi@redhat.com, aarcange@redhat.com, chrisw@redhat.com, riel@redhat.com, jeremy@goop.org, mtosatti@redhat.com, hugh@veritas.com, corbet@lwn.net, yaniv@redhat.com, dmonakhov@openvz.org, Izik Eidus <ieidus@redhat.com> Subject: [PATCH 1/4] MMU_NOTIFIERS: add set_pte_at_notify() Date: Tue, 31 Mar 2009 02:59:17 +0300 [thread overview] Message-ID: <1238457560-7613-2-git-send-email-ieidus@redhat.com> (raw) In-Reply-To: <1238457560-7613-1-git-send-email-ieidus@redhat.com> this macro allow setting the pte in the shadow page tables directly instead of flushing the shadow page table entry and then get vmexit in order to set it. This function is optimzation for kvm/users of mmu_notifiers for COW pages, it is useful for kvm when ksm is used beacuse it allow kvm not to have to recive VMEXIT and only then map the shared page into the mmu shadow pages, but instead map it directly at the same time linux map the page into the host page table. this mmu notifer macro is working by calling to callback that will map directly the physical page into the shadow page tables. (users of mmu_notifiers that didnt implement the set_pte_at_notify() call back will just recive the mmu_notifier_invalidate_page callback) Signed-off-by: Izik Eidus <ieidus@redhat.com> --- include/linux/mmu_notifier.h | 34 ++++++++++++++++++++++++++++++++++ mm/memory.c | 10 ++++++++-- mm/mmu_notifier.c | 20 ++++++++++++++++++++ 3 files changed, 62 insertions(+), 2 deletions(-) diff --git a/include/linux/mmu_notifier.h b/include/linux/mmu_notifier.h index b77486d..8bb245f 100644 --- a/include/linux/mmu_notifier.h +++ b/include/linux/mmu_notifier.h @@ -61,6 +61,15 @@ struct mmu_notifier_ops { struct mm_struct *mm, unsigned long address); + /* + * change_pte is called in cases that pte mapping into page is changed + * for example when ksm mapped pte to point into a new shared page. + */ + void (*change_pte)(struct mmu_notifier *mn, + struct mm_struct *mm, + unsigned long address, + pte_t pte); + /* * Before this is invoked any secondary MMU is still ok to * read/write to the page previously pointed to by the Linux @@ -154,6 +163,8 @@ extern void __mmu_notifier_mm_destroy(struct mm_struct *mm); extern void __mmu_notifier_release(struct mm_struct *mm); extern int __mmu_notifier_clear_flush_young(struct mm_struct *mm, unsigned long address); +extern void __mmu_notifier_change_pte(struct mm_struct *mm, + unsigned long address, pte_t pte); extern void __mmu_notifier_invalidate_page(struct mm_struct *mm, unsigned long address); extern void __mmu_notifier_invalidate_range_start(struct mm_struct *mm, @@ -175,6 +186,13 @@ static inline int mmu_notifier_clear_flush_young(struct mm_struct *mm, return 0; } +static inline void mmu_notifier_change_pte(struct mm_struct *mm, + unsigned long address, pte_t pte) +{ + if (mm_has_notifiers(mm)) + __mmu_notifier_change_pte(mm, address, pte); +} + static inline void mmu_notifier_invalidate_page(struct mm_struct *mm, unsigned long address) { @@ -236,6 +254,16 @@ static inline void mmu_notifier_mm_destroy(struct mm_struct *mm) __young; \ }) +#define set_pte_at_notify(__mm, __address, __ptep, __pte) \ +({ \ + struct mm_struct *___mm = __mm; \ + unsigned long ___address = __address; \ + pte_t ___pte = __pte; \ + \ + set_pte_at(__mm, __address, __ptep, ___pte); \ + mmu_notifier_change_pte(___mm, ___address, ___pte); \ +}) + #else /* CONFIG_MMU_NOTIFIER */ static inline void mmu_notifier_release(struct mm_struct *mm) @@ -248,6 +276,11 @@ static inline int mmu_notifier_clear_flush_young(struct mm_struct *mm, return 0; } +static inline void mmu_notifier_change_pte(struct mm_struct *mm, + unsigned long address, pte_t pte) +{ +} + static inline void mmu_notifier_invalidate_page(struct mm_struct *mm, unsigned long address) { @@ -273,6 +306,7 @@ static inline void mmu_notifier_mm_destroy(struct mm_struct *mm) #define ptep_clear_flush_young_notify ptep_clear_flush_young #define ptep_clear_flush_notify ptep_clear_flush +#define set_pte_at_notify set_pte_at #endif /* CONFIG_MMU_NOTIFIER */ diff --git a/mm/memory.c b/mm/memory.c index baa999e..0382a34 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -2031,9 +2031,15 @@ gotten: * seen in the presence of one thread doing SMC and another * thread doing COW. */ - ptep_clear_flush_notify(vma, address, page_table); + ptep_clear_flush(vma, address, page_table); page_add_new_anon_rmap(new_page, vma, address); - set_pte_at(mm, address, page_table, entry); + /* + * We call here the notify macro beacuse in cases of using + * secondary mmu page table like kvm shadow page, tables we want + * the new page to be mapped directly into the secondary page + * table + */ + set_pte_at_notify(mm, address, page_table, entry); update_mmu_cache(vma, address, entry); if (old_page) { /* diff --git a/mm/mmu_notifier.c b/mm/mmu_notifier.c index 5f4ef02..c3e8779 100644 --- a/mm/mmu_notifier.c +++ b/mm/mmu_notifier.c @@ -99,6 +99,26 @@ int __mmu_notifier_clear_flush_young(struct mm_struct *mm, return young; } +void __mmu_notifier_change_pte(struct mm_struct *mm, unsigned long address, + pte_t pte) +{ + struct mmu_notifier *mn; + struct hlist_node *n; + + rcu_read_lock(); + hlist_for_each_entry_rcu(mn, n, &mm->mmu_notifier_mm->list, hlist) { + if (mn->ops->change_pte) + mn->ops->change_pte(mn, mm, address, pte); + /* + * some drivers dont have change_pte and therefor we must call + * for invalidate_page in that case + */ + else if (mn->ops->invalidate_page) + mn->ops->invalidate_page(mn, mm, address); + } + rcu_read_unlock(); +} + void __mmu_notifier_invalidate_page(struct mm_struct *mm, unsigned long address) { -- 1.5.6.5 -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to majordomo@kvack.org. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"dont@kvack.org"> email@kvack.org </a>
next prev parent reply other threads:[~2009-03-31 0:01 UTC|newest] Thread overview: 111+ messages / expand[flat|nested] mbox.gz Atom feed top 2009-03-30 23:59 [PATCH 0/4] ksm - dynamic page sharing driver for linux Izik Eidus 2009-03-30 23:59 ` Izik Eidus 2009-03-30 23:59 ` Izik Eidus [this message] 2009-03-30 23:59 ` [PATCH 1/4] MMU_NOTIFIERS: add set_pte_at_notify() Izik Eidus 2009-03-30 23:59 ` [PATCH 2/4] add page_wrprotect(): write protecting page Izik Eidus 2009-03-30 23:59 ` Izik Eidus 2009-03-30 23:59 ` [PATCH 3/4] add replace_page(): change the page pte is pointing to Izik Eidus 2009-03-30 23:59 ` Izik Eidus 2009-03-30 23:59 ` [PATCH 4/4] add ksm kernel shared memory driver Izik Eidus 2009-03-30 23:59 ` Izik Eidus 2009-03-31 2:12 ` Anthony Liguori 2009-03-31 2:12 ` Anthony Liguori 2009-03-31 12:24 ` Izik Eidus 2009-03-31 12:24 ` Izik Eidus 2009-03-31 13:31 ` Anthony Liguori 2009-03-31 13:31 ` Anthony Liguori 2009-03-31 14:25 ` Andrea Arcangeli 2009-03-31 14:25 ` Andrea Arcangeli 2009-03-31 14:37 ` Anthony Liguori 2009-03-31 14:37 ` Anthony Liguori 2009-03-31 15:02 ` Andrea Arcangeli 2009-03-31 15:02 ` Andrea Arcangeli 2009-03-31 15:09 ` Anthony Liguori 2009-03-31 15:09 ` Anthony Liguori 2009-03-31 15:18 ` Andrea Arcangeli 2009-03-31 15:18 ` Andrea Arcangeli 2009-03-31 15:54 ` Anthony Liguori 2009-03-31 15:54 ` Anthony Liguori 2009-03-31 16:25 ` Andrea Arcangeli 2009-03-31 16:25 ` Andrea Arcangeli 2009-03-31 16:51 ` Anthony Liguori 2009-03-31 16:51 ` Anthony Liguori 2009-03-31 17:11 ` Andrea Arcangeli 2009-03-31 17:11 ` Andrea Arcangeli 2009-04-01 22:54 ` Izik Eidus 2009-04-01 22:54 ` Izik Eidus 2009-04-02 0:31 ` Anthony Liguori 2009-04-02 0:31 ` Anthony Liguori 2009-04-02 0:48 ` Chris Wright 2009-04-02 0:48 ` Chris Wright 2009-04-02 1:22 ` Chris Wright 2009-04-02 1:22 ` Chris Wright 2009-04-02 2:36 ` Anthony Liguori 2009-04-02 2:36 ` Anthony Liguori 2009-04-02 5:31 ` [PATCH 5/4] update ksm userspace interfaces Chris Wright 2009-04-02 5:31 ` Chris Wright 2009-04-02 13:32 ` Izik Eidus 2009-04-02 13:32 ` Izik Eidus 2009-04-02 15:20 ` Chris Wright 2009-04-02 15:20 ` Chris Wright 2009-04-02 15:56 ` Chris Wright 2009-04-02 15:56 ` Chris Wright 2009-04-02 15:55 ` Izik Eidus 2009-04-02 15:55 ` Izik Eidus 2009-04-03 10:16 ` Gerd Hoffmann 2009-04-03 10:16 ` Gerd Hoffmann 2009-04-03 10:49 ` Izik Eidus 2009-04-03 10:49 ` Izik Eidus 2009-04-03 11:08 ` Gerd Hoffmann 2009-04-03 11:08 ` Gerd Hoffmann 2009-04-03 16:22 ` Chris Wright 2009-04-03 16:22 ` Chris Wright 2009-04-02 14:41 ` Andrea Arcangeli 2009-04-02 14:41 ` Andrea Arcangeli 2009-04-02 15:12 ` Chris Wright 2009-04-02 15:12 ` Chris Wright 2009-04-02 15:25 ` Andrea Arcangeli 2009-04-02 15:25 ` Andrea Arcangeli 2009-04-02 5:48 ` [PATCH 4/4 alternative userspace] add ksm kernel shared memory driver Chris Wright 2009-04-02 5:48 ` Chris Wright 2009-04-02 5:57 ` Bert Wesarg 2009-04-02 5:57 ` Bert Wesarg 2009-04-02 5:57 ` Bert Wesarg 2009-04-02 5:59 ` Chris Wright 2009-04-02 5:59 ` Chris Wright 2009-04-02 6:00 ` Bert Wesarg 2009-04-02 6:00 ` Bert Wesarg 2009-04-02 7:09 ` Avi Kivity 2009-04-02 7:09 ` Avi Kivity 2009-04-02 7:24 ` [PATCH 4/4] " Avi Kivity 2009-04-02 7:24 ` Avi Kivity 2009-04-02 9:38 ` Andrea Arcangeli 2009-04-02 9:38 ` Andrea Arcangeli 2009-04-02 11:23 ` Izik Eidus 2009-04-02 11:23 ` Izik Eidus 2009-03-31 2:15 ` KAMEZAWA Hiroyuki 2009-03-31 2:15 ` KAMEZAWA Hiroyuki 2009-03-31 12:21 ` Izik Eidus 2009-03-31 12:21 ` Izik Eidus 2009-03-31 23:57 ` KAMEZAWA Hiroyuki 2009-03-31 23:57 ` KAMEZAWA Hiroyuki 2009-04-01 17:28 ` Izik Eidus 2009-04-01 17:28 ` Izik Eidus 2009-03-31 20:52 ` Andrea Arcangeli 2009-03-31 20:52 ` Andrea Arcangeli 2009-03-31 1:42 ` [PATCH 0/4] ksm - dynamic page sharing driver for linux Anthony Liguori 2009-03-31 1:42 ` Anthony Liguori 2009-03-31 12:33 ` Izik Eidus 2009-03-31 12:33 ` Izik Eidus 2009-04-02 19:22 ` Jesper Juhl 2009-04-02 19:22 ` Jesper Juhl 2009-04-02 19:38 ` Izik Eidus 2009-04-02 19:38 ` Izik Eidus 2009-04-02 19:39 ` Chris Wright 2009-04-02 19:39 ` Chris Wright 2009-04-02 19:49 ` Jesper Juhl 2009-04-02 19:49 ` Jesper Juhl 2009-04-04 14:35 [PATCH 0/4] ksm - dynamic page sharing driver for linux v2 Izik Eidus 2009-04-04 14:35 ` [PATCH 1/4] MMU_NOTIFIERS: add set_pte_at_notify() Izik Eidus 2009-04-04 14:35 ` Izik Eidus 2009-04-09 3:58 [PATCH 0/4] ksm - dynamic page sharing driver for linux v3 Izik Eidus 2009-04-09 3:58 ` [PATCH 1/4] MMU_NOTIFIERS: add set_pte_at_notify() Izik Eidus 2009-04-09 3:58 ` Izik Eidus
Reply instructions: You may reply publicly to this message via plain-text email using any one of the following methods: * Save the following mbox file, import it into your mail client, and reply-to-all from there: mbox Avoid top-posting and favor interleaved quoting: https://en.wikipedia.org/wiki/Posting_style#Interleaved_style * Reply using the --to, --cc, and --in-reply-to switches of git-send-email(1): git send-email \ --in-reply-to=1238457560-7613-2-git-send-email-ieidus@redhat.com \ --to=ieidus@redhat.com \ --cc=aarcange@redhat.com \ --cc=avi@redhat.com \ --cc=chrisw@redhat.com \ --cc=corbet@lwn.net \ --cc=dmonakhov@openvz.org \ --cc=hugh@veritas.com \ --cc=jeremy@goop.org \ --cc=kvm@vger.kernel.org \ --cc=linux-kernel@vger.kernel.org \ --cc=linux-mm@kvack.org \ --cc=mtosatti@redhat.com \ --cc=riel@redhat.com \ --cc=yaniv@redhat.com \ /path/to/YOUR_REPLY https://kernel.org/pub/software/scm/git/docs/git-send-email.html * If your mail client supports setting the In-Reply-To header via mailto: links, try the mailto: linkBe sure your reply has a Subject: header at the top and a blank line before the message body.
This is an external index of several public inboxes, see mirroring instructions on how to clone and mirror all data and code used by this external index.