From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: (majordomo@vger.kernel.org) by vger.kernel.org via listexpand id S1752650AbdLLSfb (ORCPT ); Tue, 12 Dec 2017 13:35:31 -0500 Received: from mail-yb0-f194.google.com ([209.85.213.194]:40972 "EHLO mail-yb0-f194.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP id S1752615AbdLLSfJ (ORCPT ); Tue, 12 Dec 2017 13:35:09 -0500 X-Google-Smtp-Source: ACJfBotqmlrxd3V+YbblbSLz/pZA/d7ePSNZrzeaBwngFnxd1Lhpii+FIqi6Tp/qg36eK4YGBf5TEA== From: Martin Brandenburg X-Google-Original-From: Martin Brandenburg To: hubcap@omnibond.com, devel@lists.orangefs.org, linux-fsdevel@vger.kernel.org, linux-kernel@vger.kernel.org Cc: Martin Brandenburg Subject: [PATCH 18/18] orangefs: implement xattr cache Date: Tue, 12 Dec 2017 13:34:24 -0500 Message-Id: <20171212183424.26406-19-martin@martinbrandenburg.com> X-Mailer: git-send-email 2.15.1 In-Reply-To: <20171212183424.26406-1-martin@martinbrandenburg.com> References: <20171212183424.26406-1-martin@martinbrandenburg.com> Sender: linux-kernel-owner@vger.kernel.org List-ID: X-Mailing-List: linux-kernel@vger.kernel.org This uses the same timeout as the getattr cache. This substantially increases performance when writing files with smaller buffer sizes. When writing, the size is (often) changed, which causes a call to notify_change which calls security_inode_need_killpriv which needs a getxattr. Caching it reduces traffic to the server. Without: $ time (dd if=/dev/zero of=/orangefs/foo bs=256 count=32768; sync) 32768+0 records in 32768+0 records out 8388608 bytes (8.4 MB, 8.0 MiB) copied, 11.0343 s, 760 kB/s real 0m11.788s user 0m0.013s sys 0m0.703s With: $ time (dd if=/dev/zero of=/orangefs/foo bs=256 count=32768; sync) 32768+0 records in 32768+0 records out 8388608 bytes (8.4 MB, 8.0 MiB) copied, 0.0438278 s, 191 MB/s real 0m2.181s user 0m0.002s sys 0m0.048s Signed-off-by: Martin Brandenburg --- fs/orangefs/inode.c | 1 + fs/orangefs/orangefs-kernel.h | 10 +++++++ fs/orangefs/super.c | 9 +++++++ fs/orangefs/xattr.c | 62 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 82 insertions(+) diff --git a/fs/orangefs/inode.c b/fs/orangefs/inode.c index 66c40ac7c4ac..31d655239f09 100644 --- a/fs/orangefs/inode.c +++ b/fs/orangefs/inode.c @@ -164,6 +164,7 @@ static int orangefs_set_inode(struct inode *inode, void *data) struct orangefs_object_kref *ref = (struct orangefs_object_kref *) data; ORANGEFS_I(inode)->refn.fs_id = ref->fs_id; ORANGEFS_I(inode)->refn.khandle = ref->khandle; + hash_init(ORANGEFS_I(inode)->xattr_cache); return 0; } diff --git a/fs/orangefs/orangefs-kernel.h b/fs/orangefs/orangefs-kernel.h index 4cabde0a5e9f..ff9a874440a5 100644 --- a/fs/orangefs/orangefs-kernel.h +++ b/fs/orangefs/orangefs-kernel.h @@ -211,6 +211,8 @@ struct orangefs_inode_s { unsigned long getattr_time; u32 getattr_mask; + + DECLARE_HASHTABLE(xattr_cache, 4); }; /* per superblock private orangefs info */ @@ -268,6 +270,14 @@ struct orangefs_stats { unsigned long writes; }; +struct orangefs_cached_xattr { + struct hlist_node node; + char key[ORANGEFS_MAX_XATTR_NAMELEN]; + char val[ORANGEFS_MAX_XATTR_VALUELEN]; + ssize_t length; + unsigned long timeout; +}; + extern struct orangefs_stats orangefs_stats; /* diff --git a/fs/orangefs/super.c b/fs/orangefs/super.c index 5c1a343ba026..dd1a768e84ff 100644 --- a/fs/orangefs/super.c +++ b/fs/orangefs/super.c @@ -128,6 +128,15 @@ static void orangefs_i_callback(struct rcu_head *head) { struct inode *inode = container_of(head, struct inode, i_rcu); struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode); + struct orangefs_cached_xattr *cx; + struct hlist_node *tmp; + int i; + + hash_for_each_safe(orangefs_inode->xattr_cache, i, tmp, cx, node) { + hlist_del(&cx->node); + kfree(cx); + } + kmem_cache_free(orangefs_inode_cache, orangefs_inode); } diff --git a/fs/orangefs/xattr.c b/fs/orangefs/xattr.c index 03bcb871544d..dac297870d40 100644 --- a/fs/orangefs/xattr.c +++ b/fs/orangefs/xattr.c @@ -50,6 +50,35 @@ static inline int convert_to_internal_xattr_flags(int setxattr_flags) return internal_flag; } +static unsigned int xattr_key(const char *key) +{ + unsigned int i = 0; + while (key) + i += *key++; + return i % 16; +} + +static struct orangefs_cached_xattr *find_cached_xattr(struct inode *inode, + const char *key) +{ + struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode); + struct orangefs_cached_xattr *cx; + struct hlist_head *h; + struct hlist_node *tmp; + h = &orangefs_inode->xattr_cache[xattr_key(key)]; + if (hlist_empty(h)) + return NULL; + hlist_for_each_entry_safe(cx, tmp, h, node) { + if (!time_before(jiffies, cx->timeout)) { + hlist_del(&cx->node); + kfree(cx); + continue; + } + if (!strcmp(cx->key, key)) + return cx; + } + return NULL; +} /* * Tries to get a specified key's attributes of a given @@ -65,6 +94,7 @@ ssize_t orangefs_inode_getxattr(struct inode *inode, const char *name, { struct orangefs_inode_s *orangefs_inode = ORANGEFS_I(inode); struct orangefs_kernel_op_s *new_op = NULL; + struct orangefs_cached_xattr *cx; ssize_t ret = -ENOMEM; ssize_t length = 0; int fsuid; @@ -93,6 +123,19 @@ ssize_t orangefs_inode_getxattr(struct inode *inode, const char *name, down_read(&orangefs_inode->xattr_sem); + cx = find_cached_xattr(inode, name); + if (cx) { + if (cx->length == -1) { + ret = -ENODATA; + goto out_unlock; + } else { + memcpy(buffer, cx->val, cx->length); + memset(buffer + cx->length, 0, size - cx->length); + ret = cx->length; + goto out_unlock; + } + } + new_op = op_alloc(ORANGEFS_VFS_OP_GETXATTR); if (!new_op) goto out_unlock; @@ -117,6 +160,15 @@ ssize_t orangefs_inode_getxattr(struct inode *inode, const char *name, " does not exist!\n", get_khandle_from_ino(inode), (char *)new_op->upcall.req.getxattr.key); + cx = kmalloc(sizeof *cx, GFP_KERNEL); + if (cx) { + strcpy(cx->key, name); + cx->length = -1; + cx->timeout = jiffies + + orangefs_getattr_timeout_msecs*HZ/1000; + hash_add(orangefs_inode->xattr_cache, &cx->node, + xattr_key(cx->key)); + } } goto out_release_op; } @@ -156,6 +208,16 @@ ssize_t orangefs_inode_getxattr(struct inode *inode, const char *name, ret = length; + cx = kmalloc(sizeof *cx, GFP_KERNEL); + if (cx) { + strcpy(cx->key, name); + memcpy(cx->val, name, length); + cx->length = length; + cx->timeout = jiffies + HZ; + hash_add(orangefs_inode->xattr_cache, &cx->node, + xattr_key(cx->key)); + } + out_release_op: op_release(new_op); out_unlock: -- 2.15.1