All of lore.kernel.org
 help / color / mirror / Atom feed
* High load in 2.6.27, NFS / rpcauth_lookup_credcache()?
@ 2008-10-22  9:12 ` Max Kellermann
  0 siblings, 0 replies; 29+ messages in thread
From: Max Kellermann @ 2008-10-22  9:12 UTC (permalink / raw)
  To: linux-kernel, linux-nfs

[-- Attachment #1: Type: text/plain, Size: 984 bytes --]

Hi,

after I was able to fix http://lkml.org/lkml/2008/10/17/147, the
server which was already upgraded to 2.6.27.2 still gets very high
load.  It is a web server with NFS file storage (NetApp), and while
the others in the cluster (kernel 2.6.25) have a load of 1-3, 2.6.27.2
gets 30-50.

I did an oprofile, with the following results (server just started,
load "only" 5-10):

87593    56.1116  (no location information)   vmlinux
vmlinux                  rpcauth_lookup_credcache
16037    10.2732  auth_generic.c:0            vmlinux
vmlinux                  generic_match
6460      4.1382  (no location information)   php4
php4                     (no symbols)
2478      1.5874  (no location information)   libc-2.7.so
libc-2.7.so              (no symbols)
[...]

We havn't configured any special authentication method.  It is a NFSv3
over UDP mount, but the kernel has NFSv4 and therefore KRB5 enabled.

Any ideas why rpcauth_lookup_credcache() goes overboard with CPU
usage?

Max

[-- Attachment #2: .config --]
[-- Type: text/plain, Size: 39058 bytes --]

#
# Automatically generated make config: don't edit
# Linux kernel version: 2.6.27.2
# Wed Oct 22 11:06:17 2008
#
# CONFIG_64BIT is not set
CONFIG_X86_32=y
# CONFIG_X86_64 is not set
CONFIG_X86=y
CONFIG_ARCH_DEFCONFIG="arch/x86/configs/i386_defconfig"
# CONFIG_GENERIC_LOCKBREAK is not set
CONFIG_GENERIC_TIME=y
CONFIG_GENERIC_CMOS_UPDATE=y
CONFIG_CLOCKSOURCE_WATCHDOG=y
CONFIG_GENERIC_CLOCKEVENTS=y
CONFIG_GENERIC_CLOCKEVENTS_BROADCAST=y
CONFIG_LOCKDEP_SUPPORT=y
CONFIG_STACKTRACE_SUPPORT=y
CONFIG_HAVE_LATENCYTOP_SUPPORT=y
CONFIG_FAST_CMPXCHG_LOCAL=y
CONFIG_MMU=y
CONFIG_ZONE_DMA=y
CONFIG_GENERIC_ISA_DMA=y
CONFIG_GENERIC_IOMAP=y
CONFIG_GENERIC_BUG=y
CONFIG_GENERIC_HWEIGHT=y
# CONFIG_GENERIC_GPIO is not set
CONFIG_ARCH_MAY_HAVE_PC_FDC=y
# CONFIG_RWSEM_GENERIC_SPINLOCK is not set
CONFIG_RWSEM_XCHGADD_ALGORITHM=y
# CONFIG_ARCH_HAS_ILOG2_U32 is not set
# CONFIG_ARCH_HAS_ILOG2_U64 is not set
CONFIG_ARCH_HAS_CPU_IDLE_WAIT=y
CONFIG_GENERIC_CALIBRATE_DELAY=y
# CONFIG_GENERIC_TIME_VSYSCALL is not set
CONFIG_ARCH_HAS_CPU_RELAX=y
CONFIG_ARCH_HAS_CACHE_LINE_SIZE=y
CONFIG_HAVE_SETUP_PER_CPU_AREA=y
# CONFIG_HAVE_CPUMASK_OF_CPU_MAP is not set
CONFIG_ARCH_HIBERNATION_POSSIBLE=y
CONFIG_ARCH_SUSPEND_POSSIBLE=y
# CONFIG_ZONE_DMA32 is not set
CONFIG_ARCH_POPULATES_NODE_MAP=y
# CONFIG_AUDIT_ARCH is not set
CONFIG_ARCH_SUPPORTS_AOUT=y
CONFIG_ARCH_SUPPORTS_OPTIMIZED_INLINING=y
CONFIG_GENERIC_HARDIRQS=y
CONFIG_GENERIC_IRQ_PROBE=y
CONFIG_GENERIC_PENDING_IRQ=y
CONFIG_X86_SMP=y
CONFIG_X86_32_SMP=y
CONFIG_X86_HT=y
CONFIG_X86_BIOS_REBOOT=y
CONFIG_X86_TRAMPOLINE=y
CONFIG_KTIME_SCALAR=y
CONFIG_DEFCONFIG_LIST="/lib/modules/$UNAME_RELEASE/.config"

#
# General setup
#
CONFIG_EXPERIMENTAL=y
CONFIG_LOCK_KERNEL=y
CONFIG_INIT_ENV_ARG_LIMIT=32
CONFIG_LOCALVERSION=""
# CONFIG_LOCALVERSION_AUTO is not set
CONFIG_SWAP=y
CONFIG_SYSVIPC=y
CONFIG_SYSVIPC_SYSCTL=y
CONFIG_POSIX_MQUEUE=y
# CONFIG_BSD_PROCESS_ACCT is not set
# CONFIG_TASKSTATS is not set
# CONFIG_AUDIT is not set
# CONFIG_IKCONFIG is not set
CONFIG_LOG_BUF_SHIFT=17
# CONFIG_CGROUPS is not set
CONFIG_HAVE_UNSTABLE_SCHED_CLOCK=y
# CONFIG_GROUP_SCHED is not set
CONFIG_SYSFS_DEPRECATED=y
CONFIG_SYSFS_DEPRECATED_V2=y
# CONFIG_RELAY is not set
CONFIG_NAMESPACES=y
CONFIG_UTS_NS=y
CONFIG_IPC_NS=y
CONFIG_USER_NS=y
CONFIG_PID_NS=y
# CONFIG_BLK_DEV_INITRD is not set
CONFIG_CC_OPTIMIZE_FOR_SIZE=y
CONFIG_SYSCTL=y
# CONFIG_EMBEDDED is not set
CONFIG_UID16=y
CONFIG_SYSCTL_SYSCALL=y
CONFIG_KALLSYMS=y
CONFIG_KALLSYMS_ALL=y
# CONFIG_KALLSYMS_EXTRA_PASS is not set
CONFIG_HOTPLUG=y
CONFIG_PRINTK=y
CONFIG_BUG=y
CONFIG_ELF_CORE=y
CONFIG_PCSPKR_PLATFORM=y
# CONFIG_COMPAT_BRK is not set
CONFIG_BASE_FULL=y
CONFIG_FUTEX=y
CONFIG_ANON_INODES=y
CONFIG_EPOLL=y
CONFIG_SIGNALFD=y
CONFIG_TIMERFD=y
CONFIG_EVENTFD=y
CONFIG_SHMEM=y
CONFIG_VM_EVENT_COUNTERS=y
CONFIG_SLUB_DEBUG=y
# CONFIG_SLAB is not set
CONFIG_SLUB=y
# CONFIG_SLOB is not set
CONFIG_PROFILING=y
# CONFIG_MARKERS is not set
CONFIG_OPROFILE=y
CONFIG_HAVE_OPROFILE=y
# CONFIG_KPROBES is not set
CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS=y
CONFIG_HAVE_IOREMAP_PROT=y
CONFIG_HAVE_KPROBES=y
CONFIG_HAVE_KRETPROBES=y
# CONFIG_HAVE_ARCH_TRACEHOOK is not set
# CONFIG_HAVE_DMA_ATTRS is not set
CONFIG_USE_GENERIC_SMP_HELPERS=y
# CONFIG_HAVE_CLK is not set
CONFIG_PROC_PAGE_MONITOR=y
CONFIG_HAVE_GENERIC_DMA_COHERENT=y
CONFIG_SLABINFO=y
CONFIG_RT_MUTEXES=y
# CONFIG_TINY_SHMEM is not set
CONFIG_BASE_SMALL=0
CONFIG_MODULES=y
# CONFIG_MODULE_FORCE_LOAD is not set
CONFIG_MODULE_UNLOAD=y
# CONFIG_MODULE_FORCE_UNLOAD is not set
# CONFIG_MODVERSIONS is not set
CONFIG_MODULE_SRCVERSION_ALL=y
CONFIG_KMOD=y
CONFIG_STOP_MACHINE=y
CONFIG_BLOCK=y
CONFIG_LBD=y
# CONFIG_BLK_DEV_IO_TRACE is not set
CONFIG_LSF=y
CONFIG_BLK_DEV_BSG=y
CONFIG_BLK_DEV_INTEGRITY=y

#
# IO Schedulers
#
CONFIG_IOSCHED_NOOP=y
# CONFIG_IOSCHED_AS is not set
CONFIG_IOSCHED_DEADLINE=y
CONFIG_IOSCHED_CFQ=y
# CONFIG_DEFAULT_AS is not set
# CONFIG_DEFAULT_DEADLINE is not set
CONFIG_DEFAULT_CFQ=y
# CONFIG_DEFAULT_NOOP is not set
CONFIG_DEFAULT_IOSCHED="cfq"
CONFIG_CLASSIC_RCU=y

#
# Processor type and features
#
CONFIG_TICK_ONESHOT=y
CONFIG_NO_HZ=y
CONFIG_HIGH_RES_TIMERS=y
CONFIG_GENERIC_CLOCKEVENTS_BUILD=y
CONFIG_SMP=y
CONFIG_X86_FIND_SMP_CONFIG=y
CONFIG_X86_MPPARSE=y
CONFIG_X86_PC=y
# CONFIG_X86_ELAN is not set
# CONFIG_X86_VOYAGER is not set
# CONFIG_X86_GENERICARCH is not set
# CONFIG_X86_VSMP is not set
# CONFIG_X86_VISWS is not set
# CONFIG_X86_RDC321X is not set
CONFIG_SCHED_NO_NO_OMIT_FRAME_POINTER=y
# CONFIG_PARAVIRT_GUEST is not set
# CONFIG_MEMTEST is not set
# CONFIG_M386 is not set
# CONFIG_M486 is not set
# CONFIG_M586 is not set
# CONFIG_M586TSC is not set
# CONFIG_M586MMX is not set
# CONFIG_M686 is not set
# CONFIG_MPENTIUMII is not set
# CONFIG_MPENTIUMIII is not set
# CONFIG_MPENTIUMM is not set
CONFIG_MPENTIUM4=y
# CONFIG_MK6 is not set
# CONFIG_MK7 is not set
# CONFIG_MK8 is not set
# CONFIG_MCRUSOE is not set
# CONFIG_MEFFICEON is not set
# CONFIG_MWINCHIPC6 is not set
# CONFIG_MWINCHIP2 is not set
# CONFIG_MWINCHIP3D is not set
# CONFIG_MGEODEGX1 is not set
# CONFIG_MGEODE_LX is not set
# CONFIG_MCYRIXIII is not set
# CONFIG_MVIAC3_2 is not set
# CONFIG_MVIAC7 is not set
# CONFIG_MPSC is not set
# CONFIG_MCORE2 is not set
# CONFIG_GENERIC_CPU is not set
# CONFIG_X86_GENERIC is not set
CONFIG_X86_CPU=y
CONFIG_X86_CMPXCHG=y
CONFIG_X86_L1_CACHE_SHIFT=7
CONFIG_X86_XADD=y
CONFIG_X86_WP_WORKS_OK=y
CONFIG_X86_INVLPG=y
CONFIG_X86_BSWAP=y
CONFIG_X86_POPAD_OK=y
CONFIG_X86_INTEL_USERCOPY=y
CONFIG_X86_USE_PPRO_CHECKSUM=y
CONFIG_X86_TSC=y
CONFIG_X86_CMPXCHG64=y
CONFIG_X86_CMOV=y
CONFIG_X86_MINIMUM_CPU_FAMILY=4
CONFIG_X86_DEBUGCTLMSR=y
CONFIG_HPET_TIMER=y
CONFIG_HPET_EMULATE_RTC=y
CONFIG_DMI=y
# CONFIG_IOMMU_HELPER is not set
CONFIG_NR_CPUS=8
CONFIG_SCHED_SMT=y
CONFIG_SCHED_MC=y
CONFIG_PREEMPT_NONE=y
# CONFIG_PREEMPT_VOLUNTARY is not set
# CONFIG_PREEMPT is not set
CONFIG_X86_LOCAL_APIC=y
CONFIG_X86_IO_APIC=y
CONFIG_X86_MCE=y
CONFIG_X86_MCE_NONFATAL=y
CONFIG_X86_MCE_P4THERMAL=y
CONFIG_VM86=y
# CONFIG_TOSHIBA is not set
# CONFIG_I8K is not set
# CONFIG_X86_REBOOTFIXUPS is not set
# CONFIG_MICROCODE is not set
# CONFIG_X86_MSR is not set
# CONFIG_X86_CPUID is not set
# CONFIG_NOHIGHMEM is not set
# CONFIG_HIGHMEM4G is not set
CONFIG_HIGHMEM64G=y
CONFIG_PAGE_OFFSET=0xC0000000
CONFIG_HIGHMEM=y
CONFIG_X86_PAE=y
CONFIG_ARCH_FLATMEM_ENABLE=y
CONFIG_ARCH_SPARSEMEM_ENABLE=y
CONFIG_ARCH_SELECT_MEMORY_MODEL=y
CONFIG_SELECT_MEMORY_MODEL=y
CONFIG_FLATMEM_MANUAL=y
# CONFIG_DISCONTIGMEM_MANUAL is not set
# CONFIG_SPARSEMEM_MANUAL is not set
CONFIG_FLATMEM=y
CONFIG_FLAT_NODE_MEM_MAP=y
CONFIG_SPARSEMEM_STATIC=y
# CONFIG_SPARSEMEM_VMEMMAP_ENABLE is not set
CONFIG_PAGEFLAGS_EXTENDED=y
CONFIG_SPLIT_PTLOCK_CPUS=4
CONFIG_RESOURCES_64BIT=y
CONFIG_ZONE_DMA_FLAG=1
CONFIG_BOUNCE=y
CONFIG_VIRT_TO_BUS=y
CONFIG_HIGHPTE=y
# CONFIG_MATH_EMULATION is not set
CONFIG_MTRR=y
# CONFIG_MTRR_SANITIZER is not set
CONFIG_X86_PAT=y
# CONFIG_EFI is not set
CONFIG_IRQBALANCE=y
# CONFIG_SECCOMP is not set
# CONFIG_HZ_100 is not set
CONFIG_HZ_250=y
# CONFIG_HZ_300 is not set
# CONFIG_HZ_1000 is not set
CONFIG_HZ=250
CONFIG_SCHED_HRTICK=y
CONFIG_KEXEC=y
CONFIG_CRASH_DUMP=y
CONFIG_PHYSICAL_START=0x100000
CONFIG_RELOCATABLE=y
CONFIG_PHYSICAL_ALIGN=0x100000
# CONFIG_HOTPLUG_CPU is not set
# CONFIG_COMPAT_VDSO is not set
CONFIG_ARCH_ENABLE_MEMORY_HOTPLUG=y

#
# Power management options
#
CONFIG_PM=y
# CONFIG_PM_DEBUG is not set
# CONFIG_SUSPEND is not set
# CONFIG_HIBERNATION is not set
CONFIG_ACPI=y
# CONFIG_ACPI_PROCFS is not set
# CONFIG_ACPI_PROCFS_POWER is not set
CONFIG_ACPI_SYSFS_POWER=y
CONFIG_ACPI_PROC_EVENT=y
# CONFIG_ACPI_AC is not set
# CONFIG_ACPI_BATTERY is not set
CONFIG_ACPI_BUTTON=y
CONFIG_ACPI_FAN=y
CONFIG_ACPI_DOCK=y
# CONFIG_ACPI_BAY is not set
CONFIG_ACPI_PROCESSOR=y
CONFIG_ACPI_THERMAL=y
CONFIG_ACPI_WMI=y
# CONFIG_ACPI_ASUS is not set
# CONFIG_ACPI_TOSHIBA is not set
# CONFIG_ACPI_CUSTOM_DSDT is not set
CONFIG_ACPI_BLACKLIST_YEAR=0
# CONFIG_ACPI_DEBUG is not set
CONFIG_ACPI_EC=y
CONFIG_ACPI_PCI_SLOT=y
CONFIG_ACPI_POWER=y
CONFIG_ACPI_SYSTEM=y
CONFIG_X86_PM_TIMER=y
CONFIG_ACPI_CONTAINER=y
# CONFIG_ACPI_SBS is not set

#
# CPU Frequency scaling
#
CONFIG_CPU_FREQ=y
CONFIG_CPU_FREQ_TABLE=y
# CONFIG_CPU_FREQ_DEBUG is not set
CONFIG_CPU_FREQ_STAT=y
# CONFIG_CPU_FREQ_STAT_DETAILS is not set
CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE=y
# CONFIG_CPU_FREQ_DEFAULT_GOV_POWERSAVE is not set
# CONFIG_CPU_FREQ_DEFAULT_GOV_USERSPACE is not set
# CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND is not set
# CONFIG_CPU_FREQ_DEFAULT_GOV_CONSERVATIVE is not set
CONFIG_CPU_FREQ_GOV_PERFORMANCE=y
CONFIG_CPU_FREQ_GOV_POWERSAVE=y
CONFIG_CPU_FREQ_GOV_USERSPACE=y
CONFIG_CPU_FREQ_GOV_ONDEMAND=y
# CONFIG_CPU_FREQ_GOV_CONSERVATIVE is not set

#
# CPUFreq processor drivers
#
CONFIG_X86_ACPI_CPUFREQ=y
# CONFIG_X86_POWERNOW_K6 is not set
# CONFIG_X86_POWERNOW_K7 is not set
# CONFIG_X86_POWERNOW_K8 is not set
# CONFIG_X86_GX_SUSPMOD is not set
# CONFIG_X86_SPEEDSTEP_CENTRINO is not set
# CONFIG_X86_SPEEDSTEP_ICH is not set
# CONFIG_X86_SPEEDSTEP_SMI is not set
CONFIG_X86_P4_CLOCKMOD=y
# CONFIG_X86_CPUFREQ_NFORCE2 is not set
# CONFIG_X86_LONGRUN is not set
# CONFIG_X86_LONGHAUL is not set
# CONFIG_X86_E_POWERSAVER is not set

#
# shared options
#
# CONFIG_X86_ACPI_CPUFREQ_PROC_INTF is not set
CONFIG_X86_SPEEDSTEP_LIB=y
CONFIG_CPU_IDLE=y
CONFIG_CPU_IDLE_GOV_LADDER=y
CONFIG_CPU_IDLE_GOV_MENU=y

#
# Bus options (PCI etc.)
#
CONFIG_PCI=y
# CONFIG_PCI_GOBIOS is not set
# CONFIG_PCI_GOMMCONFIG is not set
CONFIG_PCI_GODIRECT=y
# CONFIG_PCI_GOOLPC is not set
# CONFIG_PCI_GOANY is not set
CONFIG_PCI_DIRECT=y
CONFIG_PCI_DOMAINS=y
CONFIG_PCIEPORTBUS=y
CONFIG_PCIEAER=y
# CONFIG_PCIEASPM is not set
CONFIG_ARCH_SUPPORTS_MSI=y
CONFIG_PCI_MSI=y
# CONFIG_PCI_LEGACY is not set
# CONFIG_PCI_DEBUG is not set
# CONFIG_HT_IRQ is not set
CONFIG_ISA_DMA_API=y
# CONFIG_ISA is not set
# CONFIG_MCA is not set
# CONFIG_SCx200 is not set
# CONFIG_OLPC is not set
# CONFIG_PCCARD is not set
# CONFIG_HOTPLUG_PCI is not set

#
# Executable file formats / Emulations
#
CONFIG_BINFMT_ELF=y
# CONFIG_BINFMT_AOUT is not set
# CONFIG_BINFMT_MISC is not set
CONFIG_NET=y

#
# Networking options
#
CONFIG_PACKET=y
CONFIG_PACKET_MMAP=y
CONFIG_UNIX=y
# CONFIG_NET_KEY is not set
CONFIG_INET=y
CONFIG_IP_MULTICAST=y
# CONFIG_IP_ADVANCED_ROUTER is not set
CONFIG_IP_FIB_HASH=y
# CONFIG_IP_PNP is not set
# CONFIG_NET_IPIP is not set
# CONFIG_NET_IPGRE is not set
# CONFIG_IP_MROUTE is not set
# CONFIG_ARPD is not set
CONFIG_SYN_COOKIES=y
# CONFIG_INET_AH is not set
# CONFIG_INET_ESP is not set
# CONFIG_INET_IPCOMP is not set
# CONFIG_INET_XFRM_TUNNEL is not set
# CONFIG_INET_TUNNEL is not set
# CONFIG_INET_XFRM_MODE_TRANSPORT is not set
# CONFIG_INET_XFRM_MODE_TUNNEL is not set
# CONFIG_INET_XFRM_MODE_BEET is not set
CONFIG_INET_LRO=y
CONFIG_INET_DIAG=y
CONFIG_INET_TCP_DIAG=y
CONFIG_TCP_CONG_ADVANCED=y
# CONFIG_TCP_CONG_BIC is not set
# CONFIG_TCP_CONG_CUBIC is not set
# CONFIG_TCP_CONG_WESTWOOD is not set
# CONFIG_TCP_CONG_HTCP is not set
# CONFIG_TCP_CONG_HSTCP is not set
# CONFIG_TCP_CONG_HYBLA is not set
CONFIG_TCP_CONG_VEGAS=y
# CONFIG_TCP_CONG_SCALABLE is not set
# CONFIG_TCP_CONG_LP is not set
# CONFIG_TCP_CONG_VENO is not set
# CONFIG_TCP_CONG_YEAH is not set
# CONFIG_TCP_CONG_ILLINOIS is not set
# CONFIG_DEFAULT_BIC is not set
# CONFIG_DEFAULT_CUBIC is not set
# CONFIG_DEFAULT_HTCP is not set
CONFIG_DEFAULT_VEGAS=y
# CONFIG_DEFAULT_WESTWOOD is not set
# CONFIG_DEFAULT_RENO is not set
CONFIG_DEFAULT_TCP_CONG="vegas"
# CONFIG_TCP_MD5SIG is not set
# CONFIG_IP_VS is not set
# CONFIG_IPV6 is not set
# CONFIG_NETLABEL is not set
# CONFIG_NETWORK_SECMARK is not set
CONFIG_NETFILTER=y
# CONFIG_NETFILTER_DEBUG is not set
CONFIG_NETFILTER_ADVANCED=y

#
# Core Netfilter Configuration
#
CONFIG_NETFILTER_NETLINK=y
# CONFIG_NETFILTER_NETLINK_QUEUE is not set
CONFIG_NETFILTER_NETLINK_LOG=y
CONFIG_NF_CONNTRACK=y
CONFIG_NF_CT_ACCT=y
CONFIG_NF_CONNTRACK_MARK=y
CONFIG_NF_CONNTRACK_EVENTS=y
# CONFIG_NF_CT_PROTO_DCCP is not set
# CONFIG_NF_CT_PROTO_SCTP is not set
# CONFIG_NF_CT_PROTO_UDPLITE is not set
# CONFIG_NF_CONNTRACK_AMANDA is not set
CONFIG_NF_CONNTRACK_FTP=y
# CONFIG_NF_CONNTRACK_H323 is not set
# CONFIG_NF_CONNTRACK_IRC is not set
# CONFIG_NF_CONNTRACK_NETBIOS_NS is not set
# CONFIG_NF_CONNTRACK_PPTP is not set
# CONFIG_NF_CONNTRACK_SANE is not set
# CONFIG_NF_CONNTRACK_SIP is not set
# CONFIG_NF_CONNTRACK_TFTP is not set
CONFIG_NF_CT_NETLINK=y
CONFIG_NETFILTER_XTABLES=y
# CONFIG_NETFILTER_XT_TARGET_CLASSIFY is not set
CONFIG_NETFILTER_XT_TARGET_MARK=y
# CONFIG_NETFILTER_XT_TARGET_NFQUEUE is not set
CONFIG_NETFILTER_XT_TARGET_NFLOG=y
CONFIG_NETFILTER_XT_TARGET_NOTRACK=y
# CONFIG_NETFILTER_XT_TARGET_RATEEST is not set
CONFIG_NETFILTER_XT_TARGET_TRACE=y
# CONFIG_NETFILTER_XT_TARGET_TCPMSS is not set
CONFIG_NETFILTER_XT_MATCH_COMMENT=y
CONFIG_NETFILTER_XT_MATCH_CONNBYTES=y
# CONFIG_NETFILTER_XT_MATCH_CONNLIMIT is not set
CONFIG_NETFILTER_XT_MATCH_CONNMARK=y
CONFIG_NETFILTER_XT_MATCH_CONNTRACK=y
# CONFIG_NETFILTER_XT_MATCH_DCCP is not set
# CONFIG_NETFILTER_XT_MATCH_DSCP is not set
# CONFIG_NETFILTER_XT_MATCH_ESP is not set
CONFIG_NETFILTER_XT_MATCH_HELPER=y
CONFIG_NETFILTER_XT_MATCH_IPRANGE=y
CONFIG_NETFILTER_XT_MATCH_LENGTH=y
CONFIG_NETFILTER_XT_MATCH_LIMIT=y
CONFIG_NETFILTER_XT_MATCH_MAC=y
CONFIG_NETFILTER_XT_MATCH_MARK=y
CONFIG_NETFILTER_XT_MATCH_OWNER=y
CONFIG_NETFILTER_XT_MATCH_MULTIPORT=y
CONFIG_NETFILTER_XT_MATCH_PKTTYPE=y
# CONFIG_NETFILTER_XT_MATCH_QUOTA is not set
# CONFIG_NETFILTER_XT_MATCH_RATEEST is not set
CONFIG_NETFILTER_XT_MATCH_REALM=y
# CONFIG_NETFILTER_XT_MATCH_SCTP is not set
CONFIG_NETFILTER_XT_MATCH_STATE=y
# CONFIG_NETFILTER_XT_MATCH_STATISTIC is not set
CONFIG_NETFILTER_XT_MATCH_STRING=y
# CONFIG_NETFILTER_XT_MATCH_TCPMSS is not set
# CONFIG_NETFILTER_XT_MATCH_TIME is not set
# CONFIG_NETFILTER_XT_MATCH_U32 is not set
CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=y

#
# IP: Netfilter Configuration
#
CONFIG_NF_CONNTRACK_IPV4=y
# CONFIG_NF_CONNTRACK_PROC_COMPAT is not set
# CONFIG_IP_NF_QUEUE is not set
CONFIG_IP_NF_IPTABLES=y
CONFIG_IP_NF_MATCH_RECENT=y
# CONFIG_IP_NF_MATCH_ECN is not set
# CONFIG_IP_NF_MATCH_AH is not set
# CONFIG_IP_NF_MATCH_TTL is not set
CONFIG_IP_NF_MATCH_ADDRTYPE=y
CONFIG_IP_NF_FILTER=y
CONFIG_IP_NF_TARGET_REJECT=y
CONFIG_IP_NF_TARGET_LOG=y
# CONFIG_IP_NF_TARGET_ULOG is not set
CONFIG_NF_NAT=y
CONFIG_NF_NAT_NEEDED=y
# CONFIG_IP_NF_TARGET_MASQUERADE is not set
CONFIG_IP_NF_TARGET_REDIRECT=y
# CONFIG_IP_NF_TARGET_NETMAP is not set
# CONFIG_NF_NAT_SNMP_BASIC is not set
CONFIG_NF_NAT_FTP=y
# CONFIG_NF_NAT_IRC is not set
# CONFIG_NF_NAT_TFTP is not set
# CONFIG_NF_NAT_AMANDA is not set
# CONFIG_NF_NAT_PPTP is not set
# CONFIG_NF_NAT_H323 is not set
# CONFIG_NF_NAT_SIP is not set
# CONFIG_IP_NF_MANGLE is not set
CONFIG_IP_NF_RAW=y
# CONFIG_IP_NF_SECURITY is not set
# CONFIG_IP_NF_ARPTABLES is not set
# CONFIG_IP_DCCP is not set
# CONFIG_IP_SCTP is not set
# CONFIG_TIPC is not set
# CONFIG_ATM is not set
# CONFIG_BRIDGE is not set
# CONFIG_VLAN_8021Q is not set
# CONFIG_DECNET is not set
# CONFIG_LLC2 is not set
# CONFIG_IPX is not set
# CONFIG_ATALK is not set
# CONFIG_X25 is not set
# CONFIG_LAPB is not set
# CONFIG_ECONET is not set
# CONFIG_WAN_ROUTER is not set
# CONFIG_NET_SCHED is not set
CONFIG_NET_CLS_ROUTE=y

#
# Network testing
#
# CONFIG_NET_PKTGEN is not set
# CONFIG_HAMRADIO is not set
# CONFIG_CAN is not set
# CONFIG_IRDA is not set
# CONFIG_BT is not set
# CONFIG_AF_RXRPC is not set

#
# Wireless
#
# CONFIG_CFG80211 is not set
# CONFIG_WIRELESS_EXT is not set
# CONFIG_MAC80211 is not set
# CONFIG_IEEE80211 is not set
# CONFIG_RFKILL is not set
# CONFIG_NET_9P is not set

#
# Device Drivers
#

#
# Generic Driver Options
#
CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
CONFIG_STANDALONE=y
CONFIG_PREVENT_FIRMWARE_BUILD=y
CONFIG_FW_LOADER=y
CONFIG_FIRMWARE_IN_KERNEL=y
CONFIG_EXTRA_FIRMWARE=""
# CONFIG_DEBUG_DRIVER is not set
# CONFIG_DEBUG_DEVRES is not set
# CONFIG_SYS_HYPERVISOR is not set
CONFIG_CONNECTOR=y
CONFIG_PROC_EVENTS=y
# CONFIG_MTD is not set
# CONFIG_PARPORT is not set
CONFIG_PNP=y
# CONFIG_PNP_DEBUG is not set

#
# Protocols
#
CONFIG_PNPACPI=y
CONFIG_BLK_DEV=y
CONFIG_BLK_DEV_FD=y
# CONFIG_BLK_CPQ_DA is not set
CONFIG_BLK_CPQ_CISS_DA=y
# CONFIG_CISS_SCSI_TAPE is not set
# CONFIG_BLK_DEV_DAC960 is not set
# CONFIG_BLK_DEV_UMEM is not set
# CONFIG_BLK_DEV_COW_COMMON is not set
CONFIG_BLK_DEV_LOOP=y
# CONFIG_BLK_DEV_CRYPTOLOOP is not set
# CONFIG_BLK_DEV_NBD is not set
# CONFIG_BLK_DEV_SX8 is not set
# CONFIG_BLK_DEV_RAM is not set
# CONFIG_CDROM_PKTCDVD is not set
# CONFIG_ATA_OVER_ETH is not set
# CONFIG_BLK_DEV_HD is not set
# CONFIG_MISC_DEVICES is not set
CONFIG_HAVE_IDE=y
# CONFIG_IDE is not set

#
# SCSI device support
#
CONFIG_RAID_ATTRS=y
CONFIG_SCSI=y
CONFIG_SCSI_DMA=y
CONFIG_SCSI_TGT=y
CONFIG_SCSI_NETLINK=y
CONFIG_SCSI_PROC_FS=y

#
# SCSI support type (disk, tape, CD-ROM)
#
CONFIG_BLK_DEV_SD=y
# CONFIG_CHR_DEV_ST is not set
# CONFIG_CHR_DEV_OSST is not set
CONFIG_BLK_DEV_SR=y
# CONFIG_BLK_DEV_SR_VENDOR is not set
CONFIG_CHR_DEV_SG=y
# CONFIG_CHR_DEV_SCH is not set

#
# Some SCSI devices (e.g. CD jukebox) support multiple LUNs
#
CONFIG_SCSI_MULTI_LUN=y
CONFIG_SCSI_CONSTANTS=y
# CONFIG_SCSI_LOGGING is not set
CONFIG_SCSI_SCAN_ASYNC=y
CONFIG_SCSI_WAIT_SCAN=m

#
# SCSI Transports
#
CONFIG_SCSI_SPI_ATTRS=y
CONFIG_SCSI_FC_ATTRS=y
CONFIG_SCSI_FC_TGT_ATTRS=y
CONFIG_SCSI_ISCSI_ATTRS=y
CONFIG_SCSI_SAS_ATTRS=y
CONFIG_SCSI_SAS_LIBSAS=y
# CONFIG_SCSI_SAS_ATA is not set
CONFIG_SCSI_SAS_HOST_SMP=y
# CONFIG_SCSI_SAS_LIBSAS_DEBUG is not set
# CONFIG_SCSI_SRP_ATTRS is not set
CONFIG_SCSI_LOWLEVEL=y
# CONFIG_ISCSI_TCP is not set
# CONFIG_BLK_DEV_3W_XXXX_RAID is not set
# CONFIG_SCSI_3W_9XXX is not set
# CONFIG_SCSI_ACARD is not set
# CONFIG_SCSI_AACRAID is not set
# CONFIG_SCSI_AIC7XXX is not set
# CONFIG_SCSI_AIC7XXX_OLD is not set
# CONFIG_SCSI_AIC79XX is not set
# CONFIG_SCSI_AIC94XX is not set
# CONFIG_SCSI_DPT_I2O is not set
# CONFIG_SCSI_ADVANSYS is not set
# CONFIG_SCSI_ARCMSR is not set
# CONFIG_MEGARAID_NEWGEN is not set
# CONFIG_MEGARAID_LEGACY is not set
# CONFIG_MEGARAID_SAS is not set
# CONFIG_SCSI_HPTIOP is not set
# CONFIG_SCSI_BUSLOGIC is not set
# CONFIG_SCSI_DMX3191D is not set
# CONFIG_SCSI_EATA is not set
# CONFIG_SCSI_FUTURE_DOMAIN is not set
# CONFIG_SCSI_GDTH is not set
# CONFIG_SCSI_IPS is not set
# CONFIG_SCSI_INITIO is not set
# CONFIG_SCSI_INIA100 is not set
# CONFIG_SCSI_MVSAS is not set
# CONFIG_SCSI_STEX is not set
# CONFIG_SCSI_SYM53C8XX_2 is not set
# CONFIG_SCSI_IPR is not set
# CONFIG_SCSI_QLOGIC_1280 is not set
CONFIG_SCSI_QLA_FC=m
# CONFIG_SCSI_QLA_ISCSI is not set
# CONFIG_SCSI_LPFC is not set
# CONFIG_SCSI_DC395x is not set
# CONFIG_SCSI_DC390T is not set
# CONFIG_SCSI_NSP32 is not set
# CONFIG_SCSI_DEBUG is not set
# CONFIG_SCSI_SRP is not set
# CONFIG_SCSI_DH is not set
CONFIG_ATA=y
# CONFIG_ATA_NONSTANDARD is not set
CONFIG_ATA_ACPI=y
# CONFIG_SATA_PMP is not set
# CONFIG_SATA_AHCI is not set
# CONFIG_SATA_SIL24 is not set
CONFIG_ATA_SFF=y
# CONFIG_SATA_SVW is not set
# CONFIG_ATA_PIIX is not set
# CONFIG_SATA_MV is not set
# CONFIG_SATA_NV is not set
# CONFIG_PDC_ADMA is not set
# CONFIG_SATA_QSTOR is not set
# CONFIG_SATA_PROMISE is not set
# CONFIG_SATA_SX4 is not set
# CONFIG_SATA_SIL is not set
# CONFIG_SATA_SIS is not set
# CONFIG_SATA_ULI is not set
# CONFIG_SATA_VIA is not set
# CONFIG_SATA_VITESSE is not set
# CONFIG_SATA_INIC162X is not set
CONFIG_PATA_ACPI=y
# CONFIG_PATA_ALI is not set
# CONFIG_PATA_AMD is not set
# CONFIG_PATA_ARTOP is not set
# CONFIG_PATA_ATIIXP is not set
# CONFIG_PATA_CMD640_PCI is not set
# CONFIG_PATA_CMD64X is not set
# CONFIG_PATA_CS5520 is not set
# CONFIG_PATA_CS5530 is not set
# CONFIG_PATA_CS5535 is not set
# CONFIG_PATA_CS5536 is not set
# CONFIG_PATA_CYPRESS is not set
# CONFIG_PATA_EFAR is not set
# CONFIG_ATA_GENERIC is not set
# CONFIG_PATA_HPT366 is not set
# CONFIG_PATA_HPT37X is not set
# CONFIG_PATA_HPT3X2N is not set
# CONFIG_PATA_HPT3X3 is not set
# CONFIG_PATA_IT821X is not set
# CONFIG_PATA_IT8213 is not set
# CONFIG_PATA_JMICRON is not set
# CONFIG_PATA_TRIFLEX is not set
# CONFIG_PATA_MARVELL is not set
# CONFIG_PATA_MPIIX is not set
# CONFIG_PATA_OLDPIIX is not set
# CONFIG_PATA_NETCELL is not set
# CONFIG_PATA_NINJA32 is not set
# CONFIG_PATA_NS87410 is not set
# CONFIG_PATA_NS87415 is not set
# CONFIG_PATA_OPTI is not set
# CONFIG_PATA_OPTIDMA is not set
# CONFIG_PATA_PDC_OLD is not set
# CONFIG_PATA_RADISYS is not set
# CONFIG_PATA_RZ1000 is not set
# CONFIG_PATA_SC1200 is not set
CONFIG_PATA_SERVERWORKS=y
# CONFIG_PATA_PDC2027X is not set
# CONFIG_PATA_SIL680 is not set
# CONFIG_PATA_SIS is not set
# CONFIG_PATA_VIA is not set
# CONFIG_PATA_WINBOND is not set
# CONFIG_PATA_SCH is not set
CONFIG_MD=y
CONFIG_BLK_DEV_MD=m
CONFIG_MD_LINEAR=m
CONFIG_MD_RAID0=m
CONFIG_MD_RAID1=m
CONFIG_MD_RAID10=m
# CONFIG_MD_RAID456 is not set
CONFIG_MD_MULTIPATH=m
# CONFIG_MD_FAULTY is not set
CONFIG_BLK_DEV_DM=m
# CONFIG_DM_DEBUG is not set
CONFIG_DM_CRYPT=m
CONFIG_DM_SNAPSHOT=m
CONFIG_DM_MIRROR=m
CONFIG_DM_ZERO=m
CONFIG_DM_MULTIPATH=m
# CONFIG_DM_DELAY is not set
CONFIG_DM_UEVENT=y
# CONFIG_FUSION is not set

#
# IEEE 1394 (FireWire) support
#

#
# Enable only one of the two stacks, unless you know what you are doing
#
# CONFIG_FIREWIRE is not set
# CONFIG_IEEE1394 is not set
# CONFIG_I2O is not set
# CONFIG_MACINTOSH_DRIVERS is not set
CONFIG_NETDEVICES=y
CONFIG_DUMMY=y
# CONFIG_BONDING is not set
# CONFIG_MACVLAN is not set
# CONFIG_EQUALIZER is not set
CONFIG_TUN=y
CONFIG_VETH=y
# CONFIG_NET_SB1000 is not set
# CONFIG_ARCNET is not set
CONFIG_PHYLIB=y

#
# MII PHY device drivers
#
CONFIG_MARVELL_PHY=y
CONFIG_DAVICOM_PHY=y
CONFIG_QSEMI_PHY=y
CONFIG_LXT_PHY=y
CONFIG_CICADA_PHY=y
CONFIG_VITESSE_PHY=y
CONFIG_SMSC_PHY=y
CONFIG_BROADCOM_PHY=y
CONFIG_ICPLUS_PHY=y
# CONFIG_REALTEK_PHY is not set
# CONFIG_FIXED_PHY is not set
# CONFIG_MDIO_BITBANG is not set
CONFIG_NET_ETHERNET=y
CONFIG_MII=y
# CONFIG_HAPPYMEAL is not set
# CONFIG_SUNGEM is not set
# CONFIG_CASSINI is not set
# CONFIG_NET_VENDOR_3COM is not set
# CONFIG_NET_TULIP is not set
# CONFIG_HP100 is not set
# CONFIG_IBM_NEW_EMAC_ZMII is not set
# CONFIG_IBM_NEW_EMAC_RGMII is not set
# CONFIG_IBM_NEW_EMAC_TAH is not set
# CONFIG_IBM_NEW_EMAC_EMAC4 is not set
CONFIG_NET_PCI=y
# CONFIG_PCNET32 is not set
# CONFIG_AMD8111_ETH is not set
# CONFIG_ADAPTEC_STARFIRE is not set
# CONFIG_B44 is not set
# CONFIG_FORCEDETH is not set
# CONFIG_EEPRO100 is not set
CONFIG_E100=m
# CONFIG_FEALNX is not set
# CONFIG_NATSEMI is not set
# CONFIG_NE2K_PCI is not set
# CONFIG_8139CP is not set
# CONFIG_8139TOO is not set
# CONFIG_R6040 is not set
# CONFIG_SIS900 is not set
# CONFIG_EPIC100 is not set
# CONFIG_SUNDANCE is not set
# CONFIG_TLAN is not set
# CONFIG_VIA_RHINE is not set
# CONFIG_SC92031 is not set
CONFIG_NETDEV_1000=y
# CONFIG_ACENIC is not set
# CONFIG_DL2K is not set
# CONFIG_E1000 is not set
# CONFIG_E1000E is not set
# CONFIG_IP1000 is not set
# CONFIG_IGB is not set
# CONFIG_NS83820 is not set
# CONFIG_HAMACHI is not set
# CONFIG_YELLOWFIN is not set
# CONFIG_R8169 is not set
# CONFIG_SIS190 is not set
# CONFIG_SKGE is not set
# CONFIG_SKY2 is not set
# CONFIG_VIA_VELOCITY is not set
CONFIG_TIGON3=m
CONFIG_BNX2=m
# CONFIG_QLA3XXX is not set
# CONFIG_ATL1 is not set
# CONFIG_ATL1E is not set
# CONFIG_NETDEV_10000 is not set
# CONFIG_TR is not set

#
# Wireless LAN
#
# CONFIG_WLAN_PRE80211 is not set
# CONFIG_WLAN_80211 is not set
# CONFIG_IWLWIFI_LEDS is not set
# CONFIG_WAN is not set
# CONFIG_FDDI is not set
# CONFIG_HIPPI is not set
# CONFIG_PPP is not set
# CONFIG_SLIP is not set
# CONFIG_NET_FC is not set
# CONFIG_NETCONSOLE is not set
# CONFIG_NETPOLL is not set
# CONFIG_NET_POLL_CONTROLLER is not set
# CONFIG_ISDN is not set
# CONFIG_PHONE is not set

#
# Input device support
#
CONFIG_INPUT=y
# CONFIG_INPUT_FF_MEMLESS is not set
CONFIG_INPUT_POLLDEV=y

#
# Userland interfaces
#
CONFIG_INPUT_MOUSEDEV=y
# CONFIG_INPUT_MOUSEDEV_PSAUX is not set
CONFIG_INPUT_MOUSEDEV_SCREEN_X=1024
CONFIG_INPUT_MOUSEDEV_SCREEN_Y=768
# CONFIG_INPUT_JOYDEV is not set
# CONFIG_INPUT_EVDEV is not set
# CONFIG_INPUT_EVBUG is not set

#
# Input Device Drivers
#
CONFIG_INPUT_KEYBOARD=y
CONFIG_KEYBOARD_ATKBD=y
# CONFIG_KEYBOARD_SUNKBD is not set
# CONFIG_KEYBOARD_LKKBD is not set
# CONFIG_KEYBOARD_XTKBD is not set
# CONFIG_KEYBOARD_NEWTON is not set
# CONFIG_KEYBOARD_STOWAWAY is not set
# CONFIG_INPUT_MOUSE is not set
# CONFIG_INPUT_JOYSTICK is not set
# CONFIG_INPUT_TABLET is not set
# CONFIG_INPUT_TOUCHSCREEN is not set
# CONFIG_INPUT_MISC is not set

#
# Hardware I/O ports
#
CONFIG_SERIO=y
CONFIG_SERIO_I8042=y
# CONFIG_SERIO_SERPORT is not set
# CONFIG_SERIO_CT82C710 is not set
# CONFIG_SERIO_PCIPS2 is not set
CONFIG_SERIO_LIBPS2=y
# CONFIG_SERIO_RAW is not set
# CONFIG_GAMEPORT is not set

#
# Character devices
#
CONFIG_VT=y
CONFIG_CONSOLE_TRANSLATIONS=y
CONFIG_VT_CONSOLE=y
CONFIG_HW_CONSOLE=y
CONFIG_VT_HW_CONSOLE_BINDING=y
# CONFIG_DEVKMEM is not set
# CONFIG_SERIAL_NONSTANDARD is not set
# CONFIG_NOZOMI is not set

#
# Serial drivers
#
CONFIG_SERIAL_8250=y
CONFIG_SERIAL_8250_CONSOLE=y
CONFIG_FIX_EARLYCON_MEM=y
CONFIG_SERIAL_8250_PCI=y
CONFIG_SERIAL_8250_PNP=y
CONFIG_SERIAL_8250_NR_UARTS=4
CONFIG_SERIAL_8250_RUNTIME_UARTS=4
# CONFIG_SERIAL_8250_EXTENDED is not set

#
# Non-8250 serial port support
#
CONFIG_SERIAL_CORE=y
CONFIG_SERIAL_CORE_CONSOLE=y
CONFIG_CONSOLE_POLL=y
# CONFIG_SERIAL_JSM is not set
CONFIG_UNIX98_PTYS=y
# CONFIG_LEGACY_PTYS is not set
# CONFIG_IPMI_HANDLER is not set
CONFIG_HW_RANDOM=y
CONFIG_HW_RANDOM_INTEL=y
# CONFIG_HW_RANDOM_AMD is not set
# CONFIG_HW_RANDOM_GEODE is not set
CONFIG_HW_RANDOM_VIA=y
# CONFIG_NVRAM is not set
CONFIG_RTC=y
# CONFIG_R3964 is not set
# CONFIG_APPLICOM is not set
# CONFIG_SONYPI is not set
# CONFIG_MWAVE is not set
# CONFIG_PC8736x_GPIO is not set
# CONFIG_NSC_GPIO is not set
# CONFIG_CS5535_GPIO is not set
# CONFIG_RAW_DRIVER is not set
CONFIG_HPET=y
CONFIG_HPET_RTC_IRQ=y
CONFIG_HPET_MMAP=y
# CONFIG_HANGCHECK_TIMER is not set
# CONFIG_TCG_TPM is not set
# CONFIG_TELCLOCK is not set
CONFIG_DEVPORT=y
CONFIG_I2C=y
CONFIG_I2C_BOARDINFO=y
CONFIG_I2C_CHARDEV=y
CONFIG_I2C_HELPER_AUTO=y

#
# I2C Hardware Bus support
#

#
# PC SMBus host controller drivers
#
# CONFIG_I2C_ALI1535 is not set
# CONFIG_I2C_ALI1563 is not set
# CONFIG_I2C_ALI15X3 is not set
# CONFIG_I2C_AMD756 is not set
# CONFIG_I2C_AMD8111 is not set
# CONFIG_I2C_I801 is not set
CONFIG_I2C_ISCH=y
CONFIG_I2C_PIIX4=y
# CONFIG_I2C_NFORCE2 is not set
# CONFIG_I2C_SIS5595 is not set
# CONFIG_I2C_SIS630 is not set
# CONFIG_I2C_SIS96X is not set
# CONFIG_I2C_VIA is not set
# CONFIG_I2C_VIAPRO is not set

#
# I2C system bus drivers (mostly embedded / system-on-chip)
#
CONFIG_I2C_OCORES=y
# CONFIG_I2C_SIMTEC is not set

#
# External I2C/SMBus adapter drivers
#
# CONFIG_I2C_PARPORT_LIGHT is not set
# CONFIG_I2C_TAOS_EVM is not set

#
# Graphics adapter I2C/DDC channel drivers
#
# CONFIG_I2C_VOODOO3 is not set

#
# Other I2C/SMBus bus drivers
#
# CONFIG_I2C_PCA_PLATFORM is not set
# CONFIG_I2C_STUB is not set
# CONFIG_SCx200_ACB is not set

#
# Miscellaneous I2C Chip support
#
# CONFIG_DS1682 is not set
CONFIG_AT24=y
CONFIG_SENSORS_EEPROM=y
# CONFIG_SENSORS_PCF8574 is not set
# CONFIG_PCF8575 is not set
# CONFIG_SENSORS_PCA9539 is not set
# CONFIG_SENSORS_PCF8591 is not set
# CONFIG_SENSORS_MAX6875 is not set
# CONFIG_SENSORS_TSL2550 is not set
# CONFIG_I2C_DEBUG_CORE is not set
# CONFIG_I2C_DEBUG_ALGO is not set
# CONFIG_I2C_DEBUG_BUS is not set
# CONFIG_I2C_DEBUG_CHIP is not set
# CONFIG_SPI is not set
CONFIG_ARCH_WANT_OPTIONAL_GPIOLIB=y
# CONFIG_GPIOLIB is not set
# CONFIG_W1 is not set
CONFIG_POWER_SUPPLY=y
# CONFIG_POWER_SUPPLY_DEBUG is not set
# CONFIG_PDA_POWER is not set
# CONFIG_BATTERY_DS2760 is not set
CONFIG_HWMON=y
CONFIG_HWMON_VID=y
CONFIG_SENSORS_ABITUGURU=y
# CONFIG_SENSORS_ABITUGURU3 is not set
# CONFIG_SENSORS_AD7414 is not set
CONFIG_SENSORS_AD7418=y
CONFIG_SENSORS_ADM1021=y
CONFIG_SENSORS_ADM1025=y
CONFIG_SENSORS_ADM1026=y
CONFIG_SENSORS_ADM1029=y
CONFIG_SENSORS_ADM1031=y
CONFIG_SENSORS_ADM9240=y
# CONFIG_SENSORS_ADT7470 is not set
# CONFIG_SENSORS_ADT7473 is not set
CONFIG_SENSORS_K8TEMP=y
CONFIG_SENSORS_ASB100=y
CONFIG_SENSORS_ATXP1=y
CONFIG_SENSORS_DS1621=y
# CONFIG_SENSORS_I5K_AMB is not set
CONFIG_SENSORS_F71805F=y
# CONFIG_SENSORS_F71882FG is not set
# CONFIG_SENSORS_F75375S is not set
CONFIG_SENSORS_FSCHER=y
CONFIG_SENSORS_FSCPOS=y
# CONFIG_SENSORS_FSCHMD is not set
CONFIG_SENSORS_GL518SM=y
CONFIG_SENSORS_GL520SM=y
# CONFIG_SENSORS_CORETEMP is not set
CONFIG_SENSORS_IT87=y
CONFIG_SENSORS_LM63=y
CONFIG_SENSORS_LM75=y
CONFIG_SENSORS_LM77=y
CONFIG_SENSORS_LM78=y
CONFIG_SENSORS_LM80=y
CONFIG_SENSORS_LM83=y
CONFIG_SENSORS_LM85=y
CONFIG_SENSORS_LM87=y
CONFIG_SENSORS_LM90=y
CONFIG_SENSORS_LM92=y
# CONFIG_SENSORS_LM93 is not set
CONFIG_SENSORS_MAX1619=y
CONFIG_SENSORS_MAX6650=y
CONFIG_SENSORS_PC87360=y
CONFIG_SENSORS_PC87427=y
CONFIG_SENSORS_SIS5595=y
# CONFIG_SENSORS_DME1737 is not set
CONFIG_SENSORS_SMSC47M1=y
CONFIG_SENSORS_SMSC47M192=y
CONFIG_SENSORS_SMSC47B397=y
# CONFIG_SENSORS_ADS7828 is not set
# CONFIG_SENSORS_THMC50 is not set
CONFIG_SENSORS_VIA686A=y
CONFIG_SENSORS_VT1211=y
CONFIG_SENSORS_VT8231=y
CONFIG_SENSORS_W83781D=y
CONFIG_SENSORS_W83791D=y
CONFIG_SENSORS_W83792D=y
CONFIG_SENSORS_W83793=y
CONFIG_SENSORS_W83L785TS=y
# CONFIG_SENSORS_W83L786NG is not set
CONFIG_SENSORS_W83627HF=y
CONFIG_SENSORS_W83627EHF=y
CONFIG_SENSORS_HDAPS=y
# CONFIG_SENSORS_APPLESMC is not set
# CONFIG_HWMON_DEBUG_CHIP is not set
CONFIG_THERMAL=y
# CONFIG_THERMAL_HWMON is not set
# CONFIG_WATCHDOG is not set

#
# Sonics Silicon Backplane
#
CONFIG_SSB_POSSIBLE=y
# CONFIG_SSB is not set

#
# Multifunction device drivers
#
# CONFIG_MFD_CORE is not set
# CONFIG_MFD_SM501 is not set
# CONFIG_HTC_PASIC3 is not set
# CONFIG_MFD_TMIO is not set

#
# Multimedia devices
#

#
# Multimedia core support
#
# CONFIG_VIDEO_DEV is not set
# CONFIG_DVB_CORE is not set
# CONFIG_VIDEO_MEDIA is not set

#
# Multimedia drivers
#
CONFIG_DAB=y

#
# Graphics support
#
# CONFIG_AGP is not set
# CONFIG_DRM is not set
# CONFIG_VGASTATE is not set
# CONFIG_VIDEO_OUTPUT_CONTROL is not set
# CONFIG_FB is not set
# CONFIG_BACKLIGHT_LCD_SUPPORT is not set

#
# Display device support
#
# CONFIG_DISPLAY_SUPPORT is not set

#
# Console display driver support
#
CONFIG_VGA_CONSOLE=y
# CONFIG_VGACON_SOFT_SCROLLBACK is not set
CONFIG_VIDEO_SELECT=y
CONFIG_DUMMY_CONSOLE=y
# CONFIG_SOUND is not set
# CONFIG_HID_SUPPORT is not set
# CONFIG_USB_SUPPORT is not set
# CONFIG_MMC is not set
# CONFIG_MEMSTICK is not set
# CONFIG_NEW_LEDS is not set
# CONFIG_ACCESSIBILITY is not set
# CONFIG_INFINIBAND is not set
CONFIG_EDAC=y

#
# Reporting subsystems
#
# CONFIG_EDAC_DEBUG is not set
CONFIG_EDAC_MM_EDAC=y
# CONFIG_EDAC_AMD76X is not set
CONFIG_EDAC_E7XXX=y
CONFIG_EDAC_E752X=y
CONFIG_EDAC_I82875P=y
CONFIG_EDAC_I82975X=y
CONFIG_EDAC_I3000=y
CONFIG_EDAC_I82860=y
# CONFIG_EDAC_R82600 is not set
CONFIG_EDAC_I5000=y
# CONFIG_EDAC_I5100 is not set
# CONFIG_RTC_CLASS is not set
# CONFIG_UIO is not set

#
# Firmware Drivers
#
# CONFIG_EDD is not set
CONFIG_FIRMWARE_MEMMAP=y
# CONFIG_DELL_RBU is not set
# CONFIG_DCDBAS is not set
CONFIG_DMIID=y
# CONFIG_ISCSI_IBFT_FIND is not set

#
# File systems
#
CONFIG_EXT2_FS=y
# CONFIG_EXT2_FS_XATTR is not set
# CONFIG_EXT2_FS_XIP is not set
CONFIG_EXT3_FS=y
CONFIG_EXT3_FS_XATTR=y
CONFIG_EXT3_FS_POSIX_ACL=y
# CONFIG_EXT3_FS_SECURITY is not set
# CONFIG_EXT4DEV_FS is not set
CONFIG_JBD=y
# CONFIG_JBD_DEBUG is not set
CONFIG_FS_MBCACHE=y
CONFIG_REISERFS_FS=y
# CONFIG_REISERFS_CHECK is not set
# CONFIG_REISERFS_PROC_INFO is not set
CONFIG_REISERFS_FS_XATTR=y
CONFIG_REISERFS_FS_POSIX_ACL=y
# CONFIG_REISERFS_FS_SECURITY is not set
# CONFIG_JFS_FS is not set
CONFIG_FS_POSIX_ACL=y
CONFIG_XFS_FS=y
# CONFIG_XFS_QUOTA is not set
CONFIG_XFS_POSIX_ACL=y
# CONFIG_XFS_RT is not set
# CONFIG_XFS_DEBUG is not set
# CONFIG_GFS2_FS is not set
# CONFIG_OCFS2_FS is not set
CONFIG_DNOTIFY=y
CONFIG_INOTIFY=y
CONFIG_INOTIFY_USER=y
# CONFIG_QUOTA is not set
# CONFIG_AUTOFS_FS is not set
# CONFIG_AUTOFS4_FS is not set
# CONFIG_FUSE_FS is not set

#
# CD-ROM/DVD Filesystems
#
# CONFIG_ISO9660_FS is not set
# CONFIG_UDF_FS is not set

#
# DOS/FAT/NT Filesystems
#
# CONFIG_MSDOS_FS is not set
# CONFIG_VFAT_FS is not set
# CONFIG_NTFS_FS is not set

#
# Pseudo filesystems
#
CONFIG_PROC_FS=y
CONFIG_PROC_KCORE=y
CONFIG_PROC_VMCORE=y
CONFIG_PROC_SYSCTL=y
CONFIG_SYSFS=y
CONFIG_TMPFS=y
# CONFIG_TMPFS_POSIX_ACL is not set
# CONFIG_HUGETLBFS is not set
# CONFIG_HUGETLB_PAGE is not set
# CONFIG_CONFIGFS_FS is not set

#
# Miscellaneous filesystems
#
# CONFIG_ADFS_FS is not set
# CONFIG_AFFS_FS is not set
# CONFIG_HFS_FS is not set
# CONFIG_HFSPLUS_FS is not set
# CONFIG_BEFS_FS is not set
# CONFIG_BFS_FS is not set
# CONFIG_EFS_FS is not set
# CONFIG_CRAMFS is not set
# CONFIG_VXFS_FS is not set
# CONFIG_MINIX_FS is not set
# CONFIG_OMFS_FS is not set
# CONFIG_HPFS_FS is not set
# CONFIG_QNX4FS_FS is not set
# CONFIG_ROMFS_FS is not set
# CONFIG_SYSV_FS is not set
# CONFIG_UFS_FS is not set
CONFIG_NETWORK_FILESYSTEMS=y
CONFIG_NFS_FS=y
CONFIG_NFS_V3=y
CONFIG_NFS_V3_ACL=y
CONFIG_NFS_V4=y
# CONFIG_NFSD is not set
CONFIG_LOCKD=y
CONFIG_LOCKD_V4=y
CONFIG_NFS_ACL_SUPPORT=y
CONFIG_NFS_COMMON=y
CONFIG_SUNRPC=y
CONFIG_SUNRPC_GSS=y
CONFIG_RPCSEC_GSS_KRB5=y
# CONFIG_RPCSEC_GSS_SPKM3 is not set
# CONFIG_SMB_FS is not set
# CONFIG_CIFS is not set
# CONFIG_NCP_FS is not set
CONFIG_CODA_FS=y
# CONFIG_AFS_FS is not set

#
# Partition Types
#
# CONFIG_PARTITION_ADVANCED is not set
CONFIG_MSDOS_PARTITION=y
# CONFIG_NLS is not set
# CONFIG_DLM is not set

#
# Kernel hacking
#
CONFIG_TRACE_IRQFLAGS_SUPPORT=y
# CONFIG_PRINTK_TIME is not set
# CONFIG_ENABLE_WARN_DEPRECATED is not set
# CONFIG_ENABLE_MUST_CHECK is not set
CONFIG_FRAME_WARN=1024
CONFIG_MAGIC_SYSRQ=y
# CONFIG_UNUSED_SYMBOLS is not set
CONFIG_DEBUG_FS=y
# CONFIG_HEADERS_CHECK is not set
CONFIG_DEBUG_KERNEL=y
# CONFIG_DEBUG_SHIRQ is not set
CONFIG_DETECT_SOFTLOCKUP=y
# CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC is not set
CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC_VALUE=0
# CONFIG_SCHED_DEBUG is not set
# CONFIG_SCHEDSTATS is not set
# CONFIG_TIMER_STATS is not set
# CONFIG_DEBUG_OBJECTS is not set
# CONFIG_SLUB_DEBUG_ON is not set
# CONFIG_SLUB_STATS is not set
# CONFIG_DEBUG_RT_MUTEXES is not set
# CONFIG_RT_MUTEX_TESTER is not set
# CONFIG_DEBUG_SPINLOCK is not set
# CONFIG_DEBUG_MUTEXES is not set
# CONFIG_DEBUG_LOCK_ALLOC is not set
# CONFIG_PROVE_LOCKING is not set
# CONFIG_LOCK_STAT is not set
# CONFIG_DEBUG_SPINLOCK_SLEEP is not set
# CONFIG_DEBUG_LOCKING_API_SELFTESTS is not set
# CONFIG_DEBUG_KOBJECT is not set
# CONFIG_DEBUG_HIGHMEM is not set
CONFIG_DEBUG_BUGVERBOSE=y
CONFIG_DEBUG_INFO=y
# CONFIG_DEBUG_VM is not set
# CONFIG_DEBUG_WRITECOUNT is not set
CONFIG_DEBUG_MEMORY_INIT=y
# CONFIG_DEBUG_LIST is not set
# CONFIG_DEBUG_SG is not set
CONFIG_FRAME_POINTER=y
# CONFIG_BOOT_PRINTK_DELAY is not set
# CONFIG_RCU_TORTURE_TEST is not set
# CONFIG_BACKTRACE_SELF_TEST is not set
# CONFIG_FAULT_INJECTION is not set
# CONFIG_LATENCYTOP is not set
CONFIG_SYSCTL_SYSCALL_CHECK=y
CONFIG_HAVE_FTRACE=y
CONFIG_HAVE_DYNAMIC_FTRACE=y
# CONFIG_FTRACE is not set
# CONFIG_IRQSOFF_TRACER is not set
# CONFIG_SYSPROF_TRACER is not set
# CONFIG_SCHED_TRACER is not set
# CONFIG_CONTEXT_SWITCH_TRACER is not set
# CONFIG_PROVIDE_OHCI1394_DMA_INIT is not set
# CONFIG_SAMPLES is not set
CONFIG_HAVE_ARCH_KGDB=y
CONFIG_KGDB=y
CONFIG_KGDB_SERIAL_CONSOLE=y
# CONFIG_KGDB_TESTS is not set
CONFIG_STRICT_DEVMEM=y
# CONFIG_X86_VERBOSE_BOOTUP is not set
CONFIG_EARLY_PRINTK=y
# CONFIG_DEBUG_STACKOVERFLOW is not set
# CONFIG_DEBUG_STACK_USAGE is not set
# CONFIG_DEBUG_PAGEALLOC is not set
# CONFIG_DEBUG_PER_CPU_MAPS is not set
# CONFIG_X86_PTDUMP is not set
CONFIG_DEBUG_RODATA=y
# CONFIG_DEBUG_RODATA_TEST is not set
# CONFIG_DEBUG_NX_TEST is not set
CONFIG_4KSTACKS=y
CONFIG_DOUBLEFAULT=y
# CONFIG_MMIOTRACE is not set
CONFIG_IO_DELAY_TYPE_0X80=0
CONFIG_IO_DELAY_TYPE_0XED=1
CONFIG_IO_DELAY_TYPE_UDELAY=2
CONFIG_IO_DELAY_TYPE_NONE=3
CONFIG_IO_DELAY_0X80=y
# CONFIG_IO_DELAY_0XED is not set
# CONFIG_IO_DELAY_UDELAY is not set
# CONFIG_IO_DELAY_NONE is not set
CONFIG_DEFAULT_IO_DELAY_TYPE=0
# CONFIG_DEBUG_BOOT_PARAMS is not set
# CONFIG_CPA_DEBUG is not set
# CONFIG_OPTIMIZE_INLINING is not set

#
# Security options
#
# CONFIG_KEYS is not set
CONFIG_SECURITY=y
CONFIG_SECURITY_NETWORK=y
# CONFIG_SECURITY_FILE_CAPABILITIES is not set
CONFIG_SECURITY_DEFAULT_MMAP_MIN_ADDR=0
CONFIG_CRYPTO=y

#
# Crypto core or helper
#
CONFIG_CRYPTO_ALGAPI=y
CONFIG_CRYPTO_BLKCIPHER=y
CONFIG_CRYPTO_MANAGER=y
# CONFIG_CRYPTO_GF128MUL is not set
# CONFIG_CRYPTO_NULL is not set
# CONFIG_CRYPTO_CRYPTD is not set
# CONFIG_CRYPTO_AUTHENC is not set
# CONFIG_CRYPTO_TEST is not set

#
# Authenticated Encryption with Associated Data
#
# CONFIG_CRYPTO_CCM is not set
# CONFIG_CRYPTO_GCM is not set
# CONFIG_CRYPTO_SEQIV is not set

#
# Block modes
#
CONFIG_CRYPTO_CBC=y
# CONFIG_CRYPTO_CTR is not set
# CONFIG_CRYPTO_CTS is not set
# CONFIG_CRYPTO_ECB is not set
# CONFIG_CRYPTO_LRW is not set
# CONFIG_CRYPTO_PCBC is not set
# CONFIG_CRYPTO_XTS is not set

#
# Hash modes
#
# CONFIG_CRYPTO_HMAC is not set
# CONFIG_CRYPTO_XCBC is not set

#
# Digest
#
# CONFIG_CRYPTO_CRC32C is not set
# CONFIG_CRYPTO_MD4 is not set
CONFIG_CRYPTO_MD5=y
# CONFIG_CRYPTO_MICHAEL_MIC is not set
# CONFIG_CRYPTO_RMD128 is not set
# CONFIG_CRYPTO_RMD160 is not set
# CONFIG_CRYPTO_RMD256 is not set
# CONFIG_CRYPTO_RMD320 is not set
# CONFIG_CRYPTO_SHA1 is not set
# CONFIG_CRYPTO_SHA256 is not set
# CONFIG_CRYPTO_SHA512 is not set
# CONFIG_CRYPTO_TGR192 is not set
# CONFIG_CRYPTO_WP512 is not set

#
# Ciphers
#
# CONFIG_CRYPTO_AES is not set
# CONFIG_CRYPTO_AES_586 is not set
# CONFIG_CRYPTO_ANUBIS is not set
# CONFIG_CRYPTO_ARC4 is not set
# CONFIG_CRYPTO_BLOWFISH is not set
# CONFIG_CRYPTO_CAMELLIA is not set
# CONFIG_CRYPTO_CAST5 is not set
# CONFIG_CRYPTO_CAST6 is not set
CONFIG_CRYPTO_DES=y
# CONFIG_CRYPTO_FCRYPT is not set
# CONFIG_CRYPTO_KHAZAD is not set
# CONFIG_CRYPTO_SALSA20 is not set
# CONFIG_CRYPTO_SALSA20_586 is not set
# CONFIG_CRYPTO_SEED is not set
# CONFIG_CRYPTO_SERPENT is not set
# CONFIG_CRYPTO_TEA is not set
# CONFIG_CRYPTO_TWOFISH is not set
# CONFIG_CRYPTO_TWOFISH_586 is not set

#
# Compression
#
# CONFIG_CRYPTO_DEFLATE is not set
# CONFIG_CRYPTO_LZO is not set
# CONFIG_CRYPTO_HW is not set
CONFIG_HAVE_KVM=y
# CONFIG_VIRTUALIZATION is not set

#
# Library routines
#
CONFIG_BITREVERSE=y
CONFIG_GENERIC_FIND_FIRST_BIT=y
CONFIG_GENERIC_FIND_NEXT_BIT=y
# CONFIG_CRC_CCITT is not set
# CONFIG_CRC16 is not set
CONFIG_CRC_T10DIF=y
# CONFIG_CRC_ITU_T is not set
CONFIG_CRC32=y
# CONFIG_CRC7 is not set
# CONFIG_LIBCRC32C is not set
CONFIG_ZLIB_INFLATE=m
CONFIG_TEXTSEARCH=y
CONFIG_TEXTSEARCH_KMP=y
CONFIG_TEXTSEARCH_BM=y
CONFIG_TEXTSEARCH_FSM=y
CONFIG_PLIST=y
CONFIG_HAS_IOMEM=y
CONFIG_HAS_IOPORT=y
CONFIG_HAS_DMA=y

^ permalink raw reply	[flat|nested] 29+ messages in thread

* High load in 2.6.27, NFS / rpcauth_lookup_credcache()?
@ 2008-10-22  9:12 ` Max Kellermann
  0 siblings, 0 replies; 29+ messages in thread
From: Max Kellermann @ 2008-10-22  9:12 UTC (permalink / raw)
  To: linux-kernel, linux-nfs

[-- Attachment #1: Type: text/plain, Size: 984 bytes --]

Hi,

after I was able to fix http://lkml.org/lkml/2008/10/17/147, the
server which was already upgraded to 2.6.27.2 still gets very high
load.  It is a web server with NFS file storage (NetApp), and while
the others in the cluster (kernel 2.6.25) have a load of 1-3, 2.6.27.2
gets 30-50.

I did an oprofile, with the following results (server just started,
load "only" 5-10):

87593    56.1116  (no location information)   vmlinux
vmlinux                  rpcauth_lookup_credcache
16037    10.2732  auth_generic.c:0            vmlinux
vmlinux                  generic_match
6460      4.1382  (no location information)   php4
php4                     (no symbols)
2478      1.5874  (no location information)   libc-2.7.so
libc-2.7.so              (no symbols)
[...]

We havn't configured any special authentication method.  It is a NFSv3
over UDP mount, but the kernel has NFSv4 and therefore KRB5 enabled.

Any ideas why rpcauth_lookup_credcache() goes overboard with CPU
usage?

Max

[-- Attachment #2: .config --]
[-- Type: text/plain, Size: 39058 bytes --]

#
# Automatically generated make config: don't edit
# Linux kernel version: 2.6.27.2
# Wed Oct 22 11:06:17 2008
#
# CONFIG_64BIT is not set
CONFIG_X86_32=y
# CONFIG_X86_64 is not set
CONFIG_X86=y
CONFIG_ARCH_DEFCONFIG="arch/x86/configs/i386_defconfig"
# CONFIG_GENERIC_LOCKBREAK is not set
CONFIG_GENERIC_TIME=y
CONFIG_GENERIC_CMOS_UPDATE=y
CONFIG_CLOCKSOURCE_WATCHDOG=y
CONFIG_GENERIC_CLOCKEVENTS=y
CONFIG_GENERIC_CLOCKEVENTS_BROADCAST=y
CONFIG_LOCKDEP_SUPPORT=y
CONFIG_STACKTRACE_SUPPORT=y
CONFIG_HAVE_LATENCYTOP_SUPPORT=y
CONFIG_FAST_CMPXCHG_LOCAL=y
CONFIG_MMU=y
CONFIG_ZONE_DMA=y
CONFIG_GENERIC_ISA_DMA=y
CONFIG_GENERIC_IOMAP=y
CONFIG_GENERIC_BUG=y
CONFIG_GENERIC_HWEIGHT=y
# CONFIG_GENERIC_GPIO is not set
CONFIG_ARCH_MAY_HAVE_PC_FDC=y
# CONFIG_RWSEM_GENERIC_SPINLOCK is not set
CONFIG_RWSEM_XCHGADD_ALGORITHM=y
# CONFIG_ARCH_HAS_ILOG2_U32 is not set
# CONFIG_ARCH_HAS_ILOG2_U64 is not set
CONFIG_ARCH_HAS_CPU_IDLE_WAIT=y
CONFIG_GENERIC_CALIBRATE_DELAY=y
# CONFIG_GENERIC_TIME_VSYSCALL is not set
CONFIG_ARCH_HAS_CPU_RELAX=y
CONFIG_ARCH_HAS_CACHE_LINE_SIZE=y
CONFIG_HAVE_SETUP_PER_CPU_AREA=y
# CONFIG_HAVE_CPUMASK_OF_CPU_MAP is not set
CONFIG_ARCH_HIBERNATION_POSSIBLE=y
CONFIG_ARCH_SUSPEND_POSSIBLE=y
# CONFIG_ZONE_DMA32 is not set
CONFIG_ARCH_POPULATES_NODE_MAP=y
# CONFIG_AUDIT_ARCH is not set
CONFIG_ARCH_SUPPORTS_AOUT=y
CONFIG_ARCH_SUPPORTS_OPTIMIZED_INLINING=y
CONFIG_GENERIC_HARDIRQS=y
CONFIG_GENERIC_IRQ_PROBE=y
CONFIG_GENERIC_PENDING_IRQ=y
CONFIG_X86_SMP=y
CONFIG_X86_32_SMP=y
CONFIG_X86_HT=y
CONFIG_X86_BIOS_REBOOT=y
CONFIG_X86_TRAMPOLINE=y
CONFIG_KTIME_SCALAR=y
CONFIG_DEFCONFIG_LIST="/lib/modules/$UNAME_RELEASE/.config"

#
# General setup
#
CONFIG_EXPERIMENTAL=y
CONFIG_LOCK_KERNEL=y
CONFIG_INIT_ENV_ARG_LIMIT=32
CONFIG_LOCALVERSION=""
# CONFIG_LOCALVERSION_AUTO is not set
CONFIG_SWAP=y
CONFIG_SYSVIPC=y
CONFIG_SYSVIPC_SYSCTL=y
CONFIG_POSIX_MQUEUE=y
# CONFIG_BSD_PROCESS_ACCT is not set
# CONFIG_TASKSTATS is not set
# CONFIG_AUDIT is not set
# CONFIG_IKCONFIG is not set
CONFIG_LOG_BUF_SHIFT=17
# CONFIG_CGROUPS is not set
CONFIG_HAVE_UNSTABLE_SCHED_CLOCK=y
# CONFIG_GROUP_SCHED is not set
CONFIG_SYSFS_DEPRECATED=y
CONFIG_SYSFS_DEPRECATED_V2=y
# CONFIG_RELAY is not set
CONFIG_NAMESPACES=y
CONFIG_UTS_NS=y
CONFIG_IPC_NS=y
CONFIG_USER_NS=y
CONFIG_PID_NS=y
# CONFIG_BLK_DEV_INITRD is not set
CONFIG_CC_OPTIMIZE_FOR_SIZE=y
CONFIG_SYSCTL=y
# CONFIG_EMBEDDED is not set
CONFIG_UID16=y
CONFIG_SYSCTL_SYSCALL=y
CONFIG_KALLSYMS=y
CONFIG_KALLSYMS_ALL=y
# CONFIG_KALLSYMS_EXTRA_PASS is not set
CONFIG_HOTPLUG=y
CONFIG_PRINTK=y
CONFIG_BUG=y
CONFIG_ELF_CORE=y
CONFIG_PCSPKR_PLATFORM=y
# CONFIG_COMPAT_BRK is not set
CONFIG_BASE_FULL=y
CONFIG_FUTEX=y
CONFIG_ANON_INODES=y
CONFIG_EPOLL=y
CONFIG_SIGNALFD=y
CONFIG_TIMERFD=y
CONFIG_EVENTFD=y
CONFIG_SHMEM=y
CONFIG_VM_EVENT_COUNTERS=y
CONFIG_SLUB_DEBUG=y
# CONFIG_SLAB is not set
CONFIG_SLUB=y
# CONFIG_SLOB is not set
CONFIG_PROFILING=y
# CONFIG_MARKERS is not set
CONFIG_OPROFILE=y
CONFIG_HAVE_OPROFILE=y
# CONFIG_KPROBES is not set
CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS=y
CONFIG_HAVE_IOREMAP_PROT=y
CONFIG_HAVE_KPROBES=y
CONFIG_HAVE_KRETPROBES=y
# CONFIG_HAVE_ARCH_TRACEHOOK is not set
# CONFIG_HAVE_DMA_ATTRS is not set
CONFIG_USE_GENERIC_SMP_HELPERS=y
# CONFIG_HAVE_CLK is not set
CONFIG_PROC_PAGE_MONITOR=y
CONFIG_HAVE_GENERIC_DMA_COHERENT=y
CONFIG_SLABINFO=y
CONFIG_RT_MUTEXES=y
# CONFIG_TINY_SHMEM is not set
CONFIG_BASE_SMALL=0
CONFIG_MODULES=y
# CONFIG_MODULE_FORCE_LOAD is not set
CONFIG_MODULE_UNLOAD=y
# CONFIG_MODULE_FORCE_UNLOAD is not set
# CONFIG_MODVERSIONS is not set
CONFIG_MODULE_SRCVERSION_ALL=y
CONFIG_KMOD=y
CONFIG_STOP_MACHINE=y
CONFIG_BLOCK=y
CONFIG_LBD=y
# CONFIG_BLK_DEV_IO_TRACE is not set
CONFIG_LSF=y
CONFIG_BLK_DEV_BSG=y
CONFIG_BLK_DEV_INTEGRITY=y

#
# IO Schedulers
#
CONFIG_IOSCHED_NOOP=y
# CONFIG_IOSCHED_AS is not set
CONFIG_IOSCHED_DEADLINE=y
CONFIG_IOSCHED_CFQ=y
# CONFIG_DEFAULT_AS is not set
# CONFIG_DEFAULT_DEADLINE is not set
CONFIG_DEFAULT_CFQ=y
# CONFIG_DEFAULT_NOOP is not set
CONFIG_DEFAULT_IOSCHED="cfq"
CONFIG_CLASSIC_RCU=y

#
# Processor type and features
#
CONFIG_TICK_ONESHOT=y
CONFIG_NO_HZ=y
CONFIG_HIGH_RES_TIMERS=y
CONFIG_GENERIC_CLOCKEVENTS_BUILD=y
CONFIG_SMP=y
CONFIG_X86_FIND_SMP_CONFIG=y
CONFIG_X86_MPPARSE=y
CONFIG_X86_PC=y
# CONFIG_X86_ELAN is not set
# CONFIG_X86_VOYAGER is not set
# CONFIG_X86_GENERICARCH is not set
# CONFIG_X86_VSMP is not set
# CONFIG_X86_VISWS is not set
# CONFIG_X86_RDC321X is not set
CONFIG_SCHED_NO_NO_OMIT_FRAME_POINTER=y
# CONFIG_PARAVIRT_GUEST is not set
# CONFIG_MEMTEST is not set
# CONFIG_M386 is not set
# CONFIG_M486 is not set
# CONFIG_M586 is not set
# CONFIG_M586TSC is not set
# CONFIG_M586MMX is not set
# CONFIG_M686 is not set
# CONFIG_MPENTIUMII is not set
# CONFIG_MPENTIUMIII is not set
# CONFIG_MPENTIUMM is not set
CONFIG_MPENTIUM4=y
# CONFIG_MK6 is not set
# CONFIG_MK7 is not set
# CONFIG_MK8 is not set
# CONFIG_MCRUSOE is not set
# CONFIG_MEFFICEON is not set
# CONFIG_MWINCHIPC6 is not set
# CONFIG_MWINCHIP2 is not set
# CONFIG_MWINCHIP3D is not set
# CONFIG_MGEODEGX1 is not set
# CONFIG_MGEODE_LX is not set
# CONFIG_MCYRIXIII is not set
# CONFIG_MVIAC3_2 is not set
# CONFIG_MVIAC7 is not set
# CONFIG_MPSC is not set
# CONFIG_MCORE2 is not set
# CONFIG_GENERIC_CPU is not set
# CONFIG_X86_GENERIC is not set
CONFIG_X86_CPU=y
CONFIG_X86_CMPXCHG=y
CONFIG_X86_L1_CACHE_SHIFT=7
CONFIG_X86_XADD=y
CONFIG_X86_WP_WORKS_OK=y
CONFIG_X86_INVLPG=y
CONFIG_X86_BSWAP=y
CONFIG_X86_POPAD_OK=y
CONFIG_X86_INTEL_USERCOPY=y
CONFIG_X86_USE_PPRO_CHECKSUM=y
CONFIG_X86_TSC=y
CONFIG_X86_CMPXCHG64=y
CONFIG_X86_CMOV=y
CONFIG_X86_MINIMUM_CPU_FAMILY=4
CONFIG_X86_DEBUGCTLMSR=y
CONFIG_HPET_TIMER=y
CONFIG_HPET_EMULATE_RTC=y
CONFIG_DMI=y
# CONFIG_IOMMU_HELPER is not set
CONFIG_NR_CPUS=8
CONFIG_SCHED_SMT=y
CONFIG_SCHED_MC=y
CONFIG_PREEMPT_NONE=y
# CONFIG_PREEMPT_VOLUNTARY is not set
# CONFIG_PREEMPT is not set
CONFIG_X86_LOCAL_APIC=y
CONFIG_X86_IO_APIC=y
CONFIG_X86_MCE=y
CONFIG_X86_MCE_NONFATAL=y
CONFIG_X86_MCE_P4THERMAL=y
CONFIG_VM86=y
# CONFIG_TOSHIBA is not set
# CONFIG_I8K is not set
# CONFIG_X86_REBOOTFIXUPS is not set
# CONFIG_MICROCODE is not set
# CONFIG_X86_MSR is not set
# CONFIG_X86_CPUID is not set
# CONFIG_NOHIGHMEM is not set
# CONFIG_HIGHMEM4G is not set
CONFIG_HIGHMEM64G=y
CONFIG_PAGE_OFFSET=0xC0000000
CONFIG_HIGHMEM=y
CONFIG_X86_PAE=y
CONFIG_ARCH_FLATMEM_ENABLE=y
CONFIG_ARCH_SPARSEMEM_ENABLE=y
CONFIG_ARCH_SELECT_MEMORY_MODEL=y
CONFIG_SELECT_MEMORY_MODEL=y
CONFIG_FLATMEM_MANUAL=y
# CONFIG_DISCONTIGMEM_MANUAL is not set
# CONFIG_SPARSEMEM_MANUAL is not set
CONFIG_FLATMEM=y
CONFIG_FLAT_NODE_MEM_MAP=y
CONFIG_SPARSEMEM_STATIC=y
# CONFIG_SPARSEMEM_VMEMMAP_ENABLE is not set
CONFIG_PAGEFLAGS_EXTENDED=y
CONFIG_SPLIT_PTLOCK_CPUS=4
CONFIG_RESOURCES_64BIT=y
CONFIG_ZONE_DMA_FLAG=1
CONFIG_BOUNCE=y
CONFIG_VIRT_TO_BUS=y
CONFIG_HIGHPTE=y
# CONFIG_MATH_EMULATION is not set
CONFIG_MTRR=y
# CONFIG_MTRR_SANITIZER is not set
CONFIG_X86_PAT=y
# CONFIG_EFI is not set
CONFIG_IRQBALANCE=y
# CONFIG_SECCOMP is not set
# CONFIG_HZ_100 is not set
CONFIG_HZ_250=y
# CONFIG_HZ_300 is not set
# CONFIG_HZ_1000 is not set
CONFIG_HZ=250
CONFIG_SCHED_HRTICK=y
CONFIG_KEXEC=y
CONFIG_CRASH_DUMP=y
CONFIG_PHYSICAL_START=0x100000
CONFIG_RELOCATABLE=y
CONFIG_PHYSICAL_ALIGN=0x100000
# CONFIG_HOTPLUG_CPU is not set
# CONFIG_COMPAT_VDSO is not set
CONFIG_ARCH_ENABLE_MEMORY_HOTPLUG=y

#
# Power management options
#
CONFIG_PM=y
# CONFIG_PM_DEBUG is not set
# CONFIG_SUSPEND is not set
# CONFIG_HIBERNATION is not set
CONFIG_ACPI=y
# CONFIG_ACPI_PROCFS is not set
# CONFIG_ACPI_PROCFS_POWER is not set
CONFIG_ACPI_SYSFS_POWER=y
CONFIG_ACPI_PROC_EVENT=y
# CONFIG_ACPI_AC is not set
# CONFIG_ACPI_BATTERY is not set
CONFIG_ACPI_BUTTON=y
CONFIG_ACPI_FAN=y
CONFIG_ACPI_DOCK=y
# CONFIG_ACPI_BAY is not set
CONFIG_ACPI_PROCESSOR=y
CONFIG_ACPI_THERMAL=y
CONFIG_ACPI_WMI=y
# CONFIG_ACPI_ASUS is not set
# CONFIG_ACPI_TOSHIBA is not set
# CONFIG_ACPI_CUSTOM_DSDT is not set
CONFIG_ACPI_BLACKLIST_YEAR=0
# CONFIG_ACPI_DEBUG is not set
CONFIG_ACPI_EC=y
CONFIG_ACPI_PCI_SLOT=y
CONFIG_ACPI_POWER=y
CONFIG_ACPI_SYSTEM=y
CONFIG_X86_PM_TIMER=y
CONFIG_ACPI_CONTAINER=y
# CONFIG_ACPI_SBS is not set

#
# CPU Frequency scaling
#
CONFIG_CPU_FREQ=y
CONFIG_CPU_FREQ_TABLE=y
# CONFIG_CPU_FREQ_DEBUG is not set
CONFIG_CPU_FREQ_STAT=y
# CONFIG_CPU_FREQ_STAT_DETAILS is not set
CONFIG_CPU_FREQ_DEFAULT_GOV_PERFORMANCE=y
# CONFIG_CPU_FREQ_DEFAULT_GOV_POWERSAVE is not set
# CONFIG_CPU_FREQ_DEFAULT_GOV_USERSPACE is not set
# CONFIG_CPU_FREQ_DEFAULT_GOV_ONDEMAND is not set
# CONFIG_CPU_FREQ_DEFAULT_GOV_CONSERVATIVE is not set
CONFIG_CPU_FREQ_GOV_PERFORMANCE=y
CONFIG_CPU_FREQ_GOV_POWERSAVE=y
CONFIG_CPU_FREQ_GOV_USERSPACE=y
CONFIG_CPU_FREQ_GOV_ONDEMAND=y
# CONFIG_CPU_FREQ_GOV_CONSERVATIVE is not set

#
# CPUFreq processor drivers
#
CONFIG_X86_ACPI_CPUFREQ=y
# CONFIG_X86_POWERNOW_K6 is not set
# CONFIG_X86_POWERNOW_K7 is not set
# CONFIG_X86_POWERNOW_K8 is not set
# CONFIG_X86_GX_SUSPMOD is not set
# CONFIG_X86_SPEEDSTEP_CENTRINO is not set
# CONFIG_X86_SPEEDSTEP_ICH is not set
# CONFIG_X86_SPEEDSTEP_SMI is not set
CONFIG_X86_P4_CLOCKMOD=y
# CONFIG_X86_CPUFREQ_NFORCE2 is not set
# CONFIG_X86_LONGRUN is not set
# CONFIG_X86_LONGHAUL is not set
# CONFIG_X86_E_POWERSAVER is not set

#
# shared options
#
# CONFIG_X86_ACPI_CPUFREQ_PROC_INTF is not set
CONFIG_X86_SPEEDSTEP_LIB=y
CONFIG_CPU_IDLE=y
CONFIG_CPU_IDLE_GOV_LADDER=y
CONFIG_CPU_IDLE_GOV_MENU=y

#
# Bus options (PCI etc.)
#
CONFIG_PCI=y
# CONFIG_PCI_GOBIOS is not set
# CONFIG_PCI_GOMMCONFIG is not set
CONFIG_PCI_GODIRECT=y
# CONFIG_PCI_GOOLPC is not set
# CONFIG_PCI_GOANY is not set
CONFIG_PCI_DIRECT=y
CONFIG_PCI_DOMAINS=y
CONFIG_PCIEPORTBUS=y
CONFIG_PCIEAER=y
# CONFIG_PCIEASPM is not set
CONFIG_ARCH_SUPPORTS_MSI=y
CONFIG_PCI_MSI=y
# CONFIG_PCI_LEGACY is not set
# CONFIG_PCI_DEBUG is not set
# CONFIG_HT_IRQ is not set
CONFIG_ISA_DMA_API=y
# CONFIG_ISA is not set
# CONFIG_MCA is not set
# CONFIG_SCx200 is not set
# CONFIG_OLPC is not set
# CONFIG_PCCARD is not set
# CONFIG_HOTPLUG_PCI is not set

#
# Executable file formats / Emulations
#
CONFIG_BINFMT_ELF=y
# CONFIG_BINFMT_AOUT is not set
# CONFIG_BINFMT_MISC is not set
CONFIG_NET=y

#
# Networking options
#
CONFIG_PACKET=y
CONFIG_PACKET_MMAP=y
CONFIG_UNIX=y
# CONFIG_NET_KEY is not set
CONFIG_INET=y
CONFIG_IP_MULTICAST=y
# CONFIG_IP_ADVANCED_ROUTER is not set
CONFIG_IP_FIB_HASH=y
# CONFIG_IP_PNP is not set
# CONFIG_NET_IPIP is not set
# CONFIG_NET_IPGRE is not set
# CONFIG_IP_MROUTE is not set
# CONFIG_ARPD is not set
CONFIG_SYN_COOKIES=y
# CONFIG_INET_AH is not set
# CONFIG_INET_ESP is not set
# CONFIG_INET_IPCOMP is not set
# CONFIG_INET_XFRM_TUNNEL is not set
# CONFIG_INET_TUNNEL is not set
# CONFIG_INET_XFRM_MODE_TRANSPORT is not set
# CONFIG_INET_XFRM_MODE_TUNNEL is not set
# CONFIG_INET_XFRM_MODE_BEET is not set
CONFIG_INET_LRO=y
CONFIG_INET_DIAG=y
CONFIG_INET_TCP_DIAG=y
CONFIG_TCP_CONG_ADVANCED=y
# CONFIG_TCP_CONG_BIC is not set
# CONFIG_TCP_CONG_CUBIC is not set
# CONFIG_TCP_CONG_WESTWOOD is not set
# CONFIG_TCP_CONG_HTCP is not set
# CONFIG_TCP_CONG_HSTCP is not set
# CONFIG_TCP_CONG_HYBLA is not set
CONFIG_TCP_CONG_VEGAS=y
# CONFIG_TCP_CONG_SCALABLE is not set
# CONFIG_TCP_CONG_LP is not set
# CONFIG_TCP_CONG_VENO is not set
# CONFIG_TCP_CONG_YEAH is not set
# CONFIG_TCP_CONG_ILLINOIS is not set
# CONFIG_DEFAULT_BIC is not set
# CONFIG_DEFAULT_CUBIC is not set
# CONFIG_DEFAULT_HTCP is not set
CONFIG_DEFAULT_VEGAS=y
# CONFIG_DEFAULT_WESTWOOD is not set
# CONFIG_DEFAULT_RENO is not set
CONFIG_DEFAULT_TCP_CONG="vegas"
# CONFIG_TCP_MD5SIG is not set
# CONFIG_IP_VS is not set
# CONFIG_IPV6 is not set
# CONFIG_NETLABEL is not set
# CONFIG_NETWORK_SECMARK is not set
CONFIG_NETFILTER=y
# CONFIG_NETFILTER_DEBUG is not set
CONFIG_NETFILTER_ADVANCED=y

#
# Core Netfilter Configuration
#
CONFIG_NETFILTER_NETLINK=y
# CONFIG_NETFILTER_NETLINK_QUEUE is not set
CONFIG_NETFILTER_NETLINK_LOG=y
CONFIG_NF_CONNTRACK=y
CONFIG_NF_CT_ACCT=y
CONFIG_NF_CONNTRACK_MARK=y
CONFIG_NF_CONNTRACK_EVENTS=y
# CONFIG_NF_CT_PROTO_DCCP is not set
# CONFIG_NF_CT_PROTO_SCTP is not set
# CONFIG_NF_CT_PROTO_UDPLITE is not set
# CONFIG_NF_CONNTRACK_AMANDA is not set
CONFIG_NF_CONNTRACK_FTP=y
# CONFIG_NF_CONNTRACK_H323 is not set
# CONFIG_NF_CONNTRACK_IRC is not set
# CONFIG_NF_CONNTRACK_NETBIOS_NS is not set
# CONFIG_NF_CONNTRACK_PPTP is not set
# CONFIG_NF_CONNTRACK_SANE is not set
# CONFIG_NF_CONNTRACK_SIP is not set
# CONFIG_NF_CONNTRACK_TFTP is not set
CONFIG_NF_CT_NETLINK=y
CONFIG_NETFILTER_XTABLES=y
# CONFIG_NETFILTER_XT_TARGET_CLASSIFY is not set
CONFIG_NETFILTER_XT_TARGET_MARK=y
# CONFIG_NETFILTER_XT_TARGET_NFQUEUE is not set
CONFIG_NETFILTER_XT_TARGET_NFLOG=y
CONFIG_NETFILTER_XT_TARGET_NOTRACK=y
# CONFIG_NETFILTER_XT_TARGET_RATEEST is not set
CONFIG_NETFILTER_XT_TARGET_TRACE=y
# CONFIG_NETFILTER_XT_TARGET_TCPMSS is not set
CONFIG_NETFILTER_XT_MATCH_COMMENT=y
CONFIG_NETFILTER_XT_MATCH_CONNBYTES=y
# CONFIG_NETFILTER_XT_MATCH_CONNLIMIT is not set
CONFIG_NETFILTER_XT_MATCH_CONNMARK=y
CONFIG_NETFILTER_XT_MATCH_CONNTRACK=y
# CONFIG_NETFILTER_XT_MATCH_DCCP is not set
# CONFIG_NETFILTER_XT_MATCH_DSCP is not set
# CONFIG_NETFILTER_XT_MATCH_ESP is not set
CONFIG_NETFILTER_XT_MATCH_HELPER=y
CONFIG_NETFILTER_XT_MATCH_IPRANGE=y
CONFIG_NETFILTER_XT_MATCH_LENGTH=y
CONFIG_NETFILTER_XT_MATCH_LIMIT=y
CONFIG_NETFILTER_XT_MATCH_MAC=y
CONFIG_NETFILTER_XT_MATCH_MARK=y
CONFIG_NETFILTER_XT_MATCH_OWNER=y
CONFIG_NETFILTER_XT_MATCH_MULTIPORT=y
CONFIG_NETFILTER_XT_MATCH_PKTTYPE=y
# CONFIG_NETFILTER_XT_MATCH_QUOTA is not set
# CONFIG_NETFILTER_XT_MATCH_RATEEST is not set
CONFIG_NETFILTER_XT_MATCH_REALM=y
# CONFIG_NETFILTER_XT_MATCH_SCTP is not set
CONFIG_NETFILTER_XT_MATCH_STATE=y
# CONFIG_NETFILTER_XT_MATCH_STATISTIC is not set
CONFIG_NETFILTER_XT_MATCH_STRING=y
# CONFIG_NETFILTER_XT_MATCH_TCPMSS is not set
# CONFIG_NETFILTER_XT_MATCH_TIME is not set
# CONFIG_NETFILTER_XT_MATCH_U32 is not set
CONFIG_NETFILTER_XT_MATCH_HASHLIMIT=y

#
# IP: Netfilter Configuration
#
CONFIG_NF_CONNTRACK_IPV4=y
# CONFIG_NF_CONNTRACK_PROC_COMPAT is not set
# CONFIG_IP_NF_QUEUE is not set
CONFIG_IP_NF_IPTABLES=y
CONFIG_IP_NF_MATCH_RECENT=y
# CONFIG_IP_NF_MATCH_ECN is not set
# CONFIG_IP_NF_MATCH_AH is not set
# CONFIG_IP_NF_MATCH_TTL is not set
CONFIG_IP_NF_MATCH_ADDRTYPE=y
CONFIG_IP_NF_FILTER=y
CONFIG_IP_NF_TARGET_REJECT=y
CONFIG_IP_NF_TARGET_LOG=y
# CONFIG_IP_NF_TARGET_ULOG is not set
CONFIG_NF_NAT=y
CONFIG_NF_NAT_NEEDED=y
# CONFIG_IP_NF_TARGET_MASQUERADE is not set
CONFIG_IP_NF_TARGET_REDIRECT=y
# CONFIG_IP_NF_TARGET_NETMAP is not set
# CONFIG_NF_NAT_SNMP_BASIC is not set
CONFIG_NF_NAT_FTP=y
# CONFIG_NF_NAT_IRC is not set
# CONFIG_NF_NAT_TFTP is not set
# CONFIG_NF_NAT_AMANDA is not set
# CONFIG_NF_NAT_PPTP is not set
# CONFIG_NF_NAT_H323 is not set
# CONFIG_NF_NAT_SIP is not set
# CONFIG_IP_NF_MANGLE is not set
CONFIG_IP_NF_RAW=y
# CONFIG_IP_NF_SECURITY is not set
# CONFIG_IP_NF_ARPTABLES is not set
# CONFIG_IP_DCCP is not set
# CONFIG_IP_SCTP is not set
# CONFIG_TIPC is not set
# CONFIG_ATM is not set
# CONFIG_BRIDGE is not set
# CONFIG_VLAN_8021Q is not set
# CONFIG_DECNET is not set
# CONFIG_LLC2 is not set
# CONFIG_IPX is not set
# CONFIG_ATALK is not set
# CONFIG_X25 is not set
# CONFIG_LAPB is not set
# CONFIG_ECONET is not set
# CONFIG_WAN_ROUTER is not set
# CONFIG_NET_SCHED is not set
CONFIG_NET_CLS_ROUTE=y

#
# Network testing
#
# CONFIG_NET_PKTGEN is not set
# CONFIG_HAMRADIO is not set
# CONFIG_CAN is not set
# CONFIG_IRDA is not set
# CONFIG_BT is not set
# CONFIG_AF_RXRPC is not set

#
# Wireless
#
# CONFIG_CFG80211 is not set
# CONFIG_WIRELESS_EXT is not set
# CONFIG_MAC80211 is not set
# CONFIG_IEEE80211 is not set
# CONFIG_RFKILL is not set
# CONFIG_NET_9P is not set

#
# Device Drivers
#

#
# Generic Driver Options
#
CONFIG_UEVENT_HELPER_PATH="/sbin/hotplug"
CONFIG_STANDALONE=y
CONFIG_PREVENT_FIRMWARE_BUILD=y
CONFIG_FW_LOADER=y
CONFIG_FIRMWARE_IN_KERNEL=y
CONFIG_EXTRA_FIRMWARE=""
# CONFIG_DEBUG_DRIVER is not set
# CONFIG_DEBUG_DEVRES is not set
# CONFIG_SYS_HYPERVISOR is not set
CONFIG_CONNECTOR=y
CONFIG_PROC_EVENTS=y
# CONFIG_MTD is not set
# CONFIG_PARPORT is not set
CONFIG_PNP=y
# CONFIG_PNP_DEBUG is not set

#
# Protocols
#
CONFIG_PNPACPI=y
CONFIG_BLK_DEV=y
CONFIG_BLK_DEV_FD=y
# CONFIG_BLK_CPQ_DA is not set
CONFIG_BLK_CPQ_CISS_DA=y
# CONFIG_CISS_SCSI_TAPE is not set
# CONFIG_BLK_DEV_DAC960 is not set
# CONFIG_BLK_DEV_UMEM is not set
# CONFIG_BLK_DEV_COW_COMMON is not set
CONFIG_BLK_DEV_LOOP=y
# CONFIG_BLK_DEV_CRYPTOLOOP is not set
# CONFIG_BLK_DEV_NBD is not set
# CONFIG_BLK_DEV_SX8 is not set
# CONFIG_BLK_DEV_RAM is not set
# CONFIG_CDROM_PKTCDVD is not set
# CONFIG_ATA_OVER_ETH is not set
# CONFIG_BLK_DEV_HD is not set
# CONFIG_MISC_DEVICES is not set
CONFIG_HAVE_IDE=y
# CONFIG_IDE is not set

#
# SCSI device support
#
CONFIG_RAID_ATTRS=y
CONFIG_SCSI=y
CONFIG_SCSI_DMA=y
CONFIG_SCSI_TGT=y
CONFIG_SCSI_NETLINK=y
CONFIG_SCSI_PROC_FS=y

#
# SCSI support type (disk, tape, CD-ROM)
#
CONFIG_BLK_DEV_SD=y
# CONFIG_CHR_DEV_ST is not set
# CONFIG_CHR_DEV_OSST is not set
CONFIG_BLK_DEV_SR=y
# CONFIG_BLK_DEV_SR_VENDOR is not set
CONFIG_CHR_DEV_SG=y
# CONFIG_CHR_DEV_SCH is not set

#
# Some SCSI devices (e.g. CD jukebox) support multiple LUNs
#
CONFIG_SCSI_MULTI_LUN=y
CONFIG_SCSI_CONSTANTS=y
# CONFIG_SCSI_LOGGING is not set
CONFIG_SCSI_SCAN_ASYNC=y
CONFIG_SCSI_WAIT_SCAN=m

#
# SCSI Transports
#
CONFIG_SCSI_SPI_ATTRS=y
CONFIG_SCSI_FC_ATTRS=y
CONFIG_SCSI_FC_TGT_ATTRS=y
CONFIG_SCSI_ISCSI_ATTRS=y
CONFIG_SCSI_SAS_ATTRS=y
CONFIG_SCSI_SAS_LIBSAS=y
# CONFIG_SCSI_SAS_ATA is not set
CONFIG_SCSI_SAS_HOST_SMP=y
# CONFIG_SCSI_SAS_LIBSAS_DEBUG is not set
# CONFIG_SCSI_SRP_ATTRS is not set
CONFIG_SCSI_LOWLEVEL=y
# CONFIG_ISCSI_TCP is not set
# CONFIG_BLK_DEV_3W_XXXX_RAID is not set
# CONFIG_SCSI_3W_9XXX is not set
# CONFIG_SCSI_ACARD is not set
# CONFIG_SCSI_AACRAID is not set
# CONFIG_SCSI_AIC7XXX is not set
# CONFIG_SCSI_AIC7XXX_OLD is not set
# CONFIG_SCSI_AIC79XX is not set
# CONFIG_SCSI_AIC94XX is not set
# CONFIG_SCSI_DPT_I2O is not set
# CONFIG_SCSI_ADVANSYS is not set
# CONFIG_SCSI_ARCMSR is not set
# CONFIG_MEGARAID_NEWGEN is not set
# CONFIG_MEGARAID_LEGACY is not set
# CONFIG_MEGARAID_SAS is not set
# CONFIG_SCSI_HPTIOP is not set
# CONFIG_SCSI_BUSLOGIC is not set
# CONFIG_SCSI_DMX3191D is not set
# CONFIG_SCSI_EATA is not set
# CONFIG_SCSI_FUTURE_DOMAIN is not set
# CONFIG_SCSI_GDTH is not set
# CONFIG_SCSI_IPS is not set
# CONFIG_SCSI_INITIO is not set
# CONFIG_SCSI_INIA100 is not set
# CONFIG_SCSI_MVSAS is not set
# CONFIG_SCSI_STEX is not set
# CONFIG_SCSI_SYM53C8XX_2 is not set
# CONFIG_SCSI_IPR is not set
# CONFIG_SCSI_QLOGIC_1280 is not set
CONFIG_SCSI_QLA_FC=m
# CONFIG_SCSI_QLA_ISCSI is not set
# CONFIG_SCSI_LPFC is not set
# CONFIG_SCSI_DC395x is not set
# CONFIG_SCSI_DC390T is not set
# CONFIG_SCSI_NSP32 is not set
# CONFIG_SCSI_DEBUG is not set
# CONFIG_SCSI_SRP is not set
# CONFIG_SCSI_DH is not set
CONFIG_ATA=y
# CONFIG_ATA_NONSTANDARD is not set
CONFIG_ATA_ACPI=y
# CONFIG_SATA_PMP is not set
# CONFIG_SATA_AHCI is not set
# CONFIG_SATA_SIL24 is not set
CONFIG_ATA_SFF=y
# CONFIG_SATA_SVW is not set
# CONFIG_ATA_PIIX is not set
# CONFIG_SATA_MV is not set
# CONFIG_SATA_NV is not set
# CONFIG_PDC_ADMA is not set
# CONFIG_SATA_QSTOR is not set
# CONFIG_SATA_PROMISE is not set
# CONFIG_SATA_SX4 is not set
# CONFIG_SATA_SIL is not set
# CONFIG_SATA_SIS is not set
# CONFIG_SATA_ULI is not set
# CONFIG_SATA_VIA is not set
# CONFIG_SATA_VITESSE is not set
# CONFIG_SATA_INIC162X is not set
CONFIG_PATA_ACPI=y
# CONFIG_PATA_ALI is not set
# CONFIG_PATA_AMD is not set
# CONFIG_PATA_ARTOP is not set
# CONFIG_PATA_ATIIXP is not set
# CONFIG_PATA_CMD640_PCI is not set
# CONFIG_PATA_CMD64X is not set
# CONFIG_PATA_CS5520 is not set
# CONFIG_PATA_CS5530 is not set
# CONFIG_PATA_CS5535 is not set
# CONFIG_PATA_CS5536 is not set
# CONFIG_PATA_CYPRESS is not set
# CONFIG_PATA_EFAR is not set
# CONFIG_ATA_GENERIC is not set
# CONFIG_PATA_HPT366 is not set
# CONFIG_PATA_HPT37X is not set
# CONFIG_PATA_HPT3X2N is not set
# CONFIG_PATA_HPT3X3 is not set
# CONFIG_PATA_IT821X is not set
# CONFIG_PATA_IT8213 is not set
# CONFIG_PATA_JMICRON is not set
# CONFIG_PATA_TRIFLEX is not set
# CONFIG_PATA_MARVELL is not set
# CONFIG_PATA_MPIIX is not set
# CONFIG_PATA_OLDPIIX is not set
# CONFIG_PATA_NETCELL is not set
# CONFIG_PATA_NINJA32 is not set
# CONFIG_PATA_NS87410 is not set
# CONFIG_PATA_NS87415 is not set
# CONFIG_PATA_OPTI is not set
# CONFIG_PATA_OPTIDMA is not set
# CONFIG_PATA_PDC_OLD is not set
# CONFIG_PATA_RADISYS is not set
# CONFIG_PATA_RZ1000 is not set
# CONFIG_PATA_SC1200 is not set
CONFIG_PATA_SERVERWORKS=y
# CONFIG_PATA_PDC2027X is not set
# CONFIG_PATA_SIL680 is not set
# CONFIG_PATA_SIS is not set
# CONFIG_PATA_VIA is not set
# CONFIG_PATA_WINBOND is not set
# CONFIG_PATA_SCH is not set
CONFIG_MD=y
CONFIG_BLK_DEV_MD=m
CONFIG_MD_LINEAR=m
CONFIG_MD_RAID0=m
CONFIG_MD_RAID1=m
CONFIG_MD_RAID10=m
# CONFIG_MD_RAID456 is not set
CONFIG_MD_MULTIPATH=m
# CONFIG_MD_FAULTY is not set
CONFIG_BLK_DEV_DM=m
# CONFIG_DM_DEBUG is not set
CONFIG_DM_CRYPT=m
CONFIG_DM_SNAPSHOT=m
CONFIG_DM_MIRROR=m
CONFIG_DM_ZERO=m
CONFIG_DM_MULTIPATH=m
# CONFIG_DM_DELAY is not set
CONFIG_DM_UEVENT=y
# CONFIG_FUSION is not set

#
# IEEE 1394 (FireWire) support
#

#
# Enable only one of the two stacks, unless you know what you are doing
#
# CONFIG_FIREWIRE is not set
# CONFIG_IEEE1394 is not set
# CONFIG_I2O is not set
# CONFIG_MACINTOSH_DRIVERS is not set
CONFIG_NETDEVICES=y
CONFIG_DUMMY=y
# CONFIG_BONDING is not set
# CONFIG_MACVLAN is not set
# CONFIG_EQUALIZER is not set
CONFIG_TUN=y
CONFIG_VETH=y
# CONFIG_NET_SB1000 is not set
# CONFIG_ARCNET is not set
CONFIG_PHYLIB=y

#
# MII PHY device drivers
#
CONFIG_MARVELL_PHY=y
CONFIG_DAVICOM_PHY=y
CONFIG_QSEMI_PHY=y
CONFIG_LXT_PHY=y
CONFIG_CICADA_PHY=y
CONFIG_VITESSE_PHY=y
CONFIG_SMSC_PHY=y
CONFIG_BROADCOM_PHY=y
CONFIG_ICPLUS_PHY=y
# CONFIG_REALTEK_PHY is not set
# CONFIG_FIXED_PHY is not set
# CONFIG_MDIO_BITBANG is not set
CONFIG_NET_ETHERNET=y
CONFIG_MII=y
# CONFIG_HAPPYMEAL is not set
# CONFIG_SUNGEM is not set
# CONFIG_CASSINI is not set
# CONFIG_NET_VENDOR_3COM is not set
# CONFIG_NET_TULIP is not set
# CONFIG_HP100 is not set
# CONFIG_IBM_NEW_EMAC_ZMII is not set
# CONFIG_IBM_NEW_EMAC_RGMII is not set
# CONFIG_IBM_NEW_EMAC_TAH is not set
# CONFIG_IBM_NEW_EMAC_EMAC4 is not set
CONFIG_NET_PCI=y
# CONFIG_PCNET32 is not set
# CONFIG_AMD8111_ETH is not set
# CONFIG_ADAPTEC_STARFIRE is not set
# CONFIG_B44 is not set
# CONFIG_FORCEDETH is not set
# CONFIG_EEPRO100 is not set
CONFIG_E100=m
# CONFIG_FEALNX is not set
# CONFIG_NATSEMI is not set
# CONFIG_NE2K_PCI is not set
# CONFIG_8139CP is not set
# CONFIG_8139TOO is not set
# CONFIG_R6040 is not set
# CONFIG_SIS900 is not set
# CONFIG_EPIC100 is not set
# CONFIG_SUNDANCE is not set
# CONFIG_TLAN is not set
# CONFIG_VIA_RHINE is not set
# CONFIG_SC92031 is not set
CONFIG_NETDEV_1000=y
# CONFIG_ACENIC is not set
# CONFIG_DL2K is not set
# CONFIG_E1000 is not set
# CONFIG_E1000E is not set
# CONFIG_IP1000 is not set
# CONFIG_IGB is not set
# CONFIG_NS83820 is not set
# CONFIG_HAMACHI is not set
# CONFIG_YELLOWFIN is not set
# CONFIG_R8169 is not set
# CONFIG_SIS190 is not set
# CONFIG_SKGE is not set
# CONFIG_SKY2 is not set
# CONFIG_VIA_VELOCITY is not set
CONFIG_TIGON3=m
CONFIG_BNX2=m
# CONFIG_QLA3XXX is not set
# CONFIG_ATL1 is not set
# CONFIG_ATL1E is not set
# CONFIG_NETDEV_10000 is not set
# CONFIG_TR is not set

#
# Wireless LAN
#
# CONFIG_WLAN_PRE80211 is not set
# CONFIG_WLAN_80211 is not set
# CONFIG_IWLWIFI_LEDS is not set
# CONFIG_WAN is not set
# CONFIG_FDDI is not set
# CONFIG_HIPPI is not set
# CONFIG_PPP is not set
# CONFIG_SLIP is not set
# CONFIG_NET_FC is not set
# CONFIG_NETCONSOLE is not set
# CONFIG_NETPOLL is not set
# CONFIG_NET_POLL_CONTROLLER is not set
# CONFIG_ISDN is not set
# CONFIG_PHONE is not set

#
# Input device support
#
CONFIG_INPUT=y
# CONFIG_INPUT_FF_MEMLESS is not set
CONFIG_INPUT_POLLDEV=y

#
# Userland interfaces
#
CONFIG_INPUT_MOUSEDEV=y
# CONFIG_INPUT_MOUSEDEV_PSAUX is not set
CONFIG_INPUT_MOUSEDEV_SCREEN_X=1024
CONFIG_INPUT_MOUSEDEV_SCREEN_Y=768
# CONFIG_INPUT_JOYDEV is not set
# CONFIG_INPUT_EVDEV is not set
# CONFIG_INPUT_EVBUG is not set

#
# Input Device Drivers
#
CONFIG_INPUT_KEYBOARD=y
CONFIG_KEYBOARD_ATKBD=y
# CONFIG_KEYBOARD_SUNKBD is not set
# CONFIG_KEYBOARD_LKKBD is not set
# CONFIG_KEYBOARD_XTKBD is not set
# CONFIG_KEYBOARD_NEWTON is not set
# CONFIG_KEYBOARD_STOWAWAY is not set
# CONFIG_INPUT_MOUSE is not set
# CONFIG_INPUT_JOYSTICK is not set
# CONFIG_INPUT_TABLET is not set
# CONFIG_INPUT_TOUCHSCREEN is not set
# CONFIG_INPUT_MISC is not set

#
# Hardware I/O ports
#
CONFIG_SERIO=y
CONFIG_SERIO_I8042=y
# CONFIG_SERIO_SERPORT is not set
# CONFIG_SERIO_CT82C710 is not set
# CONFIG_SERIO_PCIPS2 is not set
CONFIG_SERIO_LIBPS2=y
# CONFIG_SERIO_RAW is not set
# CONFIG_GAMEPORT is not set

#
# Character devices
#
CONFIG_VT=y
CONFIG_CONSOLE_TRANSLATIONS=y
CONFIG_VT_CONSOLE=y
CONFIG_HW_CONSOLE=y
CONFIG_VT_HW_CONSOLE_BINDING=y
# CONFIG_DEVKMEM is not set
# CONFIG_SERIAL_NONSTANDARD is not set
# CONFIG_NOZOMI is not set

#
# Serial drivers
#
CONFIG_SERIAL_8250=y
CONFIG_SERIAL_8250_CONSOLE=y
CONFIG_FIX_EARLYCON_MEM=y
CONFIG_SERIAL_8250_PCI=y
CONFIG_SERIAL_8250_PNP=y
CONFIG_SERIAL_8250_NR_UARTS=4
CONFIG_SERIAL_8250_RUNTIME_UARTS=4
# CONFIG_SERIAL_8250_EXTENDED is not set

#
# Non-8250 serial port support
#
CONFIG_SERIAL_CORE=y
CONFIG_SERIAL_CORE_CONSOLE=y
CONFIG_CONSOLE_POLL=y
# CONFIG_SERIAL_JSM is not set
CONFIG_UNIX98_PTYS=y
# CONFIG_LEGACY_PTYS is not set
# CONFIG_IPMI_HANDLER is not set
CONFIG_HW_RANDOM=y
CONFIG_HW_RANDOM_INTEL=y
# CONFIG_HW_RANDOM_AMD is not set
# CONFIG_HW_RANDOM_GEODE is not set
CONFIG_HW_RANDOM_VIA=y
# CONFIG_NVRAM is not set
CONFIG_RTC=y
# CONFIG_R3964 is not set
# CONFIG_APPLICOM is not set
# CONFIG_SONYPI is not set
# CONFIG_MWAVE is not set
# CONFIG_PC8736x_GPIO is not set
# CONFIG_NSC_GPIO is not set
# CONFIG_CS5535_GPIO is not set
# CONFIG_RAW_DRIVER is not set
CONFIG_HPET=y
CONFIG_HPET_RTC_IRQ=y
CONFIG_HPET_MMAP=y
# CONFIG_HANGCHECK_TIMER is not set
# CONFIG_TCG_TPM is not set
# CONFIG_TELCLOCK is not set
CONFIG_DEVPORT=y
CONFIG_I2C=y
CONFIG_I2C_BOARDINFO=y
CONFIG_I2C_CHARDEV=y
CONFIG_I2C_HELPER_AUTO=y

#
# I2C Hardware Bus support
#

#
# PC SMBus host controller drivers
#
# CONFIG_I2C_ALI1535 is not set
# CONFIG_I2C_ALI1563 is not set
# CONFIG_I2C_ALI15X3 is not set
# CONFIG_I2C_AMD756 is not set
# CONFIG_I2C_AMD8111 is not set
# CONFIG_I2C_I801 is not set
CONFIG_I2C_ISCH=y
CONFIG_I2C_PIIX4=y
# CONFIG_I2C_NFORCE2 is not set
# CONFIG_I2C_SIS5595 is not set
# CONFIG_I2C_SIS630 is not set
# CONFIG_I2C_SIS96X is not set
# CONFIG_I2C_VIA is not set
# CONFIG_I2C_VIAPRO is not set

#
# I2C system bus drivers (mostly embedded / system-on-chip)
#
CONFIG_I2C_OCORES=y
# CONFIG_I2C_SIMTEC is not set

#
# External I2C/SMBus adapter drivers
#
# CONFIG_I2C_PARPORT_LIGHT is not set
# CONFIG_I2C_TAOS_EVM is not set

#
# Graphics adapter I2C/DDC channel drivers
#
# CONFIG_I2C_VOODOO3 is not set

#
# Other I2C/SMBus bus drivers
#
# CONFIG_I2C_PCA_PLATFORM is not set
# CONFIG_I2C_STUB is not set
# CONFIG_SCx200_ACB is not set

#
# Miscellaneous I2C Chip support
#
# CONFIG_DS1682 is not set
CONFIG_AT24=y
CONFIG_SENSORS_EEPROM=y
# CONFIG_SENSORS_PCF8574 is not set
# CONFIG_PCF8575 is not set
# CONFIG_SENSORS_PCA9539 is not set
# CONFIG_SENSORS_PCF8591 is not set
# CONFIG_SENSORS_MAX6875 is not set
# CONFIG_SENSORS_TSL2550 is not set
# CONFIG_I2C_DEBUG_CORE is not set
# CONFIG_I2C_DEBUG_ALGO is not set
# CONFIG_I2C_DEBUG_BUS is not set
# CONFIG_I2C_DEBUG_CHIP is not set
# CONFIG_SPI is not set
CONFIG_ARCH_WANT_OPTIONAL_GPIOLIB=y
# CONFIG_GPIOLIB is not set
# CONFIG_W1 is not set
CONFIG_POWER_SUPPLY=y
# CONFIG_POWER_SUPPLY_DEBUG is not set
# CONFIG_PDA_POWER is not set
# CONFIG_BATTERY_DS2760 is not set
CONFIG_HWMON=y
CONFIG_HWMON_VID=y
CONFIG_SENSORS_ABITUGURU=y
# CONFIG_SENSORS_ABITUGURU3 is not set
# CONFIG_SENSORS_AD7414 is not set
CONFIG_SENSORS_AD7418=y
CONFIG_SENSORS_ADM1021=y
CONFIG_SENSORS_ADM1025=y
CONFIG_SENSORS_ADM1026=y
CONFIG_SENSORS_ADM1029=y
CONFIG_SENSORS_ADM1031=y
CONFIG_SENSORS_ADM9240=y
# CONFIG_SENSORS_ADT7470 is not set
# CONFIG_SENSORS_ADT7473 is not set
CONFIG_SENSORS_K8TEMP=y
CONFIG_SENSORS_ASB100=y
CONFIG_SENSORS_ATXP1=y
CONFIG_SENSORS_DS1621=y
# CONFIG_SENSORS_I5K_AMB is not set
CONFIG_SENSORS_F71805F=y
# CONFIG_SENSORS_F71882FG is not set
# CONFIG_SENSORS_F75375S is not set
CONFIG_SENSORS_FSCHER=y
CONFIG_SENSORS_FSCPOS=y
# CONFIG_SENSORS_FSCHMD is not set
CONFIG_SENSORS_GL518SM=y
CONFIG_SENSORS_GL520SM=y
# CONFIG_SENSORS_CORETEMP is not set
CONFIG_SENSORS_IT87=y
CONFIG_SENSORS_LM63=y
CONFIG_SENSORS_LM75=y
CONFIG_SENSORS_LM77=y
CONFIG_SENSORS_LM78=y
CONFIG_SENSORS_LM80=y
CONFIG_SENSORS_LM83=y
CONFIG_SENSORS_LM85=y
CONFIG_SENSORS_LM87=y
CONFIG_SENSORS_LM90=y
CONFIG_SENSORS_LM92=y
# CONFIG_SENSORS_LM93 is not set
CONFIG_SENSORS_MAX1619=y
CONFIG_SENSORS_MAX6650=y
CONFIG_SENSORS_PC87360=y
CONFIG_SENSORS_PC87427=y
CONFIG_SENSORS_SIS5595=y
# CONFIG_SENSORS_DME1737 is not set
CONFIG_SENSORS_SMSC47M1=y
CONFIG_SENSORS_SMSC47M192=y
CONFIG_SENSORS_SMSC47B397=y
# CONFIG_SENSORS_ADS7828 is not set
# CONFIG_SENSORS_THMC50 is not set
CONFIG_SENSORS_VIA686A=y
CONFIG_SENSORS_VT1211=y
CONFIG_SENSORS_VT8231=y
CONFIG_SENSORS_W83781D=y
CONFIG_SENSORS_W83791D=y
CONFIG_SENSORS_W83792D=y
CONFIG_SENSORS_W83793=y
CONFIG_SENSORS_W83L785TS=y
# CONFIG_SENSORS_W83L786NG is not set
CONFIG_SENSORS_W83627HF=y
CONFIG_SENSORS_W83627EHF=y
CONFIG_SENSORS_HDAPS=y
# CONFIG_SENSORS_APPLESMC is not set
# CONFIG_HWMON_DEBUG_CHIP is not set
CONFIG_THERMAL=y
# CONFIG_THERMAL_HWMON is not set
# CONFIG_WATCHDOG is not set

#
# Sonics Silicon Backplane
#
CONFIG_SSB_POSSIBLE=y
# CONFIG_SSB is not set

#
# Multifunction device drivers
#
# CONFIG_MFD_CORE is not set
# CONFIG_MFD_SM501 is not set
# CONFIG_HTC_PASIC3 is not set
# CONFIG_MFD_TMIO is not set

#
# Multimedia devices
#

#
# Multimedia core support
#
# CONFIG_VIDEO_DEV is not set
# CONFIG_DVB_CORE is not set
# CONFIG_VIDEO_MEDIA is not set

#
# Multimedia drivers
#
CONFIG_DAB=y

#
# Graphics support
#
# CONFIG_AGP is not set
# CONFIG_DRM is not set
# CONFIG_VGASTATE is not set
# CONFIG_VIDEO_OUTPUT_CONTROL is not set
# CONFIG_FB is not set
# CONFIG_BACKLIGHT_LCD_SUPPORT is not set

#
# Display device support
#
# CONFIG_DISPLAY_SUPPORT is not set

#
# Console display driver support
#
CONFIG_VGA_CONSOLE=y
# CONFIG_VGACON_SOFT_SCROLLBACK is not set
CONFIG_VIDEO_SELECT=y
CONFIG_DUMMY_CONSOLE=y
# CONFIG_SOUND is not set
# CONFIG_HID_SUPPORT is not set
# CONFIG_USB_SUPPORT is not set
# CONFIG_MMC is not set
# CONFIG_MEMSTICK is not set
# CONFIG_NEW_LEDS is not set
# CONFIG_ACCESSIBILITY is not set
# CONFIG_INFINIBAND is not set
CONFIG_EDAC=y

#
# Reporting subsystems
#
# CONFIG_EDAC_DEBUG is not set
CONFIG_EDAC_MM_EDAC=y
# CONFIG_EDAC_AMD76X is not set
CONFIG_EDAC_E7XXX=y
CONFIG_EDAC_E752X=y
CONFIG_EDAC_I82875P=y
CONFIG_EDAC_I82975X=y
CONFIG_EDAC_I3000=y
CONFIG_EDAC_I82860=y
# CONFIG_EDAC_R82600 is not set
CONFIG_EDAC_I5000=y
# CONFIG_EDAC_I5100 is not set
# CONFIG_RTC_CLASS is not set
# CONFIG_UIO is not set

#
# Firmware Drivers
#
# CONFIG_EDD is not set
CONFIG_FIRMWARE_MEMMAP=y
# CONFIG_DELL_RBU is not set
# CONFIG_DCDBAS is not set
CONFIG_DMIID=y
# CONFIG_ISCSI_IBFT_FIND is not set

#
# File systems
#
CONFIG_EXT2_FS=y
# CONFIG_EXT2_FS_XATTR is not set
# CONFIG_EXT2_FS_XIP is not set
CONFIG_EXT3_FS=y
CONFIG_EXT3_FS_XATTR=y
CONFIG_EXT3_FS_POSIX_ACL=y
# CONFIG_EXT3_FS_SECURITY is not set
# CONFIG_EXT4DEV_FS is not set
CONFIG_JBD=y
# CONFIG_JBD_DEBUG is not set
CONFIG_FS_MBCACHE=y
CONFIG_REISERFS_FS=y
# CONFIG_REISERFS_CHECK is not set
# CONFIG_REISERFS_PROC_INFO is not set
CONFIG_REISERFS_FS_XATTR=y
CONFIG_REISERFS_FS_POSIX_ACL=y
# CONFIG_REISERFS_FS_SECURITY is not set
# CONFIG_JFS_FS is not set
CONFIG_FS_POSIX_ACL=y
CONFIG_XFS_FS=y
# CONFIG_XFS_QUOTA is not set
CONFIG_XFS_POSIX_ACL=y
# CONFIG_XFS_RT is not set
# CONFIG_XFS_DEBUG is not set
# CONFIG_GFS2_FS is not set
# CONFIG_OCFS2_FS is not set
CONFIG_DNOTIFY=y
CONFIG_INOTIFY=y
CONFIG_INOTIFY_USER=y
# CONFIG_QUOTA is not set
# CONFIG_AUTOFS_FS is not set
# CONFIG_AUTOFS4_FS is not set
# CONFIG_FUSE_FS is not set

#
# CD-ROM/DVD Filesystems
#
# CONFIG_ISO9660_FS is not set
# CONFIG_UDF_FS is not set

#
# DOS/FAT/NT Filesystems
#
# CONFIG_MSDOS_FS is not set
# CONFIG_VFAT_FS is not set
# CONFIG_NTFS_FS is not set

#
# Pseudo filesystems
#
CONFIG_PROC_FS=y
CONFIG_PROC_KCORE=y
CONFIG_PROC_VMCORE=y
CONFIG_PROC_SYSCTL=y
CONFIG_SYSFS=y
CONFIG_TMPFS=y
# CONFIG_TMPFS_POSIX_ACL is not set
# CONFIG_HUGETLBFS is not set
# CONFIG_HUGETLB_PAGE is not set
# CONFIG_CONFIGFS_FS is not set

#
# Miscellaneous filesystems
#
# CONFIG_ADFS_FS is not set
# CONFIG_AFFS_FS is not set
# CONFIG_HFS_FS is not set
# CONFIG_HFSPLUS_FS is not set
# CONFIG_BEFS_FS is not set
# CONFIG_BFS_FS is not set
# CONFIG_EFS_FS is not set
# CONFIG_CRAMFS is not set
# CONFIG_VXFS_FS is not set
# CONFIG_MINIX_FS is not set
# CONFIG_OMFS_FS is not set
# CONFIG_HPFS_FS is not set
# CONFIG_QNX4FS_FS is not set
# CONFIG_ROMFS_FS is not set
# CONFIG_SYSV_FS is not set
# CONFIG_UFS_FS is not set
CONFIG_NETWORK_FILESYSTEMS=y
CONFIG_NFS_FS=y
CONFIG_NFS_V3=y
CONFIG_NFS_V3_ACL=y
CONFIG_NFS_V4=y
# CONFIG_NFSD is not set
CONFIG_LOCKD=y
CONFIG_LOCKD_V4=y
CONFIG_NFS_ACL_SUPPORT=y
CONFIG_NFS_COMMON=y
CONFIG_SUNRPC=y
CONFIG_SUNRPC_GSS=y
CONFIG_RPCSEC_GSS_KRB5=y
# CONFIG_RPCSEC_GSS_SPKM3 is not set
# CONFIG_SMB_FS is not set
# CONFIG_CIFS is not set
# CONFIG_NCP_FS is not set
CONFIG_CODA_FS=y
# CONFIG_AFS_FS is not set

#
# Partition Types
#
# CONFIG_PARTITION_ADVANCED is not set
CONFIG_MSDOS_PARTITION=y
# CONFIG_NLS is not set
# CONFIG_DLM is not set

#
# Kernel hacking
#
CONFIG_TRACE_IRQFLAGS_SUPPORT=y
# CONFIG_PRINTK_TIME is not set
# CONFIG_ENABLE_WARN_DEPRECATED is not set
# CONFIG_ENABLE_MUST_CHECK is not set
CONFIG_FRAME_WARN=1024
CONFIG_MAGIC_SYSRQ=y
# CONFIG_UNUSED_SYMBOLS is not set
CONFIG_DEBUG_FS=y
# CONFIG_HEADERS_CHECK is not set
CONFIG_DEBUG_KERNEL=y
# CONFIG_DEBUG_SHIRQ is not set
CONFIG_DETECT_SOFTLOCKUP=y
# CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC is not set
CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC_VALUE=0
# CONFIG_SCHED_DEBUG is not set
# CONFIG_SCHEDSTATS is not set
# CONFIG_TIMER_STATS is not set
# CONFIG_DEBUG_OBJECTS is not set
# CONFIG_SLUB_DEBUG_ON is not set
# CONFIG_SLUB_STATS is not set
# CONFIG_DEBUG_RT_MUTEXES is not set
# CONFIG_RT_MUTEX_TESTER is not set
# CONFIG_DEBUG_SPINLOCK is not set
# CONFIG_DEBUG_MUTEXES is not set
# CONFIG_DEBUG_LOCK_ALLOC is not set
# CONFIG_PROVE_LOCKING is not set
# CONFIG_LOCK_STAT is not set
# CONFIG_DEBUG_SPINLOCK_SLEEP is not set
# CONFIG_DEBUG_LOCKING_API_SELFTESTS is not set
# CONFIG_DEBUG_KOBJECT is not set
# CONFIG_DEBUG_HIGHMEM is not set
CONFIG_DEBUG_BUGVERBOSE=y
CONFIG_DEBUG_INFO=y
# CONFIG_DEBUG_VM is not set
# CONFIG_DEBUG_WRITECOUNT is not set
CONFIG_DEBUG_MEMORY_INIT=y
# CONFIG_DEBUG_LIST is not set
# CONFIG_DEBUG_SG is not set
CONFIG_FRAME_POINTER=y
# CONFIG_BOOT_PRINTK_DELAY is not set
# CONFIG_RCU_TORTURE_TEST is not set
# CONFIG_BACKTRACE_SELF_TEST is not set
# CONFIG_FAULT_INJECTION is not set
# CONFIG_LATENCYTOP is not set
CONFIG_SYSCTL_SYSCALL_CHECK=y
CONFIG_HAVE_FTRACE=y
CONFIG_HAVE_DYNAMIC_FTRACE=y
# CONFIG_FTRACE is not set
# CONFIG_IRQSOFF_TRACER is not set
# CONFIG_SYSPROF_TRACER is not set
# CONFIG_SCHED_TRACER is not set
# CONFIG_CONTEXT_SWITCH_TRACER is not set
# CONFIG_PROVIDE_OHCI1394_DMA_INIT is not set
# CONFIG_SAMPLES is not set
CONFIG_HAVE_ARCH_KGDB=y
CONFIG_KGDB=y
CONFIG_KGDB_SERIAL_CONSOLE=y
# CONFIG_KGDB_TESTS is not set
CONFIG_STRICT_DEVMEM=y
# CONFIG_X86_VERBOSE_BOOTUP is not set
CONFIG_EARLY_PRINTK=y
# CONFIG_DEBUG_STACKOVERFLOW is not set
# CONFIG_DEBUG_STACK_USAGE is not set
# CONFIG_DEBUG_PAGEALLOC is not set
# CONFIG_DEBUG_PER_CPU_MAPS is not set
# CONFIG_X86_PTDUMP is not set
CONFIG_DEBUG_RODATA=y
# CONFIG_DEBUG_RODATA_TEST is not set
# CONFIG_DEBUG_NX_TEST is not set
CONFIG_4KSTACKS=y
CONFIG_DOUBLEFAULT=y
# CONFIG_MMIOTRACE is not set
CONFIG_IO_DELAY_TYPE_0X80=0
CONFIG_IO_DELAY_TYPE_0XED=1
CONFIG_IO_DELAY_TYPE_UDELAY=2
CONFIG_IO_DELAY_TYPE_NONE=3
CONFIG_IO_DELAY_0X80=y
# CONFIG_IO_DELAY_0XED is not set
# CONFIG_IO_DELAY_UDELAY is not set
# CONFIG_IO_DELAY_NONE is not set
CONFIG_DEFAULT_IO_DELAY_TYPE=0
# CONFIG_DEBUG_BOOT_PARAMS is not set
# CONFIG_CPA_DEBUG is not set
# CONFIG_OPTIMIZE_INLINING is not set

#
# Security options
#
# CONFIG_KEYS is not set
CONFIG_SECURITY=y
CONFIG_SECURITY_NETWORK=y
# CONFIG_SECURITY_FILE_CAPABILITIES is not set
CONFIG_SECURITY_DEFAULT_MMAP_MIN_ADDR=0
CONFIG_CRYPTO=y

#
# Crypto core or helper
#
CONFIG_CRYPTO_ALGAPI=y
CONFIG_CRYPTO_BLKCIPHER=y
CONFIG_CRYPTO_MANAGER=y
# CONFIG_CRYPTO_GF128MUL is not set
# CONFIG_CRYPTO_NULL is not set
# CONFIG_CRYPTO_CRYPTD is not set
# CONFIG_CRYPTO_AUTHENC is not set
# CONFIG_CRYPTO_TEST is not set

#
# Authenticated Encryption with Associated Data
#
# CONFIG_CRYPTO_CCM is not set
# CONFIG_CRYPTO_GCM is not set
# CONFIG_CRYPTO_SEQIV is not set

#
# Block modes
#
CONFIG_CRYPTO_CBC=y
# CONFIG_CRYPTO_CTR is not set
# CONFIG_CRYPTO_CTS is not set
# CONFIG_CRYPTO_ECB is not set
# CONFIG_CRYPTO_LRW is not set
# CONFIG_CRYPTO_PCBC is not set
# CONFIG_CRYPTO_XTS is not set

#
# Hash modes
#
# CONFIG_CRYPTO_HMAC is not set
# CONFIG_CRYPTO_XCBC is not set

#
# Digest
#
# CONFIG_CRYPTO_CRC32C is not set
# CONFIG_CRYPTO_MD4 is not set
CONFIG_CRYPTO_MD5=y
# CONFIG_CRYPTO_MICHAEL_MIC is not set
# CONFIG_CRYPTO_RMD128 is not set
# CONFIG_CRYPTO_RMD160 is not set
# CONFIG_CRYPTO_RMD256 is not set
# CONFIG_CRYPTO_RMD320 is not set
# CONFIG_CRYPTO_SHA1 is not set
# CONFIG_CRYPTO_SHA256 is not set
# CONFIG_CRYPTO_SHA512 is not set
# CONFIG_CRYPTO_TGR192 is not set
# CONFIG_CRYPTO_WP512 is not set

#
# Ciphers
#
# CONFIG_CRYPTO_AES is not set
# CONFIG_CRYPTO_AES_586 is not set
# CONFIG_CRYPTO_ANUBIS is not set
# CONFIG_CRYPTO_ARC4 is not set
# CONFIG_CRYPTO_BLOWFISH is not set
# CONFIG_CRYPTO_CAMELLIA is not set
# CONFIG_CRYPTO_CAST5 is not set
# CONFIG_CRYPTO_CAST6 is not set
CONFIG_CRYPTO_DES=y
# CONFIG_CRYPTO_FCRYPT is not set
# CONFIG_CRYPTO_KHAZAD is not set
# CONFIG_CRYPTO_SALSA20 is not set
# CONFIG_CRYPTO_SALSA20_586 is not set
# CONFIG_CRYPTO_SEED is not set
# CONFIG_CRYPTO_SERPENT is not set
# CONFIG_CRYPTO_TEA is not set
# CONFIG_CRYPTO_TWOFISH is not set
# CONFIG_CRYPTO_TWOFISH_586 is not set

#
# Compression
#
# CONFIG_CRYPTO_DEFLATE is not set
# CONFIG_CRYPTO_LZO is not set
# CONFIG_CRYPTO_HW is not set
CONFIG_HAVE_KVM=y
# CONFIG_VIRTUALIZATION is not set

#
# Library routines
#
CONFIG_BITREVERSE=y
CONFIG_GENERIC_FIND_FIRST_BIT=y
CONFIG_GENERIC_FIND_NEXT_BIT=y
# CONFIG_CRC_CCITT is not set
# CONFIG_CRC16 is not set
CONFIG_CRC_T10DIF=y
# CONFIG_CRC_ITU_T is not set
CONFIG_CRC32=y
# CONFIG_CRC7 is not set
# CONFIG_LIBCRC32C is not set
CONFIG_ZLIB_INFLATE=m
CONFIG_TEXTSEARCH=y
CONFIG_TEXTSEARCH_KMP=y
CONFIG_TEXTSEARCH_BM=y
CONFIG_TEXTSEARCH_FSM=y
CONFIG_PLIST=y
CONFIG_HAS_IOMEM=y
CONFIG_HAS_IOPORT=y
CONFIG_HAS_DMA=y

^ permalink raw reply	[flat|nested] 29+ messages in thread

* Re: High load in 2.6.27, NFS / rpcauth_lookup_credcache()?
@ 2008-10-22 17:56   ` J. Bruce Fields
  0 siblings, 0 replies; 29+ messages in thread
From: J. Bruce Fields @ 2008-10-22 17:56 UTC (permalink / raw)
  To: linux-kernel, linux-nfs

On Wed, Oct 22, 2008 at 11:12:08AM +0200, Max Kellermann wrote:
> Hi,
> 
> after I was able to fix http://lkml.org/lkml/2008/10/17/147, the
> server which was already upgraded to 2.6.27.2 still gets very high
> load.  It is a web server with NFS file storage (NetApp), and while
> the others in the cluster (kernel 2.6.25) have a load of 1-3, 2.6.27.2
> gets 30-50.
> 
> I did an oprofile, with the following results (server just started,
> load "only" 5-10):

Well, there's only 7 patches to net/sunrpc/auth.c between 2.6.25 and
today, so worst case a 'git bisect v2.6.25..v2.6.27 net/sunrpc/auth.c'
might not be too bad.

No particular idea why rpcauth_lookup_credcache would be a problem,
though.

--b.

> 
> 87593    56.1116  (no location information)   vmlinux
> vmlinux                  rpcauth_lookup_credcache
> 16037    10.2732  auth_generic.c:0            vmlinux
> vmlinux                  generic_match
> 6460      4.1382  (no location information)   php4
> php4                     (no symbols)
> 2478      1.5874  (no location information)   libc-2.7.so
> libc-2.7.so              (no symbols)
> [...]
> 
> We havn't configured any special authentication method.  It is a NFSv3
> over UDP mount, but the kernel has NFSv4 and therefore KRB5 enabled.
> 
> Any ideas why rpcauth_lookup_credcache() goes overboard with CPU
> usage?
> 
> Max

^ permalink raw reply	[flat|nested] 29+ messages in thread

* Re: High load in 2.6.27, NFS / rpcauth_lookup_credcache()?
@ 2008-10-22 17:56   ` J. Bruce Fields
  0 siblings, 0 replies; 29+ messages in thread
From: J. Bruce Fields @ 2008-10-22 17:56 UTC (permalink / raw)
  To: linux-kernel, linux-nfs

On Wed, Oct 22, 2008 at 11:12:08AM +0200, Max Kellermann wrote:
> Hi,
> 
> after I was able to fix http://lkml.org/lkml/2008/10/17/147, the
> server which was already upgraded to 2.6.27.2 still gets very high
> load.  It is a web server with NFS file storage (NetApp), and while
> the others in the cluster (kernel 2.6.25) have a load of 1-3, 2.6.27.2
> gets 30-50.
> 
> I did an oprofile, with the following results (server just started,
> load "only" 5-10):

Well, there's only 7 patches to net/sunrpc/auth.c between 2.6.25 and
today, so worst case a 'git bisect v2.6.25..v2.6.27 net/sunrpc/auth.c'
might not be too bad.

No particular idea why rpcauth_lookup_credcache would be a problem,
though.

--b.

> 
> 87593    56.1116  (no location information)   vmlinux
> vmlinux                  rpcauth_lookup_credcache
> 16037    10.2732  auth_generic.c:0            vmlinux
> vmlinux                  generic_match
> 6460      4.1382  (no location information)   php4
> php4                     (no symbols)
> 2478      1.5874  (no location information)   libc-2.7.so
> libc-2.7.so              (no symbols)
> [...]
> 
> We havn't configured any special authentication method.  It is a NFSv3
> over UDP mount, but the kernel has NFSv4 and therefore KRB5 enabled.
> 
> Any ideas why rpcauth_lookup_credcache() goes overboard with CPU
> usage?
> 
> Max

^ permalink raw reply	[flat|nested] 29+ messages in thread

* Re: High load in 2.6.27, NFS / rpcauth_lookup_credcache()?
@ 2008-10-23 12:36   ` Max Kellermann
  0 siblings, 0 replies; 29+ messages in thread
From: Max Kellermann @ 2008-10-23 12:36 UTC (permalink / raw)
  To: linux-kernel, linux-nfs, trond.myklebust; +Cc: harry

[-- Attachment #1: Type: text/plain, Size: 1458 bytes --]

On 2008/10/22 11:12, Max Kellermann <max@duempel.org> wrote:
> after I was able to fix http://lkml.org/lkml/2008/10/17/147, the
> server which was already upgraded to 2.6.27.2 still gets very high
> load.  It is a web server with NFS file storage (NetApp), and while
> the others in the cluster (kernel 2.6.25) have a load of 1-3, 2.6.27.2
> gets 30-50.
> 
> I did an oprofile, with the following results (server just started,
> load "only" 5-10):
> 
> 87593    56.1116  (no location information)   vmlinux
> vmlinux                  rpcauth_lookup_credcache
> 16037    10.2732  auth_generic.c:0            vmlinux
> vmlinux                  generic_match
> 6460      4.1382  (no location information)   php4
> php4                     (no symbols)
> 2478      1.5874  (no location information)   libc-2.7.so
> libc-2.7.so              (no symbols)
> [...]
> 
> We havn't configured any special authentication method.  It is a NFSv3
> over UDP mount, but the kernel has NFSv4 and therefore KRB5 enabled.
> 
> Any ideas why rpcauth_lookup_credcache() goes overboard with CPU
> usage?

I have bisected the problem: 98a8e323 is the result ("SUNRPC: Add a
helper rpcauth_lookup_generic_cred()").  5c691044 is ok.

See the attached oprofile annotation data for both commits.  I guess
that the function rpcauth_lookup_credcache() is waiting for a spinlock
too often and too long.  Trond, any idea?

Harry: added you to Cc because your problem sounds similar.

Max

[-- Attachment #2: annotate.98a8e3239427051f5d44f2025b398bdcc3918f37 --]
[-- Type: text/plain, Size: 7840 bytes --]

/* 
 * Command line: opannotate -a -i rpcauth_lookup_credcache vmlinux 
 * 
 * Interpretation of command line:
 * Output annotated assembly listing with samples
 * 
 * CPU: P4 / Xeon with 2 hyper-threads, speed 2400 MHz (estimated)
 * Counted GLOBAL_POWER_EVENTS events (time during which processor is not stopped) with a unit mask of 0x01 (mandatory) count 100000
 */
               :
               :/root/vmlinux:     file format elf32-i386
               :
               :Disassembly of section .text.head:
               :Disassembly of section .text:
               :
c03f7074 <rpcauth_lookup_credcache>: /* rpcauth_lookup_credcache total: 905790 100.000 */
  1525  0.1684 :c03f7074:	push   %ebp
   140  0.0155 :c03f7075:	mov    %esp,%ebp
    22  0.0024 :c03f7077:	push   %edi
    81  0.0089 :c03f7078:	push   %esi
    52  0.0057 :c03f7079:	push   %ebx
    38  0.0042 :c03f707a:	sub    $0x1c,%esp
               :c03f707d:	mov    %eax,0xffffffe0(%ebp)
   156  0.0172 :c03f7080:	mov    %edx,0xffffffdc(%ebp)
    67  0.0074 :c03f7083:	mov    %ecx,0xffffffd8(%ebp)
     5 5.5e-04 :c03f7086:	lea    0xffffffec(%ebp),%eax
    21  0.0023 :c03f7089:	mov    %eax,0xffffffec(%ebp)
    75  0.0083 :c03f708c:	mov    %eax,0xfffffff0(%ebp)
    16  0.0018 :c03f708f:	mov    0xffffffe0(%ebp),%eax
    60  0.0066 :c03f7092:	mov    0x1c(%eax),%eax
   101  0.0112 :c03f7095:	mov    %eax,0xffffffe4(%ebp)
    12  0.0013 :c03f7098:	imul   $0x9e370001,(%edx),%edx
   273  0.0301 :c03f709e:	shr    $0x1c,%edx
    38  0.0042 :c03f70a1:	mov    %edx,0xffffffe8(%ebp)
    64  0.0071 :c03f70a4:	mov    (%eax,%edx,4),%esi
  1254  0.1384 :c03f70a7:	jmp    c03f70ca <rpcauth_lookup_credcache+0x56>
   620  0.0684 :c03f70a9:	mov    0xffffffe4(%ebp),%eax
   132  0.0146 :c03f70ac:	add    $0x40,%eax
    20  0.0022 :c03f70af:	call   c0435d3e <_spin_lock>
  4039  0.4459 :c03f70b4:	testb  $0x4,0x28(%esi)
    34  0.0038 :c03f70b8:	jne    c03f70c2 <rpcauth_lookup_credcache+0x4e>
               :c03f70ba:	mov    0xffffffe4(%ebp),%ecx
               :c03f70bd:	incb   0x40(%ecx)
               :c03f70c0:	jmp    c03f70c8 <rpcauth_lookup_credcache+0x54>
    44  0.0049 :c03f70c2:	lock incl 0x2c(%esi)
  3854  0.4255 :c03f70c6:	jmp    c03f7147 <rpcauth_lookup_credcache+0xd3>
 11816  1.3045 :c03f70c8:	mov    (%esi),%esi
  5214  0.5756 :c03f70ca:	test   %esi,%esi
  1082  0.1195 :c03f70cc:	je     c03f719e <rpcauth_lookup_credcache+0x12a>
 22189  2.4497 :c03f70d2:	mov    (%esi),%eax
702721 77.5810 :c03f70d4:	(bad)  
               :c03f70d6:	inc    %eax
 43523  4.8050 :c03f70d7:	add    %cl,0x4d8b1c5e(%ebx)
               :c03f70dd:	fmuls  0xdc458bf2(%ecx)
  3752  0.4142 :c03f70e3:	call   *0xc(%ebx)
 22791  2.5161 :c03f70e6:	test   %eax,%eax
 10130  1.1184 :c03f70e8:	je     c03f70c8 <rpcauth_lookup_credcache+0x54>
   217  0.0240 :c03f70ea:	jmp    c03f70a9 <rpcauth_lookup_credcache+0x35>
               :c03f70ec:	mov    0xffffffe4(%ebp),%eax
               :c03f70ef:	add    $0x40,%eax
               :c03f70f2:	call   c0435d3e <_spin_lock>
     4 4.4e-04 :c03f70f7:	mov    0xffffffe8(%ebp),%edx
               :c03f70fa:	mov    0xffffffe4(%ebp),%ecx
               :c03f70fd:	mov    (%ecx,%edx,4),%esi
               :c03f7100:	jmp    c03f711e <rpcauth_lookup_credcache+0xaa>
               :c03f7102:	lock incl 0x2c(%esi)
               :c03f7106:	mov    0xfffffff0(%ebp),%ecx
               :c03f7109:	lea    0x8(%edi),%edx
               :c03f710c:	mov    %edx,0xfffffff0(%ebp)
               :c03f710f:	lea    0xffffffec(%ebp),%eax
               :c03f7112:	mov    %eax,0x8(%edi)
               :c03f7115:	mov    %ecx,0x4(%edx)
               :c03f7118:	mov    %edx,(%ecx)
               :c03f711a:	jmp    c03f7147 <rpcauth_lookup_credcache+0xd3>
    20  0.0022 :c03f711c:	mov    (%esi),%esi
               :c03f711e:	test   %esi,%esi
               :c03f7120:	je     c03f71c0 <rpcauth_lookup_credcache+0x14c>
    19  0.0021 :c03f7126:	mov    (%esi),%eax
  1279  0.1412 :c03f7128:	(bad)  
               :c03f712a:	inc    %eax
    59  0.0065 :c03f712b:	add    %cl,0x4d8b1c5e(%ebx)
               :c03f7131:	fmuls  0xdc458bf2(%ecx)
     3 3.3e-04 :c03f7137:	call   *0xc(%ebx)
    22  0.0024 :c03f713a:	test   %eax,%eax
    18  0.0020 :c03f713c:	je     c03f711c <rpcauth_lookup_credcache+0xa8>
               :c03f713e:	jmp    c03f7102 <rpcauth_lookup_credcache+0x8e>
               :c03f7140:	mov    %edi,0x4(%eax)
               :c03f7143:	mov    %edi,(%edx)
               :c03f7145:	mov    %edi,%esi
    14  0.0015 :c03f7147:	mov    0xffffffe4(%ebp),%eax
    11  0.0012 :c03f714a:	incb   0x40(%eax)
    76  0.0084 :c03f714d:	testb  $0x1,0x28(%esi)
   109  0.0120 :c03f7151:	je     c03f7192 <rpcauth_lookup_credcache+0x11e>
               :c03f7153:	mov    0x1c(%esi),%eax
               :c03f7156:	mov    0x4(%eax),%ecx
               :c03f7159:	test   %ecx,%ecx
               :c03f715b:	je     c03f7192 <rpcauth_lookup_credcache+0x11e>
               :c03f715d:	testb  $0x1,0xffffffd8(%ebp)
               :c03f7161:	jne    c03f7192 <rpcauth_lookup_credcache+0x11e>
               :c03f7163:	mov    %esi,%edx
               :c03f7165:	mov    0xffffffe0(%ebp),%eax
               :c03f7168:	call   *%ecx
               :c03f716a:	mov    %eax,%ebx
               :c03f716c:	test   %eax,%eax
               :c03f716e:	jns    c03f7192 <rpcauth_lookup_credcache+0x11e>
               :c03f7170:	mov    %esi,%eax
               :c03f7172:	call   c03f6caa <put_rpccred>
               :c03f7177:	mov    %ebx,%esi
               :c03f7179:	jmp    c03f7192 <rpcauth_lookup_credcache+0x11e>
               :c03f717b:	lea    0xfffffff8(%ebx),%eax
               :c03f717e:	mov    (%ebx),%ecx
               :c03f7180:	mov    0x4(%ebx),%edx
               :c03f7183:	mov    %edx,0x4(%ecx)
               :c03f7186:	mov    %ecx,(%edx)
               :c03f7188:	mov    %ebx,(%ebx)
               :c03f718a:	mov    %ebx,0x4(%ebx)
               :c03f718d:	call   c03f6caa <put_rpccred>
   100  0.0110 :c03f7192:	mov    0xffffffec(%ebp),%ebx
    45  0.0050 :c03f7195:	lea    0xffffffec(%ebp),%eax
     5 5.5e-04 :c03f7198:	cmp    %eax,%ebx
    49  0.0054 :c03f719a:	jne    c03f717b <rpcauth_lookup_credcache+0x107>
    29  0.0032 :c03f719c:	jmp    c03f71e3 <rpcauth_lookup_credcache+0x16f>
     1 1.1e-04 :c03f719e:	mov    0xffffffe0(%ebp),%edx
               :c03f71a1:	mov    0x10(%edx),%ebx
     2 2.2e-04 :c03f71a4:	mov    0xffffffd8(%ebp),%ecx
               :c03f71a7:	mov    0xffffffdc(%ebp),%edx
               :c03f71aa:	mov    0xffffffe0(%ebp),%eax
               :c03f71ad:	call   *0x18(%ebx)
               :c03f71b0:	mov    %eax,%edi
               :c03f71b2:	mov    %eax,%esi
               :c03f71b4:	cmp    $0xfffff000,%eax
               :c03f71b9:	ja     c03f71e3 <rpcauth_lookup_credcache+0x16f>
               :c03f71bb:	jmp    c03f70ec <rpcauth_lookup_credcache+0x78>
               :c03f71c0:	lock btsl $0x2,0x28(%edi)
     1 1.1e-04 :c03f71c6:	mov    0xffffffe8(%ebp),%ecx
               :c03f71c9:	mov    0xffffffe4(%ebp),%eax
               :c03f71cc:	lea    (%eax,%ecx,4),%edx
               :c03f71cf:	mov    (%edx),%eax
               :c03f71d1:	mov    %eax,(%edi)
               :c03f71d3:	mov    %edx,0x4(%edi)
               :c03f71d6:	test   %eax,%eax
               :c03f71d8:	jne    c03f7140 <rpcauth_lookup_credcache+0xcc>
               :c03f71de:	jmp    c03f7143 <rpcauth_lookup_credcache+0xcf>
    58  0.0064 :c03f71e3:	mov    %esi,%eax
    17  0.0019 :c03f71e5:	add    $0x1c,%esp
    13  0.0014 :c03f71e8:	pop    %ebx
    45  0.0050 :c03f71e9:	pop    %esi
    17  0.0019 :c03f71ea:	pop    %edi
     5 5.5e-04 :c03f71eb:	pop    %ebp
    38  0.0042 :c03f71ec:	ret    
               :Disassembly of section .init.text:
               :Disassembly of section .altinstr_replacement:
               :Disassembly of section .exit.text:

[-- Attachment #3: annotate.5c691044ecbca04dd558fca4c754121689fe1b34 --]
[-- Type: text/plain, Size: 7840 bytes --]

/* 
 * Command line: opannotate -a -i rpcauth_lookup_credcache vmlinux 
 * 
 * Interpretation of command line:
 * Output annotated assembly listing with samples
 * 
 * CPU: P4 / Xeon with 2 hyper-threads, speed 2400 MHz (estimated)
 * Counted GLOBAL_POWER_EVENTS events (time during which processor is not stopped) with a unit mask of 0x01 (mandatory) count 100000
 */
               :
               :/root/vmlinux:     file format elf32-i386
               :
               :Disassembly of section .text.head:
               :Disassembly of section .text:
               :
c03f7104 <rpcauth_lookup_credcache>: /* rpcauth_lookup_credcache total:    435 100.000 */
    21  4.8276 :c03f7104:	push   %ebp
     2  0.4598 :c03f7105:	mov    %esp,%ebp
               :c03f7107:	push   %edi
               :c03f7108:	push   %esi
     1  0.2299 :c03f7109:	push   %ebx
               :c03f710a:	sub    $0x1c,%esp
               :c03f710d:	mov    %eax,0xffffffe0(%ebp)
     1  0.2299 :c03f7110:	mov    %edx,0xffffffdc(%ebp)
               :c03f7113:	mov    %ecx,0xffffffd8(%ebp)
               :c03f7116:	lea    0xffffffec(%ebp),%eax
     1  0.2299 :c03f7119:	mov    %eax,0xffffffec(%ebp)
     2  0.4598 :c03f711c:	mov    %eax,0xfffffff0(%ebp)
               :c03f711f:	mov    0xffffffe0(%ebp),%eax
     3  0.6897 :c03f7122:	mov    0x1c(%eax),%eax
     3  0.6897 :c03f7125:	mov    %eax,0xffffffe4(%ebp)
               :c03f7128:	imul   $0x9e370001,(%edx),%edx
     4  0.9195 :c03f712e:	shr    $0x1c,%edx
     2  0.4598 :c03f7131:	mov    %edx,0xffffffe8(%ebp)
     2  0.4598 :c03f7134:	mov    (%eax,%edx,4),%esi
    39  8.9655 :c03f7137:	jmp    c03f715a <rpcauth_lookup_credcache+0x56>
    10  2.2989 :c03f7139:	mov    0xffffffe4(%ebp),%eax
     1  0.2299 :c03f713c:	add    $0x40,%eax
     1  0.2299 :c03f713f:	call   c0435dce <_spin_lock>
   134 30.8046 :c03f7144:	testb  $0x4,0x28(%esi)
     2  0.4598 :c03f7148:	jne    c03f7152 <rpcauth_lookup_credcache+0x4e>
               :c03f714a:	mov    0xffffffe4(%ebp),%ecx
               :c03f714d:	incb   0x40(%ecx)
               :c03f7150:	jmp    c03f7158 <rpcauth_lookup_credcache+0x54>
     3  0.6897 :c03f7152:	lock incl 0x2c(%esi)
   110 25.2874 :c03f7156:	jmp    c03f71d7 <rpcauth_lookup_credcache+0xd3>
     2  0.4598 :c03f7158:	mov    (%esi),%esi
     6  1.3793 :c03f715a:	test   %esi,%esi
               :c03f715c:	je     c03f722e <rpcauth_lookup_credcache+0x12a>
               :c03f7162:	mov    (%esi),%eax
    43  9.8851 :c03f7164:	(bad)  
               :c03f7166:	inc    %eax
    10  2.2989 :c03f7167:	add    %cl,0x4d8b1c5e(%ebx)
               :c03f716d:	fmuls  0xdc458bf2(%ecx)
               :c03f7173:	call   *0xc(%ebx)
     4  0.9195 :c03f7176:	test   %eax,%eax
     6  1.3793 :c03f7178:	je     c03f7158 <rpcauth_lookup_credcache+0x54>
     2  0.4598 :c03f717a:	jmp    c03f7139 <rpcauth_lookup_credcache+0x35>
               :c03f717c:	mov    0xffffffe4(%ebp),%eax
               :c03f717f:	add    $0x40,%eax
               :c03f7182:	call   c0435dce <_spin_lock>
               :c03f7187:	mov    0xffffffe8(%ebp),%edx
               :c03f718a:	mov    0xffffffe4(%ebp),%ecx
               :c03f718d:	mov    (%ecx,%edx,4),%esi
               :c03f7190:	jmp    c03f71ae <rpcauth_lookup_credcache+0xaa>
               :c03f7192:	lock incl 0x2c(%esi)
               :c03f7196:	mov    0xfffffff0(%ebp),%ecx
               :c03f7199:	lea    0x8(%edi),%edx
               :c03f719c:	mov    %edx,0xfffffff0(%ebp)
               :c03f719f:	lea    0xffffffec(%ebp),%eax
               :c03f71a2:	mov    %eax,0x8(%edi)
               :c03f71a5:	mov    %ecx,0x4(%edx)
               :c03f71a8:	mov    %edx,(%ecx)
               :c03f71aa:	jmp    c03f71d7 <rpcauth_lookup_credcache+0xd3>
               :c03f71ac:	mov    (%esi),%esi
               :c03f71ae:	test   %esi,%esi
               :c03f71b0:	je     c03f7250 <rpcauth_lookup_credcache+0x14c>
               :c03f71b6:	mov    (%esi),%eax
               :c03f71b8:	(bad)  
               :c03f71ba:	inc    %eax
               :c03f71bb:	add    %cl,0x4d8b1c5e(%ebx)
               :c03f71c1:	fmuls  0xdc458bf2(%ecx)
               :c03f71c7:	call   *0xc(%ebx)
               :c03f71ca:	test   %eax,%eax
               :c03f71cc:	je     c03f71ac <rpcauth_lookup_credcache+0xa8>
               :c03f71ce:	jmp    c03f7192 <rpcauth_lookup_credcache+0x8e>
               :c03f71d0:	mov    %edi,0x4(%eax)
               :c03f71d3:	mov    %edi,(%edx)
               :c03f71d5:	mov    %edi,%esi
               :c03f71d7:	mov    0xffffffe4(%ebp),%eax
     1  0.2299 :c03f71da:	incb   0x40(%eax)
     1  0.2299 :c03f71dd:	testb  $0x1,0x28(%esi)
     2  0.4598 :c03f71e1:	je     c03f7222 <rpcauth_lookup_credcache+0x11e>
               :c03f71e3:	mov    0x1c(%esi),%eax
               :c03f71e6:	mov    0x4(%eax),%ecx
               :c03f71e9:	test   %ecx,%ecx
               :c03f71eb:	je     c03f7222 <rpcauth_lookup_credcache+0x11e>
               :c03f71ed:	testb  $0x1,0xffffffd8(%ebp)
               :c03f71f1:	jne    c03f7222 <rpcauth_lookup_credcache+0x11e>
               :c03f71f3:	mov    %esi,%edx
               :c03f71f5:	mov    0xffffffe0(%ebp),%eax
               :c03f71f8:	call   *%ecx
               :c03f71fa:	mov    %eax,%ebx
               :c03f71fc:	test   %eax,%eax
               :c03f71fe:	jns    c03f7222 <rpcauth_lookup_credcache+0x11e>
               :c03f7200:	mov    %esi,%eax
               :c03f7202:	call   c03f6d3a <put_rpccred>
               :c03f7207:	mov    %ebx,%esi
               :c03f7209:	jmp    c03f7222 <rpcauth_lookup_credcache+0x11e>
               :c03f720b:	lea    0xfffffff8(%ebx),%eax
               :c03f720e:	mov    (%ebx),%ecx
               :c03f7210:	mov    0x4(%ebx),%edx
               :c03f7213:	mov    %edx,0x4(%ecx)
               :c03f7216:	mov    %ecx,(%edx)
               :c03f7218:	mov    %ebx,(%ebx)
               :c03f721a:	mov    %ebx,0x4(%ebx)
               :c03f721d:	call   c03f6d3a <put_rpccred>
     2  0.4598 :c03f7222:	mov    0xffffffec(%ebp),%ebx
     2  0.4598 :c03f7225:	lea    0xffffffec(%ebp),%eax
               :c03f7228:	cmp    %eax,%ebx
     1  0.2299 :c03f722a:	jne    c03f720b <rpcauth_lookup_credcache+0x107>
     3  0.6897 :c03f722c:	jmp    c03f7273 <rpcauth_lookup_credcache+0x16f>
               :c03f722e:	mov    0xffffffe0(%ebp),%edx
               :c03f7231:	mov    0x10(%edx),%ebx
               :c03f7234:	mov    0xffffffd8(%ebp),%ecx
               :c03f7237:	mov    0xffffffdc(%ebp),%edx
               :c03f723a:	mov    0xffffffe0(%ebp),%eax
               :c03f723d:	call   *0x18(%ebx)
               :c03f7240:	mov    %eax,%edi
               :c03f7242:	mov    %eax,%esi
               :c03f7244:	cmp    $0xfffff000,%eax
               :c03f7249:	ja     c03f7273 <rpcauth_lookup_credcache+0x16f>
               :c03f724b:	jmp    c03f717c <rpcauth_lookup_credcache+0x78>
               :c03f7250:	lock btsl $0x2,0x28(%edi)
               :c03f7256:	mov    0xffffffe8(%ebp),%ecx
               :c03f7259:	mov    0xffffffe4(%ebp),%eax
               :c03f725c:	lea    (%eax,%ecx,4),%edx
               :c03f725f:	mov    (%edx),%eax
               :c03f7261:	mov    %eax,(%edi)
               :c03f7263:	mov    %edx,0x4(%edi)
               :c03f7266:	test   %eax,%eax
               :c03f7268:	jne    c03f71d0 <rpcauth_lookup_credcache+0xcc>
               :c03f726e:	jmp    c03f71d3 <rpcauth_lookup_credcache+0xcf>
     4  0.9195 :c03f7273:	mov    %esi,%eax
               :c03f7275:	add    $0x1c,%esp
               :c03f7278:	pop    %ebx
     1  0.2299 :c03f7279:	pop    %esi
     1  0.2299 :c03f727a:	pop    %edi
               :c03f727b:	pop    %ebp
               :c03f727c:	ret    
               :Disassembly of section .init.text:
               :Disassembly of section .altinstr_replacement:
               :Disassembly of section .exit.text:

^ permalink raw reply	[flat|nested] 29+ messages in thread

* Re: High load in 2.6.27, NFS / rpcauth_lookup_credcache()?
@ 2008-10-23 12:36   ` Max Kellermann
  0 siblings, 0 replies; 29+ messages in thread
From: Max Kellermann @ 2008-10-23 12:36 UTC (permalink / raw)
  To: linux-kernel, linux-nfs, trond.myklebust
  Cc: harry-qmPYOCrcNLLyFCzt5hm0YvZ8FUJU4vz8

[-- Attachment #1: Type: text/plain, Size: 1486 bytes --]

On 2008/10/22 11:12, Max Kellermann <max-hDT0AjmEH7RAfugRpC6u6w@public.gmane.org> wrote:
> after I was able to fix http://lkml.org/lkml/2008/10/17/147, the
> server which was already upgraded to 2.6.27.2 still gets very high
> load.  It is a web server with NFS file storage (NetApp), and while
> the others in the cluster (kernel 2.6.25) have a load of 1-3, 2.6.27.2
> gets 30-50.
> 
> I did an oprofile, with the following results (server just started,
> load "only" 5-10):
> 
> 87593    56.1116  (no location information)   vmlinux
> vmlinux                  rpcauth_lookup_credcache
> 16037    10.2732  auth_generic.c:0            vmlinux
> vmlinux                  generic_match
> 6460      4.1382  (no location information)   php4
> php4                     (no symbols)
> 2478      1.5874  (no location information)   libc-2.7.so
> libc-2.7.so              (no symbols)
> [...]
> 
> We havn't configured any special authentication method.  It is a NFSv3
> over UDP mount, but the kernel has NFSv4 and therefore KRB5 enabled.
> 
> Any ideas why rpcauth_lookup_credcache() goes overboard with CPU
> usage?

I have bisected the problem: 98a8e323 is the result ("SUNRPC: Add a
helper rpcauth_lookup_generic_cred()").  5c691044 is ok.

See the attached oprofile annotation data for both commits.  I guess
that the function rpcauth_lookup_credcache() is waiting for a spinlock
too often and too long.  Trond, any idea?

Harry: added you to Cc because your problem sounds similar.

Max

[-- Attachment #2: annotate.98a8e3239427051f5d44f2025b398bdcc3918f37 --]
[-- Type: text/plain, Size: 7840 bytes --]

/* 
 * Command line: opannotate -a -i rpcauth_lookup_credcache vmlinux 
 * 
 * Interpretation of command line:
 * Output annotated assembly listing with samples
 * 
 * CPU: P4 / Xeon with 2 hyper-threads, speed 2400 MHz (estimated)
 * Counted GLOBAL_POWER_EVENTS events (time during which processor is not stopped) with a unit mask of 0x01 (mandatory) count 100000
 */
               :
               :/root/vmlinux:     file format elf32-i386
               :
               :Disassembly of section .text.head:
               :Disassembly of section .text:
               :
c03f7074 <rpcauth_lookup_credcache>: /* rpcauth_lookup_credcache total: 905790 100.000 */
  1525  0.1684 :c03f7074:	push   %ebp
   140  0.0155 :c03f7075:	mov    %esp,%ebp
    22  0.0024 :c03f7077:	push   %edi
    81  0.0089 :c03f7078:	push   %esi
    52  0.0057 :c03f7079:	push   %ebx
    38  0.0042 :c03f707a:	sub    $0x1c,%esp
               :c03f707d:	mov    %eax,0xffffffe0(%ebp)
   156  0.0172 :c03f7080:	mov    %edx,0xffffffdc(%ebp)
    67  0.0074 :c03f7083:	mov    %ecx,0xffffffd8(%ebp)
     5 5.5e-04 :c03f7086:	lea    0xffffffec(%ebp),%eax
    21  0.0023 :c03f7089:	mov    %eax,0xffffffec(%ebp)
    75  0.0083 :c03f708c:	mov    %eax,0xfffffff0(%ebp)
    16  0.0018 :c03f708f:	mov    0xffffffe0(%ebp),%eax
    60  0.0066 :c03f7092:	mov    0x1c(%eax),%eax
   101  0.0112 :c03f7095:	mov    %eax,0xffffffe4(%ebp)
    12  0.0013 :c03f7098:	imul   $0x9e370001,(%edx),%edx
   273  0.0301 :c03f709e:	shr    $0x1c,%edx
    38  0.0042 :c03f70a1:	mov    %edx,0xffffffe8(%ebp)
    64  0.0071 :c03f70a4:	mov    (%eax,%edx,4),%esi
  1254  0.1384 :c03f70a7:	jmp    c03f70ca <rpcauth_lookup_credcache+0x56>
   620  0.0684 :c03f70a9:	mov    0xffffffe4(%ebp),%eax
   132  0.0146 :c03f70ac:	add    $0x40,%eax
    20  0.0022 :c03f70af:	call   c0435d3e <_spin_lock>
  4039  0.4459 :c03f70b4:	testb  $0x4,0x28(%esi)
    34  0.0038 :c03f70b8:	jne    c03f70c2 <rpcauth_lookup_credcache+0x4e>
               :c03f70ba:	mov    0xffffffe4(%ebp),%ecx
               :c03f70bd:	incb   0x40(%ecx)
               :c03f70c0:	jmp    c03f70c8 <rpcauth_lookup_credcache+0x54>
    44  0.0049 :c03f70c2:	lock incl 0x2c(%esi)
  3854  0.4255 :c03f70c6:	jmp    c03f7147 <rpcauth_lookup_credcache+0xd3>
 11816  1.3045 :c03f70c8:	mov    (%esi),%esi
  5214  0.5756 :c03f70ca:	test   %esi,%esi
  1082  0.1195 :c03f70cc:	je     c03f719e <rpcauth_lookup_credcache+0x12a>
 22189  2.4497 :c03f70d2:	mov    (%esi),%eax
702721 77.5810 :c03f70d4:	(bad)  
               :c03f70d6:	inc    %eax
 43523  4.8050 :c03f70d7:	add    %cl,0x4d8b1c5e(%ebx)
               :c03f70dd:	fmuls  0xdc458bf2(%ecx)
  3752  0.4142 :c03f70e3:	call   *0xc(%ebx)
 22791  2.5161 :c03f70e6:	test   %eax,%eax
 10130  1.1184 :c03f70e8:	je     c03f70c8 <rpcauth_lookup_credcache+0x54>
   217  0.0240 :c03f70ea:	jmp    c03f70a9 <rpcauth_lookup_credcache+0x35>
               :c03f70ec:	mov    0xffffffe4(%ebp),%eax
               :c03f70ef:	add    $0x40,%eax
               :c03f70f2:	call   c0435d3e <_spin_lock>
     4 4.4e-04 :c03f70f7:	mov    0xffffffe8(%ebp),%edx
               :c03f70fa:	mov    0xffffffe4(%ebp),%ecx
               :c03f70fd:	mov    (%ecx,%edx,4),%esi
               :c03f7100:	jmp    c03f711e <rpcauth_lookup_credcache+0xaa>
               :c03f7102:	lock incl 0x2c(%esi)
               :c03f7106:	mov    0xfffffff0(%ebp),%ecx
               :c03f7109:	lea    0x8(%edi),%edx
               :c03f710c:	mov    %edx,0xfffffff0(%ebp)
               :c03f710f:	lea    0xffffffec(%ebp),%eax
               :c03f7112:	mov    %eax,0x8(%edi)
               :c03f7115:	mov    %ecx,0x4(%edx)
               :c03f7118:	mov    %edx,(%ecx)
               :c03f711a:	jmp    c03f7147 <rpcauth_lookup_credcache+0xd3>
    20  0.0022 :c03f711c:	mov    (%esi),%esi
               :c03f711e:	test   %esi,%esi
               :c03f7120:	je     c03f71c0 <rpcauth_lookup_credcache+0x14c>
    19  0.0021 :c03f7126:	mov    (%esi),%eax
  1279  0.1412 :c03f7128:	(bad)  
               :c03f712a:	inc    %eax
    59  0.0065 :c03f712b:	add    %cl,0x4d8b1c5e(%ebx)
               :c03f7131:	fmuls  0xdc458bf2(%ecx)
     3 3.3e-04 :c03f7137:	call   *0xc(%ebx)
    22  0.0024 :c03f713a:	test   %eax,%eax
    18  0.0020 :c03f713c:	je     c03f711c <rpcauth_lookup_credcache+0xa8>
               :c03f713e:	jmp    c03f7102 <rpcauth_lookup_credcache+0x8e>
               :c03f7140:	mov    %edi,0x4(%eax)
               :c03f7143:	mov    %edi,(%edx)
               :c03f7145:	mov    %edi,%esi
    14  0.0015 :c03f7147:	mov    0xffffffe4(%ebp),%eax
    11  0.0012 :c03f714a:	incb   0x40(%eax)
    76  0.0084 :c03f714d:	testb  $0x1,0x28(%esi)
   109  0.0120 :c03f7151:	je     c03f7192 <rpcauth_lookup_credcache+0x11e>
               :c03f7153:	mov    0x1c(%esi),%eax
               :c03f7156:	mov    0x4(%eax),%ecx
               :c03f7159:	test   %ecx,%ecx
               :c03f715b:	je     c03f7192 <rpcauth_lookup_credcache+0x11e>
               :c03f715d:	testb  $0x1,0xffffffd8(%ebp)
               :c03f7161:	jne    c03f7192 <rpcauth_lookup_credcache+0x11e>
               :c03f7163:	mov    %esi,%edx
               :c03f7165:	mov    0xffffffe0(%ebp),%eax
               :c03f7168:	call   *%ecx
               :c03f716a:	mov    %eax,%ebx
               :c03f716c:	test   %eax,%eax
               :c03f716e:	jns    c03f7192 <rpcauth_lookup_credcache+0x11e>
               :c03f7170:	mov    %esi,%eax
               :c03f7172:	call   c03f6caa <put_rpccred>
               :c03f7177:	mov    %ebx,%esi
               :c03f7179:	jmp    c03f7192 <rpcauth_lookup_credcache+0x11e>
               :c03f717b:	lea    0xfffffff8(%ebx),%eax
               :c03f717e:	mov    (%ebx),%ecx
               :c03f7180:	mov    0x4(%ebx),%edx
               :c03f7183:	mov    %edx,0x4(%ecx)
               :c03f7186:	mov    %ecx,(%edx)
               :c03f7188:	mov    %ebx,(%ebx)
               :c03f718a:	mov    %ebx,0x4(%ebx)
               :c03f718d:	call   c03f6caa <put_rpccred>
   100  0.0110 :c03f7192:	mov    0xffffffec(%ebp),%ebx
    45  0.0050 :c03f7195:	lea    0xffffffec(%ebp),%eax
     5 5.5e-04 :c03f7198:	cmp    %eax,%ebx
    49  0.0054 :c03f719a:	jne    c03f717b <rpcauth_lookup_credcache+0x107>
    29  0.0032 :c03f719c:	jmp    c03f71e3 <rpcauth_lookup_credcache+0x16f>
     1 1.1e-04 :c03f719e:	mov    0xffffffe0(%ebp),%edx
               :c03f71a1:	mov    0x10(%edx),%ebx
     2 2.2e-04 :c03f71a4:	mov    0xffffffd8(%ebp),%ecx
               :c03f71a7:	mov    0xffffffdc(%ebp),%edx
               :c03f71aa:	mov    0xffffffe0(%ebp),%eax
               :c03f71ad:	call   *0x18(%ebx)
               :c03f71b0:	mov    %eax,%edi
               :c03f71b2:	mov    %eax,%esi
               :c03f71b4:	cmp    $0xfffff000,%eax
               :c03f71b9:	ja     c03f71e3 <rpcauth_lookup_credcache+0x16f>
               :c03f71bb:	jmp    c03f70ec <rpcauth_lookup_credcache+0x78>
               :c03f71c0:	lock btsl $0x2,0x28(%edi)
     1 1.1e-04 :c03f71c6:	mov    0xffffffe8(%ebp),%ecx
               :c03f71c9:	mov    0xffffffe4(%ebp),%eax
               :c03f71cc:	lea    (%eax,%ecx,4),%edx
               :c03f71cf:	mov    (%edx),%eax
               :c03f71d1:	mov    %eax,(%edi)
               :c03f71d3:	mov    %edx,0x4(%edi)
               :c03f71d6:	test   %eax,%eax
               :c03f71d8:	jne    c03f7140 <rpcauth_lookup_credcache+0xcc>
               :c03f71de:	jmp    c03f7143 <rpcauth_lookup_credcache+0xcf>
    58  0.0064 :c03f71e3:	mov    %esi,%eax
    17  0.0019 :c03f71e5:	add    $0x1c,%esp
    13  0.0014 :c03f71e8:	pop    %ebx
    45  0.0050 :c03f71e9:	pop    %esi
    17  0.0019 :c03f71ea:	pop    %edi
     5 5.5e-04 :c03f71eb:	pop    %ebp
    38  0.0042 :c03f71ec:	ret    
               :Disassembly of section .init.text:
               :Disassembly of section .altinstr_replacement:
               :Disassembly of section .exit.text:

[-- Attachment #3: annotate.5c691044ecbca04dd558fca4c754121689fe1b34 --]
[-- Type: text/plain, Size: 7840 bytes --]

/* 
 * Command line: opannotate -a -i rpcauth_lookup_credcache vmlinux 
 * 
 * Interpretation of command line:
 * Output annotated assembly listing with samples
 * 
 * CPU: P4 / Xeon with 2 hyper-threads, speed 2400 MHz (estimated)
 * Counted GLOBAL_POWER_EVENTS events (time during which processor is not stopped) with a unit mask of 0x01 (mandatory) count 100000
 */
               :
               :/root/vmlinux:     file format elf32-i386
               :
               :Disassembly of section .text.head:
               :Disassembly of section .text:
               :
c03f7104 <rpcauth_lookup_credcache>: /* rpcauth_lookup_credcache total:    435 100.000 */
    21  4.8276 :c03f7104:	push   %ebp
     2  0.4598 :c03f7105:	mov    %esp,%ebp
               :c03f7107:	push   %edi
               :c03f7108:	push   %esi
     1  0.2299 :c03f7109:	push   %ebx
               :c03f710a:	sub    $0x1c,%esp
               :c03f710d:	mov    %eax,0xffffffe0(%ebp)
     1  0.2299 :c03f7110:	mov    %edx,0xffffffdc(%ebp)
               :c03f7113:	mov    %ecx,0xffffffd8(%ebp)
               :c03f7116:	lea    0xffffffec(%ebp),%eax
     1  0.2299 :c03f7119:	mov    %eax,0xffffffec(%ebp)
     2  0.4598 :c03f711c:	mov    %eax,0xfffffff0(%ebp)
               :c03f711f:	mov    0xffffffe0(%ebp),%eax
     3  0.6897 :c03f7122:	mov    0x1c(%eax),%eax
     3  0.6897 :c03f7125:	mov    %eax,0xffffffe4(%ebp)
               :c03f7128:	imul   $0x9e370001,(%edx),%edx
     4  0.9195 :c03f712e:	shr    $0x1c,%edx
     2  0.4598 :c03f7131:	mov    %edx,0xffffffe8(%ebp)
     2  0.4598 :c03f7134:	mov    (%eax,%edx,4),%esi
    39  8.9655 :c03f7137:	jmp    c03f715a <rpcauth_lookup_credcache+0x56>
    10  2.2989 :c03f7139:	mov    0xffffffe4(%ebp),%eax
     1  0.2299 :c03f713c:	add    $0x40,%eax
     1  0.2299 :c03f713f:	call   c0435dce <_spin_lock>
   134 30.8046 :c03f7144:	testb  $0x4,0x28(%esi)
     2  0.4598 :c03f7148:	jne    c03f7152 <rpcauth_lookup_credcache+0x4e>
               :c03f714a:	mov    0xffffffe4(%ebp),%ecx
               :c03f714d:	incb   0x40(%ecx)
               :c03f7150:	jmp    c03f7158 <rpcauth_lookup_credcache+0x54>
     3  0.6897 :c03f7152:	lock incl 0x2c(%esi)
   110 25.2874 :c03f7156:	jmp    c03f71d7 <rpcauth_lookup_credcache+0xd3>
     2  0.4598 :c03f7158:	mov    (%esi),%esi
     6  1.3793 :c03f715a:	test   %esi,%esi
               :c03f715c:	je     c03f722e <rpcauth_lookup_credcache+0x12a>
               :c03f7162:	mov    (%esi),%eax
    43  9.8851 :c03f7164:	(bad)  
               :c03f7166:	inc    %eax
    10  2.2989 :c03f7167:	add    %cl,0x4d8b1c5e(%ebx)
               :c03f716d:	fmuls  0xdc458bf2(%ecx)
               :c03f7173:	call   *0xc(%ebx)
     4  0.9195 :c03f7176:	test   %eax,%eax
     6  1.3793 :c03f7178:	je     c03f7158 <rpcauth_lookup_credcache+0x54>
     2  0.4598 :c03f717a:	jmp    c03f7139 <rpcauth_lookup_credcache+0x35>
               :c03f717c:	mov    0xffffffe4(%ebp),%eax
               :c03f717f:	add    $0x40,%eax
               :c03f7182:	call   c0435dce <_spin_lock>
               :c03f7187:	mov    0xffffffe8(%ebp),%edx
               :c03f718a:	mov    0xffffffe4(%ebp),%ecx
               :c03f718d:	mov    (%ecx,%edx,4),%esi
               :c03f7190:	jmp    c03f71ae <rpcauth_lookup_credcache+0xaa>
               :c03f7192:	lock incl 0x2c(%esi)
               :c03f7196:	mov    0xfffffff0(%ebp),%ecx
               :c03f7199:	lea    0x8(%edi),%edx
               :c03f719c:	mov    %edx,0xfffffff0(%ebp)
               :c03f719f:	lea    0xffffffec(%ebp),%eax
               :c03f71a2:	mov    %eax,0x8(%edi)
               :c03f71a5:	mov    %ecx,0x4(%edx)
               :c03f71a8:	mov    %edx,(%ecx)
               :c03f71aa:	jmp    c03f71d7 <rpcauth_lookup_credcache+0xd3>
               :c03f71ac:	mov    (%esi),%esi
               :c03f71ae:	test   %esi,%esi
               :c03f71b0:	je     c03f7250 <rpcauth_lookup_credcache+0x14c>
               :c03f71b6:	mov    (%esi),%eax
               :c03f71b8:	(bad)  
               :c03f71ba:	inc    %eax
               :c03f71bb:	add    %cl,0x4d8b1c5e(%ebx)
               :c03f71c1:	fmuls  0xdc458bf2(%ecx)
               :c03f71c7:	call   *0xc(%ebx)
               :c03f71ca:	test   %eax,%eax
               :c03f71cc:	je     c03f71ac <rpcauth_lookup_credcache+0xa8>
               :c03f71ce:	jmp    c03f7192 <rpcauth_lookup_credcache+0x8e>
               :c03f71d0:	mov    %edi,0x4(%eax)
               :c03f71d3:	mov    %edi,(%edx)
               :c03f71d5:	mov    %edi,%esi
               :c03f71d7:	mov    0xffffffe4(%ebp),%eax
     1  0.2299 :c03f71da:	incb   0x40(%eax)
     1  0.2299 :c03f71dd:	testb  $0x1,0x28(%esi)
     2  0.4598 :c03f71e1:	je     c03f7222 <rpcauth_lookup_credcache+0x11e>
               :c03f71e3:	mov    0x1c(%esi),%eax
               :c03f71e6:	mov    0x4(%eax),%ecx
               :c03f71e9:	test   %ecx,%ecx
               :c03f71eb:	je     c03f7222 <rpcauth_lookup_credcache+0x11e>
               :c03f71ed:	testb  $0x1,0xffffffd8(%ebp)
               :c03f71f1:	jne    c03f7222 <rpcauth_lookup_credcache+0x11e>
               :c03f71f3:	mov    %esi,%edx
               :c03f71f5:	mov    0xffffffe0(%ebp),%eax
               :c03f71f8:	call   *%ecx
               :c03f71fa:	mov    %eax,%ebx
               :c03f71fc:	test   %eax,%eax
               :c03f71fe:	jns    c03f7222 <rpcauth_lookup_credcache+0x11e>
               :c03f7200:	mov    %esi,%eax
               :c03f7202:	call   c03f6d3a <put_rpccred>
               :c03f7207:	mov    %ebx,%esi
               :c03f7209:	jmp    c03f7222 <rpcauth_lookup_credcache+0x11e>
               :c03f720b:	lea    0xfffffff8(%ebx),%eax
               :c03f720e:	mov    (%ebx),%ecx
               :c03f7210:	mov    0x4(%ebx),%edx
               :c03f7213:	mov    %edx,0x4(%ecx)
               :c03f7216:	mov    %ecx,(%edx)
               :c03f7218:	mov    %ebx,(%ebx)
               :c03f721a:	mov    %ebx,0x4(%ebx)
               :c03f721d:	call   c03f6d3a <put_rpccred>
     2  0.4598 :c03f7222:	mov    0xffffffec(%ebp),%ebx
     2  0.4598 :c03f7225:	lea    0xffffffec(%ebp),%eax
               :c03f7228:	cmp    %eax,%ebx
     1  0.2299 :c03f722a:	jne    c03f720b <rpcauth_lookup_credcache+0x107>
     3  0.6897 :c03f722c:	jmp    c03f7273 <rpcauth_lookup_credcache+0x16f>
               :c03f722e:	mov    0xffffffe0(%ebp),%edx
               :c03f7231:	mov    0x10(%edx),%ebx
               :c03f7234:	mov    0xffffffd8(%ebp),%ecx
               :c03f7237:	mov    0xffffffdc(%ebp),%edx
               :c03f723a:	mov    0xffffffe0(%ebp),%eax
               :c03f723d:	call   *0x18(%ebx)
               :c03f7240:	mov    %eax,%edi
               :c03f7242:	mov    %eax,%esi
               :c03f7244:	cmp    $0xfffff000,%eax
               :c03f7249:	ja     c03f7273 <rpcauth_lookup_credcache+0x16f>
               :c03f724b:	jmp    c03f717c <rpcauth_lookup_credcache+0x78>
               :c03f7250:	lock btsl $0x2,0x28(%edi)
               :c03f7256:	mov    0xffffffe8(%ebp),%ecx
               :c03f7259:	mov    0xffffffe4(%ebp),%eax
               :c03f725c:	lea    (%eax,%ecx,4),%edx
               :c03f725f:	mov    (%edx),%eax
               :c03f7261:	mov    %eax,(%edi)
               :c03f7263:	mov    %edx,0x4(%edi)
               :c03f7266:	test   %eax,%eax
               :c03f7268:	jne    c03f71d0 <rpcauth_lookup_credcache+0xcc>
               :c03f726e:	jmp    c03f71d3 <rpcauth_lookup_credcache+0xcf>
     4  0.9195 :c03f7273:	mov    %esi,%eax
               :c03f7275:	add    $0x1c,%esp
               :c03f7278:	pop    %ebx
     1  0.2299 :c03f7279:	pop    %esi
     1  0.2299 :c03f727a:	pop    %edi
               :c03f727b:	pop    %ebp
               :c03f727c:	ret    
               :Disassembly of section .init.text:
               :Disassembly of section .altinstr_replacement:
               :Disassembly of section .exit.text:

^ permalink raw reply	[flat|nested] 29+ messages in thread

* Re: High load in 2.6.27, NFS / rpcauth_lookup_credcache()?
  2008-10-23 12:36   ` Max Kellermann
  (?)
@ 2008-10-23 14:55   ` Trond Myklebust
  2008-10-24  8:39       ` Max Kellermann
  2008-12-15 23:44       ` Harry Edmon
  -1 siblings, 2 replies; 29+ messages in thread
From: Trond Myklebust @ 2008-10-23 14:55 UTC (permalink / raw)
  To: Max Kellermann; +Cc: linux-kernel, linux-nfs, harry

On Thu, 2008-10-23 at 14:36 +0200, Max Kellermann wrote:
> On 2008/10/22 11:12, Max Kellermann <max@duempel.org> wrote:
> > after I was able to fix http://lkml.org/lkml/2008/10/17/147, the
> > server which was already upgraded to 2.6.27.2 still gets very high
> > load.  It is a web server with NFS file storage (NetApp), and while
> > the others in the cluster (kernel 2.6.25) have a load of 1-3, 2.6.27.2
> > gets 30-50.
> > 
> > I did an oprofile, with the following results (server just started,
> > load "only" 5-10):
> > 
> > 87593    56.1116  (no location information)   vmlinux
> > vmlinux                  rpcauth_lookup_credcache
> > 16037    10.2732  auth_generic.c:0            vmlinux
> > vmlinux                  generic_match
> > 6460      4.1382  (no location information)   php4
> > php4                     (no symbols)
> > 2478      1.5874  (no location information)   libc-2.7.so
> > libc-2.7.so              (no symbols)
> > [...]
> > 
> > We havn't configured any special authentication method.  It is a NFSv3
> > over UDP mount, but the kernel has NFSv4 and therefore KRB5 enabled.
> > 
> > Any ideas why rpcauth_lookup_credcache() goes overboard with CPU
> > usage?
> 
> I have bisected the problem: 98a8e323 is the result ("SUNRPC: Add a
> helper rpcauth_lookup_generic_cred()").  5c691044 is ok.
> 
> See the attached oprofile annotation data for both commits.  I guess
> that the function rpcauth_lookup_credcache() is waiting for a spinlock
> too often and too long.  Trond, any idea?

Can you add a '-v' to the rpc.gssd daemon startup line? I'd like to see
how often you are creating new gss contexts.

> Harry: added you to Cc because your problem sounds similar.

Harry's problem is should be unrelated. afaik, he is seeing a problem
with userland RPC code, not kernel rpc code.

Trond


^ permalink raw reply	[flat|nested] 29+ messages in thread

* Re: High load in 2.6.27, NFS / rpcauth_lookup_credcache()?
  2008-10-23 14:55   ` Trond Myklebust
@ 2008-10-24  8:39       ` Max Kellermann
  2008-12-15 23:44       ` Harry Edmon
  1 sibling, 0 replies; 29+ messages in thread
From: Max Kellermann @ 2008-10-24  8:39 UTC (permalink / raw)
  To: Trond Myklebust; +Cc: linux-kernel, linux-nfs

On 2008/10/23 16:55, Trond Myklebust <trond.myklebust@fys.uio.no> wrote:
> Can you add a '-v' to the rpc.gssd daemon startup line? I'd like to see
> how often you are creating new gss contexts.

There is no rpc.gssd running on any of these servers, we're using
plain NFSv3 without (protocol based) authentication.  Do you want me
to start it anyway?

Max

^ permalink raw reply	[flat|nested] 29+ messages in thread

* Re: High load in 2.6.27, NFS / rpcauth_lookup_credcache()?
@ 2008-10-24  8:39       ` Max Kellermann
  0 siblings, 0 replies; 29+ messages in thread
From: Max Kellermann @ 2008-10-24  8:39 UTC (permalink / raw)
  To: Trond Myklebust; +Cc: linux-kernel, linux-nfs

On 2008/10/23 16:55, Trond Myklebust <trond.myklebust@fys.uio.no> wrote:
> Can you add a '-v' to the rpc.gssd daemon startup line? I'd like to see
> how often you are creating new gss contexts.

There is no rpc.gssd running on any of these servers, we're using
plain NFSv3 without (protocol based) authentication.  Do you want me
to start it anyway?

Max

^ permalink raw reply	[flat|nested] 29+ messages in thread

* Re: High load in 2.6.27, NFS / rpcauth_lookup_credcache()?
@ 2008-10-24 18:09         ` Trond Myklebust
  0 siblings, 0 replies; 29+ messages in thread
From: Trond Myklebust @ 2008-10-24 18:09 UTC (permalink / raw)
  To: Max Kellermann; +Cc: linux-kernel, linux-nfs

On Fri, 2008-10-24 at 10:39 +0200, Max Kellermann wrote:
> On 2008/10/23 16:55, Trond Myklebust <trond.myklebust@fys.uio.no> wrote:
> > Can you add a '-v' to the rpc.gssd daemon startup line? I'd like to see
> > how often you are creating new gss contexts.
> 
> There is no rpc.gssd running on any of these servers, we're using
> plain NFSv3 without (protocol based) authentication.  Do you want me
> to start it anyway?

Sorry. I misread your email. I read it as saying that you had krb5
enabled on your mountpoint, but what you really said was just that you
have compiled up the module.

OK, could you please describe your environment a bit. Do you have lots
of different users logged in at the same time, or do you perhaps use
newgrp or su to switch uid/gids a lot on your processes?
I'm trying to see if there might be a reason for the lookup in the
credcache being such a heavy duty operation in your setup.

Cheers
  Trond


^ permalink raw reply	[flat|nested] 29+ messages in thread

* Re: High load in 2.6.27, NFS / rpcauth_lookup_credcache()?
@ 2008-10-24 18:09         ` Trond Myklebust
  0 siblings, 0 replies; 29+ messages in thread
From: Trond Myklebust @ 2008-10-24 18:09 UTC (permalink / raw)
  To: Max Kellermann; +Cc: linux-kernel, linux-nfs

On Fri, 2008-10-24 at 10:39 +0200, Max Kellermann wrote:
> On 2008/10/23 16:55, Trond Myklebust <trond.myklebust@fys.uio.no> wrote:
> > Can you add a '-v' to the rpc.gssd daemon startup line? I'd like to see
> > how often you are creating new gss contexts.
> 
> There is no rpc.gssd running on any of these servers, we're using
> plain NFSv3 without (protocol based) authentication.  Do you want me
> to start it anyway?

Sorry. I misread your email. I read it as saying that you had krb5
enabled on your mountpoint, but what you really said was just that you
have compiled up the module.

OK, could you please describe your environment a bit. Do you have lots
of different users logged in at the same time, or do you perhaps use
newgrp or su to switch uid/gids a lot on your processes?
I'm trying to see if there might be a reason for the lookup in the
credcache being such a heavy duty operation in your setup.

Cheers
  Trond


^ permalink raw reply	[flat|nested] 29+ messages in thread

* Re: High load in 2.6.27, NFS / rpcauth_lookup_credcache()?
  2008-10-24 18:09         ` Trond Myklebust
@ 2008-10-27  9:58           ` Max Kellermann
  -1 siblings, 0 replies; 29+ messages in thread
From: Max Kellermann @ 2008-10-27  9:58 UTC (permalink / raw)
  To: Trond Myklebust; +Cc: linux-kernel, linux-nfs

On 2008/10/24 20:09, Trond Myklebust <trond.myklebust@fys.uio.no> wrote:
> OK, could you please describe your environment a bit. Do you have lots
> of different users logged in at the same time, or do you perhaps use
> newgrp or su to switch uid/gids a lot on your processes?
> I'm trying to see if there might be a reason for the lookup in the
> credcache being such a heavy duty operation in your setup.

It's a web server for shared hosting.  The web space is mounted via
NFSv3 from a NetApp.  There is a huge number of web sites on this
cluster.  All web sites are owned by the same UID, and the web server
runs as a different UID (read-only access).

Each time a CGI starts, its uid is changed to the one "owner" UID
(similar to mod_suexec, but there's only one UID for all customer
accounts).  Each time a CGI starts, its chroot (pivot_root) is
constructed with several bind mounts (in a separate namespace with
CLONE_NEWNS).

There are no new users or groups being created.  There are only 2 UIDs
accessing NFS: the webserver (ro) and CGI (rw).

Max

^ permalink raw reply	[flat|nested] 29+ messages in thread

* Re: High load in 2.6.27, NFS / rpcauth_lookup_credcache()?
@ 2008-10-27  9:58           ` Max Kellermann
  0 siblings, 0 replies; 29+ messages in thread
From: Max Kellermann @ 2008-10-27  9:58 UTC (permalink / raw)
  To: Trond Myklebust; +Cc: linux-kernel, linux-nfs

On 2008/10/24 20:09, Trond Myklebust <trond.myklebust@fys.uio.no> wrote:
> OK, could you please describe your environment a bit. Do you have lots
> of different users logged in at the same time, or do you perhaps use
> newgrp or su to switch uid/gids a lot on your processes?
> I'm trying to see if there might be a reason for the lookup in the
> credcache being such a heavy duty operation in your setup.

It's a web server for shared hosting.  The web space is mounted via
NFSv3 from a NetApp.  There is a huge number of web sites on this
cluster.  All web sites are owned by the same UID, and the web server
runs as a different UID (read-only access).

Each time a CGI starts, its uid is changed to the one "owner" UID
(similar to mod_suexec, but there's only one UID for all customer
accounts).  Each time a CGI starts, its chroot (pivot_root) is
constructed with several bind mounts (in a separate namespace with
CLONE_NEWNS).

There are no new users or groups being created.  There are only 2 UIDs
accessing NFS: the webserver (ro) and CGI (rw).

Max

^ permalink raw reply	[flat|nested] 29+ messages in thread

* Re: High load in 2.6.27, NFS / rpcauth_lookup_credcache()?
@ 2008-10-27 15:48             ` Trond Myklebust
  0 siblings, 0 replies; 29+ messages in thread
From: Trond Myklebust @ 2008-10-27 15:48 UTC (permalink / raw)
  To: Max Kellermann; +Cc: linux-kernel, linux-nfs

On Mon, 2008-10-27 at 10:58 +0100, Max Kellermann wrote:
> On 2008/10/24 20:09, Trond Myklebust <trond.myklebust@fys.uio.no> wrote:
> > OK, could you please describe your environment a bit. Do you have lots
> > of different users logged in at the same time, or do you perhaps use
> > newgrp or su to switch uid/gids a lot on your processes?
> > I'm trying to see if there might be a reason for the lookup in the
> > credcache being such a heavy duty operation in your setup.
> 
> It's a web server for shared hosting.  The web space is mounted via
> NFSv3 from a NetApp.  There is a huge number of web sites on this
> cluster.  All web sites are owned by the same UID, and the web server
> runs as a different UID (read-only access).
> 
> Each time a CGI starts, its uid is changed to the one "owner" UID
> (similar to mod_suexec, but there's only one UID for all customer
> accounts).  Each time a CGI starts, its chroot (pivot_root) is
> constructed with several bind mounts (in a separate namespace with
> CLONE_NEWNS).
> 
> There are no new users or groups being created.  There are only 2 UIDs
> accessing NFS: the webserver (ro) and CGI (rw).

OK. That points a finger at the garbage collector. Does the following
patch help at all?

Cheers
  Trond
------------------------------------------------------------------------
From: Trond Myklebust <Trond.Myklebust@netapp.com>
Date: Mon, 27 Oct 2008 11:43:48 -0400
SUNRPC: Fix rpcauth_prune_expired

We need to make sure that we don't remove creds from the cred_unused list
if they are still under the moratorium, or else they will never get
garbage collected.

Signed-off-by: Trond Myklebust <Trond.Myklebust@netapp.com>
---

 net/sunrpc/auth.c |   16 +++++++++-------
 1 files changed, 9 insertions(+), 7 deletions(-)


diff --git a/net/sunrpc/auth.c b/net/sunrpc/auth.c
index 436bf1b..a045a12 100644
--- a/net/sunrpc/auth.c
+++ b/net/sunrpc/auth.c
@@ -228,19 +228,21 @@ static int
 rpcauth_prune_expired(struct list_head *free, int nr_to_scan)
 {
 	spinlock_t *cache_lock;
-	struct rpc_cred *cred;
+	struct rpc_cred *cred, *next;
 	unsigned long expired = jiffies - RPC_AUTH_EXPIRY_MORATORIUM;
 
-	while (!list_empty(&cred_unused)) {
-		cred = list_entry(cred_unused.next, struct rpc_cred, cr_lru);
+	list_for_each_entry_safe(cred, next, &cred_unused, cr_lru) {
+
+		/* Enforce a 60 second garbage collection moratorium */
+		if (time_in_range(cred->cr_expire, expired, jiffies) &&
+		    test_bit(RPCAUTH_CRED_HASHED, &cred->cr_flags) != 0)
+			continue;
+
 		list_del_init(&cred->cr_lru);
 		number_cred_unused--;
 		if (atomic_read(&cred->cr_count) != 0)
 			continue;
-		/* Enforce a 5 second garbage collection moratorium */
-		if (time_in_range(cred->cr_expire, expired, jiffies) &&
-		    test_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags) != 0)
-			continue;
+
 		cache_lock = &cred->cr_auth->au_credcache->lock;
 		spin_lock(cache_lock);
 		if (atomic_read(&cred->cr_count) == 0) {



^ permalink raw reply related	[flat|nested] 29+ messages in thread

* Re: High load in 2.6.27, NFS / rpcauth_lookup_credcache()?
@ 2008-10-27 15:48             ` Trond Myklebust
  0 siblings, 0 replies; 29+ messages in thread
From: Trond Myklebust @ 2008-10-27 15:48 UTC (permalink / raw)
  To: Max Kellermann; +Cc: linux-kernel, linux-nfs

On Mon, 2008-10-27 at 10:58 +0100, Max Kellermann wrote:
> On 2008/10/24 20:09, Trond Myklebust <trond.myklebust@fys.uio.no> wrote:
> > OK, could you please describe your environment a bit. Do you have lots
> > of different users logged in at the same time, or do you perhaps use
> > newgrp or su to switch uid/gids a lot on your processes?
> > I'm trying to see if there might be a reason for the lookup in the
> > credcache being such a heavy duty operation in your setup.
> 
> It's a web server for shared hosting.  The web space is mounted via
> NFSv3 from a NetApp.  There is a huge number of web sites on this
> cluster.  All web sites are owned by the same UID, and the web server
> runs as a different UID (read-only access).
> 
> Each time a CGI starts, its uid is changed to the one "owner" UID
> (similar to mod_suexec, but there's only one UID for all customer
> accounts).  Each time a CGI starts, its chroot (pivot_root) is
> constructed with several bind mounts (in a separate namespace with
> CLONE_NEWNS).
> 
> There are no new users or groups being created.  There are only 2 UIDs
> accessing NFS: the webserver (ro) and CGI (rw).

OK. That points a finger at the garbage collector. Does the following
patch help at all?

Cheers
  Trond
------------------------------------------------------------------------
From: Trond Myklebust <Trond.Myklebust@netapp.com>
Date: Mon, 27 Oct 2008 11:43:48 -0400
SUNRPC: Fix rpcauth_prune_expired

We need to make sure that we don't remove creds from the cred_unused list
if they are still under the moratorium, or else they will never get
garbage collected.

Signed-off-by: Trond Myklebust <Trond.Myklebust@netapp.com>
---

 net/sunrpc/auth.c |   16 +++++++++-------
 1 files changed, 9 insertions(+), 7 deletions(-)


diff --git a/net/sunrpc/auth.c b/net/sunrpc/auth.c
index 436bf1b..a045a12 100644
--- a/net/sunrpc/auth.c
+++ b/net/sunrpc/auth.c
@@ -228,19 +228,21 @@ static int
 rpcauth_prune_expired(struct list_head *free, int nr_to_scan)
 {
 	spinlock_t *cache_lock;
-	struct rpc_cred *cred;
+	struct rpc_cred *cred, *next;
 	unsigned long expired = jiffies - RPC_AUTH_EXPIRY_MORATORIUM;
 
-	while (!list_empty(&cred_unused)) {
-		cred = list_entry(cred_unused.next, struct rpc_cred, cr_lru);
+	list_for_each_entry_safe(cred, next, &cred_unused, cr_lru) {
+
+		/* Enforce a 60 second garbage collection moratorium */
+		if (time_in_range(cred->cr_expire, expired, jiffies) &&
+		    test_bit(RPCAUTH_CRED_HASHED, &cred->cr_flags) != 0)
+			continue;
+
 		list_del_init(&cred->cr_lru);
 		number_cred_unused--;
 		if (atomic_read(&cred->cr_count) != 0)
 			continue;
-		/* Enforce a 5 second garbage collection moratorium */
-		if (time_in_range(cred->cr_expire, expired, jiffies) &&
-		    test_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags) != 0)
-			continue;
+
 		cache_lock = &cred->cr_auth->au_credcache->lock;
 		spin_lock(cache_lock);
 		if (atomic_read(&cred->cr_count) == 0) {



^ permalink raw reply related	[flat|nested] 29+ messages in thread

* Re: High load in 2.6.27, NFS / rpcauth_lookup_credcache()?
@ 2008-11-17 12:35               ` Max Kellermann
  0 siblings, 0 replies; 29+ messages in thread
From: Max Kellermann @ 2008-11-17 12:35 UTC (permalink / raw)
  To: Trond Myklebust; +Cc: linux-kernel, linux-nfs

On 2008/10/27 16:48, Trond Myklebust <trond.myklebust@fys.uio.no> wrote:
> OK. That points a finger at the garbage collector. Does the following
> patch help at all?

Hi Trond,

the server's load seems to be a little bit lower with 2.6.27.6 plus
your patch, but OProfile still gives me 45% in
rpcauth_lookup_credcache().  See:

1060627  45.1474  (no location information)   rpcauth_lookup_credcache
289362   12.3172  auth_generic.c:0            generic_match
31886     1.3573  page_alloc.c:0              get_page_from_freelist
23932     1.0187  (no location information)   page_fault
22176     0.9440  (no location information)   schedule
14613     0.6220  (no location information)   __d_lookup
[...]

Max

^ permalink raw reply	[flat|nested] 29+ messages in thread

* Re: High load in 2.6.27, NFS / rpcauth_lookup_credcache()?
@ 2008-11-17 12:35               ` Max Kellermann
  0 siblings, 0 replies; 29+ messages in thread
From: Max Kellermann @ 2008-11-17 12:35 UTC (permalink / raw)
  To: Trond Myklebust; +Cc: linux-kernel, linux-nfs

On 2008/10/27 16:48, Trond Myklebust <trond.myklebust@fys.uio.no> wrote:
> OK. That points a finger at the garbage collector. Does the following
> patch help at all?

Hi Trond,

the server's load seems to be a little bit lower with 2.6.27.6 plus
your patch, but OProfile still gives me 45% in
rpcauth_lookup_credcache().  See:

1060627  45.1474  (no location information)   rpcauth_lookup_credcache
289362   12.3172  auth_generic.c:0            generic_match
31886     1.3573  page_alloc.c:0              get_page_from_freelist
23932     1.0187  (no location information)   page_fault
22176     0.9440  (no location information)   schedule
14613     0.6220  (no location information)   __d_lookup
[...]

Max

^ permalink raw reply	[flat|nested] 29+ messages in thread

* Re: High load in 2.6.27, NFS / rpcauth_lookup_credcache()?
@ 2008-11-19 22:31                 ` Trond Myklebust
  0 siblings, 0 replies; 29+ messages in thread
From: Trond Myklebust @ 2008-11-19 22:31 UTC (permalink / raw)
  To: Max Kellermann; +Cc: linux-kernel, linux-nfs

On Mon, 2008-11-17 at 13:35 +0100, Max Kellermann wrote:
> On 2008/10/27 16:48, Trond Myklebust <trond.myklebust@fys.uio.no> wrote:
> > OK. That points a finger at the garbage collector. Does the following
> > patch help at all?
> 
> Hi Trond,
> 
> the server's load seems to be a little bit lower with 2.6.27.6 plus
> your patch, but OProfile still gives me 45% in
> rpcauth_lookup_credcache().  See:
> 
> 1060627  45.1474  (no location information)   rpcauth_lookup_credcache
> 289362   12.3172  auth_generic.c:0            generic_match
> 31886     1.3573  page_alloc.c:0              get_page_from_freelist
> 23932     1.0187  (no location information)   page_fault
> 22176     0.9440  (no location information)   schedule
> 14613     0.6220  (no location information)   __d_lookup
> [...]
> 
> Max

Looking at the above, it seems that you're spending an inordinate amount
of time in generic_match too.
Could you see if the following patch helps?

Cheers
  Trond

-------------------------------------------------------------------------------
From: Trond Myklebust <Trond.Myklebust@netapp.com>
SUNRPC: Fix up generic_match()

Cut down on the number of similar copies of group_info.

Signed-off-by: Trond Myklebust <Trond.Myklebust@netapp.com>
---

 net/sunrpc/auth_generic.c |   20 ++++++++++++++++++--
 1 files changed, 18 insertions(+), 2 deletions(-)


diff --git a/net/sunrpc/auth_generic.c b/net/sunrpc/auth_generic.c
index 744b79f..4028502 100644
--- a/net/sunrpc/auth_generic.c
+++ b/net/sunrpc/auth_generic.c
@@ -133,13 +133,29 @@ static int
 generic_match(struct auth_cred *acred, struct rpc_cred *cred, int flags)
 {
 	struct generic_cred *gcred = container_of(cred, struct generic_cred, gc_base);
+	int i;
 
 	if (gcred->acred.uid != acred->uid ||
 	    gcred->acred.gid != acred->gid ||
-	    gcred->acred.group_info != acred->group_info ||
 	    gcred->acred.machine_cred != acred->machine_cred)
-		return 0;
+		goto out_nomatch;
+
+	/* Optimisation in the case where pointers are identical... */
+	if (gcred->acred.group_info == acred->group_info)
+		goto out_match;
+
+	/* Slow path... */
+	if (gcred->acred.group_info->ngroups != acred->group_info->ngroups)
+		goto out_nomatch;
+	for (i = 0; i < gcred->acred.group_info->ngroups; i++) {
+		if (GROUP_AT(gcred->acred.group_info, i) !=
+				GROUP_AT(acred->group_info, i))
+			goto out_nomatch;
+	}
+out_match:
 	return 1;
+out_nomatch:
+	return 0;
 }
 
 void __init rpc_init_generic_auth(void)



^ permalink raw reply related	[flat|nested] 29+ messages in thread

* Re: High load in 2.6.27, NFS / rpcauth_lookup_credcache()?
@ 2008-11-19 22:31                 ` Trond Myklebust
  0 siblings, 0 replies; 29+ messages in thread
From: Trond Myklebust @ 2008-11-19 22:31 UTC (permalink / raw)
  To: Max Kellermann; +Cc: linux-kernel, linux-nfs

On Mon, 2008-11-17 at 13:35 +0100, Max Kellermann wrote:
> On 2008/10/27 16:48, Trond Myklebust <trond.myklebust@fys.uio.no> wrote:
> > OK. That points a finger at the garbage collector. Does the following
> > patch help at all?
> 
> Hi Trond,
> 
> the server's load seems to be a little bit lower with 2.6.27.6 plus
> your patch, but OProfile still gives me 45% in
> rpcauth_lookup_credcache().  See:
> 
> 1060627  45.1474  (no location information)   rpcauth_lookup_credcache
> 289362   12.3172  auth_generic.c:0            generic_match
> 31886     1.3573  page_alloc.c:0              get_page_from_freelist
> 23932     1.0187  (no location information)   page_fault
> 22176     0.9440  (no location information)   schedule
> 14613     0.6220  (no location information)   __d_lookup
> [...]
> 
> Max

Looking at the above, it seems that you're spending an inordinate amount
of time in generic_match too.
Could you see if the following patch helps?

Cheers
  Trond

-------------------------------------------------------------------------------
From: Trond Myklebust <Trond.Myklebust@netapp.com>
SUNRPC: Fix up generic_match()

Cut down on the number of similar copies of group_info.

Signed-off-by: Trond Myklebust <Trond.Myklebust@netapp.com>
---

 net/sunrpc/auth_generic.c |   20 ++++++++++++++++++--
 1 files changed, 18 insertions(+), 2 deletions(-)


diff --git a/net/sunrpc/auth_generic.c b/net/sunrpc/auth_generic.c
index 744b79f..4028502 100644
--- a/net/sunrpc/auth_generic.c
+++ b/net/sunrpc/auth_generic.c
@@ -133,13 +133,29 @@ static int
 generic_match(struct auth_cred *acred, struct rpc_cred *cred, int flags)
 {
 	struct generic_cred *gcred = container_of(cred, struct generic_cred, gc_base);
+	int i;
 
 	if (gcred->acred.uid != acred->uid ||
 	    gcred->acred.gid != acred->gid ||
-	    gcred->acred.group_info != acred->group_info ||
 	    gcred->acred.machine_cred != acred->machine_cred)
-		return 0;
+		goto out_nomatch;
+
+	/* Optimisation in the case where pointers are identical... */
+	if (gcred->acred.group_info == acred->group_info)
+		goto out_match;
+
+	/* Slow path... */
+	if (gcred->acred.group_info->ngroups != acred->group_info->ngroups)
+		goto out_nomatch;
+	for (i = 0; i < gcred->acred.group_info->ngroups; i++) {
+		if (GROUP_AT(gcred->acred.group_info, i) !=
+				GROUP_AT(acred->group_info, i))
+			goto out_nomatch;
+	}
+out_match:
 	return 1;
+out_nomatch:
+	return 0;
 }
 
 void __init rpc_init_generic_auth(void)



^ permalink raw reply related	[flat|nested] 29+ messages in thread

* Re: High load in 2.6.27, NFS / rpcauth_lookup_credcache()?
@ 2008-11-20 14:08                   ` Max Kellermann
  0 siblings, 0 replies; 29+ messages in thread
From: Max Kellermann @ 2008-11-20 14:08 UTC (permalink / raw)
  To: Trond Myklebust; +Cc: linux-kernel, linux-nfs

On 2008/11/19 23:31, Trond Myklebust <trond.myklebust@fys.uio.no> wrote:
> Looking at the above, it seems that you're spending an inordinate amount
> of time in generic_match too.
> Could you see if the following patch helps?

That fixes the problem.  The first kernel symbol in oprofile is the
tg3 network driver at 2%.

Max

^ permalink raw reply	[flat|nested] 29+ messages in thread

* Re: High load in 2.6.27, NFS / rpcauth_lookup_credcache()?
@ 2008-11-20 14:08                   ` Max Kellermann
  0 siblings, 0 replies; 29+ messages in thread
From: Max Kellermann @ 2008-11-20 14:08 UTC (permalink / raw)
  To: Trond Myklebust; +Cc: linux-kernel, linux-nfs

On 2008/11/19 23:31, Trond Myklebust <trond.myklebust@fys.uio.no> wrote:
> Looking at the above, it seems that you're spending an inordinate amount
> of time in generic_match too.
> Could you see if the following patch helps?

That fixes the problem.  The first kernel symbol in oprofile is the
tg3 network driver at 2%.

Max

^ permalink raw reply	[flat|nested] 29+ messages in thread

* Re: High load in 2.6.27, NFS / rpcauth_lookup_credcache()?
  2008-10-23 14:55   ` Trond Myklebust
@ 2008-12-15 23:44       ` Harry Edmon
  2008-12-15 23:44       ` Harry Edmon
  1 sibling, 0 replies; 29+ messages in thread
From: Harry Edmon @ 2008-12-15 23:44 UTC (permalink / raw)
  To: Trond Myklebust; +Cc: Max Kellermann, linux-kernel, linux-nfs

Trond Myklebust wrote:
> On Thu, 2008-10-23 at 14:36 +0200, Max Kellermann wrote:
>   
>> On 2008/10/22 11:12, Max Kellermann <max@duempel.org> wrote:
>>     
>>> after I was able to fix http://lkml.org/lkml/2008/10/17/147, the
>>> server which was already upgraded to 2.6.27.2 still gets very high
>>> load.  It is a web server with NFS file storage (NetApp), and while
>>> the others in the cluster (kernel 2.6.25) have a load of 1-3, 2.6.27.2
>>> gets 30-50.
>>>
>>> I did an oprofile, with the following results (server just started,
>>> load "only" 5-10):
>>>
>>> 87593    56.1116  (no location information)   vmlinux
>>> vmlinux                  rpcauth_lookup_credcache
>>> 16037    10.2732  auth_generic.c:0            vmlinux
>>> vmlinux                  generic_match
>>> 6460      4.1382  (no location information)   php4
>>> php4                     (no symbols)
>>> 2478      1.5874  (no location information)   libc-2.7.so
>>> libc-2.7.so              (no symbols)
>>> [...]
>>>
>>> We havn't configured any special authentication method.  It is a NFSv3
>>> over UDP mount, but the kernel has NFSv4 and therefore KRB5 enabled.
>>>
>>> Any ideas why rpcauth_lookup_credcache() goes overboard with CPU
>>> usage?
>>>       
>> I have bisected the problem: 98a8e323 is the result ("SUNRPC: Add a
>> helper rpcauth_lookup_generic_cred()").  5c691044 is ok.
>>
>> See the attached oprofile annotation data for both commits.  I guess
>> that the function rpcauth_lookup_credcache() is waiting for a spinlock
>> too often and too long.  Trond, any idea?
>>     
>
> Can you add a '-v' to the rpc.gssd daemon startup line? I'd like to see
> how often you are creating new gss contexts.
>
>   
>> Harry: added you to Cc because your problem sounds similar.
>>     
>
> Harry's problem is should be unrelated. afaik, he is seeing a problem
> with userland RPC code, not kernel rpc code.
>
> Trond
>
>   
I am finally getting some time to look at my problem that I originally 
reported in October (SUNRPC problem with 2.6.26 and beyond), and I am 
seeing the same behavior as Max Kellermann when my machine slows as I 
described earlier.  The system in question is currently running 
2.6.27.7.  Here is what I see when it is misbehaving:

samples  %        image name               app name                 
symbol name
11380517 57.4191  sunrpc.ko                sunrpc                   
rpcauth_lookup_credcache
3263657  16.4664  sunrpc.ko                sunrpc                   
generic_match
1081287   5.4555  vmlinux                  vmlinux                  
copy_user_generic_string
499407    2.5197  vmlinux                  vmlinux                  
__posix_lock_file
[...]

And here is what I see when I stop the programs that are chewing up all 
the system time, and then starting them up again:

samples  %        image name               app name                 
symbol name
6372650  21.7978  vmlinux                  vmlinux                  
copy_user_generic_string
5401386  18.4755  sunrpc.ko                sunrpc                   
rpcauth_lookup_credcache
3018753  10.3257  vmlinux                  vmlinux                  
__posix_lock_file
1050095   3.5919  sunrpc.ko                sunrpc                   
generic_match


and I am not using Kerberos with NFSv4 (i.e. no rpc.gssd).  Did you ever 
find a solution for this problem with rpcauth_lookup_credcache?

-- 

 Dr. Harry Edmon			E-MAIL: harry@atmos.washington.edu
 206-543-0547				harry@washington.edu
 Dept of Atmospheric Sciences		FAX:	206-543-0308
 University of Washington, Box 351640, Seattle, WA 98195-1640


^ permalink raw reply	[flat|nested] 29+ messages in thread

* Re: High load in 2.6.27, NFS / rpcauth_lookup_credcache()?
@ 2008-12-15 23:44       ` Harry Edmon
  0 siblings, 0 replies; 29+ messages in thread
From: Harry Edmon @ 2008-12-15 23:44 UTC (permalink / raw)
  To: Trond Myklebust; +Cc: Max Kellermann, linux-kernel, linux-nfs

Trond Myklebust wrote:
> On Thu, 2008-10-23 at 14:36 +0200, Max Kellermann wrote:
>   
>> On 2008/10/22 11:12, Max Kellermann <max-hDT0AjmEH7RAfugRpC6u6w@public.gmane.org> wrote:
>>     
>>> after I was able to fix http://lkml.org/lkml/2008/10/17/147, the
>>> server which was already upgraded to 2.6.27.2 still gets very high
>>> load.  It is a web server with NFS file storage (NetApp), and while
>>> the others in the cluster (kernel 2.6.25) have a load of 1-3, 2.6.27.2
>>> gets 30-50.
>>>
>>> I did an oprofile, with the following results (server just started,
>>> load "only" 5-10):
>>>
>>> 87593    56.1116  (no location information)   vmlinux
>>> vmlinux                  rpcauth_lookup_credcache
>>> 16037    10.2732  auth_generic.c:0            vmlinux
>>> vmlinux                  generic_match
>>> 6460      4.1382  (no location information)   php4
>>> php4                     (no symbols)
>>> 2478      1.5874  (no location information)   libc-2.7.so
>>> libc-2.7.so              (no symbols)
>>> [...]
>>>
>>> We havn't configured any special authentication method.  It is a NFSv3
>>> over UDP mount, but the kernel has NFSv4 and therefore KRB5 enabled.
>>>
>>> Any ideas why rpcauth_lookup_credcache() goes overboard with CPU
>>> usage?
>>>       
>> I have bisected the problem: 98a8e323 is the result ("SUNRPC: Add a
>> helper rpcauth_lookup_generic_cred()").  5c691044 is ok.
>>
>> See the attached oprofile annotation data for both commits.  I guess
>> that the function rpcauth_lookup_credcache() is waiting for a spinlock
>> too often and too long.  Trond, any idea?
>>     
>
> Can you add a '-v' to the rpc.gssd daemon startup line? I'd like to see
> how often you are creating new gss contexts.
>
>   
>> Harry: added you to Cc because your problem sounds similar.
>>     
>
> Harry's problem is should be unrelated. afaik, he is seeing a problem
> with userland RPC code, not kernel rpc code.
>
> Trond
>
>   
I am finally getting some time to look at my problem that I originally 
reported in October (SUNRPC problem with 2.6.26 and beyond), and I am 
seeing the same behavior as Max Kellermann when my machine slows as I 
described earlier.  The system in question is currently running 
2.6.27.7.  Here is what I see when it is misbehaving:

samples  %        image name               app name                 
symbol name
11380517 57.4191  sunrpc.ko                sunrpc                   
rpcauth_lookup_credcache
3263657  16.4664  sunrpc.ko                sunrpc                   
generic_match
1081287   5.4555  vmlinux                  vmlinux                  
copy_user_generic_string
499407    2.5197  vmlinux                  vmlinux                  
__posix_lock_file
[...]

And here is what I see when I stop the programs that are chewing up all 
the system time, and then starting them up again:

samples  %        image name               app name                 
symbol name
6372650  21.7978  vmlinux                  vmlinux                  
copy_user_generic_string
5401386  18.4755  sunrpc.ko                sunrpc                   
rpcauth_lookup_credcache
3018753  10.3257  vmlinux                  vmlinux                  
__posix_lock_file
1050095   3.5919  sunrpc.ko                sunrpc                   
generic_match


and I am not using Kerberos with NFSv4 (i.e. no rpc.gssd).  Did you ever 
find a solution for this problem with rpcauth_lookup_credcache?

-- 

 Dr. Harry Edmon			E-MAIL: harry-qmPYOCrcNLLyFCzt5hm0YvZ8FUJU4vz8@public.gmane.org
 206-543-0547				harry-B93hV6UPU7Z2icitjWtXSw@public.gmane.org
 Dept of Atmospheric Sciences		FAX:	206-543-0308
 University of Washington, Box 351640, Seattle, WA 98195-1640


^ permalink raw reply	[flat|nested] 29+ messages in thread

* Re: High load in 2.6.27, NFS / rpcauth_lookup_credcache()?
@ 2008-12-16 13:02         ` Trond Myklebust
  0 siblings, 0 replies; 29+ messages in thread
From: Trond Myklebust @ 2008-12-16 13:02 UTC (permalink / raw)
  To: Harry Edmon; +Cc: Max Kellermann, linux-kernel, linux-nfs

On Mon, 2008-12-15 at 15:44 -0800, Harry Edmon wrote:
> Trond Myklebust wrote:
> > On Thu, 2008-10-23 at 14:36 +0200, Max Kellermann wrote:
> >   
> >> On 2008/10/22 11:12, Max Kellermann <max@duempel.org> wrote:
> >>     
> >>> after I was able to fix http://lkml.org/lkml/2008/10/17/147, the
> >>> server which was already upgraded to 2.6.27.2 still gets very high
> >>> load.  It is a web server with NFS file storage (NetApp), and while
> >>> the others in the cluster (kernel 2.6.25) have a load of 1-3, 2.6.27.2
> >>> gets 30-50.
> >>>
> >>> I did an oprofile, with the following results (server just started,
> >>> load "only" 5-10):
> >>>
> >>> 87593    56.1116  (no location information)   vmlinux
> >>> vmlinux                  rpcauth_lookup_credcache
> >>> 16037    10.2732  auth_generic.c:0            vmlinux
> >>> vmlinux                  generic_match
> >>> 6460      4.1382  (no location information)   php4
> >>> php4                     (no symbols)
> >>> 2478      1.5874  (no location information)   libc-2.7.so
> >>> libc-2.7.so              (no symbols)
> >>> [...]
> >>>
> >>> We havn't configured any special authentication method.  It is a NFSv3
> >>> over UDP mount, but the kernel has NFSv4 and therefore KRB5 enabled.
> >>>
> >>> Any ideas why rpcauth_lookup_credcache() goes overboard with CPU
> >>> usage?
> >>>       
> >> I have bisected the problem: 98a8e323 is the result ("SUNRPC: Add a
> >> helper rpcauth_lookup_generic_cred()").  5c691044 is ok.
> >>
> >> See the attached oprofile annotation data for both commits.  I guess
> >> that the function rpcauth_lookup_credcache() is waiting for a spinlock
> >> too often and too long.  Trond, any idea?
> >>     
> >
> > Can you add a '-v' to the rpc.gssd daemon startup line? I'd like to see
> > how often you are creating new gss contexts.
> >
> >   
> >> Harry: added you to Cc because your problem sounds similar.
> >>     
> >
> > Harry's problem is should be unrelated. afaik, he is seeing a problem
> > with userland RPC code, not kernel rpc code.
> >
> > Trond
> >
> >   
> I am finally getting some time to look at my problem that I originally 
> reported in October (SUNRPC problem with 2.6.26 and beyond), and I am 
> seeing the same behavior as Max Kellermann when my machine slows as I 
> described earlier.  The system in question is currently running 
> 2.6.27.7.  Here is what I see when it is misbehaving:
> 
> samples  %        image name               app name                 
> symbol name
> 11380517 57.4191  sunrpc.ko                sunrpc                   
> rpcauth_lookup_credcache
> 3263657  16.4664  sunrpc.ko                sunrpc                   
> generic_match
> 1081287   5.4555  vmlinux                  vmlinux                  
> copy_user_generic_string
> 499407    2.5197  vmlinux                  vmlinux                  
> __posix_lock_file
> [...]
> 
> And here is what I see when I stop the programs that are chewing up all 
> the system time, and then starting them up again:
> 
> samples  %        image name               app name                 
> symbol name
> 6372650  21.7978  vmlinux                  vmlinux                  
> copy_user_generic_string
> 5401386  18.4755  sunrpc.ko                sunrpc                   
> rpcauth_lookup_credcache
> 3018753  10.3257  vmlinux                  vmlinux                  
> __posix_lock_file
> 1050095   3.5919  sunrpc.ko                sunrpc                   
> generic_match
> 
> 
> and I am not using Kerberos with NFSv4 (i.e. no rpc.gssd).  Did you ever 
> find a solution for this problem with rpcauth_lookup_credcache?

http://git.kernel.org/?p=linux/kernel/git/torvalds/linux-2.6.git&a=commitdiff&h=23918b03060f6e572168fdde1798a905679d2e06

Trond


^ permalink raw reply	[flat|nested] 29+ messages in thread

* Re: High load in 2.6.27, NFS / rpcauth_lookup_credcache()?
@ 2008-12-16 13:02         ` Trond Myklebust
  0 siblings, 0 replies; 29+ messages in thread
From: Trond Myklebust @ 2008-12-16 13:02 UTC (permalink / raw)
  To: Harry Edmon; +Cc: Max Kellermann, linux-kernel, linux-nfs

On Mon, 2008-12-15 at 15:44 -0800, Harry Edmon wrote:
> Trond Myklebust wrote:
> > On Thu, 2008-10-23 at 14:36 +0200, Max Kellermann wrote:
> >   
> >> On 2008/10/22 11:12, Max Kellermann <max-hDT0AjmEH7RAfugRpC6u6w@public.gmane.org> wrote:
> >>     
> >>> after I was able to fix http://lkml.org/lkml/2008/10/17/147, the
> >>> server which was already upgraded to 2.6.27.2 still gets very high
> >>> load.  It is a web server with NFS file storage (NetApp), and while
> >>> the others in the cluster (kernel 2.6.25) have a load of 1-3, 2.6.27.2
> >>> gets 30-50.
> >>>
> >>> I did an oprofile, with the following results (server just started,
> >>> load "only" 5-10):
> >>>
> >>> 87593    56.1116  (no location information)   vmlinux
> >>> vmlinux                  rpcauth_lookup_credcache
> >>> 16037    10.2732  auth_generic.c:0            vmlinux
> >>> vmlinux                  generic_match
> >>> 6460      4.1382  (no location information)   php4
> >>> php4                     (no symbols)
> >>> 2478      1.5874  (no location information)   libc-2.7.so
> >>> libc-2.7.so              (no symbols)
> >>> [...]
> >>>
> >>> We havn't configured any special authentication method.  It is a NFSv3
> >>> over UDP mount, but the kernel has NFSv4 and therefore KRB5 enabled.
> >>>
> >>> Any ideas why rpcauth_lookup_credcache() goes overboard with CPU
> >>> usage?
> >>>       
> >> I have bisected the problem: 98a8e323 is the result ("SUNRPC: Add a
> >> helper rpcauth_lookup_generic_cred()").  5c691044 is ok.
> >>
> >> See the attached oprofile annotation data for both commits.  I guess
> >> that the function rpcauth_lookup_credcache() is waiting for a spinlock
> >> too often and too long.  Trond, any idea?
> >>     
> >
> > Can you add a '-v' to the rpc.gssd daemon startup line? I'd like to see
> > how often you are creating new gss contexts.
> >
> >   
> >> Harry: added you to Cc because your problem sounds similar.
> >>     
> >
> > Harry's problem is should be unrelated. afaik, he is seeing a problem
> > with userland RPC code, not kernel rpc code.
> >
> > Trond
> >
> >   
> I am finally getting some time to look at my problem that I originally 
> reported in October (SUNRPC problem with 2.6.26 and beyond), and I am 
> seeing the same behavior as Max Kellermann when my machine slows as I 
> described earlier.  The system in question is currently running 
> 2.6.27.7.  Here is what I see when it is misbehaving:
> 
> samples  %        image name               app name                 
> symbol name
> 11380517 57.4191  sunrpc.ko                sunrpc                   
> rpcauth_lookup_credcache
> 3263657  16.4664  sunrpc.ko                sunrpc                   
> generic_match
> 1081287   5.4555  vmlinux                  vmlinux                  
> copy_user_generic_string
> 499407    2.5197  vmlinux                  vmlinux                  
> __posix_lock_file
> [...]
> 
> And here is what I see when I stop the programs that are chewing up all 
> the system time, and then starting them up again:
> 
> samples  %        image name               app name                 
> symbol name
> 6372650  21.7978  vmlinux                  vmlinux                  
> copy_user_generic_string
> 5401386  18.4755  sunrpc.ko                sunrpc                   
> rpcauth_lookup_credcache
> 3018753  10.3257  vmlinux                  vmlinux                  
> __posix_lock_file
> 1050095   3.5919  sunrpc.ko                sunrpc                   
> generic_match
> 
> 
> and I am not using Kerberos with NFSv4 (i.e. no rpc.gssd).  Did you ever 
> find a solution for this problem with rpcauth_lookup_credcache?

http://git.kernel.org/?p=linux/kernel/git/torvalds/linux-2.6.git&a=commitdiff&h=23918b03060f6e572168fdde1798a905679d2e06

Trond


^ permalink raw reply	[flat|nested] 29+ messages in thread

* Re: High load in 2.6.27, NFS / rpcauth_lookup_credcache()?
@ 2008-12-16 21:21           ` Willy Tarreau
  0 siblings, 0 replies; 29+ messages in thread
From: Willy Tarreau @ 2008-12-16 21:21 UTC (permalink / raw)
  To: Trond Myklebust
  Cc: Harry Edmon, Max Kellermann, linux-kernel, linux-nfs, stable

On Tue, Dec 16, 2008 at 08:02:33AM -0500, Trond Myklebust wrote:
> On Mon, 2008-12-15 at 15:44 -0800, Harry Edmon wrote:
> > Trond Myklebust wrote:
> > > On Thu, 2008-10-23 at 14:36 +0200, Max Kellermann wrote:
> > >   
> > >> On 2008/10/22 11:12, Max Kellermann <max@duempel.org> wrote:
> > >>     
> > >>> after I was able to fix http://lkml.org/lkml/2008/10/17/147, the
> > >>> server which was already upgraded to 2.6.27.2 still gets very high
> > >>> load.  It is a web server with NFS file storage (NetApp), and while
> > >>> the others in the cluster (kernel 2.6.25) have a load of 1-3, 2.6.27.2
> > >>> gets 30-50.
> > >>>
> > >>> I did an oprofile, with the following results (server just started,
> > >>> load "only" 5-10):
> > >>>
> > >>> 87593    56.1116  (no location information)   vmlinux
> > >>> vmlinux                  rpcauth_lookup_credcache
> > >>> 16037    10.2732  auth_generic.c:0            vmlinux
> > >>> vmlinux                  generic_match
> > >>> 6460      4.1382  (no location information)   php4
> > >>> php4                     (no symbols)
> > >>> 2478      1.5874  (no location information)   libc-2.7.so
> > >>> libc-2.7.so              (no symbols)
> > >>> [...]
> > >>>
> > >>> We havn't configured any special authentication method.  It is a NFSv3
> > >>> over UDP mount, but the kernel has NFSv4 and therefore KRB5 enabled.
> > >>>
> > >>> Any ideas why rpcauth_lookup_credcache() goes overboard with CPU
> > >>> usage?
> > >>>       
> > >> I have bisected the problem: 98a8e323 is the result ("SUNRPC: Add a
> > >> helper rpcauth_lookup_generic_cred()").  5c691044 is ok.
> > >>
> > >> See the attached oprofile annotation data for both commits.  I guess
> > >> that the function rpcauth_lookup_credcache() is waiting for a spinlock
> > >> too often and too long.  Trond, any idea?
> > >>     
> > >
> > > Can you add a '-v' to the rpc.gssd daemon startup line? I'd like to see
> > > how often you are creating new gss contexts.
> > >
> > >   
> > >> Harry: added you to Cc because your problem sounds similar.
> > >>     
> > >
> > > Harry's problem is should be unrelated. afaik, he is seeing a problem
> > > with userland RPC code, not kernel rpc code.
> > >
> > > Trond
> > >
> > >   
> > I am finally getting some time to look at my problem that I originally 
> > reported in October (SUNRPC problem with 2.6.26 and beyond), and I am 
> > seeing the same behavior as Max Kellermann when my machine slows as I 
> > described earlier.  The system in question is currently running 
> > 2.6.27.7.  Here is what I see when it is misbehaving:
> > 
> > samples  %        image name               app name                 
> > symbol name
> > 11380517 57.4191  sunrpc.ko                sunrpc                   
> > rpcauth_lookup_credcache
> > 3263657  16.4664  sunrpc.ko                sunrpc                   
> > generic_match
> > 1081287   5.4555  vmlinux                  vmlinux                  
> > copy_user_generic_string
> > 499407    2.5197  vmlinux                  vmlinux                  
> > __posix_lock_file
> > [...]
> > 
> > And here is what I see when I stop the programs that are chewing up all 
> > the system time, and then starting them up again:
> > 
> > samples  %        image name               app name                 
> > symbol name
> > 6372650  21.7978  vmlinux                  vmlinux                  
> > copy_user_generic_string
> > 5401386  18.4755  sunrpc.ko                sunrpc                   
> > rpcauth_lookup_credcache
> > 3018753  10.3257  vmlinux                  vmlinux                  
> > __posix_lock_file
> > 1050095   3.5919  sunrpc.ko                sunrpc                   
> > generic_match
> > 
> > 
> > and I am not using Kerberos with NFSv4 (i.e. no rpc.gssd).  Did you ever 
> > find a solution for this problem with rpcauth_lookup_credcache?
> 
> http://git.kernel.org/?p=linux/kernel/git/torvalds/linux-2.6.git&a=commitdiff&h=23918b03060f6e572168fdde1798a905679d2e06

Trond, this should be included into next stable, right ?

It's fortunate because I know someone else who recently described me the
same problem under the same circumstances when migrating from 2.6.22 to
2.6.27.

Regards,
Willy


^ permalink raw reply	[flat|nested] 29+ messages in thread

* Re: High load in 2.6.27, NFS / rpcauth_lookup_credcache()?
@ 2008-12-16 21:21           ` Willy Tarreau
  0 siblings, 0 replies; 29+ messages in thread
From: Willy Tarreau @ 2008-12-16 21:21 UTC (permalink / raw)
  To: Trond Myklebust
  Cc: Harry Edmon, Max Kellermann, linux-kernel, linux-nfs, stable

On Tue, Dec 16, 2008 at 08:02:33AM -0500, Trond Myklebust wrote:
> On Mon, 2008-12-15 at 15:44 -0800, Harry Edmon wrote:
> > Trond Myklebust wrote:
> > > On Thu, 2008-10-23 at 14:36 +0200, Max Kellermann wrote:
> > >   
> > >> On 2008/10/22 11:12, Max Kellermann <max-hDT0AjmEH7RAfugRpC6u6w@public.gmane.org> wrote:
> > >>     
> > >>> after I was able to fix http://lkml.org/lkml/2008/10/17/147, the
> > >>> server which was already upgraded to 2.6.27.2 still gets very high
> > >>> load.  It is a web server with NFS file storage (NetApp), and while
> > >>> the others in the cluster (kernel 2.6.25) have a load of 1-3, 2.6.27.2
> > >>> gets 30-50.
> > >>>
> > >>> I did an oprofile, with the following results (server just started,
> > >>> load "only" 5-10):
> > >>>
> > >>> 87593    56.1116  (no location information)   vmlinux
> > >>> vmlinux                  rpcauth_lookup_credcache
> > >>> 16037    10.2732  auth_generic.c:0            vmlinux
> > >>> vmlinux                  generic_match
> > >>> 6460      4.1382  (no location information)   php4
> > >>> php4                     (no symbols)
> > >>> 2478      1.5874  (no location information)   libc-2.7.so
> > >>> libc-2.7.so              (no symbols)
> > >>> [...]
> > >>>
> > >>> We havn't configured any special authentication method.  It is a NFSv3
> > >>> over UDP mount, but the kernel has NFSv4 and therefore KRB5 enabled.
> > >>>
> > >>> Any ideas why rpcauth_lookup_credcache() goes overboard with CPU
> > >>> usage?
> > >>>       
> > >> I have bisected the problem: 98a8e323 is the result ("SUNRPC: Add a
> > >> helper rpcauth_lookup_generic_cred()").  5c691044 is ok.
> > >>
> > >> See the attached oprofile annotation data for both commits.  I guess
> > >> that the function rpcauth_lookup_credcache() is waiting for a spinlock
> > >> too often and too long.  Trond, any idea?
> > >>     
> > >
> > > Can you add a '-v' to the rpc.gssd daemon startup line? I'd like to see
> > > how often you are creating new gss contexts.
> > >
> > >   
> > >> Harry: added you to Cc because your problem sounds similar.
> > >>     
> > >
> > > Harry's problem is should be unrelated. afaik, he is seeing a problem
> > > with userland RPC code, not kernel rpc code.
> > >
> > > Trond
> > >
> > >   
> > I am finally getting some time to look at my problem that I originally 
> > reported in October (SUNRPC problem with 2.6.26 and beyond), and I am 
> > seeing the same behavior as Max Kellermann when my machine slows as I 
> > described earlier.  The system in question is currently running 
> > 2.6.27.7.  Here is what I see when it is misbehaving:
> > 
> > samples  %        image name               app name                 
> > symbol name
> > 11380517 57.4191  sunrpc.ko                sunrpc                   
> > rpcauth_lookup_credcache
> > 3263657  16.4664  sunrpc.ko                sunrpc                   
> > generic_match
> > 1081287   5.4555  vmlinux                  vmlinux                  
> > copy_user_generic_string
> > 499407    2.5197  vmlinux                  vmlinux                  
> > __posix_lock_file
> > [...]
> > 
> > And here is what I see when I stop the programs that are chewing up all 
> > the system time, and then starting them up again:
> > 
> > samples  %        image name               app name                 
> > symbol name
> > 6372650  21.7978  vmlinux                  vmlinux                  
> > copy_user_generic_string
> > 5401386  18.4755  sunrpc.ko                sunrpc                   
> > rpcauth_lookup_credcache
> > 3018753  10.3257  vmlinux                  vmlinux                  
> > __posix_lock_file
> > 1050095   3.5919  sunrpc.ko                sunrpc                   
> > generic_match
> > 
> > 
> > and I am not using Kerberos with NFSv4 (i.e. no rpc.gssd).  Did you ever 
> > find a solution for this problem with rpcauth_lookup_credcache?
> 
> http://git.kernel.org/?p=linux/kernel/git/torvalds/linux-2.6.git&a=commitdiff&h=23918b03060f6e572168fdde1798a905679d2e06

Trond, this should be included into next stable, right ?

It's fortunate because I know someone else who recently described me the
same problem under the same circumstances when migrating from 2.6.22 to
2.6.27.

Regards,
Willy


^ permalink raw reply	[flat|nested] 29+ messages in thread

* Re: [stable] High load in 2.6.27, NFS / rpcauth_lookup_credcache()?
@ 2008-12-16 23:21             ` Greg KH
  0 siblings, 0 replies; 29+ messages in thread
From: Greg KH @ 2008-12-16 23:21 UTC (permalink / raw)
  To: Willy Tarreau
  Cc: Trond Myklebust, Harry Edmon, linux-nfs, Max Kellermann,
	linux-kernel, stable

On Tue, Dec 16, 2008 at 10:21:55PM +0100, Willy Tarreau wrote:
> On Tue, Dec 16, 2008 at 08:02:33AM -0500, Trond Myklebust wrote:
> > > and I am not using Kerberos with NFSv4 (i.e. no rpc.gssd).  Did you ever 
> > > find a solution for this problem with rpcauth_lookup_credcache?
> > 
> > http://git.kernel.org/?p=linux/kernel/git/torvalds/linux-2.6.git&a=commitdiff&h=23918b03060f6e572168fdde1798a905679d2e06
> 
> Trond, this should be included into next stable, right ?
> 
> It's fortunate because I know someone else who recently described me the
> same problem under the same circumstances when migrating from 2.6.22 to
> 2.6.27.

Now queued up in the 2.6.27-stable queue.

thanks,

greg k-h

^ permalink raw reply	[flat|nested] 29+ messages in thread

* Re: [stable] High load in 2.6.27, NFS / rpcauth_lookup_credcache()?
@ 2008-12-16 23:21             ` Greg KH
  0 siblings, 0 replies; 29+ messages in thread
From: Greg KH @ 2008-12-16 23:21 UTC (permalink / raw)
  To: Willy Tarreau
  Cc: Trond Myklebust, Harry Edmon, linux-nfs, Max Kellermann,
	linux-kernel, stable

On Tue, Dec 16, 2008 at 10:21:55PM +0100, Willy Tarreau wrote:
> On Tue, Dec 16, 2008 at 08:02:33AM -0500, Trond Myklebust wrote:
> > > and I am not using Kerberos with NFSv4 (i.e. no rpc.gssd).  Did you ever 
> > > find a solution for this problem with rpcauth_lookup_credcache?
> > 
> > http://git.kernel.org/?p=linux/kernel/git/torvalds/linux-2.6.git&a=commitdiff&h=23918b03060f6e572168fdde1798a905679d2e06
> 
> Trond, this should be included into next stable, right ?
> 
> It's fortunate because I know someone else who recently described me the
> same problem under the same circumstances when migrating from 2.6.22 to
> 2.6.27.

Now queued up in the 2.6.27-stable queue.

thanks,

greg k-h

^ permalink raw reply	[flat|nested] 29+ messages in thread

end of thread, other threads:[~2008-12-16 23:27 UTC | newest]

Thread overview: 29+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2008-10-22  9:12 High load in 2.6.27, NFS / rpcauth_lookup_credcache()? Max Kellermann
2008-10-22  9:12 ` Max Kellermann
2008-10-22 17:56 ` J. Bruce Fields
2008-10-22 17:56   ` J. Bruce Fields
2008-10-23 12:36 ` Max Kellermann
2008-10-23 12:36   ` Max Kellermann
2008-10-23 14:55   ` Trond Myklebust
2008-10-24  8:39     ` Max Kellermann
2008-10-24  8:39       ` Max Kellermann
2008-10-24 18:09       ` Trond Myklebust
2008-10-24 18:09         ` Trond Myklebust
2008-10-27  9:58         ` Max Kellermann
2008-10-27  9:58           ` Max Kellermann
2008-10-27 15:48           ` Trond Myklebust
2008-10-27 15:48             ` Trond Myklebust
2008-11-17 12:35             ` Max Kellermann
2008-11-17 12:35               ` Max Kellermann
2008-11-19 22:31               ` Trond Myklebust
2008-11-19 22:31                 ` Trond Myklebust
2008-11-20 14:08                 ` Max Kellermann
2008-11-20 14:08                   ` Max Kellermann
2008-12-15 23:44     ` Harry Edmon
2008-12-15 23:44       ` Harry Edmon
2008-12-16 13:02       ` Trond Myklebust
2008-12-16 13:02         ` Trond Myklebust
2008-12-16 21:21         ` Willy Tarreau
2008-12-16 21:21           ` Willy Tarreau
2008-12-16 23:21           ` [stable] " Greg KH
2008-12-16 23:21             ` Greg KH

This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.