* 3.10.0 i386 uniprocessor panic
@ 2013-07-18 6:13 George Spelvin
2013-07-18 8:18 ` Borislav Petkov
2013-07-19 17:17 ` H. Peter Anvin
0 siblings, 2 replies; 7+ messages in thread
From: George Spelvin @ 2013-07-18 6:13 UTC (permalink / raw)
To: linux-kernel; +Cc: linux
I ressurected an old Athlon XP box for fun, and was stress-testing it
with mprime. (It had been stable before retirement.) After 34 hours
of successful torture test (suggesting a stable memory syatem), I found
this on the screen (hand-transcribed, top scrolled off):
h_rpcgss oid_registry exportfs nfs_acl nfs lockd sunrpc loop fuse sil164 nouveau video mxm_wmi wmi ttm fbcon font bitblit softcursor drm_kms_helper drm i2c_algo_bit cfbcopyarea cfbfillrect serio_raw cfbimgblt hid_generic processor fan thermal thermal_sys button
CPU: 0 PID: 3567 Comm: mprime Not tainted 3.10.0 #4
Hardware name: /FN41 , BIOS 6.00 PG 08/23/2004
task: f31849f0 ti: f3150000 task.ti: f3150000
EIP: 0060:[<c143a091>] EFLAGS 00010286 CPU: 0
EIP is at 0xc143a091
EAX: c143a090 EBX: 00000100 ECX: f3150000 EDX: c143a090
ESI: c143a090 EDI: c143a090 EBP: c143a090 ESP: f3151eec
DS: 007b ES: 007b FS: 0000 GS: 0033 SS: 0068
CR0: 80050033 CR2: a090c143 CR3: 331c6000 CR4: 000007d0
DR0: 00000000 DR1: 00000000 DR2: 00000000 DR3: 00000000
DR6: ffff0ff0 DR7: 00000400
Stack:
c102437d b665a951 0000713f 8ae3556c 0000a66a f31849f0 00000002 c1439980
c143a080 c1024524 c143a090 c143a0a4 00000000 00000000 f3151f30 c143a190
c143a390 c143a080 e63938bc 00000001 f3150000 c1439844 00000100 c1020e8b
Call Trace:
[<c102437d>] ? call_timer_fn.isra.37+0x16/0x6d
[<c1024524>] ? run_timer_softirq+0x150/0x165
[<c1020e8b>] ? __do_softirq+0x8b/0x135
[<c1020fe4>] ? irq_exit+0x3d/0x72
[<c10021f2>] ? do_IRQ+0x69/0x7c
[<c1088524>] ? SyS_write+0x59/0x6a
[<c10015ef>] ? math_state_restore+0x73/0xcd
[<c128192c>] ? common_interrupt+0x2c/0x31
Code: 43 c1 68 a0 43 c1 68 a0 43 c1 70 a0 43 c1 70 a0 43 c1 78 a0 43 c1 78 a0 43 c1 00 00 00 00 00 02 20 00 88 a0 43 c1 88 a0 43 c1 90 <a0> 43 c1 90 a0 43 c1 98 a0 43 c1 98 a0 43 c1 a0 a0 43 c1 a0 a0
EIP: [<c143a091>] 0xc143a091 SS:ESP 0068:f3151eec
CR2: 00000000a090c143
---[ end trace 4009bf27ab8c3bf3 ]---
Kernel panic - not syncing: Fatal exception in interrupt
drm_kms_helper: panic occurred, switching back to text console
(The CR2 value looks particularly odd.)
Note that it already *was* on the console screen, with blanking disabled.
Mprime was nohupped, and tail -f on the nohup.out file was running.
Kernel is stock 3.10.0, compiled with GCC 4.8.1. I don't see anything
that looks like this in 3.10.1. Although NFS modules were loaded,
no NFS file systems were mounted or exported.
The parent of the offending function is:
0000021c <call_timer_fn.isra.37>:
21c: 56 push %esi
21d: 89 c6 mov %eax,%esi
21f: 89 d0 mov %edx,%eax
221: 53 push %ebx
222: 83 ec 18 sub $0x18,%esp
225: 89 e1 mov %esp,%ecx
227: 81 e1 00 e0 ff ff and $0xffffe000,%ecx
22d: 8b 59 14 mov 0x14(%ecx),%ebx
230: ff d6 call *%esi <-- Call site
232: 89 e2 mov %esp,%edx <-- Saved return addr
234: 81 e2 00 e0 ff ff and $0xffffe000,%edx
23a: 8b 42 14 mov 0x14(%edx),%eax
23d: 39 c3 cmp %eax,%ebx
23f: 74 42 je 283 <call_timer_fn.isra.37+0x67>
241: 80 3d 00 00 00 00 00 cmpb $0x0,0x0
248: 75 2f jne 279 <call_timer_fn.isra.37+0x5d>
24a: 89 44 24 14 mov %eax,0x14(%esp)
24e: 89 5c 24 10 mov %ebx,0x10(%esp)
252: 89 74 24 0c mov %esi,0xc(%esp)
256: c7 44 24 08 0f 00 00 movl $0xf,0x8(%esp)
25d: 00
25e: c7 44 24 04 62 04 00 movl $0x462,0x4(%esp)
265: 00
266: c7 04 24 00 00 00 00 movl $0x0,(%esp)
26d: e8 fc ff ff ff call 26e <call_timer_fn.isra.37+0x52>
272: c6 05 00 00 00 00 01 movb $0x1,0x0
279: 89 e0 mov %esp,%eax
27b: 25 00 e0 ff ff and $0xffffe000,%eax
280: 89 58 14 mov %ebx,0x14(%eax)
283: 83 c4 18 add $0x18,%esp
286: 5b pop %ebx
287: 5e pop %esi
288: c3 ret
The offending PC is boot_tvec_bases+0x711/0x1040, which is not code,
but data...
Here are the non-comment lines of .config, with CONFIG_ stripped:
X86_32=y
X86=y
INSTRUCTION_DECODER=y
OUTPUT_FORMAT="elf32-i386"
ARCH_DEFCONFIG="arch/x86/configs/i386_defconfig"
LOCKDEP_SUPPORT=y
STACKTRACE_SUPPORT=y
HAVE_LATENCYTOP_SUPPORT=y
MMU=y
NEED_SG_DMA_LENGTH=y
GENERIC_ISA_DMA=y
GENERIC_BUG=y
GENERIC_HWEIGHT=y
ARCH_MAY_HAVE_PC_FDC=y
RWSEM_XCHGADD_ALGORITHM=y
GENERIC_CALIBRATE_DELAY=y
ARCH_HAS_CPU_RELAX=y
ARCH_HAS_CACHE_LINE_SIZE=y
ARCH_HAS_CPU_AUTOPROBE=y
HAVE_SETUP_PER_CPU_AREA=y
NEED_PER_CPU_EMBED_FIRST_CHUNK=y
NEED_PER_CPU_PAGE_FIRST_CHUNK=y
ARCH_HIBERNATION_POSSIBLE=y
ARCH_SUSPEND_POSSIBLE=y
ARCH_SUPPORTS_OPTIMIZED_INLINING=y
ARCH_SUPPORTS_DEBUG_PAGEALLOC=y
X86_32_LAZY_GS=y
ARCH_HWEIGHT_CFLAGS="-fcall-saved-ecx -fcall-saved-edx"
ARCH_SUPPORTS_UPROBES=y
DEFCONFIG_LIST="/lib/modules/$UNAME_RELEASE/.config"
IRQ_WORK=y
BUILDTIME_EXTABLE_SORT=y
BROKEN_ON_SMP=y
INIT_ENV_ARG_LIMIT=32
CROSS_COMPILE=""
LOCALVERSION=""
LOCALVERSION_AUTO=y
HAVE_KERNEL_GZIP=y
HAVE_KERNEL_BZIP2=y
HAVE_KERNEL_LZMA=y
HAVE_KERNEL_XZ=y
HAVE_KERNEL_LZO=y
KERNEL_GZIP=y
DEFAULT_HOSTNAME="(none)"
SWAP=y
SYSVIPC=y
SYSVIPC_SYSCTL=y
POSIX_MQUEUE=y
POSIX_MQUEUE_SYSCTL=y
HAVE_GENERIC_HARDIRQS=y
GENERIC_HARDIRQS=y
GENERIC_IRQ_PROBE=y
GENERIC_IRQ_SHOW=y
IRQ_DOMAIN=y
IRQ_FORCED_THREADING=y
SPARSE_IRQ=y
CLOCKSOURCE_WATCHDOG=y
KTIME_SCALAR=y
GENERIC_CLOCKEVENTS=y
GENERIC_CLOCKEVENTS_BUILD=y
GENERIC_CLOCKEVENTS_MIN_ADJUST=y
GENERIC_CMOS_UPDATE=y
TICK_ONESHOT=y
NO_HZ_COMMON=y
NO_HZ_IDLE=y
NO_HZ=y
HIGH_RES_TIMERS=y
IRQ_TIME_ACCOUNTING=y
TINY_RCU=y
IKCONFIG=y
LOG_BUF_SHIFT=16
HAVE_UNSTABLE_SCHED_CLOCK=y
ARCH_SUPPORTS_NUMA_BALANCING=y
ARCH_WANTS_PROT_NUMA_PROT_NONE=y
CGROUPS=y
CGROUP_SCHED=y
FAIR_GROUP_SCHED=y
BLK_CGROUP=y
NAMESPACES=y
UIDGID_CONVERTED=y
SCHED_AUTOGROUP=y
RELAY=y
CC_OPTIMIZE_FOR_SIZE=y
SYSCTL=y
ANON_INODES=y
HAVE_UID16=y
SYSCTL_EXCEPTION_TRACE=y
HOTPLUG=y
HAVE_PCSPKR_PLATFORM=y
UID16=y
KALLSYMS=y
PRINTK=y
BUG=y
ELF_CORE=y
PCSPKR_PLATFORM=y
BASE_FULL=y
FUTEX=y
EPOLL=y
SIGNALFD=y
TIMERFD=y
EVENTFD=y
SHMEM=y
AIO=y
PCI_QUIRKS=y
HAVE_PERF_EVENTS=y
PERF_EVENTS=y
VM_EVENT_COUNTERS=y
SLUB_DEBUG=y
SLUB=y
HAVE_OPROFILE=y
OPROFILE_NMI_TIMER=y
JUMP_LABEL=y
HAVE_EFFICIENT_UNALIGNED_ACCESS=y
ARCH_USE_BUILTIN_BSWAP=y
HAVE_IOREMAP_PROT=y
HAVE_KPROBES=y
HAVE_KRETPROBES=y
HAVE_OPTPROBES=y
HAVE_KPROBES_ON_FTRACE=y
HAVE_ARCH_TRACEHOOK=y
HAVE_DMA_ATTRS=y
HAVE_DMA_CONTIGUOUS=y
GENERIC_SMP_IDLE_THREAD=y
HAVE_REGS_AND_STACK_ACCESS_API=y
HAVE_DMA_API_DEBUG=y
HAVE_HW_BREAKPOINT=y
HAVE_MIXED_BREAKPOINTS_REGS=y
HAVE_USER_RETURN_NOTIFIER=y
HAVE_PERF_EVENTS_NMI=y
HAVE_PERF_REGS=y
HAVE_PERF_USER_STACK_DUMP=y
HAVE_ARCH_JUMP_LABEL=y
ARCH_HAVE_NMI_SAFE_CMPXCHG=y
HAVE_ALIGNED_STRUCT_PAGE=y
HAVE_CMPXCHG_LOCAL=y
HAVE_CMPXCHG_DOUBLE=y
ARCH_WANT_IPC_PARSE_VERSION=y
HAVE_ARCH_SECCOMP_FILTER=y
SECCOMP_FILTER=y
HAVE_IRQ_TIME_ACCOUNTING=y
HAVE_ARCH_TRANSPARENT_HUGEPAGE=y
MODULES_USE_ELF_REL=y
CLONE_BACKWARDS=y
OLD_SIGSUSPEND3=y
OLD_SIGACTION=y
HAVE_GENERIC_DMA_COHERENT=y
SLABINFO=y
RT_MUTEXES=y
BASE_SMALL=0
MODULES=y
MODULE_UNLOAD=y
MODULE_FORCE_UNLOAD=y
MODVERSIONS=y
BLOCK=y
MSDOS_PARTITION=y
EFI_PARTITION=y
IOSCHED_NOOP=y
IOSCHED_DEADLINE=y
IOSCHED_CFQ=y
CFQ_GROUP_IOSCHED=y
DEFAULT_CFQ=y
DEFAULT_IOSCHED="cfq"
INLINE_SPIN_UNLOCK_IRQ=y
INLINE_READ_UNLOCK=y
INLINE_READ_UNLOCK_IRQ=y
INLINE_WRITE_UNLOCK=y
INLINE_WRITE_UNLOCK_IRQ=y
FREEZER=y
ZONE_DMA=y
X86_SUPPORTS_MEMORY_FAILURE=y
SCHED_OMIT_FRAME_POINTER=y
NO_BOOTMEM=y
MK7=y
X86_INTERNODE_CACHE_SHIFT=6
X86_L1_CACHE_SHIFT=6
X86_INTEL_USERCOPY=y
X86_USE_PPRO_CHECKSUM=y
X86_USE_3DNOW=y
X86_TSC=y
X86_CMOV=y
X86_MINIMUM_CPU_FAMILY=4
X86_DEBUGCTLMSR=y
CPU_SUP_INTEL=y
CPU_SUP_AMD=y
CPU_SUP_CENTAUR=y
CPU_SUP_TRANSMETA_32=y
HPET_TIMER=y
HPET_EMULATE_RTC=y
DMI=y
NR_CPUS=1
PREEMPT_VOLUNTARY=y
X86_MCE=y
VM86=y
HIGHMEM4G=y
PAGE_OFFSET=0xC0000000
HIGHMEM=y
ARCH_FLATMEM_ENABLE=y
ARCH_SPARSEMEM_ENABLE=y
ARCH_SELECT_MEMORY_MODEL=y
ILLEGAL_POINTER_VALUE=0
SELECT_MEMORY_MODEL=y
FLATMEM_MANUAL=y
FLATMEM=y
FLAT_NODE_MEM_MAP=y
SPARSEMEM_STATIC=y
HAVE_MEMBLOCK=y
HAVE_MEMBLOCK_NODE_MAP=y
ARCH_DISCARD_MEMBLOCK=y
PAGEFLAGS_EXTENDED=y
SPLIT_PTLOCK_CPUS=4
COMPACTION=y
MIGRATION=y
ZONE_DMA_FLAG=1
BOUNCE=y
VIRT_TO_BUS=y
DEFAULT_MMAP_MIN_ADDR=4096
ARCH_SUPPORTS_MEMORY_FAILURE=y
TRANSPARENT_HUGEPAGE=y
TRANSPARENT_HUGEPAGE_ALWAYS=y
NEED_PER_CPU_KM=y
X86_CHECK_BIOS_CORRUPTION=y
X86_BOOTPARAM_MEMORY_CORRUPTION_CHECK=y
X86_RESERVE_LOW=4
MTRR=y
X86_PAT=y
ARCH_USES_PG_UNCACHED=y
ARCH_RANDOM=y
X86_SMAP=y
SECCOMP=y
HZ_300=y
HZ=300
SCHED_HRTICK=y
KEXEC=y
PHYSICAL_START=0x1000000
PHYSICAL_ALIGN=0x100000
ARCH_ENABLE_MEMORY_HOTPLUG=y
SUSPEND=y
SUSPEND_FREEZER=y
HIBERNATE_CALLBACKS=y
HIBERNATION=y
PM_STD_PARTITION="/dev/hda1"
PM_SLEEP=y
PM=y
ACPI=y
ACPI_SLEEP=y
ACPI_BUTTON=m
ACPI_VIDEO=m
ACPI_FAN=m
ACPI_I2C=y
ACPI_PROCESSOR=m
ACPI_THERMAL=m
ACPI_BLACKLIST_YEAR=0
X86_PM_TIMER=y
ACPI_SBS=m
CPU_IDLE=y
CPU_IDLE_GOV_LADDER=y
CPU_IDLE_GOV_MENU=y
PCI=y
PCI_GOANY=y
PCI_BIOS=y
PCI_DIRECT=y
PCI_MMCONFIG=y
PCI_DOMAINS=y
PCI_LABEL=y
ISA_DMA_API=y
AMD_NB=y
BINFMT_ELF=y
ARCH_BINFMT_ELF_RANDOMIZE_PIE=y
CORE_DUMP_DEFAULT_ELF_HEADERS=y
BINFMT_SCRIPT=y
HAVE_AOUT=y
COREDUMP=y
HAVE_ATOMIC_IOMAP=y
HAVE_TEXT_POKE_SMP=y
NET=y
PACKET=y
PACKET_DIAG=y
UNIX=y
UNIX_DIAG=y
XFRM=y
XFRM_ALGO=m
XFRM_USER=m
XFRM_IPCOMP=m
NET_KEY=m
INET=y
IP_MULTICAST=y
SYN_COOKIES=y
INET_AH=m
INET_ESP=m
INET_IPCOMP=m
INET_XFRM_TUNNEL=m
INET_TUNNEL=m
INET_XFRM_MODE_TRANSPORT=m
INET_XFRM_MODE_TUNNEL=m
INET_XFRM_MODE_BEET=m
INET_LRO=y
INET_DIAG=m
INET_TCP_DIAG=m
INET_UDP_DIAG=m
TCP_CONG_CUBIC=y
DEFAULT_TCP_CONG="cubic"
IP_SCTP=m
SCTP_DEFAULT_COOKIE_HMAC_MD5=y
SCTP_COOKIE_HMAC_MD5=y
SCTP_COOKIE_HMAC_SHA1=y
HAVE_NET_DSA=y
DNS_RESOLVER=y
NETLINK_DIAG=y
BQL=y
WIRELESS=y
UEVENT_HELPER_PATH="/sbin/hotplug"
STANDALONE=y
PREVENT_FIRMWARE_BUILD=y
FW_LOADER=y
FIRMWARE_IN_KERNEL=y
EXTRA_FIRMWARE=""
DMA_SHARED_BUFFER=y
PNP=y
PNP_DEBUG_MESSAGES=y
PNPACPI=y
BLK_DEV=y
BLK_DEV_FD=y
BLK_DEV_LOOP=m
BLK_DEV_LOOP_MIN_COUNT=8
BLK_DEV_CRYPTOLOOP=m
CDROM_PKTCDVD=y
CDROM_PKTCDVD_BUFFERS=8
EEPROM_LEGACY=y
HAVE_IDE=y
IDE=y
IDE_XFER_MODE=y
IDE_TIMINGS=y
IDE_ATAPI=y
IDE_GD=y
IDE_GD_ATA=y
BLK_DEV_IDECD=y
BLK_DEV_IDECD_VERBOSE_ERRORS=y
IDE_TASK_IOCTL=y
IDE_GENERIC=y
BLK_DEV_IDEDMA_SFF=y
BLK_DEV_IDEPCI=y
IDEPCI_PCIBUS_ORDER=y
BLK_DEV_GENERIC=y
BLK_DEV_IDEDMA_PCI=y
BLK_DEV_AMD74XX=y
BLK_DEV_IDEDMA=y
SCSI_MOD=y
SCSI=y
SCSI_DMA=y
BLK_DEV_SD=m
NETDEVICES=y
NET_CORE=y
MII=m
ETHERNET=y
NET_VENDOR_NVIDIA=y
FORCEDETH=y
PHYLIB=y
REALTEK_PHY=y
MDIO_BITBANG=m
WLAN=y
INPUT=y
INPUT_FF_MEMLESS=m
INPUT_POLLDEV=m
INPUT_MOUSEDEV=y
INPUT_MOUSEDEV_PSAUX=y
INPUT_MOUSEDEV_SCREEN_X=1024
INPUT_MOUSEDEV_SCREEN_Y=768
INPUT_EVDEV=y
INPUT_KEYBOARD=y
KEYBOARD_ATKBD=y
INPUT_MOUSE=y
MOUSE_PS2=y
MOUSE_PS2_ALPS=y
MOUSE_PS2_LOGIPS2PP=y
MOUSE_PS2_SYNAPTICS=y
MOUSE_PS2_CYPRESS=y
MOUSE_PS2_LIFEBOOK=y
MOUSE_PS2_TRACKPOINT=y
SERIO=y
SERIO_I8042=y
SERIO_SERPORT=m
SERIO_PCIPS2=m
SERIO_LIBPS2=y
SERIO_RAW=m
TTY=y
VT=y
CONSOLE_TRANSLATIONS=y
VT_CONSOLE=y
VT_CONSOLE_SLEEP=y
HW_CONSOLE=y
UNIX98_PTYS=y
SERIAL_8250=y
SERIAL_8250_PNP=y
SERIAL_8250_CONSOLE=y
FIX_EARLYCON_MEM=y
SERIAL_8250_PCI=y
SERIAL_8250_NR_UARTS=32
SERIAL_8250_RUNTIME_UARTS=4
SERIAL_8250_EXTENDED=y
SERIAL_8250_DETECT_IRQ=y
SERIAL_CORE=y
SERIAL_CORE_CONSOLE=y
NVRAM=m
HPET=y
HPET_MMAP=y
DEVPORT=y
I2C=y
I2C_BOARDINFO=y
I2C_CHARDEV=y
I2C_HELPER_AUTO=y
I2C_ALGOBIT=m
I2C_NFORCE2=y
ARCH_WANT_OPTIONAL_GPIOLIB=y
GPIO_DEVRES=y
POWER_SUPPLY=y
HWMON=y
HWMON_VID=y
SENSORS_IT87=y
THERMAL=m
THERMAL_HWMON=y
THERMAL_DEFAULT_GOV_STEP_WISE=y
THERMAL_GOV_STEP_WISE=y
WATCHDOG=y
WATCHDOG_CORE=y
SOFT_WATCHDOG=m
ACQUIRE_WDT=m
ADVANTECH_WDT=m
ALIM1535_WDT=m
ALIM7101_WDT=m
F71808E_WDT=m
SP5100_TCO=m
SC520_WDT=m
EUROTECH_WDT=m
IB700_WDT=m
IBMASR=m
WAFER_WDT=m
I6300ESB_WDT=m
ITCO_WDT=m
IT8712F_WDT=m
IT87_WDT=m
HP_WATCHDOG=m
SC1200_WDT=m
PC87413_WDT=m
NV_TCO=m
60XX_WDT=m
SBC8360_WDT=m
SBC7240_WDT=m
CPU5_WDT=m
SMSC37B787_WDT=m
W83627HF_WDT=m
W83697HF_WDT=m
W83697UG_WDT=m
W83877F_WDT=m
W83977F_WDT=m
MACHZ_WDT=m
SBC_EPX_C3_WATCHDOG=m
PCIPCWATCHDOG=m
WDTPCI=m
USBPCWATCHDOG=m
SSB_POSSIBLE=y
BCMA_POSSIBLE=y
MFD_CORE=m
LPC_ICH=m
MEDIA_SUPPORT=y
AGP=y
AGP_INTEL=m
AGP_NVIDIA=y
VGA_ARB=y
VGA_ARB_MAX_GPUS=2
DRM=m
DRM_KMS_HELPER=m
DRM_TTM=m
DRM_I2C_CH7006=m
DRM_I2C_SIL164=m
DRM_NOUVEAU=m
NOUVEAU_DEBUG=5
NOUVEAU_DEBUG_DEFAULT=3
VGASTATE=m
VIDEO_OUTPUT_CONTROL=m
HDMI=y
FB=y
FIRMWARE_EDID=y
FB_DDC=m
FB_CFB_FILLRECT=m
FB_CFB_COPYAREA=m
FB_CFB_IMAGEBLIT=m
FB_MODE_HELPERS=y
FB_NVIDIA=m
FB_NVIDIA_I2C=y
BACKLIGHT_LCD_SUPPORT=y
BACKLIGHT_CLASS_DEVICE=y
BACKLIGHT_GENERIC=y
VGA_CONSOLE=y
DUMMY_CONSOLE=y
FRAMEBUFFER_CONSOLE=m
FRAMEBUFFER_CONSOLE_DETECT_PRIMARY=y
FONT_8x8=y
FONT_8x16=y
SOUND=y
SOUND_OSS_CORE=y
SOUND_OSS_CORE_PRECLAIM=y
SND=y
SND_TIMER=y
SND_PCM=y
SND_SEQUENCER=m
SND_SEQ_DUMMY=m
SND_OSSEMUL=y
SND_MIXER_OSS=m
SND_PCM_OSS=m
SND_PCM_OSS_PLUGINS=y
SND_SEQUENCER_OSS=y
SND_HRTIMER=y
SND_SEQ_HRTIMER_DEFAULT=y
SND_SUPPORT_OLD_API=y
SND_VERBOSE_PROCFS=y
SND_VMASTER=y
SND_DMA_SGBUF=y
SND_AC97_CODEC=y
SND_PCI=y
SND_INTEL8X0=y
AC97_BUS=y
HID=y
HIDRAW=y
HID_GENERIC=m
HID_A4TECH=y
HID_APPLE=y
HID_BELKIN=y
HID_CHERRY=y
HID_CHICONY=y
HID_CYPRESS=y
HID_DRAGONRISE=y
HID_EZKEY=y
HID_KYE=y
HID_GYRATION=y
HID_TWINHAN=y
HID_KENSINGTON=y
HID_LOGITECH=y
HID_LOGITECH_DJ=m
HID_MICROSOFT=y
HID_MONTEREY=y
HID_NTRIG=y
HID_ORTEK=y
HID_PANTHERLORD=y
HID_PETALYNX=y
HID_SAMSUNG=y
HID_SONY=y
HID_SUNPLUS=y
HID_GREENASIA=y
HID_SMARTJOYPLUS=y
HID_TOPSEED=y
HID_THRUSTMASTER=y
HID_ZEROPLUS=y
USB_HID=y
USB_HIDDEV=y
USB_ARCH_HAS_OHCI=y
USB_ARCH_HAS_EHCI=y
USB_ARCH_HAS_XHCI=y
USB_SUPPORT=y
USB_COMMON=y
USB_ARCH_HAS_HCD=y
USB=y
USB_ANNOUNCE_NEW_DEVICES=y
USB_DEFAULT_PERSIST=y
USB_MON=y
USB_EHCI_HCD=y
USB_EHCI_ROOT_HUB_TT=y
USB_EHCI_TT_NEWSCHED=y
USB_EHCI_PCI=y
USB_OHCI_HCD=y
USB_OHCI_LITTLE_ENDIAN=y
USB_ACM=m
USB_PRINTER=m
USB_WDM=m
USB_STORAGE=m
USB_MDC800=m
USB_SERIAL=m
USB_SERIAL_GENERIC=y
USB_SERIAL_AIRCABLE=m
USB_SERIAL_ARK3116=m
USB_SERIAL_BELKIN=m
USB_SERIAL_CH341=m
USB_SERIAL_DIGI_ACCELEPORT=m
USB_SERIAL_CYPRESS_M8=m
USB_SERIAL_EMPEG=m
USB_SERIAL_FTDI_SIO=m
USB_SERIAL_FUNSOFT=m
USB_SERIAL_VISOR=m
USB_SERIAL_IPAQ=m
USB_SERIAL_IR=m
USB_SERIAL_EDGEPORT=m
USB_SERIAL_EDGEPORT_TI=m
USB_SERIAL_GARMIN=m
USB_SERIAL_IPW=m
USB_SERIAL_IUU=m
USB_SERIAL_KEYSPAN_PDA=m
USB_SERIAL_KEYSPAN=m
USB_SERIAL_KLSI=m
USB_SERIAL_KOBIL_SCT=m
USB_SERIAL_MCT_U232=m
USB_SERIAL_MOS7720=m
USB_SERIAL_MOS7840=m
USB_SERIAL_MOTOROLA=m
USB_SERIAL_NAVMAN=m
USB_SERIAL_PL2303=m
USB_SERIAL_OTI6858=m
USB_SERIAL_SPCP8X5=m
USB_SERIAL_HP4X=m
USB_SERIAL_SAFE=m
USB_SERIAL_SIERRAWIRELESS=m
USB_SERIAL_CYBERJACK=m
USB_SERIAL_XIRCOM=m
USB_SERIAL_WWAN=m
USB_SERIAL_OPTION=m
USB_SERIAL_OMNINET=m
USB_SERIAL_DEBUG=m
USB_ADUTUX=m
USB_RIO500=m
USB_LEGOTOWER=m
USB_LCD=m
USB_LED=m
USB_CYPRESS_CY7C63=m
USB_CYTHERM=m
USB_IDMOUSE=m
USB_FTDI_ELAN=m
USB_APPLEDISPLAY=m
USB_SISUSBVGA=m
USB_SISUSBVGA_CON=y
USB_LD=m
USB_TRANCEVIBRATOR=m
USB_IOWARRIOR=m
USB_TEST=m
USB_ISIGHTFW=m
USB_EZUSB_FX2=m
NEW_LEDS=y
RTC_LIB=y
RTC_CLASS=y
RTC_HCTOSYS=y
RTC_SYSTOHC=y
RTC_HCTOSYS_DEVICE="rtc0"
RTC_INTF_SYSFS=y
RTC_INTF_PROC=y
RTC_INTF_DEV=y
RTC_DRV_DS1307=m
RTC_DRV_DS1374=m
RTC_DRV_DS1672=m
RTC_DRV_MAX6900=m
RTC_DRV_RS5C372=m
RTC_DRV_ISL1208=m
RTC_DRV_X1205=m
RTC_DRV_PCF8563=m
RTC_DRV_PCF8583=m
RTC_DRV_M41T80=m
RTC_DRV_S35390A=m
RTC_DRV_FM3130=m
RTC_DRV_CMOS=y
RTC_DRV_DS1511=m
RTC_DRV_DS1553=m
RTC_DRV_DS1742=m
RTC_DRV_STK17TA8=m
RTC_DRV_M48T86=m
RTC_DRV_M48T59=m
RTC_DRV_V3020=m
STAGING=y
X86_PLATFORM_DEVICES=y
ACPI_WMI=m
MXM_WMI=m
CLKSRC_I8253=y
CLKEVT_I8253=y
I8253_LOCK=y
CLKBLD_I8253=y
IOMMU_SUPPORT=y
FIRMWARE_MEMMAP=y
DMIID=y
DCACHE_WORD_ACCESS=y
EXT4_FS=y
EXT4_USE_FOR_EXT23=y
JBD2=y
FS_MBCACHE=y
FS_POSIX_ACL=y
EXPORTFS=m
FILE_LOCKING=y
FSNOTIFY=y
INOTIFY_USER=y
FANOTIFY=y
FUSE_FS=m
GENERIC_ACL=y
ISO9660_FS=m
JOLIET=y
ZISOFS=y
UDF_FS=m
UDF_NLS=y
FAT_FS=m
MSDOS_FS=m
VFAT_FS=m
FAT_DEFAULT_CODEPAGE=437
FAT_DEFAULT_IOCHARSET="utf8"
PROC_FS=y
PROC_KCORE=y
PROC_SYSCTL=y
PROC_PAGE_MONITOR=y
SYSFS=y
TMPFS=y
TMPFS_POSIX_ACL=y
TMPFS_XATTR=y
HUGETLBFS=y
HUGETLB_PAGE=y
CONFIGFS_FS=m
NETWORK_FILESYSTEMS=y
NFS_FS=m
NFS_V2=m
NFS_V3=m
NFS_V3_ACL=y
NFS_V4=m
NFS_V4_1=y
PNFS_FILE_LAYOUT=m
NFS_V4_1_IMPLEMENTATION_ID_DOMAIN="kernel.org"
NFS_USE_KERNEL_DNS=y
NFSD=m
NFSD_V2_ACL=y
NFSD_V3=y
NFSD_V3_ACL=y
NFSD_V4=y
LOCKD=m
LOCKD_V4=y
NFS_ACL_SUPPORT=m
NFS_COMMON=y
SUNRPC=m
SUNRPC_GSS=m
SUNRPC_BACKCHANNEL=y
RPCSEC_GSS_KRB5=m
CIFS=m
CIFS_WEAK_PW_HASH=y
CIFS_UPCALL=y
CIFS_XATTR=y
CIFS_POSIX=y
CIFS_DEBUG=y
CIFS_DFS_UPCALL=y
NLS=y
NLS_DEFAULT="utf8"
NLS_CODEPAGE_437=m
NLS_CODEPAGE_737=m
NLS_CODEPAGE_775=m
NLS_CODEPAGE_850=m
NLS_CODEPAGE_852=m
NLS_CODEPAGE_855=m
NLS_CODEPAGE_857=m
NLS_CODEPAGE_860=m
NLS_CODEPAGE_861=m
NLS_CODEPAGE_862=m
NLS_CODEPAGE_863=m
NLS_CODEPAGE_864=m
NLS_CODEPAGE_865=m
NLS_CODEPAGE_866=m
NLS_CODEPAGE_869=m
NLS_CODEPAGE_936=m
NLS_CODEPAGE_950=m
NLS_CODEPAGE_932=m
NLS_CODEPAGE_949=m
NLS_CODEPAGE_874=m
NLS_ISO8859_8=m
NLS_CODEPAGE_1250=m
NLS_CODEPAGE_1251=m
NLS_ASCII=m
NLS_ISO8859_1=m
NLS_ISO8859_2=m
NLS_ISO8859_3=m
NLS_ISO8859_4=m
NLS_ISO8859_5=m
NLS_ISO8859_6=m
NLS_ISO8859_7=m
NLS_ISO8859_9=m
NLS_ISO8859_13=m
NLS_ISO8859_14=m
NLS_ISO8859_15=m
NLS_KOI8_R=m
NLS_KOI8_U=m
NLS_UTF8=m
TRACE_IRQFLAGS_SUPPORT=y
PRINTK_TIME=y
DEFAULT_MESSAGE_LOGLEVEL=4
ENABLE_WARN_DEPRECATED=y
ENABLE_MUST_CHECK=y
FRAME_WARN=1024
MAGIC_SYSRQ=y
STRIP_ASM_SYMS=y
DEBUG_KERNEL=y
LOCKUP_DETECTOR=y
HARDLOCKUP_DETECTOR=y
BOOTPARAM_HARDLOCKUP_PANIC_VALUE=0
BOOTPARAM_SOFTLOCKUP_PANIC_VALUE=0
PANIC_ON_OOPS_VALUE=0
DETECT_HUNG_TASK=y
DEFAULT_HUNG_TASK_TIMEOUT=120
BOOTPARAM_HUNG_TASK_PANIC_VALUE=0
SCHED_DEBUG=y
TIMER_STATS=y
HAVE_DEBUG_KMEMLEAK=y
DEBUG_BUGVERBOSE=y
DEBUG_MEMORY_INIT=y
ARCH_WANT_FRAME_POINTERS=y
ARCH_HAS_DEBUG_STRICT_USER_COPY_CHECKS=y
USER_STACKTRACE_SUPPORT=y
HAVE_FUNCTION_TRACER=y
HAVE_FUNCTION_GRAPH_TRACER=y
HAVE_FUNCTION_GRAPH_FP_TEST=y
HAVE_FUNCTION_TRACE_MCOUNT_TEST=y
HAVE_DYNAMIC_FTRACE=y
HAVE_DYNAMIC_FTRACE_WITH_REGS=y
HAVE_FTRACE_MCOUNT_RECORD=y
HAVE_SYSCALL_TRACEPOINTS=y
HAVE_C_RECORDMCOUNT=y
TRACING_SUPPORT=y
HAVE_ARCH_KGDB=y
HAVE_ARCH_KMEMCHECK=y
X86_VERBOSE_BOOTUP=y
EARLY_PRINTK=y
DOUBLEFAULT=y
HAVE_MMIOTRACE_SUPPORT=y
IO_DELAY_TYPE_0X80=0
IO_DELAY_TYPE_0XED=1
IO_DELAY_TYPE_UDELAY=2
IO_DELAY_TYPE_NONE=3
IO_DELAY_NONE=y
DEFAULT_IO_DELAY_TYPE=3
OPTIMIZE_INLINING=y
KEYS=y
DEFAULT_SECURITY_DAC=y
DEFAULT_SECURITY=""
CRYPTO=y
CRYPTO_ALGAPI=y
CRYPTO_ALGAPI2=y
CRYPTO_AEAD=m
CRYPTO_AEAD2=y
CRYPTO_BLKCIPHER=m
CRYPTO_BLKCIPHER2=y
CRYPTO_HASH=y
CRYPTO_HASH2=y
CRYPTO_RNG=m
CRYPTO_RNG2=y
CRYPTO_PCOMP2=y
CRYPTO_MANAGER=y
CRYPTO_MANAGER2=y
CRYPTO_GF128MUL=m
CRYPTO_NULL=m
CRYPTO_WORKQUEUE=y
CRYPTO_AUTHENC=m
CRYPTO_TEST=m
CRYPTO_CCM=m
CRYPTO_GCM=m
CRYPTO_SEQIV=m
CRYPTO_CBC=m
CRYPTO_CTR=m
CRYPTO_CTS=m
CRYPTO_ECB=m
CRYPTO_LRW=m
CRYPTO_PCBC=m
CRYPTO_XTS=m
CRYPTO_HMAC=y
CRYPTO_XCBC=m
CRYPTO_CRC32C=y
CRYPTO_GHASH=m
CRYPTO_MD4=m
CRYPTO_MD5=y
CRYPTO_MICHAEL_MIC=m
CRYPTO_SHA1=m
CRYPTO_SHA256=m
CRYPTO_SHA512=m
CRYPTO_TGR192=m
CRYPTO_WP512=m
CRYPTO_AES=y
CRYPTO_AES_586=m
CRYPTO_ANUBIS=m
CRYPTO_ARC4=m
CRYPTO_BLOWFISH=m
CRYPTO_BLOWFISH_COMMON=m
CRYPTO_CAMELLIA=m
CRYPTO_CAST_COMMON=m
CRYPTO_CAST5=m
CRYPTO_CAST6=m
CRYPTO_DES=m
CRYPTO_FCRYPT=m
CRYPTO_KHAZAD=m
CRYPTO_SALSA20=m
CRYPTO_SALSA20_586=m
CRYPTO_SEED=m
CRYPTO_SERPENT=m
CRYPTO_TEA=m
CRYPTO_TWOFISH=m
CRYPTO_TWOFISH_COMMON=m
CRYPTO_TWOFISH_586=m
CRYPTO_DEFLATE=m
CRYPTO_LZO=m
HAVE_KVM=y
BITREVERSE=y
GENERIC_STRNCPY_FROM_USER=y
GENERIC_STRNLEN_USER=y
GENERIC_FIND_FIRST_BIT=y
GENERIC_PCI_IOMAP=y
GENERIC_IOMAP=y
GENERIC_IO=y
CRC_CCITT=m
CRC16=y
CRC_T10DIF=m
CRC_ITU_T=m
CRC32=y
CRC32_SLICEBY8=y
CRC7=m
LIBCRC32C=m
ZLIB_INFLATE=m
ZLIB_DEFLATE=m
LZO_COMPRESS=y
LZO_DECOMPRESS=y
HAS_IOMEM=y
HAS_IOPORT=y
HAS_DMA=y
DQL=y
NLATTR=y
ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE=y
OID_REGISTRY=m
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: 3.10.0 i386 uniprocessor panic
2013-07-18 6:13 3.10.0 i386 uniprocessor panic George Spelvin
@ 2013-07-18 8:18 ` Borislav Petkov
2013-07-19 13:21 ` Thomas Gleixner
2013-07-19 17:17 ` H. Peter Anvin
1 sibling, 1 reply; 7+ messages in thread
From: Borislav Petkov @ 2013-07-18 8:18 UTC (permalink / raw)
To: George Spelvin; +Cc: linux-kernel, Thomas Gleixner
On Thu, Jul 18, 2013 at 02:13:47AM -0400, George Spelvin wrote:
> I ressurected an old Athlon XP box for fun, and was stress-testing it
> with mprime. (It had been stable before retirement.) After 34 hours
> of successful torture test (suggesting a stable memory syatem), I found
> this on the screen (hand-transcribed, top scrolled off):
>
> h_rpcgss oid_registry exportfs nfs_acl nfs lockd sunrpc loop fuse sil164 nouveau video mxm_wmi wmi ttm fbcon font bitblit softcursor drm_kms_helper drm i2c_algo_bit cfbcopyarea cfbfillrect serio_raw cfbimgblt hid_generic processor fan thermal thermal_sys button
> CPU: 0 PID: 3567 Comm: mprime Not tainted 3.10.0 #4
> Hardware name: /FN41 , BIOS 6.00 PG 08/23/2004
> task: f31849f0 ti: f3150000 task.ti: f3150000
> EIP: 0060:[<c143a091>] EFLAGS 00010286 CPU: 0
> EIP is at 0xc143a091
> EAX: c143a090 EBX: 00000100 ECX: f3150000 EDX: c143a090
> ESI: c143a090 EDI: c143a090 EBP: c143a090 ESP: f3151eec
> DS: 007b ES: 007b FS: 0000 GS: 0033 SS: 0068
> CR0: 80050033 CR2: a090c143 CR3: 331c6000 CR4: 000007d0
> DR0: 00000000 DR1: 00000000 DR2: 00000000 DR3: 00000000
> DR6: ffff0ff0 DR7: 00000400
> Stack:
> c102437d b665a951 0000713f 8ae3556c 0000a66a f31849f0 00000002 c1439980
> c143a080 c1024524 c143a090 c143a0a4 00000000 00000000 f3151f30 c143a190
> c143a390 c143a080 e63938bc 00000001 f3150000 c1439844 00000100 c1020e8b
> Call Trace:
> [<c102437d>] ? call_timer_fn.isra.37+0x16/0x6d
> [<c1024524>] ? run_timer_softirq+0x150/0x165
> [<c1020e8b>] ? __do_softirq+0x8b/0x135
> [<c1020fe4>] ? irq_exit+0x3d/0x72
> [<c10021f2>] ? do_IRQ+0x69/0x7c
> [<c1088524>] ? SyS_write+0x59/0x6a
> [<c10015ef>] ? math_state_restore+0x73/0xcd
> [<c128192c>] ? common_interrupt+0x2c/0x31
> Code: 43 c1 68 a0 43 c1 68 a0 43 c1 70 a0 43 c1 70 a0 43 c1 78 a0 43 c1 78 a0 43 c1 00 00 00 00 00 02 20 00 88 a0 43 c1 88 a0 43 c1 90 <a0> 43 c1 90 a0 43 c1 98 a0 43 c1 98 a0 43 c1 a0 a0 43 c1 a0 a0
> EIP: [<c143a091>] 0xc143a091 SS:ESP 0068:f3151eec
> CR2: 00000000a090c143
> ---[ end trace 4009bf27ab8c3bf3 ]---
> Kernel panic - not syncing: Fatal exception in interrupt
> drm_kms_helper: panic occurred, switching back to text console
>
> (The CR2 value looks particularly odd.)
>
> Note that it already *was* on the console screen, with blanking disabled.
> Mprime was nohupped, and tail -f on the nohup.out file was running.
>
> Kernel is stock 3.10.0, compiled with GCC 4.8.1. I don't see anything
> that looks like this in 3.10.1. Although NFS modules were loaded,
> no NFS file systems were mounted or exported.
>
> The parent of the offending function is:
>
> 0000021c <call_timer_fn.isra.37>:
> 21c: 56 push %esi
> 21d: 89 c6 mov %eax,%esi
> 21f: 89 d0 mov %edx,%eax
> 221: 53 push %ebx
> 222: 83 ec 18 sub $0x18,%esp
> 225: 89 e1 mov %esp,%ecx
> 227: 81 e1 00 e0 ff ff and $0xffffe000,%ecx
> 22d: 8b 59 14 mov 0x14(%ecx),%ebx
> 230: ff d6 call *%esi <-- Call site
> 232: 89 e2 mov %esp,%edx <-- Saved return addr
> 234: 81 e2 00 e0 ff ff and $0xffffe000,%edx
> 23a: 8b 42 14 mov 0x14(%edx),%eax
> 23d: 39 c3 cmp %eax,%ebx
> 23f: 74 42 je 283 <call_timer_fn.isra.37+0x67>
> 241: 80 3d 00 00 00 00 00 cmpb $0x0,0x0
> 248: 75 2f jne 279 <call_timer_fn.isra.37+0x5d>
> 24a: 89 44 24 14 mov %eax,0x14(%esp)
> 24e: 89 5c 24 10 mov %ebx,0x10(%esp)
> 252: 89 74 24 0c mov %esi,0xc(%esp)
> 256: c7 44 24 08 0f 00 00 movl $0xf,0x8(%esp)
> 25d: 00
> 25e: c7 44 24 04 62 04 00 movl $0x462,0x4(%esp)
> 265: 00
> 266: c7 04 24 00 00 00 00 movl $0x0,(%esp)
> 26d: e8 fc ff ff ff call 26e <call_timer_fn.isra.37+0x52>
> 272: c6 05 00 00 00 00 01 movb $0x1,0x0
> 279: 89 e0 mov %esp,%eax
> 27b: 25 00 e0 ff ff and $0xffffe000,%eax
> 280: 89 58 14 mov %ebx,0x14(%eax)
> 283: 83 c4 18 add $0x18,%esp
> 286: 5b pop %ebx
> 287: 5e pop %esi
> 288: c3 ret
>
> The offending PC is boot_tvec_bases+0x711/0x1040, which is not code,
> but data...
Hmm, this actually makes sense - boot_tvec_bases contains timers and I
can very well imagine timer->function getting corrupted.
Btw, how reproducible is this?
Leaving in the rest for tglx.
> Here are the non-comment lines of .config, with CONFIG_ stripped:
>
> X86_32=y
> X86=y
> INSTRUCTION_DECODER=y
> OUTPUT_FORMAT="elf32-i386"
> ARCH_DEFCONFIG="arch/x86/configs/i386_defconfig"
> LOCKDEP_SUPPORT=y
> STACKTRACE_SUPPORT=y
> HAVE_LATENCYTOP_SUPPORT=y
> MMU=y
> NEED_SG_DMA_LENGTH=y
> GENERIC_ISA_DMA=y
> GENERIC_BUG=y
> GENERIC_HWEIGHT=y
> ARCH_MAY_HAVE_PC_FDC=y
> RWSEM_XCHGADD_ALGORITHM=y
> GENERIC_CALIBRATE_DELAY=y
> ARCH_HAS_CPU_RELAX=y
> ARCH_HAS_CACHE_LINE_SIZE=y
> ARCH_HAS_CPU_AUTOPROBE=y
> HAVE_SETUP_PER_CPU_AREA=y
> NEED_PER_CPU_EMBED_FIRST_CHUNK=y
> NEED_PER_CPU_PAGE_FIRST_CHUNK=y
> ARCH_HIBERNATION_POSSIBLE=y
> ARCH_SUSPEND_POSSIBLE=y
> ARCH_SUPPORTS_OPTIMIZED_INLINING=y
> ARCH_SUPPORTS_DEBUG_PAGEALLOC=y
> X86_32_LAZY_GS=y
> ARCH_HWEIGHT_CFLAGS="-fcall-saved-ecx -fcall-saved-edx"
> ARCH_SUPPORTS_UPROBES=y
> DEFCONFIG_LIST="/lib/modules/$UNAME_RELEASE/.config"
> IRQ_WORK=y
> BUILDTIME_EXTABLE_SORT=y
> BROKEN_ON_SMP=y
> INIT_ENV_ARG_LIMIT=32
> CROSS_COMPILE=""
> LOCALVERSION=""
> LOCALVERSION_AUTO=y
> HAVE_KERNEL_GZIP=y
> HAVE_KERNEL_BZIP2=y
> HAVE_KERNEL_LZMA=y
> HAVE_KERNEL_XZ=y
> HAVE_KERNEL_LZO=y
> KERNEL_GZIP=y
> DEFAULT_HOSTNAME="(none)"
> SWAP=y
> SYSVIPC=y
> SYSVIPC_SYSCTL=y
> POSIX_MQUEUE=y
> POSIX_MQUEUE_SYSCTL=y
> HAVE_GENERIC_HARDIRQS=y
> GENERIC_HARDIRQS=y
> GENERIC_IRQ_PROBE=y
> GENERIC_IRQ_SHOW=y
> IRQ_DOMAIN=y
> IRQ_FORCED_THREADING=y
> SPARSE_IRQ=y
> CLOCKSOURCE_WATCHDOG=y
> KTIME_SCALAR=y
> GENERIC_CLOCKEVENTS=y
> GENERIC_CLOCKEVENTS_BUILD=y
> GENERIC_CLOCKEVENTS_MIN_ADJUST=y
> GENERIC_CMOS_UPDATE=y
> TICK_ONESHOT=y
> NO_HZ_COMMON=y
> NO_HZ_IDLE=y
> NO_HZ=y
> HIGH_RES_TIMERS=y
> IRQ_TIME_ACCOUNTING=y
> TINY_RCU=y
> IKCONFIG=y
> LOG_BUF_SHIFT=16
> HAVE_UNSTABLE_SCHED_CLOCK=y
> ARCH_SUPPORTS_NUMA_BALANCING=y
> ARCH_WANTS_PROT_NUMA_PROT_NONE=y
> CGROUPS=y
> CGROUP_SCHED=y
> FAIR_GROUP_SCHED=y
> BLK_CGROUP=y
> NAMESPACES=y
> UIDGID_CONVERTED=y
> SCHED_AUTOGROUP=y
> RELAY=y
> CC_OPTIMIZE_FOR_SIZE=y
> SYSCTL=y
> ANON_INODES=y
> HAVE_UID16=y
> SYSCTL_EXCEPTION_TRACE=y
> HOTPLUG=y
> HAVE_PCSPKR_PLATFORM=y
> UID16=y
> KALLSYMS=y
> PRINTK=y
> BUG=y
> ELF_CORE=y
> PCSPKR_PLATFORM=y
> BASE_FULL=y
> FUTEX=y
> EPOLL=y
> SIGNALFD=y
> TIMERFD=y
> EVENTFD=y
> SHMEM=y
> AIO=y
> PCI_QUIRKS=y
> HAVE_PERF_EVENTS=y
> PERF_EVENTS=y
> VM_EVENT_COUNTERS=y
> SLUB_DEBUG=y
> SLUB=y
> HAVE_OPROFILE=y
> OPROFILE_NMI_TIMER=y
> JUMP_LABEL=y
> HAVE_EFFICIENT_UNALIGNED_ACCESS=y
> ARCH_USE_BUILTIN_BSWAP=y
> HAVE_IOREMAP_PROT=y
> HAVE_KPROBES=y
> HAVE_KRETPROBES=y
> HAVE_OPTPROBES=y
> HAVE_KPROBES_ON_FTRACE=y
> HAVE_ARCH_TRACEHOOK=y
> HAVE_DMA_ATTRS=y
> HAVE_DMA_CONTIGUOUS=y
> GENERIC_SMP_IDLE_THREAD=y
> HAVE_REGS_AND_STACK_ACCESS_API=y
> HAVE_DMA_API_DEBUG=y
> HAVE_HW_BREAKPOINT=y
> HAVE_MIXED_BREAKPOINTS_REGS=y
> HAVE_USER_RETURN_NOTIFIER=y
> HAVE_PERF_EVENTS_NMI=y
> HAVE_PERF_REGS=y
> HAVE_PERF_USER_STACK_DUMP=y
> HAVE_ARCH_JUMP_LABEL=y
> ARCH_HAVE_NMI_SAFE_CMPXCHG=y
> HAVE_ALIGNED_STRUCT_PAGE=y
> HAVE_CMPXCHG_LOCAL=y
> HAVE_CMPXCHG_DOUBLE=y
> ARCH_WANT_IPC_PARSE_VERSION=y
> HAVE_ARCH_SECCOMP_FILTER=y
> SECCOMP_FILTER=y
> HAVE_IRQ_TIME_ACCOUNTING=y
> HAVE_ARCH_TRANSPARENT_HUGEPAGE=y
> MODULES_USE_ELF_REL=y
> CLONE_BACKWARDS=y
> OLD_SIGSUSPEND3=y
> OLD_SIGACTION=y
> HAVE_GENERIC_DMA_COHERENT=y
> SLABINFO=y
> RT_MUTEXES=y
> BASE_SMALL=0
> MODULES=y
> MODULE_UNLOAD=y
> MODULE_FORCE_UNLOAD=y
> MODVERSIONS=y
> BLOCK=y
> MSDOS_PARTITION=y
> EFI_PARTITION=y
> IOSCHED_NOOP=y
> IOSCHED_DEADLINE=y
> IOSCHED_CFQ=y
> CFQ_GROUP_IOSCHED=y
> DEFAULT_CFQ=y
> DEFAULT_IOSCHED="cfq"
> INLINE_SPIN_UNLOCK_IRQ=y
> INLINE_READ_UNLOCK=y
> INLINE_READ_UNLOCK_IRQ=y
> INLINE_WRITE_UNLOCK=y
> INLINE_WRITE_UNLOCK_IRQ=y
> FREEZER=y
> ZONE_DMA=y
> X86_SUPPORTS_MEMORY_FAILURE=y
> SCHED_OMIT_FRAME_POINTER=y
> NO_BOOTMEM=y
> MK7=y
> X86_INTERNODE_CACHE_SHIFT=6
> X86_L1_CACHE_SHIFT=6
> X86_INTEL_USERCOPY=y
> X86_USE_PPRO_CHECKSUM=y
> X86_USE_3DNOW=y
> X86_TSC=y
> X86_CMOV=y
> X86_MINIMUM_CPU_FAMILY=4
> X86_DEBUGCTLMSR=y
> CPU_SUP_INTEL=y
> CPU_SUP_AMD=y
> CPU_SUP_CENTAUR=y
> CPU_SUP_TRANSMETA_32=y
> HPET_TIMER=y
> HPET_EMULATE_RTC=y
> DMI=y
> NR_CPUS=1
> PREEMPT_VOLUNTARY=y
> X86_MCE=y
> VM86=y
> HIGHMEM4G=y
> PAGE_OFFSET=0xC0000000
> HIGHMEM=y
> ARCH_FLATMEM_ENABLE=y
> ARCH_SPARSEMEM_ENABLE=y
> ARCH_SELECT_MEMORY_MODEL=y
> ILLEGAL_POINTER_VALUE=0
> SELECT_MEMORY_MODEL=y
> FLATMEM_MANUAL=y
> FLATMEM=y
> FLAT_NODE_MEM_MAP=y
> SPARSEMEM_STATIC=y
> HAVE_MEMBLOCK=y
> HAVE_MEMBLOCK_NODE_MAP=y
> ARCH_DISCARD_MEMBLOCK=y
> PAGEFLAGS_EXTENDED=y
> SPLIT_PTLOCK_CPUS=4
> COMPACTION=y
> MIGRATION=y
> ZONE_DMA_FLAG=1
> BOUNCE=y
> VIRT_TO_BUS=y
> DEFAULT_MMAP_MIN_ADDR=4096
> ARCH_SUPPORTS_MEMORY_FAILURE=y
> TRANSPARENT_HUGEPAGE=y
> TRANSPARENT_HUGEPAGE_ALWAYS=y
> NEED_PER_CPU_KM=y
> X86_CHECK_BIOS_CORRUPTION=y
> X86_BOOTPARAM_MEMORY_CORRUPTION_CHECK=y
> X86_RESERVE_LOW=4
> MTRR=y
> X86_PAT=y
> ARCH_USES_PG_UNCACHED=y
> ARCH_RANDOM=y
> X86_SMAP=y
> SECCOMP=y
> HZ_300=y
> HZ=300
> SCHED_HRTICK=y
> KEXEC=y
> PHYSICAL_START=0x1000000
> PHYSICAL_ALIGN=0x100000
> ARCH_ENABLE_MEMORY_HOTPLUG=y
> SUSPEND=y
> SUSPEND_FREEZER=y
> HIBERNATE_CALLBACKS=y
> HIBERNATION=y
> PM_STD_PARTITION="/dev/hda1"
> PM_SLEEP=y
> PM=y
> ACPI=y
> ACPI_SLEEP=y
> ACPI_BUTTON=m
> ACPI_VIDEO=m
> ACPI_FAN=m
> ACPI_I2C=y
> ACPI_PROCESSOR=m
> ACPI_THERMAL=m
> ACPI_BLACKLIST_YEAR=0
> X86_PM_TIMER=y
> ACPI_SBS=m
> CPU_IDLE=y
> CPU_IDLE_GOV_LADDER=y
> CPU_IDLE_GOV_MENU=y
> PCI=y
> PCI_GOANY=y
> PCI_BIOS=y
> PCI_DIRECT=y
> PCI_MMCONFIG=y
> PCI_DOMAINS=y
> PCI_LABEL=y
> ISA_DMA_API=y
> AMD_NB=y
> BINFMT_ELF=y
> ARCH_BINFMT_ELF_RANDOMIZE_PIE=y
> CORE_DUMP_DEFAULT_ELF_HEADERS=y
> BINFMT_SCRIPT=y
> HAVE_AOUT=y
> COREDUMP=y
> HAVE_ATOMIC_IOMAP=y
> HAVE_TEXT_POKE_SMP=y
> NET=y
> PACKET=y
> PACKET_DIAG=y
> UNIX=y
> UNIX_DIAG=y
> XFRM=y
> XFRM_ALGO=m
> XFRM_USER=m
> XFRM_IPCOMP=m
> NET_KEY=m
> INET=y
> IP_MULTICAST=y
> SYN_COOKIES=y
> INET_AH=m
> INET_ESP=m
> INET_IPCOMP=m
> INET_XFRM_TUNNEL=m
> INET_TUNNEL=m
> INET_XFRM_MODE_TRANSPORT=m
> INET_XFRM_MODE_TUNNEL=m
> INET_XFRM_MODE_BEET=m
> INET_LRO=y
> INET_DIAG=m
> INET_TCP_DIAG=m
> INET_UDP_DIAG=m
> TCP_CONG_CUBIC=y
> DEFAULT_TCP_CONG="cubic"
> IP_SCTP=m
> SCTP_DEFAULT_COOKIE_HMAC_MD5=y
> SCTP_COOKIE_HMAC_MD5=y
> SCTP_COOKIE_HMAC_SHA1=y
> HAVE_NET_DSA=y
> DNS_RESOLVER=y
> NETLINK_DIAG=y
> BQL=y
> WIRELESS=y
> UEVENT_HELPER_PATH="/sbin/hotplug"
> STANDALONE=y
> PREVENT_FIRMWARE_BUILD=y
> FW_LOADER=y
> FIRMWARE_IN_KERNEL=y
> EXTRA_FIRMWARE=""
> DMA_SHARED_BUFFER=y
> PNP=y
> PNP_DEBUG_MESSAGES=y
> PNPACPI=y
> BLK_DEV=y
> BLK_DEV_FD=y
> BLK_DEV_LOOP=m
> BLK_DEV_LOOP_MIN_COUNT=8
> BLK_DEV_CRYPTOLOOP=m
> CDROM_PKTCDVD=y
> CDROM_PKTCDVD_BUFFERS=8
> EEPROM_LEGACY=y
> HAVE_IDE=y
> IDE=y
> IDE_XFER_MODE=y
> IDE_TIMINGS=y
> IDE_ATAPI=y
> IDE_GD=y
> IDE_GD_ATA=y
> BLK_DEV_IDECD=y
> BLK_DEV_IDECD_VERBOSE_ERRORS=y
> IDE_TASK_IOCTL=y
> IDE_GENERIC=y
> BLK_DEV_IDEDMA_SFF=y
> BLK_DEV_IDEPCI=y
> IDEPCI_PCIBUS_ORDER=y
> BLK_DEV_GENERIC=y
> BLK_DEV_IDEDMA_PCI=y
> BLK_DEV_AMD74XX=y
> BLK_DEV_IDEDMA=y
> SCSI_MOD=y
> SCSI=y
> SCSI_DMA=y
> BLK_DEV_SD=m
> NETDEVICES=y
> NET_CORE=y
> MII=m
> ETHERNET=y
> NET_VENDOR_NVIDIA=y
> FORCEDETH=y
> PHYLIB=y
> REALTEK_PHY=y
> MDIO_BITBANG=m
> WLAN=y
> INPUT=y
> INPUT_FF_MEMLESS=m
> INPUT_POLLDEV=m
> INPUT_MOUSEDEV=y
> INPUT_MOUSEDEV_PSAUX=y
> INPUT_MOUSEDEV_SCREEN_X=1024
> INPUT_MOUSEDEV_SCREEN_Y=768
> INPUT_EVDEV=y
> INPUT_KEYBOARD=y
> KEYBOARD_ATKBD=y
> INPUT_MOUSE=y
> MOUSE_PS2=y
> MOUSE_PS2_ALPS=y
> MOUSE_PS2_LOGIPS2PP=y
> MOUSE_PS2_SYNAPTICS=y
> MOUSE_PS2_CYPRESS=y
> MOUSE_PS2_LIFEBOOK=y
> MOUSE_PS2_TRACKPOINT=y
> SERIO=y
> SERIO_I8042=y
> SERIO_SERPORT=m
> SERIO_PCIPS2=m
> SERIO_LIBPS2=y
> SERIO_RAW=m
> TTY=y
> VT=y
> CONSOLE_TRANSLATIONS=y
> VT_CONSOLE=y
> VT_CONSOLE_SLEEP=y
> HW_CONSOLE=y
> UNIX98_PTYS=y
> SERIAL_8250=y
> SERIAL_8250_PNP=y
> SERIAL_8250_CONSOLE=y
> FIX_EARLYCON_MEM=y
> SERIAL_8250_PCI=y
> SERIAL_8250_NR_UARTS=32
> SERIAL_8250_RUNTIME_UARTS=4
> SERIAL_8250_EXTENDED=y
> SERIAL_8250_DETECT_IRQ=y
> SERIAL_CORE=y
> SERIAL_CORE_CONSOLE=y
> NVRAM=m
> HPET=y
> HPET_MMAP=y
> DEVPORT=y
> I2C=y
> I2C_BOARDINFO=y
> I2C_CHARDEV=y
> I2C_HELPER_AUTO=y
> I2C_ALGOBIT=m
> I2C_NFORCE2=y
> ARCH_WANT_OPTIONAL_GPIOLIB=y
> GPIO_DEVRES=y
> POWER_SUPPLY=y
> HWMON=y
> HWMON_VID=y
> SENSORS_IT87=y
> THERMAL=m
> THERMAL_HWMON=y
> THERMAL_DEFAULT_GOV_STEP_WISE=y
> THERMAL_GOV_STEP_WISE=y
> WATCHDOG=y
> WATCHDOG_CORE=y
> SOFT_WATCHDOG=m
> ACQUIRE_WDT=m
> ADVANTECH_WDT=m
> ALIM1535_WDT=m
> ALIM7101_WDT=m
> F71808E_WDT=m
> SP5100_TCO=m
> SC520_WDT=m
> EUROTECH_WDT=m
> IB700_WDT=m
> IBMASR=m
> WAFER_WDT=m
> I6300ESB_WDT=m
> ITCO_WDT=m
> IT8712F_WDT=m
> IT87_WDT=m
> HP_WATCHDOG=m
> SC1200_WDT=m
> PC87413_WDT=m
> NV_TCO=m
> 60XX_WDT=m
> SBC8360_WDT=m
> SBC7240_WDT=m
> CPU5_WDT=m
> SMSC37B787_WDT=m
> W83627HF_WDT=m
> W83697HF_WDT=m
> W83697UG_WDT=m
> W83877F_WDT=m
> W83977F_WDT=m
> MACHZ_WDT=m
> SBC_EPX_C3_WATCHDOG=m
> PCIPCWATCHDOG=m
> WDTPCI=m
> USBPCWATCHDOG=m
> SSB_POSSIBLE=y
> BCMA_POSSIBLE=y
> MFD_CORE=m
> LPC_ICH=m
> MEDIA_SUPPORT=y
> AGP=y
> AGP_INTEL=m
> AGP_NVIDIA=y
> VGA_ARB=y
> VGA_ARB_MAX_GPUS=2
> DRM=m
> DRM_KMS_HELPER=m
> DRM_TTM=m
> DRM_I2C_CH7006=m
> DRM_I2C_SIL164=m
> DRM_NOUVEAU=m
> NOUVEAU_DEBUG=5
> NOUVEAU_DEBUG_DEFAULT=3
> VGASTATE=m
> VIDEO_OUTPUT_CONTROL=m
> HDMI=y
> FB=y
> FIRMWARE_EDID=y
> FB_DDC=m
> FB_CFB_FILLRECT=m
> FB_CFB_COPYAREA=m
> FB_CFB_IMAGEBLIT=m
> FB_MODE_HELPERS=y
> FB_NVIDIA=m
> FB_NVIDIA_I2C=y
> BACKLIGHT_LCD_SUPPORT=y
> BACKLIGHT_CLASS_DEVICE=y
> BACKLIGHT_GENERIC=y
> VGA_CONSOLE=y
> DUMMY_CONSOLE=y
> FRAMEBUFFER_CONSOLE=m
> FRAMEBUFFER_CONSOLE_DETECT_PRIMARY=y
> FONT_8x8=y
> FONT_8x16=y
> SOUND=y
> SOUND_OSS_CORE=y
> SOUND_OSS_CORE_PRECLAIM=y
> SND=y
> SND_TIMER=y
> SND_PCM=y
> SND_SEQUENCER=m
> SND_SEQ_DUMMY=m
> SND_OSSEMUL=y
> SND_MIXER_OSS=m
> SND_PCM_OSS=m
> SND_PCM_OSS_PLUGINS=y
> SND_SEQUENCER_OSS=y
> SND_HRTIMER=y
> SND_SEQ_HRTIMER_DEFAULT=y
> SND_SUPPORT_OLD_API=y
> SND_VERBOSE_PROCFS=y
> SND_VMASTER=y
> SND_DMA_SGBUF=y
> SND_AC97_CODEC=y
> SND_PCI=y
> SND_INTEL8X0=y
> AC97_BUS=y
> HID=y
> HIDRAW=y
> HID_GENERIC=m
> HID_A4TECH=y
> HID_APPLE=y
> HID_BELKIN=y
> HID_CHERRY=y
> HID_CHICONY=y
> HID_CYPRESS=y
> HID_DRAGONRISE=y
> HID_EZKEY=y
> HID_KYE=y
> HID_GYRATION=y
> HID_TWINHAN=y
> HID_KENSINGTON=y
> HID_LOGITECH=y
> HID_LOGITECH_DJ=m
> HID_MICROSOFT=y
> HID_MONTEREY=y
> HID_NTRIG=y
> HID_ORTEK=y
> HID_PANTHERLORD=y
> HID_PETALYNX=y
> HID_SAMSUNG=y
> HID_SONY=y
> HID_SUNPLUS=y
> HID_GREENASIA=y
> HID_SMARTJOYPLUS=y
> HID_TOPSEED=y
> HID_THRUSTMASTER=y
> HID_ZEROPLUS=y
> USB_HID=y
> USB_HIDDEV=y
> USB_ARCH_HAS_OHCI=y
> USB_ARCH_HAS_EHCI=y
> USB_ARCH_HAS_XHCI=y
> USB_SUPPORT=y
> USB_COMMON=y
> USB_ARCH_HAS_HCD=y
> USB=y
> USB_ANNOUNCE_NEW_DEVICES=y
> USB_DEFAULT_PERSIST=y
> USB_MON=y
> USB_EHCI_HCD=y
> USB_EHCI_ROOT_HUB_TT=y
> USB_EHCI_TT_NEWSCHED=y
> USB_EHCI_PCI=y
> USB_OHCI_HCD=y
> USB_OHCI_LITTLE_ENDIAN=y
> USB_ACM=m
> USB_PRINTER=m
> USB_WDM=m
> USB_STORAGE=m
> USB_MDC800=m
> USB_SERIAL=m
> USB_SERIAL_GENERIC=y
> USB_SERIAL_AIRCABLE=m
> USB_SERIAL_ARK3116=m
> USB_SERIAL_BELKIN=m
> USB_SERIAL_CH341=m
> USB_SERIAL_DIGI_ACCELEPORT=m
> USB_SERIAL_CYPRESS_M8=m
> USB_SERIAL_EMPEG=m
> USB_SERIAL_FTDI_SIO=m
> USB_SERIAL_FUNSOFT=m
> USB_SERIAL_VISOR=m
> USB_SERIAL_IPAQ=m
> USB_SERIAL_IR=m
> USB_SERIAL_EDGEPORT=m
> USB_SERIAL_EDGEPORT_TI=m
> USB_SERIAL_GARMIN=m
> USB_SERIAL_IPW=m
> USB_SERIAL_IUU=m
> USB_SERIAL_KEYSPAN_PDA=m
> USB_SERIAL_KEYSPAN=m
> USB_SERIAL_KLSI=m
> USB_SERIAL_KOBIL_SCT=m
> USB_SERIAL_MCT_U232=m
> USB_SERIAL_MOS7720=m
> USB_SERIAL_MOS7840=m
> USB_SERIAL_MOTOROLA=m
> USB_SERIAL_NAVMAN=m
> USB_SERIAL_PL2303=m
> USB_SERIAL_OTI6858=m
> USB_SERIAL_SPCP8X5=m
> USB_SERIAL_HP4X=m
> USB_SERIAL_SAFE=m
> USB_SERIAL_SIERRAWIRELESS=m
> USB_SERIAL_CYBERJACK=m
> USB_SERIAL_XIRCOM=m
> USB_SERIAL_WWAN=m
> USB_SERIAL_OPTION=m
> USB_SERIAL_OMNINET=m
> USB_SERIAL_DEBUG=m
> USB_ADUTUX=m
> USB_RIO500=m
> USB_LEGOTOWER=m
> USB_LCD=m
> USB_LED=m
> USB_CYPRESS_CY7C63=m
> USB_CYTHERM=m
> USB_IDMOUSE=m
> USB_FTDI_ELAN=m
> USB_APPLEDISPLAY=m
> USB_SISUSBVGA=m
> USB_SISUSBVGA_CON=y
> USB_LD=m
> USB_TRANCEVIBRATOR=m
> USB_IOWARRIOR=m
> USB_TEST=m
> USB_ISIGHTFW=m
> USB_EZUSB_FX2=m
> NEW_LEDS=y
> RTC_LIB=y
> RTC_CLASS=y
> RTC_HCTOSYS=y
> RTC_SYSTOHC=y
> RTC_HCTOSYS_DEVICE="rtc0"
> RTC_INTF_SYSFS=y
> RTC_INTF_PROC=y
> RTC_INTF_DEV=y
> RTC_DRV_DS1307=m
> RTC_DRV_DS1374=m
> RTC_DRV_DS1672=m
> RTC_DRV_MAX6900=m
> RTC_DRV_RS5C372=m
> RTC_DRV_ISL1208=m
> RTC_DRV_X1205=m
> RTC_DRV_PCF8563=m
> RTC_DRV_PCF8583=m
> RTC_DRV_M41T80=m
> RTC_DRV_S35390A=m
> RTC_DRV_FM3130=m
> RTC_DRV_CMOS=y
> RTC_DRV_DS1511=m
> RTC_DRV_DS1553=m
> RTC_DRV_DS1742=m
> RTC_DRV_STK17TA8=m
> RTC_DRV_M48T86=m
> RTC_DRV_M48T59=m
> RTC_DRV_V3020=m
> STAGING=y
> X86_PLATFORM_DEVICES=y
> ACPI_WMI=m
> MXM_WMI=m
> CLKSRC_I8253=y
> CLKEVT_I8253=y
> I8253_LOCK=y
> CLKBLD_I8253=y
> IOMMU_SUPPORT=y
> FIRMWARE_MEMMAP=y
> DMIID=y
> DCACHE_WORD_ACCESS=y
> EXT4_FS=y
> EXT4_USE_FOR_EXT23=y
> JBD2=y
> FS_MBCACHE=y
> FS_POSIX_ACL=y
> EXPORTFS=m
> FILE_LOCKING=y
> FSNOTIFY=y
> INOTIFY_USER=y
> FANOTIFY=y
> FUSE_FS=m
> GENERIC_ACL=y
> ISO9660_FS=m
> JOLIET=y
> ZISOFS=y
> UDF_FS=m
> UDF_NLS=y
> FAT_FS=m
> MSDOS_FS=m
> VFAT_FS=m
> FAT_DEFAULT_CODEPAGE=437
> FAT_DEFAULT_IOCHARSET="utf8"
> PROC_FS=y
> PROC_KCORE=y
> PROC_SYSCTL=y
> PROC_PAGE_MONITOR=y
> SYSFS=y
> TMPFS=y
> TMPFS_POSIX_ACL=y
> TMPFS_XATTR=y
> HUGETLBFS=y
> HUGETLB_PAGE=y
> CONFIGFS_FS=m
> NETWORK_FILESYSTEMS=y
> NFS_FS=m
> NFS_V2=m
> NFS_V3=m
> NFS_V3_ACL=y
> NFS_V4=m
> NFS_V4_1=y
> PNFS_FILE_LAYOUT=m
> NFS_V4_1_IMPLEMENTATION_ID_DOMAIN="kernel.org"
> NFS_USE_KERNEL_DNS=y
> NFSD=m
> NFSD_V2_ACL=y
> NFSD_V3=y
> NFSD_V3_ACL=y
> NFSD_V4=y
> LOCKD=m
> LOCKD_V4=y
> NFS_ACL_SUPPORT=m
> NFS_COMMON=y
> SUNRPC=m
> SUNRPC_GSS=m
> SUNRPC_BACKCHANNEL=y
> RPCSEC_GSS_KRB5=m
> CIFS=m
> CIFS_WEAK_PW_HASH=y
> CIFS_UPCALL=y
> CIFS_XATTR=y
> CIFS_POSIX=y
> CIFS_DEBUG=y
> CIFS_DFS_UPCALL=y
> NLS=y
> NLS_DEFAULT="utf8"
> NLS_CODEPAGE_437=m
> NLS_CODEPAGE_737=m
> NLS_CODEPAGE_775=m
> NLS_CODEPAGE_850=m
> NLS_CODEPAGE_852=m
> NLS_CODEPAGE_855=m
> NLS_CODEPAGE_857=m
> NLS_CODEPAGE_860=m
> NLS_CODEPAGE_861=m
> NLS_CODEPAGE_862=m
> NLS_CODEPAGE_863=m
> NLS_CODEPAGE_864=m
> NLS_CODEPAGE_865=m
> NLS_CODEPAGE_866=m
> NLS_CODEPAGE_869=m
> NLS_CODEPAGE_936=m
> NLS_CODEPAGE_950=m
> NLS_CODEPAGE_932=m
> NLS_CODEPAGE_949=m
> NLS_CODEPAGE_874=m
> NLS_ISO8859_8=m
> NLS_CODEPAGE_1250=m
> NLS_CODEPAGE_1251=m
> NLS_ASCII=m
> NLS_ISO8859_1=m
> NLS_ISO8859_2=m
> NLS_ISO8859_3=m
> NLS_ISO8859_4=m
> NLS_ISO8859_5=m
> NLS_ISO8859_6=m
> NLS_ISO8859_7=m
> NLS_ISO8859_9=m
> NLS_ISO8859_13=m
> NLS_ISO8859_14=m
> NLS_ISO8859_15=m
> NLS_KOI8_R=m
> NLS_KOI8_U=m
> NLS_UTF8=m
> TRACE_IRQFLAGS_SUPPORT=y
> PRINTK_TIME=y
> DEFAULT_MESSAGE_LOGLEVEL=4
> ENABLE_WARN_DEPRECATED=y
> ENABLE_MUST_CHECK=y
> FRAME_WARN=1024
> MAGIC_SYSRQ=y
> STRIP_ASM_SYMS=y
> DEBUG_KERNEL=y
> LOCKUP_DETECTOR=y
> HARDLOCKUP_DETECTOR=y
> BOOTPARAM_HARDLOCKUP_PANIC_VALUE=0
> BOOTPARAM_SOFTLOCKUP_PANIC_VALUE=0
> PANIC_ON_OOPS_VALUE=0
> DETECT_HUNG_TASK=y
> DEFAULT_HUNG_TASK_TIMEOUT=120
> BOOTPARAM_HUNG_TASK_PANIC_VALUE=0
> SCHED_DEBUG=y
> TIMER_STATS=y
> HAVE_DEBUG_KMEMLEAK=y
> DEBUG_BUGVERBOSE=y
> DEBUG_MEMORY_INIT=y
> ARCH_WANT_FRAME_POINTERS=y
> ARCH_HAS_DEBUG_STRICT_USER_COPY_CHECKS=y
> USER_STACKTRACE_SUPPORT=y
> HAVE_FUNCTION_TRACER=y
> HAVE_FUNCTION_GRAPH_TRACER=y
> HAVE_FUNCTION_GRAPH_FP_TEST=y
> HAVE_FUNCTION_TRACE_MCOUNT_TEST=y
> HAVE_DYNAMIC_FTRACE=y
> HAVE_DYNAMIC_FTRACE_WITH_REGS=y
> HAVE_FTRACE_MCOUNT_RECORD=y
> HAVE_SYSCALL_TRACEPOINTS=y
> HAVE_C_RECORDMCOUNT=y
> TRACING_SUPPORT=y
> HAVE_ARCH_KGDB=y
> HAVE_ARCH_KMEMCHECK=y
> X86_VERBOSE_BOOTUP=y
> EARLY_PRINTK=y
> DOUBLEFAULT=y
> HAVE_MMIOTRACE_SUPPORT=y
> IO_DELAY_TYPE_0X80=0
> IO_DELAY_TYPE_0XED=1
> IO_DELAY_TYPE_UDELAY=2
> IO_DELAY_TYPE_NONE=3
> IO_DELAY_NONE=y
> DEFAULT_IO_DELAY_TYPE=3
> OPTIMIZE_INLINING=y
> KEYS=y
> DEFAULT_SECURITY_DAC=y
> DEFAULT_SECURITY=""
> CRYPTO=y
> CRYPTO_ALGAPI=y
> CRYPTO_ALGAPI2=y
> CRYPTO_AEAD=m
> CRYPTO_AEAD2=y
> CRYPTO_BLKCIPHER=m
> CRYPTO_BLKCIPHER2=y
> CRYPTO_HASH=y
> CRYPTO_HASH2=y
> CRYPTO_RNG=m
> CRYPTO_RNG2=y
> CRYPTO_PCOMP2=y
> CRYPTO_MANAGER=y
> CRYPTO_MANAGER2=y
> CRYPTO_GF128MUL=m
> CRYPTO_NULL=m
> CRYPTO_WORKQUEUE=y
> CRYPTO_AUTHENC=m
> CRYPTO_TEST=m
> CRYPTO_CCM=m
> CRYPTO_GCM=m
> CRYPTO_SEQIV=m
> CRYPTO_CBC=m
> CRYPTO_CTR=m
> CRYPTO_CTS=m
> CRYPTO_ECB=m
> CRYPTO_LRW=m
> CRYPTO_PCBC=m
> CRYPTO_XTS=m
> CRYPTO_HMAC=y
> CRYPTO_XCBC=m
> CRYPTO_CRC32C=y
> CRYPTO_GHASH=m
> CRYPTO_MD4=m
> CRYPTO_MD5=y
> CRYPTO_MICHAEL_MIC=m
> CRYPTO_SHA1=m
> CRYPTO_SHA256=m
> CRYPTO_SHA512=m
> CRYPTO_TGR192=m
> CRYPTO_WP512=m
> CRYPTO_AES=y
> CRYPTO_AES_586=m
> CRYPTO_ANUBIS=m
> CRYPTO_ARC4=m
> CRYPTO_BLOWFISH=m
> CRYPTO_BLOWFISH_COMMON=m
> CRYPTO_CAMELLIA=m
> CRYPTO_CAST_COMMON=m
> CRYPTO_CAST5=m
> CRYPTO_CAST6=m
> CRYPTO_DES=m
> CRYPTO_FCRYPT=m
> CRYPTO_KHAZAD=m
> CRYPTO_SALSA20=m
> CRYPTO_SALSA20_586=m
> CRYPTO_SEED=m
> CRYPTO_SERPENT=m
> CRYPTO_TEA=m
> CRYPTO_TWOFISH=m
> CRYPTO_TWOFISH_COMMON=m
> CRYPTO_TWOFISH_586=m
> CRYPTO_DEFLATE=m
> CRYPTO_LZO=m
> HAVE_KVM=y
> BITREVERSE=y
> GENERIC_STRNCPY_FROM_USER=y
> GENERIC_STRNLEN_USER=y
> GENERIC_FIND_FIRST_BIT=y
> GENERIC_PCI_IOMAP=y
> GENERIC_IOMAP=y
> GENERIC_IO=y
> CRC_CCITT=m
> CRC16=y
> CRC_T10DIF=m
> CRC_ITU_T=m
> CRC32=y
> CRC32_SLICEBY8=y
> CRC7=m
> LIBCRC32C=m
> ZLIB_INFLATE=m
> ZLIB_DEFLATE=m
> LZO_COMPRESS=y
> LZO_DECOMPRESS=y
> HAS_IOMEM=y
> HAS_IOPORT=y
> HAS_DMA=y
> DQL=y
> NLATTR=y
> ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE=y
> OID_REGISTRY=m
> --
> To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
> the body of a message to majordomo@vger.kernel.org
> More majordomo info at http://vger.kernel.org/majordomo-info.html
> Please read the FAQ at http://www.tux.org/lkml/
>
--
Regards/Gruss,
Boris.
Sent from a fat crate under my desk. Formatting is fine.
--
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: 3.10.0 i386 uniprocessor panic
2013-07-18 8:18 ` Borislav Petkov
@ 2013-07-19 13:21 ` Thomas Gleixner
0 siblings, 0 replies; 7+ messages in thread
From: Thomas Gleixner @ 2013-07-19 13:21 UTC (permalink / raw)
To: Borislav Petkov; +Cc: George Spelvin, linux-kernel
On Thu, 18 Jul 2013, Borislav Petkov wrote:
> On Thu, Jul 18, 2013 at 02:13:47AM -0400, George Spelvin wrote:
> > I ressurected an old Athlon XP box for fun, and was stress-testing it
> > with mprime. (It had been stable before retirement.) After 34 hours
> > of successful torture test (suggesting a stable memory syatem), I found
> > this on the screen (hand-transcribed, top scrolled off):
> > Call Trace:
> > [<c102437d>] ? call_timer_fn.isra.37+0x16/0x6d
> > [<c1024524>] ? run_timer_softirq+0x150/0x165
> > [<c1020e8b>] ? __do_softirq+0x8b/0x135
Can you please enable debugobjects. That should catch the offender.
Thanks,
tglx
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: 3.10.0 i386 uniprocessor panic
2013-07-18 6:13 3.10.0 i386 uniprocessor panic George Spelvin
2013-07-18 8:18 ` Borislav Petkov
@ 2013-07-19 17:17 ` H. Peter Anvin
2013-07-19 21:00 ` George Spelvin
1 sibling, 1 reply; 7+ messages in thread
From: H. Peter Anvin @ 2013-07-19 17:17 UTC (permalink / raw)
To: George Spelvin; +Cc: linux-kernel
On 07/17/2013 11:13 PM, George Spelvin wrote:
> I ressurected an old Athlon XP box for fun, and was stress-testing it
> with mprime. (It had been stable before retirement.) After 34 hours
> of successful torture test (suggesting a stable memory syatem), I found
> this on the screen (hand-transcribed, top scrolled off):
>
> h_rpcgss oid_registry exportfs nfs_acl nfs lockd sunrpc loop fuse sil164 nouveau video mxm_wmi wmi ttm fbcon font bitblit softcursor drm_kms_helper drm i2c_algo_bit cfbcopyarea cfbfillrect serio_raw cfbimgblt hid_generic processor fan thermal thermal_sys button
> CPU: 0 PID: 3567 Comm: mprime Not tainted 3.10.0 #4
> Hardware name: /FN41 , BIOS 6.00 PG 08/23/2004
> task: f31849f0 ti: f3150000 task.ti: f3150000
> EIP: 0060:[<c143a091>] EFLAGS 00010286 CPU: 0
> EIP is at 0xc143a091
> EAX: c143a090 EBX: 00000100 ECX: f3150000 EDX: c143a090
> ESI: c143a090 EDI: c143a090 EBP: c143a090 ESP: f3151eec
> DS: 007b ES: 007b FS: 0000 GS: 0033 SS: 0068
> CR0: 80050033 CR2: a090c143 CR3: 331c6000 CR4: 000007d0
> DR0: 00000000 DR1: 00000000 DR2: 00000000 DR3: 00000000
> DR6: ffff0ff0 DR7: 00000400
> Stack:
> c102437d b665a951 0000713f 8ae3556c 0000a66a f31849f0 00000002 c1439980
> c143a080 c1024524 c143a090 c143a0a4 00000000 00000000 f3151f30 c143a190
> c143a390 c143a080 e63938bc 00000001 f3150000 c1439844 00000100 c1020e8b
> Call Trace:
> [<c102437d>] ? call_timer_fn.isra.37+0x16/0x6d
> [<c1024524>] ? run_timer_softirq+0x150/0x165
> [<c1020e8b>] ? __do_softirq+0x8b/0x135
> [<c1020fe4>] ? irq_exit+0x3d/0x72
> [<c10021f2>] ? do_IRQ+0x69/0x7c
> [<c1088524>] ? SyS_write+0x59/0x6a
> [<c10015ef>] ? math_state_restore+0x73/0xcd
> [<c128192c>] ? common_interrupt+0x2c/0x31
> Code: 43 c1 68 a0 43 c1 68 a0 43 c1 70 a0 43 c1 70 a0 43 c1 78 a0 43 c1 78 a0 43 c1 00 00 00 00 00 02 20 00 88 a0 43 c1 88 a0 43 c1 90 <a0> 43 c1 90 a0 43 c1 98 a0 43 c1 98 a0 43 c1 a0 a0 43 c1 a0 a0
> EIP: [<c143a091>] 0xc143a091 SS:ESP 0068:f3151eec
> CR2: 00000000a090c143
> ---[ end trace 4009bf27ab8c3bf3 ]---
> Kernel panic - not syncing: Fatal exception in interrupt
> drm_kms_helper: panic occurred, switching back to text console
>
> (The CR2 value looks particularly odd.)
>
Indeed it does; it is a user space value, but it doesn't look like
either a normal user space value nor really as a trivially buggered-up
kernel pointer value, unless the 0xc143... at the bottom is the upper
half of a kernel pointer, in which case we probably obtained this value
from a corrupt, misaligned pointer.
-hpa
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: 3.10.0 i386 uniprocessor panic
2013-07-19 17:17 ` H. Peter Anvin
@ 2013-07-19 21:00 ` George Spelvin
2013-07-19 22:25 ` H. Peter Anvin
0 siblings, 1 reply; 7+ messages in thread
From: George Spelvin @ 2013-07-19 21:00 UTC (permalink / raw)
To: hpa, linux; +Cc: linux-kernel
>> EIP is at 0xc143a091
>> EAX: c143a090 EBX: 00000100 ECX: f3150000 EDX: c143a090
>> ESI: c143a090 EDI: c143a090 EBP: c143a090 ESP: f3151eec
>> DS: 007b ES: 007b FS: 0000 GS: 0033 SS: 0068
>> CR0: 80050033 CR2: a090c143 CR3: 331c6000 CR4: 000007d0
>> DR0: 00000000 DR1: 00000000 DR2: 00000000 DR3: 00000000
>> DR6: ffff0ff0 DR7: 00000400
>> (The CR2 value looks particularly odd.)
> Indeed it does; it is a user space value, but it doesn't look like
> either a normal user space value nor really as a trivially buggered-up
> kernel pointer value, unless the 0xc143... at the bottom is the upper
> half of a kernel pointer, in which case we probably obtained this value
> from a corrupt, misaligned pointer.
Er... I assumed you'd see instantly that it was the 0xc143a090 value
that's in 5 registers (EAX/EDX/ESI/EDI/EBP), and IP-1, but with the
halves swapped.
How the heck the halves got swapped is confusing me...
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: 3.10.0 i386 uniprocessor panic
2013-07-19 21:00 ` George Spelvin
@ 2013-07-19 22:25 ` H. Peter Anvin
2013-07-19 23:45 ` George Spelvin
0 siblings, 1 reply; 7+ messages in thread
From: H. Peter Anvin @ 2013-07-19 22:25 UTC (permalink / raw)
To: George Spelvin; +Cc: linux-kernel
On 07/19/2013 02:00 PM, George Spelvin wrote:
>>> EIP is at 0xc143a091
>>> EAX: c143a090 EBX: 00000100 ECX: f3150000 EDX: c143a090
>>> ESI: c143a090 EDI: c143a090 EBP: c143a090 ESP: f3151eec
>>> DS: 007b ES: 007b FS: 0000 GS: 0033 SS: 0068
>>> CR0: 80050033 CR2: a090c143 CR3: 331c6000 CR4: 000007d0
>>> DR0: 00000000 DR1: 00000000 DR2: 00000000 DR3: 00000000
>>> DR6: ffff0ff0 DR7: 00000400
>
>>> (The CR2 value looks particularly odd.)
>
>> Indeed it does; it is a user space value, but it doesn't look like
>> either a normal user space value nor really as a trivially buggered-up
>> kernel pointer value, unless the 0xc143... at the bottom is the upper
>> half of a kernel pointer, in which case we probably obtained this value
>> from a corrupt, misaligned pointer.
>
> Er... I assumed you'd see instantly that it was the 0xc143a090 value
> that's in 5 registers (EAX/EDX/ESI/EDI/EBP), and IP-1, but with the
> halves swapped.
That would have requiring me to actually pay attention. I claim
undercaffeination.
> How the heck the halves got swapped is confusing me...
Disassembling the "code" (which is really data) makes it kind of obvious:
C143A090 90 nop
C143A091 A043C190A0 mov al,[0xa090c143] ; fault here
We jumped into data which contained a series of self-pointers
(presumably empty double-linked lists), and the first two bytes became
opcodes...
Unfortunately the disassembly of call_timer_fn.isra.37 doesn't really
tell us anything other than that the passed-in value of %eax was bogus.
It is *very* interesting, though, that that value is present in so many
registers (in fact, the ONLY GPRs which didn't have that value are %ebx
and %ecx, which are set by that function itself.)
A disassembly of the calling function, i.e.:
[<c1024524>] ? run_timer_softirq+0x150/0x165
... would be a good idea, at least.
-hpa
^ permalink raw reply [flat|nested] 7+ messages in thread
* Re: 3.10.0 i386 uniprocessor panic
2013-07-19 22:25 ` H. Peter Anvin
@ 2013-07-19 23:45 ` George Spelvin
0 siblings, 0 replies; 7+ messages in thread
From: George Spelvin @ 2013-07-19 23:45 UTC (permalink / raw)
To: hpa, linux; +Cc: linux-kernel
> A disassembly of the calling function, i.e.:
>
> [<c1024524>] ? run_timer_softirq+0x150/0x165
>
> ... would be a good idea, at least.
Here you go. Note that the machine isn't doing anything of interest,
so after a quick search for residual personal data I could give you a
root login on it. (Just send ssh key.)
kernel/timer.o: file format elf32-i386
Disassembly of section .text:
[snip...]
00000289 <run_timer_softirq>:
289: 55 push %ebp
28a: 57 push %edi
28b: 56 push %esi
28c: 53 push %ebx
28d: 83 ec 24 sub $0x24,%esp
290: 8b 1d 00 00 00 00 mov 0x0,%ebx
296: e8 fc ff ff ff call 297 <run_timer_softirq+0xe>
29b: a1 00 00 00 00 mov 0x0,%eax
2a0: 3b 43 04 cmp 0x4(%ebx),%eax
2a3: 0f 88 3d 01 00 00 js 3e6 <run_timer_softirq+0x15d>
2a9: fa cli
2aa: 8d 83 10 08 00 00 lea 0x810(%ebx),%eax
2b0: 89 44 24 14 mov %eax,0x14(%esp)
2b4: 8d 83 10 0a 00 00 lea 0xa10(%ebx),%eax
2ba: 89 44 24 18 mov %eax,0x18(%esp)
2be: a1 00 00 00 00 mov 0x0,%eax
2c3: 8b 4b 04 mov 0x4(%ebx),%ecx
2c6: 39 c8 cmp %ecx,%eax
2c8: 0f 88 11 01 00 00 js 3df <run_timer_softirq+0x156>
2ce: 0f b6 f1 movzbl %cl,%esi
2d1: 85 f6 test %esi,%esi
2d3: 75 5a jne 32f <run_timer_softirq+0xa6>
2d5: 8b 54 24 14 mov 0x14(%esp),%edx
2d9: c1 e9 08 shr $0x8,%ecx
2dc: 89 d8 mov %ebx,%eax
2de: 83 e1 3f and $0x3f,%ecx
2e1: e8 e3 fe ff ff call 1c9 <cascade>
2e6: 85 c0 test %eax,%eax
2e8: 75 45 jne 32f <run_timer_softirq+0xa6>
2ea: 8b 4b 04 mov 0x4(%ebx),%ecx
2ed: 8b 54 24 18 mov 0x18(%esp),%edx
2f1: 89 d8 mov %ebx,%eax
2f3: c1 e9 0e shr $0xe,%ecx
2f6: 83 e1 3f and $0x3f,%ecx
2f9: e8 cb fe ff ff call 1c9 <cascade>
2fe: 85 c0 test %eax,%eax
300: 75 2d jne 32f <run_timer_softirq+0xa6>
302: 8b 4b 04 mov 0x4(%ebx),%ecx
305: 8d 93 10 0c 00 00 lea 0xc10(%ebx),%edx
30b: 89 d8 mov %ebx,%eax
30d: c1 e9 14 shr $0x14,%ecx
310: 83 e1 3f and $0x3f,%ecx
313: e8 b1 fe ff ff call 1c9 <cascade>
318: 85 c0 test %eax,%eax
31a: 75 13 jne 32f <run_timer_softirq+0xa6>
31c: 8b 4b 04 mov 0x4(%ebx),%ecx
31f: 8d 93 10 0e 00 00 lea 0xe10(%ebx),%edx
325: 89 d8 mov %ebx,%eax
327: c1 e9 1a shr $0x1a,%ecx
32a: e8 9a fe ff ff call 1c9 <cascade>
32f: 8d 0c f3 lea (%ebx,%esi,8),%ecx
332: ff 43 04 incl 0x4(%ebx)
335: 8d 54 24 1c lea 0x1c(%esp),%edx
339: 8b 71 10 mov 0x10(%ecx),%esi
33c: 8d 41 10 lea 0x10(%ecx),%eax
33f: 89 54 24 10 mov %edx,0x10(%esp)
343: 89 56 04 mov %edx,0x4(%esi)
346: 89 74 24 1c mov %esi,0x1c(%esp)
34a: 8b 71 14 mov 0x14(%ecx),%esi
34d: 89 74 24 20 mov %esi,0x20(%esp)
351: 89 16 mov %edx,(%esi)
353: 89 41 10 mov %eax,0x10(%ecx)
356: 89 40 04 mov %eax,0x4(%eax)
359: 8b 74 24 1c mov 0x1c(%esp),%esi
35d: 3b 74 24 10 cmp 0x10(%esp),%esi
361: 0f 84 57 ff ff ff je 2be <run_timer_softirq+0x35>
367: 8b 46 0c mov 0xc(%esi),%eax
36a: 8b 4e 20 mov 0x20(%esi),%ecx
36d: 8b 7e 10 mov 0x10(%esi),%edi
370: 8b 6e 14 mov 0x14(%esi),%ebp
373: 89 c2 mov %eax,%edx
375: 83 e2 02 and $0x2,%edx
378: 85 c9 test %ecx,%ecx
37a: 89 54 24 0c mov %edx,0xc(%esp)
37e: 74 1b je 39b <run_timer_softirq+0x112>
380: 83 e0 01 and $0x1,%eax
383: 8b 56 1c mov 0x1c(%esi),%edx
386: 89 3c 24 mov %edi,(%esp)
389: 89 44 24 08 mov %eax,0x8(%esp)
38d: 8d 46 24 lea 0x24(%esi),%eax
390: 89 44 24 04 mov %eax,0x4(%esp)
394: 89 f0 mov %esi,%eax
396: e8 fc ff ff ff call 397 <run_timer_softirq+0x10e>
39b: 89 33 mov %esi,(%ebx)
39d: 8b 16 mov (%esi),%edx
39f: 8b 46 04 mov 0x4(%esi),%eax
3a2: 89 42 04 mov %eax,0x4(%edx)
3a5: 89 10 mov %edx,(%eax)
3a7: f6 46 0c 01 testb $0x1,0xc(%esi)
3ab: c7 06 00 00 00 00 movl $0x0,(%esi)
3b1: c7 46 04 00 02 20 00 movl $0x200200,0x4(%esi)
3b8: 75 03 jne 3bd <run_timer_softirq+0x134>
3ba: ff 4b 0c decl 0xc(%ebx)
3bd: 83 7c 24 0c 00 cmpl $0x0,0xc(%esp)
3c2: 74 0b je 3cf <run_timer_softirq+0x146>
3c4: 89 ea mov %ebp,%edx
3c6: 89 f8 mov %edi,%eax
3c8: e8 4f fe ff ff call 21c <call_timer_fn.isra.37>
3cd: eb 0b jmp 3da <run_timer_softirq+0x151>
3cf: fb sti
3d0: 89 ea mov %ebp,%edx
3d2: 89 f8 mov %edi,%eax
3d4: e8 43 fe ff ff call 21c <call_timer_fn.isra.37>
3d9: fa cli
3da: e9 7a ff ff ff jmp 359 <run_timer_softirq+0xd0>
3df: c7 03 00 00 00 00 movl $0x0,(%ebx)
3e5: fb sti
3e6: 83 c4 24 add $0x24,%esp
3e9: 5b pop %ebx
3ea: 5e pop %esi
3eb: 5f pop %edi
3ec: 5d pop %ebp
3ed: c3 ret
^ permalink raw reply [flat|nested] 7+ messages in thread
end of thread, other threads:[~2013-07-19 23:45 UTC | newest]
Thread overview: 7+ messages (download: mbox.gz / follow: Atom feed)
-- links below jump to the message on this page --
2013-07-18 6:13 3.10.0 i386 uniprocessor panic George Spelvin
2013-07-18 8:18 ` Borislav Petkov
2013-07-19 13:21 ` Thomas Gleixner
2013-07-19 17:17 ` H. Peter Anvin
2013-07-19 21:00 ` George Spelvin
2013-07-19 22:25 ` H. Peter Anvin
2013-07-19 23:45 ` George Spelvin
This is an external index of several public inboxes,
see mirroring instructions on how to clone and mirror
all data and code used by this external index.