[PATCH]: allow individual core dump methods to be unlimited when sending to a pipe

[PATCH]: allow individual core dump methods to be unlimited when sending to a pipe

[Neil Horman]

Hey-
	This is a follow on to the patch entitled:
update-coredump-path-in-kernel-to-not-check-coredump-rlim-if-core_pattern-is-a-pipe.patch
It allows individual core dump methods to have the core dump size limit passed
into them.  Its more efficient than each method having to retrieve it on their
own, and it allows for do_coredump to specify a infinite limit in the event that
a pipe is configured in /proc/sys/kernel/core_pattern, in which case ulimit -c
should not apply.

Regards
Neil

Signed-off-by: Neil Horman <nhorman@tuxdriver.com>


 arch/mips/kernel/irixelf.c          |    5 ++---
 arch/sparc64/kernel/binfmt_aout32.c |    8 +++-----
 arch/x86_64/ia32/ia32_aout.c        |   10 ++++------
 fs/binfmt_aout.c                    |   16 ++++++----------
 fs/binfmt_elf.c                     |    5 ++---
 fs/binfmt_elf_fdpic.c               |    5 ++---
 fs/binfmt_flat.c                    |    4 ++--
 fs/binfmt_som.c                     |    2 +-
 fs/exec.c                           |    6 ++++--
 include/linux/binfmts.h             |    2 +-
 10 files changed, 27 insertions(+), 36 deletions(-)


diff --git a/arch/mips/kernel/irixelf.c b/arch/mips/kernel/irixelf.c
index 403d96f..5f11d6e 100644
--- a/arch/mips/kernel/irixelf.c
+++ b/arch/mips/kernel/irixelf.c
@@ -44,7 +44,7 @@
 static int load_irix_binary(struct linux_binprm * bprm, struct pt_regs * regs);
 static int load_irix_library(struct file *);
 static int irix_core_dump(long signr, struct pt_regs * regs,
-                          struct file *file);
+                          struct file *file, int limit);
 
 static struct linux_binfmt irix_format = {
 	NULL, THIS_MODULE, load_irix_binary, load_irix_library,
@@ -1088,7 +1088,7 @@ end_coredump:
  * and then they are actually written out.  If we run out of core limit
  * we just truncate.
  */
-static int irix_core_dump(long signr, struct pt_regs * regs, struct file *file)
+static int irix_core_dump(long signr, struct pt_regs * regs, struct file *file, int limit)
 {
 	int has_dumped = 0;
 	mm_segment_t fs;
@@ -1098,7 +1098,6 @@ static int irix_core_dump(long signr, struct pt_regs * regs, struct file *file)
 	struct vm_area_struct *vma;
 	struct elfhdr elf;
 	off_t offset = 0, dataoff;
-	int limit = current->signal->rlim[RLIMIT_CORE].rlim_cur;
 	int numnote = 3;
 	struct memelfnote notes[3];
 	struct elf_prstatus prstatus;	/* NT_PRSTATUS */
diff --git a/arch/sparc64/kernel/binfmt_aout32.c b/arch/sparc64/kernel/binfmt_aout32.c
index f205fc7..d71691f 100644
--- a/arch/sparc64/kernel/binfmt_aout32.c
+++ b/arch/sparc64/kernel/binfmt_aout32.c
@@ -83,7 +83,7 @@ if (file->f_op->llseek) { \
  * dumping of the process results in another error..
  */
 
-static int aout32_core_dump(long signr, struct pt_regs *regs, struct file *file)
+static int aout32_core_dump(long signr, struct pt_regs *regs, struct file *file, u32 limit)
 {
 	mm_segment_t fs;
 	int has_dumped = 0;
@@ -102,13 +102,11 @@ static int aout32_core_dump(long signr, struct pt_regs *regs, struct file *file)
 
 /* If the size of the dump file exceeds the rlimit, then see what would happen
    if we wrote the stack, but not the data area.  */
-	if ((dump.u_dsize+dump.u_ssize) >
-	    current->signal->rlim[RLIMIT_CORE].rlim_cur)
+	if ((dump.u_dsize+dump.u_ssize) > limit)
 		dump.u_dsize = 0;
 
 /* Make sure we have enough room to write the stack and data areas. */
-	if ((dump.u_ssize) >
-	    current->signal->rlim[RLIMIT_CORE].rlim_cur)
+	if ((dump.u_ssize) > limit)
 		dump.u_ssize = 0;
 
 /* make sure we actually have a data and stack area to dump */
diff --git a/arch/x86_64/ia32/ia32_aout.c b/arch/x86_64/ia32/ia32_aout.c
index 0878137..5967cc5 100644
--- a/arch/x86_64/ia32/ia32_aout.c
+++ b/arch/x86_64/ia32/ia32_aout.c
@@ -40,7 +40,7 @@ static int load_aout_binary(struct linux_binprm *, struct pt_regs * regs);
 static int load_aout_library(struct file*);
 
 #ifdef CORE_DUMP
-static int aout_core_dump(long signr, struct pt_regs * regs, struct file *file);
+static int aout_core_dump(long signr, struct pt_regs * regs, struct file *file, u32 limit);
 
 /*
  * fill in the user structure for a core dump..
@@ -148,7 +148,7 @@ if (file->f_op->llseek) { \
  * dumping of the process results in another error..
  */
 
-static int aout_core_dump(long signr, struct pt_regs * regs, struct file *file)
+static int aout_core_dump(long signr, struct pt_regs * regs, struct file *file, u32 limit)
 {
 	mm_segment_t fs;
 	int has_dumped = 0;
@@ -168,13 +168,11 @@ static int aout_core_dump(long signr, struct pt_regs * regs, struct file *file)
 
 /* If the size of the dump file exceeds the rlimit, then see what would happen
    if we wrote the stack, but not the data area.  */
-	if ((dump.u_dsize+dump.u_ssize+1) * PAGE_SIZE >
-	    current->signal->rlim[RLIMIT_CORE].rlim_cur)
+	if ((dump.u_dsize+dump.u_ssize+1) * PAGE_SIZE > limit)
 		dump.u_dsize = 0;
 
 /* Make sure we have enough room to write the stack and data areas. */
-	if ((dump.u_ssize+1) * PAGE_SIZE >
-	    current->signal->rlim[RLIMIT_CORE].rlim_cur)
+	if ((dump.u_ssize+1) * PAGE_SIZE > limit)
 		dump.u_ssize = 0;
 
 /* make sure we actually have a data and stack area to dump */
diff --git a/fs/binfmt_aout.c b/fs/binfmt_aout.c
index 813a887..e247d80 100644
--- a/fs/binfmt_aout.c
+++ b/fs/binfmt_aout.c
@@ -31,7 +31,7 @@
 
 static int load_aout_binary(struct linux_binprm *, struct pt_regs * regs);
 static int load_aout_library(struct file*);
-static int aout_core_dump(long signr, struct pt_regs * regs, struct file *file);
+static int aout_core_dump(long signr, struct pt_regs * regs, struct file *file, u32 limit);
 
 static struct linux_binfmt aout_format = {
 	.module		= THIS_MODULE,
@@ -88,7 +88,7 @@ if (file->f_op->llseek) { \
  * dumping of the process results in another error..
  */
 
-static int aout_core_dump(long signr, struct pt_regs * regs, struct file *file)
+static int aout_core_dump(long signr, struct pt_regs * regs, struct file *file, u32 limit)
 {
 	mm_segment_t fs;
 	int has_dumped = 0;
@@ -123,23 +123,19 @@ static int aout_core_dump(long signr, struct pt_regs * regs, struct file *file)
 /* If the size of the dump file exceeds the rlimit, then see what would happen
    if we wrote the stack, but not the data area.  */
 #ifdef __sparc__
-	if ((dump.u_dsize+dump.u_ssize) >
-	    current->signal->rlim[RLIMIT_CORE].rlim_cur)
+	if ((dump.u_dsize+dump.u_ssize) > limit)
 		dump.u_dsize = 0;
 #else
-	if ((dump.u_dsize+dump.u_ssize+1) * PAGE_SIZE >
-	    current->signal->rlim[RLIMIT_CORE].rlim_cur)
+	if ((dump.u_dsize+dump.u_ssize+1) * PAGE_SIZE > limit)
 		dump.u_dsize = 0;
 #endif
 
 /* Make sure we have enough room to write the stack and data areas. */
 #ifdef __sparc__
-	if ((dump.u_ssize) >
-	    current->signal->rlim[RLIMIT_CORE].rlim_cur)
+	if ((dump.u_ssize) > limit)
 		dump.u_ssize = 0;
 #else
-	if ((dump.u_ssize+1) * PAGE_SIZE >
-	    current->signal->rlim[RLIMIT_CORE].rlim_cur)
+	if ((dump.u_ssize+1) * PAGE_SIZE > limit)
 		dump.u_ssize = 0;
 #endif
 
diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c
index ba24cb2..b1c70cb 100644
--- a/fs/binfmt_elf.c
+++ b/fs/binfmt_elf.c
@@ -52,7 +52,7 @@ static unsigned long elf_map (struct file *, unsigned long, struct elf_phdr *, i
  * don't even try.
  */
 #if defined(USE_ELF_CORE_DUMP) && defined(CONFIG_ELF_CORE)
-static int elf_core_dump(long signr, struct pt_regs *regs, struct file *file);
+static int elf_core_dump(long signr, struct pt_regs *regs, struct file *file, u32 limit);
 #else
 #define elf_core_dump	NULL
 #endif
@@ -1551,7 +1551,7 @@ static struct vm_area_struct *next_vma(struct vm_area_struct *this_vma,
  * and then they are actually written out.  If we run out of core limit
  * we just truncate.
  */
-static int elf_core_dump(long signr, struct pt_regs *regs, struct file *file)
+static int elf_core_dump(long signr, struct pt_regs *regs, struct file *file, u32 limit)
 {
 #define	NUM_NOTES	6
 	int has_dumped = 0;
@@ -1562,7 +1562,6 @@ static int elf_core_dump(long signr, struct pt_regs *regs, struct file *file)
 	struct vm_area_struct *vma, *gate_vma;
 	struct elfhdr *elf = NULL;
 	loff_t offset = 0, dataoff, foffset;
-	unsigned long limit = current->signal->rlim[RLIMIT_CORE].rlim_cur;
 	int numnote;
 	struct memelfnote *notes = NULL;
 	struct elf_prstatus *prstatus = NULL;	/* NT_PRSTATUS */
diff --git a/fs/binfmt_elf_fdpic.c b/fs/binfmt_elf_fdpic.c
index 2f5d8db..fb4ad53 100644
--- a/fs/binfmt_elf_fdpic.c
+++ b/fs/binfmt_elf_fdpic.c
@@ -75,7 +75,7 @@ static int elf_fdpic_map_file_by_direct_mmap(struct elf_fdpic_params *,
 					     struct file *, struct mm_struct *);
 
 #if defined(USE_ELF_CORE_DUMP) && defined(CONFIG_ELF_CORE)
-static int elf_fdpic_core_dump(long, struct pt_regs *, struct file *);
+static int elf_fdpic_core_dump(long, struct pt_regs *, struct file *, u32 limit);
 #endif
 
 static struct linux_binfmt elf_fdpic_format = {
@@ -1552,7 +1552,7 @@ static int elf_fdpic_dump_segments(struct file *file, size_t *size,
  * we just truncate.
  */
 static int elf_fdpic_core_dump(long signr, struct pt_regs *regs,
-			       struct file *file)
+			       struct file *file, u32 limit)
 {
 #define	NUM_NOTES	6
 	int has_dumped = 0;
@@ -1563,7 +1563,6 @@ static int elf_fdpic_core_dump(long signr, struct pt_regs *regs,
 	struct vm_area_struct *vma;
 	struct elfhdr *elf = NULL;
 	loff_t offset = 0, dataoff;
-	unsigned long limit = current->signal->rlim[RLIMIT_CORE].rlim_cur;
 	int numnote;
 	struct memelfnote *notes = NULL;
 	struct elf_prstatus *prstatus = NULL;	/* NT_PRSTATUS */
diff --git a/fs/binfmt_flat.c b/fs/binfmt_flat.c
index 861141b..1a17556 100644
--- a/fs/binfmt_flat.c
+++ b/fs/binfmt_flat.c
@@ -75,7 +75,7 @@ static int load_flat_shared_library(int id, struct lib_info *p);
 #endif
 
 static int load_flat_binary(struct linux_binprm *, struct pt_regs * regs);
-static int flat_core_dump(long signr, struct pt_regs * regs, struct file *file);
+static int flat_core_dump(long signr, struct pt_regs * regs, struct file *file, u32 limit);
 
 static struct linux_binfmt flat_format = {
 	.module		= THIS_MODULE,
@@ -90,7 +90,7 @@ static struct linux_binfmt flat_format = {
  * Currently only a stub-function.
  */
 
-static int flat_core_dump(long signr, struct pt_regs * regs, struct file *file)
+static int flat_core_dump(long signr, struct pt_regs * regs, struct file *file, u32 limit)
 {
 	printk("Process %s:%d received signr %d and should have core dumped\n",
 			current->comm, current->pid, (int) signr);
diff --git a/fs/binfmt_som.c b/fs/binfmt_som.c
index 5bcdaaf..0da1fae 100644
--- a/fs/binfmt_som.c
+++ b/fs/binfmt_som.c
@@ -44,7 +44,7 @@ static int load_som_library(struct file *);
  * don't even try.
  */
 #if 0
-static int som_core_dump(long signr, struct pt_regs * regs);
+static int som_core_dump(long signr, struct pt_regs * regs, u32 limit);
 #else
 #define som_core_dump	NULL
 #endif
diff --git a/fs/exec.c b/fs/exec.c
index e82a01d..3b2e1e7 100644
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -1726,6 +1726,7 @@ int do_coredump(long signr, int exit_code, struct pt_regs * regs)
 	int fsuid = current->fsuid;
 	int flag = 0;
 	int ispipe = 0;
+	u32 core_limit = current->signal->rlim[RLIMIT_CORE].rlim_cur;
 
 	audit_core_dumps(signr);
 
@@ -1775,10 +1776,11 @@ int do_coredump(long signr, int exit_code, struct pt_regs * regs)
 	 * as it does with any other process
 	 */
 	if ((!ispipe) &&
-	   (current->signal->rlim[RLIMIT_CORE].rlim_cur < binfmt->min_coredump))
+	   (core_limit < binfmt->min_coredump))
 		goto fail_unlock;
 
  	if (ispipe) {
+		core_limit = RLIM_INFINITY;
 		/* SIGPIPE can happen, but it's just never processed */
  		if(call_usermodehelper_pipe(corename+1, NULL, NULL, &file)) {
  			printk(KERN_INFO "Core dump to %s pipe failed\n",
@@ -1808,7 +1810,7 @@ int do_coredump(long signr, int exit_code, struct pt_regs * regs)
 	if (!ispipe && do_truncate(file->f_path.dentry, 0, 0, file) != 0)
 		goto close_fail;
 
-	retval = binfmt->core_dump(signr, regs, file);
+	retval = binfmt->core_dump(signr, regs, file, core_limit);
 
 	if (retval)
 		current->signal->group_exit_code |= 0x80;
diff --git a/include/linux/binfmts.h b/include/linux/binfmts.h
index 91c8c07..3188627 100644
--- a/include/linux/binfmts.h
+++ b/include/linux/binfmts.h
@@ -67,7 +67,7 @@ struct linux_binfmt {
 	struct module *module;
 	int (*load_binary)(struct linux_binprm *, struct  pt_regs * regs);
 	int (*load_shlib)(struct file *);
-	int (*core_dump)(long signr, struct pt_regs * regs, struct file * file);
+	int (*core_dump)(long signr, struct pt_regs * regs, struct file * file, u32 limit);
 	unsigned long min_coredump;	/* minimal dump size */
 	int hasvdso;
 };
-- 
/***************************************************
 *Neil Horman
 *Software Engineer
 *Red Hat, Inc.
 *nhorman@tuxdriver.com
 *gpg keyid: 1024D / 0x92A74FA1
 *http://pgp.mit.edu
 ***************************************************/

Re: [PATCH]: allow individual core dump methods to be unlimited when sending to a pipe

[Andrew Morton]

On Sun, 22 Jul 2007 11:39:47 -0400 Neil Horman <nhorman@tuxdriver.com> wrote:

> 	This is a follow on to the patch entitled:
> update-coredump-path-in-kernel-to-not-check-coredump-rlim-if-core_pattern-is-a-pipe.patch
> It allows individual core dump methods to have the core dump size limit passed
> into them.  Its more efficient than each method having to retrieve it on their
> own, and it allows for do_coredump to specify a infinite limit in the event that
> a pipe is configured in /proc/sys/kernel/core_pattern, in which case ulimit -c
> should not apply.

Needed this warning fix:

fs/exec.c: In function 'do_coredump':
fs/exec.c:1761: warning: large integer implicitly truncated to unsigned type

diff -puN fs/exec.c~allow-individual-core-dump-methods-to-be-unlimited-when-sending-to-a-pipe-fix fs/exec.c
--- a/fs/exec.c~allow-individual-core-dump-methods-to-be-unlimited-when-sending-to-a-pipe-fix
+++ a/fs/exec.c
@@ -1703,7 +1703,7 @@ int do_coredump(long signr, int exit_cod
 	int fsuid = current->fsuid;
 	int flag = 0;
 	int ispipe = 0;
-	u32 core_limit = current->signal->rlim[RLIMIT_CORE].rlim_cur;
+	unsigned long core_limit = current->signal->rlim[RLIMIT_CORE].rlim_cur;
 
 	audit_core_dumps(signr);
 
_

which makes me worry about how much care was taken here on 32/64 issues,
and crsos-arch issues (the underlying type for compat_ulong_t, for one).

Could you please go over it with a toothcomb, see if there are any other
slipups there?


Thanks.

Re: [PATCH]: allow individual core dump methods to be unlimited when sending to a pipe

[Neil Horman]

On Tue, Jul 24, 2007 at 02:09:39AM -0700, Andrew Morton wrote:
> On Sun, 22 Jul 2007 11:39:47 -0400 Neil Horman <nhorman@tuxdriver.com> wrote:
> 
> > 	This is a follow on to the patch entitled:
> > update-coredump-path-in-kernel-to-not-check-coredump-rlim-if-core_pattern-is-a-pipe.patch
> > It allows individual core dump methods to have the core dump size limit passed
> > into them.  Its more efficient than each method having to retrieve it on their
> > own, and it allows for do_coredump to specify a infinite limit in the event that
> > a pipe is configured in /proc/sys/kernel/core_pattern, in which case ulimit -c
> > should not apply.
> 
> Needed this warning fix:
> 
> fs/exec.c: In function 'do_coredump':
> fs/exec.c:1761: warning: large integer implicitly truncated to unsigned type
> 
> diff -puN fs/exec.c~allow-individual-core-dump-methods-to-be-unlimited-when-sending-to-a-pipe-fix fs/exec.c
> --- a/fs/exec.c~allow-individual-core-dump-methods-to-be-unlimited-when-sending-to-a-pipe-fix
> +++ a/fs/exec.c
> @@ -1703,7 +1703,7 @@ int do_coredump(long signr, int exit_cod
>  	int fsuid = current->fsuid;
>  	int flag = 0;
>  	int ispipe = 0;
> -	u32 core_limit = current->signal->rlim[RLIMIT_CORE].rlim_cur;
> +	unsigned long core_limit = current->signal->rlim[RLIMIT_CORE].rlim_cur;
>  
>  	audit_core_dumps(signr);
>  
> _
> 
> which makes me worry about how much care was taken here on 32/64 issues,
> and crsos-arch issues (the underlying type for compat_ulong_t, for one).
> 
> Could you please go over it with a toothcomb, see if there are any other
> slipups there?
> 
Yes, I had noticed this yesterday aternoon as well, Andrew. Sorry about that.
Upon looking at it using unsigned long should be the superset of all the
definitions, so I'm working on a patch to align that, and am testing it on all
the arches I have available here (x86, x86_64, ia64, and perhaps ppc64 if I can
get hold of it).  I also have a few more bugs that need fixing in the
core_pattern/pipe code as well as one more feature enhancement (the ability to
pass an argv array).  I'll fix up the bugs first, and you can expect these
patches in the next day or two.

Thanks & Regards
Neil
> 
> Thanks.

-- 
/***************************************************
 *Neil Horman
 *Software Engineer
 *Red Hat, Inc.
 *nhorman@tuxdriver.com
 *gpg keyid: 1024D / 0x92A74FA1
 *http://pgp.mit.edu
 ***************************************************/

Re: [PATCH]: allow individual core dump methods to be unlimited when sending to a pipe

[Neil Horman]

On Tue, Jul 24, 2007 at 02:09:39AM -0700, Andrew Morton wrote:
> On Sun, 22 Jul 2007 11:39:47 -0400 Neil Horman <nhorman@tuxdriver.com> wrote:
> 
> > 	This is a follow on to the patch entitled:
> > update-coredump-path-in-kernel-to-not-check-coredump-rlim-if-core_pattern-is-a-pipe.patch
> > It allows individual core dump methods to have the core dump size limit passed
> > into them.  Its more efficient than each method having to retrieve it on their
> > own, and it allows for do_coredump to specify a infinite limit in the event that
> > a pipe is configured in /proc/sys/kernel/core_pattern, in which case ulimit -c
> > should not apply.
> 
> Needed this warning fix:
> 
> fs/exec.c: In function 'do_coredump':
> fs/exec.c:1761: warning: large integer implicitly truncated to unsigned type
> 
> diff -puN fs/exec.c~allow-individual-core-dump-methods-to-be-unlimited-when-sending-to-a-pipe-fix fs/exec.c
> --- a/fs/exec.c~allow-individual-core-dump-methods-to-be-unlimited-when-sending-to-a-pipe-fix
> +++ a/fs/exec.c
> @@ -1703,7 +1703,7 @@ int do_coredump(long signr, int exit_cod
>  	int fsuid = current->fsuid;
>  	int flag = 0;
>  	int ispipe = 0;
> -	u32 core_limit = current->signal->rlim[RLIMIT_CORE].rlim_cur;
> +	unsigned long core_limit = current->signal->rlim[RLIMIT_CORE].rlim_cur;
>  
>  	audit_core_dumps(signr);
>  
> _
> 
> which makes me worry about how much care was taken here on 32/64 issues,
> and crsos-arch issues (the underlying type for compat_ulong_t, for one).
> 
> Could you please go over it with a toothcomb, see if there are any other
> slipups there?
> 
> 
> Thanks.

Heres the subsequent patch that I promised you.  It changes all the limit
passing to be unsigned longs.  I've built it on x86, x86_64 and ia64 and it
seems to be good to go.  It leaves off the bit that you have above, since you've
already included that.

Thanks & Regards
Neil


Signed-off-by: Neil Horman <nhorman@tuxdriver.com>


 arch/x86_64/ia32/ia32_aout.c |    2 +-
 fs/binfmt_aout.c             |    2 +-
 fs/binfmt_elf.c              |    4 ++--
 fs/binfmt_elf_fdpic.c        |    2 +-
 fs/binfmt_flat.c             |    2 +-
 fs/binfmt_som.c              |    2 +-
 fs/exec.c                    |    2 +-
 include/linux/binfmts.h      |    2 +-
 8 files changed, 9 insertions(+), 9 deletions(-)



diff --git a/arch/x86_64/ia32/ia32_aout.c b/arch/x86_64/ia32/ia32_aout.c
index 5967cc5..61acc5c 100644
--- a/arch/x86_64/ia32/ia32_aout.c
+++ b/arch/x86_64/ia32/ia32_aout.c
@@ -148,7 +148,7 @@ if (file->f_op->llseek) { \
  * dumping of the process results in another error..
  */
 
-static int aout_core_dump(long signr, struct pt_regs * regs, struct file *file, u32 limit)
+static int aout_core_dump(long signr, struct pt_regs * regs, struct file *file, unsigned long limit)
 {
 	mm_segment_t fs;
 	int has_dumped = 0;
diff --git a/fs/binfmt_aout.c b/fs/binfmt_aout.c
index e247d80..e914b82 100644
--- a/fs/binfmt_aout.c
+++ b/fs/binfmt_aout.c
@@ -88,7 +88,7 @@ if (file->f_op->llseek) { \
  * dumping of the process results in another error..
  */
 
-static int aout_core_dump(long signr, struct pt_regs * regs, struct file *file, u32 limit)
+static int aout_core_dump(long signr, struct pt_regs * regs, struct file *file, unsigned long limit)
 {
 	mm_segment_t fs;
 	int has_dumped = 0;
diff --git a/fs/binfmt_elf.c b/fs/binfmt_elf.c
index cd81def..cbda09b 100644
--- a/fs/binfmt_elf.c
+++ b/fs/binfmt_elf.c
@@ -52,7 +52,7 @@ static unsigned long elf_map (struct file *, unsigned long, struct elf_phdr *, i
  * don't even try.
  */
 #if defined(USE_ELF_CORE_DUMP) && defined(CONFIG_ELF_CORE)
-static int elf_core_dump(long signr, struct pt_regs *regs, struct file *file, u32 limit);
+static int elf_core_dump(long signr, struct pt_regs *regs, struct file *file, unsigned long limit);
 #else
 #define elf_core_dump	NULL
 #endif
@@ -1488,7 +1488,7 @@ static struct vm_area_struct *next_vma(struct vm_area_struct *this_vma,
  * and then they are actually written out.  If we run out of core limit
  * we just truncate.
  */
-static int elf_core_dump(long signr, struct pt_regs *regs, struct file *file, u32 limit)
+static int elf_core_dump(long signr, struct pt_regs *regs, struct file *file, unsigned long limit)
 {
 #define	NUM_NOTES	6
 	int has_dumped = 0;
diff --git a/fs/binfmt_elf_fdpic.c b/fs/binfmt_elf_fdpic.c
index fb4ad53..f2871e5 100644
--- a/fs/binfmt_elf_fdpic.c
+++ b/fs/binfmt_elf_fdpic.c
@@ -1552,7 +1552,7 @@ static int elf_fdpic_dump_segments(struct file *file, size_t *size,
  * we just truncate.
  */
 static int elf_fdpic_core_dump(long signr, struct pt_regs *regs,
-			       struct file *file, u32 limit)
+			       struct file *file, unsigned long limit)
 {
 #define	NUM_NOTES	6
 	int has_dumped = 0;
diff --git a/fs/binfmt_flat.c b/fs/binfmt_flat.c
index 1a17556..e8f8198 100644
--- a/fs/binfmt_flat.c
+++ b/fs/binfmt_flat.c
@@ -90,7 +90,7 @@ static struct linux_binfmt flat_format = {
  * Currently only a stub-function.
  */
 
-static int flat_core_dump(long signr, struct pt_regs * regs, struct file *file, u32 limit)
+static int flat_core_dump(long signr, struct pt_regs * regs, struct file *file, unsigned long limit)
 {
 	printk("Process %s:%d received signr %d and should have core dumped\n",
 			current->comm, current->pid, (int) signr);
diff --git a/fs/binfmt_som.c b/fs/binfmt_som.c
index 0da1fae..7474970 100644
--- a/fs/binfmt_som.c
+++ b/fs/binfmt_som.c
@@ -44,7 +44,7 @@ static int load_som_library(struct file *);
  * don't even try.
  */
 #if 0
-static int som_core_dump(long signr, struct pt_regs * regs, u32 limit);
+static int som_core_dump(long signr, struct pt_regs * regs, unsigned long limit);
 #else
 #define som_core_dump	NULL
 #endif
diff --git a/fs/exec.c b/fs/exec.c
index 3b2e1e7..c0b5def 100644
--- a/fs/exec.c
+++ b/fs/exec.c
@@ -1726,7 +1726,7 @@ int do_coredump(long signr, int exit_code, struct pt_regs * regs)
 	int fsuid = current->fsuid;
 	int flag = 0;
 	int ispipe = 0;
-	u32 core_limit = current->signal->rlim[RLIMIT_CORE].rlim_cur;
+	unsigned long core_limit = current->signal->rlim[RLIMIT_CORE].rlim_cur;
 
 	audit_core_dumps(signr);
 
diff --git a/include/linux/binfmts.h b/include/linux/binfmts.h
index 3188627..fb62eda 100644
--- a/include/linux/binfmts.h
+++ b/include/linux/binfmts.h
@@ -67,7 +67,7 @@ struct linux_binfmt {
 	struct module *module;
 	int (*load_binary)(struct linux_binprm *, struct  pt_regs * regs);
 	int (*load_shlib)(struct file *);
-	int (*core_dump)(long signr, struct pt_regs * regs, struct file * file, u32 limit);
+	int (*core_dump)(long signr, struct pt_regs * regs, struct file * file, unsigned long limit);
 	unsigned long min_coredump;	/* minimal dump size */
 	int hasvdso;
 };
/***************************************************
 *Neil Horman
 *Software Engineer
 *Red Hat, Inc.
 *nhorman@tuxdriver.com
 *gpg keyid: 1024D / 0x92A74FA1
 *http://pgp.mit.edu
 ***************************************************/

Re: [PATCH]: allow individual core dump methods to be unlimited when sending to a pipe

[Andrew Morton]

SuperH allmodconfig broke:

fs/binfmt_flat.c:83: warning: initialization from incompatible pointer type
fs/binfmt_flat.c:94: error: conflicting types for 'flat_core_dump'
fs/binfmt_flat.c:78: error: previous declaration of 'flat_core_dump' was here
fs/binfmt_flat.c:94: error: conflicting types for 'flat_core_dump'
fs/binfmt_flat.c:78: error: previous declaration of 'flat_core_dump' was here
fs/binfmt_flat.c: In function `decompress_exec':
fs/binfmt_flat.c:293: warning: label `out' defined but not used
fs/binfmt_flat.c: In function `load_flat_file':
fs/binfmt_flat.c:462: warning: unsigned int format, long int arg (arg 3)
fs/binfmt_flat.c:462: warning: unsigned int format, long int arg (arg 4)
fs/binfmt_flat.c:518: warning: comparison of distinct pointer types lacks a cast
fs/binfmt_flat.c:549: warning: passing arg 1 of `ksize' makes pointer from integer without a cast
fs/binfmt_flat.c:601: warning: passing arg 1 of `ksize' makes pointer from integer without a cast
fs/binfmt_flat.c: At top level:
fs/binfmt_flat.c:78: warning: 'flat_core_dump' used but never defined
fs/binfmt_flat.c:94: warning: 'flat_core_dump' defined but not used

Re: [PATCH]: allow individual core dump methods to be unlimited when sending to a pipe

[Neil Horman]

On Tue, Jul 24, 2007 at 10:03:58PM -0700, Andrew Morton wrote:
> 
> SuperH allmodconfig broke:
> 
> fs/binfmt_flat.c:83: warning: initialization from incompatible pointer type
> fs/binfmt_flat.c:94: error: conflicting types for 'flat_core_dump'
> fs/binfmt_flat.c:78: error: previous declaration of 'flat_core_dump' was here
> fs/binfmt_flat.c:94: error: conflicting types for 'flat_core_dump'
> fs/binfmt_flat.c:78: error: previous declaration of 'flat_core_dump' was here
> fs/binfmt_flat.c: In function `decompress_exec':
> fs/binfmt_flat.c:293: warning: label `out' defined but not used
> fs/binfmt_flat.c: In function `load_flat_file':
> fs/binfmt_flat.c:462: warning: unsigned int format, long int arg (arg 3)
> fs/binfmt_flat.c:462: warning: unsigned int format, long int arg (arg 4)
> fs/binfmt_flat.c:518: warning: comparison of distinct pointer types lacks a cast
> fs/binfmt_flat.c:549: warning: passing arg 1 of `ksize' makes pointer from integer without a cast
> fs/binfmt_flat.c:601: warning: passing arg 1 of `ksize' makes pointer from integer without a cast
> fs/binfmt_flat.c: At top level:
> fs/binfmt_flat.c:78: warning: 'flat_core_dump' used but never defined
> fs/binfmt_flat.c:94: warning: 'flat_core_dump' defined but not used

Damn!  Sorry about that.  At least I have a mediocre excuse in that it was on an
arch I didn't have hardware to test with.  Should have caught that sooner
though.  Heres the patch for it.

Regards
Neil

Signed-off-by: Neil Horman <nhorman@tuxdriver.com>


 binfmt_flat.c |    2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/fs/binfmt_flat.c b/fs/binfmt_flat.c
index e8f8198..cc24c5d 100644
--- a/fs/binfmt_flat.c
+++ b/fs/binfmt_flat.c
@@ -75,7 +75,7 @@ static int load_flat_shared_library(int id, struct lib_info *p);
 #endif
 
 static int load_flat_binary(struct linux_binprm *, struct pt_regs * regs);
-static int flat_core_dump(long signr, struct pt_regs * regs, struct file *file, u32 limit);
+static int flat_core_dump(long signr, struct pt_regs * regs, struct file *file, unsigned long limit);
 
 static struct linux_binfmt flat_format = {
 	.module		= THIS_MODULE,
-- 
/***************************************************
 *Neil Horman
 *Software Engineer
 *Red Hat, Inc.
 *nhorman@tuxdriver.com
 *gpg keyid: 1024D / 0x92A74FA1
 *http://pgp.mit.edu
 ***************************************************/