RCU stalls when running out of memory on 3.14-rc4 w/ NFS and kernel threads priorities changed

* RCU stalls when running out of memory on 3.14-rc4 w/ NFS and kernel threads priorities changed
@ 2014-03-04 23:55 Florian Fainelli
  2014-03-05  0:48   ` Eric Dumazet
  0 siblings, 1 reply; 22+ messages in thread
From: Florian Fainelli @ 2014-03-04 23:55 UTC (permalink / raw)
  To: linux-kernel, linux-mm, paulmck, linux-nfs, trond.myklebust

[-- Attachment #1: Type: text/plain, Size: 1251 bytes --]

Hi all,

I am seeing the following RCU stalls messages appearing on an ARMv7
4xCPUs system running 3.14-rc4:

[   42.974327] INFO: rcu_sched detected stalls on CPUs/tasks:
[   42.979839]  (detected by 0, t=2102 jiffies, g=4294967082,
c=4294967081, q=516)
[   42.987169] INFO: Stall ended before state dump start

this is happening under the following conditions:

- the attached bumper.c binary alters various kernel thread priorities
based on the contents of bumpup.cfg and
- malloc_crazy is running from a NFS share
- malloc_crazy.c is running in a loop allocating chunks of memory but
never freeing it

when the priorities are altered, instead of getting the OOM killer to
be invoked, the RCU stalls are happening. Taking NFS out of the
equation does not allow me to reproduce the problem even with the
priorities altered.

This "problem" seems to have been there for quite a while now since I
was able to get 3.8.13 to trigger that bug as well, with a slightly
more detailed RCU debugging trace which points the finger at kswapd0.

You should be able to get that reproduced under QEMU with the
Versatile Express platform emulating a Cortex A15 CPU and the attached
files.

Any help or suggestions would be greatly appreciated. Thanks!
-- 
Florian

[-- Attachment #2: malloc_crazy.c --]
[-- Type: text/x-csrc, Size: 405 bytes --]

#include <stdlib.h>
#include <stdio.h>
#include <string.h>

int main(int argc, char **argv)
{
	int i;
	int *buf;
	int max = 3048;

	if (argc == 2)
		max = atoi(argv[1]);

	for(i = 0; i < max; i++) {
		buf = (void *)malloc(1048576);
		if(! buf) {
			printf("malloc returned NULL\n");
			return(0);
		}
		memset(buf, 0x11, 1048576);
		*buf = i;
		printf("%d\n", i);
	}
	printf("finished\n");

	return(1);
}

[-- Attachment #3: rcu_test.sh --]
[-- Type: application/x-sh, Size: 280 bytes --]

[-- Attachment #4: bumpup.cfg --]
[-- Type: application/octet-stream, Size: 255 bytes --]

ksoftirqd/0,RR,97
events/0,RR,97
khelper,RR,97
kthread,RR,97
kblockd/0,RR,97
pdflush,RR,97
kswapd0,RR,97

aio/0,RR,97
ata/0,RR,97
ata_aux,RR,97
scsi_eh_0,RR,97
scsi_eh_1,RR,97
mtdblockd,RR,97

inetd,RR,97
telnetd,RR,34
portmap,RR,97
sh,RR,97

khubd,RR,96

[-- Attachment #5: bumper.c --]
[-- Type: text/x-csrc, Size: 6782 bytes --]

/***************************************************************************
 *   Copyright (C) 2010 by BSkyB
 *   Richard Parsons
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 *   This program is distributed in the hope that it will be useful,       *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
 *   GNU General Public License for more details.                          *
 *                                                                         *
 *   You should have received a copy of the GNU General Public License     *
 *   along with this program; if not, write to the                         *
 *   Free Software Foundation, Inc.,                                       *
 *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.             *
 ***************************************************************************/

/* N.B. */
/* this program is supplied WITHOUT any warranty whatsoever		  */

/*==========================================================================

This program is used for the mass changing of process priorities by name
The syntax is as follows
	./bumber [cfgfile]

The config file uses the following syntax
#this is a comment
process_name,POLICY,priority

Default is RR , but valid policies are
  RR, FIFO and OTHER

priority is a value from 0..99

e.g.

#Telnet priority
telnetd,RR,37

=============================================================================*/

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include <stdio.h>
#include <stdlib.h>
#include <dirent.h>
#include <string.h>
#include <sched.h>
#include "bumper.h"

int main ( int argc, char *argv[] )
{

	if ( argc != 2 )
	{
		printf ( "Error: Usage is %s [config file]\n",argv[0] );
		return EXIT_FAILURE;
	} 
	processList = init_application();
	if ( readInputFile ( argv[1] ) < 0 )
	{
		printf ( "Error: Could not open %s\n",argv[1] );
		return EXIT_FAILURE;
	}
	getProcessID ( processList );
	destory_application(processList);
	return EXIT_SUCCESS;
}

void *init_application ( void )
{
  return NULL;
}

void destory_application(struct params *head)
{
    struct params *next;

    if (head == NULL)
      return;

   next = head; 

   while (next != NULL)
     {
        void *nptr = next->next; 
	free(next->process);
        free(next);
        next = nptr;
     } 
}

int readInputFile ( char *inFile )
{
	FILE *fp;
	int rc = -1;
	char inLine[101];

	if ( ( fp = fopen ( inFile, "r" ) ) == NULL )
		return rc;

	while ( fgets ( inLine, 100, fp ) !=NULL )
	{
		if ( inLine[0] == '#' )
			continue;

		processList = addEntry ( inLine,processList );
	}
	fclose ( fp );
	return 1;
}

void *addEntry ( char *szParameters,struct params *head )
{
	struct params *entry;
	struct params *lastPtr=head;
	void *retPtr=head;
	char process[512];
	char policy[512];
	char prio[512];
	if ( strlen ( szParameters ) > 511 )  // small saniy check..
		return head;
	if ( strlen ( szParameters ) < 5 )  // small saniy check..
		return head;

	int rc = sscanf ( szParameters,"%[^','],%[^','],%[^',']",process,policy,prio );
	if ( rc == 3 )
	{
		int pVal = atoi(prio);
		if ( ( pVal < 0 ) || ( pVal > 100 ) )
		{
			printf ( "Error : Process %s Priority %i OUT OF BOUNDS of 0..99\n",process,pVal );
			return head;
		}
		entry = malloc ( sizeof ( struct params ) );
		entry->next=head;
		entry->process = malloc ( strlen ( process ) +1 );
		strcpy ( entry->process,process );
		entry->prio = pVal;
		entry->policy = SCHED_RR;
		entry->processCount=0;

		if ( !strcasecmp ( policy,"RR" ) )
		{
			entry->policy = SCHED_RR;
		}
		else if ( !strcasecmp ( policy,"OTHER" ) )
		{
			entry->policy = SCHED_OTHER;
		}
		else if ( !strcasecmp ( policy,"FIFO" ) )
		{
			entry->policy = SCHED_FIFO;
		}
		printf ( "Process %s [Priority %i] [Policy %s (%i)]\n",entry->process,entry->prio,policy,entry->policy );
	}
	else
	{
		return head;
	}
	return entry;
}

int checkProcessName ( struct params *head, char *process, int *prio, int *policy )
{
	int rc = -1;
	struct params *itr = head;

	for ( itr=head;itr != NULL;itr=itr->next )
	{
		if ( strstr ( process,itr->process ) != NULL )
		{
			*prio = itr->prio;
			*policy = itr->policy;
			itr->processCount++;
			rc = 1;
			break;
		}
	}
	return rc;
}

int findChar ( char *inString,char character )
{
	int a;

	for ( a=0;a<strlen ( inString );a++ )
	{
		if ( inString[a] == character )
			return a;
	}
	return -1;
}

int readStat ( char *exe, char *target_name )
{
	FILE *fp;
	int rc = -1;
	char inLine[512];

	if ( ( fp = fopen ( exe, "r" ) ) == NULL )
	{
		printf ( "Failed to open %s\n",exe );
		return rc;
	}

	char *retChar = fgets ( inLine, 100, fp );
	if ( retChar == NULL )
	{
		close ( fp );
		return rc;
	}

	int start = findChar ( inLine,'(' );
	int end = findChar ( inLine,')' );

	if ( ( start > 0 ) && ( end >0 ) )
	{
		memset ( target_name,0, ( end-start ) +1 );
		memcpy ( target_name,inLine+start+1, ( end-start )-1 );
		rc = 1;
	}

	fclose ( fp );
	return rc;
}

unsigned int getProcessID ( struct params *head )
{
	DIR *dir_p;
	struct dirent *dir_entry_p;
	char dir_name[512]; // Horrible.. but will do
	char target_name[512];  // Horrible.. but will do
	int target_result;
	char exe_link[512];  // yuk
	int errorcount;
	int result;
	int prio;
	int policy;

	errorcount=0;
	result=0;
	dir_p = opendir ( "/proc/" );
	while ( NULL != ( dir_entry_p = readdir ( dir_p ) ) )
	{
		if ( strspn ( dir_entry_p->d_name, "0123456789" ) == strlen ( dir_entry_p->d_name ) )
		{
			strcpy ( dir_name, "/proc/" );
			strcat ( dir_name, dir_entry_p->d_name );
			strcat ( dir_name, "/" );
			exe_link[0] = 0;
			strcat ( exe_link, dir_name );
			strcat ( exe_link, "stat" );
			target_result = readStat ( exe_link, target_name );
			if ( target_result > 0 )
			{
				//printf ( "Checking for %s\n",target_name );

				if ( checkProcessName ( head,target_name,&prio,&policy ) > 0 )
				{
					result = atoi ( dir_entry_p->d_name );
					struct sched_param sp;
					sp.sched_priority = prio;
					if ( sched_setscheduler ( result, policy, &sp ) < 0 )
					{
						printf ( "Failed to set %s (%i) [Policy %i] [Priority %i]\n", target_name, result,policy,prio );
					}
					else
						printf ( "Priority set %s (%i)\n", target_name, result );
				}
			}
		}
	}
	closedir ( dir_p );
	return result;
}

[-- Attachment #6: bumper.h --]
[-- Type: text/x-chdr, Size: 534 bytes --]

#ifndef _BUMPER_H
#define _BUMPER_H

struct params {
  char *process;
  int  prio;
  int  policy;
  int  processCount;
struct params *next; // single entry linked list
};

unsigned int getProcessID ( struct params *head );
void *addEntry ( char *szParameters,struct params *head );
int checkProcessName ( struct params *head, char *process, int *prio, int *policy );
int readInputFile ( char *inFile );
void *init_application ( void );
void destory_application(struct params *head);

// Globals 
struct params *processList;

#endif

^ permalink raw reply	[flat|nested] 22+ messages in thread