A filename to label translation daemon

A filename to label translation daemon

[Eric Paris]

[-- Attachment #1: Type: text/plain, Size: 2222 bytes --]

We know that utilities like install disable their SELinux support
because of the enormous amount of time it takes to load the matchpathcon
regex database.  We know that systemd spends time loading the database
at least twice.  Other utilities like the krb5libs complain about the
size and time it takes to load the database.  We've added hacks (I
believe all in Fedora, but maybe upstream as well) which try to pare
down the database to some prefix(es) on database load.  If systemd only
needs to label in /var why load all the stuff for /etc?  These prefix
hacks don't work particularly well as fallback labels (such as
default_t) are hard to capture and the prefixes cannot be long as the
regexes are usually quite short.  They also don't work well with label
equivalencies.

So today I wrote a little daemon which listens in the abstract namespace
for requests and returns the context.  It really really rough, I admit,
but it works quite well.  My first perf numbers looking at /home/eparis
make sense:

$ ./initonce /home/eparis
 0.180 seconds used by the processor.
$ ./initalways /home/eparis
 19.200 seconds used by the processor.
$ ./client /home/eparis
 0.570 seconds used by the processor.

If I init the DB one time and do the same lookup (for /home/eparis) 1000
times it takes .18 seconds.  Doing 1000 lookups init-ing and fini-ing
the db every time it took 19.2.  Connecting to the server and asking
1000 times took .57 seconds.  This means that if you have to do about 48
lookups, it's faster to do your own init.  If <48, you should use the
server.

The I tried again with a different pathname (and get very different
results)

$ ./initonce /var/www/html/cgi-bin
 1.510 seconds used by the processor.
$ ./initalways /var/www/html/cgi-bin
 42.790 seconds used by the processor.
$ ./client /var/www/html/cgi-bin
 0.600 seconds used by the processor.

These I cannot explain.  How the heck is local slower when the time to
init the db is not taken into account at all?  I'm clueless here.  But
still, the client server model doesn't look like a bad idea.

I'm attaching my server, my client, and my 2 local test programs.
Thoughts?

*having the daemon listen and update the db on policy load is a todo

-Eric

[-- Attachment #2: client.c --]
[-- Type: text/x-csrc, Size: 2458 bytes --]

#include <errno.h>
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include <sys/socket.h>
#include <sys/types.h>
#include <sys/un.h>

#define SOCKET_PATH "selinux.filename.label"
#define MAX_REQUEST_LEN 8192

#define offsetof(st, m)	((size_t)((char *)&((st *)0)->m - (char *)0))

#include <time.h>
clock_t startm, stopm;
#define START if ( (startm = clock()) == -1) {printf("Error calling clock");exit(1);}
#define STOP if ( (stopm = clock()) == -1) {printf("Error calling clock");exit(1);}
#define PRINTTIME printf( "%6.3f seconds used by the processor.\n", ((double)stopm-startm)/CLOCKS_PER_SEC);

static int make_request(char *pathname, mode_t mode)
{
	int rc, sockfd;
	struct sockaddr_un sockaddr_un;
	ssize_t buflen, len;
	char buffer[MAX_REQUEST_LEN];

	/* put the mode at the front of the buffer */
	buflen = snprintf(buffer, sizeof(buffer), "%x %s", mode, pathname);
	if (buflen < 0)
		return -1;
	if (buflen == sizeof(buffer)) {
		errno = -ENOSPC;
		return -1;
	}

	/* create socket to talk to server */
	sockfd = socket(AF_UNIX, SOCK_SEQPACKET, 0);
	if (sockfd < 0)
		return -1;

	/* set up address of the server */
	sockaddr_un.sun_family = AF_UNIX;
	sockaddr_un.sun_path[0] = '\0';
	memcpy(&sockaddr_un.sun_path[1], SOCKET_PATH, strlen(SOCKET_PATH));

	/* this size of addr is really screwy to calculate, there be dragons here */
	len = offsetof(struct sockaddr_un, sun_path) + strlen(SOCKET_PATH) + 1;

	/* connect to server */
	rc = connect(sockfd, (const struct sockaddr *)&sockaddr_un, len);
	if (rc < 0)
		return -1;

	/* send the request */
	len = send(sockfd, buffer, buflen + 1, 0);
	if (len <= 0)
		return -1;

	/* get the context */
	len = recv(sockfd, buffer, MAX_REQUEST_LEN, 0);
	if (len <= 0)
		return -1;

#ifdef VERBOSE
	printf("pathname=%s mode=%x context=%s\n", pathname, mode, buffer);
#endif

	return 0;
}

int main(int argc, char *argv[])
{
	mode_t mode = 0;
	int rc, i;

	if (argc < 2) {
		printf("usage: %s filename [mode]\n", argv[0]);
		return 0;
	}

	if (argc >= 3) {
		unsigned long model;

		errno = 0;    /* To distinguish success/failure after call */
		model = strtoul(argv[2], NULL, 16);
		if ((errno == ERANGE && model == ULONG_MAX) ||
		    (errno != 0 && model == 0)) {
			perror("strtol");
			exit(EXIT_FAILURE);
		}
		mode = (mode_t)model;
	}


	START
	for (i = 0; i < 1000; i++) {
		rc = make_request(argv[1], mode);
		if (rc)
			return -1;
	}
	STOP
	PRINTTIME

	return 0;
}

[-- Attachment #3: initalways.c --]
[-- Type: text/x-csrc, Size: 1438 bytes --]

#include <errno.h>
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

#include <selinux/selinux.h>

#include <sys/socket.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/un.h>

#define SOCKET_PATH "selinux.filename.label"
#define MAX_LABEL_LEN 4096

#include <time.h>
clock_t startm, stopm;
#define START if ( (startm = clock()) == -1) {printf("Error calling clock");exit(1);}
#define STOP if ( (stopm = clock()) == -1) {printf("Error calling clock");exit(1);}
#define PRINTTIME printf( "%6.3f seconds used by the processor.\n", ((double)stopm-startm)/CLOCKS_PER_SEC);

int main(int argc, char *argv[])
{
	int rc, i;
	mode_t mode = 0;
	security_context_t con;

	if (argc < 2) {
		printf("usage: %s filename [mode]\n", argv[0]);
		return 0;
	}

	if (argc >= 3) {
		unsigned long model;

		errno = 0;    /* To distinguish success/failure after call */
		model = strtoul(argv[2], NULL, 16);
		if ((errno == ERANGE && model == ULONG_MAX) ||
		    (errno != 0 && model == 0)) {
			perror("strtol");
			exit(EXIT_FAILURE);
		}
		mode = (mode_t)model;
	}

	START
	for (i = 0; i < 1000; i++) {
		rc = matchpathcon_init(NULL);
		if (rc < 0)
			return -1;

		rc = matchpathcon(argv[1], mode, &con);
		if (rc < 0)
			return -2;
#ifdef VERBOSE
		printf("pathname=%s mode=%x context=%s\n", argv[1], mode, con);
#endif
		free(con);
		matchpathcon_fini();
	}
	STOP
	PRINTTIME

	return rc;
}

[-- Attachment #4: initonce.c --]
[-- Type: text/x-csrc, Size: 1434 bytes --]

#include <errno.h>
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

#include <selinux/selinux.h>

#include <sys/socket.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/un.h>

#define SOCKET_PATH "selinux.filename.label"
#define MAX_LABEL_LEN 4096

#include <time.h>
clock_t startm, stopm;
#define START if ( (startm = clock()) == -1) {printf("Error calling clock");exit(1);}
#define STOP if ( (stopm = clock()) == -1) {printf("Error calling clock");exit(1);}
#define PRINTTIME printf( "%6.3f seconds used by the processor.\n", ((double)stopm-startm)/CLOCKS_PER_SEC);

int main(int argc, char *argv[])
{
	int rc, i;
	mode_t mode = 0;
	security_context_t con;

	if (argc < 2) {
		printf("usage: %s filename [mode]\n", argv[0]);
		return 0;
	}

	if (argc >= 3) {
		unsigned long model;

		errno = 0;    /* To distinguish success/failure after call */
		model = strtoul(argv[2], NULL, 16);
		if ((errno == ERANGE && model == ULONG_MAX) ||
		    (errno != 0 && model == 0)) {
			perror("strtol");
			exit(EXIT_FAILURE);
		}
		mode = (mode_t)model;
	}

	rc = matchpathcon_init(NULL);
	if (rc < 0)
		return -1;

	START
	for (i = 0; i < 1000; i++) {
		rc = matchpathcon(argv[1], mode, &con);
		if (rc < 0)
			return -2;
#ifdef VERBOSE
		printf("pathname=%s mode=%x context=%s\n", argv[1], mode, con);
#endif
		free(con);
	}
	STOP
	PRINTTIME

	matchpathcon_fini();
	return rc;
}

[-- Attachment #5: server.c --]
[-- Type: text/x-csrc, Size: 4622 bytes --]

#include <ctype.h>
#include <errno.h>
#include <getopt.h>
#include <limits.h>
#include <poll.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

#include <selinux/selinux.h>

#include <sys/socket.h>
#include <sys/types.h>
#include <sys/un.h>

#include <systemd/sd-daemon.h>

#define SOCKET_PATH "selinux.filename.label"
#define MAX_REQUEST_LEN 8192
#define MAX_BACKLOG 100
#define RECV_TIMEOUT 250 /* milliseconds */

#define offsetof(st, m)	((size_t)((char *)&((st *)0)->m - (char *)0))

/*
 * Requests are send to the server as single packet of the form:
 * 	snprintf("%x %s", mode, pathname);
 * Responses are sent to the client as string representing the raw context
 */
static void handle_request(int fd)
{
	int rc;
	ssize_t len;
	char buffer[MAX_REQUEST_LEN + 1];
	char *pathname;
	unsigned long model;
	mode_t mode;
	security_context_t con = NULL;
	struct pollfd pollfd;

	pollfd.fd = fd;
	pollfd.events = POLLIN;

	/* don't hang forever if the client sux */
	rc = poll(&pollfd, 1, RECV_TIMEOUT);
	if (rc <= 0)
		goto out;

	/* get the request */
	len = recv(fd, buffer, sizeof(buffer), 0);
	if (len <= 0)
		goto out;

	/* just to be safe! */
	buffer[MAX_REQUEST_LEN] = '\0';

	/*
	 * Yes, I could use sscanf("%x %s") but it would needlessly duplicate
	 * pathname.  Whereas strtoul leaves me a pointer to it in buffer
	 */

	/* parse the mode portion to a mode_t */
	errno = 0;    /* To distinguish success/failure after call */
	model = strtoul(buffer, &pathname, 16);
	if ((errno == ERANGE && model == ULONG_MAX) ||
	    (errno != 0 && model == 0))
		goto out;
	mode = (mode_t)model;

	/* skip whitespace between mode and pathname */
	while (pathname < &buffer[MAX_REQUEST_LEN - 1] && isspace(pathname[0]))
		pathname++;

	/* pathname now points at the pathname part of the buffer! */
	if (pathname >= &buffer[MAX_REQUEST_LEN - 1] || !pathname[0])
		goto out;

	/* do the lookup */
	rc = matchpathcon(pathname, mode, &con);
	if (rc < 0) {
		if (errno == ENOENT) {
			con = strdup("<<none>>");
			if (!con)
				goto out;
		} else {
			goto out;
		}
	}

	/* send the response */
	len = send(fd, con, strlen(con) + 1, 0);
	if (len <= 0)
		goto out;

#ifdef VERBOSE
	printf("pathname=%s mode=%x context=%s\n", pathname, mode, con);
#endif
out:
	free(con);
	close(fd);
}

static void usage(char *progname)
{
	fprintf(stderr, "usage: %s [-f,--foreground] [-s,--socket fd]\n", progname);
	exit(EXIT_FAILURE);
}

int main(int argc, char *argv[])
{
	int rc, clientfd, c;
	int serverfd = -1;
	size_t len;
	struct sockaddr_un sockaddr_un;
	int option_index = 0;
	int foreground = 0;
	static struct option long_options[] = {
		{"foreground", no_argument, 0, 'f'},
		{"socket", required_argument, 0, 's'},
		{0, 0, 0, 0}
	};

	while (1) {
		c = getopt_long(argc, argv, "fs:", long_options, &option_index);
		if (c == -1)
			break;

		switch (c) {
		case 'f':
			foreground = 1;
			break;
		case 's':
			serverfd = strtol(optarg, NULL, 0);
			if (serverfd < 0) {
				fprintf(stderr, "invalid socket fd\n");
				usage(argv[0]);
			}
			break;
		case '?':
		default:
			usage(argv[0]);
			break;
		};
	}

	c = sd_listen_fds(0);
	if (c > 1) {
		fprintf(stderr, "Too many file descriptors received.\n");
		exit(EXIT_FAILURE);
	} else if (c == 1)
		serverfd = SD_LISTEN_FDS_START + 0;

	if (!foreground)
		daemon(0, 0);

	/* init the label regex db */
	rc = matchpathcon_init(NULL);
	if (rc < 0) {
		perror("matchpathcon_init");
		return errno;
	}

	/* if we didn't get the serverfd from systemd or command line, do it outselves */
	if (serverfd == -1) {
		/* create the socket on which we will listen */
		serverfd = socket(AF_UNIX, SOCK_SEQPACKET, 0);
		if (serverfd < 0) {
			perror("socket");
			return errno;
		}

		/* set up out address */
		sockaddr_un.sun_family = AF_UNIX;
		sockaddr_un.sun_path[0] = '\0';
		memcpy(&sockaddr_un.sun_path[1], SOCKET_PATH, strlen(SOCKET_PATH));

		/* calculating the length of the address is magic since it starts with nul.
	 	* there be dragons in here! */
		len = offsetof(struct sockaddr_un, sun_path) + strlen(SOCKET_PATH) + 1;

		/* bind the socket to the address */
		rc = bind(serverfd, (const struct sockaddr *)&sockaddr_un, len);
		if (rc < 0) {
			perror("bind");
			return errno;
		}

		/* start listening for connections */
		rc = listen(serverfd, MAX_BACKLOG);
		if (rc < 0) {
			perror("listen");
			return errno;
		}
	} /* if (serverfd == -1) */

	/* work yo! */
	while (1) {
		clientfd = accept(serverfd, NULL, 0);
		if (clientfd < 0) {
			perror("accept");
			return errno;
		}

		handle_request(clientfd);
	}
	close(serverfd);
	return rc;
}

Re: A filename to label translation daemon

[Stephen Smalley]

On Wed, 2012-08-08 at 15:31 -0400, Eric Paris wrote:
> We know that utilities like install disable their SELinux support
> because of the enormous amount of time it takes to load the matchpathcon
> regex database.  We know that systemd spends time loading the database
> at least twice.  Other utilities like the krb5libs complain about the
> size and time it takes to load the database.  We've added hacks (I
> believe all in Fedora, but maybe upstream as well) which try to pare
> down the database to some prefix(es) on database load.  If systemd only
> needs to label in /var why load all the stuff for /etc?  These prefix
> hacks don't work particularly well as fallback labels (such as
> default_t) are hard to capture and the prefixes cannot be long as the
> regexes are usually quite short.  They also don't work well with label
> equivalencies.
> 
> So today I wrote a little daemon which listens in the abstract namespace
> for requests and returns the context.  It really really rough, I admit,
> but it works quite well.  My first perf numbers looking at /home/eparis
> make sense:
> 
> $ ./initonce /home/eparis
>  0.180 seconds used by the processor.
> $ ./initalways /home/eparis
>  19.200 seconds used by the processor.
> $ ./client /home/eparis
>  0.570 seconds used by the processor.
> 
> If I init the DB one time and do the same lookup (for /home/eparis) 1000
> times it takes .18 seconds.  Doing 1000 lookups init-ing and fini-ing
> the db every time it took 19.2.  Connecting to the server and asking
> 1000 times took .57 seconds.  This means that if you have to do about 48
> lookups, it's faster to do your own init.  If <48, you should use the
> server.
> 
> The I tried again with a different pathname (and get very different
> results)
> 
> $ ./initonce /var/www/html/cgi-bin
>  1.510 seconds used by the processor.
> $ ./initalways /var/www/html/cgi-bin
>  42.790 seconds used by the processor.
> $ ./client /var/www/html/cgi-bin
>  0.600 seconds used by the processor.
> 
> These I cannot explain.  How the heck is local slower when the time to
> init the db is not taken into account at all?  I'm clueless here.  But
> still, the client server model doesn't look like a bad idea.
> 
> I'm attaching my server, my client, and my 2 local test programs.
> Thoughts?
> 
> *having the daemon listen and update the db on policy load is a todo

Not sure how this helps systemd, as it runs first (by definition) and
loads the file_contexts configuration before it starts any other
daemons, right?  Now if you wanted systemd to export this as a service
to everything else, that might make sense.

-- 
Stephen Smalley
National Security Agency


--
This message was distributed to subscribers of the selinux mailing list.
If you no longer wish to subscribe, send mail to majordomo@tycho.nsa.gov with
the words "unsubscribe selinux" without quotes as the message.

Re: A filename to label translation daemon

[Daniel J Walsh]

-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1

On 08/08/2012 04:05 PM, Stephen Smalley wrote:
> On Wed, 2012-08-08 at 15:31 -0400, Eric Paris wrote:
>> We know that utilities like install disable their SELinux support because
>> of the enormous amount of time it takes to load the matchpathcon regex
>> database.  We know that systemd spends time loading the database at least
>> twice.  Other utilities like the krb5libs complain about the size and
>> time it takes to load the database.  We've added hacks (I believe all in
>> Fedora, but maybe upstream as well) which try to pare down the database
>> to some prefix(es) on database load.  If systemd only needs to label in
>> /var why load all the stuff for /etc?  These prefix hacks don't work
>> particularly well as fallback labels (such as default_t) are hard to
>> capture and the prefixes cannot be long as the regexes are usually quite
>> short.  They also don't work well with label equivalencies.
>> 
>> So today I wrote a little daemon which listens in the abstract namespace 
>> for requests and returns the context.  It really really rough, I admit, 
>> but it works quite well.  My first perf numbers looking at /home/eparis 
>> make sense:
>> 
>> $ ./initonce /home/eparis 0.180 seconds used by the processor. $
>> ./initalways /home/eparis 19.200 seconds used by the processor. $
>> ./client /home/eparis 0.570 seconds used by the processor.
>> 
>> If I init the DB one time and do the same lookup (for /home/eparis) 1000 
>> times it takes .18 seconds.  Doing 1000 lookups init-ing and fini-ing the
>> db every time it took 19.2.  Connecting to the server and asking 1000
>> times took .57 seconds.  This means that if you have to do about 48 
>> lookups, it's faster to do your own init.  If <48, you should use the 
>> server.
>> 
>> The I tried again with a different pathname (and get very different 
>> results)
>> 
>> $ ./initonce /var/www/html/cgi-bin 1.510 seconds used by the processor. $
>> ./initalways /var/www/html/cgi-bin 42.790 seconds used by the processor. 
>> $ ./client /var/www/html/cgi-bin 0.600 seconds used by the processor.
>> 
>> These I cannot explain.  How the heck is local slower when the time to 
>> init the db is not taken into account at all?  I'm clueless here.  But 
>> still, the client server model doesn't look like a bad idea.
>> 
>> I'm attaching my server, my client, and my 2 local test programs. 
>> Thoughts?
>> 
>> *having the daemon listen and update the db on policy load is a todo
> 
> Not sure how this helps systemd, as it runs first (by definition) and loads
> the file_contexts configuration before it starts any other daemons, right?
> Now if you wanted systemd to export this as a service to everything else,
> that might make sense.
> 

That is our goal, to have systemd launch this as a service that it could use
for systemd-udev and friends, and then other apps that use kerberos libraries
or tools like install could start to take advantage.

We would still want to allow a tool like restorecon to override the behaviour
since a tools that is doing hundreds or thousands of matchpathcons is faster
if it loads the regex.
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.4.12 (GNU/Linux)
Comment: Using GnuPG with Mozilla - http://enigmail.mozdev.org/

iEYEARECAAYFAlAi0agACgkQrlYvE4MpobPawwCg4IEWgc6bGRhoflU3CCi/5cZg
OFwAoK3q2rZpg1dJQ9DryhiVJrCDvegx
=A4k6
-----END PGP SIGNATURE-----

--
This message was distributed to subscribers of the selinux mailing list.
If you no longer wish to subscribe, send mail to majordomo@tycho.nsa.gov with
the words "unsubscribe selinux" without quotes as the message.

Re: A filename to label translation daemon

[Eric Paris]

On Wed, 2012-08-08 at 16:05 -0400, Stephen Smalley wrote:
> On Wed, 2012-08-08 at 15:31 -0400, Eric Paris wrote:

> Not sure how this helps systemd, as it runs first (by definition) and
> loads the file_contexts configuration before it starts any other
> daemons, right?  Now if you wanted systemd to export this as a service
> to everything else, that might make sense.

If we agree a label daemon is useful and practical I'm sure we can find
a way to get systemd to either use it or be the label daemon.  It might
be as easy as getting systemd to activate it earlier than it normally
activates things.  Although maybe the code needs to live in systemd
itself.  I don't know right now.  I'm still in the, "is this as good an
idea as it seems" stage?

-Eric


--
This message was distributed to subscribers of the selinux mailing list.
If you no longer wish to subscribe, send mail to majordomo@tycho.nsa.gov with
the words "unsubscribe selinux" without quotes as the message.

Re: A filename to label translation daemon

[Colin Walters]

Seems to make sense...though someone could also probably get fairly far
by writing a regular expression optimizer.  It might not even be that
hard to write a multi-regexp matching engine which took a set of regexps
at once and constructed a single matching DFA for them.

On Wed, 2012-08-08 at 15:31 -0400, Eric Paris wrote:
>         /* just to be safe! */
>         buffer[MAX_REQUEST_LEN] = '\0'; 

Should be buffer[len] = '\0';  right?

>                /* calculating the length of the address is magic since it starts with nul.
>                * there be dragons in here! */

See also https://bugzilla.gnome.org/show_bug.cgi?id=615960



--
This message was distributed to subscribers of the selinux mailing list.
If you no longer wish to subscribe, send mail to majordomo@tycho.nsa.gov with
the words "unsubscribe selinux" without quotes as the message.

Re: A filename to label translation daemon

[Russell Coker]

On Thu, 9 Aug 2012, Colin Walters <walters@verbum.org> wrote:
> Seems to make sense...though someone could also probably get fairly far
> by writing a regular expression optimizer.  It might not even be that
> hard to write a multi-regexp matching engine which took a set of regexps
> at once and constructed a single matching DFA for them.

Is this really going to help?  My slowest system is a P3-866 which takes less 
than 30ms of user time for "restorecon /bin/bash" and takes a total of 136ms 
of wall time if the cache is cold.  On a 1.8GHz 64bit system it's only 8ms of 
user time.

What benefit are we expecting to get here?

-- 
My Main Blog         http://etbe.coker.com.au/
My Documents Blog    http://doc.coker.com.au/

--
This message was distributed to subscribers of the selinux mailing list.
If you no longer wish to subscribe, send mail to majordomo@tycho.nsa.gov with
the words "unsubscribe selinux" without quotes as the message.

Re: A filename to label translation daemon

[Daniel J Walsh]

-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1

On 08/09/2012 10:37 AM, Russell Coker wrote:
> On Thu, 9 Aug 2012, Colin Walters <walters@verbum.org> wrote:
>> Seems to make sense...though someone could also probably get fairly far 
>> by writing a regular expression optimizer.  It might not even be that 
>> hard to write a multi-regexp matching engine which took a set of regexps 
>> at once and constructed a single matching DFA for them.
> 
> Is this really going to help?  My slowest system is a P3-866 which takes
> less than 30ms of user time for "restorecon /bin/bash" and takes a total of
> 136ms of wall time if the cache is cold.  On a 1.8GHz 64bit system it's
> only 8ms of user time.
> 
> What benefit are we expecting to get here?
> 
kerberos library currently does a matchpathcon on /tmp/BLAH files and sets the
label correctly.  With this change in the library we are seeing huge
performance hits of apache services caused by loading the regex.

Running make install has caused a huge hit if you are running thousands of
install commands which caused the remove of labeling from the install command.

Systemd has been is executing the load load many many times and is showing up
to 1 second slow down on startup.  If the startup is 10 seconds, it is kind of
hard to justify 10% slowdown on boot.

I believe we just add support for this service and have the labeling fall back
to the default if the labeling socket does not exists, and then distributions
can decide whether or not they want to use it.
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.4.12 (GNU/Linux)
Comment: Using GnuPG with Mozilla - http://enigmail.mozdev.org/

iEYEARECAAYFAlAj7hsACgkQrlYvE4MpobM/BACfdD0TsYmGFyRc6vh+P4xIMcUB
wzEAn2fTC1sAO7MsA7xlBZoAvmfJsBDI
=bIvH
-----END PGP SIGNATURE-----

--
This message was distributed to subscribers of the selinux mailing list.
If you no longer wish to subscribe, send mail to majordomo@tycho.nsa.gov with
the words "unsubscribe selinux" without quotes as the message.

Re: A filename to label translation daemon

[Colin Walters]

On Thu, 2012-08-09 at 13:06 -0400, Daniel J Walsh wrote:

> I believe we just add support for this service and have the labeling fall back
> to the default if the labeling socket does not exists, and then distributions
> can decide whether or not they want to use it.

There are other possible intermediate steps though - for example,
caching the precompiled regular expressions in a file accessible via
mmap().

Basically:

* Your mmap file is in some data format - you can make up your own, but
  I like using http://developer.gnome.org/glib/stable/glib-GVariant.html
* Check the timestamp on the regexp text file versus the cached copy, if
  newer, use the text file
* Otherwise, mmap the cached blob, loop through each regexp, passing
  a pointer to the mmap cache file for regexec()

The mmap cache file would probably need to be tied to a specific version
of glibc though; you wouldn't want to upgrade and use old compiled
regexps that the new glibc doesn't understand.





--
This message was distributed to subscribers of the selinux mailing list.
If you no longer wish to subscribe, send mail to majordomo@tycho.nsa.gov with
the words "unsubscribe selinux" without quotes as the message.

Re: A filename to label translation daemon

[Russell Coker]

On Fri, 10 Aug 2012, Daniel J Walsh <dwalsh@redhat.com> wrote:
> On 08/09/2012 10:37 AM, Russell Coker wrote:
> > On Thu, 9 Aug 2012, Colin Walters <walters@verbum.org> wrote:
> >> Seems to make sense...though someone could also probably get fairly far
> >> by writing a regular expression optimizer.  It might not even be that
> >> hard to write a multi-regexp matching engine which took a set of regexps
> >> at once and constructed a single matching DFA for them.
> > 
> > Is this really going to help?  My slowest system is a P3-866 which takes
> > less than 30ms of user time for "restorecon /bin/bash" and takes a total
> > of 136ms of wall time if the cache is cold.  On a 1.8GHz 64bit system
> > it's only 8ms of user time.
> > 
> > What benefit are we expecting to get here?
> 
> kerberos library currently does a matchpathcon on /tmp/BLAH files and sets
> the label correctly.  With this change in the library we are seeing huge
> performance hits of apache services caused by loading the regex.

What is kerberos doing under /tmp and why is it being done repeatedly by 
different processes?

> Running make install has caused a huge hit if you are running thousands of
> install commands which caused the remove of labeling from the install
> command.

http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=638304

I believe that is a design bug in the SE Linux code in install, I've filed the 
above Debian bug report about it.

I think that correct design of install wouldn't have a "make install" 
performed as part of a dpkg or rpm build do any SE Linux checks.  That would 
be faster than any other option.

> Systemd has been is executing the load load many many times and is showing
> up to 1 second slow down on startup.  If the startup is 10 seconds, it is
> kind of hard to justify 10% slowdown on boot.

Wow, who's got a 10 second boot?

Is systemd loading the file contexts 100 times?  If not then why is it taking 
a second?

> I believe we just add support for this service and have the labeling fall
> back to the default if the labeling socket does not exists, and then
> distributions can decide whether or not they want to use it.

I'm not opposed to making such changes and a 10% performance improvement is 
definitely worth getting.  But first I think we should consider other 
approaches to some of these issues such as changing the way install works.

-- 
My Main Blog         http://etbe.coker.com.au/
My Documents Blog    http://doc.coker.com.au/

--
This message was distributed to subscribers of the selinux mailing list.
If you no longer wish to subscribe, send mail to majordomo@tycho.nsa.gov with
the words "unsubscribe selinux" without quotes as the message.

Re: A filename to label translation daemon

[Daniel J Walsh]

-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1

On 08/09/2012 10:28 PM, Russell Coker wrote:
> On Fri, 10 Aug 2012, Daniel J Walsh <dwalsh@redhat.com> wrote:
>> On 08/09/2012 10:37 AM, Russell Coker wrote:
>>> On Thu, 9 Aug 2012, Colin Walters <walters@verbum.org> wrote:
>>>> Seems to make sense...though someone could also probably get fairly
>>>> far by writing a regular expression optimizer.  It might not even be
>>>> that hard to write a multi-regexp matching engine which took a set of
>>>> regexps at once and constructed a single matching DFA for them.
>>> 
>>> Is this really going to help?  My slowest system is a P3-866 which
>>> takes less than 30ms of user time for "restorecon /bin/bash" and takes
>>> a total of 136ms of wall time if the cache is cold.  On a 1.8GHz 64bit
>>> system it's only 8ms of user time.
>>> 
>>> What benefit are we expecting to get here?
>> 
>> kerberos library currently does a matchpathcon on /tmp/BLAH files and
>> sets the label correctly.  With this change in the library we are seeing
>> huge performance hits of apache services caused by loading the regex.
> 
> What is kerberos doing under /tmp and why is it being done repeatedly by 
> different processes?
> 
Actually /var/tmp/HOST_0 /var/tmp/HTTP_23 ...  Kerberos Replay Cache.   Every
time someone contacts an apache server using kerberos it needs to update this
file, it does this via mktemp (/tmpHTTPD_23XXXX), rename.

/var/cache/krb5rcache(/.*)?	system_u:object_r:krb5_host_rcache_t:s0
/var/tmp/nfs_0	--	system_u:object_r:krb5_host_rcache_t:s0
/var/tmp/host_0	--	system_u:object_r:krb5_host_rcache_t:s0
/var/tmp/HTTP_23	--	system_u:object_r:krb5_host_rcache_t:s0
/var/tmp/HTTP_48	--	system_u:object_r:krb5_host_rcache_t:s0
/var/tmp/ldap_55	--	system_u:object_r:krb5_host_rcache_t:s0
/var/tmp/ldap_487	--	system_u:object_r:krb5_host_rcache_t:s0
/var/tmp/ldapmap1_0	--	system_u:object_r:krb5_host_rcache_t:s0

>> Running make install has caused a huge hit if you are running thousands
>> of install commands which caused the remove of labeling from the install 
>> command.
> 
> http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=638304
> 
> I believe that is a design bug in the SE Linux code in install, I've filed
> the above Debian bug report about it.
> 
> I think that correct design of install wouldn't have a "make install" 
> performed as part of a dpkg or rpm build do any SE Linux checks.  That
> would be faster than any other option.
> 
Programmers and testers regularly run make install and this ends up badly
mislabling files all over the place, telling everyone they have to use rpm or
dpkg is not going to fly.
>> Systemd has been is executing the load load many many times and is
>> showing up to 1 second slow down on startup.  If the startup is 10
>> seconds, it is kind of hard to justify 10% slowdown on boot.
> 
> Wow, who's got a 10 second boot?
> 
> Is systemd loading the file contexts 100 times?  If not then why is it
> taking a second?
> 
10 second boot is available with solid state machines.  Systemd has not seen
the kind of performance that you are and obviously this is sensitive to the
speed of the CPU/Memory.
>> I believe we just add support for this service and have the labeling
>> fall back to the default if the labeling socket does not exists, and
>> then distributions can decide whether or not they want to use it.
> 
> I'm not opposed to making such changes and a 10% performance improvement is
>  definitely worth getting.  But first I think we should consider other 
> approaches to some of these issues such as changing the way install works.
> 

I would prefer Colin suggestion to pre-compiling the regex, if this is
possible.  Then we could do it in the libsemanage any time file context got
updated and put a watch into selinux-policy package to recompile any time gcc
got updated.  I just have no idea how to do this.
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.4.12 (GNU/Linux)
Comment: Using GnuPG with Mozilla - http://enigmail.mozdev.org/

iEYEARECAAYFAlAlAO0ACgkQrlYvE4MpobNyigCgzMUbwkHn+PRkvSHOzrwsNyfk
MkEAnjGO44MUdXtG+x97C3cpY75jDNdP
=1AdS
-----END PGP SIGNATURE-----

--
This message was distributed to subscribers of the selinux mailing list.
If you no longer wish to subscribe, send mail to majordomo@tycho.nsa.gov with
the words "unsubscribe selinux" without quotes as the message.

Re: A filename to label translation daemon

[Russell Coker]

On Fri, 10 Aug 2012, Daniel J Walsh <dwalsh@redhat.com> wrote:
> > What is kerberos doing under /tmp and why is it being done repeatedly by
> > different processes?
> 
> Actually /var/tmp/HOST_0 /var/tmp/HTTP_23 ...  Kerberos Replay Cache.  
> Every time someone contacts an apache server using kerberos it needs to
> update this file, it does this via mktemp (/tmpHTTPD_23XXXX), rename.

When replacing an existing file wouldn't it be better to just copy the context 
of the existing file when creating the replacement?  If there was some good 
reason for running chcon on such a file (and I can't imagine a reason but it's 
best to leave the options open IMHO) then having the context change back the 
next time someone connects seems like a bug.

> >> Running make install has caused a huge hit if you are running thousands
> >> of install commands which caused the remove of labeling from the install
> >> command.
> > 
> > http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=638304
> > 
> > I believe that is a design bug in the SE Linux code in install, I've
> > filed the above Debian bug report about it.
> > 
> > I think that correct design of install wouldn't have a "make install"
> > performed as part of a dpkg or rpm build do any SE Linux checks.  That
> > would be faster than any other option.
> 
> Programmers and testers regularly run make install and this ends up badly
> mislabling files all over the place, telling everyone they have to use rpm
> or dpkg is not going to fly.

The current behavior of dpkg-buildpackage (and presumably a similar RPM build) 
producing lots of warning messages from install (depending on where you build 
it) isn't great either.

Would it be possible to have dpkg-buildpackage (and other relevant programs) 
set an environment variable to modify the behavior of install in such 
situations?

Also note that as install doesn't apply such contexts when creating 
directories the problem of testers running "make install" isn't properly 
solved with the current code.

> >> Systemd has been is executing the load load many many times and is
> >> showing up to 1 second slow down on startup.  If the startup is 10
> >> seconds, it is kind of hard to justify 10% slowdown on boot.
> > 
> > Wow, who's got a 10 second boot?
> > 
> > Is systemd loading the file contexts 100 times?  If not then why is it
> > taking a second?
> 
> 10 second boot is available with solid state machines.  Systemd has not
> seen the kind of performance that you are and obviously this is sensitive
> to the speed of the CPU/Memory.

Why isn't systemd seeing the same performance?

-- 
My Main Blog         http://etbe.coker.com.au/
My Documents Blog    http://doc.coker.com.au/

--
This message was distributed to subscribers of the selinux mailing list.
If you no longer wish to subscribe, send mail to majordomo@tycho.nsa.gov with
the words "unsubscribe selinux" without quotes as the message.

Re: A filename to label translation daemon

[Daniel J Walsh]

-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1

On 08/10/2012 09:35 AM, Russell Coker wrote:
> On Fri, 10 Aug 2012, Daniel J Walsh <dwalsh@redhat.com> wrote:
>>> What is kerberos doing under /tmp and why is it being done repeatedly
>>> by different processes?
>> 
>> Actually /var/tmp/HOST_0 /var/tmp/HTTP_23 ...  Kerberos Replay Cache. 
>> Every time someone contacts an apache server using kerberos it needs to 
>> update this file, it does this via mktemp (/tmpHTTPD_23XXXX), rename.
> 
> When replacing an existing file wouldn't it be better to just copy the
> context of the existing file when creating the replacement?  If there was
> some good reason for running chcon on such a file (and I can't imagine a
> reason but it's best to leave the options open IMHO) then having the
> context change back the next time someone connects seems like a bug.
> 
They use setfscreatecon, if file does not exist, it gets labeled incorrectly.
>>>> Running make install has caused a huge hit if you are running
>>>> thousands of install commands which caused the remove of labeling
>>>> from the install command.
>>> 
>>> http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=638304
>>> 
>>> I believe that is a design bug in the SE Linux code in install, I've 
>>> filed the above Debian bug report about it.
>>> 
>>> I think that correct design of install wouldn't have a "make install" 
>>> performed as part of a dpkg or rpm build do any SE Linux checks.  That 
>>> would be faster than any other option.
>> 
>> Programmers and testers regularly run make install and this ends up
>> badly mislabling files all over the place, telling everyone they have to
>> use rpm or dpkg is not going to fly.
> 
> The current behavior of dpkg-buildpackage (and presumably a similar RPM
> build) producing lots of warning messages from install (depending on where
> you build it) isn't great either.
> 
> Would it be possible to have dpkg-buildpackage (and other relevant
> programs) set an environment variable to modify the behavior of install in
> such situations?
> 
> Also note that as install doesn't apply such contexts when creating 
> directories the problem of testers running "make install" isn't properly 
> solved with the current code.
> 
>>>> Systemd has been is executing the load load many many times and is 
>>>> showing up to 1 second slow down on startup.  If the startup is 10 
>>>> seconds, it is kind of hard to justify 10% slowdown on boot.
>>> 
>>> Wow, who's got a 10 second boot?
>>> 
>>> Is systemd loading the file contexts 100 times?  If not then why is it 
>>> taking a second?
>> 
>> 10 second boot is available with solid state machines.  Systemd has not 
>> seen the kind of performance that you are and obviously this is
>> sensitive to the speed of the CPU/Memory.
> 
> Why isn't systemd seeing the same performance?
> 

-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.4.12 (GNU/Linux)
Comment: Using GnuPG with Mozilla - http://enigmail.mozdev.org/

iEYEARECAAYFAlAnjTQACgkQrlYvE4MpobM2YQCgpErvn+KDksgOSNHiGJfiJ+x0
07cAn2atoTxS/sAAR3/lOtsOIfAeygmM
=IrXz
-----END PGP SIGNATURE-----

--
This message was distributed to subscribers of the selinux mailing list.
If you no longer wish to subscribe, send mail to majordomo@tycho.nsa.gov with
the words "unsubscribe selinux" without quotes as the message.

Re: A filename to label translation daemon

[Daniel J Walsh]

-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1

On 08/10/2012 10:05 AM, Lennart Poettering wrote:
> On Fri, 10.08.12 08:39, Daniel J Walsh (dwalsh@redhat.com) wrote:
> 
>>>>> What benefit are we expecting to get here?
>>>> 
>>>> kerberos library currently does a matchpathcon on /tmp/BLAH files
>>>> and sets the label correctly.  With this change in the library we are
>>>> seeing huge performance hits of apache services caused by loading the
>>>> regex.
>>> 
>>> What is kerberos doing under /tmp and why is it being done repeatedly
>>> by different processes?
>>> 
>> Actually /var/tmp/HOST_0 /var/tmp/HTTP_23 ...  Kerberos Replay Cache.
>> Every time someone contacts an apache server using kerberos it needs to
>> update this file, it does this via mktemp (/tmpHTTPD_23XXXX), rename.
> 
> Hmm, but the ultimate name is still guessable? That sounds really 
> dangerous. Guessable names in /tmp (or /var/tmp) are prone to DoS 
> attacks...
> 
> Lennart .
One would guess that the Kerberos Libraries handle this situation, since it
has been doing it for years.
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.4.12 (GNU/Linux)
Comment: Using GnuPG with Mozilla - http://enigmail.mozdev.org/

iEYEARECAAYFAlAnjZcACgkQrlYvE4MpobOMFACeJQf3CDtrM5qjk8X6LWYAlstn
1o0AoJsmWO7cOrCGhrkOD8gQ+5+envFI
=rbUn
-----END PGP SIGNATURE-----

--
This message was distributed to subscribers of the selinux mailing list.
If you no longer wish to subscribe, send mail to majordomo@tycho.nsa.gov with
the words "unsubscribe selinux" without quotes as the message.

Re: A filename to label translation daemon

[Daniel J Walsh]

-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1

On 08/13/2012 11:18 AM, Nalin Dahyabhai wrote:
> On Sun, Aug 12, 2012 at 07:03:52AM -0400, Daniel J Walsh wrote:
>> -----BEGIN PGP SIGNED MESSAGE----- Hash: SHA1
>> 
>> On 08/10/2012 10:05 AM, Lennart Poettering wrote:
>>> On Fri, 10.08.12 08:39, Daniel J Walsh (dwalsh@redhat.com) wrote:
>>> 
>>>>>>> What benefit are we expecting to get here?
>>>>>> 
>>>>>> kerberos library currently does a matchpathcon on /tmp/BLAH
>>>>>> files and sets the label correctly.  With this change in the
>>>>>> library we are seeing huge performance hits of apache services
>>>>>> caused by loading the regex.
>>>>> 
>>>>> What is kerberos doing under /tmp and why is it being done
>>>>> repeatedly by different processes?
>>>>> 
>>>> Actually /var/tmp/HOST_0 /var/tmp/HTTP_23 ...  Kerberos Replay
>>>> Cache. Every time someone contacts an apache server using kerberos it
>>>> needs to update this file, it does this via mktemp
>>>> (/tmpHTTPD_23XXXX), rename.
>>> 
>>> Hmm, but the ultimate name is still guessable? That sounds really 
>>> dangerous. Guessable names in /tmp (or /var/tmp) are prone to DoS 
>>> attacks...
>>> 
>>> Lennart .
>> One would guess that the Kerberos Libraries handle this situation, since
>> it has been doing it for years.
> 
> No, the library pretty much just fails if it detects shenanigans.  You get
> an error, you call up the admin, they nuke the suspicious file and then go
> yell at someone.
> 
> If /run/user/$UID is available to non-users without them having to log in
> and trigger its creation first, it's probably worth moving.  Or we should
> arrange to have $KRB5RCACHEDIR set to a better location when we start a
> daemon.
> 
> Nalin
> 
Lennart is there something you can add to the unit file to create a UID
directory in /run/user?



-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.4.12 (GNU/Linux)
Comment: Using GnuPG with Mozilla - http://enigmail.mozdev.org/

iEYEARECAAYFAlApLToACgkQrlYvE4MpobOf1gCePMKaq0ctD5cbzRTbNAblcSrr
/aUAn00eVbt+Qrlzx/m5n5EG8Z/KK3Mt
=7+F0
-----END PGP SIGNATURE-----

--
This message was distributed to subscribers of the selinux mailing list.
If you no longer wish to subscribe, send mail to majordomo@tycho.nsa.gov with
the words "unsubscribe selinux" without quotes as the message.

Re: A filename to label translation daemon

[Daniel J Walsh]

-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1

On 08/10/2012 10:17 AM, Lennart Poettering wrote:
> On Fri, 10.08.12 16:11, Lennart Poettering (lennart@poettering.net) wrote:
> 
>>> * Your mmap file is in some data format - you can make up your own,
>>> but I like using
>>> http://developer.gnome.org/glib/stable/glib-GVariant.html * Check the
>>> timestamp on the regexp text file versus the cached copy, if newer, use
>>> the text file * Otherwise, mmap the cached blob, loop through each
>>> regexp, passing a pointer to the mmap cache file for regexec()
>> 
>> Well, the main bits are not actually stored in regex_t afaics. They are 
>> stored in regex_t::buffer. But that makes this kind of caching really 
>> difficult, because that is a pointer and storing a pointer in an mmap() 
>> region makes no sense.
>> 
>> Dunno, maybe another regex library is more suitable for this...
> 
> PCRE appears to be perfectly suited for this. See pcreprecompile(3). They
> have an interface explicitly designed for this. And PCRE is used by
> numerous projects already, so should be safe to use for SELinux too?
> 
> Lennart
> 
This seems like the best solution?  If upstream will accept it.  We could
rebuild the regex data when semanage modifies the file context.

-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.4.12 (GNU/Linux)
Comment: Using GnuPG with Mozilla - http://enigmail.mozdev.org/

iEYEARECAAYFAlApOxEACgkQrlYvE4MpobOtgwCdGLgwt60AY3L+pHuQzQnrREEW
eQ4AoIgVktEglFqzInQJSZpUQLfEFk9H
=YWaU
-----END PGP SIGNATURE-----

--
This message was distributed to subscribers of the selinux mailing list.
If you no longer wish to subscribe, send mail to majordomo@tycho.nsa.gov with
the words "unsubscribe selinux" without quotes as the message.

Re: A filename to label translation daemon

[Colin Walters]

On Mon, 2012-08-13 at 13:36 -0400, Daniel J Walsh wrote:

> This seems like the best solution?  If upstream will accept it.  We could
> rebuild the regex data when semanage modifies the file context.

One thing that will make me mildly sad about this is that now in GNOME
processes we'll have *three* regexp libraries linked in: libc, glib's
PCRE fork (it's ancient history now), and PCRE via libselinux.

I wonder how hard it would be to get a pcre_precompile equivalent into
libc.

Really though in the big picture, while the file context regexps were
probably an OK solution way back when SELinux was a "proof of concept"
prototype, the current policy generating 5000 of them is just crazy...

One other possibility - I bet one could get a huge speedup in some cases
by splitting up the regexp set based on common prefixes.  For example,
if you're trying to match /tmp/krb5cc, there's no reason to run over all
2000 regexps which start with /usr.  This solution is kind of an
intermediate step between "run 5000 regexps serially" and "write custom
code to compile 5000 regexps into a DFA that returns a context".





--
This message was distributed to subscribers of the selinux mailing list.
If you no longer wish to subscribe, send mail to majordomo@tycho.nsa.gov with
the words "unsubscribe selinux" without quotes as the message.

Re: A filename to label translation daemon

[Daniel J Walsh]

-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1

On 08/13/2012 01:55 PM, Colin Walters wrote:
> On Mon, 2012-08-13 at 13:36 -0400, Daniel J Walsh wrote:
> 
>> This seems like the best solution?  If upstream will accept it.  We
>> could rebuild the regex data when semanage modifies the file context.
> 
> One thing that will make me mildly sad about this is that now in GNOME 
> processes we'll have *three* regexp libraries linked in: libc, glib's PCRE
> fork (it's ancient history now), and PCRE via libselinux.
> 
> I wonder how hard it would be to get a pcre_precompile equivalent into 
> libc.
> 
> Really though in the big picture, while the file context regexps were 
> probably an OK solution way back when SELinux was a "proof of concept" 
> prototype, the current policy generating 5000 of them is just crazy...
> 
> One other possibility - I bet one could get a huge speedup in some cases by
> splitting up the regexp set based on common prefixes.  For example, if
> you're trying to match /tmp/krb5cc, there's no reason to run over all 2000
> regexps which start with /usr.  This solution is kind of an intermediate
> step between "run 5000 regexps serially" and "write custom code to compile
> 5000 regexps into a DFA that returns a context".
> 
> 
> 
> 
> 
> -- This message was distributed to subscribers of the selinux mailing
> list. If you no longer wish to subscribe, send mail to
> majordomo@tycho.nsa.gov with the words "unsubscribe selinux" without quotes
> as the message.
> 

We have had a solution for this using prefixes and were trying to add some
intelligence to the library, but we are now thinking this is not a good
solution since we are running into potential problems with substitutions.

-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.4.12 (GNU/Linux)
Comment: Using GnuPG with Mozilla - http://enigmail.mozdev.org/

iEYEARECAAYFAlApQjMACgkQrlYvE4MpobNznACgxFNRIS8PmrgKjSLI+sKisyNn
elEAoJ7qx9TI7c9lCllt5UOGmMxOEFZ9
=36+R
-----END PGP SIGNATURE-----

--
This message was distributed to subscribers of the selinux mailing list.
If you no longer wish to subscribe, send mail to majordomo@tycho.nsa.gov with
the words "unsubscribe selinux" without quotes as the message.

Re: A filename to label translation daemon

[Russell Coker]

On Tue, 14 Aug 2012, Colin Walters <walters@verbum.org> wrote:
> Really though in the big picture, while the file context regexps were
> probably an OK solution way back when SELinux was a "proof of concept"
> prototype, the current policy generating 5000 of them is just crazy...

Actually the situation is way better than it was in the early days.

When I first started working on SE Linux the software wasn't as optimised and 
the hardware was way slower.  A restorecon type operation would be 99% user 
CPU time, taking more than 20 minutes of CPU time for relabelling a relatively 
small filesystem was common.

Having 5000 on a modern for argument sake (it's 1923 on my system, but that 
depends on whether you load a policy with everything or just the modules you 
need) is a lot easier than the situation in the early days with fewer regular 
expressions.

> One other possibility - I bet one could get a huge speedup in some cases
> by splitting up the regexp set based on common prefixes.  For example,
> if you're trying to match /tmp/krb5cc, there's no reason to run over all
> 2000 regexps which start with /usr.  This solution is kind of an
> intermediate step between "run 5000 regexps serially" and "write custom
> code to compile 5000 regexps into a DFA that returns a context".

Yes, I wrote code to do that many years ago.  Any regex which had a fixed 
string for the first subdirectory from root would only be called for a filename 
which was in the same subdirectory.  The prefixes were indexed so an integer 
compare would be used to determine whether a regex would be called.  Regexes 
which applied to multiple prefixes (EG "/.*") would be applied to all files.

But I believe that the kerberos performance problem is not calling the regexes 
but loading.  The current code (unless it's changed recently) will compile all 
regexes, so when kerberos loads the file contexts for a check on /tmp then it 
will compile all regexes under /usr, /var, and other common prefixes even when 
they won't be used.  I don't know how much time can be saved by skipping the 
compile of those.

Another thing that could be done is that we could have an interface for 
loading a file_contexts file for a specific prefix.  Then the code which generates 
the file_contexts file could generate files such as file_contexts_tmp which only 
has entries which match /tmp (10 for the policy I use, maybe 50 or so for the 
one you use) and which match everything (EG "/.*").  On my system there are 9 
file_contexts entries which are not prefix specific of which one is required 
("/.*") and of the others /vmlinux.* and /initrd\.img.* are obsolete and the 
other 6 could be easily split to be prefix specific.

So with a minor change to the library interface (adding a new entry point so 
the new library could work with old apps) we could have a program which knows 
that it will only label files under /tmp only checking 11 regexes on my system 
or maybe 50 on your system.

-- 
My Main Blog         http://etbe.coker.com.au/
My Documents Blog    http://doc.coker.com.au/

--
This message was distributed to subscribers of the selinux mailing list.
If you no longer wish to subscribe, send mail to majordomo@tycho.nsa.gov with
the words "unsubscribe selinux" without quotes as the message.

Re: A filename to label translation daemon

[Eric Paris]

[-- Attachment #1: Type: text/plain, Size: 3655 bytes --]

On Tue, Aug 14, 2012 at 7:18 AM, Russell Coker <russell@coker.com.au> wrote:
> On Tue, 14 Aug 2012, Colin Walters <walters@verbum.org> wrote:
>> Really though in the big picture, while the file context regexps were
>> probably an OK solution way back when SELinux was a "proof of concept"
>> prototype, the current policy generating 5000 of them is just crazy...
>
> Actually the situation is way better than it was in the early days.
>
> When I first started working on SE Linux the software wasn't as optimised and
> the hardware was way slower.  A restorecon type operation would be 99% user
> CPU time, taking more than 20 minutes of CPU time for relabelling a relatively
> small filesystem was common.
>
> Having 5000 on a modern for argument sake (it's 1923 on my system, but that
> depends on whether you load a policy with everything or just the modules you
> need) is a lot easier than the situation in the early days with fewer regular
> expressions.
>
>> One other possibility - I bet one could get a huge speedup in some cases
>> by splitting up the regexp set based on common prefixes.  For example,
>> if you're trying to match /tmp/krb5cc, there's no reason to run over all
>> 2000 regexps which start with /usr.  This solution is kind of an
>> intermediate step between "run 5000 regexps serially" and "write custom
>> code to compile 5000 regexps into a DFA that returns a context".
>
> Yes, I wrote code to do that many years ago.  Any regex which had a fixed
> string for the first subdirectory from root would only be called for a filename
> which was in the same subdirectory.  The prefixes were indexed so an integer
> compare would be used to determine whether a regex would be called.  Regexes
> which applied to multiple prefixes (EG "/.*") would be applied to all files.
>
> But I believe that the kerberos performance problem is not calling the regexes
> but loading.  The current code (unless it's changed recently) will compile all
> regexes, so when kerberos loads the file contexts for a check on /tmp then it
> will compile all regexes under /usr, /var, and other common prefixes even when
> they won't be used.  I don't know how much time can be saved by skipping the
> compile of those.
>
> Another thing that could be done is that we could have an interface for
> loading a file_contexts file for a specific prefix.  Then the code which generates
> the file_contexts file could generate files such as file_contexts_tmp which only
> has entries which match /tmp (10 for the policy I use, maybe 50 or so for the
> one you use) and which match everything (EG "/.*").  On my system there are 9
> file_contexts entries which are not prefix specific of which one is required
> ("/.*") and of the others /vmlinux.* and /initrd\.img.* are obsolete and the
> other 6 could be easily split to be prefix specific.
>
> So with a minor change to the library interface (adding a new entry point so
> the new library could work with old apps) we could have a program which knows
> that it will only label files under /tmp only checking 11 regexes on my system
> or maybe 50 on your system.

I have code that does just that.  Dan and I both wrote a version.
I'll attach it.  I didn't find the speedups we were hoping for and it
didn't work correctly/completely in the face of file context
equivalencies.  Although that is likely fixable.  I was just looking
at all of the stem code (and wondered who wrote it but it was
pre-git).  I'm surprised it made a big difference.  Wouldn't the regex
code be able to return extremely quickly if it didn't match?  Anyway.
I'm writing some test programs to look at all of the possibilities.

-Eric

[-- Attachment #2: 1.patch --]
[-- Type: application/octet-stream, Size: 9364 bytes --]

commit 799c25920f00afa45a5962df9b670e9c5c5766e0
Author: Dan Walsh <dwalsh@redhat.com>
Date:   Fri Sep 2 08:54:12 2011 -0400

    libselinux: FIXME support multiple prefix/subsets labeling
    
    to reduce memory and increase speed apps that support labeling on a subset of directories
    
    [Needs review]
    
    Signed-off-by: Dan Walsh <dwalsh@redhat.com>
    NOT-Signed-off-by: Eric Paris <eparis@redhat.com>
    Acked-by: Dan Walsh <dwalsh@redhat.com>

diff --git a/libselinux/include/selinux/label.h b/libselinux/include/selinux/label.h
index 8263f32..9ef10fc 100644
--- a/libselinux/include/selinux/label.h
+++ b/libselinux/include/selinux/label.h
@@ -48,8 +48,10 @@ struct selabel_handle;
 #define SELABEL_OPT_PATH	3
 /* select a subset of the search space as an optimization (file backend) */
 #define SELABEL_OPT_SUBSET	4
+/* like subset, but an array of subsets */
+#define SELABEL_OPT_PREFIXES	5
 /* total number of options */
-#define SELABEL_NOPT		5
+#define SELABEL_NOPT		6
 
 /*
  * Label operations
diff --git a/libselinux/include/selinux/selinux.h b/libselinux/include/selinux/selinux.h
index 3202039..b1002ac 100644
--- a/libselinux/include/selinux/selinux.h
+++ b/libselinux/include/selinux/selinux.h
@@ -139,7 +139,10 @@ struct av_decision {
 /* Structure for passing options, used by AVC and label subsystems */
 struct selinux_opt {
 	int type;
-	const char *value;
+	union {
+		const char *value;
+		const char **values;
+	};
 };
 
 /* Callback facilities */
@@ -420,6 +423,11 @@ extern int matchpathcon_init(const char *path);
    regexes that have stems that are prefixes of 'prefix'. */
 extern int matchpathcon_init_prefix(const char *path, const char *prefix);
 
+/* Same as matchpathcon_init, but only load entries with
+ * regexes that have stems that are prefixes of the 'prefixes'
+ * array of entries.  The last entry must be NULL. */
+extern int matchpathcon_init_prefixes(const char *patch, const char **prefixes);
+
 /* Free the memory allocated by matchpathcon_init. */
 extern void matchpathcon_fini(void);
 
diff --git a/libselinux/man/man3/matchpathcon.3 b/libselinux/man/man3/matchpathcon.3
index cdbb252..b6814ed 100644
--- a/libselinux/man/man3/matchpathcon.3
+++ b/libselinux/man/man3/matchpathcon.3
@@ -8,7 +8,9 @@ matchpathcon, matchpathcon_index \- get the default SELinux security context for
 
 .BI "int matchpathcon_init(const char *" path ");"
 
-.BI "int matchpathcon_init_prefix(const char *" path ", const char *" subset ");"
+.BI "int matchpathcon_init_prefix(const char *" path ", const char *" prefix ");"
+
+.BI "int matchpathcon_init_prefixes(const char *" path ", const char **" prefixes ");"
 
 .BI "int matchpathcon_fini(void);"
 .sp
@@ -50,6 +52,14 @@ by
 .I prefix.
 
 .sp
+.B matchpathcon_init_prefixes
+is the same as
+.B matchpathcon_init_prefix
+but takes an array of
+.I prefixes
+instead of a single prefix.  The last entry in the array must be NULL.
+
+.sp
 .B matchpathcon_fini
 frees the memory allocated by a prior call to
 .B matchpathcon_init.
diff --git a/libselinux/man/man3/selabel_open.3 b/libselinux/man/man3/selabel_open.3
index 8674e37..89bb4d3 100644
--- a/libselinux/man/man3/selabel_open.3
+++ b/libselinux/man/man3/selabel_open.3
@@ -66,6 +66,13 @@ A non-null value for this option enables context validation.  By default,
 is used; a custom validation function can be provided via
 .BR selinux_set_callback (3).
 Note that an invalid context may not be treated as an error unless it is actually encountered during a lookup operation.
+.TP
+.B SELABEL_OPT_SUBSET
+A ":" separates string of path prefixes that tell the system to only loads entries with regular expressions that could match this strings. For example "/dev:/var/run:/tmp".  This option can cause the system to use less memory and work faster, but you should only use paths that begin with a prefix.
+.TP
+.B SELABEL_OPT_PATH
+A string representing an alternate path the the regular expressions.
+.sp
 
 .SH "BACKENDS"
 
@@ -99,4 +106,3 @@ Eamon Walsh <ewalsh@tycho.nsa.gov>
 .BR selabel_stats (3),
 .BR selinux_set_callback (3),
 .BR selinux (8)
-
diff --git a/libselinux/src/label_file.c b/libselinux/src/label_file.c
index 7bc46cc..82a608c 100644
--- a/libselinux/src/label_file.c
+++ b/libselinux/src/label_file.c
@@ -27,6 +27,7 @@
  * Internals, mostly moved over from matchpathcon.c
  */
 
+#define MAX_PREFIX 100
 /* A file security context specification. */
 typedef struct spec {
 	struct selabel_lookup_rec lr;	/* holds contexts for lookup result */
@@ -276,7 +277,7 @@ static int compile_regex(struct saved_data *data, spec_t *spec, char **errbuf)
 
 
 static int process_line(struct selabel_handle *rec,
-			const char *path, const char *prefix,
+			const char *path, const char **prefix_array,
 			char *line_buf, int pass, unsigned lineno)
 {
 	int items, len;
@@ -310,12 +311,24 @@ static int process_line(struct selabel_handle *rec,
 	}
 
 	len = get_stem_from_spec(regex);
-	if (len && prefix && strncmp(prefix, regex, len)) {
-		/* Stem of regex does not match requested prefix, discard. */
-		free(regex);
-		free(type);
-		free(context);
-		return 0;
+	if (len && prefix_array[0]) {
+		int i = 0;
+		int found = 0;
+		while (i < MAX_PREFIX && prefix_array[i]) {
+			if (strncmp(prefix_array[i], regex, len) == 0) {
+				found = 1;
+				break;
+			}
+			i++;
+		}
+
+		if (! found) {
+			/* Stem of regex does not match requested prefix, discard. */
+			free(regex);
+			free(type);
+			free(context);
+			return 0;
+		}
 	}
 
 	if (pass == 1) {
@@ -397,7 +410,8 @@ static int init(struct selabel_handle *rec, struct selinux_opt *opts,
 {
 	struct saved_data *data = (struct saved_data *)rec->data;
 	const char *path = NULL;
-	const char *prefix = NULL;
+	const char *static_prefix_array[2] = {NULL, };
+	const char **prefix_array = static_prefix_array;
 	FILE *fp;
 	FILE *localfp = NULL;
 	FILE *homedirfp = NULL;
@@ -418,7 +432,10 @@ static int init(struct selabel_handle *rec, struct selinux_opt *opts,
 			path = opts[n].value;
 			break;
 		case SELABEL_OPT_SUBSET:
-			prefix = opts[n].value;
+			static_prefix_array[0] = opts[n].value;
+			break;
+		case SELABEL_OPT_PREFIXES:
+			prefix_array = opts[n].values;
 			break;
 		case SELABEL_OPT_BASEONLY:
 			baseonly = !!opts[n].value;
@@ -481,7 +498,7 @@ static int init(struct selabel_handle *rec, struct selinux_opt *opts,
 		while (getline(&line_buf, &line_len, fp) > 0) {
 			if (data->nspec >= maxnspec)
 				break;
-			status = process_line(rec, path, prefix, line_buf, pass, ++lineno);
+			status = process_line(rec, path, prefix_array, line_buf, pass, ++lineno);
 			if (status)
 				goto finish;
 		}
@@ -497,7 +514,7 @@ static int init(struct selabel_handle *rec, struct selinux_opt *opts,
 			while (getline(&line_buf, &line_len, homedirfp) > 0) {
 				if (data->nspec >= maxnspec)
 					break;
-				status = process_line(rec, homedir_path, prefix, line_buf, pass, ++lineno);
+				status = process_line(rec, homedir_path, prefix_array, line_buf, pass, ++lineno);
 				if (status)
 					goto finish;
 			}
@@ -507,7 +524,7 @@ static int init(struct selabel_handle *rec, struct selinux_opt *opts,
 			while (getline(&line_buf, &line_len, localfp) > 0) {
 				if (data->nspec >= maxnspec)
 					break;
-				status = process_line(rec, local_path, prefix, line_buf, pass, ++lineno);
+				status = process_line(rec, local_path, prefix_array, line_buf, pass, ++lineno);
 				if (status)
 					goto finish;
 			}
diff --git a/libselinux/src/matchpathcon.c b/libselinux/src/matchpathcon.c
index 2d7369e..c461e43 100644
--- a/libselinux/src/matchpathcon.c
+++ b/libselinux/src/matchpathcon.c
@@ -306,7 +306,7 @@ static void matchpathcon_init_once(void)
 		destructor_key_initialized = 1;
 }
 
-int matchpathcon_init_prefix(const char *path, const char *subset)
+int matchpathcon_init_prefixes(const char *path, const char **prefixes)
 {
 	if (!mycanoncon)
 		mycanoncon = default_canoncon;
@@ -314,15 +314,22 @@ int matchpathcon_init_prefix(const char *path, const char *subset)
 	__selinux_once(once, matchpathcon_init_once);
 	__selinux_setspecific(destructor_key, (void *)1);
 
-	options[SELABEL_OPT_SUBSET].type = SELABEL_OPT_SUBSET;
-	options[SELABEL_OPT_SUBSET].value = subset;
+	options[SELABEL_OPT_PREFIXES].type = SELABEL_OPT_PREFIXES;
+	options[SELABEL_OPT_PREFIXES].values = prefixes;
 	options[SELABEL_OPT_PATH].type = SELABEL_OPT_PATH;
 	options[SELABEL_OPT_PATH].value = path;
 
 	hnd = selabel_open(SELABEL_CTX_FILE, options, SELABEL_NOPT);
 	return hnd ? 0 : -1;
 }
+hidden_def(matchpathcon_init_prefixes)
 
+int matchpathcon_init_prefix(const char *path, const char *prefix)
+{
+	const char *prefixes[2] = { prefix, NULL };
+
+	return matchpathcon_init_prefixes(path, prefixes);
+}
 hidden_def(matchpathcon_init_prefix)
 
 int matchpathcon_init(const char *path)
diff --git a/libselinux/src/selinux_internal.h b/libselinux/src/selinux_internal.h
index ab80dba..1bbc395 100644
--- a/libselinux/src/selinux_internal.h
+++ b/libselinux/src/selinux_internal.h
@@ -83,6 +83,7 @@ hidden_proto(selinux_mkload_policy)
     hidden_proto(selinux_path)
     hidden_proto(selinux_check_passwd_access)
     hidden_proto(selinux_check_securetty_context)
+    hidden_proto(matchpathcon_init_prefixes)
     hidden_proto(matchpathcon_init_prefix)
     hidden_proto(selinux_users_path)
     hidden_proto(selinux_usersconf_path);

[-- Attachment #3: 2.patch --]
[-- Type: application/octet-stream, Size: 8750 bytes --]

commit 623b3c80a2efaede2d3d08989407862734ec65b9
Author: Dan Walsh <dwalsh@redhat.com>
Date:   Fri Sep 2 08:55:47 2011 -0400

    policycoreutils: setfiles: FIXME Allow setfiles/restorecon to take advantage of new subset handling in libselinux
    
    Needs review.
    
    Signed-off-by: Dan Walsh <dwalsh@redhat.com>
    NOT-Signed-off-by: Eric Paris <eparis@redhat.com>
    Acked-by: Dan Walsh <dwalsh@redhat.com>

diff --git a/libselinux/man/man3/selabel_open.3 b/libselinux/man/man3/selabel_open.3
index 89bb4d3..fc5b120 100644
--- a/libselinux/man/man3/selabel_open.3
+++ b/libselinux/man/man3/selabel_open.3
@@ -37,8 +37,11 @@ structures of length
 .ta 4n 16n 24n
 .nf
 struct selinux_opt {
-	int	type;
-	const char	*value;
+	int type;
+	union {
+		const char *value;
+		const char **values;
+	};
 };
 .fi
 .ta
diff --git a/policycoreutils/restorecond/restorecond.c b/policycoreutils/restorecond/restorecond.c
index 9bdd89d..1762850 100644
--- a/policycoreutils/restorecond/restorecond.c
+++ b/policycoreutils/restorecond/restorecond.c
@@ -139,6 +139,7 @@ int main(int argc, char **argv)
 {
 	int opt;
 	struct sigaction sa;
+	const char *null_array[1] = { NULL };
 
 	memset(&r_opts, 0, sizeof(r_opts));
 
@@ -159,6 +160,7 @@ int main(int argc, char **argv)
 	r_opts.fts_flags = FTS_PHYSICAL;
 	r_opts.selabel_opt_validate = NULL;
 	r_opts.selabel_opt_path = NULL;
+	r_opts.selabel_opt_prefixes = null_array;
 	r_opts.ignore_enoent = 1;
 
 	restore_init(&r_opts);
diff --git a/policycoreutils/scripts/genhomedircon b/policycoreutils/scripts/genhomedircon
index ab696a7..58b19cd 100644
--- a/policycoreutils/scripts/genhomedircon
+++ b/policycoreutils/scripts/genhomedircon
@@ -1,2 +1,3 @@
 #!/bin/sh
+
 /usr/sbin/semodule -Bn
diff --git a/policycoreutils/setfiles/restore.c b/policycoreutils/setfiles/restore.c
index 4c62b41..1b461f6 100644
--- a/policycoreutils/setfiles/restore.c
+++ b/policycoreutils/setfiles/restore.c
@@ -58,11 +58,16 @@ void remove_exclude(const char *directory)
 void restore_init(struct restore_opts *opts)
 {	
 	r_opts = opts;
-	struct selinux_opt selinux_opts[] = {
-		{ SELABEL_OPT_VALIDATE, r_opts->selabel_opt_validate },
-		{ SELABEL_OPT_PATH, r_opts->selabel_opt_path }
-	};
-	r_opts->hnd = selabel_open(SELABEL_CTX_FILE, selinux_opts, 2);
+	struct selinux_opt selinux_opts[3];
+
+	selinux_opts[0].type = SELABEL_OPT_VALIDATE;
+	selinux_opts[0].value = r_opts->selabel_opt_validate;
+	selinux_opts[1].type = SELABEL_OPT_PATH;
+	selinux_opts[1].value = r_opts->selabel_opt_path;
+	selinux_opts[2].type = SELABEL_OPT_PREFIXES;
+	selinux_opts[2].values = r_opts->selabel_opt_prefixes;
+
+	r_opts->hnd = selabel_open(SELABEL_CTX_FILE, selinux_opts, 3);
 	if (!r_opts->hnd) {
 		perror(r_opts->selabel_opt_path);
 		exit(1);
diff --git a/policycoreutils/setfiles/restore.h b/policycoreutils/setfiles/restore.h
index ac27222..4b39972 100644
--- a/policycoreutils/setfiles/restore.h
+++ b/policycoreutils/setfiles/restore.h
@@ -40,6 +40,7 @@ struct restore_opts {
 	int fts_flags; /* Flags to fts, e.g. follow links, follow mounts */
 	const char *selabel_opt_validate;
 	const char *selabel_opt_path;
+	const char **selabel_opt_prefixes;
 };
 
 void restore_init(struct restore_opts *opts);
diff --git a/policycoreutils/setfiles/restorecon.8 b/policycoreutils/setfiles/restorecon.8
index 35c8a9c..65a59de 100644
--- a/policycoreutils/setfiles/restorecon.8
+++ b/policycoreutils/setfiles/restorecon.8
@@ -4,10 +4,10 @@ restorecon \- restore file(s) default SELinux security contexts.
 
 .SH "SYNOPSIS"
 .B restorecon
-.I [\-o outfilename ] [\-R] [\-n] [\-p] [\-v] [\-e directory ] pathname...
+.I [\-o outfilename ] [\-R] [\-n] [\-p] [\-v] [\-e directory ] [\-L labelprefix ] pathname...
 .P
 .B restorecon
-.I \-f infilename [\-o outfilename ] [\-e directory ] [\-R] [\-n] [\-p] [\-v] [\-F]
+.I \-f infilename [\-o outfilename ] [\-e directory ] [\-L labelprefix ] [\-R] [\-n] [\-p] [\-v] [\-F]
 
 .SH "DESCRIPTION"
 This manual page describes the
@@ -37,6 +37,12 @@ infilename contains a list of files to be processed by application. Use \- for s
 .B \-e directory
 directory to exclude (repeat option for more than one directory.)
 .TP 
+.B \-L labelprefix
+Tells selinux to only use the file context that match this prefix for labeling,  -L can be called multiple times.  Can speed up labeling if you are only doing one directory.
+
+# restorecon -R -v -L /dev /dev
+
+.TP
 .B \-R \-r
 change files and directories file labels recursively
 .TP 
diff --git a/policycoreutils/setfiles/setfiles.8 b/policycoreutils/setfiles/setfiles.8
index 5ef2dba..2741919 100644
--- a/policycoreutils/setfiles/setfiles.8
+++ b/policycoreutils/setfiles/setfiles.8
@@ -4,7 +4,7 @@ setfiles \- set file SELinux security contexts.
 
 .SH "SYNOPSIS"
 .B setfiles
-.I [\-c policy ] [\-d] [\-l] [\-n] [\-e directory ] [\-o filename ] [\-q] [\-s] [\-v] [\-W] [\-F] spec_file pathname...
+.I [\-c policy ] [\-d] [\-l] [\-n] [\-e directory ] [\-o filename ] [\-L labelprefix ] [\-q] [\-s] [\-v] [\-W] [\-F] spec_file pathname...
 .SH "DESCRIPTION"
 This manual page describes the
 .BR setfiles
@@ -52,6 +52,9 @@ directory to exclude (repeat option for more than one directory.)
 .B \-F
 Force reset of context to match file_context for customizable files, and the default file context, changing the user, role, range portion as well as the type.
 .TP 
+.B \-L labelprefix
+Tells selinux to only use the file context that match this prefix for labeling,  -L can be called multiple times.  Can speed up labeling if you are only doing one directory.
+.TP
 .B \-o filename
 save list of files with incorrect context in filename.
 .TP 
diff --git a/policycoreutils/setfiles/setfiles.c b/policycoreutils/setfiles/setfiles.c
index 8d2eadf..36fc4e8 100644
--- a/policycoreutils/setfiles/setfiles.c
+++ b/policycoreutils/setfiles/setfiles.c
@@ -39,7 +39,7 @@ void usage(const char *const name)
 {
 	if (iamrestorecon) {
 		fprintf(stderr,
-			"usage:  %s [-iFnprRv0] [-e excludedir ] [-o filename ] [-f filename | pathname... ]\n",
+			"usage:  %s [-iFnprRv0] [ -L labelprefix ] [-e excludedir ] [-o filename ] [-f filename | pathname... ]\n",
 			name);
 	} else {
 		fprintf(stderr,
@@ -137,7 +137,7 @@ static void maybe_audit_mass_relabel(int mass_relabel, int mass_relabel_errs)
 int main(int argc, char **argv)
 {
 	struct stat sb;
-	int opt, i = 0;
+	int opt, i;
 	char *input_filename = NULL;
 	int use_input_file = 0;
 	char *buf = NULL;
@@ -145,6 +145,8 @@ int main(int argc, char **argv)
 	int recurse; /* Recursive descent. */
 	char *base;
 	int mass_relabel = 0, errors = 0;
+	int num_prefixes = 0;
+	const char *null_array[1] = { NULL };
 	
 	memset(&r_opts, 0, sizeof(r_opts));
 
@@ -160,6 +162,7 @@ int main(int argc, char **argv)
 	r_opts.outfile = NULL;
 	r_opts.force = 0;
 	r_opts.hard_links = 1;
+	r_opts.selabel_opt_prefixes = null_array;
 
 	altpath = NULL;
 
@@ -217,7 +220,7 @@ int main(int argc, char **argv)
 	exclude_non_seclabel_mounts();
 
 	/* Process any options. */
-	while ((opt = getopt(argc, argv, "c:de:f:ilnpqrsvo:FRW0")) > 0) {
+	while ((opt = getopt(argc, argv, "c:de:f:ilnpqrsvo:FL:RW0")) > 0) {
 		switch (opt) {
 		case 'c':
 			{
@@ -280,6 +283,35 @@ int main(int argc, char **argv)
 		case 'n':
 			r_opts.change = 0;
 			break;
+		case 'L':
+		{
+			char **new_prefixes;
+
+			/* we need 1 for this entry and 1 for the NULL entry */
+			new_prefixes = malloc(sizeof(*new_prefixes) * (num_prefixes + 2));
+			if (!new_prefixes) {
+				fprintf(stderr, "Can't allocate memory for labeling prefix %s:%s\n",
+				optarg, strerror(errno));
+				exit(1);
+			}
+
+			memcpy(new_prefixes, r_opts.selabel_opt_prefixes, sizeof(*new_prefixes) * num_prefixes);
+			new_prefixes[num_prefixes] = strdup(optarg);
+			if (!new_prefixes[num_prefixes]) {
+				fprintf(stderr, "Can't allocate memory for labeling prefix %s:%s\n",
+				optarg, strerror(errno));
+				exit(1);
+			}
+
+			new_prefixes[num_prefixes + 1] = NULL;
+			num_prefixes++;
+
+			if (r_opts.selabel_opt_prefixes != null_array)
+				free(r_opts.selabel_opt_prefixes);
+
+			r_opts.selabel_opt_prefixes = (const char **)new_prefixes;
+			break;
+		}
 		case 'o':
 			if (strcmp(optarg, "-") == 0) {
 				r_opts.outfile = stdout;
@@ -433,7 +465,15 @@ int main(int argc, char **argv)
 	if (r_opts.outfile)
 		fclose(r_opts.outfile);
 
-       if (r_opts.progress && r_opts.count >= STAR_COUNT)
-               printf("\n");
+	if (r_opts.progress && r_opts.count >= STAR_COUNT)
+		printf("\n");
+
+	free(r_opts.progname);
+	i = 0;
+	while (r_opts.selabel_opt_prefixes[i])
+		free((void *)r_opts.selabel_opt_prefixes[i++]);
+	if (r_opts.selabel_opt_prefixes != null_array)
+		free(r_opts.selabel_opt_prefixes);
+	free(r_opts.rootpath);
 	exit(errors);
 }

[-- Attachment #4: 3.patch --]
[-- Type: application/octet-stream, Size: 2228 bytes --]

commit a9f302cf1eff9df2d412c85ff8f0817deae0e8e5
Author: Dan Walsh <dwalsh@redhat.com>
Date:   Wed May 30 16:08:05 2012 -0400

    libselinux: matchpathcon: use multiple prefixes FIXME
    
    [This patch does not do length checking on a static array]
    
    Allow multiple -p prefixes to limit the database.
    
    NOT-Signed-off-by: Eric Paris <eparis@redhat.com>
    Acked-by: Dan Walsh <dwalsh@redhat.com>

diff --git a/libselinux/utils/matchpathcon.c b/libselinux/utils/matchpathcon.c
index b1adadd..b0dc01d 100644
--- a/libselinux/utils/matchpathcon.c
+++ b/libselinux/utils/matchpathcon.c
@@ -45,12 +45,15 @@ static int printmatchpathcon(const char *path, int header, int mode)
 
 int main(int argc, char **argv)
 {
-	int i, init = 0;
+	int i;
 	int header = 1, opt;
 	int verify = 0;
 	int notrans = 0;
 	int error = 0;
 	int quiet = 0;
+	const char *prefixes[100];
+	int file=0;
+	int prefix=0;
 
 	if (argc < 2)
 		usage(argv[0]);
@@ -68,13 +71,13 @@ int main(int argc, char **argv)
 			set_matchpathcon_flags(MATCHPATHCON_NOTRANS);
 			break;
 		case 'f':
-			if (init) {
+			if (prefix) {
 				fprintf(stderr,
 					"%s:  -f and -p are exclusive\n",
 					argv[0]);
 				exit(1);
 			}
-			init = 1;
+			file = 1;
 			if (matchpathcon_init(optarg)) {
 				fprintf(stderr,
 					"Error while processing %s:  %s\n",
@@ -84,20 +87,15 @@ int main(int argc, char **argv)
 			}
 			break;
 		case 'p':
-			if (init) {
+			if (file) {
 				fprintf(stderr,
 					"%s:  -f and -p are exclusive\n",
 					argv[0]);
 				exit(1);
 			}
-			init = 1;
-			if (matchpathcon_init_prefix(NULL, optarg)) {
-				fprintf(stderr,
-					"Error while processing %s:  %s\n",
-					optarg,
-					errno ? strerror(errno) : "invalid");
-				exit(1);
-			}
+			prefixes[prefix]=optarg;
+			prefix++;
+			prefixes[prefix]=NULL;
 			break;
 		case 'q':
 			quiet = 1;
@@ -106,6 +104,16 @@ int main(int argc, char **argv)
 			usage(argv[0]);
 		}
 	}
+	if (prefix > 0) {
+		if (matchpathcon_init_prefixes(NULL, prefixes)) {
+			fprintf(stderr,
+				"Error while processing %s:  %s\n",
+				optarg,
+				errno ? strerror(errno) : "invalid");
+			exit(1);
+		}
+	}
+
 	for (i = optind; i < argc; i++) {
 		int rc, mode = 0;
 		struct stat buf;

Re: A filename to label translation daemon

[Eric Paris]

Initial really really rough testing using pcre:

Load and free DB 100 times using GLIBC:
 8.370 seconds used by the processor.

Load and free DB 100 times using pcre:
 1.800 seconds used by the processor.

This is just using PCRE to load and compile the DB. Not using an
mmap'd precompiled version.  That's next on the testing list.  But
switching to pcre is already a huge time win it seems....

-Eric

--
This message was distributed to subscribers of the selinux mailing list.
If you no longer wish to subscribe, send mail to majordomo@tycho.nsa.gov with
the words "unsubscribe selinux" without quotes as the message.

Re: A filename to label translation daemon

[Stephen Smalley]

On Tue, 2012-08-14 at 10:01 -0400, Eric Paris wrote:
> Initial really really rough testing using pcre:
> 
> Load and free DB 100 times using GLIBC:
>  8.370 seconds used by the processor.
> 
> Load and free DB 100 times using pcre:
>  1.800 seconds used by the processor.
> 
> This is just using PCRE to load and compile the DB. Not using an
> mmap'd precompiled version.  That's next on the testing list.  But
> switching to pcre is already a huge time win it seems....

Do you need to convert any of the regexes in file_contexts to address
the differences between PCRE and POSIX regexes?

You should check that setfiles using glibc regex followed by setfiles
using PCRE does not change anything on the filesystem, i.e. run the
existing setfiles on all filesystems, then run the setfiles using PCRE
with -nv to report any changes that would be made without applying them.

-- 
Stephen Smalley
National Security Agency


--
This message was distributed to subscribers of the selinux mailing list.
If you no longer wish to subscribe, send mail to majordomo@tycho.nsa.gov with
the words "unsubscribe selinux" without quotes as the message.

Re: A filename to label translation daemon

[Eric Paris]

Testing is FAR FAR FAR from complete.  PCRE was able to find 2 invalid
specifications that the posix API didn't complain about.  Dan has
fixed those.  I will verify any changes with a complete relabel before
submitting.

-Eric

On Tue, Aug 14, 2012 at 12:48 PM, Stephen Smalley <sds@tycho.nsa.gov> wrote:
> On Tue, 2012-08-14 at 10:01 -0400, Eric Paris wrote:
>> Initial really really rough testing using pcre:
>>
>> Load and free DB 100 times using GLIBC:
>>  8.370 seconds used by the processor.
>>
>> Load and free DB 100 times using pcre:
>>  1.800 seconds used by the processor.
>>
>> This is just using PCRE to load and compile the DB. Not using an
>> mmap'd precompiled version.  That's next on the testing list.  But
>> switching to pcre is already a huge time win it seems....
>
> Do you need to convert any of the regexes in file_contexts to address
> the differences between PCRE and POSIX regexes?
>
> You should check that setfiles using glibc regex followed by setfiles
> using PCRE does not change anything on the filesystem, i.e. run the
> existing setfiles on all filesystems, then run the setfiles using PCRE
> with -nv to report any changes that would be made without applying them.
>
> --
> Stephen Smalley
> National Security Agency
>

--
This message was distributed to subscribers of the selinux mailing list.
If you no longer wish to subscribe, send mail to majordomo@tycho.nsa.gov with
the words "unsubscribe selinux" without quotes as the message.

Re: A filename to label translation daemon

[Russell Coker]

On Wed, 15 Aug 2012, Stephen Smalley <sds@tycho.nsa.gov> wrote:
> You should check that setfiles using glibc regex followed by setfiles
> using PCRE does not change anything on the filesystem, i.e. run the
> existing setfiles on all filesystems, then run the setfiles using PCRE
> with -nv to report any changes that would be made without applying them.

Of course that test would only really work for files that exist on the current 
filesystem.  But that should be a good enough approximation to be useful.

In terms of breakage in Debian, I'm not too bothered about that.  Before every 
release we need to do a lot of testing anyway.  The changes in this regard 
will be a lot less likely to break things than the regular churn of policy and 
daemon changes.

-- 
My Main Blog         http://etbe.coker.com.au/
My Documents Blog    http://doc.coker.com.au/

--
This message was distributed to subscribers of the selinux mailing list.
If you no longer wish to subscribe, send mail to majordomo@tycho.nsa.gov with
the words "unsubscribe selinux" without quotes as the message.

Re: A filename to label translation daemon

[Russell Coker]

On Tue, 14 Aug 2012, Eric Paris <eparis@parisplace.org> wrote:
> I have code that does just that.  Dan and I both wrote a version.
> I'll attach it.  I didn't find the speedups we were hoping for and it
> didn't work correctly/completely in the face of file context
> equivalencies.  Although that is likely fixable.  I was just looking
> at all of the stem code (and wondered who wrote it but it was
> pre-git).  I'm surprised it made a big difference.  Wouldn't the regex
> code be able to return extremely quickly if it didn't match?  Anyway.
> I'm writing some test programs to look at all of the possibilities.

At the time I wrote the code I didn't attempt to micro-benchmark it.  As the 
use case that mattered most was a full relabel of a root filesystem (between 
40,000 and 120,000 files in the common case) and the number of regexes was 
smaller than it is today the win was fairly obvious.

>From memory the stem compression change typically improved performance by a 
factor of 3-5 depending on what you were doing.  That was on systems like a 
Pentium-166 or P2-400.  The performance benefits on faster systems (800MHz 
Athlon and better) were less important to me at the time.

Prior to stem compression the relabel was entirely CPU bottlenecked on all 
systems.  After stem compression and some new CPU releases from Intel and AMD 
I wrote the code to have two processes doing disk IO (one stat'ing the files 
and the other applying labels) so that we could have some parallelism between 
regex checks and seeks on disk.

A theoretical regex library wouldn't be much slower than the integer checks we 
are doing, it could just use a 32bit integer compare to check the first 4 bytes 
which would compare to the integer stem check we do.  Does one of the current 
regex libraries do such optimisations where you have a common case of a non-
match on a relatively simple regex?

-- 
My Main Blog         http://etbe.coker.com.au/
My Documents Blog    http://doc.coker.com.au/


--
This message was distributed to subscribers of the selinux mailing list.
If you no longer wish to subscribe, send mail to majordomo@tycho.nsa.gov with
the words "unsubscribe selinux" without quotes as the message.

Re: A filename to label translation daemon

[Eric Paris]

[-- Attachment #1: Type: text/plain, Size: 2835 bytes --]

On Wed, Aug 15, 2012 at 1:56 AM, Russell Coker <russell@coker.com.au> wrote:
> On Tue, 14 Aug 2012, Eric Paris <eparis@parisplace.org> wrote:
>> I have code that does just that.  Dan and I both wrote a version.
>> I'll attach it.  I didn't find the speedups we were hoping for and it
>> didn't work correctly/completely in the face of file context
>> equivalencies.  Although that is likely fixable.  I was just looking
>> at all of the stem code (and wondered who wrote it but it was
>> pre-git).  I'm surprised it made a big difference.  Wouldn't the regex
>> code be able to return extremely quickly if it didn't match?  Anyway.
>> I'm writing some test programs to look at all of the possibilities.
>
> At the time I wrote the code I didn't attempt to micro-benchmark it.  As the
> use case that mattered most was a full relabel of a root filesystem (between
> 40,000 and 120,000 files in the common case) and the number of regexes was
> smaller than it is today the win was fairly obvious.
>
> From memory the stem compression change typically improved performance by a
> factor of 3-5 depending on what you were doing.  That was on systems like a
> Pentium-166 or P2-400.  The performance benefits on faster systems (800MHz
> Athlon and better) were less important to me at the time.
>
> Prior to stem compression the relabel was entirely CPU bottlenecked on all
> systems.  After stem compression and some new CPU releases from Intel and AMD
> I wrote the code to have two processes doing disk IO (one stat'ing the files
> and the other applying labels) so that we could have some parallelism between
> regex checks and seeks on disk.
>
> A theoretical regex library wouldn't be much slower than the integer checks we
> are doing, it could just use a 32bit integer compare to check the first 4 bytes
> which would compare to the integer stem check we do.  Does one of the current
> regex libraries do such optimisations where you have a common case of a non-
> match on a relatively simple regex?

Couldn't say.  But I can say that pcre seems to kick the crap out of
glibc at every turn.  My test programs aren't doing any of the stem
stuff.  So I can't give a perfect comparison yet.  I'll pull that in
today to see if it seems reasonable to drop all of it and simplify the
code.  My current testing is to run in a loop 100 times load db, look
up one semi-complex path (/var/www/html/cgi-bin/mail.pl), and free the
db.  The timing for each method is as follows:

glibc - 16.16 sec
pcre - 2.08 sec
pcre-mmap - 0.23 sec

I made up my own mmap format instead of adding glib and gVariant as a
build requirement.  I'm not opposed to it.  Us reusing code seems like
a very good idea, kinda weird to have that circular dependancy thing
though.  I'll play with that today as well.

You'll find my current test programs attached.
-Eric

[-- Attachment #2: glibc.c --]
[-- Type: text/x-csrc, Size: 2312 bytes --]

#include <regex.h>
#include <stdio.h>

#include "test-regex.h"

struct spec {
	char *context;
	char *regex;
	mode_t mode;
	regex_t reg;
};

static int process_file(FILE *context_file, unsigned int file_len, struct spec **out_spec)
{
	struct spec *specs;
	unsigned int line_num;
	char *line_buf = NULL;
	size_t line_len;
	ssize_t len;

	specs = calloc(file_len, sizeof(*specs));
	if (!specs) {
		perror("calloc");
		exit(EXIT_FAILURE);
	}

	line_num = 0;
	while ((len = getline(&line_buf, &line_len, context_file)) != -1) {
		char *context;
		char *mode;
		char *regex;
		int items, rc;

		items = sscanf(line_buf, "%ms %ms %ms", &regex, &mode, &context);
		if (items < 2 || items > 3) {
			fprintf(stderr, "invalid entry, skipping:%s", line_buf);
			continue;
		}

		if (items == 2) {
			context = mode;
			mode = NULL;
		}

		specs[line_num].context = context;
		specs[line_num].mode = string_to_mode(mode);
		free(mode);
		specs[line_num].regex = regex;

		rc = regcomp(&specs[line_num].reg, regex, REG_EXTENDED | REG_NOSUB);
		if (rc < 0)
			return rc;
		line_num++;
	}

	rewind(context_file);
	*out_spec = specs;
	return 0;
}

static int free_specs(struct spec *specs, unsigned int num_specs)
{
	unsigned int i;

	for (i = 0; i < num_specs; i++) {
		free(specs[i].context);
		free(specs[i].regex);
		regfree(&specs[i].reg);
	}

	free(specs);
	return 0;
}

static int test_match(struct spec *specs, unsigned int num_specs)
{
	unsigned int i;
	unsigned int matches = 0;
	int rc;

	for (i = 0; i < num_specs; i++) {
		rc = regexec(&specs[i].reg, TEST_PATH, 0, NULL, 0);
		if (rc == 0)
			matches++;
		else if (rc == REG_NOMATCH)
			continue;
		else
			return -1;
	}

	if (matches != TEST_PATH_MATCHES) {
		fprintf(stderr, "Found %d matches for %s\n", matches, TEST_PATH);
		return -1;
	}
	return 0;
}

int main(void)
{
	FILE *context_file;
	unsigned int num_specs;
	struct spec *specs;
	int rc, i;

	context_file = get_context_file();

	num_specs = lines_in_file(context_file);
	if (num_specs <= 0)
		exit(EXIT_FAILURE);

START;
	for (i = 0; i < NUM_RUNS; i++) {
		rc = process_file(context_file, num_specs, &specs);
		if (rc < 0)
			return rc;

		rc = test_match(specs, num_specs);
		if (rc < 0)
			return rc;

		rc = free_specs(specs, num_specs);
		if (rc < 0)
			return rc;
	}
STOP;
PRINTTIME;

	return 0;
}

[-- Attachment #3: Makefile --]
[-- Type: application/octet-stream, Size: 506 bytes --]

CFLAGS = -Wall -W -Werror -g
PCRE_LDFLAGS = -lpcre

INCLUDES = $(wildcard *.h)
DEPS = $(INCLUDES) Makefile

SRCS = $(wildcard *.c)
OBJS = $(SRCS:.c=)

all: $(OBJS)

clean:
	rm -rf $(OBJS) *.o

glibc: glibc.c $(DEPS)
	$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $<

pcre: pcre.c $(DEPS)
	$(CC) $(CFLAGS) $(LDFLAGS) $(PCRE_LDFLAGS) -o $@ $<

pcre-export: pcre-export.c $(DEPS)
	$(CC) $(CFLAGS) $(LDFLAGS) $(PCRE_LDFLAGS) -o $@ $<

pcre-import: pcre-import.c $(DEPS)
	$(CC) $(CFLAGS) $(LDFLAGS) $(PCRE_LDFLAGS) -o $@ $<

[-- Attachment #4: pcre.c --]
[-- Type: text/x-csrc, Size: 2739 bytes --]

#include <pcre.h>
#include <stdio.h>

#include "test-regex.h"

struct spec {
	char *context;
	char *regex;
	mode_t mode;
	pcre *re;
	pcre_extra *sd;
};

static int process_file(FILE *context_file, unsigned int file_len, struct spec **out_spec)
{
	struct spec *specs;
	unsigned int line_num;
	char *line_buf = NULL;
	size_t line_len;
	ssize_t len;

	specs = calloc(file_len, sizeof(*specs));
	if (!specs) {
		perror("calloc");
		exit(EXIT_FAILURE);
	}

	line_num = 0;
	while ((len = getline(&line_buf, &line_len, context_file)) != -1) {
		char *context;
		char *mode;
		char *regex;
		pcre *re;
		pcre_extra *sd;
		const char *err;
		int items, erroff;

		items = sscanf(line_buf, "%ms %ms %ms", &regex, &mode, &context);
		if (items < 2 || items > 3) {
			fprintf(stderr, "invalid entry, skipping:%s", line_buf);
			continue;
		}

		if (items == 2) {
			context = mode;
			mode = NULL;
		}

		specs[line_num].context=context;
		specs[line_num].mode = string_to_mode(mode);
		specs[line_num].regex = regex;

		re = pcre_compile(regex, 0, &err, &erroff, NULL);
		if (!re) {
			fprintf(stderr, "PCRE compilation failed for %s at offset %d: %s\n", regex, erroff, err);
			return -1;
		}
		specs[line_num].re = re;

		sd = pcre_study(re, 0, &err);
		if (!sd) {
			fprintf(stderr, "PCRE study failed for %s: %s\n", regex, err);
			return -1;
		}
		specs[line_num].sd = sd;

		line_num++;
	}

	rewind(context_file);
	*out_spec = specs;
	return 0;
}

static int test_match(struct spec *specs, unsigned int num_specs)
{
	unsigned int i;
	unsigned int matches = 0;
	int rc;

	for (i = 0; i < num_specs; i++) {
		pcre *re = specs[i].re;
		pcre_extra *sd = specs[i].sd;

		rc = pcre_exec(re, sd, TEST_PATH, strlen(TEST_PATH), 0, 0, NULL, 0);
		if (rc == 0)
			matches++;
		else if (rc == PCRE_ERROR_NOMATCH)
			continue;
		else
			return -1;
	}

	if (matches != TEST_PATH_MATCHES) {
		fprintf(stderr, "Found %d matches for %s\n", matches, TEST_PATH);
		return -1;
	}
	return 0;
}

static int free_specs(struct spec *specs, unsigned int num_specs)
{
	unsigned int i;

	for (i = 0; i < num_specs; i++) {
		free(specs[i].context);
		free(specs[i].regex);
		pcre_free(specs[i].re);
		pcre_free_study(specs[i].sd);
	}

	free(specs);
	return 0;
}

int main(void)
{
	FILE *context_file;
	unsigned int num_specs;
	struct spec *specs;
	int rc, i;

	context_file = get_context_file();

	num_specs = lines_in_file(context_file);
	if (num_specs <= 0)
		exit(EXIT_FAILURE);

START;
	for (i = 0; i < NUM_RUNS; i++) {
		rc = process_file(context_file, num_specs, &specs);
		if (rc < 0)
			return rc;

		rc = test_match(specs, num_specs);
		if (rc < 0)
			return rc;

		rc = free_specs(specs, num_specs);
		if (rc < 0)
			return rc;
	}
STOP;
PRINTTIME;

	return 0;
}

[-- Attachment #5: pcre-export.c --]
[-- Type: text/x-csrc, Size: 3983 bytes --]

#include <pcre.h>
#include <stdio.h>

#include "test-regex.h"

struct spec {
	char *context;
	char *regex;
	mode_t mode;
	pcre *re;
	pcre_extra *sd;
};

static int process_file(FILE *context_file, unsigned int file_len, struct spec **out_spec)
{
	struct spec *specs;
	unsigned int line_num;
	char *line_buf = NULL;
	size_t line_len;
	ssize_t len;

	specs = calloc(file_len, sizeof(*specs));
	if (!specs) {
		perror("calloc");
		exit(EXIT_FAILURE);
	}

	line_num = 0;
	while ((len = getline(&line_buf, &line_len, context_file)) != -1) {
		char *context;
		char *mode;
		char *regex;
		pcre *re;
		pcre_extra *sd;
		const char *err;
		int items, erroff;

		items = sscanf(line_buf, "%ms %ms %ms", &regex, &mode, &context);
		if (items < 2 || items > 3) {
			fprintf(stderr, "invalid entry, skipping:%s", line_buf);
			continue;
		}

		if (items == 2) {
			context = mode;
			mode = NULL;
		}

		specs[line_num].context=context;
		specs[line_num].mode = string_to_mode(mode);
		specs[line_num].regex = regex;

		re = pcre_compile(regex, 0, &err, &erroff, NULL);
		if (!re) {
			fprintf(stderr, "PCRE compilation failed for %s at offset %d: %s\n", regex, erroff, err);
			return -1;
		}
		specs[line_num].re = re;

		sd = pcre_study(re, 0, &err);
		if (!sd) {
			fprintf(stderr, "PCRE study failed for %s: %s\n", regex, err);
			return -1;
		}
		specs[line_num].sd = sd;

		line_num++;
	}

	rewind(context_file);
	*out_spec = specs;
	return 0;
}

static int write_binary_file(struct spec *specs, unsigned int num_specs)
{
	FILE *bin_file;
	size_t len;
	unsigned int magic = 0xdeadbeef;
	unsigned int i;

	bin_file = fopen(BIN_FILE_PATH, "w");
	if (!bin_file) {
		perror("fopen binfile");
		exit(EXIT_FAILURE);
	}

	len = fwrite(&magic, sizeof(magic), 1, bin_file);
	if (len != 1)
		return -1;

	len = fwrite(&num_specs, sizeof(num_specs), 1, bin_file);
	if (len != 1)
		return -1;

	for (i = 0; i < num_specs; i++) {
		char *context = specs[i].context;
		char *regex = specs[i].regex;
		mode_t mode = specs[i].mode;
		pcre *re = specs[i].re;
		pcre_extra *sd = specs[i].sd;
		size_t to_write, size;
		int rc;

		to_write = strlen(context) + 1;
		len = fwrite(&to_write, sizeof(to_write), 1, bin_file);
		if (len != 1)
			return -1;

		len = fwrite(context, sizeof(*context), to_write, bin_file);
		if (len != to_write)
			return -1;

		to_write = strlen(regex) + 1;
		len = fwrite(&to_write, sizeof(to_write), 1, bin_file);
		if (len != 1)
			return -1;

		len = fwrite(regex, sizeof(*regex), to_write, bin_file);
		if (len != to_write)
			return -1;

		len = fwrite(&mode, sizeof(mode), 1, bin_file);
		if (len != 1)
			return -1;

		rc = pcre_fullinfo(re, NULL, PCRE_INFO_SIZE, &size);
		if (rc < 0)
			return -1;

		to_write = size;
		len = fwrite(&to_write, sizeof(to_write), 1, bin_file);
		if (len != 1)
			return -1;

		len = fwrite(re, 1, to_write, bin_file);
		if (len != to_write)
			return -1;

		rc = pcre_fullinfo(re, sd, PCRE_INFO_STUDYSIZE, &size);
		if (rc < 0)
			return -1;

		to_write = size;
		len = fwrite(&to_write, sizeof(to_write), 1, bin_file);
		if (len != 1)
			return -1;

		len = fwrite(sd->study_data, 1, to_write, bin_file);
		if (len != to_write)
			return -1;
	}
	return 0;
}

static int free_specs(struct spec *specs, unsigned int num_specs)
{
	unsigned int i;

	for (i = 0; i < num_specs; i++) {
		free(specs[i].context);
		free(specs[i].regex);
		pcre_free(specs[i].re);
		pcre_free_study(specs[i].sd);
	}

	free(specs);
	return 0;
}

int main(void)
{
	FILE *context_file;
	unsigned int num_specs;
	struct spec *specs;
	int rc, i;

	context_file = get_context_file();

	num_specs = lines_in_file(context_file);
	if (num_specs <= 0)
		exit(EXIT_FAILURE);

START;
	for (i = 0; i < NUM_RUNS; i++) {
		rc = process_file(context_file, num_specs, &specs);
		if (rc < 0)
			return rc;

		rc = write_binary_file(specs, num_specs);
		if (rc < 0)
			return rc;

		rc = free_specs(specs, num_specs);
		if (rc < 0)
			return rc;
	}
STOP;
PRINTTIME;

	return 0;
}

[-- Attachment #6: pcre-import.c --]
[-- Type: text/x-csrc, Size: 2864 bytes --]

#include <fcntl.h>
#include <pcre.h>
#include <stdio.h>
#include <unistd.h>

#include <sys/mman.h>
#include <sys/stat.h>
#include <sys/types.h>

#include "test-regex.h"

struct spec_data {
	void *addr;
	size_t len;
};

struct spec {
	char *context;
	char *regex;
	mode_t *mode;
	pcre *re;
	pcre_extra sd;
};

static int read_binary_file(struct spec **out_specs, unsigned int *out_num_specs, struct spec_data *data)
{
	int fd, rc;
	size_t len;
	size_t *plen;
	unsigned int *magic;
	unsigned int i;
	unsigned int *num_specs;
	struct spec *specs;
	struct stat stat;
	char *addr;

	fd = open(BIN_FILE_PATH, O_RDONLY);
	if (fd < 0)
		return -1;

	rc = fstat(fd, &stat);
	if (rc < 0)
		return -1;

	len = stat.st_size;
	len += (sysconf(_SC_PAGE_SIZE) - 1);
	len &= ~(sysconf(_SC_PAGE_SIZE) - 1);

	addr = mmap(NULL, len, PROT_READ, MAP_PRIVATE, fd, 0);
	if (addr == MAP_FAILED) {
		perror("mmap");
		return -1;
	}

	data->addr = addr;
	data->len = len;
	close(fd);

	magic = (unsigned int *)addr;
	if (*magic != 0xdeadbeef)
		return -1;
	addr += sizeof(*magic);

	num_specs = (unsigned int *)addr;
	addr += sizeof(*num_specs);

	specs = calloc(*num_specs, sizeof(*specs));
	if (!specs)
		return -1;

	for (i = 0; i < *num_specs; i++) {
		plen = (size_t *)addr;
		addr += sizeof(*plen);
		specs[i].context = (char *)addr;
		addr += *plen;

		plen = (size_t *)addr;
		addr += sizeof(*plen);
		specs[i].regex = (char *)addr;
		addr += *plen;

		specs[i].mode = (mode_t *)addr;
		addr += sizeof(*specs[i].mode);

		plen = (size_t *)addr;
		addr += sizeof(*plen);
		specs[i].re = (pcre *)addr;
		addr += *plen;

		plen = (size_t *)addr;
		addr += sizeof(*plen);
		specs[i].sd.study_data = (void *)addr;
		specs[i].sd.flags |= PCRE_EXTRA_STUDY_DATA;
		addr += *plen;
	}
	
	*out_num_specs = *num_specs;
	*out_specs = specs;
	return 0;
}

static int test_match(struct spec *specs, unsigned int num_specs)
{
	unsigned int i;
	unsigned int matches = 0;
	int rc;

	for (i = 0; i < num_specs; i++) {
		pcre *re = specs[i].re;
		pcre_extra *sd = &specs[i].sd;

		rc = pcre_exec(re, sd, TEST_PATH, strlen(TEST_PATH), 0, 0, NULL, 0);
		if (rc == 0)
			matches++;
		else if (rc == PCRE_ERROR_NOMATCH)
			continue;
		else
			return -1;
	}

	if (matches != TEST_PATH_MATCHES) {
		fprintf(stderr, "Found %d matches for %s\n", matches, TEST_PATH);
		return -1;
	}
	return 0;
}

static int free_specs(struct spec *specs, struct spec_data *data)
{
	free(specs);
	munmap(data->addr, data->len);

	return 0;
}

int main(void)
{
	unsigned int num_specs;
	struct spec *specs;
	struct spec_data data;
	int rc, i;

START;
	for (i = 0; i < NUM_RUNS; i++) {
		rc = read_binary_file(&specs, &num_specs, &data);
		if (rc < 0)
			return rc;

		rc = test_match(specs, num_specs);
		if (rc < 0)
			return rc;

		rc = free_specs(specs, &data);
		if (rc < 0)
			return rc;
	}
STOP;
PRINTTIME;

	return 0;
}

[-- Attachment #7: test-regex.h --]
[-- Type: text/x-chdr, Size: 1666 bytes --]

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>

#include <sys/stat.h>

clock_t startm, stopm;
#define START if ( (startm = clock()) == -1) {printf("Error calling clock");exit(1);}
#define STOP if ( (stopm = clock()) == -1) {printf("Error calling clock");exit(1);}
#define PRINTTIME printf( "%6.3f seconds used by the processor.\n", ((double)stopm-startm)/CLOCKS_PER_SEC);

#define CONTEXT_PATH "/etc/selinux/targeted/contexts/files/file_contexts"
#define BIN_FILE_PATH "bin_file"
#define NUM_RUNS 100

#define TEST_PATH "/var/www/html/cgi-bin/mail.pl"
#define TEST_PATH_MATCHES 7

static inline void usage(char *prog)
{
	fprintf(stderr, "usage: %s\n", prog);
	exit(EXIT_FAILURE);
}

static inline FILE *get_context_file(void)
{
	FILE *f;

	f = fopen(CONTEXT_PATH, "r");
	if (!f) {
		perror("fopen");
		exit(EXIT_FAILURE);
	}

	return f;
}

static inline unsigned int lines_in_file(FILE *f)
{
	unsigned int lines = 0;
	int ch;

	while (EOF != (ch=fgetc(f)))
	if (ch=='\n')
		lines++;

	rewind(f);
	return lines;
}

static inline mode_t string_to_mode(char *smode)
{
	mode_t mode = 0;
	size_t len;

	if (!smode)
		return mode;

	len = strlen(smode);
	if (len != 2 || smode[0] != '-') {
		fprintf(stderr, "Illegal file type %s\n", smode);
		return -1;
	}

	switch (smode[1]) {
	case 'b':
		mode = S_IFBLK;
		break;
	case 'c':
		mode = S_IFCHR;
		break;
	case 'd':
		mode = S_IFDIR;
		break;
	case 'p':
		mode = S_IFIFO;
		break;
	case 'l':
		mode = S_IFLNK;
		break;
	case 's':
		mode = S_IFSOCK;
		break;
	case '-':
		mode = S_IFREG;
		break;
	default:
		fprintf(stderr, "Illegal file type %s\n", smode);
		return -1;
	}

	return mode;
}

Re: A filename to label translation daemon

[Eric Paris]

Latest update was trying to decide if keeping Russell's stem work was
worth the code complexity given the speedups I can get just moving to
pcre and using a precompiled data format.  I'm thinking they aren't as
huge a win, but it's still clearly a win.  My testing is:

start timer
for (i = 0; i < 100; i++)
    initialize db;
    look up one path;
    clean up db
stop timer

So the stems code will help even more if you do more than 1 lookup per
db load.  Results at the moment for that testing looks like:

 $ ./glibc
13.390 seconds used by the processor.
$ ./glibc-stems
 8.750 seconds used by the processor.
$ ./pcre
 2.160 seconds used by the processor.
$ ./pcre-stems
 2.040 seconds used by the processor.
$ ./pcre-mmap
 0.230 seconds used by the processor.
$ ./pcre-mmap-stems
 0.170 seconds used by the processor.

So we're talking about a x50 speedup here for my best case vs
something analogous to what we are doing today.

-Eric

--
This message was distributed to subscribers of the selinux mailing list.
If you no longer wish to subscribe, send mail to majordomo@tycho.nsa.gov with
the words "unsubscribe selinux" without quotes as the message.