[PATCH 0/6] dump macros definitions

[PATCH 0/6] dump macros definitions

[Luc Van Oostenryck]

The goal of this series is to add support for GCC's -dD.

This implementation is quite straightforward:
- record all macro's name
- dump the tokens corresponding to each macros
  without argument expansion.

This series is available at:
	git://github.com/lucvoo/sparse.git dump-macros
based on commit:
	14964df5373292af78b29529d4fc7e1a26b67a97 (sparse-next @ 2017-03-31)
up to commit:
	9a01ec7d42d6013ead12cb111b1a1879d0c856ab


Luc Van Oostenryck (6):
  define ident_list
  teach sparse how to dump macro definitions
  teach sparse how to handle -dD flag
  let -dD report macro defintions
  let -dD report #undef too
  fix -dD for never-defined #undef

 lib.c                                       | 21 ++++++++++
 lib.h                                       |  9 +++++
 pre-process.c                               | 61 ++++++++++++++++++++++++++++-
 validation/empty-file                       |  0
 validation/preprocessor/dump-macros-empty.c |  7 ++++
 validation/preprocessor/dump-macros-multi.c |  7 ++++
 validation/preprocessor/dump-macros.c       | 23 +++++++++++
 7 files changed, 126 insertions(+), 2 deletions(-)
 create mode 100644 validation/empty-file
 create mode 100644 validation/preprocessor/dump-macros-empty.c
 create mode 100644 validation/preprocessor/dump-macros-multi.c
 create mode 100644 validation/preprocessor/dump-macros.c


-- Luc Van Oostenryck

[PATCH 1/6] define ident_list

[Luc Van Oostenryck]

Signed-off-by: Luc Van Oostenryck <luc.vanoostenryck@gmail.com>
---
 lib.h | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/lib.h b/lib.h
index 134e56040..19b5cb0b3 100644
--- a/lib.h
+++ b/lib.h
@@ -75,6 +75,7 @@ DECLARE_PTR_LIST(basic_block_list, struct basic_block);
 DECLARE_PTR_LIST(instruction_list, struct instruction);
 DECLARE_PTR_LIST(multijmp_list, struct multijmp);
 DECLARE_PTR_LIST(pseudo_list, struct pseudo);
+DECLARE_PTR_LIST(ident_list, struct ident);
 DECLARE_PTR_LIST(string_list, char);
 
 typedef struct pseudo *pseudo_t;
@@ -244,6 +245,11 @@ static inline void add_expression(struct expression_list **list, struct expressi
 	add_ptr_list(list, expr);
 }
 
+static inline void add_ident(struct ident_list **list, struct ident *ident)
+{
+	add_ptr_list(list, ident);
+}
+
 #define hashval(x) ((unsigned long)(x))
 
 #endif
-- 
2.12.0

[PATCH 2/6] teach sparse how to dump macro definitions

[Luc Van Oostenryck]

Signed-off-by: Luc Van Oostenryck <luc.vanoostenryck@gmail.com>
---
 lib.h         |  1 +
 pre-process.c | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+)

diff --git a/lib.h b/lib.h
index 19b5cb0b3..f2df4aa1d 100644
--- a/lib.h
+++ b/lib.h
@@ -141,6 +141,7 @@ extern int arch_m64;
 
 extern void declare_builtin_functions(void);
 extern void create_builtin_stream(void);
+extern void dump_macro_definitions(void);
 extern struct symbol_list *sparse_initialize(int argc, char **argv, struct string_list **files);
 extern struct symbol_list *__sparse(char *filename);
 extern struct symbol_list *sparse_keep_tokens(char *filename);
diff --git a/pre-process.c b/pre-process.c
index 7c57ba1cd..74414dfeb 100644
--- a/pre-process.c
+++ b/pre-process.c
@@ -44,6 +44,7 @@
 #include "expression.h"
 #include "scope.h"
 
+static struct ident_list *macros;	// only needed for -dD
 static int false_nesting = 0;
 static int counter_macro = 0;		// __COUNTER__ expansion
 
@@ -1351,6 +1352,7 @@ static int do_handle_define(struct stream *stream, struct token **line, struct t
 	if (!sym || sym->scope != file_scope) {
 		sym = alloc_symbol(left->pos, SYM_NODE);
 		bind_symbol(sym, name, NS_MACRO);
+		add_ident(&macros, name);
 		ret = 0;
 	}
 
@@ -2010,3 +2012,56 @@ struct token * preprocess(struct token *token)
 
 	return token;
 }
+
+static void dump_macro(struct symbol *sym)
+{
+	int nargs = sym->arglist ? sym->arglist->count.normal : 0;
+	struct token *args[nargs];
+	struct token *token;
+
+	printf("#define %s", show_ident(sym->ident));
+	token = sym->arglist;
+	if (token) {
+		const char *sep = "";
+		int narg = 0;
+		putchar('(');
+		for (; !eof_token(token); token = token->next) {
+			if (token_type(token) == TOKEN_ARG_COUNT)
+				continue;
+			printf("%s%s", sep, show_token(token));
+			args[narg++] = token;
+			sep = ", ";
+		}
+		putchar(')');
+	}
+	putchar(' ');
+
+	token = sym->expansion;
+	while (!eof_token(token)) {
+		struct token *next = token->next;
+		switch (token_type(token)) {
+		case TOKEN_UNTAINT:
+			break;
+		case TOKEN_MACRO_ARGUMENT:
+			token = args[token->argnum];
+			/* fall-through */
+		default:
+			printf("%s", show_token(token));
+			if (next->pos.whitespace)
+				putchar(' ');
+		}
+		token = next;
+	}
+	putchar('\n');
+}
+
+void dump_macro_definitions(void)
+{
+	struct ident *name;
+
+	FOR_EACH_PTR(macros, name) {
+		struct symbol *sym = lookup_macro(name);
+		if (sym)
+			dump_macro(sym);
+	} END_FOR_EACH_PTR(name);
+}
-- 
2.12.0

Re: [PATCH 2/6] teach sparse how to dump macro definitions

[Christopher Li]

On Tue, Apr 4, 2017 at 2:49 PM, Luc Van Oostenryck
<luc.vanoostenryck@gmail.com> wrote:
> diff --git a/pre-process.c b/pre-process.c
> index 7c57ba1cd..74414dfeb 100644
> --- a/pre-process.c
> +++ b/pre-process.c
> @@ -44,6 +44,7 @@
>  #include "expression.h"
>  #include "scope.h"
>
> +static struct ident_list *macros;      // only needed for -dD

I think the macros list should be initialized as NULL.

Chris

Re: [PATCH 2/6] teach sparse how to dump macro definitions

[Luc Van Oostenryck]

On Thu, Jun 1, 2017 at 8:56 AM, Christopher Li <sparse@chrisli.org> wrote:
> On Tue, Apr 4, 2017 at 2:49 PM, Luc Van Oostenryck
> <luc.vanoostenryck@gmail.com> wrote:
>> diff --git a/pre-process.c b/pre-process.c
>> index 7c57ba1cd..74414dfeb 100644
>> --- a/pre-process.c
>> +++ b/pre-process.c
>> @@ -44,6 +44,7 @@
>>  #include "expression.h"
>>  #include "scope.h"
>>
>> +static struct ident_list *macros;      // only needed for -dD
>
> I think the macros list should be initialized as NULL.

It certainly doesn't need to be explicitly initialized to NULL since
all top-level variables that are not explicitly initialized are initialized
to zero by the environment before main() is called.

-- Luc

[PATCH 3/6] teach sparse how to handle -dD flag

[Luc Van Oostenryck]

Signed-off-by: Luc Van Oostenryck <luc.vanoostenryck@gmail.com>
---
 lib.c | 16 ++++++++++++++++
 lib.h |  2 ++
 2 files changed, 18 insertions(+)

diff --git a/lib.c b/lib.c
index 272d2c88a..991ead9bd 100644
--- a/lib.c
+++ b/lib.c
@@ -250,6 +250,8 @@ int Wvla = 1;
 int dbg_entry = 0;
 int dbg_dead = 0;
 
+int dump_macro_defs = 0;
+
 int preprocess_only;
 
 static enum { STANDARD_C89,
@@ -567,6 +569,19 @@ static char **handle_switch_v(char *arg, char **next)
 	return next;
 }
 
+static struct warning dumps[] = {
+	{ "D", &dump_macro_defs},
+};
+
+static char **handle_switch_d(char *arg, char **next)
+{
+	char ** ret = handle_onoff_switch(arg, next, dumps, ARRAY_SIZE(dumps));
+	if (ret)
+		return ret;
+
+	return next;
+}
+
 
 static void handle_onoff_switch_finalize(const struct warning warnings[], int n)
 {
@@ -800,6 +815,7 @@ static char **handle_switch(char *arg, char **next)
 	switch (*arg) {
 	case 'a': return handle_switch_a(arg, next);
 	case 'D': return handle_switch_D(arg, next);
+	case 'd': return handle_switch_d(arg, next);
 	case 'E': return handle_switch_E(arg, next);
 	case 'f': return handle_switch_f(arg, next);
 	case 'g': return handle_switch_g(arg, next);
diff --git a/lib.h b/lib.h
index f2df4aa1d..1b21de0c2 100644
--- a/lib.h
+++ b/lib.h
@@ -137,6 +137,8 @@ extern int Wvla;
 extern int dbg_entry;
 extern int dbg_dead;
 
+extern int dump_macro_defs;
+
 extern int arch_m64;
 
 extern void declare_builtin_functions(void);
-- 
2.12.0

[PATCH 4/6] let -dD report macro defintions

[Luc Van Oostenryck]

Signed-off-by: Luc Van Oostenryck <luc.vanoostenryck@gmail.com>
---
 lib.c                                       |  5 +++++
 validation/empty-file                       |  0
 validation/preprocessor/dump-macros-empty.c |  7 +++++++
 validation/preprocessor/dump-macros-multi.c |  7 +++++++
 validation/preprocessor/dump-macros.c       | 18 ++++++++++++++++++
 5 files changed, 37 insertions(+)
 create mode 100644 validation/empty-file
 create mode 100644 validation/preprocessor/dump-macros-empty.c
 create mode 100644 validation/preprocessor/dump-macros-multi.c
 create mode 100644 validation/preprocessor/dump-macros.c

diff --git a/lib.c b/lib.c
index 991ead9bd..700681c01 100644
--- a/lib.c
+++ b/lib.c
@@ -1075,9 +1075,14 @@ void create_builtin_stream(void)
 
 static struct symbol_list *sparse_tokenstream(struct token *token)
 {
+	int builtin = token && !token->pos.stream;
+
 	// Preprocess the stream
 	token = preprocess(token);
 
+	if (dump_macro_defs && !builtin)
+		dump_macro_definitions();
+
 	if (preprocess_only) {
 		while (!eof_token(token)) {
 			int prec = 1;
diff --git a/validation/empty-file b/validation/empty-file
new file mode 100644
index 000000000..e69de29bb
diff --git a/validation/preprocessor/dump-macros-empty.c b/validation/preprocessor/dump-macros-empty.c
new file mode 100644
index 000000000..672c66c71
--- /dev/null
+++ b/validation/preprocessor/dump-macros-empty.c
@@ -0,0 +1,7 @@
+/*
+ * check-name: dump-macros with empty file
+ * check-command: sparse -E -dD empty-file
+ *
+ * check-output-ignore
+check-output-pattern-1-times: #define __CHECKER__ 1
+ */
diff --git a/validation/preprocessor/dump-macros-multi.c b/validation/preprocessor/dump-macros-multi.c
new file mode 100644
index 000000000..2f6e8d04f
--- /dev/null
+++ b/validation/preprocessor/dump-macros-multi.c
@@ -0,0 +1,7 @@
+/*
+ * check-name: dump-macros with multiple files
+ * check-command: sparse -E -dD empty-file $file
+ *
+ * check-output-ignore
+check-output-pattern-2-times: #define __CHECKER__ 1
+ */
diff --git a/validation/preprocessor/dump-macros.c b/validation/preprocessor/dump-macros.c
new file mode 100644
index 000000000..79f3de6a2
--- /dev/null
+++ b/validation/preprocessor/dump-macros.c
@@ -0,0 +1,18 @@
+#define ABC abc
+#undef ABC
+
+#define	DEF def
+#undef DEF
+#define DEF xyz
+
+#define NYDEF ydef
+/*
+ * check-name: dump-macros
+ * check-command: sparse -E -dD -DIJK=ijk -UNDEF -UNYDEF $file
+ *
+ * check-output-ignore
+check-output-pattern-1-times: #define __CHECKER__ 1
+check-output-contains: #define IJK ijk
+check-output-contains: #define DEF xyz
+check-output-contains: #define NYDEF ydef
+ */
-- 
2.12.0

[PATCH 5/6] let -dD report #undef too

[Luc Van Oostenryck]

Signed-off-by: Luc Van Oostenryck <luc.vanoostenryck@gmail.com>
---
 pre-process.c                         | 3 +++
 validation/preprocessor/dump-macros.c | 1 +
 2 files changed, 4 insertions(+)

diff --git a/pre-process.c b/pre-process.c
index 74414dfeb..63df3f71c 100644
--- a/pre-process.c
+++ b/pre-process.c
@@ -1406,6 +1406,7 @@ static int do_handle_undef(struct stream *stream, struct token **line, struct to
 	if (!sym || sym->scope != file_scope) {
 		sym = alloc_symbol(left->pos, SYM_NODE);
 		bind_symbol(sym, left->ident, NS_MACRO);
+		add_ident(&macros, left->ident);
 	}
 
 	sym->namespace = NS_UNDEF;
@@ -2063,5 +2064,7 @@ void dump_macro_definitions(void)
 		struct symbol *sym = lookup_macro(name);
 		if (sym)
 			dump_macro(sym);
+		else
+			printf("#undef %s\n", show_ident(name));
 	} END_FOR_EACH_PTR(name);
 }
diff --git a/validation/preprocessor/dump-macros.c b/validation/preprocessor/dump-macros.c
index 79f3de6a2..5508233aa 100644
--- a/validation/preprocessor/dump-macros.c
+++ b/validation/preprocessor/dump-macros.c
@@ -15,4 +15,5 @@ check-output-pattern-1-times: #define __CHECKER__ 1
 check-output-contains: #define IJK ijk
 check-output-contains: #define DEF xyz
 check-output-contains: #define NYDEF ydef
+check-output-contains: #undef ABC
  */
-- 
2.12.0

[PATCH 6/6] fix -dD for never-defined #undef

[Luc Van Oostenryck]

Note: to support this, we have to create a symbol where
      it wasn't needed before.

Signed-off-by: Luc Van Oostenryck <luc.vanoostenryck@gmail.com>
---
 pre-process.c                         | 3 +--
 validation/preprocessor/dump-macros.c | 6 +++++-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/pre-process.c b/pre-process.c
index 63df3f71c..5067b59b2 100644
--- a/pre-process.c
+++ b/pre-process.c
@@ -1400,8 +1400,7 @@ static int do_handle_undef(struct stream *stream, struct token **line, struct to
 			return 1;
 		if (attr == sym->attr && sym->namespace == NS_UNDEF)
 			return 1;
-	} else if (attr <= SYM_ATTR_NORMAL)
-		return 1;
+	}
 
 	if (!sym || sym->scope != file_scope) {
 		sym = alloc_symbol(left->pos, SYM_NODE);
diff --git a/validation/preprocessor/dump-macros.c b/validation/preprocessor/dump-macros.c
index 5508233aa..a10325171 100644
--- a/validation/preprocessor/dump-macros.c
+++ b/validation/preprocessor/dump-macros.c
@@ -6,9 +6,11 @@
 #define DEF xyz
 
 #define NYDEF ydef
+
+#undef	NDEF2
 /*
  * check-name: dump-macros
- * check-command: sparse -E -dD -DIJK=ijk -UNDEF -UNYDEF $file
+ * check-command: sparse -E -dD -DIJK=ijk -UNDEF1 -UNYDEF $file
  *
  * check-output-ignore
 check-output-pattern-1-times: #define __CHECKER__ 1
@@ -16,4 +18,6 @@ check-output-contains: #define IJK ijk
 check-output-contains: #define DEF xyz
 check-output-contains: #define NYDEF ydef
 check-output-contains: #undef ABC
+check-output-contains: #undef NDEF1
+check-output-contains: #undef NDEF2
  */
-- 
2.12.0

Re: [PATCH 0/6] dump macros definitions

[Christopher Li]

On Tue, Apr 4, 2017 at 2:49 PM, Luc Van Oostenryck
<luc.vanoostenryck@gmail.com> wrote:
> The goal of this series is to add support for GCC's -dD.
>
> This implementation is quite straightforward:
> - record all macro's name
> - dump the tokens corresponding to each macros
>   without argument expansion.
>
> This series is available at:
>         git://github.com/lucvoo/sparse.git dump-macros
> based on commit:
>         14964df5373292af78b29529d4fc7e1a26b67a97 (sparse-next @ 2017-03-31)
> up to commit:
>         9a01ec7d42d6013ead12cb111b1a1879d0c856ab

Just a heads up this week I am very occupied, don't have much
time for sparse at all. I will resume sparse hacking in the week
end. Hopefully make some progress on sparse-next.

Sorry for the delay guys.

Chris

Re: [PATCH 0/6] dump macros definitions

[Luc Van Oostenryck]

On Thu, Apr 6, 2017 at 10:05 AM, Christopher Li <sparse@chrisli.org> wrote:
> Just a heads up this week I am very occupied, don't have much
> time for sparse at all. I will resume sparse hacking in the week
> end. Hopefully make some progress on sparse-next.
>
> Sorry for the delay guys.
>
> Chris

No problem.

-- Luc