* [LTP] [PATCH v2 1/7] docparse: Implement #define and #include
2021-11-01 14:53 [LTP] [PATCH v2 0/7] docparse improvements Cyril Hrubis
@ 2021-11-01 14:53 ` Cyril Hrubis
2021-11-02 10:05 ` Richard Palethorpe
2021-11-01 14:53 ` [LTP] [PATCH v2 2/7] docparse: Add tests Cyril Hrubis
` (6 subsequent siblings)
7 siblings, 1 reply; 27+ messages in thread
From: Cyril Hrubis @ 2021-11-01 14:53 UTC (permalink / raw)
To: ltp
We ignore most of the include statements and we attempt to parse only
header files that reside in the same directory as the test source code,
that is since we are not interested in any system or library headers as
we are only looking for constants used in the tst_test structure that
are always either directly in the test source or in header in the same
directory.
The macro support is very simple as well, it's a single pass as we are
not interested in intricate macros. We just need values for constants
that are used in the tst_test structure intializations.
+ Also add -v verbose mode that prints included files and defined macros
Signed-off-by: Cyril Hrubis <chrubis@suse.cz>
---
docparse/docparse.c | 234 ++++++++++++++++++++++++++++++++++++++++++--
1 file changed, 224 insertions(+), 10 deletions(-)
diff --git a/docparse/docparse.c b/docparse/docparse.c
index 8cd0d0eef..64f9d08d9 100644
--- a/docparse/docparse.c
+++ b/docparse/docparse.c
@@ -1,9 +1,12 @@
// SPDX-License-Identifier: GPL-2.0-or-later
/*
- * Copyright (c) 2019 Cyril Hrubis <chrubis@suse.cz>
+ * Copyright (c) 2019-2021 Cyril Hrubis <chrubis@suse.cz>
* Copyright (c) 2020 Petr Vorel <pvorel@suse.cz>
*/
+#define _GNU_SOURCE
+
+#include <search.h>
#include <stdio.h>
#include <string.h>
#include <libgen.h>
@@ -12,6 +15,9 @@
#include "data_storage.h"
+static int verbose;
+static char *includepath;
+
#define WARN(str) fprintf(stderr, "WARNING: " str "\n")
static void oneline_comment(FILE *f)
@@ -126,7 +132,7 @@ static void maybe_comment(FILE *f, struct data_node *doc)
}
}
-const char *next_token(FILE *f, struct data_node *doc)
+static char *next_token(FILE *f, struct data_node *doc)
{
size_t i = 0;
static char buf[4096];
@@ -159,6 +165,7 @@ const char *next_token(FILE *f, struct data_node *doc)
case ',':
case '[':
case ']':
+ case '#':
if (i) {
ungetc(c, f);
goto exit;
@@ -197,6 +204,46 @@ exit:
return buf;
}
+static FILE *open_include(const char *includepath, FILE *f)
+{
+ char buf[256];
+ char *path;
+ FILE *inc;
+
+ if (!fscanf(f, "%s\n", buf))
+ return NULL;
+
+ if (buf[0] != '"')
+ return NULL;
+
+ char *filename = buf + 1;
+
+ if (!buf[0])
+ return NULL;
+
+ filename[strlen(filename)-1] = 0;
+
+ if (asprintf(&path, "%s/%s", includepath, filename) < 0)
+ return NULL;
+
+ inc = fopen(path, "r");
+
+ if (inc && verbose)
+ fprintf(stderr, "INCLUDE %s\n", path);
+
+ free(path);
+
+ return inc;
+}
+
+static void close_include(FILE *inc)
+{
+ if (verbose)
+ fprintf(stderr, "INCLUDE END\n");
+
+ fclose(inc);
+}
+
static int parse_array(FILE *f, struct data_node *node)
{
const char *token;
@@ -234,9 +281,28 @@ static int parse_array(FILE *f, struct data_node *node)
return 0;
}
+static void try_apply_macro(char **res)
+{
+ ENTRY macro = {
+ .key = *res,
+ };
+
+ ENTRY *ret;
+
+ ret = hsearch(macro, FIND);
+
+ if (!ret)
+ return;
+
+ if (verbose)
+ fprintf(stderr, "APPLYING MACRO %s=%s\n", ret->key, (char*)ret->data);
+
+ *res = ret->data;
+}
+
static int parse_test_struct(FILE *f, struct data_node *doc, struct data_node *node)
{
- const char *token;
+ char *token;
char *id = NULL;
int state = 0;
struct data_node *ret;
@@ -280,6 +346,7 @@ static int parse_test_struct(FILE *f, struct data_node *doc, struct data_node *n
ret = data_node_array();
parse_array(f, ret);
} else {
+ try_apply_macro(&token);
ret = data_node_string(token);
}
@@ -302,6 +369,114 @@ static const char *tokens[] = {
"{",
};
+static void macro_get_string(FILE *f, char *buf, char *buf_end)
+{
+ int c;
+
+ for (;;) {
+ c = fgetc(f);
+
+ switch (c) {
+ case '"':
+ case EOF:
+ *buf = 0;
+ return;
+ default:
+ if (buf < buf_end)
+ *(buf++) = c;
+ }
+ }
+}
+
+static void macro_get_val(FILE *f, char *buf, size_t buf_len)
+{
+ int c, prev = 0;
+ char *buf_end = buf + buf_len - 1;
+
+ c = fgetc(f);
+ if (c == '"') {
+ macro_get_string(f, buf, buf_end);
+ return;
+ }
+
+ for (;;) {
+ switch (c) {
+ case '\n':
+ if (prev == '\\') {
+ buf--;
+ } else {
+ *buf = 0;
+ return;
+ }
+ break;
+ case EOF:
+ *buf = 0;
+ return;
+ case ' ':
+ case '\t':
+ break;
+ default:
+ if (buf < buf_end)
+ *(buf++) = c;
+ }
+
+ prev = c;
+ c = fgetc(f);
+ }
+}
+
+static void parse_macro(FILE *f)
+{
+ char name[128];
+ char val[256];
+
+ if (!fscanf(f, "%s[^\n]", name))
+ return;
+
+ if (fgetc(f) == '\n')
+ return;
+
+ macro_get_val(f, val, sizeof(val));
+
+ ENTRY e = {
+ .key = strdup(name),
+ .data = strdup(val),
+ };
+
+ if (verbose)
+ fprintf(stderr, " MACRO %s=%s\n", e.key, (char*)e.data);
+
+ hsearch(e, ENTER);
+}
+
+static void parse_include_macros(FILE *f)
+{
+ FILE *inc;
+ const char *token;
+ int hash = 0;
+
+ inc = open_include(includepath, f);
+ if (!inc)
+ return;
+
+ while ((token = next_token(inc, NULL))) {
+ if (token[0] == '#') {
+ hash = 1;
+ continue;
+ }
+
+ if (!hash)
+ continue;
+
+ if (!strcmp(token, "define"))
+ parse_macro(inc);
+
+ hash = 0;
+ }
+
+ close_include(inc);
+}
+
static struct data_node *parse_file(const char *fname)
{
int state = 0, found = 0;
@@ -314,14 +489,28 @@ static struct data_node *parse_file(const char *fname)
FILE *f = fopen(fname, "r");
+ includepath = dirname(strdup(fname));
+
struct data_node *res = data_node_hash();
struct data_node *doc = data_node_array();
while ((token = next_token(f, doc))) {
- if (state < 6 && !strcmp(tokens[state], token))
+ if (state < 6 && !strcmp(tokens[state], token)) {
state++;
- else
+ } else {
+ if (token[0] == '#') {
+ token = next_token(f, doc);
+ if (token) {
+ if (!strcmp(token, "define"))
+ parse_macro(f);
+
+ if (!strcmp(token, "include"))
+ parse_include_macros(f);
+ }
+ }
+
state = 0;
+ }
if (state < 6)
continue;
@@ -386,17 +575,42 @@ const char *strip_name(char *path)
return name;
}
+static void print_help(const char *prgname)
+{
+ printf("usage: %s [-vh] input.c\n\n", prgname);
+ printf("-v sets verbose mode\n");
+ printf("-h prints this help\n\n");
+ exit(0);
+}
+
int main(int argc, char *argv[])
{
unsigned int i, j;
struct data_node *res;
+ int opt;
+
+ while ((opt = getopt(argc, argv, "hv")) != -1) {
+ switch (opt) {
+ case 'h':
+ print_help(argv[0]);
+ break;
+ case 'v':
+ verbose = 1;
+ break;
+ }
+ }
+
+ if (optind >= argc) {
+ fprintf(stderr, "No input filename.c\n");
+ return 1;
+ }
- if (argc != 2) {
- fprintf(stderr, "Usage: docparse filename.c\n");
+ if (!hcreate(128)) {
+ fprintf(stderr, "Failed to initialize hash table\n");
return 1;
}
- res = parse_file(argv[1]);
+ res = parse_file(argv[optind]);
if (!res)
return 0;
@@ -425,8 +639,8 @@ int main(int argc, char *argv[])
}
}
- data_node_hash_add(res, "fname", data_node_string(argv[1]));
- printf(" \"%s\": ", strip_name(argv[1]));
+ data_node_hash_add(res, "fname", data_node_string(argv[optind]));
+ printf(" \"%s\": ", strip_name(argv[optind]));
data_to_json(res, stdout, 2);
data_node_free(res);
--
2.32.0
--
Mailing list info: https://lists.linux.it/listinfo/ltp
^ permalink raw reply related [flat|nested] 27+ messages in thread* Re: [LTP] [PATCH v2 1/7] docparse: Implement #define and #include
2021-11-01 14:53 ` [LTP] [PATCH v2 1/7] docparse: Implement #define and #include Cyril Hrubis
@ 2021-11-02 10:05 ` Richard Palethorpe
2021-11-02 11:21 ` Cyril Hrubis
0 siblings, 1 reply; 27+ messages in thread
From: Richard Palethorpe @ 2021-11-02 10:05 UTC (permalink / raw)
To: Cyril Hrubis; +Cc: ltp
Cyril Hrubis <chrubis@suse.cz> writes:
> We ignore most of the include statements and we attempt to parse only
> header files that reside in the same directory as the test source code,
> that is since we are not interested in any system or library headers as
> we are only looking for constants used in the tst_test structure that
> are always either directly in the test source or in header in the same
> directory.
>
> The macro support is very simple as well, it's a single pass as we are
> not interested in intricate macros. We just need values for constants
> that are used in the tst_test structure intializations.
>
> + Also add -v verbose mode that prints included files and defined macros
>
> Signed-off-by: Cyril Hrubis <chrubis@suse.cz>
I don't see any issues that are likely to cause trouble
immediately. However please check the comments below to ensure they are
out-of-scope.
Reviewed-by: rpalethorpe@suse.com
> ---
> docparse/docparse.c | 234 ++++++++++++++++++++++++++++++++++++++++++--
> 1 file changed, 224 insertions(+), 10 deletions(-)
>
> diff --git a/docparse/docparse.c b/docparse/docparse.c
> index 8cd0d0eef..64f9d08d9 100644
> --- a/docparse/docparse.c
> +++ b/docparse/docparse.c
> @@ -1,9 +1,12 @@
> // SPDX-License-Identifier: GPL-2.0-or-later
> /*
> - * Copyright (c) 2019 Cyril Hrubis <chrubis@suse.cz>
> + * Copyright (c) 2019-2021 Cyril Hrubis <chrubis@suse.cz>
> * Copyright (c) 2020 Petr Vorel <pvorel@suse.cz>
> */
>
> +#define _GNU_SOURCE
> +
> +#include <search.h>
> #include <stdio.h>
> #include <string.h>
> #include <libgen.h>
> @@ -12,6 +15,9 @@
>
> #include "data_storage.h"
>
> +static int verbose;
> +static char *includepath;
> +
> #define WARN(str) fprintf(stderr, "WARNING: " str "\n")
>
> static void oneline_comment(FILE *f)
> @@ -126,7 +132,7 @@ static void maybe_comment(FILE *f, struct data_node *doc)
> }
> }
>
> -const char *next_token(FILE *f, struct data_node *doc)
> +static char *next_token(FILE *f, struct data_node *doc)
> {
> size_t i = 0;
> static char buf[4096];
> @@ -159,6 +165,7 @@ const char *next_token(FILE *f, struct data_node *doc)
> case ',':
> case '[':
> case ']':
> + case '#':
> if (i) {
> ungetc(c, f);
> goto exit;
> @@ -197,6 +204,46 @@ exit:
> return buf;
> }
>
> +static FILE *open_include(const char *includepath, FILE *f)
> +{
> + char buf[256];
> + char *path;
> + FILE *inc;
> +
> + if (!fscanf(f, "%s\n", buf))
> + return NULL;
> +
> + if (buf[0] != '"')
> + return NULL;
> +
> + char *filename = buf + 1;
> +
> + if (!buf[0])
> + return NULL;
> +
> + filename[strlen(filename)-1] = 0;
> +
> + if (asprintf(&path, "%s/%s", includepath, filename) < 0)
> + return NULL;
> +
> + inc = fopen(path, "r");
> +
> + if (inc && verbose)
> + fprintf(stderr, "INCLUDE %s\n", path);
> +
> + free(path);
> +
> + return inc;
> +}
> +
> +static void close_include(FILE *inc)
> +{
> + if (verbose)
> + fprintf(stderr, "INCLUDE END\n");
> +
> + fclose(inc);
> +}
> +
> static int parse_array(FILE *f, struct data_node *node)
> {
> const char *token;
> @@ -234,9 +281,28 @@ static int parse_array(FILE *f, struct data_node *node)
> return 0;
> }
>
> +static void try_apply_macro(char **res)
> +{
> + ENTRY macro = {
> + .key = *res,
> + };
> +
> + ENTRY *ret;
> +
> + ret = hsearch(macro, FIND);
> +
> + if (!ret)
> + return;
> +
> + if (verbose)
> + fprintf(stderr, "APPLYING MACRO %s=%s\n", ret->key, (char*)ret->data);
> +
> + *res = ret->data;
> +}
> +
> static int parse_test_struct(FILE *f, struct data_node *doc, struct data_node *node)
> {
> - const char *token;
> + char *token;
> char *id = NULL;
> int state = 0;
> struct data_node *ret;
> @@ -280,6 +346,7 @@ static int parse_test_struct(FILE *f, struct data_node *doc, struct data_node *n
> ret = data_node_array();
> parse_array(f, ret);
> } else {
> + try_apply_macro(&token);
> ret = data_node_string(token);
> }
>
> @@ -302,6 +369,114 @@ static const char *tokens[] = {
> "{",
> };
>
> +static void macro_get_string(FILE *f, char *buf, char *buf_end)
> +{
> + int c;
> +
> + for (;;) {
> + c = fgetc(f);
> +
> + switch (c) {
> + case '"':
Luckily there are no instances of '#define MACRO "...\"...\"..."' in LTP
AFAICT. Also there don't appear to be any '#define MACRO "..." \\n' that
we would care about.
> + case EOF:
> + *buf = 0;
> + return;
> + default:
> + if (buf < buf_end)
> + *(buf++) = c;
> + }
> + }
> +}
> +
> +static void macro_get_val(FILE *f, char *buf, size_t buf_len)
> +{
> + int c, prev = 0;
> + char *buf_end = buf + buf_len - 1;
> +
> + c = fgetc(f);
> + if (c == '"') {
I guess this could be whitespace unless scanf slurps any trailing
whitespace?
Again no actual instances of this AFAICT.
> + macro_get_string(f, buf, buf_end);
> + return;
> + }
> +
> + for (;;) {
> + switch (c) {
> + case '\n':
> + if (prev == '\\') {
> + buf--;
> + } else {
> + *buf = 0;
> + return;
> + }
> + break;
> + case EOF:
> + *buf = 0;
> + return;
> + case ' ':
> + case '\t':
> + break;
> + default:
> + if (buf < buf_end)
> + *(buf++) = c;
> + }
> +
> + prev = c;
> + c = fgetc(f);
> + }
> +}
> +
> +static void parse_macro(FILE *f)
> +{
> + char name[128];
> + char val[256];
> +
> + if (!fscanf(f, "%s[^\n]", name))
> + return;
> +
> + if (fgetc(f) == '\n')
> + return;
> +
> + macro_get_val(f, val, sizeof(val));
> +
> + ENTRY e = {
> + .key = strdup(name),
> + .data = strdup(val),
> + };
> +
> + if (verbose)
> + fprintf(stderr, " MACRO %s=%s\n", e.key, (char*)e.data);
> +
> + hsearch(e, ENTER);
> +}
> +
> +static void parse_include_macros(FILE *f)
> +{
> + FILE *inc;
> + const char *token;
> + int hash = 0;
> +
> + inc = open_include(includepath, f);
> + if (!inc)
> + return;
> +
> + while ((token = next_token(inc, NULL))) {
> + if (token[0] == '#') {
> + hash = 1;
> + continue;
> + }
> +
> + if (!hash)
> + continue;
> +
> + if (!strcmp(token, "define"))
> + parse_macro(inc);
> +
> + hash = 0;
> + }
> +
> + close_include(inc);
> +}
> +
> static struct data_node *parse_file(const char *fname)
> {
> int state = 0, found = 0;
> @@ -314,14 +489,28 @@ static struct data_node *parse_file(const char *fname)
>
> FILE *f = fopen(fname, "r");
>
> + includepath = dirname(strdup(fname));
> +
> struct data_node *res = data_node_hash();
> struct data_node *doc = data_node_array();
>
> while ((token = next_token(f, doc))) {
> - if (state < 6 && !strcmp(tokens[state], token))
> + if (state < 6 && !strcmp(tokens[state], token)) {
> state++;
> - else
> + } else {
> + if (token[0] == '#') {
> + token = next_token(f, doc);
> + if (token) {
> + if (!strcmp(token, "define"))
> + parse_macro(f);
> +
> + if (!strcmp(token, "include"))
> + parse_include_macros(f);
> + }
> + }
> +
> state = 0;
> + }
>
> if (state < 6)
> continue;
> @@ -386,17 +575,42 @@ const char *strip_name(char *path)
> return name;
> }
>
> +static void print_help(const char *prgname)
> +{
> + printf("usage: %s [-vh] input.c\n\n", prgname);
> + printf("-v sets verbose mode\n");
> + printf("-h prints this help\n\n");
> + exit(0);
> +}
> +
> int main(int argc, char *argv[])
> {
> unsigned int i, j;
> struct data_node *res;
> + int opt;
> +
> + while ((opt = getopt(argc, argv, "hv")) != -1) {
> + switch (opt) {
> + case 'h':
> + print_help(argv[0]);
> + break;
> + case 'v':
> + verbose = 1;
> + break;
> + }
> + }
> +
> + if (optind >= argc) {
> + fprintf(stderr, "No input filename.c\n");
> + return 1;
> + }
>
> - if (argc != 2) {
> - fprintf(stderr, "Usage: docparse filename.c\n");
> + if (!hcreate(128)) {
> + fprintf(stderr, "Failed to initialize hash table\n");
> return 1;
> }
>
> - res = parse_file(argv[1]);
> + res = parse_file(argv[optind]);
> if (!res)
> return 0;
>
> @@ -425,8 +639,8 @@ int main(int argc, char *argv[])
> }
> }
>
> - data_node_hash_add(res, "fname", data_node_string(argv[1]));
> - printf(" \"%s\": ", strip_name(argv[1]));
> + data_node_hash_add(res, "fname", data_node_string(argv[optind]));
> + printf(" \"%s\": ", strip_name(argv[optind]));
> data_to_json(res, stdout, 2);
> data_node_free(res);
>
> --
> 2.32.0
--
Thank you,
Richard.
--
Mailing list info: https://lists.linux.it/listinfo/ltp
^ permalink raw reply [flat|nested] 27+ messages in thread* Re: [LTP] [PATCH v2 1/7] docparse: Implement #define and #include
2021-11-02 10:05 ` Richard Palethorpe
@ 2021-11-02 11:21 ` Cyril Hrubis
2021-11-02 14:54 ` Petr Vorel
0 siblings, 1 reply; 27+ messages in thread
From: Cyril Hrubis @ 2021-11-02 11:21 UTC (permalink / raw)
To: Richard Palethorpe; +Cc: ltp
Hi!
> > +static void macro_get_string(FILE *f, char *buf, char *buf_end)
> > +{
> > + int c;
> > +
> > + for (;;) {
> > + c = fgetc(f);
> > +
> > + switch (c) {
> > + case '"':
>
> Luckily there are no instances of '#define MACRO "...\"...\"..."' in LTP
> AFAICT. Also there don't appear to be any '#define MACRO "..." \\n' that
> we would care about.
Well I can fix that and add a test to to be sure.
> > + case EOF:
> > + *buf = 0;
> > + return;
> > + default:
> > + if (buf < buf_end)
> > + *(buf++) = c;
> > + }
> > + }
> > +}
> > +
> > +static void macro_get_val(FILE *f, char *buf, size_t buf_len)
> > +{
> > + int c, prev = 0;
> > + char *buf_end = buf + buf_len - 1;
> > +
> > + c = fgetc(f);
> > + if (c == '"') {
>
> I guess this could be whitespace unless scanf slurps any trailing
> whitespace?
The scanf does not slurp any trainling whitespaces, so this should be
fixed by:
while (isspace(c = fgetc(f)));
With that we get slightly better output, so I will add that before
applying.
--
Cyril Hrubis
chrubis@suse.cz
--
Mailing list info: https://lists.linux.it/listinfo/ltp
^ permalink raw reply [flat|nested] 27+ messages in thread* Re: [LTP] [PATCH v2 1/7] docparse: Implement #define and #include
2021-11-02 11:21 ` Cyril Hrubis
@ 2021-11-02 14:54 ` Petr Vorel
2021-11-02 15:10 ` Cyril Hrubis
0 siblings, 1 reply; 27+ messages in thread
From: Petr Vorel @ 2021-11-02 14:54 UTC (permalink / raw)
To: Cyril Hrubis; +Cc: ltp
Hi Cyril, Richie,
> Hi!
> > > +static void macro_get_string(FILE *f, char *buf, char *buf_end)
> > > +{
> > > + int c;
> > > +
> > > + for (;;) {
> > > + c = fgetc(f);
> > > +
> > > + switch (c) {
> > > + case '"':
> > Luckily there are no instances of '#define MACRO "...\"...\"..."' in LTP
> > AFAICT. Also there don't appear to be any '#define MACRO "..." \\n' that
> > we would care about.
Good catch!
Unless you send v3 feel free to add
Reviewed-by: Petr Vorel <pvorel@suse.cz>
> Well I can fix that and add a test to to be sure.
Thanks!
...
> > > +static void macro_get_val(FILE *f, char *buf, size_t buf_len)
> > > +{
> > > + int c, prev = 0;
> > > + char *buf_end = buf + buf_len - 1;
> > > +
> > > + c = fgetc(f);
> > > + if (c == '"') {
> > I guess this could be whitespace unless scanf slurps any trailing
> > whitespace?
> The scanf does not slurp any trainling whitespaces, so this should be
> fixed by:
> while (isspace(c = fgetc(f)));
> With that we get slightly better output, so I will add that before
> applying.
+1
Kind regards,
Petr
--
Mailing list info: https://lists.linux.it/listinfo/ltp
^ permalink raw reply [flat|nested] 27+ messages in thread
* [LTP] [PATCH v2 2/7] docparse: Add tests
2021-11-01 14:53 [LTP] [PATCH v2 0/7] docparse improvements Cyril Hrubis
2021-11-01 14:53 ` [LTP] [PATCH v2 1/7] docparse: Implement #define and #include Cyril Hrubis
@ 2021-11-01 14:53 ` Cyril Hrubis
2021-11-02 11:09 ` Richard Palethorpe
2021-11-01 14:53 ` [LTP] [PATCH v2 3/7] docparse: data_storage: Add integer type node Cyril Hrubis
` (5 subsequent siblings)
7 siblings, 1 reply; 27+ messages in thread
From: Cyril Hrubis @ 2021-11-01 14:53 UTC (permalink / raw)
To: ltp
This add a simple tests for the docparse parser.
Signed-off-by: Cyril Hrubis <chrubis@suse.cz>
---
docparse/tests/Makefile | 4 ++++
docparse/tests/empty_struct.c | 2 ++
docparse/tests/empty_struct.c.json | 3 +++
docparse/tests/expand_flags.c | 3 +++
docparse/tests/expand_flags.c.json | 6 ++++++
docparse/tests/include.c | 5 +++++
docparse/tests/include.c.json | 4 ++++
docparse/tests/include.h | 1 +
docparse/tests/macro.c | 5 +++++
docparse/tests/macro.c.json | 4 ++++
docparse/tests/multiline_macro.c | 6 ++++++
docparse/tests/multiline_macro.c.json | 4 ++++
docparse/tests/tags.c | 7 +++++++
docparse/tests/tags.c.json | 13 +++++++++++++
docparse/tests/test.sh | 18 ++++++++++++++++++
15 files changed, 85 insertions(+)
create mode 100644 docparse/tests/Makefile
create mode 100644 docparse/tests/empty_struct.c
create mode 100644 docparse/tests/empty_struct.c.json
create mode 100644 docparse/tests/expand_flags.c
create mode 100644 docparse/tests/expand_flags.c.json
create mode 100644 docparse/tests/include.c
create mode 100644 docparse/tests/include.c.json
create mode 100644 docparse/tests/include.h
create mode 100644 docparse/tests/macro.c
create mode 100644 docparse/tests/macro.c.json
create mode 100644 docparse/tests/multiline_macro.c
create mode 100644 docparse/tests/multiline_macro.c.json
create mode 100644 docparse/tests/tags.c
create mode 100644 docparse/tests/tags.c.json
create mode 100755 docparse/tests/test.sh
diff --git a/docparse/tests/Makefile b/docparse/tests/Makefile
new file mode 100644
index 000000000..b5c8c4668
--- /dev/null
+++ b/docparse/tests/Makefile
@@ -0,0 +1,4 @@
+all:
+
+test:
+ @./test.sh
diff --git a/docparse/tests/empty_struct.c b/docparse/tests/empty_struct.c
new file mode 100644
index 000000000..e5d986413
--- /dev/null
+++ b/docparse/tests/empty_struct.c
@@ -0,0 +1,2 @@
+static struct tst_test test = {
+};
diff --git a/docparse/tests/empty_struct.c.json b/docparse/tests/empty_struct.c.json
new file mode 100644
index 000000000..9f49f5332
--- /dev/null
+++ b/docparse/tests/empty_struct.c.json
@@ -0,0 +1,3 @@
+ "empty_struct": {
+ "fname": "empty_struct.c"
+ }
\ No newline at end of file
diff --git a/docparse/tests/expand_flags.c b/docparse/tests/expand_flags.c
new file mode 100644
index 000000000..64f6da64e
--- /dev/null
+++ b/docparse/tests/expand_flags.c
@@ -0,0 +1,3 @@
+static struct tst_test test = {
+ .all_filesystems = 1,
+};
diff --git a/docparse/tests/expand_flags.c.json b/docparse/tests/expand_flags.c.json
new file mode 100644
index 000000000..cd79dd296
--- /dev/null
+++ b/docparse/tests/expand_flags.c.json
@@ -0,0 +1,6 @@
+ "expand_flags": {
+ "all_filesystems": "1",
+ "needs_device": "1",
+ "needs_tmpdir": "1",
+ "fname": "expand_flags.c"
+ }
\ No newline at end of file
diff --git a/docparse/tests/include.c b/docparse/tests/include.c
new file mode 100644
index 000000000..15377e339
--- /dev/null
+++ b/docparse/tests/include.c
@@ -0,0 +1,5 @@
+# include "include.h"
+
+static struct tst_test test = {
+ .test_variants = TEST_VARIANTS,
+};
diff --git a/docparse/tests/include.c.json b/docparse/tests/include.c.json
new file mode 100644
index 000000000..b4ef1ccda
--- /dev/null
+++ b/docparse/tests/include.c.json
@@ -0,0 +1,4 @@
+ "include": {
+ "test_variants": "10",
+ "fname": "include.c"
+ }
\ No newline at end of file
diff --git a/docparse/tests/include.h b/docparse/tests/include.h
new file mode 100644
index 000000000..efa11d24f
--- /dev/null
+++ b/docparse/tests/include.h
@@ -0,0 +1 @@
+#define TEST_VARIANTS 10
diff --git a/docparse/tests/macro.c b/docparse/tests/macro.c
new file mode 100644
index 000000000..296da12f5
--- /dev/null
+++ b/docparse/tests/macro.c
@@ -0,0 +1,5 @@
+#define TEST_VARIANTS 10
+
+static struct tst_test test = {
+ .test_variants = TEST_VARIANTS,
+};
diff --git a/docparse/tests/macro.c.json b/docparse/tests/macro.c.json
new file mode 100644
index 000000000..0dc73d8ec
--- /dev/null
+++ b/docparse/tests/macro.c.json
@@ -0,0 +1,4 @@
+ "macro": {
+ "test_variants": "10",
+ "fname": "macro.c"
+ }
\ No newline at end of file
diff --git a/docparse/tests/multiline_macro.c b/docparse/tests/multiline_macro.c
new file mode 100644
index 000000000..2de58112b
--- /dev/null
+++ b/docparse/tests/multiline_macro.c
@@ -0,0 +1,6 @@
+#define TEST_VARIANTS \
+ 10
+
+static struct tst_test test = {
+ .test_variants = TEST_VARIANTS,
+};
diff --git a/docparse/tests/multiline_macro.c.json b/docparse/tests/multiline_macro.c.json
new file mode 100644
index 000000000..bafd037da
--- /dev/null
+++ b/docparse/tests/multiline_macro.c.json
@@ -0,0 +1,4 @@
+ "multiline_macro": {
+ "test_variants": "10",
+ "fname": "multiline_macro.c"
+ }
\ No newline at end of file
diff --git a/docparse/tests/tags.c b/docparse/tests/tags.c
new file mode 100644
index 000000000..ade3974ff
--- /dev/null
+++ b/docparse/tests/tags.c
@@ -0,0 +1,7 @@
+static struct tst_test test = {
+ .tags = (const struct tst_tag[]) {
+ {"tag-name-1", "tag-value-1"},
+ {"tag-name-2", "tag-value-2"},
+ {}
+ }
+};
diff --git a/docparse/tests/tags.c.json b/docparse/tests/tags.c.json
new file mode 100644
index 000000000..14cc14f1c
--- /dev/null
+++ b/docparse/tests/tags.c.json
@@ -0,0 +1,13 @@
+ "tags": {
+ "tags": [
+ [
+ "tag-name-1",
+ "tag-value-1"
+ ],
+ [
+ "tag-name-2",
+ "tag-value-2"
+ ]
+ ],
+ "fname": "tags.c"
+ }
\ No newline at end of file
diff --git a/docparse/tests/test.sh b/docparse/tests/test.sh
new file mode 100755
index 000000000..767cc464c
--- /dev/null
+++ b/docparse/tests/test.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+
+fail=0
+
+for i in *.c; do
+ ../docparse $i > tmp.json
+ if ! diff tmp.json $i.json &> /dev/null; then
+ echo "***"
+ echo "$i output differs!"
+ diff -u tmp.json $i.json
+ echo "***"
+ fail=1
+ fi
+done
+
+rm -f tmp.json
+
+exit $fail
--
2.32.0
--
Mailing list info: https://lists.linux.it/listinfo/ltp
^ permalink raw reply related [flat|nested] 27+ messages in thread* [LTP] [PATCH v2 3/7] docparse: data_storage: Add integer type node
2021-11-01 14:53 [LTP] [PATCH v2 0/7] docparse improvements Cyril Hrubis
2021-11-01 14:53 ` [LTP] [PATCH v2 1/7] docparse: Implement #define and #include Cyril Hrubis
2021-11-01 14:53 ` [LTP] [PATCH v2 2/7] docparse: Add tests Cyril Hrubis
@ 2021-11-01 14:53 ` Cyril Hrubis
2021-11-02 11:14 ` Richard Palethorpe
2021-11-01 14:53 ` [LTP] [PATCH v2 4/7] docparse: Implement ARRAY_SIZE() Cyril Hrubis
` (4 subsequent siblings)
7 siblings, 1 reply; 27+ messages in thread
From: Cyril Hrubis @ 2021-11-01 14:53 UTC (permalink / raw)
To: ltp
Signed-off-by: Cyril Hrubis <chrubis@suse.cz>
---
docparse/data_storage.h | 29 +++++++++++++++++++++++++++++
1 file changed, 29 insertions(+)
diff --git a/docparse/data_storage.h b/docparse/data_storage.h
index 339450c8b..117c1d127 100644
--- a/docparse/data_storage.h
+++ b/docparse/data_storage.h
@@ -15,6 +15,7 @@ enum data_type {
DATA_ARRAY,
DATA_HASH,
DATA_STRING,
+ DATA_INT,
};
struct data_node_array {
@@ -41,12 +42,18 @@ struct data_node_string {
char val[];
};
+struct data_node_int {
+ enum data_type type;
+ long val;
+};
+
struct data_node {
union {
enum data_type type;
struct data_node_hash hash;
struct data_node_array array;
struct data_node_string string;
+ struct data_node_int i;
};
};
@@ -64,6 +71,19 @@ static inline struct data_node *data_node_string(const char *string)
return node;
}
+static inline struct data_node *data_node_int(long i)
+{
+ struct data_node *node = malloc(sizeof(struct data_node_int));
+
+ if (!node)
+ return NULL;
+
+ node->type = DATA_INT;
+ node->i.val = i;
+
+ return node;
+}
+
#define MAX_ELEMS 100
static inline struct data_node *data_node_hash(void)
@@ -122,6 +142,7 @@ static inline void data_node_free(struct data_node *self)
switch (self->type) {
case DATA_STRING:
+ case DATA_INT:
break;
case DATA_HASH:
for (i = 0; i < self->hash.elems_used; i++) {
@@ -209,6 +230,10 @@ static inline void data_node_print_(struct data_node *self, unsigned int padd)
unsigned int i;
switch (self->type) {
+ case DATA_INT:
+ data_print_padd(padd);
+ printf("%li\n", self->i.val);
+ break;
case DATA_STRING:
data_print_padd(padd);
printf("'%s'\n", self->string.val);
@@ -295,6 +320,10 @@ static inline void data_to_json_(struct data_node *self, FILE *f, unsigned int p
unsigned int i;
switch (self->type) {
+ case DATA_INT:
+ padd = do_padd ? padd : 0;
+ data_fprintf(f, padd, "%li", self->i.val);
+ break;
case DATA_STRING:
padd = do_padd ? padd : 0;
data_fprintf_esc(f, padd, self->string.val);
--
2.32.0
--
Mailing list info: https://lists.linux.it/listinfo/ltp
^ permalink raw reply related [flat|nested] 27+ messages in thread* [LTP] [PATCH v2 4/7] docparse: Implement ARRAY_SIZE()
2021-11-01 14:53 [LTP] [PATCH v2 0/7] docparse improvements Cyril Hrubis
` (2 preceding siblings ...)
2021-11-01 14:53 ` [LTP] [PATCH v2 3/7] docparse: data_storage: Add integer type node Cyril Hrubis
@ 2021-11-01 14:53 ` Cyril Hrubis
2021-11-02 11:39 ` Richard Palethorpe
2021-11-01 14:53 ` [LTP] [PATCH v2 5/7] docparse: Add type normalization Cyril Hrubis
` (3 subsequent siblings)
7 siblings, 1 reply; 27+ messages in thread
From: Cyril Hrubis @ 2021-11-01 14:53 UTC (permalink / raw)
To: ltp
Adds a special handlingn for ARRAY_SIZE() macro.
If we stumble upon ARRAY_SIZE() in the tst_test structure we try to
lookup the array and count its members.
Proper parsing of .test_variants also requires that we add -I switch to
the docparse to be able to specify include paths on a command line since
some variants are stuck in top level include while others are in
testcases/kernel/syscalls/utils/.
+ tests
Signed-off-by: Cyril Hrubis <chrubis@suse.cz>
---
docparse/docparse.c | 221 ++++++++++++++++++++++++++---
docparse/parse.sh | 2 +-
docparse/tests/array_size01.c | 5 +
docparse/tests/array_size01.c.json | 4 +
docparse/tests/array_size02.c | 9 ++
docparse/tests/array_size02.c.json | 4 +
docparse/tests/array_size03.c | 10 ++
docparse/tests/array_size03.c.json | 4 +
docparse/tests/array_size04.c | 5 +
docparse/tests/array_size04.c.json | 4 +
docparse/tests/include.h | 6 +
11 files changed, 255 insertions(+), 19 deletions(-)
create mode 100644 docparse/tests/array_size01.c
create mode 100644 docparse/tests/array_size01.c.json
create mode 100644 docparse/tests/array_size02.c
create mode 100644 docparse/tests/array_size02.c.json
create mode 100644 docparse/tests/array_size03.c
create mode 100644 docparse/tests/array_size03.c.json
create mode 100644 docparse/tests/array_size04.c
create mode 100644 docparse/tests/array_size04.c.json
diff --git a/docparse/docparse.c b/docparse/docparse.c
index 64f9d08d9..9133accf5 100644
--- a/docparse/docparse.c
+++ b/docparse/docparse.c
@@ -15,7 +15,11 @@
#include "data_storage.h"
+#define INCLUDE_PATH_MAX 5
+
static int verbose;
+static char *cmdline_includepath[INCLUDE_PATH_MAX];
+static unsigned int cmdline_includepaths;
static char *includepath;
#define WARN(str) fprintf(stderr, "WARNING: " str "\n")
@@ -132,13 +136,14 @@ static void maybe_comment(FILE *f, struct data_node *doc)
}
}
-static char *next_token(FILE *f, struct data_node *doc)
+static char *next_token2(FILE *f, char *buf, size_t buf_len, struct data_node *doc)
{
size_t i = 0;
- static char buf[4096];
int c;
int in_str = 0;
+ buf_len--;
+
for (;;) {
c = fgetc(f);
@@ -151,7 +156,8 @@ static char *next_token(FILE *f, struct data_node *doc)
goto exit;
}
- buf[i++] = c;
+ if (i < buf_len)
+ buf[i++] = c;
continue;
}
@@ -171,7 +177,8 @@ static char *next_token(FILE *f, struct data_node *doc)
goto exit;
}
- buf[i++] = c;
+ if (i < buf_len)
+ buf[i++] = c;
goto exit;
case '0' ... '9':
case 'a' ... 'z':
@@ -204,11 +211,33 @@ exit:
return buf;
}
-static FILE *open_include(const char *includepath, FILE *f)
+static char *next_token(FILE *f, struct data_node *doc)
+{
+ static char buf[4096];
+
+ return next_token2(f, buf, sizeof(buf), doc);
+}
+
+static FILE *open_file(const char *dir, const char *fname)
{
- char buf[256];
+ FILE *f;
char *path;
+
+ if (asprintf(&path, "%s/%s", dir, fname) < 0)
+ return NULL;
+
+ f = fopen(path, "r");
+
+ free(path);
+
+ return f;
+}
+
+static FILE *open_include(FILE *f)
+{
+ char buf[256], *fname;
FILE *inc;
+ unsigned int i;
if (!fscanf(f, "%s\n", buf))
return NULL;
@@ -216,24 +245,36 @@ static FILE *open_include(const char *includepath, FILE *f)
if (buf[0] != '"')
return NULL;
- char *filename = buf + 1;
+ fname = buf + 1;
if (!buf[0])
return NULL;
- filename[strlen(filename)-1] = 0;
+ fname[strlen(fname)-1] = 0;
- if (asprintf(&path, "%s/%s", includepath, filename) < 0)
- return NULL;
+ inc = open_file(includepath, fname);
+ if (inc) {
+ if (verbose)
+ fprintf(stderr, "INCLUDE %s/%s\n", includepath, fname);
- inc = fopen(path, "r");
+ return inc;
+ }
- if (inc && verbose)
- fprintf(stderr, "INCLUDE %s\n", path);
+ for (i = 0; i < cmdline_includepaths; i++) {
+ inc = open_file(cmdline_includepath[i], fname);
- free(path);
+ if (!inc)
+ continue;
- return inc;
+ if (verbose) {
+ fprintf(stderr, "INCLUDE %s/%s\n",
+ cmdline_includepath[i], fname);
+ }
+
+ return inc;
+ }
+
+ return NULL;
}
static void close_include(FILE *inc)
@@ -300,6 +341,136 @@ static void try_apply_macro(char **res)
*res = ret->data;
}
+static int parse_get_array_len(FILE *f)
+{
+ const char *token;
+ int cnt = 0, depth = 0, prev_comma = 0;
+
+ if (!(token = next_token(f, NULL)))
+ return 0;
+
+ if (strcmp(token, "{"))
+ return 0;
+
+ for (;;) {
+ if (!(token = next_token(f, NULL)))
+ return 0;
+
+ if (!strcmp(token, "{"))
+ depth++;
+
+ if (!strcmp(token, "}"))
+ depth--;
+ else
+ prev_comma = 0;
+
+ if (!strcmp(token, ",") && !depth) {
+ prev_comma = 1;
+ cnt++;
+ }
+
+ if (depth < 0)
+ return cnt + !prev_comma;
+ }
+}
+
+static void look_for_array_size(FILE *f, const char *arr_id, struct data_node **res)
+{
+ const char *token;
+ char buf[2][2048] = {};
+ int cur_buf = 0;
+ int prev_buf = 1;
+
+ for (;;) {
+ if (!(token = next_token2(f, buf[cur_buf], 2048, NULL)))
+ break;
+
+ if (!strcmp(token, "=") && !strcmp(buf[prev_buf], arr_id)) {
+ int arr_len = parse_get_array_len(f);
+
+ if (verbose)
+ fprintf(stderr, "ARRAY %s LENGTH = %i\n", arr_id, arr_len);
+
+ *res = data_node_int(arr_len);
+
+ break;
+ }
+
+ if (strcmp(buf[cur_buf], "]") && strcmp(buf[cur_buf], "[")) {
+ cur_buf = !cur_buf;
+ prev_buf = !prev_buf;
+ }
+ }
+}
+
+static int parse_array_size(FILE *f, struct data_node **res)
+{
+ const char *token;
+ char *arr_id;
+ long pos;
+ int hash = 0;
+
+ *res = NULL;
+
+ if (!(token = next_token(f, NULL)))
+ return 1;
+
+ if (strcmp(token, "("))
+ return 1;
+
+ if (!(token = next_token(f, NULL)))
+ return 1;
+
+ arr_id = strdup(token);
+
+ if (verbose)
+ fprintf(stderr, "COMPUTING ARRAY '%s' LENGHT\n", arr_id);
+
+ pos = ftell(f);
+
+ rewind(f);
+
+ look_for_array_size(f, arr_id, res);
+
+ if (!*res) {
+ FILE *inc;
+
+ rewind(f);
+
+ for (;;) {
+ if (!(token = next_token(f, NULL)))
+ break;
+
+ if (token[0] == '#') {
+ hash = 1;
+ continue;
+ }
+
+ if (!hash)
+ continue;
+
+ if (!strcmp(token, "include")) {
+ inc = open_include(f);
+
+ if (inc) {
+ look_for_array_size(inc, arr_id, res);
+ close_include(inc);
+ }
+ }
+
+ if (*res)
+ break;
+ }
+ }
+
+ free(arr_id);
+
+ if (fseek(f, pos, SEEK_SET))
+ return 1;
+
+ return 0;
+}
+
static int parse_test_struct(FILE *f, struct data_node *doc, struct data_node *node)
{
char *token;
@@ -345,11 +516,17 @@ static int parse_test_struct(FILE *f, struct data_node *doc, struct data_node *n
if (!strcmp(token, "{")) {
ret = data_node_array();
parse_array(f, ret);
+ } else if (!strcmp(token, "ARRAY_SIZE")) {
+ if (parse_array_size(f, &ret))
+ return 1;
} else {
try_apply_macro(&token);
ret = data_node_string(token);
}
+ if (!ret)
+ continue;
+
const char *key = id;
if (key[0] == '.')
key++;
@@ -455,7 +632,7 @@ static void parse_include_macros(FILE *f)
const char *token;
int hash = 0;
- inc = open_include(includepath, f);
+ inc = open_include(f);
if (!inc)
return;
@@ -519,7 +696,6 @@ static struct data_node *parse_file(const char *fname)
parse_test_struct(f, doc, res);
}
-
if (data_node_array_len(doc)) {
data_node_hash_add(res, "doc", doc);
found = 1;
@@ -579,6 +755,7 @@ static void print_help(const char *prgname)
{
printf("usage: %s [-vh] input.c\n\n", prgname);
printf("-v sets verbose mode\n");
+ printf("-I add include path\n");
printf("-h prints this help\n\n");
exit(0);
}
@@ -589,11 +766,19 @@ int main(int argc, char *argv[])
struct data_node *res;
int opt;
- while ((opt = getopt(argc, argv, "hv")) != -1) {
+ while ((opt = getopt(argc, argv, "hI:v")) != -1) {
switch (opt) {
case 'h':
print_help(argv[0]);
break;
+ case 'I':
+ if (cmdline_includepaths >= INCLUDE_PATH_MAX) {
+ fprintf(stderr, "Too much include paths!");
+ exit(1);
+ }
+
+ cmdline_includepath[cmdline_includepaths++] = optarg;
+ break;
case 'v':
verbose = 1;
break;
diff --git a/docparse/parse.sh b/docparse/parse.sh
index 79011bc10..2ace34fa0 100755
--- a/docparse/parse.sh
+++ b/docparse/parse.sh
@@ -26,7 +26,7 @@ echo ' "tests": {'
first=1
for test in `find testcases/ -name '*.c'`; do
- a=$($top_builddir/docparse/docparse "$test")
+ a=$($top_builddir/docparse/docparse -Iinclude -Itestcases/kernel/syscalls/utils/ "$test")
if [ -n "$a" ]; then
if [ -z "$first" ]; then
echo ','
diff --git a/docparse/tests/array_size01.c b/docparse/tests/array_size01.c
new file mode 100644
index 000000000..5f182bc7c
--- /dev/null
+++ b/docparse/tests/array_size01.c
@@ -0,0 +1,5 @@
+static int variants[] = {1};
+
+static struct tst_test test = {
+ .test_variants = ARRAY_SIZE(variants),
+};
diff --git a/docparse/tests/array_size01.c.json b/docparse/tests/array_size01.c.json
new file mode 100644
index 000000000..ec364be12
--- /dev/null
+++ b/docparse/tests/array_size01.c.json
@@ -0,0 +1,4 @@
+ "array_size01": {
+ "test_variants": 1,
+ "fname": "array_size01.c"
+ }
\ No newline at end of file
diff --git a/docparse/tests/array_size02.c b/docparse/tests/array_size02.c
new file mode 100644
index 000000000..ffa37a5c6
--- /dev/null
+++ b/docparse/tests/array_size02.c
@@ -0,0 +1,9 @@
+struct foo {
+ int val;
+};
+
+static struct foo variants[] = {{1}, {2}, {3}};
+
+static struct tst_test test = {
+ .test_variants = ARRAY_SIZE(variants),
+};
diff --git a/docparse/tests/array_size02.c.json b/docparse/tests/array_size02.c.json
new file mode 100644
index 000000000..122686952
--- /dev/null
+++ b/docparse/tests/array_size02.c.json
@@ -0,0 +1,4 @@
+ "array_size02": {
+ "test_variants": 3,
+ "fname": "array_size02.c"
+ }
\ No newline at end of file
diff --git a/docparse/tests/array_size03.c b/docparse/tests/array_size03.c
new file mode 100644
index 000000000..9058db813
--- /dev/null
+++ b/docparse/tests/array_size03.c
@@ -0,0 +1,10 @@
+static struct foo variants[] = {
+#ifdef FOO
+ {.bar = 11},
+#endif
+ {.bar = 10},
+};
+
+static struct tst_test test = {
+ .test_variants = ARRAY_SIZE(variants),
+};
diff --git a/docparse/tests/array_size03.c.json b/docparse/tests/array_size03.c.json
new file mode 100644
index 000000000..bb690c9f5
--- /dev/null
+++ b/docparse/tests/array_size03.c.json
@@ -0,0 +1,4 @@
+ "array_size03": {
+ "test_variants": 2,
+ "fname": "array_size03.c"
+ }
\ No newline at end of file
diff --git a/docparse/tests/array_size04.c b/docparse/tests/array_size04.c
new file mode 100644
index 000000000..5f1d9986e
--- /dev/null
+++ b/docparse/tests/array_size04.c
@@ -0,0 +1,5 @@
+#include "include.h"
+
+static struct tst_test test = {
+ .test_variants = ARRAY_SIZE(variants),
+};
diff --git a/docparse/tests/array_size04.c.json b/docparse/tests/array_size04.c.json
new file mode 100644
index 000000000..6b8d41792
--- /dev/null
+++ b/docparse/tests/array_size04.c.json
@@ -0,0 +1,4 @@
+ "array_size04": {
+ "test_variants": 3,
+ "fname": "array_size04.c"
+ }
\ No newline at end of file
diff --git a/docparse/tests/include.h b/docparse/tests/include.h
index efa11d24f..fbc69a561 100644
--- a/docparse/tests/include.h
+++ b/docparse/tests/include.h
@@ -1 +1,7 @@
#define TEST_VARIANTS 10
+
+static struct variants[] = {
+ {.bar = 10},
+ {.bar = 11},
+ {.bar = 12}
+};
--
2.32.0
--
Mailing list info: https://lists.linux.it/listinfo/ltp
^ permalink raw reply related [flat|nested] 27+ messages in thread* Re: [LTP] [PATCH v2 4/7] docparse: Implement ARRAY_SIZE()
2021-11-01 14:53 ` [LTP] [PATCH v2 4/7] docparse: Implement ARRAY_SIZE() Cyril Hrubis
@ 2021-11-02 11:39 ` Richard Palethorpe
2021-11-02 12:07 ` Cyril Hrubis
0 siblings, 1 reply; 27+ messages in thread
From: Richard Palethorpe @ 2021-11-02 11:39 UTC (permalink / raw)
To: Cyril Hrubis; +Cc: ltp
Hello,
Cyril Hrubis <chrubis@suse.cz> writes:
> Adds a special handlingn for ARRAY_SIZE() macro.
>
> If we stumble upon ARRAY_SIZE() in the tst_test structure we try to
> lookup the array and count its members.
>
> Proper parsing of .test_variants also requires that we add -I switch to
> the docparse to be able to specify include paths on a command line since
> some variants are stuck in top level include while others are in
> testcases/kernel/syscalls/utils/.
>
> + tests
>
> Signed-off-by: Cyril Hrubis <chrubis@suse.cz>
Again I don't see anything that would presently prevent parsing. But see
note below
Reviewed-by: rpalethorpe@suse.com
> ---
> docparse/docparse.c | 221 ++++++++++++++++++++++++++---
> docparse/parse.sh | 2 +-
> docparse/tests/array_size01.c | 5 +
> docparse/tests/array_size01.c.json | 4 +
> docparse/tests/array_size02.c | 9 ++
> docparse/tests/array_size02.c.json | 4 +
> docparse/tests/array_size03.c | 10 ++
> docparse/tests/array_size03.c.json | 4 +
> docparse/tests/array_size04.c | 5 +
> docparse/tests/array_size04.c.json | 4 +
> docparse/tests/include.h | 6 +
> 11 files changed, 255 insertions(+), 19 deletions(-)
> create mode 100644 docparse/tests/array_size01.c
> create mode 100644 docparse/tests/array_size01.c.json
> create mode 100644 docparse/tests/array_size02.c
> create mode 100644 docparse/tests/array_size02.c.json
> create mode 100644 docparse/tests/array_size03.c
> create mode 100644 docparse/tests/array_size03.c.json
> create mode 100644 docparse/tests/array_size04.c
> create mode 100644 docparse/tests/array_size04.c.json
>
> diff --git a/docparse/docparse.c b/docparse/docparse.c
> index 64f9d08d9..9133accf5 100644
> --- a/docparse/docparse.c
> +++ b/docparse/docparse.c
> @@ -15,7 +15,11 @@
>
> #include "data_storage.h"
>
> +#define INCLUDE_PATH_MAX 5
> +
> static int verbose;
> +static char *cmdline_includepath[INCLUDE_PATH_MAX];
> +static unsigned int cmdline_includepaths;
> static char *includepath;
>
> #define WARN(str) fprintf(stderr, "WARNING: " str "\n")
> @@ -132,13 +136,14 @@ static void maybe_comment(FILE *f, struct data_node *doc)
> }
> }
>
> -static char *next_token(FILE *f, struct data_node *doc)
> +static char *next_token2(FILE *f, char *buf, size_t buf_len, struct data_node *doc)
> {
> size_t i = 0;
> - static char buf[4096];
> int c;
> int in_str = 0;
>
> + buf_len--;
> +
> for (;;) {
> c = fgetc(f);
>
> @@ -151,7 +156,8 @@ static char *next_token(FILE *f, struct data_node *doc)
> goto exit;
> }
>
> - buf[i++] = c;
> + if (i < buf_len)
> + buf[i++] = c;
> continue;
> }
>
> @@ -171,7 +177,8 @@ static char *next_token(FILE *f, struct data_node *doc)
> goto exit;
> }
>
> - buf[i++] = c;
> + if (i < buf_len)
> + buf[i++] = c;
> goto exit;
> case '0' ... '9':
> case 'a' ... 'z':
> @@ -204,11 +211,33 @@ exit:
> return buf;
> }
>
> -static FILE *open_include(const char *includepath, FILE *f)
> +static char *next_token(FILE *f, struct data_node *doc)
> +{
> + static char buf[4096];
> +
> + return next_token2(f, buf, sizeof(buf), doc);
> +}
> +
> +static FILE *open_file(const char *dir, const char *fname)
> {
> - char buf[256];
> + FILE *f;
> char *path;
> +
> + if (asprintf(&path, "%s/%s", dir, fname) < 0)
> + return NULL;
> +
> + f = fopen(path, "r");
> +
> + free(path);
> +
> + return f;
> +}
> +
> +static FILE *open_include(FILE *f)
> +{
> + char buf[256], *fname;
> FILE *inc;
> + unsigned int i;
>
> if (!fscanf(f, "%s\n", buf))
> return NULL;
> @@ -216,24 +245,36 @@ static FILE *open_include(const char *includepath, FILE *f)
> if (buf[0] != '"')
> return NULL;
>
> - char *filename = buf + 1;
> + fname = buf + 1;
>
> if (!buf[0])
> return NULL;
>
> - filename[strlen(filename)-1] = 0;
> + fname[strlen(fname)-1] = 0;
>
> - if (asprintf(&path, "%s/%s", includepath, filename) < 0)
> - return NULL;
> + inc = open_file(includepath, fname);
> + if (inc) {
> + if (verbose)
> + fprintf(stderr, "INCLUDE %s/%s\n", includepath, fname);
>
> - inc = fopen(path, "r");
> + return inc;
> + }
>
> - if (inc && verbose)
> - fprintf(stderr, "INCLUDE %s\n", path);
> + for (i = 0; i < cmdline_includepaths; i++) {
> + inc = open_file(cmdline_includepath[i], fname);
>
> - free(path);
> + if (!inc)
> + continue;
>
> - return inc;
> + if (verbose) {
> + fprintf(stderr, "INCLUDE %s/%s\n",
> + cmdline_includepath[i], fname);
> + }
> +
> + return inc;
> + }
> +
> + return NULL;
> }
>
> static void close_include(FILE *inc)
> @@ -300,6 +341,136 @@ static void try_apply_macro(char **res)
> *res = ret->data;
> }
>
> +static int parse_get_array_len(FILE *f)
> +{
> + const char *token;
> + int cnt = 0, depth = 0, prev_comma = 0;
> +
> + if (!(token = next_token(f, NULL)))
> + return 0;
> +
> + if (strcmp(token, "{"))
> + return 0;
> +
> + for (;;) {
> + if (!(token = next_token(f, NULL)))
> + return 0;
> +
> + if (!strcmp(token, "{"))
> + depth++;
> +
> + if (!strcmp(token, "}"))
> + depth--;
> + else
> + prev_comma = 0;
> +
> + if (!strcmp(token, ",") && !depth) {
> + prev_comma = 1;
> + cnt++;
> + }
> +
> + if (depth < 0)
> + return cnt + !prev_comma;
> + }
> +}
> +
> +static void look_for_array_size(FILE *f, const char *arr_id, struct data_node **res)
> +{
> + const char *token;
> + char buf[2][2048] = {};
> + int cur_buf = 0;
> + int prev_buf = 1;
> +
> + for (;;) {
> + if (!(token = next_token2(f, buf[cur_buf], 2048, NULL)))
> + break;
> +
> + if (!strcmp(token, "=") && !strcmp(buf[prev_buf], arr_id)) {
> + int arr_len = parse_get_array_len(f);
> +
> + if (verbose)
> + fprintf(stderr, "ARRAY %s LENGTH = %i\n", arr_id, arr_len);
> +
> + *res = data_node_int(arr_len);
> +
> + break;
> + }
> +
> + if (strcmp(buf[cur_buf], "]") && strcmp(buf[cur_buf], "[")) {
So this seems to mean we would silently ignore a variants array which
was declared with an explicit size. A quick grep doesn't turn up any
instances of that. However this would be an unexpected result. People
may intuitively think adding an explicity size would make parsing
easier.
--
Thank you,
Richard.
--
Mailing list info: https://lists.linux.it/listinfo/ltp
^ permalink raw reply [flat|nested] 27+ messages in thread* Re: [LTP] [PATCH v2 4/7] docparse: Implement ARRAY_SIZE()
2021-11-02 11:39 ` Richard Palethorpe
@ 2021-11-02 12:07 ` Cyril Hrubis
0 siblings, 0 replies; 27+ messages in thread
From: Cyril Hrubis @ 2021-11-02 12:07 UTC (permalink / raw)
To: Richard Palethorpe; +Cc: ltp
Hi!
> > +
> > + if (strcmp(buf[cur_buf], "]") && strcmp(buf[cur_buf], "[")) {
>
> So this seems to mean we would silently ignore a variants array which
> was declared with an explicit size. A quick grep doesn't turn up any
> instances of that. However this would be an unexpected result. People
> may intuitively think adding an explicity size would make parsing
> easier.
I would say that adding explicit size to arrays whose size is defined
by their initialize is a mistake, at least in our case so I'm inclined
to keep this as it is.
--
Cyril Hrubis
chrubis@suse.cz
--
Mailing list info: https://lists.linux.it/listinfo/ltp
^ permalink raw reply [flat|nested] 27+ messages in thread
* [LTP] [PATCH v2 5/7] docparse: Add type normalization
2021-11-01 14:53 [LTP] [PATCH v2 0/7] docparse improvements Cyril Hrubis
` (3 preceding siblings ...)
2021-11-01 14:53 ` [LTP] [PATCH v2 4/7] docparse: Implement ARRAY_SIZE() Cyril Hrubis
@ 2021-11-01 14:53 ` Cyril Hrubis
2021-11-02 11:52 ` Richard Palethorpe
2021-11-01 14:53 ` [LTP] [PATCH v2 6/7] docparse: Group data to 'testsuite' and 'defaults' Cyril Hrubis
` (2 subsequent siblings)
7 siblings, 1 reply; 27+ messages in thread
From: Cyril Hrubis @ 2021-11-01 14:53 UTC (permalink / raw)
To: ltp
For now just for .test_variants.
There are two reasons for this:
- This code makes sure that we get right value parsed and aborts the
compilation if the parser got confused. This part is important since
if the testrunners are going to use the metadata the data in there
must be correct.
- And much less important it makes the resulting json nicer to read
Signed-off-by: Cyril Hrubis <chrubis@suse.cz>
---
docparse/data_storage.h | 16 ++++++++
docparse/docparse.c | 59 +++++++++++++++++++++++++++
docparse/tests/include.c.json | 2 +-
docparse/tests/macro.c.json | 2 +-
docparse/tests/multiline_macro.c.json | 2 +-
5 files changed, 78 insertions(+), 3 deletions(-)
diff --git a/docparse/data_storage.h b/docparse/data_storage.h
index 117c1d127..91ea70a02 100644
--- a/docparse/data_storage.h
+++ b/docparse/data_storage.h
@@ -57,6 +57,22 @@ struct data_node {
};
};
+static inline const char* data_type_name(enum data_type type)
+{
+ switch (type) {
+ case DATA_ARRAY:
+ return "array";
+ case DATA_HASH:
+ return "hash";
+ case DATA_STRING:
+ return "string";
+ case DATA_INT:
+ return "int";
+ default:
+ return "???";
+ }
+}
+
static inline struct data_node *data_node_string(const char *string)
{
size_t size = sizeof(struct data_node_string) + strlen(string) + 1;
diff --git a/docparse/docparse.c b/docparse/docparse.c
index 9133accf5..9c3eaff4c 100644
--- a/docparse/docparse.c
+++ b/docparse/docparse.c
@@ -12,6 +12,7 @@
#include <libgen.h>
#include <ctype.h>
#include <unistd.h>
+#include <errno.h>
#include "data_storage.h"
@@ -713,6 +714,61 @@ static struct data_node *parse_file(const char *fname)
return res;
}
+static struct typemap {
+ const char *id;
+ enum data_type type;
+} tst_test_typemap[] = {
+ {.id = "test_variants", .type = DATA_INT},
+ {}
+};
+
+static void convert_str2int(struct data_node *res, const char *id, const char *str_val)
+{
+ long val;
+ char *endptr;
+
+ errno = 0;
+ val = strtol(str_val, &endptr, 10);
+
+ if (errno || *endptr) {
+ fprintf(stderr, "Cannot convert %s value %s to int!", id, str_val);
+ exit(1);
+ }
+
+ if (verbose)
+ fprintf(stderr, "NORMALIZING %s TO INT %li", id, val);
+
+ data_node_hash_del(res, id);
+ data_node_hash_add(res, id, data_node_int(val));
+}
+
+static void check_normalize_types(struct data_node *res)
+{
+ unsigned int i;
+
+ for (i = 0; tst_test_typemap[i].id; i++) {
+ struct data_node *n;
+ struct typemap *typemap = &tst_test_typemap[i];
+
+ n = data_node_hash_get(res, typemap->id);
+ if (!n)
+ continue;
+
+ if (n->type == typemap->type)
+ continue;
+
+ if (n->type == DATA_STRING && typemap->type == DATA_INT) {
+ convert_str2int(res, typemap->id, n->string.val);
+ continue;
+ }
+
+ fprintf(stderr, "Cannot convert %s from %s to %s!",
+ typemap->id, data_type_name(n->type),
+ data_type_name(typemap->type));
+ exit(1);
+ }
+}
+
static const char *filter_out[] = {
"bufs",
"cleanup",
@@ -814,6 +870,9 @@ int main(int argc, char *argv[])
}
}
+ /* Normalize types */
+ check_normalize_types(res);
+
for (i = 0; implies[i].flag; i++) {
if (data_node_hash_get(res, implies[i].flag)) {
for (j = 0; implies[i].implies[j]; j++) {
diff --git a/docparse/tests/include.c.json b/docparse/tests/include.c.json
index b4ef1ccda..b7c636e82 100644
--- a/docparse/tests/include.c.json
+++ b/docparse/tests/include.c.json
@@ -1,4 +1,4 @@
"include": {
- "test_variants": "10",
+ "test_variants": 10,
"fname": "include.c"
}
\ No newline at end of file
diff --git a/docparse/tests/macro.c.json b/docparse/tests/macro.c.json
index 0dc73d8ec..c3f53ae43 100644
--- a/docparse/tests/macro.c.json
+++ b/docparse/tests/macro.c.json
@@ -1,4 +1,4 @@
"macro": {
- "test_variants": "10",
+ "test_variants": 10,
"fname": "macro.c"
}
\ No newline at end of file
diff --git a/docparse/tests/multiline_macro.c.json b/docparse/tests/multiline_macro.c.json
index bafd037da..634516242 100644
--- a/docparse/tests/multiline_macro.c.json
+++ b/docparse/tests/multiline_macro.c.json
@@ -1,4 +1,4 @@
"multiline_macro": {
- "test_variants": "10",
+ "test_variants": 10,
"fname": "multiline_macro.c"
}
\ No newline at end of file
--
2.32.0
--
Mailing list info: https://lists.linux.it/listinfo/ltp
^ permalink raw reply related [flat|nested] 27+ messages in thread* Re: [LTP] [PATCH v2 5/7] docparse: Add type normalization
2021-11-01 14:53 ` [LTP] [PATCH v2 5/7] docparse: Add type normalization Cyril Hrubis
@ 2021-11-02 11:52 ` Richard Palethorpe
2021-11-02 12:06 ` Cyril Hrubis
0 siblings, 1 reply; 27+ messages in thread
From: Richard Palethorpe @ 2021-11-02 11:52 UTC (permalink / raw)
To: Cyril Hrubis; +Cc: ltp
Hello,
Cyril Hrubis <chrubis@suse.cz> writes:
> For now just for .test_variants.
>
> There are two reasons for this:
>
> - This code makes sure that we get right value parsed and aborts the
> compilation if the parser got confused. This part is important since
> if the testrunners are going to use the metadata the data in there
> must be correct.
I can't resist saying this would likely be solved by Sparse and its
understanding of C types. However in the present context it makes
perfect sense.
>
> - And much less important it makes the resulting json nicer to read
>
> Signed-off-by: Cyril Hrubis <chrubis@suse.cz>
Reviewed-by: rpalethorpe@suse.com
--
Thank you,
Richard.
--
Mailing list info: https://lists.linux.it/listinfo/ltp
^ permalink raw reply [flat|nested] 27+ messages in thread
* Re: [LTP] [PATCH v2 5/7] docparse: Add type normalization
2021-11-02 11:52 ` Richard Palethorpe
@ 2021-11-02 12:06 ` Cyril Hrubis
0 siblings, 0 replies; 27+ messages in thread
From: Cyril Hrubis @ 2021-11-02 12:06 UTC (permalink / raw)
To: Richard Palethorpe; +Cc: ltp
Hi!
> > - This code makes sure that we get right value parsed and aborts the
> > compilation if the parser got confused. This part is important since
> > if the testrunners are going to use the metadata the data in there
> > must be correct.
>
> I can't resist saying this would likely be solved by Sparse and its
> understanding of C types. However in the present context it makes
> perfect sense.
We would still have to check if we got right results, but yes, this is
one of the places where a proper parser would make things easier.
--
Cyril Hrubis
chrubis@suse.cz
--
Mailing list info: https://lists.linux.it/listinfo/ltp
^ permalink raw reply [flat|nested] 27+ messages in thread
* [LTP] [PATCH v2 6/7] docparse: Group data to 'testsuite' and 'defaults'
2021-11-01 14:53 [LTP] [PATCH v2 0/7] docparse improvements Cyril Hrubis
` (4 preceding siblings ...)
2021-11-01 14:53 ` [LTP] [PATCH v2 5/7] docparse: Add type normalization Cyril Hrubis
@ 2021-11-01 14:53 ` Cyril Hrubis
2021-11-01 14:53 ` [LTP] [PATCH v2 7/7] docparse: Split into metadata and docparse Cyril Hrubis
2021-11-01 14:55 ` [LTP] [PATCH v2 0/7] docparse improvements Cyril Hrubis
7 siblings, 0 replies; 27+ messages in thread
From: Cyril Hrubis @ 2021-11-01 14:53 UTC (permalink / raw)
To: ltp
Group all data belonging to testsuite info to 'testsuite' object and
move default timeout to 'defaults' object. This makes the JSON structure
a bit cleaner and easier to understand.
Signed-off-by: Cyril Hrubis <chrubis@suse.cz>
---
docparse/parse.sh | 16 ++++++++++------
docparse/testinfo.pl | 16 ++++++++--------
2 files changed, 18 insertions(+), 14 deletions(-)
diff --git a/docparse/parse.sh b/docparse/parse.sh
index 2ace34fa0..52d9a5cbf 100755
--- a/docparse/parse.sh
+++ b/docparse/parse.sh
@@ -15,12 +15,16 @@ if [ -d .git ]; then
fi
echo '{'
-echo ' "testsuite": "Linux Test Project",'
-echo ' "testsuite_short": "LTP",'
-echo ' "url": "https://github.com/linux-test-project/ltp/",'
-echo ' "scm_url_base": "https://github.com/linux-test-project/ltp/tree/master/",'
-echo ' "timeout": 300,'
-echo " \"version\": \"$version\","
+echo ' "testsuite": {'
+echo ' "name": "Linux Test Project",'
+echo ' "short_name": "LTP",'
+echo ' "url": "https://github.com/linux-test-project/ltp/",'
+echo ' "scm_url_base": "https://github.com/linux-test-project/ltp/tree/master/",'
+echo " \"version\": \"$version\""
+echo ' },'
+echo ' "defaults": {'
+echo ' "timeout": 300'
+echo ' },'
echo ' "tests": {'
first=1
diff --git a/docparse/testinfo.pl b/docparse/testinfo.pl
index c11064c05..891619532 100755
--- a/docparse/testinfo.pl
+++ b/docparse/testinfo.pl
@@ -164,9 +164,9 @@ sub content_about
my $json = shift;
my $content;
- $content .= print_defined("URL", $json->{'url'});
- $content .= print_defined("Version", $json->{'version'});
- $content .= print_defined("Default timeout", $json->{'timeout'}, "seconds");
+ $content .= print_defined("URL", $json->{'testsuite'}->{'url'});
+ $content .= print_defined("Version", $json->{'testsuite'}->{'version'});
+ $content .= print_defined("Default timeout", $json->{'defaults'}->{'timeout'}, "seconds");
return $content;
}
@@ -360,10 +360,10 @@ sub content_all_tests
$content .= h3($name);
$content .= $letters;
- if (defined($json->{'scm_url_base'}) &&
+ if (defined($json->{'testsuite'}->{'scm_url_base'}) &&
defined($json->{'tests'}{$name}{fname})) {
$content .= paragraph(html_a(tag_url("fname", $json->{'tests'}{$name}{fname},
- $json->{'scm_url_base'}), "source"));
+ $json->{'testsuite'}->{'scm_url_base'}), "source"));
}
if (defined $json->{'tests'}{$name}{doc}) {
@@ -386,7 +386,7 @@ sub content_all_tests
$content .= paragraph("Test timeout is $json->{'tests'}{$name}{timeout} seconds");
}
} else {
- $content .= paragraph("Test timeout defaults to $json->{'timeout'} seconds");
+ $content .= paragraph("Test timeout defaults to $json->{'defaults'}->{'timeout'} seconds");
}
my $tmp2 = undef;
@@ -463,7 +463,7 @@ my $json = decode_json(load_json($ARGV[0]));
my $config = [
{
file => "about.txt",
- title => h2("About $json->{'testsuite'}"),
+ title => h2("About $json->{'testsuite'}->{'name'}"),
content => \&content_about,
},
{
@@ -495,7 +495,7 @@ EOL
for my $c (@{$config}) {
$content .= "include::$c->{'file'}\[\]\n";
}
- print_asciidoc_page($fh, $json, h1($json->{'testsuite_short'} . " test catalog"), $content);
+ print_asciidoc_page($fh, $json, h1($json->{'testsuite'}->{'short_name'} . " test catalog"), $content);
}
for my $c (@{$config}) {
--
2.32.0
--
Mailing list info: https://lists.linux.it/listinfo/ltp
^ permalink raw reply related [flat|nested] 27+ messages in thread* [LTP] [PATCH v2 7/7] docparse: Split into metadata and docparse
2021-11-01 14:53 [LTP] [PATCH v2 0/7] docparse improvements Cyril Hrubis
` (5 preceding siblings ...)
2021-11-01 14:53 ` [LTP] [PATCH v2 6/7] docparse: Group data to 'testsuite' and 'defaults' Cyril Hrubis
@ 2021-11-01 14:53 ` Cyril Hrubis
2021-11-01 15:14 ` Cyril Hrubis
2021-11-02 15:22 ` Petr Vorel
2021-11-01 14:55 ` [LTP] [PATCH v2 0/7] docparse improvements Cyril Hrubis
7 siblings, 2 replies; 27+ messages in thread
From: Cyril Hrubis @ 2021-11-01 14:53 UTC (permalink / raw)
To: ltp
That way the metadata are build and installed unconditionally as they
are going to be integral part of the test execution framework.
The metadata file is also renamed to ltp.json and installed into
$DESTDIR/metadata/ltp.json.
The docparse build is triggered from the metadata Makefile since it has
to be done once the ltp.json is fully generated.
Signed-off-by: Cyril Hrubis <chrubis@suse.cz>
---
Makefile | 5 +---
docparse/.gitignore | 2 --
docparse/Makefile | 12 ++-------
metadata/.gitignore | 2 ++
metadata/Makefile | 26 +++++++++++++++++++
{docparse => metadata}/data_storage.h | 0
docparse/docparse.c => metadata/metaparse.c | 0
{docparse => metadata}/parse.sh | 0
{docparse => metadata}/tests/Makefile | 0
{docparse => metadata}/tests/array_size01.c | 0
.../tests/array_size01.c.json | 0
{docparse => metadata}/tests/array_size02.c | 0
.../tests/array_size02.c.json | 0
{docparse => metadata}/tests/array_size03.c | 0
.../tests/array_size03.c.json | 0
{docparse => metadata}/tests/array_size04.c | 0
.../tests/array_size04.c.json | 0
{docparse => metadata}/tests/empty_struct.c | 0
.../tests/empty_struct.c.json | 0
{docparse => metadata}/tests/expand_flags.c | 0
.../tests/expand_flags.c.json | 0
{docparse => metadata}/tests/include.c | 0
{docparse => metadata}/tests/include.c.json | 0
{docparse => metadata}/tests/include.h | 0
{docparse => metadata}/tests/macro.c | 0
{docparse => metadata}/tests/macro.c.json | 0
.../tests/multiline_macro.c | 0
.../tests/multiline_macro.c.json | 0
{docparse => metadata}/tests/tags.c | 0
{docparse => metadata}/tests/tags.c.json | 0
{docparse => metadata}/tests/test.sh | 2 +-
31 files changed, 32 insertions(+), 17 deletions(-)
create mode 100644 metadata/.gitignore
create mode 100644 metadata/Makefile
rename {docparse => metadata}/data_storage.h (100%)
rename docparse/docparse.c => metadata/metaparse.c (100%)
rename {docparse => metadata}/parse.sh (100%)
rename {docparse => metadata}/tests/Makefile (100%)
rename {docparse => metadata}/tests/array_size01.c (100%)
rename {docparse => metadata}/tests/array_size01.c.json (100%)
rename {docparse => metadata}/tests/array_size02.c (100%)
rename {docparse => metadata}/tests/array_size02.c.json (100%)
rename {docparse => metadata}/tests/array_size03.c (100%)
rename {docparse => metadata}/tests/array_size03.c.json (100%)
rename {docparse => metadata}/tests/array_size04.c (100%)
rename {docparse => metadata}/tests/array_size04.c.json (100%)
rename {docparse => metadata}/tests/empty_struct.c (100%)
rename {docparse => metadata}/tests/empty_struct.c.json (100%)
rename {docparse => metadata}/tests/expand_flags.c (100%)
rename {docparse => metadata}/tests/expand_flags.c.json (100%)
rename {docparse => metadata}/tests/include.c (100%)
rename {docparse => metadata}/tests/include.c.json (100%)
rename {docparse => metadata}/tests/include.h (100%)
rename {docparse => metadata}/tests/macro.c (100%)
rename {docparse => metadata}/tests/macro.c.json (100%)
rename {docparse => metadata}/tests/multiline_macro.c (100%)
rename {docparse => metadata}/tests/multiline_macro.c.json (100%)
rename {docparse => metadata}/tests/tags.c (100%)
rename {docparse => metadata}/tests/tags.c.json (100%)
rename {docparse => metadata}/tests/test.sh (88%)
diff --git a/Makefile b/Makefile
index 4e37362f9..63010d531 100644
--- a/Makefile
+++ b/Makefile
@@ -62,10 +62,7 @@ $(1):: | $$(abs_top_builddir)/$$(basename $$(subst -,.,$(1)))
endif
endef
-COMMON_TARGETS += testcases tools
-ifeq ($(WITH_METADATA),yes)
-COMMON_TARGETS += docparse
-endif
+COMMON_TARGETS += testcases tools metadata
# Don't want to nuke the original files if we're installing in-build-tree.
ifneq ($(BUILD_TREE_STATE),$(BUILD_TREE_SRCDIR_INSTALL))
diff --git a/docparse/.gitignore b/docparse/.gitignore
index 7a87b4234..d786a4762 100644
--- a/docparse/.gitignore
+++ b/docparse/.gitignore
@@ -1,7 +1,5 @@
/*.txt
/docbook-xsl.css
-/docparse
-/metadata.json
/metadata.html
/metadata.pdf
/metadata.chunked/
diff --git a/docparse/Makefile b/docparse/Makefile
index e2defad38..ab30efc80 100644
--- a/docparse/Makefile
+++ b/docparse/Makefile
@@ -29,7 +29,6 @@ METADATA_GENERATOR_PARAMS += -v
endif
CLEAN_TARGETS := *.css *.js *.txt
-MAKE_TARGETS := metadata.json
ifeq ($(WITH_METADATA_HTML),yes)
MAKE_TARGETS += metadata.html
@@ -42,8 +41,6 @@ ifeq ($(WITH_METADATA_PDF),yes)
MAKE_TARGETS += metadata.pdf
endif
-HOST_MAKE_TARGETS := docparse
-
INSTALL_DIR = metadata
INSTALL_TARGETS = *.css *.js
@@ -51,13 +48,8 @@ ifndef METADATA_GENERATOR
METADATA_GENERATOR := asciidoctor
endif
-.PHONY: metadata.json
-
-metadata.json: docparse
- $(abs_srcdir)/parse.sh > metadata.json
-
-txt: metadata.json
- $(abs_srcdir)/testinfo.pl metadata.json
+txt: ../metadata/ltp.json
+ $(abs_srcdir)/testinfo.pl ../metadata/ltp.json
ifeq ($(WITH_METADATA_HTML),yes)
metadata.html: txt
diff --git a/metadata/.gitignore b/metadata/.gitignore
new file mode 100644
index 000000000..07d2fd6ff
--- /dev/null
+++ b/metadata/.gitignore
@@ -0,0 +1,2 @@
+metaparse
+ltp.json
diff --git a/metadata/Makefile b/metadata/Makefile
new file mode 100644
index 000000000..76a3aed94
--- /dev/null
+++ b/metadata/Makefile
@@ -0,0 +1,26 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+# Copyright (c) 2020 Cyril Hrubis <chrubis@suse.cz>
+
+top_srcdir ?= ..
+
+include $(top_srcdir)/include/mk/env_pre.mk
+include $(top_srcdir)/include/mk/functions.mk
+
+MAKE_TARGETS := ltp.json docparse
+HOST_MAKE_TARGETS := metaparse
+INSTALL_DIR = metadata
+
+.PHONY: ltp.json
+
+ltp.json: metaparse
+ $(abs_srcdir)/parse.sh > ltp.json
+
+docparse: ltp.json
+ifeq ($(WITH_METADATA),yes)
+ $(MAKE) -C ../docparse/
+endif
+
+test:
+ $(MAKE) -C tests/ test
+
+include $(top_srcdir)/include/mk/generic_leaf_target.mk
diff --git a/docparse/data_storage.h b/metadata/data_storage.h
similarity index 100%
rename from docparse/data_storage.h
rename to metadata/data_storage.h
diff --git a/docparse/docparse.c b/metadata/metaparse.c
similarity index 100%
rename from docparse/docparse.c
rename to metadata/metaparse.c
diff --git a/docparse/parse.sh b/metadata/parse.sh
similarity index 100%
rename from docparse/parse.sh
rename to metadata/parse.sh
diff --git a/docparse/tests/Makefile b/metadata/tests/Makefile
similarity index 100%
rename from docparse/tests/Makefile
rename to metadata/tests/Makefile
diff --git a/docparse/tests/array_size01.c b/metadata/tests/array_size01.c
similarity index 100%
rename from docparse/tests/array_size01.c
rename to metadata/tests/array_size01.c
diff --git a/docparse/tests/array_size01.c.json b/metadata/tests/array_size01.c.json
similarity index 100%
rename from docparse/tests/array_size01.c.json
rename to metadata/tests/array_size01.c.json
diff --git a/docparse/tests/array_size02.c b/metadata/tests/array_size02.c
similarity index 100%
rename from docparse/tests/array_size02.c
rename to metadata/tests/array_size02.c
diff --git a/docparse/tests/array_size02.c.json b/metadata/tests/array_size02.c.json
similarity index 100%
rename from docparse/tests/array_size02.c.json
rename to metadata/tests/array_size02.c.json
diff --git a/docparse/tests/array_size03.c b/metadata/tests/array_size03.c
similarity index 100%
rename from docparse/tests/array_size03.c
rename to metadata/tests/array_size03.c
diff --git a/docparse/tests/array_size03.c.json b/metadata/tests/array_size03.c.json
similarity index 100%
rename from docparse/tests/array_size03.c.json
rename to metadata/tests/array_size03.c.json
diff --git a/docparse/tests/array_size04.c b/metadata/tests/array_size04.c
similarity index 100%
rename from docparse/tests/array_size04.c
rename to metadata/tests/array_size04.c
diff --git a/docparse/tests/array_size04.c.json b/metadata/tests/array_size04.c.json
similarity index 100%
rename from docparse/tests/array_size04.c.json
rename to metadata/tests/array_size04.c.json
diff --git a/docparse/tests/empty_struct.c b/metadata/tests/empty_struct.c
similarity index 100%
rename from docparse/tests/empty_struct.c
rename to metadata/tests/empty_struct.c
diff --git a/docparse/tests/empty_struct.c.json b/metadata/tests/empty_struct.c.json
similarity index 100%
rename from docparse/tests/empty_struct.c.json
rename to metadata/tests/empty_struct.c.json
diff --git a/docparse/tests/expand_flags.c b/metadata/tests/expand_flags.c
similarity index 100%
rename from docparse/tests/expand_flags.c
rename to metadata/tests/expand_flags.c
diff --git a/docparse/tests/expand_flags.c.json b/metadata/tests/expand_flags.c.json
similarity index 100%
rename from docparse/tests/expand_flags.c.json
rename to metadata/tests/expand_flags.c.json
diff --git a/docparse/tests/include.c b/metadata/tests/include.c
similarity index 100%
rename from docparse/tests/include.c
rename to metadata/tests/include.c
diff --git a/docparse/tests/include.c.json b/metadata/tests/include.c.json
similarity index 100%
rename from docparse/tests/include.c.json
rename to metadata/tests/include.c.json
diff --git a/docparse/tests/include.h b/metadata/tests/include.h
similarity index 100%
rename from docparse/tests/include.h
rename to metadata/tests/include.h
diff --git a/docparse/tests/macro.c b/metadata/tests/macro.c
similarity index 100%
rename from docparse/tests/macro.c
rename to metadata/tests/macro.c
diff --git a/docparse/tests/macro.c.json b/metadata/tests/macro.c.json
similarity index 100%
rename from docparse/tests/macro.c.json
rename to metadata/tests/macro.c.json
diff --git a/docparse/tests/multiline_macro.c b/metadata/tests/multiline_macro.c
similarity index 100%
rename from docparse/tests/multiline_macro.c
rename to metadata/tests/multiline_macro.c
diff --git a/docparse/tests/multiline_macro.c.json b/metadata/tests/multiline_macro.c.json
similarity index 100%
rename from docparse/tests/multiline_macro.c.json
rename to metadata/tests/multiline_macro.c.json
diff --git a/docparse/tests/tags.c b/metadata/tests/tags.c
similarity index 100%
rename from docparse/tests/tags.c
rename to metadata/tests/tags.c
diff --git a/docparse/tests/tags.c.json b/metadata/tests/tags.c.json
similarity index 100%
rename from docparse/tests/tags.c.json
rename to metadata/tests/tags.c.json
diff --git a/docparse/tests/test.sh b/metadata/tests/test.sh
similarity index 88%
rename from docparse/tests/test.sh
rename to metadata/tests/test.sh
index 767cc464c..c11f0e496 100755
--- a/docparse/tests/test.sh
+++ b/metadata/tests/test.sh
@@ -3,7 +3,7 @@
fail=0
for i in *.c; do
- ../docparse $i > tmp.json
+ ../metaparse $i > tmp.json
if ! diff tmp.json $i.json &> /dev/null; then
echo "***"
echo "$i output differs!"
--
2.32.0
--
Mailing list info: https://lists.linux.it/listinfo/ltp
^ permalink raw reply related [flat|nested] 27+ messages in thread* Re: [LTP] [PATCH v2 7/7] docparse: Split into metadata and docparse
2021-11-01 14:53 ` [LTP] [PATCH v2 7/7] docparse: Split into metadata and docparse Cyril Hrubis
@ 2021-11-01 15:14 ` Cyril Hrubis
2021-11-02 15:22 ` Petr Vorel
1 sibling, 0 replies; 27+ messages in thread
From: Cyril Hrubis @ 2021-11-01 15:14 UTC (permalink / raw)
To: ltp
Hi!
> That way the metadata are build and installed unconditionally as they
> are going to be integral part of the test execution framework.
>
> The metadata file is also renamed to ltp.json and installed into
> $DESTDIR/metadata/ltp.json.
>
> The docparse build is triggered from the metadata Makefile since it has
> to be done once the ltp.json is fully generated.
I have forgotten to update the parse.sh script, this patch is broken
without:
diff --git a/metadata/parse.sh b/metadata/parse.sh
index 52d9a5cbf..b43d024c6 100755
--- a/metadata/parse.sh
+++ b/metadata/parse.sh
@@ -30,7 +30,7 @@ echo ' "tests": {'
first=1
for test in `find testcases/ -name '*.c'`; do
- a=$($top_builddir/docparse/docparse -Iinclude -Itestcases/kernel/syscalls/utils/ "$test")
+ a=$($top_builddir/metadata/metaparse -Iinclude -Itestcases/kernel/syscalls/utils/ "$test")
if [ -n "$a" ]; then
if [ -z "$first" ]; then
Sorry for that.
--
Cyril Hrubis
chrubis@suse.cz
--
Mailing list info: https://lists.linux.it/listinfo/ltp
^ permalink raw reply related [flat|nested] 27+ messages in thread* Re: [LTP] [PATCH v2 7/7] docparse: Split into metadata and docparse
2021-11-01 14:53 ` [LTP] [PATCH v2 7/7] docparse: Split into metadata and docparse Cyril Hrubis
2021-11-01 15:14 ` Cyril Hrubis
@ 2021-11-02 15:22 ` Petr Vorel
2021-11-02 15:28 ` Cyril Hrubis
1 sibling, 1 reply; 27+ messages in thread
From: Petr Vorel @ 2021-11-02 15:22 UTC (permalink / raw)
To: Cyril Hrubis; +Cc: ltp
Hi Cyril,
> That way the metadata are build and installed unconditionally as they
> are going to be integral part of the test execution framework.
> The metadata file is also renamed to ltp.json and installed into
> $DESTDIR/metadata/ltp.json.
> The docparse build is triggered from the metadata Makefile since it has
> to be done once the ltp.json is fully generated.
While I agree with this, there is a dependency loop. Or am I missing something?
ltp.git/metadata $ make
HOSTCC metadata/metaparse
ltp.git/metadata/parse.sh > ltp.json
ltp.git/metadata/parse.sh: line 33: ltp.git/metadata/../docparse/docparse: No such file or directory
# => OK let's go to docparse and make first
ltp.git/metadata $ cd ../docparse/
ltp.git/docparse $ make
docparse/testinfo.pl ../metadata/ltp.json
'"' expected, at character offset 294 (before "(end of string)") at docparse/testinfo.pl line 461.
# => OK, let's cleanup docparse first
ltp.git/docparse $ cd ../metadata
ltp.git/metadata $ rm -rf *; git reset --hard
ltp.git/metadata $ cd ../docparse/
ltp.git/docparse $ make
make: *** No rule to make target '../metadata/ltp.json', needed by 'txt'. Stop.
# => huh, dependency loop?
And indeed CI confirms that:
https://github.com/pevik/ltp/actions/runs/1412787433
Kind regards,
Petr
--
Mailing list info: https://lists.linux.it/listinfo/ltp
^ permalink raw reply [flat|nested] 27+ messages in thread
* Re: [LTP] [PATCH v2 7/7] docparse: Split into metadata and docparse
2021-11-02 15:22 ` Petr Vorel
@ 2021-11-02 15:28 ` Cyril Hrubis
2021-11-02 15:39 ` Petr Vorel
0 siblings, 1 reply; 27+ messages in thread
From: Cyril Hrubis @ 2021-11-02 15:28 UTC (permalink / raw)
To: Petr Vorel; +Cc: ltp
Hi!
> ltp.git/metadata $ make
> HOSTCC metadata/metaparse
> ltp.git/metadata/parse.sh > ltp.json
> ltp.git/metadata/parse.sh: line 33: ltp.git/metadata/../docparse/docparse: No such file or directory
> # => OK let's go to docparse and make first
The docparse build should be triggered from the metadata directory,
since the ltp.json has to be generated first.
But I have forgotten to update the parse.sh script. Have you seen the
reply to this patch?
--
Cyril Hrubis
chrubis@suse.cz
--
Mailing list info: https://lists.linux.it/listinfo/ltp
^ permalink raw reply [flat|nested] 27+ messages in thread
* Re: [LTP] [PATCH v2 7/7] docparse: Split into metadata and docparse
2021-11-02 15:28 ` Cyril Hrubis
@ 2021-11-02 15:39 ` Petr Vorel
2021-11-02 16:04 ` Cyril Hrubis
0 siblings, 1 reply; 27+ messages in thread
From: Petr Vorel @ 2021-11-02 15:39 UTC (permalink / raw)
To: Cyril Hrubis; +Cc: ltp
> Hi!
> > ltp.git/metadata $ make
> > HOSTCC metadata/metaparse
> > ltp.git/metadata/parse.sh > ltp.json
> > ltp.git/metadata/parse.sh: line 33: ltp.git/metadata/../docparse/docparse: No such file or directory
> > # => OK let's go to docparse and make first
> The docparse build should be triggered from the metadata directory,
> since the ltp.json has to be generated first.
> But I have forgotten to update the parse.sh script. Have you seen the
> reply to this patch?
I see, thx!
Kind regards,
Petr
--
Mailing list info: https://lists.linux.it/listinfo/ltp
^ permalink raw reply [flat|nested] 27+ messages in thread
* Re: [LTP] [PATCH v2 7/7] docparse: Split into metadata and docparse
2021-11-02 15:39 ` Petr Vorel
@ 2021-11-02 16:04 ` Cyril Hrubis
2021-11-03 11:39 ` Cyril Hrubis
0 siblings, 1 reply; 27+ messages in thread
From: Cyril Hrubis @ 2021-11-02 16:04 UTC (permalink / raw)
To: Petr Vorel; +Cc: ltp
Hi!
> > > ltp.git/metadata $ make
> > > HOSTCC metadata/metaparse
> > > ltp.git/metadata/parse.sh > ltp.json
> > > ltp.git/metadata/parse.sh: line 33: ltp.git/metadata/../docparse/docparse: No such file or directory
> > > # => OK let's go to docparse and make first
>
> > The docparse build should be triggered from the metadata directory,
> > since the ltp.json has to be generated first.
>
> > But I have forgotten to update the parse.sh script. Have you seen the
> > reply to this patch?
>
> I see, thx!
Also looking at the Makefile it may not work with out-of-tree build, so
we may need this as well:
diff --git a/metadata/Makefile b/metadata/Makefile
index 76a3aed94..9b88d8302 100644
--- a/metadata/Makefile
+++ b/metadata/Makefile
@@ -17,10 +17,10 @@ ltp.json: metaparse
docparse: ltp.json
ifeq ($(WITH_METADATA),yes)
- $(MAKE) -C ../docparse/
+ $(MAKE) -C $(top_srcdir)/docparse/
endif
test:
- $(MAKE) -C tests/ test
+ $(MAKE) -C $(abs_srcdir)/tests/ test
include $(top_srcdir)/include/mk/generic_leaf_target.mk
--
Cyril Hrubis
chrubis@suse.cz
--
Mailing list info: https://lists.linux.it/listinfo/ltp
^ permalink raw reply related [flat|nested] 27+ messages in thread
* Re: [LTP] [PATCH v2 7/7] docparse: Split into metadata and docparse
2021-11-02 16:04 ` Cyril Hrubis
@ 2021-11-03 11:39 ` Cyril Hrubis
0 siblings, 0 replies; 27+ messages in thread
From: Cyril Hrubis @ 2021-11-03 11:39 UTC (permalink / raw)
To: Petr Vorel; +Cc: ltp
Hi!
> Also looking at the Makefile it may not work with out-of-tree build, so
> we may need this as well:
>
> diff --git a/metadata/Makefile b/metadata/Makefile
> index 76a3aed94..9b88d8302 100644
> --- a/metadata/Makefile
> +++ b/metadata/Makefile
> @@ -17,10 +17,10 @@ ltp.json: metaparse
>
> docparse: ltp.json
> ifeq ($(WITH_METADATA),yes)
> - $(MAKE) -C ../docparse/
> + $(MAKE) -C $(top_srcdir)/docparse/
> endif
>
> test:
> - $(MAKE) -C tests/ test
> + $(MAKE) -C $(abs_srcdir)/tests/ test
>
> include $(top_srcdir)/include/mk/generic_leaf_target.mk
And it's more complex than this, I guess that I will fix the out-of-tree
build send send another version of the patchset.
--
Cyril Hrubis
chrubis@suse.cz
--
Mailing list info: https://lists.linux.it/listinfo/ltp
^ permalink raw reply [flat|nested] 27+ messages in thread
* Re: [LTP] [PATCH v2 0/7] docparse improvements
2021-11-01 14:53 [LTP] [PATCH v2 0/7] docparse improvements Cyril Hrubis
` (6 preceding siblings ...)
2021-11-01 14:53 ` [LTP] [PATCH v2 7/7] docparse: Split into metadata and docparse Cyril Hrubis
@ 2021-11-01 14:55 ` Cyril Hrubis
7 siblings, 0 replies; 27+ messages in thread
From: Cyril Hrubis @ 2021-11-01 14:55 UTC (permalink / raw)
To: ltp
Hi!
> v2:
> - Cleaned up the patchset a bit
>
> pvorel:
> - Added Makefile to metadata tests
> - Rename the expected output files to foo.c.json
+ Split the docparse directory into metadata and docparse
instead of removing the configure checks
--
Cyril Hrubis
chrubis@suse.cz
--
Mailing list info: https://lists.linux.it/listinfo/ltp
^ permalink raw reply [flat|nested] 27+ messages in thread