[igt-dev] [PATCH i-g-t 1/2] runner: Ensure generated json is properly UTF8-encoded

* [igt-dev] [PATCH i-g-t 1/2] runner: Ensure generated json is properly UTF8-encoded
@ 2019-12-05 13:02 Petri Latvala
  2019-12-05 13:02 ` [igt-dev] [PATCH i-g-t 2/2] runner/json_tests: Test handling of unprintable output from tests Petri Latvala
                   ` (2 more replies)
  0 siblings, 3 replies; 5+ messages in thread
From: Petri Latvala @ 2019-12-05 13:02 UTC (permalink / raw)
  To: igt-dev; +Cc: Petri Latvala

Sometimes tests output garbage (e.g. due to extreme occurrences of
https://gitlab.freedesktop.org/drm/igt-gpu-tools/issues/55) but we
need to present the garbage as results.

We already ignore any test output after the first \0, and for the rest
of the bytes that are not directly UTF-8 as-is, we can quite easily
represent them with two-byte UTF-8 encoding.

libjson-c already expects the string you feed it through
json_object_new_string* functions to be UTF-8.

Signed-off-by: Petri Latvala <petri.latvala@intel.com>
Cc: Arkadiusz Hiler <arkadiusz.hiler@intel.com>
---
 runner/resultgen.c | 53 ++++++++++++++++++++++++++++++++++------------
 1 file changed, 40 insertions(+), 13 deletions(-)

diff --git a/runner/resultgen.c b/runner/resultgen.c
index aa438000..92030431 100644
--- a/runner/resultgen.c
+++ b/runner/resultgen.c
@@ -365,6 +365,33 @@ static void free_matches(struct matches *matches)
 	free(matches->items);
 }
 
+static struct json_object *new_escaped_json_string(const char *buf, size_t len)
+{
+	struct json_object *obj;
+	char *str = NULL;
+	size_t strsize = 0;
+	size_t i;
+
+	for (i = 0; i < len; i++) {
+		if (buf[i] > 0 && buf[i] < 128) {
+			str = realloc(str, strsize + 1);
+			str[strsize] = buf[i];
+			++strsize;
+		} else {
+			/* Encode > 128 character to UTF-8. */
+			str = realloc(str, strsize + 2);
+			str[strsize] = ((unsigned char)buf[i] >> 6) | 0xC0;
+			str[strsize + 1] = ((unsigned char)buf[i] & 0x3F) | 0x80;
+			strsize += 2;
+		}
+	}
+
+	obj = json_object_new_string_len(str, strsize);
+	free(str);
+
+	return obj;
+}
+
 static bool fill_from_output(int fd, const char *binary, const char *key,
 			     struct subtest_list *subtests,
 			     struct json_object *tests)
@@ -419,11 +446,11 @@ static bool fill_from_output(int fd, const char *binary, const char *key,
 		current_test = get_or_create_json_object(tests, piglit_name);
 
 		json_object_object_add(current_test, key,
-				       json_object_new_string_len(buf, statbuf.st_size));
+				       new_escaped_json_string(buf, statbuf.st_size));
 		if (igt_version)
 			json_object_object_add(current_test, "igt-version",
-					       json_object_new_string_len(igt_version,
-									  igt_version_len));
+					       new_escaped_json_string(igt_version,
+								       igt_version_len));
 
 		return true;
 	}
@@ -520,12 +547,12 @@ static bool fill_from_output(int fd, const char *binary, const char *key,
 		}
 
 		json_object_object_add(current_test, key,
-				       json_object_new_string_len(beg, end - beg));
+				       new_escaped_json_string(beg, end - beg));
 
 		if (igt_version) {
 			json_object_object_add(current_test, "igt-version",
-					       json_object_new_string_len(igt_version,
-									  igt_version_len));
+					       new_escaped_json_string(igt_version,
+								       igt_version_len));
 		}
 
 		if (!json_object_object_get_ex(current_test, "result", NULL)) {
@@ -607,11 +634,11 @@ static bool fill_from_output(int fd, const char *binary, const char *key,
 				current_dynamic_test = get_or_create_json_object(tests, dynamic_piglit_name);
 
 				json_object_object_add(current_dynamic_test, key,
-						       json_object_new_string_len(dynbeg, dynend - dynbeg));
+						       new_escaped_json_string(dynbeg, dynend - dynbeg));
 				if (igt_version)
 					json_object_object_add(current_dynamic_test, "igt-version",
-							       json_object_new_string_len(igt_version,
-											  igt_version_len));
+							       new_escaped_json_string(igt_version,
+										       igt_version_len));
 
 				if (!json_object_object_get_ex(current_dynamic_test, "result", NULL)) {
 					const char *dynresulttext;
@@ -764,11 +791,11 @@ static void add_dmesg(struct json_object *obj,
 		      const char *warnings, size_t warningslen)
 {
 	json_object_object_add(obj, "dmesg",
-			       json_object_new_string_len(dmesg, dmesglen));
+			       new_escaped_json_string(dmesg, dmesglen));
 
 	if (warnings) {
 		json_object_object_add(obj, "dmesg-warnings",
-				       json_object_new_string_len(warnings, warningslen));
+				       new_escaped_json_string(warnings, warningslen));
 	}
 }
 
@@ -1361,7 +1388,7 @@ struct json_object *generate_results_json(int dirfd)
 			r--;
 
 		json_object_object_add(obj, "uname",
-				       json_object_new_string_len(buf, r));
+				       new_escaped_json_string(buf, r));
 		close(fd);
 	}
 
@@ -1424,7 +1451,7 @@ struct json_object *generate_results_json(int dirfd)
 		s = read(fd, buf, sizeof(buf));
 
 		json_object_object_add(aborttest, "out",
-				       json_object_new_string_len(buf, s));
+				       new_escaped_json_string(buf, s));
 		json_object_object_add(aborttest, "err",
 				       json_object_new_string(""));
 		json_object_object_add(aborttest, "dmesg",
-- 
2.19.1

_______________________________________________
igt-dev mailing list
igt-dev@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/igt-dev

^ permalink raw reply related	[flat|nested] 5+ messages in thread