[Cocci] [PATCH 0/2] Add "use-patchdiff" option

[Cocci] [PATCH 0/2] Add "use-patchdiff" option

[Sumera Priyadarsini]

This patchset adds a feature to enable Coccinelle
to only check all those files in a directory which were
modified. It parses all the files obtained from the
output of "git diff" and checks them against the specified
cocci script.

An example for passing the "use-patchdiff" option is:

	spatch -D report --sp-file for_each_child.cocci --use-patchdiff drivers/gpu

One thing to note while using the command is that while it can be
invoked outside the target project directory, an absolute path
to the target folder should be specified in such cases as.

Sumera Priyadarsini (2):
  parsing_cocci: Add feature to check only modified files
  docs: manual: Add option description in spatch_options

 Makefile                        |   2 +-
 docs/manual/spatch_options.tex  |   5 ++
 enter.ml                        |  17 +++--
 globals/flag.ml                 |   2 +-
 globals/flag.mli                |   2 +-
 ocaml/coccilib.mli              |   1 +
 parsing_cocci/get_constants2.ml |   1 +
 parsing_cocci/patch_diff.ml     | 118 ++++++++++++++++++++++++++++++++
 parsing_cocci/patch_diff.mli    |   9 +++
 9 files changed, 148 insertions(+), 9 deletions(-)
 create mode 100755 parsing_cocci/patch_diff.ml
 create mode 100644 parsing_cocci/patch_diff.mli

-- 
2.31.1

_______________________________________________
Cocci mailing list
Cocci@systeme.lip6.fr
https://systeme.lip6.fr/mailman/listinfo/cocci

[Cocci] [PATCH 1/2] parsing_cocci: Add feature to check only modified files

[Sumera Priyadarsini]

This patch adds an option "use-patchdiff" to allow for
applying a semantic patch to only those files in a directory
where code additions have been made.

Usage: spatch -D <mode> --sp-file <cocci script> --use-patchdiff <target
directory>

Example: spatch -D report --sp-file for_each_child.cocci --use-patchdiff drivers/gpu

Signed-off-by: Sumera Priyadarsini <sylphrenadin@gmail.com>
---
 Makefile                        |   2 +-
 enter.ml                        |  17 +++--
 globals/flag.ml                 |   2 +-
 globals/flag.mli                |   2 +-
 ocaml/coccilib.mli              |   1 +
 parsing_cocci/get_constants2.ml |   1 +
 parsing_cocci/patch_diff.ml     | 118 ++++++++++++++++++++++++++++++++
 parsing_cocci/patch_diff.mli    |   9 +++
 8 files changed, 143 insertions(+), 9 deletions(-)
 create mode 100755 parsing_cocci/patch_diff.ml
 create mode 100644 parsing_cocci/patch_diff.mli

diff --git a/Makefile b/Makefile
index f8d3424c0..455e92395 100644
--- a/Makefile
+++ b/Makefile
@@ -44,7 +44,7 @@ SOURCES_parsing_cocci := \
 	parse_printf.ml parse_aux.ml cleanup_rules.ml disjdistr.ml \
 	parser_cocci_menhir.mly lexer_cocci.mll \
 	lexer_cli.mll lexer_script.mll \
-	cocci_grep.ml dpll.ml get_constants2.ml id_utils.ml git_grep.ml \
+	cocci_grep.ml dpll.ml get_constants2.ml id_utils.ml git_grep.ml patch_diff.ml \
 	adjacency.ml commas_on_lists.ml re_constraints.ml parse_cocci.ml \
 	command_line.ml cocci_args.ml
 SOURCES_parsing_c := \
diff --git a/enter.ml b/enter.ml
index 9643df782..587071aa8 100644
--- a/enter.ml
+++ b/enter.ml
@@ -18,9 +18,7 @@ module Inc = Includes
  * globals/config.ml, mainly a standard.h and standard.iso file *)
 
 let cocci_file = ref ""
-
 let opt_c_files = ref []
-
 let output_file = ref "" (* resulting code *)
 let tmp_dir = ref "" (* temporary files for parallelism *)
 let aux_file_suffix =
@@ -283,7 +281,6 @@ let print_version () =
 let short_options = [
   "--sp-file",  Arg.Set_string cocci_file,
   " <file> the semantic patch file";
-
   "--opt-c",
   Arg.String (fun filename ->
     if Sys.file_exists filename
@@ -370,6 +367,9 @@ let short_options = [
   "--use-coccigrep",
   Arg.Unit (function _ -> Flag.scanner := Flag.CocciGrep),
   "    find relevant files using cocci grep";
+  "--use-patchdiff",
+  Arg.Unit (function _ -> Flag.scanner := Flag.PatchDiff),
+  "    process files in the diff for a directory";
   "--patch",
     Arg.String (function s -> Flag.patch := Some (Cocci.normalize_path s)),
   ("    <dir> path name with respect to which a patch should be created\n"^
@@ -940,13 +940,18 @@ let idutils_filter (_,_,_,query) dir =
       Some
 	(files +>
 	 List.filter
-	   (fun file -> List.mem (Common.filesuffix file) suffixes))
+     (fun file -> List.mem (Common.filesuffix file) suffixes))
+
+let patchdiff_filter _ dir =
+  let struc = Patch_diff.getpatchdiff dir in
+  Some (List.map (function x -> x.Patch_diff.file_name) struc)
 
 let scanner_to_interpreter = function
     Flag.Glimpse -> glimpse_filter
   | Flag.IdUtils -> idutils_filter
   | Flag.CocciGrep -> coccigrep_filter
   | Flag.GitGrep -> gitgrep_filter
+  | Flag.PatchDiff -> patchdiff_filter
   | _ -> failwith "impossible"
 
 (*****************************************************************************)
@@ -1079,7 +1084,7 @@ let rec main_action xs =
 		     " or multiple files")
               | _, false, _, _, _ -> [List.map (fun x -> (x,None)) (x::xs)]
 	      |	_, true, "",
-		  (Flag.Glimpse|Flag.IdUtils|Flag.CocciGrep|Flag.GitGrep),
+		  (Flag.Glimpse|Flag.IdUtils|Flag.CocciGrep|Flag.GitGrep|Flag.PatchDiff),
 		  [] ->
 		    let interpreter = scanner_to_interpreter !Flag.scanner in
 		    let files =
@@ -1088,7 +1093,7 @@ let rec main_action xs =
 		      | Some files -> files in
                     files +> List.map (fun x -> [(x,None)])
               | _, true, s,
-		  (Flag.Glimpse|Flag.IdUtils|Flag.CocciGrep|Flag.GitGrep), _
+		  (Flag.Glimpse|Flag.IdUtils|Flag.CocciGrep|Flag.GitGrep|Flag.PatchDiff), _
 		when s <> "" ->
                   failwith "--use-xxx filters do not work with --kbuild"
                   (* normal *)
diff --git a/globals/flag.ml b/globals/flag.ml
index e1d01cb4c..dffe6cd80 100644
--- a/globals/flag.ml
+++ b/globals/flag.ml
@@ -16,7 +16,7 @@ let track_iso_usage = ref false
 
 let worth_trying_opt = ref true
 
-type scanner = IdUtils | Glimpse | CocciGrep | GitGrep | NoScanner
+type scanner = IdUtils | Glimpse | CocciGrep | GitGrep | PatchDiff | NoScanner
 let scanner = ref NoScanner
 
 let pyoutput = ref "coccilib.output.Console"
diff --git a/globals/flag.mli b/globals/flag.mli
index dadc7b6fc..0e8a3b063 100644
--- a/globals/flag.mli
+++ b/globals/flag.mli
@@ -4,7 +4,7 @@ val show_transinfo : bool ref
 val show_trying : bool ref
 val track_iso_usage : bool ref
 val worth_trying_opt : bool ref
-type scanner = IdUtils | Glimpse | CocciGrep | GitGrep | NoScanner
+type scanner = IdUtils | Glimpse | CocciGrep | GitGrep | PatchDiff | NoScanner
 val scanner : scanner ref
 val pyoutput : string ref
 val ocamlc : string ref
diff --git a/ocaml/coccilib.mli b/ocaml/coccilib.mli
index 3759e7787..41f89ccde 100644
--- a/ocaml/coccilib.mli
+++ b/ocaml/coccilib.mli
@@ -1372,6 +1372,7 @@ module Flag :
       | Glimpse
       | CocciGrep
       | GitGrep
+      | PatchDiff
       | NoScanner
     val scanner : scanner ref
     val pyoutput : string ref
diff --git a/parsing_cocci/get_constants2.ml b/parsing_cocci/get_constants2.ml
index 0f18aa037..2a02c2b6c 100644
--- a/parsing_cocci/get_constants2.ml
+++ b/parsing_cocci/get_constants2.ml
@@ -866,5 +866,6 @@ let get_constants rules neg_pos_vars virt =
     | Flag.IdUtils ->
 	(grep,None,coccigrep,interpret_idutils res)
     | Flag.CocciGrep | Flag.GitGrep -> (grep,None,coccigrep,None)
+    | Flag.PatchDiff -> (None, None, None, None)
     end
   else (None,None,None,None)
diff --git a/parsing_cocci/patch_diff.ml b/parsing_cocci/patch_diff.ml
new file mode 100755
index 000000000..a117349ab
--- /dev/null
+++ b/parsing_cocci/patch_diff.ml
@@ -0,0 +1,118 @@
+open Printf
+open Str
+
+(*Read file contents*)
+let read_whole_file filename =
+  let ch = open_in filename in
+  let s = really_input_string ch (in_channel_length ch) in
+  close_in ch;
+  s
+
+(*Breakdown split_result_list type to primtive type*)
+let rec decompose_list (l: Str.split_result list)  =
+  match l with
+  | [] -> []
+  | Text hd :: tl ->  decompose_list tl
+  | Delim hd :: tl -> hd ::  decompose_list tl
+;;
+
+(*Fetch output for bash commands*)
+let read_bash_help command =
+  let ic = Unix.open_process_in command in
+  let all_input = ref [] in
+  try
+    while true do
+      all_input := input_line ic :: !all_input
+    done;
+    !all_input
+  with
+    End_of_file -> close_in ic;
+    List.rev !all_input;;
+
+let get_root fpath = 
+  let current_dir = List.hd (read_bash_help "pwd") in
+  let command_get_root = "cd " ^ fpath ^ " && git rev-parse --show-toplevel" in
+  let root = List.hd (read_bash_help command_get_root) in
+  let command_return = "cd " ^ current_dir in
+  let ic = Unix.open_process_in command_return in
+  close_in ic;
+  root
+
+  let extract_numbers line =
+  let sep = String.split_on_char ',' line in
+  match sep with
+  | [l] -> (int_of_string l, int_of_string l)
+  | [f ; e] -> (int_of_string f, int_of_string e)
+  | _ -> failwith "no line numbers found, sorry"
+;;
+
+let fetch_file_name line =
+  let pat_filename = Str.regexp "\\(+++ b\\)/\\(.+\\)\\.[a-z]+" in
+  let s = Str.full_split pat_filename line in
+  decompose_list s
+;;
+
+let fetch_line_number line  =
+  let pat_line_num = Str.regexp "\\(\\+\\([0-9]+,[0-9]+\\)\\)" in
+  let s = Str.full_split pat_line_num line in
+  let fa = List.hd (decompose_list s) in
+  let (b, e) = extract_numbers fa in
+  (b, (b + e)) (*start of diff, end of diff *)
+;;
+
+type diff_info =
+{
+    file_name: string;
+    line_no: (int * int) list;
+}
+
+type patch_info = No_info | File_info of string list | Line_info of (int * int)
+
+(*check line and extract file name or file number*)
+let extract_info line =
+  if Str.string_match (Str.regexp_string "+++") line 0 then
+    File_info (fetch_file_name line)
+  else if Str.string_match (Str.regexp_string "@@ ") line 0 then
+    Line_info (fetch_line_number line)
+  else
+    No_info
+
+let rec reorg_helper new_list = function
+[] -> (List.rev new_list, [])
+| No_info :: tl -> reorg_helper new_list tl
+| File_info file_list :: tl -> (List.rev new_list, File_info file_list :: tl)
+| Line_info line_list :: tl ->  reorg_helper (line_list :: new_list) tl
+;;
+
+let rec reorg fpath toproot = function
+[] -> []
+| No_info :: tl -> reorg fpath toproot tl
+| File_info [file] :: tl ->
+  let lines, rest = reorg_helper [] tl in
+  {
+    file_name = Str.replace_first (Str.regexp "+++ b") toproot file;
+    line_no = lines;
+  } :: reorg fpath toproot rest
+| File_info file_list :: tl ->
+    let lines, rest = reorg_helper [] tl in
+    reorg fpath toproot rest
+| Line_info line_list :: tl ->  failwith "bad case"
+
+let rec mlines lines =
+  match lines with
+  | [] -> []
+  | hd :: tl -> extract_info hd :: mlines tl
+
+let final_info dir = 
+  let git_root = get_root dir in
+  let git_read_command = "cd " ^ git_root ^ " && git diff " ^ dir ^ " | egrep '^+++|^@'" in
+  let list_diff = reorg dir git_root (mlines (read_bash_help git_read_command)) in
+  list_diff;;
+
+let rec print_tuple_list l =
+  match l with
+  | [] -> ()
+  | (a,b) :: tl -> printf "\n%d, %d" a b; print_tuple_list tl
+
+let getpatchdiff dir = final_info dir
+
diff --git a/parsing_cocci/patch_diff.mli b/parsing_cocci/patch_diff.mli
new file mode 100644
index 000000000..b059509ef
--- /dev/null
+++ b/parsing_cocci/patch_diff.mli
@@ -0,0 +1,9 @@
+
+
+type diff_info =
+{
+    file_name: string;
+    line_no: (int * int) list;
+}
+
+val getpatchdiff : string -> diff_info list
-- 
2.31.1

_______________________________________________
Cocci mailing list
Cocci@systeme.lip6.fr
https://systeme.lip6.fr/mailman/listinfo/cocci

[Cocci] [PATCH 2/2] docs: manual: Add option description in spatch_options

[Sumera Priyadarsini]

Add documentation for the "use-patchdiff" option introduced in
the first patch of this patchset. This option allows for
applying a semantic patch to only those files in a directory
where code additions have been made.

Signed-off-by: Sumera Priyadarsini <sylphrenadin@gmail.com>
---
 docs/manual/spatch_options.tex | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/docs/manual/spatch_options.tex b/docs/manual/spatch_options.tex
index 223365bb5..afc55aa22 100644
--- a/docs/manual/spatch_options.tex
+++ b/docs/manual/spatch_options.tex
@@ -217,6 +217,11 @@ intereted relative to the target directory.  If the filename is an absolute
 path name, beginning with /, it is used as is.
 }
 
+\normal{-{}-use-patchdiff}{ This option allows for applying a semantic patch
+  to only the files that have been modified in a directory. The directory needs
+  to be specified by the user. Note that an absolute path needs to be used if
+  spatch is called from outside the target project directory. }
+
 \normal{-{}-use-coccigrep}{ Use a version of grep implemented in Coccinelle
   to check that selected files are relevant to the semantic patch.  This
   option is only relevant to the case of working on a complete directory,
-- 
2.31.1

_______________________________________________
Cocci mailing list
Cocci@systeme.lip6.fr
https://systeme.lip6.fr/mailman/listinfo/cocci

Re: [Cocci] [PATCH 0/2] Add "use-patchdiff" option

[Markus Elfring]

> This patchset adds a feature to enable Coccinelle
> to only check all those files in a directory which were
> modified. It parses all the files obtained from the
> output of "git diff" and checks them against the specified
> cocci script.
>
> An example for passing the "use-patchdiff" option is:

How do you think about to use the parameter name “use-files-from-diff”?

Regards,
Markus
_______________________________________________
Cocci mailing list
Cocci@systeme.lip6.fr
https://systeme.lip6.fr/mailman/listinfo/cocci

Re: [Cocci] [PATCH 0/2] Add "use-patchdiff" option

[Sumera Priyadarsini]

On Wed, May 26, 2021 at 10:13 PM Markus Elfring <Markus.Elfring@web.de> wrote:
>
> > This patchset adds a feature to enable Coccinelle
> > to only check all those files in a directory which were
> > modified. It parses all the files obtained from the
> > output of "git diff" and checks them against the specified
> > cocci script.
> >
> > An example for passing the "use-patchdiff" option is:
>
> How do you think about to use the parameter name “use-files-from-diff”?

I would prefer something shorter, like "use-filesdiff" but I am okay
with either name as
long as the maintainers are okay with it. :)

Julia, what do you think?  I will send a v2 with any of the above name options
(and/or any other changes that are suggested.)

Thanks,
Sumera

>
> Regards,
> Markus
_______________________________________________
Cocci mailing list
Cocci@systeme.lip6.fr
https://systeme.lip6.fr/mailman/listinfo/cocci

Re: [Cocci] [PATCH 0/2] Add "use-patchdiff" option

[Julia Lawall]

[-- Attachment #1: Type: text/plain, Size: 951 bytes --]



On Fri, 2 Jul 2021, Sumera Priyadarsini wrote:

> On Wed, May 26, 2021 at 10:13 PM Markus Elfring <Markus.Elfring@web.de> wrote:
> >
> > > This patchset adds a feature to enable Coccinelle
> > > to only check all those files in a directory which were
> > > modified. It parses all the files obtained from the
> > > output of "git diff" and checks them against the specified
> > > cocci script.
> > >
> > > An example for passing the "use-patchdiff" option is:
> >
> > How do you think about to use the parameter name “use-files-from-diff”?
>
> I would prefer something shorter, like "use-filesdiff" but I am okay
> with either name as
> long as the maintainers are okay with it. :)
>
> Julia, what do you think?  I will send a v2 with any of the above name options
> (and/or any other changes that are suggested.)

I think that shorter names are better.  The current name could be ok, but
a - between patch and diff might be more natural.

julia

[-- Attachment #2: Type: text/plain, Size: 136 bytes --]

_______________________________________________
Cocci mailing list
Cocci@systeme.lip6.fr
https://systeme.lip6.fr/mailman/listinfo/cocci