[RFC PATCH 12/21] parallel-checkout: add configuration options

From: Matheus Tavares <matheus.bernardino@usp.br>
To: git@vger.kernel.org
Cc: stolee@gmail.com, jeffhost@microsoft.com,
	"Nguyễn Thái Ngọc Duy" <pclouds@gmail.com>,
	"Junio C Hamano" <gitster@pobox.com>,
	"René Scharfe" <l.s.r@web.de>,
	"Stefan Beller" <stefanbeller@gmail.com>
Subject: [RFC PATCH 12/21] parallel-checkout: add configuration options
Date: Mon, 10 Aug 2020 18:33:20 -0300	[thread overview]
Message-ID: <1263342110bfc96480146df0f876a5c936b61ce1.1597093021.git.matheus.bernardino@usp.br> (raw)
In-Reply-To: <cover.1597093021.git.matheus.bernardino@usp.br>

Add the checkout.workers and checkout.workerThreshold settings, which
allow users to configure and/or disable the parallel checkout feature as
desired. The first setting defines the number of workers and the second
defines the minimum number of entries to attempt parallel checkout.

Co-authored-by: Jeff Hostetler <jeffhost@microsoft.com>
Signed-off-by: Matheus Tavares <matheus.bernardino@usp.br>
---

I still have to evaluate what is the best default value for
checkout.workersThreshold. For now, I used 0 so that the test suite uses
parallel-checkout by default, exercising the new code. I'm open to
suggestions on how we can improve testing for it, once 
checkout.workersThreshold is no longer 0.

Note: the default number of workers can probably be better calculated as
well, multiplying the number of cores by some factor. My machine, for
example, has 8 logical cores but 10 workers leads to the fastest
execution.

 Documentation/config/checkout.txt | 16 ++++++++++++++++
 parallel-checkout.c               | 26 +++++++++++++++++++++-----
 parallel-checkout.h               | 11 +++++++++--
 unpack-trees.c                    | 10 +++++++---
 4 files changed, 53 insertions(+), 10 deletions(-)

diff --git a/Documentation/config/checkout.txt b/Documentation/config/checkout.txt
index 6b646813ab..9dabdf9231 100644
--- a/Documentation/config/checkout.txt
+++ b/Documentation/config/checkout.txt
@@ -16,3 +16,19 @@ will checkout the '<something>' branch on another remote,
 and by linkgit:git-worktree[1] when 'git worktree add' refers to a
 remote branch. This setting might be used for other checkout-like
 commands or functionality in the future.
+
+checkout.workers::
+	The number of worker processes to use when updating the working tree.
+	If unset (or set to a value less than one), Git will use as many
+	workers as the number of logical cores available. One means sequential
+	execution. This and the checkout.workersThreshold settings affect all
+	commands which perform checkout. E.g. checkout, switch, clone,
+	sparse-checkout, read-tree, etc.
+
+checkout.workersThreshold::
+	If set to a positive number, parallel checkout will not be attempted
+	when the number of files to be updated is less than the defined limit.
+	When set to a negative number or unset, defaults to 0. The reasoning
+	behind this config is that, when modifying a small number of files, a
+	sequential execution might be faster, as it avoids the cost of spawning
+	subprocesses and inter-process communication.
diff --git a/parallel-checkout.c b/parallel-checkout.c
index ec42342bc8..e0fca4d380 100644
--- a/parallel-checkout.c
+++ b/parallel-checkout.c
@@ -4,6 +4,8 @@
 #include "pkt-line.h"
 #include "run-command.h"
 #include "streaming.h"
+#include "thread-utils.h"
+#include "config.h"
 
 struct parallel_checkout {
 	struct checkout_item *items;
@@ -18,6 +20,19 @@ enum pc_status parallel_checkout_status(void)
 	return pc_status;
 }
 
+#define DEFAULT_WORKERS_THRESHOLD 0
+
+void get_parallel_checkout_configs(int *num_workers, int *threshold)
+{
+	if (git_config_get_int("checkout.workers", num_workers) ||
+	    *num_workers < 1)
+		*num_workers = online_cpus();
+
+	if (git_config_get_int("checkout.workersThreshold", threshold) ||
+	    *threshold < 0)
+		*threshold = DEFAULT_WORKERS_THRESHOLD;
+}
+
 void init_parallel_checkout(void)
 {
 	if (parallel_checkout)
@@ -480,22 +495,23 @@ static int run_checkout_sequentially(struct checkout *state)
 	return handle_results(state);
 }
 
-static const int workers_threshold = 0;
-
-int run_parallel_checkout(struct checkout *state)
+int run_parallel_checkout(struct checkout *state, int num_workers, int threshold)
 {
-	int num_workers = online_cpus();
 	int ret = 0;
 	struct child_process *workers;
 
 	if (!parallel_checkout)
 		BUG("cannot run parallel checkout: not initialized yet");
 
+	if (num_workers < 1)
+		BUG("invalid number of workers for run_parallel_checkout: %d",
+		    num_workers);
+
 	pc_status = PC_RUNNING;
 
 	if (parallel_checkout->nr == 0) {
 		goto done;
-	} else if (parallel_checkout->nr < workers_threshold || num_workers == 1) {
+	} else if (parallel_checkout->nr < threshold || num_workers == 1) {
 		ret = run_checkout_sequentially(state);
 		goto done;
 	}
diff --git a/parallel-checkout.h b/parallel-checkout.h
index f25f2874ae..b4d412c8b5 100644
--- a/parallel-checkout.h
+++ b/parallel-checkout.h
@@ -18,6 +18,9 @@ enum pc_status {
 enum pc_status parallel_checkout_status(void);
 void init_parallel_checkout(void);
 
+/* Reads the checkout.workers and checkout.workersThreshold settings. */
+void get_parallel_checkout_configs(int *num_workers, int *threshold);
+
 /*
  * Return -1 if parallel checkout is currently not enabled or if the entry is
  * not eligible for parallel checkout. Otherwise, enqueue the entry for later
@@ -25,8 +28,12 @@ void init_parallel_checkout(void);
  */
 int enqueue_checkout(struct cache_entry *ce, struct conv_attrs *ca);
 
-/* Write all the queued entries, returning 0 on success. */
-int run_parallel_checkout(struct checkout *state);
+/*
+ * Write all the queued entries, returning 0 on success. If the number of
+ * entries is below the specified threshold, the operation is performed
+ * sequentially.
+ */
+int run_parallel_checkout(struct checkout *state, int num_workers, int threshold);
 
 /****************************************************************
  * Interface with checkout--helper
diff --git a/unpack-trees.c b/unpack-trees.c
index 1b1da7485a..117ed42370 100644
--- a/unpack-trees.c
+++ b/unpack-trees.c
@@ -399,7 +399,7 @@ static int check_updates(struct unpack_trees_options *o,
 	int errs = 0;
 	struct progress *progress;
 	struct checkout state = CHECKOUT_INIT;
-	int i;
+	int i, pc_workers, pc_threshold;
 
 	trace_performance_enter();
 	state.force = 1;
@@ -462,8 +462,11 @@ static int check_updates(struct unpack_trees_options *o,
 		oid_array_clear(&to_fetch);
 	}
 
+	get_parallel_checkout_configs(&pc_workers, &pc_threshold);
+
 	enable_delayed_checkout(&state);
-	init_parallel_checkout();
+	if (pc_workers > 1)
+		init_parallel_checkout();
 	for (i = 0; i < index->cache_nr; i++) {
 		struct cache_entry *ce = index->cache[i];
 
@@ -477,7 +480,8 @@ static int check_updates(struct unpack_trees_options *o,
 		}
 	}
 	stop_progress(&progress);
-	errs |= run_parallel_checkout(&state);
+	if (pc_workers > 1)
+		errs |= run_parallel_checkout(&state, pc_workers, pc_threshold);
 	errs |= finish_delayed_checkout(&state, NULL);
 	git_attr_set_direction(GIT_ATTR_CHECKIN);
 
-- 
2.27.0

