From 6713544de2c12b0f2d4fb7b57e031b70c43c62c0 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Tue, 7 Apr 2026 09:17:11 -0400 Subject: [PATCH 1/3] gvfs-helper: separate packfile extraction from indexing Refactor install_prefetch() to process prefetch packs in two distinct phases: Phase 1 (extraction): Read the multipack stream sequentially, copying each packfile to its own temp file and recording its checksum and timestamp in a prefetch_entry array. This must be sequential because the multipack is a single byte stream. Phase 2 (indexing): Run 'git index-pack' on each extracted temp file and finalize it into the ODB. Today this still runs sequentially, but the separation makes it straightforward to parallelize in a subsequent commit. The new extract_packfile_from_multipack() only does I/O against the multipack fd plus temp-file creation. The new index_and_finalize_packfile() only does the index-pack and rename work. Neither depends on the other's state, so they can operate on different entries concurrently once the extraction phase completes. No behavioral change; this is a pure refactor. Signed-off-by: Derrick Stolee --- gvfs-helper.c | 163 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 119 insertions(+), 44 deletions(-) diff --git a/gvfs-helper.c b/gvfs-helper.c index 41d87aa0cee549..9334c7b0a3c553 100644 --- a/gvfs-helper.c +++ b/gvfs-helper.c @@ -2253,39 +2253,56 @@ struct ph { }; /* - * Extract the next packfile from the multipack. - * Install {.pack, .idx, .keep} set. + * Per-packfile metadata collected during the extraction phase + * of prefetch installation. After all packfiles are extracted + * from the multipack, each entry is handed to index-pack. + */ +struct prefetch_entry { + struct strbuf temp_path_pack; + struct strbuf temp_path_idx; + char hex_checksum[GIT_MAX_HEXSZ + 1]; + timestamp_t timestamp; +}; + +#define PREFETCH_ENTRY_INIT { \ + .temp_path_pack = STRBUF_INIT, \ + .temp_path_idx = STRBUF_INIT, \ + .hex_checksum = {0}, \ + .timestamp = 0, \ +} + +static void prefetch_entry_release(struct prefetch_entry *pe) +{ + strbuf_release(&pe->temp_path_pack); + strbuf_release(&pe->temp_path_idx); +} + +/* + * Extract the next packfile from the multipack into a temp file. + * Populate `entry` with the temp path and checksum, then advance + * the fd past any trailing .idx data. * - * Mark each successfully installed prefetch pack as .keep it as installed - * in case we have errors decoding/indexing later packs within the received - * multipart file. (A later pass can delete the unnecessary .keep files - * from this and any previous invocations.) + * This is the I/O-bound phase that must run sequentially because + * the multipack is a single stream. */ static void extract_packfile_from_multipack( - struct gh__request_params *params, struct gh__response_status *status, int fd_multipack, - unsigned short k) + unsigned short k, + struct prefetch_entry *entry) { struct ph ph; struct tempfile *tempfile_pack = NULL; int result = -1; int b_no_idx_in_multipack; struct object_id packfile_checksum; - char hex_checksum[GIT_MAX_HEXSZ + 1]; - struct strbuf buf_timestamp = STRBUF_INIT; - struct strbuf temp_path_pack = STRBUF_INIT; - struct strbuf temp_path_idx = STRBUF_INIT; - struct strbuf final_path_pack = STRBUF_INIT; - struct strbuf final_path_idx = STRBUF_INIT; - struct strbuf final_filename = STRBUF_INIT; if (xread(fd_multipack, &ph, sizeof(ph)) != sizeof(ph)) { strbuf_addf(&status->error_message, "could not read header for packfile[%d] in multipack", k); status->ec = GH__ERROR_CODE__COULD_NOT_INSTALL_PREFETCH; - goto done; + return; } ph.timestamp = my_get_le64(ph.timestamp); @@ -2296,7 +2313,7 @@ static void extract_packfile_from_multipack( strbuf_addf(&status->error_message, "packfile[%d]: zero length packfile?", k); status->ec = GH__ERROR_CODE__COULD_NOT_INSTALL_PREFETCH; - goto done; + return; } b_no_idx_in_multipack = (ph.idx_len == maximum_unsigned_value_of_type(uint64_t) || @@ -2309,7 +2326,7 @@ static void extract_packfile_from_multipack( */ my_create_tempfile(status, 0, "pack", &tempfile_pack, NULL, NULL); if (!tempfile_pack) - goto done; + return; /* * Copy the current packfile from the open stream and capture @@ -2325,29 +2342,23 @@ static void extract_packfile_from_multipack( GIT_SHA1_RAWSZ); packfile_checksum.algo = GIT_HASH_SHA1; - if (result < 0){ + if (result < 0) { strbuf_addf(&status->error_message, "could not extract packfile[%d] from multipack", k); - goto done; + delete_tempfile(&tempfile_pack); + return; } - strbuf_addstr(&temp_path_pack, get_tempfile_path(tempfile_pack)); + strbuf_addstr(&entry->temp_path_pack, get_tempfile_path(tempfile_pack)); close_tempfile_gently(tempfile_pack); - oid_to_hex_r(hex_checksum, &packfile_checksum); - - /* - * Always compute the .idx file from the .pack file. - */ - strbuf_addbuf(&temp_path_idx, &temp_path_pack); - strbuf_strip_suffix(&temp_path_idx, ".pack"); - strbuf_addstr(&temp_path_idx, ".idx"); + oid_to_hex_r(entry->hex_checksum, &packfile_checksum); + entry->timestamp = (timestamp_t)ph.timestamp; - my_run_index_pack(params, status, - &temp_path_pack, &temp_path_idx, - NULL); - if (status->ec != GH__ERROR_CODE__OK) - goto done; + /* Derive the .idx temp path from the .pack temp path. */ + strbuf_addbuf(&entry->temp_path_idx, &entry->temp_path_pack); + strbuf_strip_suffix(&entry->temp_path_idx, ".pack"); + strbuf_addstr(&entry->temp_path_idx, ".idx"); if (!b_no_idx_in_multipack) { /* @@ -2359,25 +2370,43 @@ static void extract_packfile_from_multipack( "could not skip index[%d] in multipack", k); status->ec = GH__ERROR_CODE__COULD_NOT_INSTALL_PREFETCH; - goto done; + return; } } +} - strbuf_addf(&buf_timestamp, "%u", (unsigned int)ph.timestamp); - create_final_packfile_pathnames("prefetch", buf_timestamp.buf, hex_checksum, +/* + * Run index-pack on a previously extracted temp packfile, then + * finalize (move) it into the ODB. + */ +static void index_and_finalize_packfile(struct gh__request_params *params, + struct gh__response_status *status, + struct prefetch_entry *entry) +{ + struct strbuf buf_timestamp = STRBUF_INIT; + struct strbuf final_path_pack = STRBUF_INIT; + struct strbuf final_path_idx = STRBUF_INIT; + struct strbuf final_filename = STRBUF_INIT; + + my_run_index_pack(params, status, + &entry->temp_path_pack, &entry->temp_path_idx, + NULL); + if (status->ec != GH__ERROR_CODE__OK) + goto done; + + strbuf_addf(&buf_timestamp, "%u", (unsigned int)entry->timestamp); + create_final_packfile_pathnames("prefetch", buf_timestamp.buf, + entry->hex_checksum, &final_path_pack, &final_path_idx, &final_filename); - strbuf_release(&buf_timestamp); my_finalize_packfile(params, status, 1, - &temp_path_pack, &temp_path_idx, + &entry->temp_path_pack, &entry->temp_path_idx, &final_path_pack, &final_path_idx, &final_filename); done: - delete_tempfile(&tempfile_pack); - strbuf_release(&temp_path_pack); - strbuf_release(&temp_path_idx); + strbuf_release(&buf_timestamp); strbuf_release(&final_path_pack); strbuf_release(&final_path_idx); strbuf_release(&final_filename); @@ -2453,6 +2482,8 @@ static void install_prefetch(struct gh__request_params *params, int fd = -1; int nr_installed = 0; + struct prefetch_entry *entries = NULL; + struct strbuf temp_path_mp = STRBUF_INIT; /* @@ -2488,11 +2519,49 @@ static void install_prefetch(struct gh__request_params *params, trace2_data_intmax(TR2_CAT, NULL, "prefetch/packfile_count", np); + if (!np) + goto cleanup; + + CALLOC_ARRAY(entries, np); + for (k = 0; k < np; k++) { + struct prefetch_entry pe = PREFETCH_ENTRY_INIT; + entries[k] = pe; + } + + /* + * Phase 1: extract all packfiles from the multipack into + * individual temp files. This must be sequential because + * the multipack is a single byte stream. + */ if (gh__cmd_opts.show_progress) - params->progress = start_progress(the_repository, "Installing prefetch packfiles", np); + params->progress = start_progress( + the_repository, "Extracting prefetch packfiles", np); for (k = 0; k < np; k++) { - extract_packfile_from_multipack(params, status, fd, k); + extract_packfile_from_multipack(status, fd, k, &entries[k]); + display_progress(params->progress, k + 1); + if (status->ec != GH__ERROR_CODE__OK) + break; + } + stop_progress(¶ms->progress); + + /* The multipack fd is no longer needed after extraction. */ + close(fd); + fd = -1; + + if (status->ec != GH__ERROR_CODE__OK) + goto cleanup; + + /* + * Phase 2: run index-pack on each extracted packfile and + * finalize it into the ODB. + */ + if (gh__cmd_opts.show_progress) + params->progress = start_progress( + the_repository, "Installing prefetch packfiles", np); + + for (k = 0; k < np; k++) { + index_and_finalize_packfile(params, status, &entries[k]); display_progress(params->progress, k + 1); if (status->ec != GH__ERROR_CODE__OK) break; @@ -2504,6 +2573,12 @@ static void install_prefetch(struct gh__request_params *params, delete_stale_keep_files(params, status); cleanup: + if (entries) { + for (k = 0; k < np; k++) + prefetch_entry_release(&entries[k]); + free(entries); + } + if (fd != -1) close(fd); From ae9e1e629f84abf02290a919a4a5cc4c59a35edf Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Tue, 7 Apr 2026 11:23:06 -0400 Subject: [PATCH 2/3] gvfs-helper: run prefetch index-pack in parallel Replace the sequential index-pack loop in install_prefetch() with run_processes_parallel(), spawning up to four concurrent 'git index-pack' workers. The packfiles are already ordered by timestamp (oldest first) in the multipack response. In the common fresh-clone scenario the oldest pack is by far the largest, so it starts indexing immediately on the first worker while the remaining workers cycle through the smaller daily and hourly packs. Note that this works for the GVFS prefetch endpoint as all prefetch packfiles are non-thin packs. The bundle URI feature uses thin bundles that must be unpacked sequentially. Worker count is min(np, PREFETCH_MAX_WORKERS) where PREFETCH_MAX_WORKERS is 4, so we never create more workers than there are packfiles. When there is only a single packfile the parallel infrastructure is skipped entirely and index-pack runs directly. The default grouped mode of run_processes_parallel() is used so that child-process completion is detected via poll() on stderr pipes rather than the ungroup mode's aggressive mark-all-slots-WAIT_CLEANUP approach, which can misfire on slots that never started a process. The run_processes_parallel() callbacks are always invoked from the main thread, so finalize_prefetch_packfile() (which renames files into the ODB) needs no locking. If any index-pack fails, the error is recorded and remaining tasks still complete so that successfully-indexed packs are not lost. I performed manual performance testing on Linux using an internal monorepo. I deleted a set of recent prefetch packfiles, leading to a download of a couple daily packfiles and several hourly packfiles. This led to an improvement from 85.2 seconds to 40.3 seconds. Signed-off-by: Derrick Stolee --- gvfs-helper.c | 157 ++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 132 insertions(+), 25 deletions(-) diff --git a/gvfs-helper.c b/gvfs-helper.c index 9334c7b0a3c553..3d1afa760470a2 100644 --- a/gvfs-helper.c +++ b/gvfs-helper.c @@ -2376,24 +2376,18 @@ static void extract_packfile_from_multipack( } /* - * Run index-pack on a previously extracted temp packfile, then - * finalize (move) it into the ODB. + * Finalize a prefetch packfile after index-pack has already run: + * compute final pathnames and move .pack/.idx/.keep into the ODB. */ -static void index_and_finalize_packfile(struct gh__request_params *params, - struct gh__response_status *status, - struct prefetch_entry *entry) +static void finalize_prefetch_packfile(struct gh__request_params *params, + struct gh__response_status *status, + struct prefetch_entry *entry) { struct strbuf buf_timestamp = STRBUF_INIT; struct strbuf final_path_pack = STRBUF_INIT; struct strbuf final_path_idx = STRBUF_INIT; struct strbuf final_filename = STRBUF_INIT; - my_run_index_pack(params, status, - &entry->temp_path_pack, &entry->temp_path_idx, - NULL); - if (status->ec != GH__ERROR_CODE__OK) - goto done; - strbuf_addf(&buf_timestamp, "%u", (unsigned int)entry->timestamp); create_final_packfile_pathnames("prefetch", buf_timestamp.buf, entry->hex_checksum, @@ -2405,13 +2399,89 @@ static void index_and_finalize_packfile(struct gh__request_params *params, &final_path_pack, &final_path_idx, &final_filename); -done: strbuf_release(&buf_timestamp); strbuf_release(&final_path_pack); strbuf_release(&final_path_idx); strbuf_release(&final_filename); } +#define PREFETCH_MAX_WORKERS 4 + +/* + * Context for parallel index-pack execution. + * + * The run_processes_parallel() callbacks are always called from + * the main thread, so no locking is needed for these fields. + */ +struct prefetch_parallel_ctx { + struct prefetch_entry *entries; + unsigned short np; + unsigned short next; + + struct gh__request_params *params; + struct gh__response_status *status; + + struct progress *progress; + int nr_finished; + int nr_installed; +}; + +static int prefetch_get_next_task(struct child_process *cp, + struct strbuf *out UNUSED, + void *pp_cb, + void **pp_task_cb) +{ + struct prefetch_parallel_ctx *ctx = pp_cb; + struct prefetch_entry *entry; + + if (ctx->next >= ctx->np) + return 0; + + entry = &ctx->entries[ctx->next]; + *pp_task_cb = entry; + ctx->next++; + + cp->git_cmd = 1; + strvec_push(&cp->args, "index-pack"); + strvec_push(&cp->args, "--no-rev-index"); + strvec_pushl(&cp->args, "-o", entry->temp_path_idx.buf, NULL); + strvec_push(&cp->args, entry->temp_path_pack.buf); + cp->no_stdin = 1; + + return 1; +} + +static int prefetch_task_finished(int result, + struct strbuf *out UNUSED, + void *pp_cb, + void *pp_task_cb) +{ + struct prefetch_parallel_ctx *ctx = pp_cb; + struct prefetch_entry *entry = pp_task_cb; + + ctx->nr_finished++; + display_progress(ctx->progress, ctx->nr_finished); + + if (result) { + unlink(entry->temp_path_pack.buf); + unlink(entry->temp_path_idx.buf); + + if (ctx->status->ec == GH__ERROR_CODE__OK) { + strbuf_addf(&ctx->status->error_message, + "index-pack failed on '%s'", + entry->temp_path_pack.buf); + ctx->status->ec = GH__ERROR_CODE__INDEX_PACK_FAILED; + } + return 0; + } + + finalize_prefetch_packfile(ctx->params, ctx->status, entry); + if (ctx->status->ec == GH__ERROR_CODE__OK) + ctx->nr_installed++; + + return 0; +} + struct keep_files_data { timestamp_t max_timestamp; int pos_of_max; @@ -2553,21 +2623,58 @@ static void install_prefetch(struct gh__request_params *params, goto cleanup; /* - * Phase 2: run index-pack on each extracted packfile and - * finalize it into the ODB. + * Phase 2: run index-pack on the extracted packfiles in + * parallel and finalize each into the ODB. + * + * Use up to PREFETCH_MAX_WORKERS concurrent index-pack + * processes. The entries are already in timestamp order + * (oldest first), so the largest pack—the one that takes + * the longest—starts immediately while the remaining + * workers cycle through the smaller daily/hourly packs. + * + * When there is only one packfile there is no benefit from + * the parallel infrastructure, so fall through to a simple + * sequential index-pack + finalize. */ - if (gh__cmd_opts.show_progress) - params->progress = start_progress( - the_repository, "Installing prefetch packfiles", np); - - for (k = 0; k < np; k++) { - index_and_finalize_packfile(params, status, &entries[k]); - display_progress(params->progress, k + 1); - if (status->ec != GH__ERROR_CODE__OK) - break; - nr_installed++; + if (np == 1) { + my_run_index_pack(params, status, + &entries[0].temp_path_pack, + &entries[0].temp_path_idx, + NULL); + if (status->ec == GH__ERROR_CODE__OK) { + finalize_prefetch_packfile(params, status, &entries[0]); + if (status->ec == GH__ERROR_CODE__OK) + nr_installed++; + } + } else { + struct prefetch_parallel_ctx pctx = { + .entries = entries, + .np = np, + .next = 0, + .params = params, + .status = status, + .nr_finished = 0, + .nr_installed = 0, + }; + struct run_process_parallel_opts pp_opts = { + .tr2_category = TR2_CAT, + .tr2_label = "prefetch/index-pack", + .processes = MY_MIN(np, PREFETCH_MAX_WORKERS), + .get_next_task = prefetch_get_next_task, + .task_finished = prefetch_task_finished, + .data = &pctx, + }; + + if (gh__cmd_opts.show_progress) + pctx.progress = start_progress( + the_repository, + "Installing prefetch packfiles", np); + + run_processes_parallel(&pp_opts); + + stop_progress(&pctx.progress); + nr_installed = pctx.nr_installed; } - stop_progress(¶ms->progress); if (nr_installed) delete_stale_keep_files(params, status); From e82fa3315df1358fbb5a28180f7d978cb42ea591 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Tue, 21 Apr 2026 10:53:24 -0400 Subject: [PATCH 3/3] gvfs-helper: add gvfs.prefetchThreads config for parallel prefetch Introduce a new config option, gvfs.prefetchThreads, that controls the number of parallel index-pack processes used when installing prefetch packfiles. The default value is 1, which processes packfiles sequentially without any parallel infrastructure, preserving the existing behavior as a safety measure. When set to a value greater than 1, the prefetch installation phase uses run_processes_parallel() with up to that many concurrent index-pack workers. This can significantly speed up installation when multiple prefetch packs are downloaded. The sequential path (threads=1) loops through each extracted packfile one at a time using my_run_index_pack() + finalize, while the parallel path (threads>1) dispatches index-pack processes via the existing parallel process machinery. Add trace2 data points (prefetch/install_mode) to log which code path is taken, enabling tests to verify the correct mode is active. Add t5797-gvfs-helper-prefetch-threads.sh with tests that exercise both sequential and parallel modes by looping over threads=1 and threads=4. Each iteration runs four scenarios: fetch-all, fetch-since, up-to-date re-fetch, and corrupt-pack error handling. Trace2 assertions confirm the expected code path is taken in each mode. Signed-off-by: Derrick Stolee --- Documentation/config/gvfs.adoc | 9 ++ gvfs-helper.c | 73 ++++++++---- t/meson.build | 1 + t/t5797-gvfs-helper-prefetch-threads.sh | 149 ++++++++++++++++++++++++ 4 files changed, 210 insertions(+), 22 deletions(-) create mode 100755 t/t5797-gvfs-helper-prefetch-threads.sh diff --git a/Documentation/config/gvfs.adoc b/Documentation/config/gvfs.adoc index a4a060b570f148..b38c7ac9eb8937 100644 --- a/Documentation/config/gvfs.adoc +++ b/Documentation/config/gvfs.adoc @@ -34,3 +34,12 @@ gvfs.sessionKey:: across client and server behavior, including any End User Pseodynomous Identifiers (EUPI) that may be configured. The `X-Session-Id` will include the SID of the current process in either case. + +gvfs.prefetchThreads:: + Set the number of parallel `index-pack` processes that run when + installing prefetch packfiles. The default value is `1`, which + processes packfiles sequentially without any thread infrastructure. + Setting this to a higher value (for example `4`) enables parallel + index-pack execution, which can significantly speed up the + installation of multiple prefetch packs. Values less than `1` are + treated as `1`. diff --git a/gvfs-helper.c b/gvfs-helper.c index 3d1afa760470a2..12a8b7de1d3591 100644 --- a/gvfs-helper.c +++ b/gvfs-helper.c @@ -388,6 +388,8 @@ static struct gh__global { int main_creds_need_approval; /* try to only approve them once */ unsigned long connect_timeout_ms; + + int prefetch_threads; } gh__global; enum gh__server_type { @@ -2405,8 +2407,6 @@ static void finalize_prefetch_packfile(struct gh__request_params *params, strbuf_release(&final_filename); } -#define PREFETCH_MAX_WORKERS 4 - /* * Context for parallel index-pack execution. * @@ -2623,29 +2623,44 @@ static void install_prefetch(struct gh__request_params *params, goto cleanup; /* - * Phase 2: run index-pack on the extracted packfiles in - * parallel and finalize each into the ODB. + * Phase 2: run index-pack on the extracted packfiles and + * finalize each into the ODB. * - * Use up to PREFETCH_MAX_WORKERS concurrent index-pack - * processes. The entries are already in timestamp order - * (oldest first), so the largest pack—the one that takes - * the longest—starts immediately while the remaining - * workers cycle through the smaller daily/hourly packs. + * When gvfs.prefetchThreads is 1 (the default), process + * packfiles sequentially without any thread infrastructure. + * When set to a higher value, use up to that many concurrent + * index-pack processes. * - * When there is only one packfile there is no benefit from - * the parallel infrastructure, so fall through to a simple - * sequential index-pack + finalize. + * The entries are already in timestamp order (oldest first), + * so the largest pack—the one that takes the longest—starts + * immediately while the remaining workers cycle through the + * smaller daily/hourly packs. */ - if (np == 1) { - my_run_index_pack(params, status, - &entries[0].temp_path_pack, - &entries[0].temp_path_idx, - NULL); - if (status->ec == GH__ERROR_CODE__OK) { - finalize_prefetch_packfile(params, status, &entries[0]); - if (status->ec == GH__ERROR_CODE__OK) - nr_installed++; + if (gh__global.prefetch_threads <= 1) { + trace2_data_intmax(TR2_CAT, NULL, + "prefetch/install_mode", 1); + + if (gh__cmd_opts.show_progress) + params->progress = start_progress( + the_repository, + "Installing prefetch packfiles", np); + + for (k = 0; k < np; k++) { + my_run_index_pack(params, status, + &entries[k].temp_path_pack, + &entries[k].temp_path_idx, + NULL); + if (status->ec == GH__ERROR_CODE__OK) { + finalize_prefetch_packfile(params, status, + &entries[k]); + if (status->ec == GH__ERROR_CODE__OK) + nr_installed++; + } + display_progress(params->progress, k + 1); + if (status->ec != GH__ERROR_CODE__OK) + break; } + stop_progress(¶ms->progress); } else { struct prefetch_parallel_ctx pctx = { .entries = entries, @@ -2659,12 +2674,16 @@ static void install_prefetch(struct gh__request_params *params, struct run_process_parallel_opts pp_opts = { .tr2_category = TR2_CAT, .tr2_label = "prefetch/index-pack", - .processes = MY_MIN(np, PREFETCH_MAX_WORKERS), + .processes = MY_MIN(np, gh__global.prefetch_threads), .get_next_task = prefetch_get_next_task, .task_finished = prefetch_task_finished, .data = &pctx, }; + trace2_data_intmax(TR2_CAT, NULL, + "prefetch/install_mode", + gh__global.prefetch_threads); + if (gh__cmd_opts.show_progress) pctx.progress = start_progress( the_repository, @@ -4661,6 +4680,16 @@ int cmd_main(int argc, const char **argv) repo_config(the_repository, git_default_config, NULL); + /* + * Read gvfs.prefetchThreads to control parallel index-pack + * during prefetch. Default to 1 (sequential) for safety. + */ + gh__global.prefetch_threads = 1; + repo_config_get_int(the_repository, "gvfs.prefetchthreads", + &gh__global.prefetch_threads); + if (gh__global.prefetch_threads < 1) + gh__global.prefetch_threads = 1; + argc = parse_options(argc, argv, NULL, main_options, main_usage, PARSE_OPT_STOP_AT_NON_OPTION); if (argc == 0) diff --git a/t/meson.build b/t/meson.build index e218971c30593a..33466436bbb53a 100644 --- a/t/meson.build +++ b/t/meson.build @@ -764,6 +764,7 @@ integration_tests = [ 't5793-gvfs-helper-integration.sh', 't5794-gvfs-helper-packfiles.sh', 't5795-gvfs-helper-verb-cache.sh', + 't5797-gvfs-helper-prefetch-threads.sh', 't5801-remote-helpers.sh', 't5802-connect-helper.sh', 't5810-proto-disable-local.sh', diff --git a/t/t5797-gvfs-helper-prefetch-threads.sh b/t/t5797-gvfs-helper-prefetch-threads.sh new file mode 100755 index 00000000000000..33efdb4b2160b9 --- /dev/null +++ b/t/t5797-gvfs-helper-prefetch-threads.sh @@ -0,0 +1,149 @@ +#!/bin/sh + +test_description='gvfs-helper prefetch with gvfs.prefetchThreads config + +Verify that the prefetch verb works correctly in both sequential +(gvfs.prefetchThreads=1) and parallel (gvfs.prefetchThreads=4) modes. +Each test is run under both configurations to ensure identical results +and to exercise both code paths in install_prefetch(). +' + +. ./test-lib.sh + +. "$TEST_DIRECTORY"/lib-gvfs-helper.sh + +# Helper: run a prefetch that fetches all 3 epoch packs (no --since). +# +do_prefetch_all () { + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + prefetch >OUT.output 2>OUT.stderr && + + verify_received_packfile_count 3 && + verify_prefetch_keeps 1200000000 +} + +# Helper: run a prefetch with --since to get 2 of 3 packs. +# +do_prefetch_since () { + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + prefetch --since="1000000000" >OUT.output 2>OUT.stderr && + + verify_received_packfile_count 2 && + verify_prefetch_keeps 1200000000 +} + +# Helper: prefetch then re-prefetch to verify up-to-date handling. +# +do_prefetch_up_to_date () { + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + prefetch --since="1000000000" >OUT.output 2>OUT.stderr && + + verify_received_packfile_count 2 && + verify_prefetch_keeps 1200000000 && + + # Re-fetch; should find nothing new. + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + prefetch >OUT.output 2>OUT.stderr && + + verify_received_packfile_count 0 && + verify_prefetch_keeps 1200000000 +} + +# Helper: prefetch corrupt pack (error path). +# Requires the server to be started with the appropriate mayhem. +# +do_prefetch_corrupt_pack () { + test_must_fail \ + git -C "$REPO_T1" gvfs-helper \ + --cache-server=disable \ + --remote=origin \ + --no-progress \ + prefetch \ + --max-retries=0 \ + --since="1000000000" \ + >OUT.output 2>OUT.stderr && + + test_grep "error: .* index-pack failed" OUT.stderr +} + +for threads in 1 4 +do + # Describe the mode for readable test names. + if test "$threads" = "1" + then + mode="sequential" + # The sequential path logs install_mode=1. + expected_mode=1 + else + mode="parallel" + expected_mode=$threads + fi + + test_expect_success "prefetch all packs ($mode, threads=$threads)" ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + git -C "$REPO_T1" config gvfs.prefetchThreads '$threads' && + + GIT_TRACE2_EVENT="$(pwd)/trace-$test_count.txt" && + export GIT_TRACE2_EVENT && + + do_prefetch_all && + + stop_gvfs_protocol_server && + + test_trace2_data gvfs-helper prefetch/install_mode '$expected_mode' \ + <"trace-$test_count.txt" + ' + + test_expect_success "prefetch with --since ($mode, threads=$threads)" ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + git -C "$REPO_T1" config gvfs.prefetchThreads '$threads' && + + GIT_TRACE2_EVENT="$(pwd)/trace-$test_count.txt" && + export GIT_TRACE2_EVENT && + + do_prefetch_since && + + stop_gvfs_protocol_server && + + test_trace2_data gvfs-helper prefetch/install_mode '$expected_mode' \ + <"trace-$test_count.txt" + ' + + test_expect_success "prefetch up-to-date ($mode, threads=$threads)" ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server && + git -C "$REPO_T1" config gvfs.prefetchThreads '$threads' && + + do_prefetch_up_to_date && + + stop_gvfs_protocol_server + ' + + test_expect_success "prefetch corrupt pack ($mode, threads=$threads)" ' + test_when_finished "per_test_cleanup" && + start_gvfs_protocol_server_with_mayhem \ + bad_prefetch_pack_sha \ + no_prefetch_idx && + git -C "$REPO_T1" config gvfs.prefetchThreads '$threads' && + + do_prefetch_corrupt_pack && + + stop_gvfs_protocol_server + ' +done + +test_done