diff options
90 files changed, 1848 insertions, 1157 deletions
diff --git a/ChangeLog b/ChangeLog index 1ec90b7bccc8..0a5eced2d439 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,88 @@ +2025-06-18 Simon J Gerraty <sjg@beast.crufty.net> + + * VERSION (_MAKE_VERSION): 20250618 + Merge with NetBSD make, pick up + o parse.c: in a warning without location information, + print the stack trace + +2025-06-15 Simon J Gerraty <sjg@beast.crufty.net> + + * VERSION (_MAKE_VERSION): 20250615 + Merge with NetBSD make, pick up + o add on-demand inter-process stack traces + o job.c,meta.c: do not discard empty lines in the output of a command + o job.c: add job prefix if necessary in non-default filtered mode + o parse.c,var.c: skip inter-process stack trace when + MAKE_STACK_TRACE=no + +2025-06-12 Simon J Gerraty <sjg@beast.crufty.net> + + * VERSION (_MAKE_VERSION): 20250612 + Merge with NetBSD make, pick up + o use a common style for unexpected error messages + o parse.c: add program name to stack traces from sub-makes + add quotes to "in directory" line in stack traces + o var.c: check variable names for invalid characters when there + are no modifiers to apply. This detects and warns about gmake + syntax like: $(addprefix -I, $(LIST)) + +2025-06-09 Simon J Gerraty <sjg@beast.crufty.net> + + * VERSION (_MAKE_VERSION): 20250606 + Merge with NetBSD make, pick up + o main.c: fix bug in handling of output of children in jobs mode + +2025-05-28 Simon J Gerraty <sjg@beast.crufty.net> + + * VERSION (_MAKE_VERSION): 20250528 + Merge with NetBSD make, pick up + o show contents of MAKEFLAGS in the stack trace. + o main.c: delay warning about bogus -J flag, if we end up in + compat mode before the call to InitMaxJobs, the warning isn't + necessary. + +2025-05-25 Simon J Gerraty <sjg@beast.crufty.net> + + * VERSION (_MAKE_VERSION): 20250525 + Merge with NetBSD make, pick up + o main.c: set .CURDIR earlier so it can be reported in some errors. + +2025-05-20 Simon J Gerraty <sjg@beast.crufty.net> + + * VERSION (_MAKE_VERSION): 20250520 + Merge with NetBSD make, pick up + o rename variables, remove now-redundant comments + o job.c: clean up building the shell commands in parallel mode + remove timeout for polling in parallel mode + o main.c: clean up error message for malformed internal -J option + +2025-05-11 Simon J Gerraty <sjg@beast.crufty.net> + + * VERSION (_MAKE_VERSION): 20250511 + Merge with NetBSD make, pick up + o job.c: rename token pool variables to be more descriptive + move ContinueJobs further up, to eliminate a forward declaration + error out if writing to an internal pipe fails + clean up constant names and function names + use uniform debug log messages for the token pool + in the debug log, replace magic numbers with identifiers + o main.c: clean up error message for malformed internal -J option + o make.c: replace bitset in trace output with descriptive node + attributes + o targ.c: add end marker for -dg1, -dg2 and -dg3 debug log + o var.c: fix order of error messages in the ":?" modifier + +2025-04-25 Simon J Gerraty <sjg@beast.crufty.net> + + * VERSION (_MAKE_VERSION): 20250424 + Merge with NetBSD make, pick up + o cleanup; replace unsigned int with just unsigned + Inline the TMPPAT macro, as it is only needed in a single place + o move struct Job from job.h to job.c + o job.c: group the code for handling the job token pool + avoid excessive values of -j + o make.c: fix grammar in debug log message + 2025-04-14 Simon J Gerraty <sjg@beast.crufty.net> * VERSION (_MAKE_VERSION): 20250414 @@ -76,6 +76,7 @@ unit-tests/archive-suffix.exp unit-tests/archive-suffix.mk unit-tests/archive.exp unit-tests/archive.mk +unit-tests/check-expect.lua unit-tests/cmd-errors-jobs.exp unit-tests/cmd-errors-jobs.mk unit-tests/cmd-errors-lint.exp @@ -410,6 +411,8 @@ unit-tests/job-output-long-lines.exp unit-tests/job-output-long-lines.mk unit-tests/job-output-null.exp unit-tests/job-output-null.mk +unit-tests/job-output.exp +unit-tests/job-output.mk unit-tests/jobs-empty-commands-error.exp unit-tests/jobs-empty-commands-error.mk unit-tests/jobs-empty-commands.exp @@ -859,6 +862,8 @@ unit-tests/varname-make_print_var_on_error-jobs.exp unit-tests/varname-make_print_var_on_error-jobs.mk unit-tests/varname-make_print_var_on_error.exp unit-tests/varname-make_print_var_on_error.mk +unit-tests/varname-make_stack_trace.exp +unit-tests/varname-make_stack_trace.mk unit-tests/varname-makefile.exp unit-tests/varname-makefile.mk unit-tests/varname-makeflags.exp @@ -1,2 +1,2 @@ # keep this compatible with sh and make -_MAKE_VERSION=20250414 +_MAKE_VERSION=20250618 @@ -1,4 +1,4 @@ -.\" $NetBSD: make.1,v 1.384 2025/04/04 18:36:47 sjg Exp $ +.\" $NetBSD: make.1,v 1.385 2025/06/13 03:51:18 rillig Exp $ .\" .\" Copyright (c) 1990, 1993 .\" The Regents of the University of California. All rights reserved. @@ -29,7 +29,7 @@ .\" .\" from: @(#)make.1 8.4 (Berkeley) 3/19/94 .\" -.Dd April 4, 2025 +.Dd June 12, 2025 .Dt BMAKE 1 .Os .Sh NAME @@ -2688,6 +2688,7 @@ uses the following environment variables, if they exist: .Ev MAKEOBJDIR , .Ev MAKEOBJDIRPREFIX , .Ev MAKESYSPATH , +.Ev MAKE_STACK_TRACE , .Ev PWD , and .Ev TMPDIR . @@ -2707,6 +2708,12 @@ very early and the target is used to reset .Sq Va .OBJDIR , there may be unexpected side effects. +.Pp +If the +.Ev MAKE_STACK_TRACE +environment variable is set to +.Dq yes , +any stack traces include the call chain of the parent processes. .Sh FILES .Bl -tag -width /usr/share/mk -compact .It .depend diff --git a/bmake.cat1 b/bmake.cat1 index e1340e1c78e0..667c80898def 100644 --- a/bmake.cat1 +++ b/bmake.cat1 @@ -1728,7 +1728,7 @@ SSPPEECCIIAALL TTAARRGGEETTSS EENNVVIIRROONNMMEENNTT bbmmaakkee uses the following environment variables, if they exist: MACHINE, MACHINE_ARCH, MAKE, MAKEFLAGS, MAKEOBJDIR, MAKEOBJDIRPREFIX, MAKESYSPATH, - PWD, and TMPDIR. + MAKE_STACK_TRACE, PWD, and TMPDIR. MAKEOBJDIRPREFIX and MAKEOBJDIR should be set in the environment or on the command line to bbmmaakkee and not as makefile variables; see the @@ -1736,6 +1736,9 @@ EENNVVIIRROONNMMEENNTT via makefile variables but unless done very early and the `..OOBBJJDDIIRR' target is used to reset `_._O_B_J_D_I_R', there may be unexpected side effects. + If the MAKE_STACK_TRACE environment variable is set to "yes", any stack + traces include the call chain of the parent processes. + FFIILLEESS .depend list of dependencies makefile first default makefile if no makefile is specified on the @@ -1828,4 +1831,4 @@ BBUUGGSS attempt to suppress a cascade of unnecessary errors, can result in a seemingly unexplained `*** Error code 6' -FreeBSD 14.2-RELEASE-p1 April 4, 2025 FreeBSD 14.2-RELEASE-p1 +FreeBSD 14.2-RELEASE-p1 June 12, 2025 FreeBSD 14.2-RELEASE-p1 @@ -1,4 +1,4 @@ -/* $NetBSD: compat.c,v 1.262 2025/01/19 10:57:10 rillig Exp $ */ +/* $NetBSD: compat.c,v 1.267 2025/06/13 03:51:18 rillig Exp $ */ /* * Copyright (c) 1988, 1989, 1990 The Regents of the University of California. @@ -90,13 +90,16 @@ #include "make.h" #include "dir.h" #include "job.h" +#ifdef USE_META +# include "meta.h" +#endif #include "metachar.h" #include "pathnames.h" /* "@(#)compat.c 8.2 (Berkeley) 3/19/94" */ -MAKE_RCSID("$NetBSD: compat.c,v 1.262 2025/01/19 10:57:10 rillig Exp $"); +MAKE_RCSID("$NetBSD: compat.c,v 1.267 2025/06/13 03:51:18 rillig Exp $"); -static GNode *curTarg = NULL; +static GNode *curTarg; static pid_t compatChild; static int compatSigno; @@ -107,7 +110,7 @@ static int compatSigno; static void CompatDeleteTarget(GNode *gn) { - if (gn != NULL && !GNode_IsPrecious(gn) && + if (!GNode_IsPrecious(gn) && (gn->type & OP_PHONY) == 0) { const char *file = GNode_VarTarget(gn); if (!opts.noExecute && unlink_file(file) == 0) @@ -127,11 +130,9 @@ CompatDeleteTarget(GNode *gn) static void CompatInterrupt(int signo) { - CompatDeleteTarget(curTarg); - - if (curTarg != NULL && !GNode_IsPrecious(curTarg)) { - /* Run .INTERRUPT only if hit with interrupt signal. */ - if (signo == SIGINT) { + if (curTarg != NULL) { + CompatDeleteTarget(curTarg); + if (signo == SIGINT && !GNode_IsPrecious(curTarg)) { GNode *gn = Targ_FindNode(".INTERRUPT"); if (gn != NULL) Compat_Make(gn, gn); @@ -309,8 +310,6 @@ Compat_RunCommand(const char *cmdp, GNode *gn, StringListNode *ln) cmd++; } - while (ch_isspace(*cmd)) - cmd++; if (cmd[0] == '\0') goto register_command; @@ -337,7 +336,7 @@ Compat_RunCommand(const char *cmdp, GNode *gn, StringListNode *ln) if (Cmd_Argv(cmd, cmd_len, shargv, 5, cmd_file, sizeof(cmd_file), - (errCheck && shellErrFlag != NULL), + errCheck && shellErrFlag != NULL, DEBUG(SHELL)) < 0) Fatal("cannot run \"%s\"", cmd); av = shargv; @@ -356,6 +355,7 @@ Compat_RunCommand(const char *cmdp, GNode *gn, StringListNode *ln) #endif Var_ReexportVars(gn); + Var_ExportStackTrace(gn->name, cmd); compatChild = Compat_Spawn(av); free(mav); @@ -730,7 +730,7 @@ InitSignals(void) } void -Compat_MakeAll(GNodeList *targs) +Compat_MakeAll(GNodeList *targets) { GNode *errorNode = NULL; @@ -753,10 +753,10 @@ Compat_MakeAll(GNodeList *targs) * Expand .USE nodes right now, because they can modify the structure * of the tree. */ - Make_ExpandUse(targs); + Make_ExpandUse(targets); - while (!Lst_IsEmpty(targs)) { - GNode *gn = Lst_Dequeue(targs); + while (!Lst_IsEmpty(targets)) { + GNode *gn = Lst_Dequeue(targets); Compat_Make(gn, gn); if (gn->made == UPTODATE) { @@ -1,4 +1,4 @@ -/* $NetBSD: cond.c,v 1.372 2025/04/10 21:41:35 rillig Exp $ */ +/* $NetBSD: cond.c,v 1.373 2025/04/22 19:28:50 rillig Exp $ */ /* * Copyright (c) 1988, 1989, 1990 The Regents of the University of California. @@ -91,7 +91,7 @@ #include "dir.h" /* "@(#)cond.c 8.2 (Berkeley) 1/2/94" */ -MAKE_RCSID("$NetBSD: cond.c,v 1.372 2025/04/10 21:41:35 rillig Exp $"); +MAKE_RCSID("$NetBSD: cond.c,v 1.373 2025/04/22 19:28:50 rillig Exp $"); /* * Conditional expressions conform to this grammar: @@ -166,7 +166,7 @@ typedef struct CondParser { static CondResult CondParser_Or(CondParser *, bool); -unsigned int cond_depth = 0; /* current .if nesting level */ +unsigned cond_depth = 0; /* current .if nesting level */ /* Names for ComparisonOp. */ static const char opname[][3] = { "<", "<=", ">", ">=", "==", "!=" }; @@ -1028,7 +1028,7 @@ Cond_EvalLine(const char *line) } IfState; static enum IfState *cond_states = NULL; - static unsigned int cond_states_cap = 128; + static unsigned cond_states_cap = 128; bool plain; bool (*evalBare)(const char *); @@ -1221,7 +1221,7 @@ found_variable: void Cond_EndFile(void) { - unsigned int open_conds = cond_depth - CurFile_CondMinDepth(); + unsigned open_conds = cond_depth - CurFile_CondMinDepth(); if (open_conds != 0) { Parse_Error(PARSE_FATAL, "%u open conditional%s", @@ -1,4 +1,4 @@ -/* $NetBSD: dir.c,v 1.296 2025/04/11 17:21:31 rillig Exp $ */ +/* $NetBSD: dir.c,v 1.297 2025/06/12 18:51:05 rillig Exp $ */ /* * Copyright (c) 1988, 1989, 1990 The Regents of the University of California. @@ -132,7 +132,7 @@ #include "job.h" /* "@(#)dir.c 8.2 (Berkeley) 1/2/94" */ -MAKE_RCSID("$NetBSD: dir.c,v 1.296 2025/04/11 17:21:31 rillig Exp $"); +MAKE_RCSID("$NetBSD: dir.c,v 1.297 2025/06/12 18:51:05 rillig Exp $"); /* * A search path is a list of CachedDir structures. A CachedDir has in it the @@ -492,7 +492,7 @@ Dir_InitDot(void) dir = SearchPath_Add(NULL, "."); if (dir == NULL) { - Error("Cannot open `.' (%s)", strerror(errno)); + Error("Cannot open \".\": %s", strerror(errno)); exit(2); /* Not 1 so -q can distinguish error */ } @@ -1,4 +1,4 @@ -/* $NetBSD: for.c,v 1.184 2025/04/11 18:08:17 rillig Exp $ */ +/* $NetBSD: for.c,v 1.185 2025/04/22 19:28:50 rillig Exp $ */ /* * Copyright (c) 1992, The Regents of the University of California. @@ -58,14 +58,14 @@ #include "make.h" /* "@(#)for.c 8.1 (Berkeley) 6/6/93" */ -MAKE_RCSID("$NetBSD: for.c,v 1.184 2025/04/11 18:08:17 rillig Exp $"); +MAKE_RCSID("$NetBSD: for.c,v 1.185 2025/04/22 19:28:50 rillig Exp $"); typedef struct ForLoop { Vector /* of 'char *' */ vars; /* Iteration variables */ SubstringWords items; /* Substitution items */ Buffer body; /* Unexpanded body of the loop */ - unsigned int nextItem; /* Where to continue iterating */ + unsigned nextItem; /* Where to continue iterating */ } ForLoop; @@ -330,7 +330,7 @@ ExprLen(const char *s, const char *e) * by ApplyModifier_Defined. */ static void -AddEscaped(Buffer *cmds, Substring item, char endc) +AddEscaped(Buffer *body, Substring item, char endc) { const char *p; char ch; @@ -344,18 +344,18 @@ AddEscaped(Buffer *cmds, Substring item, char endc) * XXX: Should a '\' be added here? * See directive-for-escape.mk, ExprLen. */ - Buf_AddBytes(cmds, p, 1 + len); + Buf_AddBytes(body, p, 1 + len); p += 1 + len; continue; } - Buf_AddByte(cmds, '\\'); + Buf_AddByte(body, '\\'); } else if (ch == ':' || ch == '\\' || ch == endc) - Buf_AddByte(cmds, '\\'); + Buf_AddByte(body, '\\'); else if (ch == '\n') { Parse_Error(PARSE_FATAL, "newline in .for value"); ch = ' '; /* prevent newline injection */ } - Buf_AddByte(cmds, ch); + Buf_AddByte(body, ch); p++; } } @@ -365,7 +365,7 @@ AddEscaped(Buffer *cmds, Substring item, char endc) * expression like ${i} or ${i:...} or $(i) or $(i:...) with ":Uvalue". */ static void -ForLoop_SubstVarLong(ForLoop *f, unsigned int firstItem, Buffer *body, +ForLoop_SubstVarLong(ForLoop *f, unsigned firstItem, Buffer *body, const char **pp, char endc, const char **inout_mark) { size_t i; @@ -400,7 +400,7 @@ ForLoop_SubstVarLong(ForLoop *f, unsigned int firstItem, Buffer *body, * expressions like $i with their ${:U...} expansion. */ static void -ForLoop_SubstVarShort(ForLoop *f, unsigned int firstItem, Buffer *body, +ForLoop_SubstVarShort(ForLoop *f, unsigned firstItem, Buffer *body, const char *p, const char **inout_mark) { char ch = *p; @@ -444,7 +444,7 @@ found: * See unit-tests/directive-for-escape.mk. */ static void -ForLoop_SubstBody(ForLoop *f, unsigned int firstItem, Buffer *body) +ForLoop_SubstBody(ForLoop *f, unsigned firstItem, Buffer *body) { const char *p, *end; const char *mark; /* where the last substitution left off */ @@ -479,8 +479,8 @@ For_NextIteration(ForLoop *f, Buffer *body) if (f->nextItem == f->items.len) return false; - f->nextItem += (unsigned int)f->vars.len; - ForLoop_SubstBody(f, f->nextItem - (unsigned int)f->vars.len, body); + f->nextItem += (unsigned)f->vars.len; + ForLoop_SubstBody(f, f->nextItem - (unsigned)f->vars.len, body); if (DEBUG(FOR)) { char *details = ForLoop_Details(f); debug_printf("For: loop body with %s:\n%s", @@ -494,7 +494,7 @@ For_NextIteration(ForLoop *f, Buffer *body) void For_Break(ForLoop *f) { - f->nextItem = (unsigned int)f->items.len; + f->nextItem = (unsigned)f->items.len; } /* Run the .for loop, imitating the actions of an include file. */ @@ -1,4 +1,4 @@ -/* $NetBSD: hash.c,v 1.79 2024/07/07 09:37:00 rillig Exp $ */ +/* $NetBSD: hash.c,v 1.80 2025/04/22 19:28:50 rillig Exp $ */ /* * Copyright (c) 1988, 1989, 1990 The Regents of the University of California. @@ -74,7 +74,7 @@ #include "make.h" /* "@(#)hash.c 8.1 (Berkeley) 6/6/93" */ -MAKE_RCSID("$NetBSD: hash.c,v 1.79 2024/07/07 09:37:00 rillig Exp $"); +MAKE_RCSID("$NetBSD: hash.c,v 1.80 2025/04/22 19:28:50 rillig Exp $"); /* * The ratio of # entries to # buckets at which we rebuild the table to @@ -83,10 +83,10 @@ MAKE_RCSID("$NetBSD: hash.c,v 1.79 2024/07/07 09:37:00 rillig Exp $"); #define rebuildLimit 3 /* This hash function matches Gosling's Emacs and java.lang.String. */ -static unsigned int +static unsigned Hash_String(const char *key, const char **out_keyEnd) { - unsigned int h; + unsigned h; const char *p; h = 0; @@ -98,10 +98,10 @@ Hash_String(const char *key, const char **out_keyEnd) } /* This hash function matches Gosling's Emacs and java.lang.String. */ -unsigned int +unsigned Hash_Substring(Substring key) { - unsigned int h; + unsigned h; const char *p; h = 0; @@ -111,7 +111,7 @@ Hash_Substring(Substring key) } static HashEntry * -HashTable_Find(HashTable *t, Substring key, unsigned int h) +HashTable_Find(HashTable *t, Substring key, unsigned h) { HashEntry *he; size_t keyLen = Substring_Length(key); @@ -135,7 +135,7 @@ HashTable_Find(HashTable *t, Substring key, unsigned int h) void HashTable_Init(HashTable *t) { - unsigned int n = 16, i; + unsigned n = 16, i; HashEntry **buckets = bmake_malloc(sizeof *buckets * n); for (i = 0; i < n; i++) buckets[i] = NULL; @@ -176,7 +176,7 @@ HashEntry * HashTable_FindEntry(HashTable *t, const char *key) { const char *keyEnd; - unsigned int h = Hash_String(key, &keyEnd); + unsigned h = Hash_String(key, &keyEnd); return HashTable_Find(t, Substring_Init(key, keyEnd), h); } @@ -193,7 +193,7 @@ HashTable_FindValue(HashTable *t, const char *key) * or return NULL. */ void * -HashTable_FindValueBySubstringHash(HashTable *t, Substring key, unsigned int h) +HashTable_FindValueBySubstringHash(HashTable *t, Substring key, unsigned h) { HashEntry *he = HashTable_Find(t, key, h); return he != NULL ? he->value : NULL; @@ -220,10 +220,10 @@ HashTable_MaxChain(const HashTable *t) static void HashTable_Enlarge(HashTable *t) { - unsigned int oldSize = t->bucketsSize; + unsigned oldSize = t->bucketsSize; HashEntry **oldBuckets = t->buckets; - unsigned int newSize = 2 * oldSize; - unsigned int newMask = newSize - 1; + unsigned newSize = 2 * oldSize; + unsigned newMask = newSize - 1; HashEntry **newBuckets = bmake_malloc(sizeof *newBuckets * newSize); size_t i; @@ -257,7 +257,7 @@ HashEntry * HashTable_CreateEntry(HashTable *t, const char *key, bool *out_isNew) { const char *keyEnd; - unsigned int h = Hash_String(key, &keyEnd); + unsigned h = Hash_String(key, &keyEnd); HashEntry *he = HashTable_Find(t, Substring_Init(key, keyEnd), h); if (he != NULL) { @@ -313,7 +313,7 @@ HashIter_Next(HashIter *hi) HashTable *t = hi->table; HashEntry *he = hi->entry; HashEntry **buckets = t->buckets; - unsigned int bucketsSize = t->bucketsSize; + unsigned bucketsSize = t->bucketsSize; if (he != NULL) he = he->next; /* skip the most recently returned entry */ @@ -1,4 +1,4 @@ -/* $NetBSD: hash.h,v 1.51 2024/07/07 09:37:00 rillig Exp $ */ +/* $NetBSD: hash.h,v 1.52 2025/04/22 19:28:50 rillig Exp $ */ /* * Copyright (c) 1988, 1989, 1990 The Regents of the University of California. @@ -82,22 +82,22 @@ typedef struct HashEntry { struct HashEntry *next; /* Used to link together all the entries * associated with the same bucket. */ void *value; - unsigned int hash; /* hash value of the key */ + unsigned hash; /* hash value of the key */ char key[1]; /* key string, variable length */ } HashEntry; /* The hash table containing the entries. */ typedef struct HashTable { HashEntry **buckets; - unsigned int bucketsSize; - unsigned int numEntries; - unsigned int bucketsMask; /* Used to select the bucket for a hash. */ + unsigned bucketsSize; + unsigned numEntries; + unsigned bucketsMask; /* Used to select the bucket for a hash. */ } HashTable; /* State of an iteration over all entries in a table. */ typedef struct HashIter { HashTable *table; /* Table being searched. */ - unsigned int nextBucket; /* Next bucket to check (after current). */ + unsigned nextBucket; /* Next bucket to check (after current). */ HashEntry *entry; /* Next entry to check in current bucket. */ } HashIter; @@ -131,8 +131,8 @@ void HashTable_Init(HashTable *); void HashTable_Done(HashTable *); HashEntry *HashTable_FindEntry(HashTable *, const char *) MAKE_ATTR_USE; void *HashTable_FindValue(HashTable *, const char *) MAKE_ATTR_USE; -unsigned int Hash_Substring(Substring) MAKE_ATTR_USE; -void *HashTable_FindValueBySubstringHash(HashTable *, Substring, unsigned int) +unsigned Hash_Substring(Substring) MAKE_ATTR_USE; +void *HashTable_FindValueBySubstringHash(HashTable *, Substring, unsigned) MAKE_ATTR_USE; HashEntry *HashTable_CreateEntry(HashTable *, const char *, bool *); void HashTable_Set(HashTable *, const char *, void *); @@ -1,4 +1,4 @@ -/* $NetBSD: job.c,v 1.492 2025/04/12 13:00:21 rillig Exp $ */ +/* $NetBSD: job.c,v 1.516 2025/06/13 06:13:19 rillig Exp $ */ /* * Copyright (c) 1988, 1989, 1990 The Regents of the University of California. @@ -73,45 +73,29 @@ * Create child processes and collect their output. * * Interface: - * Job_Init Called to initialize this module. In addition, - * the .BEGIN target is made, including all of its - * dependencies before this function returns. - * Hence, the makefiles must have been parsed - * before this function is called. + * Job_Init Initialize this module and make the .BEGIN target. * * Job_End Clean up any memory used. * - * Job_Make Start the creation of the given target. + * Job_Make Start making the given target. * * Job_CatchChildren - * Check for and handle the termination of any - * children. This must be called reasonably - * frequently to keep the whole make going at - * a decent clip, since job table entries aren't - * removed until their process is caught this way. + * Handle the termination of any children. * * Job_CatchOutput - * Print any output our children have produced. - * Should also be called fairly frequently to - * keep the user informed of what's going on. - * If no output is waiting, it will block for - * a time given by the SEL_* constants, below, - * or until output is ready. + * Print any output the child processes have produced. * * Job_ParseShell Given a special dependency line with target '.SHELL', * define the shell that is used for the creation * commands in jobs mode. * - * Job_Finish Make the .END target. Should only be called when the + * Job_Finish Make the .END target. Must only be called when the * job table is empty. * - * Job_AbortAll Abort all currently running jobs. Do not handle - * output or do anything for the jobs, just kill them. - * Should only be called in an emergency. + * Job_AbortAll Kill all currently running jobs, in an emergency. * * Job_CheckCommands - * Verify that the commands for a target are - * ok. Provide them if necessary and possible. + * Add fallback commands to a target, if necessary. * * Job_Touch Update a target without really updating it. * @@ -123,7 +107,6 @@ #endif #include <sys/types.h> #include <sys/stat.h> -#include <sys/file.h> #include <sys/time.h> #include "wait.h" @@ -147,11 +130,111 @@ #include "make.h" #include "dir.h" #include "job.h" +#ifdef USE_META +# include "meta.h" +#endif #include "pathnames.h" #include "trace.h" /* "@(#)job.c 8.2 (Berkeley) 3/19/94" */ -MAKE_RCSID("$NetBSD: job.c,v 1.492 2025/04/12 13:00:21 rillig Exp $"); +MAKE_RCSID("$NetBSD: job.c,v 1.516 2025/06/13 06:13:19 rillig Exp $"); + + +#ifdef USE_SELECT +/* + * Emulate poll() in terms of select(). This is not a complete + * emulation but it is sufficient for make's purposes. + */ + +#define poll emul_poll +#define pollfd emul_pollfd + +struct emul_pollfd { + int fd; + short events; + short revents; +}; + +#define POLLIN 0x0001 +#define POLLOUT 0x0004 + +int emul_poll(struct pollfd *, int, int); +#endif + +struct pollfd; + + +enum JobStatus { + JOB_ST_FREE, /* Job is available */ + JOB_ST_SET_UP, /* Job is allocated but otherwise invalid */ + JOB_ST_RUNNING, /* Job is running, pid valid */ + JOB_ST_FINISHED /* Job is done (i.e. after SIGCHLD) */ +}; + +static const char JobStatus_Name[][9] = { + "free", + "set-up", + "running", + "finished", +}; + +/* + * A Job manages the shell commands that are run to create a single target. + * Each job is run in a separate subprocess by a shell. Several jobs can run + * in parallel. + * + * The shell commands for the target are written to a temporary file, + * then the shell is run with the temporary file as stdin, and the output + * of that shell is captured via a pipe. + * + * When a job is finished, Make_Update updates all parents of the node + * that was just remade, marking them as ready to be made next if all + * other dependencies are finished as well. + */ +struct Job { + /* The process ID of the shell running the commands */ + int pid; + + /* The target the child is making */ + GNode *node; + + /* + * If one of the shell commands is "...", all following commands are + * delayed until the .END node is made. This list node points to the + * first of these commands, if any. + */ + StringListNode *tailCmds; + + /* This is where the shell commands go. */ + FILE *cmdFILE; + + int exit_status; /* from wait4() in signal handler */ + + enum JobStatus status; + + bool suspended; + + /* Ignore non-zero exits */ + bool ignerr; + /* Output the command before or instead of running it. */ + bool echo; + /* Target is a special one. */ + bool special; + + int inPipe; /* Pipe for reading output from job */ + int outPipe; /* Pipe for writing control commands */ + struct pollfd *inPollfd; /* pollfd associated with inPipe */ + +#define JOB_BUFSIZE 1024 + /* Buffer for storing the output of the job, line by line. */ + char outBuf[JOB_BUFSIZE + 1]; + size_t outBufLen; + +#ifdef USE_META + struct BuildMon bm; +#endif +}; + /* * A shell defines how the commands are run. All commands for a target are @@ -264,6 +347,8 @@ static enum { /* Why is the make aborting? */ } aborting = ABORT_NONE; #define JOB_TOKENS "+EI+" /* Token to requeue for each abort state */ +static const char aborting_name[][6] = { "NONE", "ERROR", "INTR", "WAIT" }; + /* Tracks the number of tokens currently "out" to build jobs. */ int jobTokensRunning = 0; @@ -426,11 +511,12 @@ static void watchfd(Job *); static void clearfd(Job *); static char *targPrefix = NULL; /* To identify a job change in the output. */ -static Job tokenWaitJob; /* token wait pseudo-job */ + +static Job tokenPoolJob; /* token wait pseudo-job */ static Job childExitJob; /* child exit pseudo-job */ -#define CHILD_EXIT "." -#define DO_JOB_RESUME "R" +#define CEJ_CHILD_EXITED '.' +#define CEJ_RESUME_JOBS 'R' enum { npseudojobs = 2 /* number of pseudo-jobs */ @@ -441,7 +527,6 @@ static volatile sig_atomic_t caught_sigchld; static void CollectOutput(Job *, bool); static void JobInterrupt(bool, int) MAKE_ATTR_DEAD; -static void JobRestartJobs(void); static void JobSigReset(void); static void @@ -477,17 +562,38 @@ Job_FlagsToString(const Job *job, char *buf, size_t bufsize) job->special ? 'S' : '-'); } +#ifdef USE_META +struct BuildMon * +Job_BuildMon(Job *job) +{ + return &job->bm; +} +#endif + +GNode * +Job_Node(Job *job) +{ + return job->node; +} + +int +Job_Pid(Job *job) +{ + return job->pid; +} + static void DumpJobs(const char *where) { - Job *job; + const Job *job; char flags[4]; - debug_printf("job table @ %s\n", where); + debug_printf("%s, job table:\n", where); for (job = job_table; job < job_table_end; job++) { Job_FlagsToString(job, flags, sizeof flags); - debug_printf("job %d, status %d, flags %s, pid %d\n", - (int)(job - job_table), job->status, flags, job->pid); + debug_printf("job %d, status %s, flags %s, pid %d\n", + (int)(job - job_table), JobStatus_Name[job->status], + flags, job->pid); } } @@ -514,39 +620,44 @@ JobDeleteTarget(GNode *gn) Error("*** %s removed", file); } -/* - * JobSigLock/JobSigUnlock - * - * Signal lock routines to get exclusive access. Currently used to - * protect `jobs' and `stoppedJobs' list manipulations. - */ +/* Lock the jobs table and the jobs therein. */ static void -JobSigLock(sigset_t *omaskp) +JobsTable_Lock(sigset_t *omaskp) { - if (sigprocmask(SIG_BLOCK, &caught_signals, omaskp) != 0) { - Punt("JobSigLock: sigprocmask: %s", strerror(errno)); - sigemptyset(omaskp); - } + if (sigprocmask(SIG_BLOCK, &caught_signals, omaskp) != 0) + Punt("JobsTable_Lock: %s", strerror(errno)); } +/* Unlock the jobs table and the jobs therein. */ static void -JobSigUnlock(sigset_t *omaskp) +JobsTable_Unlock(sigset_t *omaskp) { (void)sigprocmask(SIG_SETMASK, omaskp, NULL); } static void +SetNonblocking(int fd) +{ + int flags = fcntl(fd, F_GETFL, 0); + if (flags == -1) + Punt("SetNonblocking.get: %s", strerror(errno)); + flags |= O_NONBLOCK; + if (fcntl(fd, F_SETFL, flags) == -1) + Punt("SetNonblocking.set: %s", strerror(errno)); +} + +static void JobCreatePipe(Job *job, int minfd) { - int i, fd, flags; + int i; int pipe_fds[2]; if (pipe(pipe_fds) == -1) - Punt("Cannot create pipe: %s", strerror(errno)); + Punt("JobCreatePipe: %s", strerror(errno)); for (i = 0; i < 2; i++) { /* Avoid using low-numbered fds */ - fd = fcntl(pipe_fds[i], F_DUPFD, minfd); + int fd = fcntl(pipe_fds[i], F_DUPFD, minfd); if (fd != -1) { close(pipe_fds[i]); pipe_fds[i] = fd; @@ -557,9 +668,9 @@ JobCreatePipe(Job *job, int minfd) job->outPipe = pipe_fds[1]; if (fcntl(job->inPipe, F_SETFD, FD_CLOEXEC) == -1) - Punt("Cannot set close-on-exec: %s", strerror(errno)); + Punt("SetCloseOnExec: %s", strerror(errno)); if (fcntl(job->outPipe, F_SETFD, FD_CLOEXEC) == -1) - Punt("Cannot set close-on-exec: %s", strerror(errno)); + Punt("SetCloseOnExec: %s", strerror(errno)); /* * We mark the input side of the pipe non-blocking; we poll(2) the @@ -567,12 +678,7 @@ JobCreatePipe(Job *job, int minfd) * race for the token when a new one becomes available, so the read * from the pipe should not block. */ - flags = fcntl(job->inPipe, F_GETFL, 0); - if (flags == -1) - Punt("Cannot get flags: %s", strerror(errno)); - flags |= O_NONBLOCK; - if (fcntl(job->inPipe, F_SETFL, flags) == -1) - Punt("Cannot set flags: %s", strerror(errno)); + SetNonblocking(job->inPipe); } /* Pass the signal to each running job. */ @@ -581,43 +687,36 @@ JobCondPassSig(int signo) { Job *job; - DEBUG1(JOB, "JobCondPassSig(%d) called.\n", signo); + DEBUG1(JOB, "JobCondPassSig: signal %d\n", signo); for (job = job_table; job < job_table_end; job++) { if (job->status != JOB_ST_RUNNING) continue; - DEBUG2(JOB, "JobCondPassSig passing signal %d to child %d.\n", + DEBUG2(JOB, "JobCondPassSig passing signal %d to pid %d\n", signo, job->pid); KILLPG(job->pid, signo); } } -/* - * SIGCHLD handler. - * - * Sends a token on the child exit pipe to wake us up from select()/poll(). - */ static void -JobChildSig(int signo MAKE_ATTR_UNUSED) +WriteOrDie(int fd, char ch) { - caught_sigchld = 1; - while (write(childExitJob.outPipe, CHILD_EXIT, 1) == -1 && - errno == EAGAIN) - continue; + if (write(fd, &ch, 1) != 1) + execDie("write", "child"); } +static void +HandleSIGCHLD(int signo MAKE_ATTR_UNUSED) +{ + caught_sigchld = 1; + /* Wake up from poll(). */ + WriteOrDie(childExitJob.outPipe, CEJ_CHILD_EXITED); +} -/* Resume all stopped jobs. */ static void -JobContinueSig(int signo MAKE_ATTR_UNUSED) +HandleSIGCONT(int signo MAKE_ATTR_UNUSED) { - /* - * Defer sending SIGCONT to our stopped children until we return - * from the signal handler. - */ - while (write(childExitJob.outPipe, DO_JOB_RESUME, 1) == -1 && - errno == EAGAIN) - continue; + WriteOrDie(childExitJob.outPipe, CEJ_RESUME_JOBS); } /* @@ -669,7 +768,7 @@ JobPassSig_suspend(int signo) act.sa_flags = 0; (void)sigaction(signo, &act, NULL); - DEBUG1(JOB, "JobPassSig_suspend passing signal %d to self.\n", signo); + DEBUG1(JOB, "JobPassSig_suspend passing signal %d to self\n", signo); (void)kill(getpid(), signo); @@ -698,7 +797,7 @@ JobPassSig_suspend(int signo) } static Job * -JobFindPid(int pid, JobStatus status, bool isJobs) +JobFindPid(int pid, enum JobStatus status, bool isJobs) { Job *job; @@ -733,8 +832,6 @@ ParseCommandFlags(char **pp, CommandFlags *out_cmdFlags) p++; } - pp_skip_whitespace(&p); - *pp = p; } @@ -876,17 +973,13 @@ JobWriteSpecials(Job *job, ShellWriter *wr, const char *escCmd, bool run, * given to make, stick a shell-specific echoOff command in the script. * * If the command starts with '-' and the shell has no error control (none - * of the predefined shells has that), ignore errors for the entire job. - * - * XXX: Why ignore errors for the entire job? This is even documented in the - * manual page, but without any rationale since there is no known rationale. + * of the predefined shells has that), ignore errors for the rest of the job. * - * XXX: The manual page says the '-' "affects the entire job", but that's not - * accurate. The '-' does not affect the commands before the '-'. + * XXX: Why ignore errors for the entire job? This is documented in the + * manual page, but without giving a rationale. * - * If the command is just "...", skip all further commands of this job. These - * commands are attached to the .END node instead and will be run by - * Job_Finish after all other targets have been made. + * If the command is just "...", attach all further commands of this job to + * the .END node instead, see Job_Finish. */ static void JobWriteCommand(Job *job, ShellWriter *wr, StringListNode *ln, const char *ucmd) @@ -1037,7 +1130,7 @@ JobSaveCommands(Job *job) } -/* Called to close both input and output pipes when a job is finished. */ +/* Close both input and output pipes when a job is finished. */ static void JobClosePipes(Job *job) { @@ -1103,14 +1196,15 @@ JobFinishDoneExitedError(Job *job, WAIT_T *inout_status) static void JobFinishDoneExited(Job *job, WAIT_T *inout_status) { - DEBUG2(JOB, "Process %d [%s] exited.\n", job->pid, job->node->name); + DEBUG2(JOB, "Target %s, pid %d exited\n", + job->node->name, job->pid); if (WEXITSTATUS(*inout_status) != 0) JobFinishDoneExitedError(job, inout_status); else if (DEBUG(JOB)) { SwitchOutputTo(job->node); - (void)printf("*** [%s] Completed successfully\n", - job->node->name); + (void)printf("Target %s, pid %d exited successfully\n", + job->node->name, job->pid); } } @@ -1136,25 +1230,16 @@ JobFinishDone(Job *job, WAIT_T *inout_status) } /* - * Do final processing for the given job including updating parent nodes and - * starting new jobs as available/necessary. - * - * Deferred commands for the job are placed on the .END node. - * - * If there was a serious error (job_errors != 0; not an ignored one), no more - * jobs will be started. - * - * Input: - * job job to finish - * status sub-why job went away + * Finish the job, add deferred commands to the .END node, mark the job as + * free, update parent nodes and start new jobs as available/necessary. */ static void JobFinish (Job *job, WAIT_T status) { bool done, return_job_token; - DEBUG3(JOB, "JobFinish: %d [%s], status %d\n", - job->pid, job->node->name, status); + DEBUG3(JOB, "JobFinish: target %s, pid %d, status %#x\n", + job->node->name, job->pid, status); if ((WIFEXITED(status) && ((WEXITSTATUS(status) != 0 && !job->ignerr))) || @@ -1164,7 +1249,7 @@ JobFinish (Job *job, WAIT_T status) JobClosePipes(job); if (job->cmdFILE != NULL && job->cmdFILE != stdout) { if (fclose(job->cmdFILE) != 0) - Punt("Cannot write shell script for '%s': %s", + Punt("Cannot write shell script for \"%s\": %s", job->node->name, strerror(errno)); job->cmdFILE = NULL; } @@ -1207,11 +1292,6 @@ JobFinish (Job *job, WAIT_T status) if (aborting != ABORT_ERROR && aborting != ABORT_INTERRUPT && (WAIT_STATUS(status) == 0)) { - /* - * As long as we aren't aborting and the job didn't return a - * non-zero status that we shouldn't ignore, we call - * Make_Update to update the parents. - */ JobSaveCommands(job); job->node->made = MADE; if (!job->special) @@ -1229,13 +1309,11 @@ JobFinish (Job *job, WAIT_T status) } if (return_job_token) - Job_TokenReturn(); + TokenPool_Return(); if (aborting == ABORT_ERROR && jobTokensRunning == 0) { - if (shouldDieQuietly(NULL, -1)) { - Job_Wait(); + if (shouldDieQuietly(NULL, -1)) exit(2); - } Fatal("%d error%s", job_errors, job_errors == 1 ? "" : "s"); } } @@ -1320,7 +1398,7 @@ Job_Touch(GNode *gn, bool echo) * abortProc Function to abort with message * * Results: - * true if the commands list is/was ok. + * true if the commands are ok. */ bool Job_CheckCommands(GNode *gn, void (*abortProc)(const char *, ...)) @@ -1406,9 +1484,9 @@ JobExec(Job *job, char **argv) int i; debug_printf("Running %s\n", job->node->name); - debug_printf("\tCommand: "); + debug_printf("\tCommand:"); for (i = 0; argv[i] != NULL; i++) { - debug_printf("%s ", argv[i]); + debug_printf(" %s", argv[i]); } debug_printf("\n"); } @@ -1422,17 +1500,18 @@ JobExec(Job *job, char **argv) if (job->echo) SwitchOutputTo(job->node); - /* No interruptions until this job is on the `jobs' list */ - JobSigLock(&mask); + /* No interruptions until this job is in the jobs table. */ + JobsTable_Lock(&mask); /* Pre-emptively mark job running, pid still zero though */ job->status = JOB_ST_RUNNING; Var_ReexportVars(job->node); + Var_ExportStackTrace(job->node->name, NULL); cpid = FORK_FUNCTION(); if (cpid == -1) - Punt("Cannot fork: %s", strerror(errno)); + Punt("fork: %s", strerror(errno)); if (cpid == 0) { /* Child */ @@ -1448,37 +1527,26 @@ JobExec(Job *job, char **argv) */ JobSigReset(); - /* Now unblock signals */ sigemptyset(&tmask); - JobSigUnlock(&tmask); + JobsTable_Unlock(&tmask); - /* - * Must duplicate the input stream down to the child's input - * and reset it to the beginning (again). Since the stream - * was marked close-on-exec, we must clear that bit in the - * new input. - */ if (dup2(fileno(job->cmdFILE), STDIN_FILENO) == -1) execDie("dup2", "job->cmdFILE"); if (fcntl(STDIN_FILENO, F_SETFD, 0) == -1) - execDie("fcntl clear close-on-exec", "stdin"); + execDie("clear close-on-exec", "stdin"); if (lseek(STDIN_FILENO, 0, SEEK_SET) == -1) execDie("lseek to 0", "stdin"); if (job->node->type & (OP_MAKE | OP_SUBMAKE)) { /* Pass job token pipe to submakes. */ - if (fcntl(tokenWaitJob.inPipe, F_SETFD, 0) == -1) + if (fcntl(tokenPoolJob.inPipe, F_SETFD, 0) == -1) execDie("clear close-on-exec", - "tokenWaitJob.inPipe"); - if (fcntl(tokenWaitJob.outPipe, F_SETFD, 0) == -1) + "tokenPoolJob.inPipe"); + if (fcntl(tokenPoolJob.outPipe, F_SETFD, 0) == -1) execDie("clear close-on-exec", - "tokenWaitJob.outPipe"); + "tokenPoolJob.outPipe"); } - /* - * Set up the child's output to be routed through the pipe - * we've created for it. - */ if (dup2(job->outPipe, STDOUT_FILENO) == -1) execDie("dup2", "job->outPipe"); @@ -1524,36 +1592,31 @@ JobExec(Job *job, char **argv) meta_job_parent(job, cpid); #endif - /* - * Set the current position in the buffer to the beginning - * and mark another stream to watch in the outputs mask - */ - job->curPos = 0; + job->outBufLen = 0; watchfd(job); if (job->cmdFILE != NULL && job->cmdFILE != stdout) { if (fclose(job->cmdFILE) != 0) - Punt("Cannot write shell script for '%s': %s", + Punt("Cannot write shell script for \"%s\": %s", job->node->name, strerror(errno)); job->cmdFILE = NULL; } - /* Now that the job is actually running, add it to the table. */ if (DEBUG(JOB)) { - debug_printf("JobExec(%s): pid %d added to jobs table\n", + debug_printf( + "JobExec: target %s, pid %d added to jobs table\n", job->node->name, job->pid); DumpJobs("job started"); } - JobSigUnlock(&mask); + JobsTable_Unlock(&mask); } -/* Create the argv needed to execute the shell for a given job. */ static void -JobMakeArgv(Job *job, char **argv) +BuildArgv(Job *job, char **argv) { int argc; - static char args[10]; /* For merged arguments */ + static char args[10]; argv[0] = UNCONST(shellName); argc = 1; @@ -1571,11 +1634,10 @@ JobMakeArgv(Job *job, char **argv) * practically relevant. */ (void)snprintf(args, sizeof args, "-%s%s", - (job->ignerr ? "" : - (shell->errFlag != NULL ? shell->errFlag : "")), - (!job->echo ? "" : - (shell->echoFlag != NULL ? shell->echoFlag : ""))); - + !job->ignerr && shell->errFlag != NULL + ? shell->errFlag : "", + job->echo && shell->echoFlag != NULL + ? shell->echoFlag : ""); if (args[1] != '\0') { argv[argc] = args; argc++; @@ -1596,21 +1658,16 @@ JobMakeArgv(Job *job, char **argv) static void JobWriteShellCommands(Job *job, GNode *gn, bool *out_run) { - /* - * tfile is the name of a file into which all shell commands - * are put. It is removed before the child shell is executed, - * unless DEBUG(SCRIPT) is set. - */ - char tfile[MAXPATHLEN]; - int tfd; /* File descriptor to the temp file */ + char fname[MAXPATHLEN]; + int fd; - tfd = Job_TempFile(TMPPAT, tfile, sizeof tfile); + fd = Job_TempFile(NULL, fname, sizeof fname); - job->cmdFILE = fdopen(tfd, "w+"); + job->cmdFILE = fdopen(fd, "w+"); if (job->cmdFILE == NULL) - Punt("Could not fdopen %s", tfile); + Punt("Could not fdopen %s", fname); - (void)fcntl(fileno(job->cmdFILE), F_SETFD, FD_CLOEXEC); + (void)fcntl(fd, F_SETFD, FD_CLOEXEC); #ifdef USE_META if (useMeta) { @@ -1626,8 +1683,8 @@ JobWriteShellCommands(Job *job, GNode *gn, bool *out_run) void Job_Make(GNode *gn) { - Job *job; /* new job descriptor */ - char *argv[10]; /* Argument vector to shell */ + Job *job; + char *argv[10]; bool cmdsOK; /* true if the nodes commands were all right */ bool run; @@ -1659,31 +1716,16 @@ Job_Make(GNode *gn) job->cmdFILE = stdout; run = false; - /* - * We're serious here, but if the commands were bogus, we're - * also dead... - */ if (!cmdsOK) { - PrintOnError(gn, "\n"); /* provide some clue */ + PrintOnError(gn, "\n"); DieHorribly(); } } else if (((gn->type & OP_MAKE) && !opts.noRecursiveExecute) || (!opts.noExecute && !opts.touch)) { - /* - * The above condition looks very similar to - * GNode_ShouldExecute but is subtly different. It prevents - * that .MAKE targets are touched since these are usually - * virtual targets. - */ - int parseErrorsBefore; - /* - * We're serious here, but if the commands were bogus, we're - * also dead... - */ if (!cmdsOK) { - PrintOnError(gn, "\n"); /* provide some clue */ + PrintOnError(gn, "\n"); DieHorribly(); } @@ -1693,10 +1735,6 @@ Job_Make(GNode *gn) run = false; (void)fflush(job->cmdFILE); } else if (!GNode_ShouldExecute(gn)) { - /* - * Just write all the commands to stdout in one fell swoop. - * This still sets up job->tailCmds correctly. - */ SwitchOutputTo(gn); job->cmdFILE = stdout; if (cmdsOK) @@ -1708,20 +1746,15 @@ Job_Make(GNode *gn) run = false; } - /* If we're not supposed to execute a shell, don't. */ if (!run) { if (!job->special) - Job_TokenReturn(); - /* Unlink and close the command file if we opened one */ + TokenPool_Return(); + if (job->cmdFILE != NULL && job->cmdFILE != stdout) { (void)fclose(job->cmdFILE); job->cmdFILE = NULL; } - /* - * We only want to work our way up the graph if we aren't - * here because the commands for the job were no good. - */ if (cmdsOK && aborting == ABORT_NONE) { JobSaveCommands(job); job->node->made = MADE; @@ -1731,15 +1764,8 @@ Job_Make(GNode *gn) return; } - /* - * Set up the control arguments to the shell. This is based on the - * flags set earlier for this job. - */ - JobMakeArgv(job, argv); - - /* Create the pipe by which we'll get the shell's output. */ + BuildArgv(job, argv); JobCreatePipe(job, 3); - JobExec(job, argv); } @@ -1751,197 +1777,111 @@ Job_Make(GNode *gn) * Return the part of the output that the calling function needs to output by * itself. */ -static char * -PrintFilteredOutput(char *p, const char *endp) /* XXX: p should be const */ +static const char * +PrintFilteredOutput(Job *job, size_t len) { - char *ep; /* XXX: should be const */ + const char *p = job->outBuf, *ep, *endp; if (shell->noPrint == NULL || shell->noPrint[0] == '\0') return p; - /* - * XXX: What happens if shell->noPrint occurs on the boundary of - * the buffer? To work correctly in all cases, this should rather - * be a proper stream filter instead of doing string matching on - * selected chunks of the output. - */ - while ((ep = strstr(p, shell->noPrint)) != NULL) { - if (ep != p) { - *ep = '\0'; /* XXX: avoid writing to the buffer */ - /* - * The only way there wouldn't be a newline after - * this line is if it were the last in the buffer. - * however, since the noPrint output comes after it, - * there must be a newline, so we don't print one. - */ - /* XXX: What about null bytes in the output? */ - (void)fprintf(stdout, "%s", p); + endp = p + len; + while ((ep = strstr(p, shell->noPrint)) != NULL && ep < endp) { + if (ep > p) { + if (!opts.silent) + SwitchOutputTo(job->node); + (void)fwrite(p, 1, (size_t)(ep - p), stdout); (void)fflush(stdout); } p = ep + shell->noPrintLen; if (p == endp) break; p++; /* skip over the (XXX: assumed) newline */ - pp_skip_whitespace(&p); + cpp_skip_whitespace(&p); } return p; } /* - * This function is called whenever there is something to read on the pipe. - * We collect more output from the given job and store it in the job's - * outBuf. If this makes up a line, we print it tagged by the job's - * identifier, as necessary. + * Collect output from the job. Print any complete lines. * * In the output of the shell, the 'noPrint' lines are removed. If the * command is not alone on the line (the character after it is not \0 or * \n), we do print whatever follows it. * - * Input: - * job the job whose output needs printing - * finish true if this is the last time we'll be called - * for this job + * If finish is true, collect all remaining output for the job. */ static void CollectOutput(Job *job, bool finish) { - bool gotNL; /* true if got a newline */ - bool fbuf; /* true if our buffer filled up */ + const char *p; size_t nr; /* number of bytes read */ size_t i; /* auxiliary index into outBuf */ size_t max; /* limit for i (end of current data) */ - ssize_t nRead; /* (Temporary) number of bytes read */ - /* Read as many bytes as will fit in the buffer. */ again: - gotNL = false; - fbuf = false; - - nRead = read(job->inPipe, &job->outBuf[job->curPos], - JOB_BUFSIZE - job->curPos); - if (nRead < 0) { + nr = (size_t)read(job->inPipe, job->outBuf + job->outBufLen, + JOB_BUFSIZE - job->outBufLen); + if (nr == (size_t)-1) { if (errno == EAGAIN) return; if (DEBUG(JOB)) perror("CollectOutput(piperead)"); nr = 0; - } else - nr = (size_t)nRead; + } if (nr == 0) finish = false; /* stop looping */ - /* - * If we hit the end-of-file (the job is dead), we must flush its - * remaining output, so pretend we read a newline if there's any - * output remaining in the buffer. - */ - if (nr == 0 && job->curPos != 0) { - job->outBuf[job->curPos] = '\n'; + if (nr == 0 && job->outBufLen > 0) { + job->outBuf[job->outBufLen] = '\n'; nr = 1; } - max = job->curPos + nr; - for (i = job->curPos; i < max; i++) + max = job->outBufLen + nr; + job->outBuf[max] = '\0'; + + for (i = job->outBufLen; i < max; i++) if (job->outBuf[i] == '\0') job->outBuf[i] = ' '; - /* Look for the last newline in the bytes we just got. */ - for (i = job->curPos + nr - 1; - i >= job->curPos && i != (size_t)-1; i--) { - if (job->outBuf[i] == '\n') { - gotNL = true; + for (i = max; i > job->outBufLen; i--) + if (job->outBuf[i - 1] == '\n') break; - } - } - if (!gotNL) { - job->curPos += nr; - if (job->curPos == JOB_BUFSIZE) { - /* - * If we've run out of buffer space, we have no choice - * but to print the stuff. sigh. - */ - fbuf = true; - i = job->curPos; - } + if (i == job->outBufLen) { + job->outBufLen = max; + if (max < JOB_BUFSIZE) + goto unfinished_line; + i = max; } - if (gotNL || fbuf) { - /* - * Need to send the output to the screen. Null terminate it - * first, overwriting the newline character if there was one. - * So long as the line isn't one we should filter (according - * to the shell description), we print the line, preceded - * by a target banner if this target isn't the same as the - * one for which we last printed something. - * The rest of the data in the buffer are then shifted down - * to the start of the buffer and curPos is set accordingly. - */ - job->outBuf[i] = '\0'; - if (i >= job->curPos) { - char *p; - - /* - * FIXME: SwitchOutputTo should be here, according to - * the comment above. But since PrintOutput does not - * do anything in the default shell, this bug has gone - * unnoticed until now. - */ - p = PrintFilteredOutput(job->outBuf, &job->outBuf[i]); - /* - * There's still more in the output buffer. This time, - * though, we know there's no newline at the end, so - * we add one of our own free will. - */ - if (*p != '\0') { - if (!opts.silent) - SwitchOutputTo(job->node); + p = PrintFilteredOutput(job, i); + if (*p != '\0') { + if (!opts.silent) + SwitchOutputTo(job->node); #ifdef USE_META - if (useMeta) { - meta_job_output(job, p, - gotNL ? "\n" : ""); - } + if (useMeta) + meta_job_output(job, p); #endif - (void)fprintf(stdout, "%s%s", p, - gotNL ? "\n" : ""); - (void)fflush(stdout); - } - } - /* - * max is the last offset still in the buffer. Move any - * remaining characters to the start of the buffer and - * update the end marker curPos. - */ - if (i < max) { - (void)memmove(job->outBuf, &job->outBuf[i + 1], - max - (i + 1)); - job->curPos = max - (i + 1); - } else { - assert(i == max); - job->curPos = 0; - } + (void)fwrite(p, 1, (size_t)(job->outBuf + i - p), stdout); + (void)fflush(stdout); } - if (finish) { - /* - * If the finish flag is true, we must loop until we hit - * end-of-file on the pipe. This is guaranteed to happen - * eventually since the other end of the pipe is now closed - * (we closed it explicitly and the child has exited). When - * we do get an EOF, finish will be set false and we'll fall - * through and out. - */ + memmove(job->outBuf, job->outBuf + i, max - i); + job->outBufLen = max - i; + +unfinished_line: + if (finish) goto again; - } } static void -JobRun(GNode *targ) +JobRun(GNode *target) { /* Don't let these special jobs overlap with other unrelated jobs. */ - Compat_Make(targ, targ); - if (GNode_IsError(targ)) { - PrintOnError(targ, "\n\nStop.\n"); + Compat_Make(target, target); + if (GNode_IsError(target)) { + PrintOnError(target, "\n\nStop.\n"); exit(1); } } @@ -1959,7 +1899,8 @@ Job_CatchChildren(void) caught_sigchld = 0; while ((pid = waitpid((pid_t)-1, &status, WNOHANG | WUNTRACED)) > 0) { - DEBUG2(JOB, "Process %d exited/stopped status %x.\n", + DEBUG2(JOB, + "Process with pid %d exited/stopped with status %#x.\n", pid, WAIT_STATUS(status)); JobReapChild(pid, status, true); } @@ -1980,14 +1921,14 @@ JobReapChild(pid_t pid, WAIT_T status, bool isJobs) job = JobFindPid(pid, JOB_ST_RUNNING, isJobs); if (job == NULL) { if (isJobs && !lurking_children) - Error("Child (%d) status %x not in table?", + Error("Child with pid %d and status %#x not in table?", pid, status); return; } if (WIFSTOPPED(status)) { - DEBUG2(JOB, "Process %d (%s) stopped.\n", - job->pid, job->node->name); + DEBUG2(JOB, "Process for target %s, pid %d stopped\n", + job->node->name, job->pid); if (!make_suspended) { switch (WSTOPSIG(status)) { case SIGTSTP: @@ -2016,19 +1957,47 @@ JobReapChild(pid_t pid, WAIT_T status, bool isJobs) JobFinish(job, status); } +static void +Job_Continue(Job *job) +{ + DEBUG1(JOB, "Continuing pid %d\n", job->pid); + if (job->suspended) { + (void)printf("*** [%s] Continued\n", job->node->name); + (void)fflush(stdout); + job->suspended = false; + } + if (KILLPG(job->pid, SIGCONT) != 0) + DEBUG1(JOB, "Failed to send SIGCONT to pid %d\n", job->pid); +} + +static void +ContinueJobs(void) +{ + Job *job; + + for (job = job_table; job < job_table_end; job++) { + if (job->status == JOB_ST_RUNNING && + (make_suspended || job->suspended)) + Job_Continue(job); + else if (job->status == JOB_ST_FINISHED) + JobFinish(job, job->exit_status); + } + make_suspended = false; +} + void Job_CatchOutput(void) { int nready; Job *job; - unsigned int i; + unsigned i; (void)fflush(stdout); do { /* Maybe skip the job token pipe. */ nfds_t skip = wantToken ? 0 : 1; - nready = poll(fds + skip, fdsLen - skip, POLL_MSEC); + nready = poll(fds + skip, fdsLen - skip, -1); } while (nready < 0 && errno == EINTR); if (nready < 0) @@ -2037,17 +2006,11 @@ Job_CatchOutput(void) if (nready > 0 && childExitJob.inPollfd->revents & POLLIN) { char token; ssize_t count = read(childExitJob.inPipe, &token, 1); - if (count == 1) { - if (token == DO_JOB_RESUME[0]) - /* - * Complete relay requested from our SIGCONT - * handler. - */ - JobRestartJobs(); - } else if (count == 0) - Punt("unexpected eof on token pipe"); - else if (errno != EAGAIN) - Punt("token pipe read: %s", strerror(errno)); + if (count != 1) + Punt("childExitJob.read: %s", + count == 0 ? "EOF" : strerror(errno)); + if (token == CEJ_RESUME_JOBS) + ContinueJobs(); nready--; } @@ -2196,11 +2159,11 @@ Job_Init(void) } /* These are permanent entries and take slots 0 and 1 */ - watchfd(&tokenWaitJob); + watchfd(&tokenPoolJob); watchfd(&childExitJob); sigemptyset(&caught_signals); - (void)bmake_signal(SIGCHLD, JobChildSig); + (void)bmake_signal(SIGCHLD, HandleSIGCHLD); sigaddset(&caught_signals, SIGCHLD); /* Handle the signals specified by POSIX. */ @@ -2218,7 +2181,7 @@ Job_Init(void) AddSig(SIGTTOU, JobPassSig_suspend); AddSig(SIGTTIN, JobPassSig_suspend); AddSig(SIGWINCH, JobCondPassSig); - AddSig(SIGCONT, JobContinueSig); + AddSig(SIGCONT, HandleSIGCONT); (void)Job_RunTarget(".BEGIN", NULL); /* Create the .END node, see Targ_GetEndNode in Compat_MakeAll. */ @@ -2452,7 +2415,7 @@ JobInterrupt(bool runINTERRUPT, int signo) aborting = ABORT_INTERRUPT; - JobSigLock(&mask); + JobsTable_Lock(&mask); for (job = job_table; job < job_table_end; job++) { if (job->status == JOB_ST_RUNNING && job->pid != 0) { @@ -2471,7 +2434,7 @@ JobInterrupt(bool runINTERRUPT, int signo) } } - JobSigUnlock(&mask); + JobsTable_Unlock(&mask); if (runINTERRUPT && !opts.touch) { interrupt = Targ_FindNode(".INTERRUPT"); @@ -2512,9 +2475,8 @@ void Job_Wait(void) { aborting = ABORT_WAIT; /* Prevent other jobs from starting. */ - while (jobTokensRunning != 0) { + while (jobTokensRunning != 0) Job_CatchOutput(); - } aborting = ABORT_NONE; } @@ -2544,42 +2506,6 @@ Job_AbortAll(void) continue; } -/* - * Tries to restart stopped jobs if there are slots available. - * Called in response to a SIGCONT. - */ -static void -JobRestartJobs(void) -{ - Job *job; - - for (job = job_table; job < job_table_end; job++) { - if (job->status == JOB_ST_RUNNING && - (make_suspended || job->suspended)) { - DEBUG1(JOB, "Restarting stopped job pid %d.\n", - job->pid); - if (job->suspended) { - (void)printf("*** [%s] Continued\n", - job->node->name); - (void)fflush(stdout); - } - job->suspended = false; - if (KILLPG(job->pid, SIGCONT) != 0 && DEBUG(JOB)) { - debug_printf("Failed to send SIGCONT to %d\n", - job->pid); - } - } - if (job->status == JOB_ST_FINISHED) { - /* - * Job exit deferred after calling waitpid() in a - * signal handler - */ - JobFinish(job, job->exit_status); - } - } - make_suspended = false; -} - static void watchfd(Job *job) { @@ -2632,59 +2558,67 @@ clearfd(Job *job) job->inPollfd = NULL; } +int +Job_TempFile(const char *pattern, char *tfile, size_t tfile_sz) +{ + int fd; + sigset_t mask; + + JobsTable_Lock(&mask); + fd = mkTempFile(pattern, tfile, tfile_sz); + if (tfile != NULL && !DEBUG(SCRIPT)) + unlink(tfile); + JobsTable_Unlock(&mask); + + return fd; +} + +static void +TokenPool_Write(char tok) +{ + if (write(tokenPoolJob.outPipe, &tok, 1) != 1) + Punt("Cannot write \"%c\" to the token pool: %s", + tok, strerror(errno)); +} + /* - * Put a token (back) into the job pipe. + * Put a token (back) into the job token pool. * This allows a make process to start a build job. */ static void -JobTokenAdd(void) +TokenPool_Add(void) { char tok = JOB_TOKENS[aborting], tok1; /* If we are depositing an error token, flush everything else. */ - while (tok != '+' && read(tokenWaitJob.inPipe, &tok1, 1) == 1) + while (tok != '+' && read(tokenPoolJob.inPipe, &tok1, 1) == 1) continue; - DEBUG3(JOB, "(%d) aborting %d, deposit token %c\n", - getpid(), aborting, tok); - while (write(tokenWaitJob.outPipe, &tok, 1) == -1 && errno == EAGAIN) - continue; + DEBUG3(JOB, "TokenPool_Add: pid %d, aborting %s, token %c\n", + getpid(), aborting_name[aborting], tok); + TokenPool_Write(tok); } -int -Job_TempFile(const char *pattern, char *tfile, size_t tfile_sz) +static void +TokenPool_InitClient(int tokenPoolReader, int tokenPoolWriter) { - int fd; - sigset_t mask; - - JobSigLock(&mask); - fd = mkTempFile(pattern, tfile, tfile_sz); - if (tfile != NULL && !DEBUG(SCRIPT)) - unlink(tfile); - JobSigUnlock(&mask); - - return fd; + tokenPoolJob.inPipe = tokenPoolReader; + tokenPoolJob.outPipe = tokenPoolWriter; + (void)fcntl(tokenPoolReader, F_SETFD, FD_CLOEXEC); + (void)fcntl(tokenPoolWriter, F_SETFD, FD_CLOEXEC); } /* Prepare the job token pipe in the root make process. */ -void -Job_ServerStart(int max_tokens, int jp_0, int jp_1) +static void +TokenPool_InitServer(int maxJobTokens) { int i; char jobarg[64]; - if (jp_0 >= 0 && jp_1 >= 0) { - tokenWaitJob.inPipe = jp_0; - tokenWaitJob.outPipe = jp_1; - (void)fcntl(jp_0, F_SETFD, FD_CLOEXEC); - (void)fcntl(jp_1, F_SETFD, FD_CLOEXEC); - return; - } - - JobCreatePipe(&tokenWaitJob, 15); + JobCreatePipe(&tokenPoolJob, 15); snprintf(jobarg, sizeof jobarg, "%d,%d", - tokenWaitJob.inPipe, tokenWaitJob.outPipe); + tokenPoolJob.inPipe, tokenPoolJob.outPipe); Global_Append(MAKEFLAGS, "-J"); Global_Append(MAKEFLAGS, jobarg); @@ -2692,68 +2626,74 @@ Job_ServerStart(int max_tokens, int jp_0, int jp_1) /* * Preload the job pipe with one token per job, save the one * "extra" token for the primary job. - * - * XXX should clip maxJobs against PIPE_BUF -- if max_tokens is - * larger than the write buffer size of the pipe, we will - * deadlock here. */ - for (i = 1; i < max_tokens; i++) - JobTokenAdd(); + SetNonblocking(tokenPoolJob.outPipe); + for (i = 1; i < maxJobTokens; i++) + TokenPool_Add(); } -/* Return a withdrawn token to the pool. */ void -Job_TokenReturn(void) +TokenPool_Init(int maxJobTokens, int tokenPoolReader, int tokenPoolWriter) +{ + if (tokenPoolReader >= 0 && tokenPoolWriter >= 0) + TokenPool_InitClient(tokenPoolReader, tokenPoolWriter); + else + TokenPool_InitServer(maxJobTokens); +} + +/* Return a taken token to the pool. */ +void +TokenPool_Return(void) { jobTokensRunning--; if (jobTokensRunning < 0) Punt("token botch"); if (jobTokensRunning != 0 || JOB_TOKENS[aborting] != '+') - JobTokenAdd(); + TokenPool_Add(); } /* - * Attempt to withdraw a token from the pool. + * Attempt to take a token from the pool. * * If the pool is empty, set wantToken so that we wake up when a token is * released. * - * Returns true if a token was withdrawn, and false if the pool is currently + * Returns true if a token was taken, and false if the pool is currently * empty. */ bool -Job_TokenWithdraw(void) +TokenPool_Take(void) { char tok, tok1; ssize_t count; wantToken = false; - DEBUG3(JOB, "Job_TokenWithdraw(%d): aborting %d, running %d\n", - getpid(), aborting, jobTokensRunning); + DEBUG3(JOB, "TokenPool_Take: pid %d, aborting %s, running %d\n", + getpid(), aborting_name[aborting], jobTokensRunning); - if (aborting != ABORT_NONE || (jobTokensRunning >= opts.maxJobs)) + if (aborting != ABORT_NONE || jobTokensRunning >= opts.maxJobs) return false; - count = read(tokenWaitJob.inPipe, &tok, 1); + count = read(tokenPoolJob.inPipe, &tok, 1); if (count == 0) - Fatal("eof on job pipe!"); + Fatal("eof on job pipe"); if (count < 0 && jobTokensRunning != 0) { if (errno != EAGAIN) Fatal("job pipe read: %s", strerror(errno)); - DEBUG1(JOB, "(%d) blocked for token\n", getpid()); + DEBUG1(JOB, "TokenPool_Take: pid %d blocked for token\n", + getpid()); wantToken = true; return false; } if (count == 1 && tok != '+') { /* make being aborted - remove any other job tokens */ - DEBUG2(JOB, "(%d) aborted by token %c\n", getpid(), tok); - while (read(tokenWaitJob.inPipe, &tok1, 1) == 1) + DEBUG2(JOB, "TokenPool_Take: pid %d aborted by token %c\n", + getpid(), tok); + while (read(tokenPoolJob.inPipe, &tok1, 1) == 1) continue; /* And put the stopper back */ - while (write(tokenWaitJob.outPipe, &tok, 1) == -1 && - errno == EAGAIN) - continue; + TokenPool_Write(tok); if (shouldDieQuietly(NULL, 1)) { Job_Wait(); exit(6); @@ -2764,12 +2704,10 @@ Job_TokenWithdraw(void) if (count == 1 && jobTokensRunning == 0) /* We didn't want the token really */ - while (write(tokenWaitJob.outPipe, &tok, 1) == -1 && - errno == EAGAIN) - continue; + TokenPool_Write(tok); jobTokensRunning++; - DEBUG1(JOB, "(%d) withdrew token\n", getpid()); + DEBUG1(JOB, "TokenPool_Take: pid %d took a token\n", getpid()); return true; } @@ -1,4 +1,4 @@ -/* $NetBSD: job.h,v 1.81 2025/01/03 04:51:42 rillig Exp $ */ +/* $NetBSD: job.h,v 1.84 2025/04/22 19:28:50 rillig Exp $ */ /* * Copyright (c) 1988, 1989, 1990 The Regents of the University of California. @@ -78,106 +78,7 @@ #ifndef MAKE_JOB_H #define MAKE_JOB_H -#define TMPPAT "makeXXXXXX" /* relative to tmpdir */ - -#ifdef USE_SELECT -/* - * Emulate poll() in terms of select(). This is not a complete - * emulation but it is sufficient for make's purposes. - */ - -#define poll emul_poll -#define pollfd emul_pollfd - -struct emul_pollfd { - int fd; - short events; - short revents; -}; - -#define POLLIN 0x0001 -#define POLLOUT 0x0004 - -int emul_poll(struct pollfd *, int, int); -#endif - -/* - * The POLL_MSEC constant determines the maximum number of milliseconds spent - * in poll before coming out to see if a child has finished. - */ -#define POLL_MSEC 5000 - -struct pollfd; - - -#ifdef USE_META -# include "meta.h" -#endif - -typedef enum JobStatus { - JOB_ST_FREE = 0, /* Job is available */ - JOB_ST_SET_UP = 1, /* Job is allocated but otherwise invalid */ - /* XXX: What about the 2? */ - JOB_ST_RUNNING = 3, /* Job is running, pid valid */ - JOB_ST_FINISHED = 4 /* Job is done (ie after SIGCHLD) */ -} JobStatus; - -/* - * A Job manages the shell commands that are run to create a single target. - * Each job is run in a separate subprocess by a shell. Several jobs can run - * in parallel. - * - * The shell commands for the target are written to a temporary file, - * then the shell is run with the temporary file as stdin, and the output - * of that shell is captured via a pipe. - * - * When a job is finished, Make_Update updates all parents of the node - * that was just remade, marking them as ready to be made next if all - * other dependencies are finished as well. - */ -typedef struct Job { - /* The process ID of the shell running the commands */ - int pid; - - /* The target the child is making */ - GNode *node; - - /* - * If one of the shell commands is "...", all following commands are - * delayed until the .END node is made. This list node points to the - * first of these commands, if any. - */ - StringListNode *tailCmds; - - /* This is where the shell commands go. */ - FILE *cmdFILE; - - int exit_status; /* from wait4() in signal handler */ - - JobStatus status; - - bool suspended; - - /* Ignore non-zero exits */ - bool ignerr; - /* Output the command before or instead of running it. */ - bool echo; - /* Target is a special one. */ - bool special; - - int inPipe; /* Pipe for reading output from job */ - int outPipe; /* Pipe for writing control commands */ - struct pollfd *inPollfd; /* pollfd associated with inPipe */ - -#define JOB_BUFSIZE 1024 - /* Buffer for storing the output of the job, line by line. */ - char outBuf[JOB_BUFSIZE + 1]; - size_t curPos; /* Current position in outBuf. */ - -#ifdef USE_META - struct BuildMon bm; -#endif -} Job; +typedef struct Job Job; extern char *shellPath; extern const char *shellName; @@ -187,6 +88,11 @@ extern int jobTokensRunning; /* tokens currently "out" */ void Shell_Init(void); const char *Shell_GetNewline(void) MAKE_ATTR_USE; + +void TokenPool_Init(int, int, int); +bool TokenPool_Take(void) MAKE_ATTR_USE; +void TokenPool_Return(void); + void Job_Touch(GNode *, bool); bool Job_CheckCommands(GNode *, void (*abortProc)(const char *, ...)) MAKE_ATTR_USE; @@ -201,12 +107,14 @@ void Job_End(void); #endif void Job_Wait(void); void Job_AbortAll(void); -void Job_TokenReturn(void); -bool Job_TokenWithdraw(void) MAKE_ATTR_USE; -void Job_ServerStart(int, int, int); void Job_SetPrefix(void); bool Job_RunTarget(const char *, const char *); void Job_FlagsToString(const Job *, char *, size_t); int Job_TempFile(const char *, char *, size_t) MAKE_ATTR_USE; +#ifdef USE_META +struct BuildMon *Job_BuildMon(Job *) MAKE_ATTR_USE; +#endif +GNode *Job_Node(Job *) MAKE_ATTR_USE; +int Job_Pid(Job *) MAKE_ATTR_USE; #endif @@ -1,4 +1,4 @@ -/* $NetBSD: main.c,v 1.641 2025/03/31 14:35:22 riastradh Exp $ */ +/* $NetBSD: main.c,v 1.659 2025/06/13 05:41:36 rillig Exp $ */ /* * Copyright (c) 1988, 1989, 1990, 1993 @@ -104,11 +104,14 @@ #include "make.h" #include "dir.h" #include "job.h" +#ifdef USE_META +# include "meta.h" +#endif #include "pathnames.h" #include "trace.h" /* "@(#)main.c 8.3 (Berkeley) 3/19/94" */ -MAKE_RCSID("$NetBSD: main.c,v 1.641 2025/03/31 14:35:22 riastradh Exp $"); +MAKE_RCSID("$NetBSD: main.c,v 1.659 2025/06/13 05:41:36 rillig Exp $"); #if defined(MAKE_NATIVE) __COPYRIGHT("@(#) Copyright (c) 1988, 1989, 1990, 1993 " "The Regents of the University of California. " @@ -127,8 +130,9 @@ bool deleteOnError; /* .DELETE_ON_ERROR: set */ static int maxJobTokens; /* -j argument */ static bool enterFlagObj; /* -w and objdir != srcdir */ +static bool bogusJflag; /* -J invalid */ -static int jp_0 = -1, jp_1 = -1; /* ends of parent job pipe */ +static int tokenPoolReader = -1, tokenPoolWriter = -1; bool doing_depend; /* Set while reading .depend */ static bool jobsRunning; /* true if the jobs might be running */ static const char *tracefile; @@ -361,7 +365,8 @@ MainParseArgChdir(const char *argvalue) exit(2); /* Not 1 so -q can distinguish error */ } if (getcwd(curdir, MAXPATHLEN) == NULL) { - (void)fprintf(stderr, "%s: %s.\n", progname, strerror(errno)); + (void)fprintf(stderr, "%s: getcwd: %s\n", + progname, strerror(errno)); exit(2); } if (!IsRelativePath(argvalue) && @@ -377,17 +382,19 @@ static void MainParseArgJobsInternal(const char *argvalue) { char end; - if (sscanf(argvalue, "%d,%d%c", &jp_0, &jp_1, &end) != 2) { + if (sscanf(argvalue, "%d,%d%c", + &tokenPoolReader, &tokenPoolWriter, &end) != 2) { (void)fprintf(stderr, - "%s: internal error -- J option malformed (%s)\n", - progname, argvalue); - usage(); + "%s: error: invalid internal option " + "\"-J %s\" in \"%s\"\n", + progname, argvalue, curdir); + exit(2); } - if ((fcntl(jp_0, F_GETFD, 0) < 0) || - (fcntl(jp_1, F_GETFD, 0) < 0)) { - jp_0 = -1; - jp_1 = -1; - opts.compatMake = true; + if ((fcntl(tokenPoolReader, F_GETFD, 0) < 0) || + (fcntl(tokenPoolWriter, F_GETFD, 0) < 0)) { + tokenPoolReader = -1; + tokenPoolWriter = -1; + bogusJflag = true; } else { Global_Append(MAKEFLAGS, "-J"); Global_Append(MAKEFLAGS, argvalue); @@ -708,7 +715,9 @@ Main_ParseArgLine(const char *line) return; } free(buf); + EvalStack_PushMakeflags(line); MainParseArgs((int)words.len, words.words); + EvalStack_Pop(); Words_Free(words); } @@ -738,7 +747,7 @@ Main_SetObjdir(bool writable, const char *fmt, ...) return false; if ((writable && access(path, W_OK) != 0) || chdir(path) != 0) { - (void)fprintf(stderr, "%s: warning: %s: %s.\n", + (void)fprintf(stderr, "%s: warning: %s: %s\n", progname, path, strerror(errno)); /* Allow debugging how we got here - not always obvious */ if (GetBooleanExpr("${MAKE_DEBUG_OBJDIR_CHECK_WRITABLE}", @@ -828,7 +837,7 @@ MakeMode(void) } static void -PrintVar(const char *varname, bool expandVars) +PrintVariable(const char *varname, bool expandVars) { if (strchr(varname, '$') != NULL) { char *evalue = Var_Subst(varname, SCOPE_GLOBAL, VARE_EVAL); @@ -872,7 +881,7 @@ GetBooleanExpr(const char *expr, bool fallback) } static void -doPrintVars(void) +PrintVariables(void) { StringListNode *ln; bool expandVars; @@ -885,49 +894,33 @@ doPrintVars(void) expandVars = GetBooleanExpr("${.MAKE.EXPAND_VARIABLES}", false); - for (ln = opts.variables.first; ln != NULL; ln = ln->next) { - const char *varname = ln->datum; - PrintVar(varname, expandVars); - } + for (ln = opts.variables.first; ln != NULL; ln = ln->next) + PrintVariable(ln->datum, expandVars); } static bool -runTargets(void) +MakeTargets(void) { - GNodeList targs = LST_INIT; /* target nodes to create */ + GNodeList targets = LST_INIT; bool outOfDate; /* false if all targets up to date */ - /* - * Have now read the entire graph and need to make a list of - * targets to create. If none was given on the command line, - * we consult the parsing module to find the main target(s) - * to create. - */ if (Lst_IsEmpty(&opts.create)) - Parse_MainName(&targs); + Parse_MainName(&targets); else - Targ_FindList(&targs, &opts.create); + Targ_FindList(&targets, &opts.create); if (!opts.compatMake) { - /* - * Initialize job module before traversing the graph - * now that any .BEGIN and .END targets have been read. - * This is done only if the -q flag wasn't given - * (to prevent the .BEGIN from being executed should - * it exist). - */ if (!opts.query) { Job_Init(); jobsRunning = true; } - /* Traverse the graph, checking on all the targets */ - outOfDate = Make_Run(&targs); + outOfDate = Make_MakeParallel(&targets); } else { - Compat_MakeAll(&targs); + Compat_MakeAll(&targets); outOfDate = false; } - Lst_Done(&targs); /* Don't free the targets themselves. */ + Lst_Done(&targets); /* Don't free the targets themselves. */ return outOfDate; } @@ -958,7 +951,7 @@ InitRandom(void) struct timeval tv; gettimeofday(&tv, NULL); - srandom((unsigned int)(tv.tv_sec + tv.tv_usec)); + srandom((unsigned)(tv.tv_sec + tv.tv_usec)); } static const char * @@ -999,9 +992,9 @@ InitVarMachineArch(void) const int mib[2] = { CTL_HW, HW_MACHINE_ARCH }; size_t len = sizeof machine_arch_buf; - if (sysctl(mib, (unsigned int)__arraycount(mib), + if (sysctl(mib, (unsigned)__arraycount(mib), machine_arch_buf, &len, NULL, 0) < 0) { - (void)fprintf(stderr, "%s: sysctl failed (%s).\n", + (void)fprintf(stderr, "%s: sysctl: %s\n", progname, strerror(errno)); exit(2); } @@ -1225,6 +1218,30 @@ InitMaxJobs(void) char *value; int n; + if (bogusJflag && !opts.compatMake) { + opts.compatMake = true; + Parse_Error(PARSE_WARNING, + "internal option \"-J\" in \"%s\" " + "refers to unopened file descriptors; " + "falling back to compat mode.\n" + "\t" + "To run the target even in -n mode, " + "add the .MAKE pseudo-source to the target.\n" + "\t" + "To run the target in default mode only, " + "add a ${:D make} marker to a target's command. " + "(This marker expression expands to an empty string.)\n" + "\t" + "To make the sub-make run in compat mode, add -B to " + "its invocation.\n" + "\t" + "To make the sub-make independent from the parent make, " + "unset the MAKEFLAGS environment variable in the " + "target's commands.", + curdir); + PrintStackTrace(true); + return; + } if (forceJobs || opts.compatMake || !Var_Exists(SCOPE_GLOBAL, ".MAKE.JOBS")) return; @@ -1345,7 +1362,7 @@ main_Init(int argc, char **argv) UnlimitFiles(); if (uname(&utsname) == -1) { - (void)fprintf(stderr, "%s: uname failed (%s).\n", progname, + (void)fprintf(stderr, "%s: uname: %s\n", progname, strerror(errno)); exit(2); } @@ -1353,7 +1370,7 @@ main_Init(int argc, char **argv) machine = InitVarMachine(&utsname); machine_arch = InitVarMachineArch(); - myPid = getpid(); /* remember this for vFork() */ + myPid = getpid(); /* Just in case MAKEOBJDIR wants us to do something tricky. */ Targ_Init(); @@ -1439,25 +1456,25 @@ main_Init(int argc, char **argv) #endif Dir_Init(); + if (getcwd(curdir, MAXPATHLEN) == NULL) { + (void)fprintf(stderr, "%s: getcwd: %s\n", + progname, strerror(errno)); + exit(2); + } + { char *makeflags = explode(getenv("MAKEFLAGS")); Main_ParseArgLine(makeflags); free(makeflags); } - if (getcwd(curdir, MAXPATHLEN) == NULL) { - (void)fprintf(stderr, "%s: getcwd: %s.\n", - progname, strerror(errno)); - exit(2); - } - MainParseArgs(argc, argv); if (opts.enterFlag) printf("%s: Entering directory `%s'\n", progname, curdir); if (stat(curdir, &sa) == -1) { - (void)fprintf(stderr, "%s: %s: %s.\n", + (void)fprintf(stderr, "%s: stat %s: %s\n", progname, curdir, strerror(errno)); exit(2); } @@ -1538,9 +1555,10 @@ main_PrepareMaking(void) opts.compatMake = true; if (!opts.compatMake) - Job_ServerStart(maxJobTokens, jp_0, jp_1); + TokenPool_Init(maxJobTokens, tokenPoolReader, tokenPoolWriter); DEBUG5(JOB, "job_pipe %d %d, maxjobs %d, tokens %d, compat %d\n", - jp_0, jp_1, opts.maxJobs, maxJobTokens, opts.compatMake ? 1 : 0); + tokenPoolReader, tokenPoolWriter, opts.maxJobs, maxJobTokens, + opts.compatMake ? 1 : 0); if (opts.printVars == PVM_NONE) Main_ExportMAKEFLAGS(true); /* initial export */ @@ -1564,20 +1582,17 @@ main_PrepareMaking(void) } /* - * Make the targets. - * If the -v or -V options are given, print variables instead. + * Make the targets, or print variables. * Return whether any of the targets is out-of-date. */ static bool main_Run(void) { if (opts.printVars != PVM_NONE) { - /* print the values of any variables requested by the user */ - doPrintVars(); + PrintVariables(); return false; - } else { - return runTargets(); - } + } else + return MakeTargets(); } /* Clean up after making the targets. */ @@ -1620,9 +1635,8 @@ main_CleanUp(void) #endif } -/* Determine the exit code. */ static int -main_Exit(bool outOfDate) +main_ExitCode(bool outOfDate) { if ((opts.strict && main_errors > 0) || parseErrors > 0) return 2; /* Not 1 so -q can distinguish error */ @@ -1639,7 +1653,7 @@ main(int argc, char **argv) main_PrepareMaking(); outOfDate = main_Run(); main_CleanUp(); - return main_Exit(outOfDate); + return main_ExitCode(outOfDate); } /* @@ -1798,6 +1812,7 @@ Cmd_Exec(const char *cmd, char **error) } Var_ReexportVars(SCOPE_GLOBAL); + Var_ExportStackTrace(NULL, cmd); switch (cpid = FORK_FUNCTION()) { case 0: @@ -2002,7 +2017,7 @@ execDie(const char *func, const char *arg) char msg[1024]; int len; - len = snprintf(msg, sizeof(msg), "%s: %s(%s) failed (%s)\n", + len = snprintf(msg, sizeof(msg), "%s: %s(%s): %s\n", progname, func, arg, strerror(errno)); write_all(STDERR_FILENO, msg, (size_t)len); _exit(1); @@ -2218,7 +2233,7 @@ mkTempFile(const char *pattern, char *tfile, size_t tfile_sz) int fd; if (pattern == NULL) - pattern = TMPPAT; + pattern = "makeXXXXXX"; if (tmpdir == NULL) tmpdir = getTmpdir(); if (tfile == NULL) { @@ -2232,8 +2247,7 @@ mkTempFile(const char *pattern, char *tfile, size_t tfile_sz) snprintf(tfile, tfile_sz, "%s%s", tmpdir, pattern); if ((fd = mkstemp(tfile)) < 0) - Punt("Could not create temporary file %s: %s", tfile, - strerror(errno)); + Punt("mkstemp %s: %s", tfile, strerror(errno)); if (tfile == tbuf) unlink(tfile); /* we just want the descriptor */ @@ -1,4 +1,4 @@ -.\" $NetBSD: make.1,v 1.384 2025/04/04 18:36:47 sjg Exp $ +.\" $NetBSD: make.1,v 1.385 2025/06/13 03:51:18 rillig Exp $ .\" .\" Copyright (c) 1990, 1993 .\" The Regents of the University of California. All rights reserved. @@ -29,7 +29,7 @@ .\" .\" from: @(#)make.1 8.4 (Berkeley) 3/19/94 .\" -.Dd April 4, 2025 +.Dd June 12, 2025 .Dt MAKE 1 .Os .Sh NAME @@ -2688,6 +2688,7 @@ uses the following environment variables, if they exist: .Ev MAKEOBJDIR , .Ev MAKEOBJDIRPREFIX , .Ev MAKESYSPATH , +.Ev MAKE_STACK_TRACE , .Ev PWD , and .Ev TMPDIR . @@ -2707,6 +2708,12 @@ very early and the target is used to reset .Sq Va .OBJDIR , there may be unexpected side effects. +.Pp +If the +.Ev MAKE_STACK_TRACE +environment variable is set to +.Dq yes , +any stack traces include the call chain of the parent processes. .Sh FILES .Bl -tag -width /usr/share/mk -compact .It .depend @@ -1,4 +1,4 @@ -/* $NetBSD: make.c,v 1.264 2024/06/02 15:31:26 rillig Exp $ */ +/* $NetBSD: make.c,v 1.272 2025/05/18 07:02:00 rillig Exp $ */ /* * Copyright (c) 1988, 1989, 1990, 1993 @@ -72,8 +72,8 @@ * Examination of targets and their suitability for creation. * * Interface: - * Make_Run Initialize things for the module. Returns true if - * work was (or would have been) done. + * Make_MakeParallel + * Make the targets in parallel mode. * * Make_Update After a target is made, update all its parents. * Perform various bookkeeping chores like the updating @@ -102,12 +102,15 @@ #include "make.h" #include "dir.h" #include "job.h" +#ifdef USE_META +# include "meta.h" +#endif /* "@(#)make.c 8.1 (Berkeley) 6/6/93" */ -MAKE_RCSID("$NetBSD: make.c,v 1.264 2024/06/02 15:31:26 rillig Exp $"); +MAKE_RCSID("$NetBSD: make.c,v 1.272 2025/05/18 07:02:00 rillig Exp $"); /* Sequence # to detect recursion. */ -static unsigned int checked_seqno = 1; +static unsigned checked_seqno = 1; /* * The current fringe of the graph. @@ -127,7 +130,7 @@ debug_printf(const char *fmt, ...) va_end(ap); } -static char * +char * GNodeType_ToString(GNodeType type) { Buffer buf; @@ -250,7 +253,7 @@ IsOODateRegular(GNode *gn) /* * See if the node is out of date with respect to its sources. * - * Used by Make_Run when deciding which nodes to place on the + * Used by Make_MakeParallel when deciding which nodes to place on the * toBeMade queue initially and by Make_Update to screen out .USE and * .EXEC nodes. In the latter case, however, any other sort of node * must be considered out-of-date since at least one of its children @@ -392,9 +395,9 @@ PretendAllChildrenAreMade(GNode *pgn) } /* - * Called by Make_Run and SuffApplyTransform on the downward pass to handle - * .USE and transformation nodes, by copying the child node's commands, type - * flags and children to the parent node. + * Called by Make_MakeParallel and SuffApplyTransform on the downward pass to + * handle .USE and transformation nodes, by copying the child node's commands, + * type flags and children to the parent node. * * A .USE node is much like an explicit transformation rule, except its * commands are always added to the target node, even if the target already @@ -462,8 +465,8 @@ Make_HandleUse(GNode *cgn, GNode *pgn) } /* - * Used by Make_Run on the downward pass to handle .USE nodes. Should be - * called before the children are enqueued to be looked at by MakeAddChild. + * Used by Make_MakeParallel on the downward pass to handle .USE nodes. Should + * be called before the children are enqueued to be looked at by MakeAddChild. * * For a .USE child, the commands, type flags and children are copied to the * parent node, and since the relation to the .USE node is then no longer @@ -487,13 +490,6 @@ MakeHandleUse(GNode *cgn, GNode *pgn, GNodeListNode *ln) if (unmarked) Make_HandleUse(cgn, pgn); - /* - * This child node is now "made", so we decrement the count of - * unmade children in the parent... We also remove the child - * from the parent's list to accurately reflect the number of decent - * children the parent has. This is used by Make_Run to decide - * whether to queue the parent or examine its children... - */ Lst_Remove(&pgn->children, ln); pgn->unmade--; } @@ -1024,7 +1020,7 @@ MakeStartJobs(void) * Get token now to avoid cycling job-list when we only * have 1 token */ - if (!have_token && !Job_TokenWithdraw()) + if (!have_token && !TokenPool_Take()) break; have_token = true; @@ -1045,25 +1041,18 @@ MakeStartJobs(void) * We've already looked at this node since a job * finished... */ - DEBUG2(MAKE, "already checked %s%s\n", gn->name, - gn->cohort_num); + DEBUG2(MAKE, "already checked %s%s\n", + gn->name, gn->cohort_num); gn->made = DEFERRED; continue; } gn->checked_seqno = checked_seqno; if (gn->unmade != 0) { - /* - * We can't build this yet, add all unmade children - * to toBeMade, just before the current first element. - */ gn->made = DEFERRED; - MakeChildren(gn); - - /* and drop this node on the floor */ - DEBUG2(MAKE, "dropped %s%s\n", gn->name, - gn->cohort_num); + DEBUG2(MAKE, "deferred %s%s\n", + gn->name, gn->cohort_num); continue; } @@ -1093,7 +1082,7 @@ MakeStartJobs(void) } if (have_token) - Job_TokenReturn(); + TokenPool_Return(); return false; } @@ -1105,12 +1094,12 @@ MakePrintStatusOrderNode(GNode *ogn, GNode *gn) if (!GNode_IsWaitingFor(ogn)) return; - printf(" `%s%s' has .ORDER dependency against %s%s ", + printf(" `%s%s' has .ORDER dependency on %s%s ", gn->name, gn->cohort_num, ogn->name, ogn->cohort_num); GNode_FprintDetails(stdout, "(", ogn, ")\n"); if (DEBUG(MAKE) && opts.debug_file != stdout) { - debug_printf(" `%s%s' has .ORDER dependency against %s%s ", + debug_printf(" `%s%s' has .ORDER dependency on %s%s ", gn->name, gn->cohort_num, ogn->name, ogn->cohort_num); GNode_FprintDetails(opts.debug_file, "(", ogn, ")\n"); } @@ -1236,17 +1225,12 @@ ExamineLater(GNodeList *examine, GNodeList *toBeExamined) } } -/* - * Expand .USE nodes and create a new targets list. - * - * Input: - * targs the initial list of targets - */ +/* Expand .USE nodes and create a new targets list. */ void -Make_ExpandUse(GNodeList *targs) +Make_ExpandUse(GNodeList *targets) { GNodeList examine = LST_INIT; /* Queue of targets to examine */ - Lst_AppendAll(&examine, targs); + Lst_AppendAll(&examine, targets); /* * Make an initial downward pass over the graph, marking nodes to @@ -1256,15 +1240,15 @@ Make_ExpandUse(GNodeList *targs) * children for it if it has none and also has no commands. If the * node is a leaf, we stick it on the toBeMade queue to be looked * at in a minute, otherwise we add its children to our queue and - * go on about our business. + * go on. */ while (!Lst_IsEmpty(&examine)) { GNode *gn = Lst_Dequeue(&examine); if (gn->flags.remake) - /* We've looked at this one already */ continue; gn->flags.remake = true; + DEBUG2(MAKE, "Make_ExpandUse: examine %s%s\n", gn->name, gn->cohort_num); @@ -1318,31 +1302,26 @@ Make_ExpandUse(GNodeList *targs) /* Make the .WAIT node depend on the previous children */ static void -add_wait_dependency(GNodeListNode *owln, GNode *wn) +AddWaitDependency(GNodeListNode *prevWaitNode, GNode *waitNode) { - GNodeListNode *cln; - GNode *cn; - - for (cln = owln; (cn = cln->datum) != wn; cln = cln->next) { - DEBUG3(MAKE, ".WAIT: add dependency %s%s -> %s\n", - cn->name, cn->cohort_num, wn->name); + GNodeListNode *ln; - /* - * XXX: This pattern should be factored out, it repeats often - */ - Lst_Append(&wn->children, cn); - wn->unmade++; - Lst_Append(&cn->parents, wn); + for (ln = prevWaitNode; ln->datum != waitNode; ln = ln->next) { + GNode *gn = ln->datum; + DEBUG3(MAKE, ".WAIT: add dependency \"%s: %s%s\"\n", + waitNode->name, gn->name, gn->cohort_num); + Lst_Append(&waitNode->children, gn); + Lst_Append(&gn->parents, waitNode); + waitNode->unmade++; } } /* Convert .WAIT nodes into dependencies. */ static void -Make_ProcessWait(GNodeList *targs) +Make_ProcessWait(GNodeList *targets) { GNode *pgn; /* 'parent' node we are examining */ - GNodeListNode *owln; /* Previous .WAIT node */ - GNodeList examine; /* List of targets to examine */ + GNodeList examine; /* * We need all the nodes to have a common parent in order for the @@ -1358,7 +1337,7 @@ Make_ProcessWait(GNodeList *targs) { GNodeListNode *ln; - for (ln = targs->first; ln != NULL; ln = ln->next) { + for (ln = targets->first; ln != NULL; ln = ln->next) { GNode *cgn = ln->datum; Lst_Append(&pgn->children, cgn); @@ -1374,7 +1353,7 @@ Make_ProcessWait(GNodeList *targs) Lst_Append(&examine, pgn); while (!Lst_IsEmpty(&examine)) { - GNodeListNode *ln; + GNodeListNode *waitNode, *ln; pgn = Lst_Dequeue(&examine); @@ -1387,85 +1366,39 @@ Make_ProcessWait(GNodeList *targs) if (pgn->type & OP_DOUBLEDEP) Lst_PrependAll(&examine, &pgn->cohorts); - owln = pgn->children.first; + waitNode = pgn->children.first; for (ln = pgn->children.first; ln != NULL; ln = ln->next) { GNode *cgn = ln->datum; if (cgn->type & OP_WAIT) { - add_wait_dependency(owln, cgn); - owln = ln; - } else { + AddWaitDependency(waitNode, cgn); + waitNode = ln; + } else Lst_Append(&examine, cgn); - } } } Lst_Done(&examine); } -/* - * Initialize the nodes to remake and the list of nodes which are ready to - * be made by doing a breadth-first traversal of the graph starting from the - * nodes in the given list. Once this traversal is finished, all the 'leaves' - * of the graph are in the toBeMade queue. - * - * Using this queue and the Job module, work back up the graph, calling on - * MakeStartJobs to keep the job table as full as possible. - * - * Input: - * targs the initial list of targets - * - * Results: - * True if work was done, false otherwise. - * - * Side Effects: - * The make field of all nodes involved in the creation of the given - * targets is set to 1. The toBeMade list is set to contain all the - * 'leaves' of these subgraphs. - */ bool -Make_Run(GNodeList *targs) +Make_MakeParallel(GNodeList *targets) { int errors; /* Number of errors the Job module reports */ - /* Start trying to make the current targets... */ Lst_Init(&toBeMade); - Make_ExpandUse(targs); - Make_ProcessWait(targs); + Make_ExpandUse(targets); + Make_ProcessWait(targets); if (DEBUG(MAKE)) { debug_printf("#***# full graph\n"); Targ_PrintGraph(1); } - if (opts.query) { - /* - * We wouldn't do any work unless we could start some jobs - * in the next loop... (we won't actually start any, of - * course, this is just to see if any of the targets was out - * of date) - */ + if (opts.query) return MakeStartJobs(); - } - /* - * Initialization. At the moment, no jobs are running and until some - * get started, nothing will happen since the remaining upward - * traversal of the graph is performed by the routines in job.c upon - * the finishing of a job. So we fill the Job table as much as we can - * before going into our loop. - */ - (void)MakeStartJobs(); - /* - * Main Loop: The idea here is that the ending of jobs will take - * care of the maintenance of data structures and the waiting for - * output will cause us to be idle most of the time while our - * children run as much as possible. Because the job table is kept - * as full as possible, the only time when it will be empty is when - * all the jobs which need running have been run, so that is the end - * condition of this loop. Note that the Job module will exit if - * there were any errors unless the keepgoing flag was given. - */ + (void)MakeStartJobs(); while (!Lst_IsEmpty(&toBeMade) || jobTokensRunning > 0) { Job_CatchOutput(); (void)MakeStartJobs(); @@ -1473,13 +1406,9 @@ Make_Run(GNodeList *targs) errors = Job_Finish(); - /* - * Print the final status of each target. E.g. if it wasn't made - * because some inferior reported an error. - */ DEBUG1(MAKE, "done: errors %d\n", errors); if (errors == 0) { - MakePrintStatusList(targs, &errors); + MakePrintStatusList(targets, &errors); if (DEBUG(MAKE)) { debug_printf("done: errors %d\n", errors); if (errors > 0) @@ -1,4 +1,4 @@ -/* $NetBSD: make.h,v 1.352 2025/03/30 21:24:57 sjg Exp $ */ +/* $NetBSD: make.h,v 1.360 2025/06/13 18:31:08 rillig Exp $ */ /* * Copyright (c) 1988, 1989, 1990, 1993 @@ -517,7 +517,7 @@ typedef struct GNode { struct GNode *centurion; /* Last time (sequence number) we tried to make this node */ - unsigned int checked_seqno; + unsigned checked_seqno; /* * The "local" variables that are specific to this target and this @@ -840,7 +840,7 @@ void Compat_MakeAll(GNodeList *); void Compat_Make(GNode *, GNode *); /* cond.c */ -extern unsigned int cond_depth; +extern unsigned cond_depth; CondResult Cond_EvalCondition(const char *) MAKE_ATTR_USE; CondResult Cond_EvalLine(const char *) MAKE_ATTR_USE; Guard *Cond_ExtractGuard(const char *) MAKE_ATTR_USE; @@ -886,6 +886,7 @@ void JobReapChild(pid_t, int, bool); void Main_ParseArgLine(const char *); int Cmd_Argv(const char *, size_t, const char **, size_t, char *, size_t, bool, bool); char *Cmd_Exec(const char *, char **) MAKE_ATTR_USE; +void Var_ExportStackTrace(const char *, const char *); void Error(const char *, ...) MAKE_ATTR_PRINTFLIKE(1, 2); void Fatal(const char *, ...) MAKE_ATTR_PRINTFLIKE(1, 2) MAKE_ATTR_DEAD; void Punt(const char *, ...) MAKE_ATTR_PRINTFLIKE(1, 2) MAKE_ATTR_DEAD; @@ -905,6 +906,8 @@ void Parse_End(void); #endif void PrintLocation(FILE *, bool, const GNode *); +const char *GetParentStackTrace(void); +char *GetStackTrace(bool); void PrintStackTrace(bool); void Parse_Error(ParseErrorLevel, const char *, ...) MAKE_ATTR_PRINTFLIKE(2, 3); bool Parse_VarAssign(const char *, bool, GNode *) MAKE_ATTR_USE; @@ -912,7 +915,7 @@ void Parse_File(const char *, int); void Parse_PushInput(const char *, unsigned, unsigned, Buffer, struct ForLoop *); void Parse_MainName(GNodeList *); -unsigned int CurFile_CondMinDepth(void) MAKE_ATTR_USE; +unsigned CurFile_CondMinDepth(void) MAKE_ATTR_USE; void Parse_GuardElse(void); void Parse_GuardEndif(void); @@ -1079,7 +1082,9 @@ void Global_Append(const char *, const char *); void Global_Delete(const char *); void Global_Set_ReadOnly(const char *, const char *); -bool EvalStack_PrintDetails(void) MAKE_ATTR_USE; +void EvalStack_PushMakeflags(const char *); +void EvalStack_Pop(void); +bool EvalStack_Details(Buffer *buf) MAKE_ATTR_USE; /* util.c */ typedef void (*SignalProc)(int); @@ -1093,7 +1098,7 @@ time_t Make_Recheck(GNode *) MAKE_ATTR_USE; void Make_HandleUse(GNode *, GNode *); void Make_Update(GNode *); void GNode_SetLocalVars(GNode *); -bool Make_Run(GNodeList *); +bool Make_MakeParallel(GNodeList *); bool shouldDieQuietly(GNode *, int) MAKE_ATTR_USE; void PrintOnError(GNode *, const char *); void Main_ExportMAKEFLAGS(bool); @@ -1102,6 +1107,7 @@ int mkTempFile(const char *, char *, size_t) MAKE_ATTR_USE; void AppendWords(StringList *, char *); void GNode_FprintDetails(FILE *, const char *, const GNode *, const char *); bool GNode_ShouldExecute(GNode *gn) MAKE_ATTR_USE; +char *GNodeType_ToString(GNodeType); #ifndef HAVE_STRLCPY size_t strlcpy(char *, const char *, size_t); @@ -1208,6 +1214,8 @@ ch_isdigit(char ch) { return isdigit((unsigned char)ch) != 0; } MAKE_INLINE bool MAKE_ATTR_USE ch_islower(char ch) { return islower((unsigned char)ch) != 0; } MAKE_INLINE bool MAKE_ATTR_USE +ch_isprint(char ch) { return isprint((unsigned char)ch) != 0; } +MAKE_INLINE bool MAKE_ATTR_USE ch_isspace(char ch) { return isspace((unsigned char)ch) != 0; } MAKE_INLINE bool MAKE_ATTR_USE ch_isupper(char ch) { return isupper((unsigned char)ch) != 0; } diff --git a/make_malloc.c b/make_malloc.c index ea347e0ec2ca..d7a735b8b08e 100644 --- a/make_malloc.c +++ b/make_malloc.c @@ -1,4 +1,4 @@ -/* $NetBSD: make_malloc.c,v 1.26 2022/01/07 08:30:04 rillig Exp $ */ +/* $NetBSD: make_malloc.c,v 1.27 2025/06/12 18:51:05 rillig Exp $ */ /* * Copyright (c) 2009 The NetBSD Foundation, Inc. @@ -30,7 +30,7 @@ #include "make.h" -MAKE_RCSID("$NetBSD: make_malloc.c,v 1.26 2022/01/07 08:30:04 rillig Exp $"); +MAKE_RCSID("$NetBSD: make_malloc.c,v 1.27 2025/06/12 18:51:05 rillig Exp $"); #ifndef USE_EMALLOC @@ -38,7 +38,7 @@ MAKE_RCSID("$NetBSD: make_malloc.c,v 1.26 2022/01/07 08:30:04 rillig Exp $"); static MAKE_ATTR_DEAD void enomem(void) { - (void)fprintf(stderr, "%s: %s.\n", progname, strerror(ENOMEM)); + (void)fprintf(stderr, "%s: %s\n", progname, strerror(errno)); exit(2); } @@ -1,4 +1,4 @@ -/* $NetBSD: meta.c,v 1.211 2025/04/11 17:33:47 rillig Exp $ */ +/* $NetBSD: meta.c,v 1.215 2025/06/13 06:13:19 rillig Exp $ */ /* * Implement 'meta' mode. @@ -52,6 +52,7 @@ char * dirname(char *); #include "make.h" #include "dir.h" #include "job.h" +#include "meta.h" #ifdef USE_FILEMON #include "filemon/filemon.h" @@ -646,7 +647,7 @@ MAKE_INLINE BuildMon * BM(Job *job) { - return ((job != NULL) ? &job->bm : &Mybm); + return job != NULL ? Job_BuildMon(job) : &Mybm; } /* @@ -748,7 +749,7 @@ meta_job_error(Job *job, GNode *gn, bool ignerr, int status) pbm = BM(job); if (job != NULL && gn == NULL) - gn = job->node; + gn = Job_Node(job); if (pbm->mfp != NULL) { fprintf(pbm->mfp, "\n*** Error code %d%s\n", status, ignerr ? "(ignored)" : ""); @@ -756,7 +757,7 @@ meta_job_error(Job *job, GNode *gn, bool ignerr, int status) if (gn != NULL) Global_Set(".ERROR_TARGET", GNode_Path(gn)); if (getcwd(cwd, sizeof cwd) == NULL) - Punt("Cannot get cwd: %s", strerror(errno)); + Punt("getcwd: %s", strerror(errno)); Global_Set(".ERROR_CWD", cwd); if (pbm->meta_fname[0] != '\0') { @@ -766,7 +767,7 @@ meta_job_error(Job *job, GNode *gn, bool ignerr, int status) } void -meta_job_output(Job *job, char *cp, const char *nl) +meta_job_output(Job *job, const char *cp) { BuildMon *pbm; @@ -777,15 +778,10 @@ meta_job_output(Job *job, char *cp, const char *nl) static size_t meta_prefix_len; if (meta_prefix == NULL) { - char *cp2; - meta_prefix = Var_Subst("${" MAKE_META_PREFIX "}", SCOPE_GLOBAL, VARE_EVAL); /* TODO: handle errors */ - if ((cp2 = strchr(meta_prefix, '$')) != NULL) - meta_prefix_len = (size_t)(cp2 - meta_prefix); - else - meta_prefix_len = strlen(meta_prefix); + meta_prefix_len = strcspn(meta_prefix, "$"); } if (strncmp(cp, meta_prefix, meta_prefix_len) == 0) { cp = strchr(cp + 1, '\n'); @@ -794,7 +790,7 @@ meta_job_output(Job *job, char *cp, const char *nl) cp++; } } - fprintf(pbm->mfp, "%s%s", cp, nl); + fprintf(pbm->mfp, "%s", cp); } } @@ -1643,7 +1639,7 @@ meta_compat_start(void) } #endif if (pipe(childPipe) < 0) - Punt("Cannot create pipe: %s", strerror(errno)); + Punt("pipe: %s", strerror(errno)); /* Set close-on-exec flag for both */ (void)fcntl(childPipe[0], F_SETFD, FD_CLOEXEC); (void)fcntl(childPipe[1], F_SETFD, FD_CLOEXEC); @@ -1707,7 +1703,7 @@ meta_compat_parent(pid_t child) fwrite(buf, 1, (size_t)nread, stdout); fflush(stdout); buf[nread] = '\0'; - meta_job_output(NULL, buf, ""); + meta_job_output(NULL, buf); } while (false); if (metafd != -1 && FD_ISSET(metafd, &readfds) != 0) { if (meta_job_event(NULL) <= 0) @@ -1,4 +1,4 @@ -/* $NetBSD: meta.h,v 1.11 2021/12/15 09:53:41 rillig Exp $ */ +/* $NetBSD: meta.h,v 1.13 2025/06/13 06:13:19 rillig Exp $ */ /* * Things needed for 'meta' mode. @@ -49,7 +49,7 @@ void meta_job_parent(struct Job *, pid_t); int meta_job_fd(struct Job *) MAKE_ATTR_USE; int meta_job_event(struct Job *) MAKE_ATTR_USE; void meta_job_error(struct Job *, GNode *, bool, int); -void meta_job_output(struct Job *, char *, const char *); +void meta_job_output(struct Job *, const char *); int meta_cmd_finish(void *); int meta_job_finish(struct Job *); bool meta_oodate(GNode *, bool) MAKE_ATTR_USE; diff --git a/mk/ChangeLog b/mk/ChangeLog index c457d3aab9c2..1822f917a138 100644 --- a/mk/ChangeLog +++ b/mk/ChangeLog @@ -1,3 +1,43 @@ +2025-05-28 Simon J Gerraty <sjg@beast.crufty.net> + + * install-mk (MK_VERSION): 20250528 + + * add dirdeps2dplibs.mk + +2025-05-18 Simon J Gerraty <sjg@beast.crufty.net> + + * install-mk (MK_VERSION): 20250518 + + * meta.autodep.mk (META_FILES): re-work to fix filtering. + if OPTIMIZE_OBJECT_META_FILES==yes + provide a default META_FILE_OBJ_FILTER that selects a valid + .SUFFIX to match *o.meta, there's no guarantee that it will be as + simple as .o or .So etc. + We have to defer evaluation until the target script is run + for any of these filters to have any effect. + Use :S,${.OBJDIR}/,, rather than :T incase there are objects + in sub-dirs. + + * lib.mk: leverage ${.SUFFIXES} when setting dependencies. + + * add UPDATE_DEPENDFILE as a dependent option - follows + DIRDEPS_BUILD and use MK_UPDATE_DEPENDFILE as default for + UPDATE_DEPENDFILE when we think it should be yes. + This allows override with -DWITH[OUT]_UPDATE_DEPENDFILE + without overriding UPDATE_DEPENDFILE directly - which can lead to + trouble. + +2025-05-16 Simon J Gerraty <sjg@beast.crufty.net> + + * install-mk (MK_VERSION): 20250515 + + * meta2deps.py: resolve the target of a Move or Link first + and track the last path resolved, then if the src is a relative + path we can easily use that last path to resolve the src correctly. + + * meta2deps.sh: for a Move or Link add the dir of target path to + the list used to resolve the src path. + 2025-04-18 Simon J Gerraty <sjg@beast.crufty.net> * init.mk: include Skipping ${RELDIR} when _SKIP_BUILD is not empty. @@ -2,25 +2,32 @@ ChangeLog FILES LICENSE README +auto.dep.mk auto.obj.mk autoconf.mk autodep.mk -auto.dep.mk cc-wrap.mk ccm.dep.mk compiler.mk cython.mk dep.mk +dirdeps-cache-update.mk +dirdeps-options.mk +dirdeps-targets.mk +dirdeps.mk +dirdeps2dplibs.mk doc.mk dpadd.mk files.mk final.mk +gendirdeps.mk genfiles.mk host-target.mk host.libnames.mk inc.mk init.mk install-mk +install-new.mk install-sh java.mk jobs.mk @@ -31,6 +38,12 @@ libs.mk links.mk man.mk manifest.mk +meta.autodep.mk +meta.stage.mk +meta.subdir.mk +meta.sys.mk +meta2deps.py +meta2deps.sh mk-files.txt mkopt.sh newlog.sh @@ -50,11 +63,11 @@ srctop.mk stage-install.sh subdir.mk suffixes.mk -sys.mk sys.clean-env.mk sys.debug.mk sys.dependfile.mk sys.dirdeps.mk +sys.mk sys.vars.mk sys/AIX.mk sys/Cygwin.mk @@ -73,15 +86,3 @@ target-flags.mk warnings.mk whats.mk yacc.mk -dirdeps.mk -dirdeps-cache-update.mk -dirdeps-options.mk -dirdeps-targets.mk -gendirdeps.mk -install-new.mk -meta2deps.py -meta2deps.sh -meta.sys.mk -meta.autodep.mk -meta.stage.mk -meta.subdir.mk diff --git a/mk/auto.obj.mk b/mk/auto.obj.mk index 9ae7ebe3af56..4b8c5325b71f 100644 --- a/mk/auto.obj.mk +++ b/mk/auto.obj.mk @@ -1,8 +1,8 @@ # SPDX-License-Identifier: BSD-2-Clause # -# $Id: auto.obj.mk,v 1.19 2025/03/27 15:51:06 sjg Exp $ +# $Id: auto.obj.mk,v 1.20 2025/05/17 15:29:55 sjg Exp $ # -# @(#) Copyright (c) 2004, Simon J. Gerraty +# @(#) Copyright (c) 2004-2025, Simon J. Gerraty # # This file is provided in the hope that it will # be of use. There is absolutely NO WARRANTY. diff --git a/mk/dirdeps2dplibs.mk b/mk/dirdeps2dplibs.mk new file mode 100644 index 000000000000..cecf70be7477 --- /dev/null +++ b/mk/dirdeps2dplibs.mk @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: BSD-2-Clause +# +# $Id: dirdeps2dplibs.mk,v 1.3 2025/05/29 18:32:53 sjg Exp $ + +# DIRDEPS generally reflects things *actually used* by RELDIR. +# dirdeps2dplibs allows us to turn DIRDEPS into a DPLIBS list +# The order will however be sorted, so some +# manual tweaking may be needed. +# + +dirdeps2dplibs: + +.if ${.MAKE.LEVEL} > 0 +# for customization +.-include <local.dirdeps2dplibs.mk> + +_DEPENDFILE ?= ${.MAKE.DEPENDFILE} + +.dinclude "${_DEPENDFILE}" + +INCS_DIRS += h include incs +DIRDEPS2DPLIBS_FILTER += C;/(${INCS_DIRS:O:u:ts|})(\.common.*)*$$;; + +dirdeps2dplibs: + @echo +.if ${DEBUG_DIRDEPS2DPLIBS:Uno:@x@${RELDIR:M$x}@} != "" + @echo "# DIRDEPS=${DIRDEPS:M*lib*}" +.endif + @echo -n 'DPLIBS += \'; \ + echo '${DIRDEPS:M*lib*:${DIRDEPS2DPLIBS_FILTER:ts:}:T:O:u:tu:@d@${.newline}${.tab}_{LIB${d:S,^LIB,,}} \\@}' | \ + sed 's,_{,$${,g'; \ + echo + + +.endif diff --git a/mk/gendirdeps.mk b/mk/gendirdeps.mk index 53e736da3391..b52c9ca0eba3 100644 --- a/mk/gendirdeps.mk +++ b/mk/gendirdeps.mk @@ -1,8 +1,8 @@ -# $Id: gendirdeps.mk,v 1.51 2025/01/05 01:16:19 sjg Exp $ +# $Id: gendirdeps.mk,v 1.53 2025/05/20 17:42:49 sjg Exp $ # SPDX-License-Identifier: BSD-2-Clause # -# Copyright (c) 2011-2020, Simon J. Gerraty +# Copyright (c) 2011-2025, Simon J. Gerraty # Copyright (c) 2010-2018, Juniper Networks, Inc. # All rights reserved. # @@ -76,6 +76,12 @@ # .MAIN: all +.if ${DEBUG_GENDIRDEPS:Uno:@m@${RELDIR:M$m}@} != "" +_debug.gendirdeps = 1 +.else +_debug.gendirdeps = 0 +.endif + # keep this simple .MAKE.MODE = compat @@ -108,6 +114,9 @@ META_FILES += ${META_XTRAS:N\*.meta} .endif .if !empty(META_FILES) +.if ${_debug.gendirdeps} && ${DEBUG_GENDIRDEPS:Mmeta*} != "" +.info ${RELDIR}: META_FILES=${META_FILES} +.endif .if ${.MAKE.LEVEL} > 0 && !empty(GENDIRDEPS_FILTER) # so we can compare below @@ -146,7 +155,7 @@ GENDIRDEPS_FILTER += ${GENDIRDEPS_FILTER_VARS:@v@S,/${$v}/,/_{${v}}/,@:NS,//,*:u META2DEPS ?= ${.PARSEDIR}/meta2deps.sh META2DEPS := ${META2DEPS} -.if ${DEBUG_GENDIRDEPS:Uno:@x@${RELDIR:M$x}@} != "" && ${DEBUG_GENDIRDEPS:Uno:Mmeta2d*} != "" +.if ${_debug.gendirdeps} && ${DEBUG_GENDIRDEPS:Mmeta2d*} != "" _time = time _sh_x = sh -x _py_d = -ddd @@ -260,7 +269,7 @@ dpadd_dir_list += ${f:H:tA} ddeps != cat ${ddep_list:O:u} | ${META2DEPS_FILTER} ${_skip_gendirdeps} \ sed ${GENDIRDEPS_SEDCMDS} -.if ${DEBUG_GENDIRDEPS:Uno:@x@${RELDIR:M$x}@} != "" +.if ${_debug.gendirdeps} .info ${RELDIR}: raw_dir_list='${dir_list}' .info ${RELDIR}: ddeps='${ddeps}' .endif @@ -294,7 +303,7 @@ skip_ql= ${SRCTOP}* ${_objtops:@o@$o*@} # we need := so only skip_ql to this point applies ql.$o := ${dir_list:${skip_ql:${M_ListToSkip}}:M$o*/*/*:C,$o([^/]+)/(.*),\2.\1,:S,.${HOST_TARGET},.host,} qualdir_list += ${ql.$o} -.if ${DEBUG_GENDIRDEPS:Uno:@x@${RELDIR:M$x}@} != "" +.if ${_debug.gendirdeps} .info ${RELDIR}: o=$o ${ql.$o qualdir_list:L:@v@$v=${$v}@} .endif skip_ql+= $o* @@ -323,7 +332,7 @@ DIRDEPS += \ GENDIRDEPS_FILTER_MASK += @CMNS DIRDEPS := ${DIRDEPS:${GENDIRDEPS_FILTER:UNno:M[${GENDIRDEPS_FILTER_MASK:O:u:ts}]*:ts:}:C,//+,/,g:O:u} -.if ${DEBUG_GENDIRDEPS:Uno:@x@${RELDIR:M$x}@} != "" +.if ${_debug.gendirdeps} .info ${RELDIR}: M2D_OBJROOTS=${M2D_OBJROOTS} .info ${RELDIR}: M2D_EXCLUDES=${M2D_EXCLUDES} .info ${RELDIR}: dir_list='${dir_list}' diff --git a/mk/install-mk b/mk/install-mk index de5056a37042..8d354de17cb9 100755..100644 --- a/mk/install-mk +++ b/mk/install-mk @@ -59,7 +59,7 @@ # Simon J. Gerraty <sjg@crufty.net> # RCSid: -# $Id: install-mk,v 1.264 2025/03/26 18:30:18 sjg Exp $ +# $Id: install-mk,v 1.266 2025/05/29 01:48:06 sjg Exp $ # # @(#) Copyright (c) 1994-2025 Simon J. Gerraty # @@ -74,7 +74,7 @@ # sjg@crufty.net # -MK_VERSION=20250326 +MK_VERSION=20250528 OWNER= GROUP= MODE=444 diff --git a/mk/lib.mk b/mk/lib.mk index 3f6a749a12d1..708a2a1994cc 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -1,4 +1,4 @@ -# $Id: lib.mk,v 1.85 2024/12/12 19:56:36 sjg Exp $ +# $Id: lib.mk,v 1.86 2025/05/20 17:19:37 sjg Exp $ # should be set properly in sys.mk _this ?= ${.PARSEFILE:S,bsd.,,} @@ -396,7 +396,7 @@ realbuild: ${_LIBS} all: _SUBDIRUSE .for s in ${SRCS:${OBJS_SRCS_PRE_FILTER:ts:}:M*/*} -${.o .po .lo:L:@o@${s:${OBJS_SRCS_FILTER:ts:}}$o@}: $s +${.SUFFIXES:U.o .po .lo:M*o:@o@${s:${OBJS_SRCS_FILTER:ts:}}$o@}: $s .endfor OBJS_SRCS = ${SRCS:${OBJS_SRCS_FILTER:ts:}} diff --git a/mk/libs.mk b/mk/libs.mk index fef339c03bce..6814916657ec 100644 --- a/mk/libs.mk +++ b/mk/libs.mk @@ -1,6 +1,6 @@ # SPDX-License-Identifier: BSD-2-Clause # -# $Id: libs.mk,v 1.7 2024/02/17 17:26:57 sjg Exp $ +# $Id: libs.mk,v 1.8 2025/05/19 19:15:22 sjg Exp $ # # @(#) Copyright (c) 2006, Simon J. Gerraty # @@ -59,7 +59,7 @@ $v += ${${v}_${LIB}:U${${v}.${LIB}}} # for meta mode, there can be only one! .if ${LIB} == ${UPDATE_DEPENDFILE_LIB:Uno} -UPDATE_DEPENDFILE ?= yes +UPDATE_DEPENDFILE ?= ${MK_UPDATE_DEPENDFILE:Uyes} .endif UPDATE_DEPENDFILE ?= NO diff --git a/mk/meta.autodep.mk b/mk/meta.autodep.mk index b94891b1b93f..ce16ac843dc3 100644 --- a/mk/meta.autodep.mk +++ b/mk/meta.autodep.mk @@ -1,6 +1,6 @@ # SPDX-License-Identifier: BSD-2-Clause # -# $Id: meta.autodep.mk,v 1.65 2025/03/14 20:28:42 sjg Exp $ +# $Id: meta.autodep.mk,v 1.70 2025/05/28 20:03:00 sjg Exp $ # # @(#) Copyright (c) 2010-2025, Simon J. Gerraty @@ -22,6 +22,12 @@ __${_this}__: .NOTMAIN .-include <local.autodep.mk> +.if ${DEBUG_AUTODEP:Uno:@m@${RELDIR:M$m}@} != "" +_debug.autodep = 1 +.else +_debug.autodep = 0 +.endif + PICO?= .pico .if defined(SRCS) @@ -85,9 +91,9 @@ UPDATE_DEPENDFILE = NO _bootstrap_dirdeps = yes .endif _bootstrap_dirdeps ?= no -UPDATE_DEPENDFILE ?= yes +UPDATE_DEPENDFILE ?= ${MK_UPDATE_DEPENDFILE:Uyes} -.if ${DEBUG_AUTODEP:Uno:@m@${RELDIR:M$m}@} != "" +.if ${_debug.autodep} .info ${_DEPENDFILE:S,${SRCTOP}/,,} update=${UPDATE_DEPENDFILE} .endif @@ -111,7 +117,7 @@ WANT_UPDATE_DEPENDFILE ?= yes UPDATE_DEPENDFILE = no .endif -.if ${DEBUG_AUTODEP:Uno:@m@${RELDIR:M$m}@} != "" +.if ${_debug.autodep} .info ${_DEPENDFILE:S,${SRCTOP}/,,} update=${UPDATE_DEPENDFILE} .endif @@ -207,7 +213,7 @@ CAT_DEPEND = /dev/null _depend = .endif -.if ${DEBUG_AUTODEP:Uno:@m@${RELDIR:M$m}@} != "" +.if ${_debug.autodep} .info ${_DEPENDFILE:S,${SRCTOP}/,,} _depend=${_depend} .endif @@ -253,19 +259,33 @@ _gendirdeps_mutex = ${GENDIRDEPS_MUTEXER} ${GENDIRDEPS_MUTEX:U${_CURDIR}/Makefil # but we need to behave as if we did. # Avoid adding glob patterns to .MAKE.META.CREATED though. .MAKE.META.CREATED += ${META_XTRAS:N*\**:O:u} - -.if make(gendirdeps) -META_FILES = *.meta -.elif ${OPTIMIZE_OBJECT_META_FILES:Uno:tl} == "no" -META_FILES = ${.MAKE.META.FILES:T:N.depend*:O:u} +OPTIMIZE_OBJECT_META_FILES ?= no + +.if ${OPTIMIZE_OBJECT_META_FILES} == "yes" +# If we have lots of .o.meta, ${PICO}.meta etc we need only look at one set. +# If META_FILE_OBJ_FILTER is not already set, we default it to a +# .SUFFIX which matches the first *o.meta. +# There is no guarantee it will be just .o or .So etc, +META_FILE_OBJ_FILTER ?= \ + ${.SUFFIXES:M*o:@o@${"${.MAKE.META.FILES:T:M*$o.meta:[1]}":?M*$o.meta:}@:[1]} +.endif + +# parent may have set META_FILE_OBJ_FILTER +.if ${OPTIMIZE_OBJECT_META_FILES} == "yes" || !empty(META_FILE_OBJ_FILTER) +META_FILES = \ + ${.MAKE.META.FILES:N.depend*:N*o.meta} \ + ${.MAKE.META.FILES:${META_FILE_OBJ_FILTER}} .else -# if we have 1000's of .o.meta, ${PICO}.meta etc we need only look at one set -# it is left as an exercise for the reader to work out what this does -META_FILES = ${.MAKE.META.FILES:T:N.depend*:N*o.meta:O:u} \ - ${.MAKE.META.FILES:T:M*.${.MAKE.META.FILES:M*o.meta:R:E:O:u:[1]}.meta:O:u} +META_FILES = ${.MAKE.META.FILES:N.depend*} .endif +# ensure this is not empty (this will sort after any M and N +# we use S,${_OBJDIR}/,, rather than :T since some makefiles have +# objects in subdirs +META_FILE_FILTER += S,${_OBJDIR}/,,:O:u +# we have to defer evaluation until the target script runs +GENDIRDEPS_ENV += META_FILES="${META_FILES:${META_FILE_FILTER:O:u:ts:}}}" -.if ${DEBUG_AUTODEP:Uno:@m@${RELDIR:M$m}@} != "" +.if ${_debug.autodep} .info ${_DEPENDFILE:S,${SRCTOP}/,,}: ${_depend} ${.PARSEDIR}/gendirdeps.mk ${META2DEPS} xtras=${META_XTRAS} .endif @@ -276,9 +296,6 @@ META_FILES = ${.MAKE.META.FILES:T:N.depend*:N*o.meta:O:u} \ .if !empty(GENDIRDEPS_FILTER) .export GENDIRDEPS_FILTER .endif -# export to avoid blowing command line limit -META_FILES := ${META_XTRAS:U:O:u} ${META_FILES:U:T:O:u:${META_FILE_FILTER:ts:}} -.export META_FILES .endif _this_dir := ${_PARSEDIR} @@ -301,7 +318,7 @@ ${_DEPENDFILE}: ${_depend} ${.PARSEDIR}/gendirdeps.mk ${META2DEPS} $${.MAKE.MET @(cd . && ${GENDIRDEPS_ENV} \ SKIP_GENDIRDEPS='${SKIP_GENDIRDEPS:O:u}' \ DPADD='${FORCE_DPADD:O:u}' ${_gendirdeps_mutex} \ - ${.MAKE} -f gendirdeps.mk RELDIR=${RELDIR} _DEPENDFILE=${_DEPENDFILE}) + ${.MAKE} -B -f gendirdeps.mk RELDIR=${RELDIR} _DEPENDFILE=${_DEPENDFILE}) @test -s $@ && touch $@; : .endif diff --git a/mk/meta2deps.py b/mk/meta2deps.py index 44c752d0e7eb..70b121003988 100755 --- a/mk/meta2deps.py +++ b/mk/meta2deps.py @@ -39,9 +39,9 @@ We only pay attention to a subset of the information in the SPDX-License-Identifier: BSD-2-Clause RCSid: - $Id: meta2deps.py,v 1.50 2024/09/27 00:08:36 sjg Exp $ + $Id: meta2deps.py,v 1.51 2025/05/16 20:03:43 sjg Exp $ - Copyright (c) 2011-2020, Simon J. Gerraty + Copyright (c) 2011-2025, Simon J. Gerraty Copyright (c) 2011-2017, Juniper Networks, Inc. All rights reserved. @@ -543,8 +543,8 @@ class MetaFile: if w[0] in 'ML': # these are special, tread src as read and # target as write - self.parse_path(w[2].strip("'"), cwd, 'R', w) self.parse_path(w[3].strip("'"), cwd, 'W', w) + self.parse_path(w[2].strip("'"), cwd, 'R', w) continue elif w[0] in 'ERWS': path = w[2] @@ -611,9 +611,19 @@ class MetaFile: return # we don't want to resolve the last component if it is # a symlink - path = resolve(path, cwd, self.last_dir, self.debug, self.debug_out) - if not path: - return + npath = resolve(path, cwd, self.last_dir, self.debug, self.debug_out) + if not npath: + if len(w) > 3 and w[0] in 'ML' and op == 'R' and path.startswith('../'): + # we already resolved the target of the M/L + # so it makes sense to try and resolve relative to that dir. + if os.path.isdir(self.last_path): + dir = self.last_path + else: + dir,junk = os.path.split(self.last_path) + npath = resolve(path, cwd, dir, self.debug, self.debug_out) + if not npath: + return + path = npath dir,base = os.path.split(path) if dir in self.seen: if self.debug > 2: @@ -631,6 +641,7 @@ class MetaFile: rdir = None # now put path back together path = '/'.join([dir,base]) + self.last_path = path if self.debug > 1: print("raw=%s rdir=%s dir=%s path=%s" % (w[2], rdir, dir, path), file=self.debug_out) if op in 'RWS': diff --git a/mk/meta2deps.sh b/mk/meta2deps.sh index f031165e1d32..4af15971b84f 100755 --- a/mk/meta2deps.sh +++ b/mk/meta2deps.sh @@ -77,10 +77,11 @@ # RCSid: -# $Id: meta2deps.sh,v 1.21 2024/02/17 17:26:57 sjg Exp $ +# $Id: meta2deps.sh,v 1.22 2025/05/16 20:03:43 sjg Exp $ # SPDX-License-Identifier: BSD-2-Clause # +# Copyright (c) 2011-2025, Simon J. Gerraty # Copyright (c) 2010-2013, Juniper Networks, Inc. # All rights reserved. # @@ -252,9 +253,9 @@ meta2deps() { esac 2> /dev/null | sed -e 's,^CWD,C C,;/^[#CREFLMVX] /!d' -e "s,',,g" | $_excludes | ( version=no epids= xpids= eof_token=no - while read op pid path junk + while read op pid path path2 do - : op=$op pid=$pid path=$path + : op=$op pid=$pid path=$path path2=$path2 # we track cwd and ldir (of interest) per pid # CWD is bmake's cwd case "$lpid,$pid" in @@ -321,9 +322,14 @@ meta2deps() { $src_re|$obj_re) ;; /*/stage/*) ;; /*) continue;; - *) for path in $ldir/$path $cwd/$path + *) + rlist="$ldir/$path $cwd/$path" + case "$op,$path" in + [ML],../*) rlist="$rlist $path2/$path `dirname $path2`/$path";; + esac + for path in $rlist do - test -e $path && break + test -e $path && break done dir=${path%/*} ;; diff --git a/mk/mkopt.sh b/mk/mkopt.sh index 24320c257250..ec425440570b 100644 --- a/mk/mkopt.sh +++ b/mk/mkopt.sh @@ -2,9 +2,9 @@ # SPDX-License-Identifier: BSD-2-Clause # -# $Id: mkopt.sh,v 1.16 2024/02/17 17:26:57 sjg Exp $ +# $Id: mkopt.sh,v 1.17 2025/05/22 22:35:14 sjg Exp $ # -# @(#) Copyright (c) 2014-2022, Simon J. Gerraty +# @(#) Copyright (c) 2014-2025, Simon J. Gerraty # # This file is provided in the hope that it will # be of use. There is absolutely NO WARRANTY. @@ -89,18 +89,19 @@ _mk_opts_defaults() { # _mk_cmdline_opts opt ... # look at the command line (saved in _cmdline) # to see any options we care about are being set with -DWITH* -# or MK_*= if 'opt' is '*' then all options are of interest. +# or MK_*= and WITH[OUT]_*= if 'opt' is '*' then all options are of interest. _cmdline="$0 $@" _mk_cmdline_opts() { for _x in $_cmdline do case "$_x" in - -DWITH*|${_MKOPT_PREFIX:-MK_}*) + -DWITH*|WITH*=*|${_MKOPT_PREFIX:-MK_}*=*) for _o in "$@" do case "$_x" in -DWITH_$_o|-DWITHOUT_$_o) eval ${_x#-D}=1;; -DWITH_$_o=*|-DWITHOUT_$_o=*) eval ${_x#-D};; + WITH_$_o=*|WITHOUT_$_o=*) eval "$_x";; ${_MKOPT_PREFIX:-MK_}$_o=*) eval "$_x";; esac done diff --git a/mk/newlog.sh b/mk/newlog.sh index 5c65e5dd5fb6..fbf347ee2746 100755 --- a/mk/newlog.sh +++ b/mk/newlog.sh @@ -15,11 +15,12 @@ # # -C "compress" # Compact old logs (other than .0) with "compress" -# (default is 'gzip' or 'compress' if no 'gzip'). +# (default is "$NEWLOG_COMPRESS" 'gzip' or 'compress' if +# no 'gzip'). # # -E "ext" # If "compress" produces a file extention other than -# '.Z' or '.gz' we need to know. +# '.Z' or '.gz' we need to know ("$NEWLOG_EXT"). # # -G "gens" # "gens" is a comma separated list of "log":"num" pairs @@ -39,6 +40,7 @@ # uniquely name it when using the '-S' option. # If a "log" is saved more than once per second we add # an extra suffix of our process-id. +# The default can be set in the env via "$NEWLOG_FMT". # # -d The "log" to be rotated/saved is a directory. # We leave the mode of old directories alone. @@ -50,21 +52,24 @@ # Set the group of "log" to "group". # # -m "mode" -# Set the mode of "log". +# Set the mode of "log" ("$NEWLOG_MODE"). # # -M "mode" -# Set the mode of old logs (default 444). +# Set the mode of old logs (default "$NEWLOG_OLD_MODE" +# or 444). # # -n "num" -# Keep "num" generations of "log". +# Keep "num" generations of "log" ("$NEWLOG_NUM"). # # -o "owner" # Set the owner of "log". # -# Regardless of whether '-R' or '-S' is provided, we attempt to -# choose the correct behavior based on observation of "log.0" if -# it exists; if it is a symbolic link, we save, otherwise -# we rotate. +# The default method for dealing with logs can be set via +# "$NEWLOG_METHOD" ('save' or 'rotate'). +# Regardless of "$NEWLOG_METHOD" or whether '-R' or '-S' is +# provided, we attempt to choose the correct behavior based on +# observation of "log.0" if it exists; if it is a symbolic link, +# we 'save', otherwise we 'rotate'. # # BUGS: # 'Newlog.sh' tries to avoid being fooled by symbolic links, but @@ -76,11 +81,11 @@ # # RCSid: -# $Id: newlog.sh,v 1.27 2024/02/17 17:26:57 sjg Exp $ +# $Id: newlog.sh,v 1.30 2025/06/01 05:07:48 sjg Exp $ # # SPDX-License-Identifier: BSD-2-Clause # -# @(#) Copyright (c) 1993-2016 Simon J. Gerraty +# @(#) Copyright (c) 1993-2025 Simon J. Gerraty # # This file is provided in the hope that it will # be of use. There is absolutely NO WARRANTY. @@ -168,6 +173,10 @@ get_mode() { $STAT -f %Op $1 | sed 's,.*\(....\),\1,' return ;; + Linux,$STAT) # works on Ubuntu + $STAT -c %a $1 2> /dev/null && + return + ;; esac # fallback to find fmode `find $1 -ls -prune | awk '{ print $3 }'` @@ -179,21 +188,33 @@ get_mtime_suffix() { $STAT -t "${2:-$opt_f}" -f %Sm $1 return ;; + Linux,*) # works on Ubuntu + mtime=`$STAT --format=%Y $1 2> /dev/null` + if [ ${mtime:-0} -gt 1 ]; then + date --date=@$mtime "+${2:-$opt_f}" 2> /dev/null && + return + fi + ;; esac # this will have to do date "+${2:-$opt_f}" } case /$0 in -*/newlog*) rotate_func=rotate_log;; +*/newlog*) rotate_func=${NEWLOG_METHOD:-rotate_log};; */save*) rotate_func=save_log;; -*) rotate_func=rotate_log;; +*) rotate_func=${NEWLOG_METHOD:-rotate_log};; +esac +case "$rotate_func" in +save|rotate) rotate_func=${rotate_func}_log;; esac -opt_n=7 -opt_m= -opt_M=444 -opt_f=%Y%m%d.%H%M%S +opt_C=${NEWLOG_COMPRESS} +opt_E=${NEWLOG_EXT} +opt_n=${NEWLOG_NUM:-7} +opt_m=${NEWLOG_MODE} +opt_M=${NEWLOG_OLD_MODE:-444} +opt_f=${NEWLOG_FMT:-%Y-%m-%dT%T} # rfc3339 opt_str=dNn:o:g:G:C:M:m:eE:f:RS . setopts.sh @@ -241,7 +262,7 @@ case "${opt_R:-0}" in esac case "${opt_S:-0}" in 0) ;; -*) rotate_func=save_log opt_S=;; +*) rotate_func=save_log;; esac # see whether test handles -h or -L @@ -345,12 +366,7 @@ save_log() { $ECHO rm $rm_f `'ls' -1td $log.* 2> /dev/null | sed "1,${n}d"` mode=${opt_m:-`get_mode $log`} - # this is our default suffix - opt_S=${opt_S:-`get_mtime_suffix $log $fmt`} - case "$fmt" in - ""|$opt_f) suffix=$opt_S;; - *) suffix=`get_mtime_suffix $log $fmt`;; - esac + suffix=`get_mtime_suffix $log $fmt` # find a unique name to save current log as for nlog in $log.$suffix $log.$suffix.$$ diff --git a/mk/progs.mk b/mk/progs.mk index ada942db8621..fe8cad4b5c26 100644 --- a/mk/progs.mk +++ b/mk/progs.mk @@ -1,6 +1,6 @@ # SPDX-License-Identifier: BSD-2-Clause # -# $Id: progs.mk,v 1.18 2024/04/09 17:18:24 sjg Exp $ +# $Id: progs.mk,v 1.19 2025/05/19 19:15:22 sjg Exp $ # # @(#) Copyright (c) 2006, Simon J. Gerraty # @@ -68,7 +68,7 @@ $v += ${${v}_${PROG}:U${${v}.${PROG}}} # for meta mode, there can be only one! .if ${PROG} == ${UPDATE_DEPENDFILE_PROG:Uno} -UPDATE_DEPENDFILE ?= yes +UPDATE_DEPENDFILE ?= ${MK_UPDATE_DEPENDFILE:Uyes} .endif UPDATE_DEPENDFILE ?= NO diff --git a/mk/setopts.sh b/mk/setopts.sh index 91c65c776438..5fccb0bcb6fe 100644 --- a/mk/setopts.sh +++ b/mk/setopts.sh @@ -12,17 +12,17 @@ # This module sets shell variables for each option specified in # "opt_str". # -# If the option is followed by a ``:'' it requires an argument. +# If the option is followed by a ':' it requires an argument. # It defaults to an empty string and specifying that option on # the command line overrides the current value. # -# If the option is followed by a ``.'' then it is treated as for -# ``:'' except that any argument provided on the command line is -# appended to the current value using the value of "opt_dot" as -# separator (default is a space). +# If the option "o" is followed by a '.' then it is treated as for +# ':' except that any argument provided on the command line is +# appended to the current value using the value of "opt_dot_$o" +# if set, or "opt_dot" as separator (default is a space). # -# If the option is followed by a ``,'' then it is treated as for -# a ``.'' except that the separator is "opt_comma" (default ,). +# If the option is followed by a ',' then it is treated as for +# a '.' except that the separator is "opt_comma" (default ','). # # If the option is followed by ``='' it requires an argument # of the form "var=val" which will be evaluated. @@ -50,9 +50,9 @@ # # RCSid: -# $Id: setopts.sh,v 1.13 2023/02/20 19:30:06 sjg Exp $ +# $Id: setopts.sh,v 1.15 2025/06/01 02:10:31 sjg Exp $ # -# @(#) Copyright (c) 1995-2023 Simon J. Gerraty +# @(#) Copyright (c) 1995-2025 Simon J. Gerraty # # This file is provided in the hope that it will # be of use. There is absolutely NO WARRANTY. @@ -87,7 +87,7 @@ set1opt() { case "$opt_str" in *${o}:*) eval "opt_$o=\"$a\"";; - *${o}.*) eval "opt_$o=\"\${opt_$o}\${opt_$o:+$opt_dot}$a\"";; + *${o}.*) eval "opt_$o=\"\${opt_$o}\${opt_$o:+\${opt_dot_$o:-$opt_dot}}$a\"";; *${o},*) eval "opt_$o=\"\${opt_$o}\${opt_$o:+$opt_comma}$a\"";; *${o}=*) case "$a" in diff --git a/mk/sys.mk b/mk/sys.mk index cf6ef061f406..d05bd62e10c0 100644 --- a/mk/sys.mk +++ b/mk/sys.mk @@ -1,6 +1,6 @@ # SPDX-License-Identifier: BSD-2-Clause # -# $Id: sys.mk,v 1.61 2024/10/30 23:46:26 sjg Exp $ +# $Id: sys.mk,v 1.62 2025/05/19 19:15:22 sjg Exp $ # # @(#) Copyright (c) 2003-2023, Simon J. Gerraty # @@ -101,6 +101,7 @@ OPTIONS_DEFAULT_DEPENDENT += \ META_MODE/DIRDEPS_BUILD \ STAGING/DIRDEPS_BUILD \ STATIC_DIRDEPS_CACHE/DIRDEPS_CACHE \ + UPDATE_DEPENDFILE/DIRDEPS_BUILD \ .-include <options.mk> @@ -1,4 +1,4 @@ -/* $NetBSD: parse.c,v 1.743 2025/04/13 09:34:43 rillig Exp $ */ +/* $NetBSD: parse.c,v 1.752 2025/06/16 18:20:00 rillig Exp $ */ /* * Copyright (c) 1988, 1989, 1990, 1993 @@ -110,7 +110,7 @@ #include "pathnames.h" /* "@(#)parse.c 8.3 (Berkeley) 3/19/94" */ -MAKE_RCSID("$NetBSD: parse.c,v 1.743 2025/04/13 09:34:43 rillig Exp $"); +MAKE_RCSID("$NetBSD: parse.c,v 1.752 2025/06/16 18:20:00 rillig Exp $"); /* Detects a multiple-inclusion guard in a makefile. */ typedef enum { @@ -130,7 +130,7 @@ typedef struct IncludedFile { unsigned forBodyReadLines; /* the number of physical lines that have * been read from the file above the body of * the .for loop */ - unsigned int condMinDepth; /* depth of nested 'if' directives, at the + unsigned condMinDepth; /* depth of nested 'if' directives, at the * beginning of the file */ bool depending; /* state of doing_depend on EOF */ @@ -342,7 +342,7 @@ CurFile(void) return GetInclude(includes.len - 1); } -unsigned int +unsigned CurFile_CondMinDepth(void) { return CurFile()->condMinDepth; @@ -372,7 +372,7 @@ LoadFile(const char *path, int fd) assert(buf.len < buf.cap); n = read(fd, buf.data + buf.len, buf.cap - buf.len); if (n < 0) { - Error("%s: read error: %s", path, strerror(errno)); + Error("%s: %s", path, strerror(errno)); exit(2); /* Not 1 so -q can distinguish error */ } if (n == 0) @@ -388,23 +388,43 @@ LoadFile(const char *path, int fd) return buf; /* may not be null-terminated */ } +const char * +GetParentStackTrace(void) +{ + static bool initialized; + static const char *parentStackTrace; + + if (!initialized) { + const char *env = getenv("MAKE_STACK_TRACE"); + parentStackTrace = env == NULL ? NULL + : env[0] == '\t' ? bmake_strdup(env) + : strcmp(env, "yes") == 0 ? bmake_strdup("") + : NULL; + initialized = true; + } + return parentStackTrace; +} + /* * Print the current chain of .include and .for directives. In Parse_Fatal * or other functions that already print the location, includingInnermost * would be redundant, but in other cases like Error or Fatal it needs to be * included. */ -void -PrintStackTrace(bool includingInnermost) +char * +GetStackTrace(bool includingInnermost) { + const char *parentStackTrace; + Buffer buffer, *buf = &buffer; const IncludedFile *entries; size_t i, n; + bool hasDetails; - bool hasDetails = EvalStack_PrintDetails(); - + Buf_Init(buf); + hasDetails = EvalStack_Details(buf); n = includes.len; if (n == 0) - return; + goto add_parent_stack_trace; entries = GetInclude(0); if (!includingInnermost && !(hasDetails && n > 1) @@ -424,16 +444,49 @@ PrintStackTrace(bool includingInnermost) if (entry->forLoop != NULL) { char *details = ForLoop_Details(entry->forLoop); - debug_printf("\tin .for loop from %s:%u with %s\n", - fname, entry->forHeadLineno, details); + Buf_AddStr(buf, "\tin .for loop from "); + Buf_AddStr(buf, fname); + Buf_AddStr(buf, ":"); + Buf_AddInt(buf, (int)entry->forHeadLineno); + Buf_AddStr(buf, " with "); + Buf_AddStr(buf, details); + Buf_AddStr(buf, "\n"); free(details); } else if (i + 1 < n && entries[i + 1].forLoop != NULL) { /* entry->lineno is not a useful line number */ - } else - debug_printf("\tin %s:%u\n", fname, entry->lineno); + } else { + Buf_AddStr(buf, "\tin "); + Buf_AddStr(buf, fname); + Buf_AddStr(buf, ":"); + Buf_AddInt(buf, (int)entry->lineno); + Buf_AddStr(buf, "\n"); + } + } + +add_parent_stack_trace: + parentStackTrace = GetParentStackTrace(); + if ((makelevel > 0 && (n > 0 || !includingInnermost)) + || parentStackTrace != NULL) { + Buf_AddStr(buf, "\tin "); + Buf_AddStr(buf, progname); + Buf_AddStr(buf, " in directory \""); + Buf_AddStr(buf, curdir); + Buf_AddStr(buf, "\"\n"); } - if (makelevel > 0) - debug_printf("\tin directory %s\n", curdir); + + if (parentStackTrace != NULL) + Buf_AddStr(buf, parentStackTrace); + + return Buf_DoneData(buf); +} + +void +PrintStackTrace(bool includingInnermost) +{ + char *stackTrace = GetStackTrace(includingInnermost); + fprintf(stderr, "%s", stackTrace); + fflush(stderr); + free(stackTrace); } /* Check if the current character is escaped on the current line. */ @@ -547,7 +600,8 @@ ParseVErrorInternal(FILE *f, bool useVars, const GNode *gn, parseErrors++; } - if (level == PARSE_FATAL || DEBUG(PARSE)) + if (level == PARSE_FATAL || DEBUG(PARSE) + || (gn == NULL && includes.len == 0 /* see PrintLocation */)) PrintStackTrace(false); } @@ -713,7 +767,7 @@ TryApplyDependencyOperator(GNode *gn, GNodeType op) cohort->centurion = gn; gn->unmade_cohorts++; snprintf(cohort->cohort_num, sizeof cohort->cohort_num, "#%d", - (unsigned int)gn->unmade_cohorts % 1000000); + (unsigned)gn->unmade_cohorts % 1000000); } else { gn->type |= op; /* preserve any previous flags */ } @@ -1,4 +1,4 @@ -/* $NetBSD: suff.c,v 1.383 2025/01/14 21:39:24 rillig Exp $ */ +/* $NetBSD: suff.c,v 1.384 2025/05/18 06:24:27 rillig Exp $ */ /* * Copyright (c) 1988, 1989, 1990, 1993 @@ -115,7 +115,7 @@ #include "dir.h" /* "@(#)suff.c 8.4 (Berkeley) 3/21/94" */ -MAKE_RCSID("$NetBSD: suff.c,v 1.383 2025/01/14 21:39:24 rillig Exp $"); +MAKE_RCSID("$NetBSD: suff.c,v 1.384 2025/05/18 06:24:27 rillig Exp $"); typedef List SuffixList; typedef ListNode SuffixListNode; @@ -1027,16 +1027,16 @@ CandidateList_AddCandidatesFor(CandidateList *list, Candidate *cand) * Return whether a candidate was removed. */ static bool -RemoveCandidate(CandidateList *srcs) +RemoveCandidate(CandidateList *sources) { CandidateListNode *ln; #ifdef DEBUG_SRC - debug_printf("cleaning list %p:", srcs); - CandidateList_PrintAddrs(srcs); + debug_printf("cleaning list %p:", sources); + CandidateList_PrintAddrs(sources); #endif - for (ln = srcs->first; ln != NULL; ln = ln->next) { + for (ln = sources->first; ln != NULL; ln = ln->next) { Candidate *src = ln->datum; if (src->numChildren == 0) { @@ -1056,10 +1056,10 @@ RemoveCandidate(CandidateList *srcs) } #ifdef DEBUG_SRC debug_printf("free: list %p src %p:%s children %d\n", - srcs, src, src->file, src->numChildren); + sources, src, src->file, src->numChildren); Lst_Done(&src->childrenList); #endif - Lst_Remove(srcs, ln); + Lst_Remove(sources, ln); free(src->file); free(src); return true; @@ -1067,7 +1067,7 @@ RemoveCandidate(CandidateList *srcs) #ifdef DEBUG_SRC else { debug_printf("keep: list %p src %p:%s children %d:", - srcs, src, src->file, src->numChildren); + sources, src, src->file, src->numChildren); CandidateList_PrintAddrs(&src->childrenList); } #endif @@ -1076,20 +1076,20 @@ RemoveCandidate(CandidateList *srcs) return false; } -/* Find the first existing file/target in srcs. */ +/* Find the first existing file/target in sources. */ static Candidate * -FindThem(CandidateList *srcs, CandidateSearcher *cs) +FindThem(CandidateList *sources, CandidateSearcher *cs) { HashSet seen; HashSet_Init(&seen); - while (!Lst_IsEmpty(srcs)) { - Candidate *src = Lst_Dequeue(srcs); + while (!Lst_IsEmpty(sources)) { + Candidate *src = Lst_Dequeue(sources); #ifdef DEBUG_SRC debug_printf("remove from list %p src %p:%s\n", - srcs, src, src->file); + sources, src, src->file); #endif DEBUG1(SUFF, "\ttrying %s...", src->file); @@ -1116,7 +1116,7 @@ FindThem(CandidateList *srcs, CandidateSearcher *cs) DEBUG0(SUFF, "not there\n"); if (HashSet_Add(&seen, src->file)) - CandidateList_AddCandidatesFor(srcs, src); + CandidateList_AddCandidatesFor(sources, src); else { DEBUG1(SUFF, "FindThem: skipping duplicate \"%s\"\n", src->file); @@ -1650,7 +1650,7 @@ FindDepsLib(GNode *gn) static void FindDepsRegularKnown(const char *name, size_t nameLen, GNode *gn, - CandidateList *srcs, CandidateList *targs) + CandidateList *sources, CandidateList *targets) { SuffixListNode *ln; Candidate *targ; @@ -1665,20 +1665,20 @@ FindDepsRegularKnown(const char *name, size_t nameLen, GNode *gn, targ = Candidate_New(bmake_strdup(gn->name), pref, suff, NULL, gn); - CandidateList_AddCandidatesFor(srcs, targ); + CandidateList_AddCandidatesFor(sources, targ); /* Record the target so we can nuke it. */ - Lst_Append(targs, targ); + Lst_Append(targets, targ); } } static void FindDepsRegularUnknown(GNode *gn, const char *sopref, - CandidateList *srcs, CandidateList *targs) + CandidateList *sources, CandidateList *targets) { Candidate *targ; - if (!Lst_IsEmpty(targs) || nullSuff == NULL) + if (!Lst_IsEmpty(targets) || nullSuff == NULL) return; DEBUG1(SUFF, "\tNo known suffix on %s. Using .NULL suffix\n", gn->name); @@ -1693,14 +1693,14 @@ FindDepsRegularUnknown(GNode *gn, const char *sopref, * this anymore. */ if (Lst_IsEmpty(&gn->commands)) - CandidateList_AddCandidatesFor(srcs, targ); + CandidateList_AddCandidatesFor(sources, targ); else { DEBUG0(SUFF, "not "); } DEBUG0(SUFF, "adding suffix rules\n"); - Lst_Append(targs, targ); + Lst_Append(targets, targ); } /* @@ -1762,12 +1762,12 @@ static void FindDepsRegular(GNode *gn, CandidateSearcher *cs) { /* List of sources at which to look */ - CandidateList srcs = LST_INIT; + CandidateList sources = LST_INIT; /* * List of targets to which things can be transformed. * They all have the same file, but different suff and prefix fields. */ - CandidateList targs = LST_INIT; + CandidateList targets = LST_INIT; Candidate *bottom; /* Start of found transformation path */ Candidate *src; Candidate *targ; @@ -1803,25 +1803,25 @@ FindDepsRegular(GNode *gn, CandidateSearcher *cs) if (!(gn->type & OP_PHONY)) { - FindDepsRegularKnown(name, nameLen, gn, &srcs, &targs); + FindDepsRegularKnown(name, nameLen, gn, &sources, &targets); /* Handle target of unknown suffix... */ - FindDepsRegularUnknown(gn, name, &srcs, &targs); + FindDepsRegularUnknown(gn, name, &sources, &targets); /* * Using the list of possible sources built up from the target * suffix(es), try and find an existing file/target that * matches. */ - bottom = FindThem(&srcs, cs); + bottom = FindThem(&sources, cs); if (bottom == NULL) { /* * No known transformations -- use the first suffix * found for setting the local variables. */ - if (targs.first != NULL) - targ = targs.first->datum; + if (targets.first != NULL) + targ = targets.first->datum; else targ = NULL; } else { @@ -1940,11 +1940,11 @@ sfnd_return: if (bottom != NULL) CandidateSearcher_AddIfNew(cs, bottom); - while (RemoveCandidate(&srcs) || RemoveCandidate(&targs)) + while (RemoveCandidate(&sources) || RemoveCandidate(&targets)) continue; - CandidateSearcher_MoveAll(cs, &srcs); - CandidateSearcher_MoveAll(cs, &targs); + CandidateSearcher_MoveAll(cs, &sources); + CandidateSearcher_MoveAll(cs, &targets); } static void @@ -1,4 +1,4 @@ -/* $NetBSD: targ.c,v 1.184 2024/07/07 09:54:12 rillig Exp $ */ +/* $NetBSD: targ.c,v 1.185 2025/05/07 19:49:00 rillig Exp $ */ /* * Copyright (c) 1988, 1989, 1990, 1993 @@ -107,7 +107,7 @@ #include "dir.h" /* "@(#)targ.c 8.2 (Berkeley) 3/19/94" */ -MAKE_RCSID("$NetBSD: targ.c,v 1.184 2024/07/07 09:54:12 rillig Exp $"); +MAKE_RCSID("$NetBSD: targ.c,v 1.185 2025/05/07 19:49:00 rillig Exp $"); /* * All target nodes that appeared on the left-hand side of one of the @@ -548,7 +548,8 @@ PrintOnlySources(void) void Targ_PrintGraph(int pass) { - debug_printf("#*** Input graph:\n"); + debug_printf("#*** Begin input graph for pass %d in %s:\n", + pass, curdir); Targ_PrintNodes(&allTargets, pass); debug_printf("\n"); debug_printf("\n"); @@ -568,6 +569,8 @@ Targ_PrintGraph(int pass) debug_printf("\n"); Suff_PrintAll(); + debug_printf("#*** End input graph for pass %d in %s:\n", + pass, curdir); } /* @@ -1,4 +1,4 @@ -/* $NetBSD: trace.c,v 1.33 2023/03/28 14:39:31 rillig Exp $ */ +/* $NetBSD: trace.c,v 1.35 2025/05/09 18:42:56 rillig Exp $ */ /* * Copyright (c) 2000 The NetBSD Foundation, Inc. @@ -48,7 +48,7 @@ #include "job.h" #include "trace.h" -MAKE_RCSID("$NetBSD: trace.c,v 1.33 2023/03/28 14:39:31 rillig Exp $"); +MAKE_RCSID("$NetBSD: trace.c,v 1.35 2025/05/09 18:42:56 rillig Exp $"); static FILE *trfile; static pid_t trpid; @@ -102,11 +102,13 @@ Trace_Log(TrEvent event, Job *job) evname[event], trpid, trwd); #endif if (job != NULL) { + GNode *gn = Job_Node(job); + char *type = GNodeType_ToString(gn->type); char flags[4]; - Job_FlagsToString(job, flags, sizeof flags); - fprintf(trfile, " %s %d %s %x", job->node->name, - job->pid, flags, job->node->type); + fprintf(trfile, " %s %d %s %s", + gn->name, Job_Pid(job), flags, type); + free(type); } fputc('\n', trfile); fflush(trfile); diff --git a/unit-tests/Makefile b/unit-tests/Makefile index 7f1fe0a26c29..618a5959e304 100644 --- a/unit-tests/Makefile +++ b/unit-tests/Makefile @@ -1,6 +1,6 @@ -# $Id: Makefile,v 1.233 2025/04/14 16:02:33 sjg Exp $ +# $Id: Makefile,v 1.239 2025/06/15 21:32:16 sjg Exp $ # -# $NetBSD: Makefile,v 1.358 2025/04/13 09:29:32 rillig Exp $ +# $NetBSD: Makefile,v 1.367 2025/06/13 20:23:16 rillig Exp $ # # Unit tests for make(1) # @@ -222,6 +222,7 @@ TESTS+= hanoi-include TESTS+= impsrc TESTS+= include-main TESTS+= job-flags +TESTS+= job-output TESTS+= job-output-long-lines TESTS+= job-output-null TESTS+= jobs-empty-commands @@ -459,6 +460,7 @@ TESTS+= varname-dot-suffixes TESTS+= varname-dot-targets TESTS+= varname-empty TESTS+= varname-make +TESTS+= varname-make_stack_trace TESTS+= varname-make_print_var_on_error TESTS+= varname-make_print_var_on_error-jobs TESTS+= varname-makefile @@ -619,13 +621,7 @@ SED_CMDS.opt-debug-graph1= ${STD_SED_CMDS.dg1} SED_CMDS.opt-debug-graph2= ${STD_SED_CMDS.dg2} SED_CMDS.opt-debug-graph3= ${STD_SED_CMDS.dg3} SED_CMDS.opt-debug-hash= -e 's,\(entries\)=[1-9][0-9],\1=<entries>,' -SED_CMDS.opt-debug-jobs= -e 's,([0-9][0-9]*),(<pid>),' -SED_CMDS.opt-debug-jobs+= -e 's,pid [0-9][0-9]*,pid <pid>,' -SED_CMDS.opt-debug-jobs+= -e 's,Process [0-9][0-9]*,Process <pid>,' -SED_CMDS.opt-debug-jobs+= -e 's,JobFinish: [0-9][0-9]*,JobFinish: <pid>,' -SED_CMDS.opt-debug-jobs+= -e 's,Command: ${.SHELL:T},Command: <shell>,' -# The "-q" may be there or not, see jobs.c, variable shells. -SED_CMDS.opt-debug-jobs+= -e 's,^\(.Command: <shell>\) -q,\1,' +SED_CMDS.opt-debug-jobs= ${STD_SED_CMDS.dj} SED_CMDS.opt-debug-lint+= ${STD_SED_CMDS.regex} SED_CMDS.opt-jobs-no-action= ${STD_SED_CMDS.hide-from-output} SED_CMDS.opt-no-action-runflags= ${STD_SED_CMDS.hide-from-output} @@ -633,9 +629,12 @@ SED_CMDS.opt-where-am-i= -e '/usr.obj/d' # For Compat_RunCommand, useShell == false. SED_CMDS.sh-dots= -e 's,^.*\.\.\.:.*,<not found: ...>,' # For Compat_RunCommand, useShell == true. -SED_CMDS.sh-dots+= -e 's,^make: exec(\(.*\)) failed (.*)$$,<not found: \1>,' +SED_CMDS.sh-dots+= -e 's,^make: exec(\(.*\)): .*$$,<not found: \1>,' SED_CMDS.sh-dots+= -e 's,^\(\*\*\* Error code \)[1-9][0-9]*,\1<nonzero>,' +# Race condition between the child's stdout and make's status. SED_CMDS.sh-errctl= ${STD_SED_CMDS.dj} +SED_CMDS.sh-errctl+= -e '/^Process with pid/d' +SED_CMDS.sh-errctl+= -e '/^JobFinish:/d' SED_CMDS.sh-flags= ${STD_SED_CMDS.hide-from-output} SED_CMDS.shell-csh= ${STD_SED_CMDS.white-space} SED_CMDS.sh-leading-hyphen= ${STD_SED_CMDS.shell} @@ -646,7 +645,7 @@ SED_CMDS.var-op-shell+= ${STD_SED_CMDS.shell} SED_CMDS.var-op-shell+= -e '/command/s,No such.*,not found,' SED_CMDS.var-op-shell+= ${STD_SED_CMDS.white-space} SED_CMDS.vardebug+= -e 's,${.SHELL},</path/to/shell>,' -SED_CMDS.varmod-mtime+= -e "s,\(.*\)': .*,\1': <ENOENT>," +SED_CMDS.varmod-mtime+= -e "s,\(mtime for .*\): .*,\1: <ENOENT>," SED_CMDS.varmod-subst-regex+= ${STD_SED_CMDS.regex} SED_CMDS.varparse-errors+= ${STD_SED_CMDS.timestamp} SED_CMDS.varname-dot-make-meta-ignore_filter+= ${SED_CMDS.meta-ignore} @@ -662,7 +661,7 @@ SED_CMDS.varname-empty= ${.OBJDIR .PARSEDIR .PATH .SHELL .SYSPATH:L:@v@-e '/\\$ # Some tests need an additional round of postprocessing. POSTPROC.depsrc-wait= sed -e '/^---/d' -e 's,^\(: Making 3[abc]\)[123]$$,\1,' POSTPROC.deptgt-suffixes= awk '/^\#\*\*\* Suffixes/,/^never-stop/' -POSTPROC.gnode-submake= awk '/Input graph/, /^$$/' +POSTPROC.gnode-submake= awk '/Begin input graph/, /^$$/' POSTPROC.varname-dot-make-mode= sed 's,^\(: Making [abc]\)[123]$$,\1,' # Some tests reuse other tests, which makes them unnecessarily fragile. @@ -698,12 +697,10 @@ STD_SED_CMDS.dg2+= -e 's,\(last modified\) ..:..:.. ... ..\, ....,\1 <timestamp> STD_SED_CMDS.dg3= ${STD_SED_CMDS.dg2} # Omit details such as process IDs from the output of the -dj option. -STD_SED_CMDS.dj= \ - -e '/Process/d;/JobFinish:/d' \ - -e 's,^\(Job_TokenWithdraw\)([0-9]*),\1(<pid>),' \ - -e 's,^([0-9][0-9]*) \(withdrew token\),(<pid>) \1,' \ - -e 's, \(pid\) [0-9][0-9]*, \1 <pid>,' \ - -e 's,^\( Command:\) .*,\1 <shell>,' +STD_SED_CMDS.dj= -e 's, pid [0-9][0-9]*, pid <pid>,' +STD_SED_CMDS.dj+= -e 's,^\(.Command\): ${.SHELL:T},\1: <shell>,' +# The "-q" may be there or not, see jobs.c, variable shells. +STD_SED_CMDS.dj+= -e 's,^\(.Command: <shell>\) -q,\1,' # Reduce the noise for tests running with the -n option, since there is no # other way to suppress the echoing of the commands. @@ -850,7 +847,7 @@ _SED_CMDS+= -e 's,${.OBJDIR},<curdir>,g' -e 's,${.OBJDIR:tA},<curdir>,g' _SED_CMDS+= -e 's,^${TEST_MAKE:T:S,.,\\.,g}[][0-9]*:,make:,' _SED_CMDS+= -e 's,${TEST_MAKE:S,.,\\.,g},make,' _SED_CMDS+= -e 's,^usage: ${TEST_MAKE:T:S,.,\\.,g} ,usage: make ,' -_SED_CMDS+= -e 's,${TEST_MAKE:T:S,.,\\.,g}\(\[[1-9][0-9]*\]:\),make\1,' +_SED_CMDS+= -e 's,${TEST_MAKE:T:S,.,\\.,g}\(\[[1-9][0-9]*\][: ]\),make\1,' _SED_CMDS+= -e 's,<curdir>/,,g' _SED_CMDS+= -e 's,${UNIT_TESTS:S,.,\\.,g}/,,g' _SED_CMDS+= -e '/MAKE_VERSION/d' diff --git a/unit-tests/archive.exp b/unit-tests/archive.exp index 67b3d60ea9db..ea81a3589ff8 100644 --- a/unit-tests/archive.exp +++ b/unit-tests/archive.exp @@ -19,13 +19,13 @@ list-archive-wildcard: archive.mk list-archive-wildcard: ternary.mk make: archive.mk:61: Error in source archive spec "libprog.a${UNDEF}(archive.mk) pre post" - in directory <curdir> + in make[1] in directory "<curdir>" make: Fatal errors encountered -- cannot continue make: stopped making "list-archive-undef-archive" in unit-tests exit 1 make: archive.mk:68: Error in source archive spec "libprog.a" - in directory <curdir> + in make[1] in directory "<curdir>" make: Fatal errors encountered -- cannot continue make: stopped making "list-archive-undef-member" in unit-tests exit 1 diff --git a/unit-tests/check-expect.lua b/unit-tests/check-expect.lua new file mode 100644 index 000000000000..218056fbc021 --- /dev/null +++ b/unit-tests/check-expect.lua @@ -0,0 +1,190 @@ +#! /usr/bin/lua +-- $NetBSD: check-expect.lua,v 1.13 2025/04/13 09:29:32 rillig Exp $ + +--[[ + +usage: lua ./check-expect.lua *.mk + +Check that the various 'expect' comments in the .mk files produce the +expected text in the corresponding .exp file. + +# expect: <line> + All of these lines must occur in the .exp file, in the same order as + in the .mk file. + +# expect-reset + Search the following 'expect:' comments from the top of the .exp + file again. + +# expect[+-]offset: <message> + Each message must occur in the .exp file and refer back to the + source line in the .mk file. + +# expect-not: <substring> + The substring must not occur as part of any line of the .exp file. + +# expect-not-matches: <pattern> + The pattern (see https://lua.org/manual/5.4/manual.html#6.4.1) + must not occur as part of any line of the .exp file. +]] + + +local had_errors = false +---@param fmt string +function print_error(fmt, ...) + print(fmt:format(...)) + had_errors = true +end + + +---@return nil | string[] +local function load_lines(fname) + local lines = {} + + local f = io.open(fname, "r") + if f == nil then return nil end + + for line in f:lines() do + table.insert(lines, line) + end + f:close() + + return lines +end + + +---@param exp_lines string[] +local function collect_lineno_diagnostics(exp_lines) + ---@type table<string, string[]> + local by_location = {} + + for _, line in ipairs(exp_lines) do + ---@type string | nil, string, string + local l_fname, l_lineno, l_msg = + line:match('^make: ([^:]+):(%d+): (.*)') + if l_fname ~= nil then + local location = ("%s:%d"):format(l_fname, l_lineno) + if by_location[location] == nil then + by_location[location] = {} + end + table.insert(by_location[location], l_msg) + end + end + + return by_location +end + + +local function missing(by_location) + ---@type {filename: string, lineno: number, location: string, message: string}[] + local missing_expectations = {} + + for location, messages in pairs(by_location) do + for _, message in ipairs(messages) do + if message ~= "" and location:find(".mk:") then + local filename, lineno = location:match("^(%S+):(%d+)$") + table.insert(missing_expectations, { + filename = filename, + lineno = tonumber(lineno), + location = location, + message = message + }) + end + end + end + table.sort(missing_expectations, function(a, b) + if a.filename ~= b.filename then + return a.filename < b.filename + end + return a.lineno < b.lineno + end) + return missing_expectations +end + + +local function check_mk(mk_fname) + local exp_fname = mk_fname:gsub("%.mk$", ".exp") + local mk_lines = load_lines(mk_fname) + local exp_lines = load_lines(exp_fname) + if exp_lines == nil then return end + local by_location = collect_lineno_diagnostics(exp_lines) + local prev_expect_line = 0 + + for mk_lineno, mk_line in ipairs(mk_lines) do + + for text in mk_line:gmatch("#%s*expect%-not:%s*(.*)") do + local i = 1 + while i <= #exp_lines and not exp_lines[i]:find(text, 1, true) do + i = i + 1 + end + if i <= #exp_lines then + print_error("error: %s:%d: %s must not contain '%s'", + mk_fname, mk_lineno, exp_fname, text) + end + end + + for text in mk_line:gmatch("#%s*expect%-not%-matches:%s*(.*)") do + local i = 1 + while i <= #exp_lines and not exp_lines[i]:find(text) do + i = i + 1 + end + if i <= #exp_lines then + print_error("error: %s:%d: %s must not match '%s'", + mk_fname, mk_lineno, exp_fname, text) + end + end + + for text in mk_line:gmatch("#%s*expect:%s*(.*)") do + local i = prev_expect_line + -- As of 2022-04-15, some lines in the .exp files contain trailing + -- whitespace. If possible, this should be avoided by rewriting the + -- debug logging. When done, the trailing gsub can be removed. + -- See deptgt-phony.exp lines 14 and 15. + while i < #exp_lines and text ~= exp_lines[i + 1]:gsub("^%s*", ""):gsub("%s*$", "") do + i = i + 1 + end + if i < #exp_lines then + prev_expect_line = i + 1 + else + print_error("error: %s:%d: '%s:%d+' must contain '%s'", + mk_fname, mk_lineno, exp_fname, prev_expect_line + 1, text) + end + end + if mk_line:match("^#%s*expect%-reset$") then + prev_expect_line = 0 + end + + ---@param text string + for offset, text in mk_line:gmatch("#%s*expect([+%-]%d+):%s*(.*)") do + local location = ("%s:%d"):format(mk_fname, mk_lineno + tonumber(offset)) + + local found = false + if by_location[location] ~= nil then + for i, message in ipairs(by_location[location]) do + if message == text then + by_location[location][i] = "" + found = true + break + elseif message ~= "" then + print_error("error: %s:%d: out-of-order '%s'", + mk_fname, mk_lineno, message) + end + end + end + + if not found then + print_error("error: %s:%d: %s must contain '%s'", + mk_fname, mk_lineno, exp_fname, text) + end + end + end + + for _, m in ipairs(missing(by_location)) do + print_error("missing: %s: # expect+1: %s", m.location, m.message) + end +end + +for _, fname in ipairs(arg) do + check_mk(fname) +end +os.exit(not had_errors) diff --git a/unit-tests/cmd-errors-jobs.exp b/unit-tests/cmd-errors-jobs.exp index a535f428e890..bd3ae1e51332 100644 --- a/unit-tests/cmd-errors-jobs.exp +++ b/unit-tests/cmd-errors-jobs.exp @@ -10,28 +10,34 @@ begin parse-error-direct make: Unclosed variable "UNCLOSED" in command ": unexpected $@-${UNCLOSED" in target "parse-error-unclosed-expression" + in make[1] in directory "<curdir>" make: Unclosed expression, expecting '}' while evaluating variable "UNCLOSED" with value "" in command ": unexpected $@-${UNCLOSED:" in target "parse-error-unclosed-modifier" + in make[1] in directory "<curdir>" make: Unknown modifier ":Z" while evaluating variable "UNKNOWN" with value "" in command ": unexpected $@-${UNKNOWN:Z}-eol" in target "parse-error-unknown-modifier" + in make[1] in directory "<curdir>" end parse-error-direct with status 2 begin parse-error-indirect make: Unclosed variable "UNCLOSED" in command ": unexpected $@-${UNCLOSED" in target "parse-error-unclosed-expression" + in make[1] in directory "<curdir>" make: Unclosed expression, expecting '}' while evaluating variable "UNCLOSED" with value "" in command ": unexpected $@-${UNCLOSED:" in target "parse-error-unclosed-modifier" + in make[1] in directory "<curdir>" make: Unknown modifier ":Z" while evaluating variable "UNKNOWN" with value "" in command ": unexpected $@-${UNKNOWN:Z}-eol" in target "parse-error-unknown-modifier" + in make[1] in directory "<curdir>" end parse-error-indirect with status 2 begin begin-direct diff --git a/unit-tests/cond-func-empty.exp b/unit-tests/cond-func-empty.exp index fce2bc443f6b..7861edc74ba2 100644 --- a/unit-tests/cond-func-empty.exp +++ b/unit-tests/cond-func-empty.exp @@ -1,4 +1,5 @@ -make: cond-func-empty.mk:167: Unclosed variable "WORD" +make: cond-func-empty.mk:156: warning: Invalid character " " in variable name " WORD " +make: cond-func-empty.mk:168: Unclosed variable "WORD" make: Fatal errors encountered -- cannot continue make: stopped in unit-tests exit status 1 diff --git a/unit-tests/cond-func-empty.mk b/unit-tests/cond-func-empty.mk index 17e76a63f2e8..ab097b026dcb 100644 --- a/unit-tests/cond-func-empty.mk +++ b/unit-tests/cond-func-empty.mk @@ -1,4 +1,4 @@ -# $NetBSD: cond-func-empty.mk,v 1.28 2025/01/11 20:54:45 rillig Exp $ +# $NetBSD: cond-func-empty.mk,v 1.29 2025/06/11 18:49:58 sjg Exp $ # # Tests for the empty() function in .if conditions, which tests an # expression for emptiness. @@ -152,6 +152,7 @@ ${:U }= space # There may be spaces outside the parentheses. # Spaces inside the parentheses are interpreted as part of the variable name. +# expect+1: warning: Invalid character " " in variable name " WORD " .if ! empty ( WORD ) . error .endif diff --git a/unit-tests/cond-late.exp b/unit-tests/cond-late.exp index 6cd20d797d3e..13846e8c822a 100644 --- a/unit-tests/cond-late.exp +++ b/unit-tests/cond-late.exp @@ -1,7 +1,7 @@ make: cond-late.mk:38: Bad condition while evaluating condition " != "no"" while evaluating variable "VAR" with value "${${UNDEF} != "no":?:}" - in directory <curdir> + in make[1] in directory "<curdir>" make: Fatal errors encountered -- cannot continue make: stopped making "do-parse-time" in unit-tests yes diff --git a/unit-tests/dep-op-missing.exp b/unit-tests/dep-op-missing.exp index 5ae39b5fd256..8521dbf79792 100644 --- a/unit-tests/dep-op-missing.exp +++ b/unit-tests/dep-op-missing.exp @@ -1,5 +1,5 @@ make: dep-op-missing.tmp:1: Invalid line 'target' - in directory <curdir> + in make[1] in directory "<curdir>" make: Fatal errors encountered -- cannot continue make: stopped in unit-tests exit status 0 diff --git a/unit-tests/deptgt-suffixes.exp b/unit-tests/deptgt-suffixes.exp index 512e6d44a8be..9ab59fcd4810 100644 --- a/unit-tests/deptgt-suffixes.exp +++ b/unit-tests/deptgt-suffixes.exp @@ -26,6 +26,7 @@ .src-right.tgt-left: : Making ${.TARGET} from ${.IMPSRC}. +#*** End input graph for pass 1 in <curdir>: : Making deptgt-suffixes.src-left out of nothing. : Making deptgt-suffixes.tgt-right from deptgt-suffixes.src-left. : Making deptgt-suffixes.src-right out of nothing. diff --git a/unit-tests/directive-for-errors.exp b/unit-tests/directive-for-errors.exp index 6064f35a1944..39929864314d 100644 --- a/unit-tests/directive-for-errors.exp +++ b/unit-tests/directive-for-errors.exp @@ -8,7 +8,7 @@ make: directive-for-errors.mk:44: invalid character '$' in .for loop variable na make: directive-for-errors.mk:52: no iteration variables in for make: directive-for-errors.mk:64: Wrong number of words (5) in .for substitution list with 3 variables make: directive-for-errors.mk:78: missing `in' in for -make: directive-for-errors.mk:86: Unknown modifier ":Z" +make: directive-for-errors.mk:85: Unknown modifier ":Z" while evaluating "${:U3:Z} 4" with value "3" make: Fatal errors encountered -- cannot continue make: stopped in unit-tests diff --git a/unit-tests/directive-for-errors.mk b/unit-tests/directive-for-errors.mk index 42f93b0212f2..a58b8294289b 100644 --- a/unit-tests/directive-for-errors.mk +++ b/unit-tests/directive-for-errors.mk @@ -1,4 +1,4 @@ -# $NetBSD: directive-for-errors.mk,v 1.16 2025/03/30 16:43:10 rillig Exp $ +# $NetBSD: directive-for-errors.mk,v 1.17 2025/05/03 08:18:33 rillig Exp $ # # Tests for error handling in .for loops. @@ -73,11 +73,10 @@ ${:U\\}= backslash # see whether the "variable" '\' is local .endfor -# A missing 'in' should parse the .for loop but skip the body. +# A missing 'in' parses the .for loop but skips the body. # expect+1: missing `in' in for .for i over k -# XXX: As of 2020-12-31, this line is reached once. -. warning Should not be reached. +. error .endfor diff --git a/unit-tests/directive-for-null.exp b/unit-tests/directive-for-null.exp index 0a94c7d9e451..60caebee238f 100644 --- a/unit-tests/directive-for-null.exp +++ b/unit-tests/directive-for-null.exp @@ -1,5 +1,5 @@ make: (stdin):2: Zero byte read from file - in directory <curdir> + in make[1] in directory "<curdir>" *** Error code 2 (continuing) Stop. diff --git a/unit-tests/gnode-submake.exp b/unit-tests/gnode-submake.exp index ea00e8d76c11..c9cfada91c69 100644 --- a/unit-tests/gnode-submake.exp +++ b/unit-tests/gnode-submake.exp @@ -1,4 +1,4 @@ -#*** Input graph: +#*** Begin input graph for pass 1 in <curdir>: # all, unmade, type OP_DEPENDS, flags none # makeinfo, unmade, type OP_DEPENDS|OP_HAS_COMMANDS, flags none # make-index, unmade, type OP_DEPENDS|OP_SUBMAKE|OP_HAS_COMMANDS, flags none diff --git a/unit-tests/job-output.exp b/unit-tests/job-output.exp new file mode 100644 index 000000000000..1891021bf3e7 --- /dev/null +++ b/unit-tests/job-output.exp @@ -0,0 +1,13 @@ +begin empty-lines + + +end empty-lines +begin stdout-and-stderr +only stdout: +This is stdout. +This is stderr. +only stderr: +end stdout-and-stderr +This is stdout. +This is stderr. +exit status 0 diff --git a/unit-tests/job-output.mk b/unit-tests/job-output.mk new file mode 100644 index 000000000000..ae4708a5b2f8 --- /dev/null +++ b/unit-tests/job-output.mk @@ -0,0 +1,41 @@ +# $NetBSD: job-output.mk,v 1.2 2025/06/13 06:13:20 rillig Exp $ +# +# Tests for handling the output in parallel mode. + +all: .PHONY + @${MAKE} -f ${MAKEFILE} -j1 empty-lines + @${MAKE} -f ${MAKEFILE} -j1 stdout-and-stderr + @${MAKE} -f ${MAKEFILE} -j1 echo-on-stdout-and-stderr + +# By sleeping for a second, the output of the child process is written byte +# by byte, to be consumed in small pieces by make. No matter what the chunk +# size is, the empty lines must not be discarded. +empty-lines: .PHONY + @echo begin $@ + @sleep 1 + @echo + @sleep 1 + @echo + @sleep 1 + @echo end $@ + +# In parallel mode, both stdout and stderr from the child process are +# collected in a local buffer and then written to make's stdout. +# +# expect: begin stdout-and-stderr +# expect: only stdout: +# expect: This is stdout. +# expect: This is stderr. +# expect: only stderr: +# expect: end stdout-and-stderr +stdout-and-stderr: + @echo begin $@ + @echo only stdout: + @${MAKE} -f ${MAKEFILE} echo-on-stdout-and-stderr 2>/dev/null + @echo only stderr: + @${MAKE} -f ${MAKEFILE} echo-on-stdout-and-stderr 1>/dev/null + @echo end $@ + +echo-on-stdout-and-stderr: .PHONY + @echo This is stdout. + @echo This is stderr. 1>&2 diff --git a/unit-tests/objdir-writable.exp b/unit-tests/objdir-writable.exp index dc5cd706349e..3fbf82c0522a 100644 --- a/unit-tests/objdir-writable.exp +++ b/unit-tests/objdir-writable.exp @@ -1,4 +1,4 @@ -make: warning: <tmpdir>/roobj: Permission denied. +make: warning: <tmpdir>/roobj: Permission denied <tmpdir> <tmpdir>/roobj <tmpdir>/roobj diff --git a/unit-tests/opt-debug-graph1.exp b/unit-tests/opt-debug-graph1.exp index d01a98a31fdb..9dae95302318 100644 --- a/unit-tests/opt-debug-graph1.exp +++ b/unit-tests/opt-debug-graph1.exp @@ -1,4 +1,4 @@ -#*** Input graph: +#*** Begin input graph for pass 1 in <curdir>: # all, unmade, type OP_DEPENDS|OP_HAS_COMMANDS, flags none # made-target, unmade, type OP_DEPENDS, flags none # made-target-no-sources, unmade, type OP_DEPENDS, flags none @@ -50,4 +50,5 @@ MFLAGS = -r -k -d g1 #*** Suffixes: #*** Transformations: +#*** End input graph for pass 1 in <curdir>: exit status 0 diff --git a/unit-tests/opt-debug-graph2.exp b/unit-tests/opt-debug-graph2.exp index b590e7379ddb..e4160e413787 100644 --- a/unit-tests/opt-debug-graph2.exp +++ b/unit-tests/opt-debug-graph2.exp @@ -4,7 +4,7 @@ false false *** Error code 1 (continuing) `all' not remade because of errors. -#*** Input graph: +#*** Begin input graph for pass 2 in <curdir>: # made-target, made, type OP_DEPENDS|OP_PHONY|OP_HAS_COMMANDS|OP_DEPS_FOUND|OP_MARK, flags REMAKE|DONE_ALLSRC # # *** MAIN TARGET *** @@ -85,6 +85,7 @@ MFLAGS = -r -k -d g2 #*** Suffixes: #*** Transformations: +#*** End input graph for pass 2 in <curdir>: Stop. make: stopped making "all" in unit-tests diff --git a/unit-tests/opt-debug-graph3.exp b/unit-tests/opt-debug-graph3.exp index d1be69f89630..ccebfd7b16bc 100644 --- a/unit-tests/opt-debug-graph3.exp +++ b/unit-tests/opt-debug-graph3.exp @@ -4,7 +4,7 @@ false false *** Error code 1 (continuing) `all' not remade because of errors. -#*** Input graph: +#*** Begin input graph for pass 3 in <curdir>: # made-target, made, type OP_DEPENDS|OP_PHONY|OP_HAS_COMMANDS|OP_DEPS_FOUND|OP_MARK, flags REMAKE|DONE_ALLSRC # # *** MAIN TARGET *** @@ -85,6 +85,7 @@ MFLAGS = -r -k -d g3 #*** Suffixes: #*** Transformations: +#*** End input graph for pass 3 in <curdir>: Stop. make: stopped making "all" in unit-tests diff --git a/unit-tests/opt-debug-jobs.exp b/unit-tests/opt-debug-jobs.exp index e79d8e94a952..6cda45107702 100644 --- a/unit-tests/opt-debug-jobs.exp +++ b/unit-tests/opt-debug-jobs.exp @@ -1,6 +1,6 @@ job_pipe -1 -1, maxjobs 1, tokens 1, compat 0 -Job_TokenWithdraw(<pid>): aborting 0, running 0 -(<pid>) withdrew token +TokenPool_Take: pid <pid>, aborting NONE, running 0 +TokenPool_Take: pid <pid> took a token echo ": expanded expression" { : expanded expression } || exit $? @@ -13,15 +13,15 @@ echo ": 'single' and \"double\" quotes" { sleep 1 } || exit $? Running all - Command: <shell> -JobExec(all): pid <pid> added to jobs table -job table @ job started -job 0, status 3, flags ---, pid <pid> + Command: <shell> +JobExec: target all, pid <pid> added to jobs table +job started, job table: +job 0, status running, flags ---, pid <pid> : expanded expression : variable : 'single' and "double" quotes -Process <pid> exited/stopped status 0. -JobFinish: <pid> [all], status 0 -Job_TokenWithdraw(<pid>): aborting 0, running 0 -(<pid>) withdrew token +Process with pid <pid> exited/stopped with status 0. +JobFinish: target all, pid <pid>, status 0 +TokenPool_Take: pid <pid>, aborting NONE, running 0 +TokenPool_Take: pid <pid> took a token exit status 0 diff --git a/unit-tests/opt-file.exp b/unit-tests/opt-file.exp index 3087ab790d2e..d915f9fe0170 100644 --- a/unit-tests/opt-file.exp +++ b/unit-tests/opt-file.exp @@ -2,7 +2,7 @@ value value line-with-trailing-whitespace make: (stdin):1: Zero byte read from file - in directory <curdir> + in make[1] in directory "<curdir>" *** Error code 2 (continuing) `all' not remade because of errors. diff --git a/unit-tests/opt-jobs-internal.exp b/unit-tests/opt-jobs-internal.exp index 470bdbddd0f8..e3e8ee498224 100644 --- a/unit-tests/opt-jobs-internal.exp +++ b/unit-tests/opt-jobs-internal.exp @@ -1,6 +1,27 @@ -make: internal error -- J option malformed (garbage) -usage: make [-BeikNnqrSstWwX] - [-C directory] [-D variable] [-d flags] [-f makefile] - [-I directory] [-J private] [-j max_jobs] [-m directory] [-T file] - [-V variable] [-v variable] [variable=value] [target ...] -exit status 2 +direct: mode=parallel +make: error: invalid internal option "-J garbage" in "<curdir>" +make: warning: internal option "-J" in "<curdir>" refers to unopened file descriptors; falling back to compat mode. + To run the target even in -n mode, add the .MAKE pseudo-source to the target. + To run the target in default mode only, add a ${:D make} marker to a target's command. (This marker expression expands to an empty string.) + To make the sub-make run in compat mode, add -B to its invocation. + To make the sub-make independent from the parent make, unset the MAKEFLAGS environment variable in the target's commands. + in make[2] in directory "<curdir>" +direct-open: mode=compat +make: warning: internal option "-J" in "<curdir>" refers to unopened file descriptors; falling back to compat mode. + To run the target even in -n mode, add the .MAKE pseudo-source to the target. + To run the target in default mode only, add a ${:D make} marker to a target's command. (This marker expression expands to an empty string.) + To make the sub-make run in compat mode, add -B to its invocation. + To make the sub-make independent from the parent make, unset the MAKEFLAGS environment variable in the target's commands. + in make[2] in directory "<curdir>" +indirect-open: mode=compat +indirect-expr: mode=parallel +make: warning: internal option "-J" in "<curdir>" refers to unopened file descriptors; falling back to compat mode. + To run the target even in -n mode, add the .MAKE pseudo-source to the target. + To run the target in default mode only, add a ${:D make} marker to a target's command. (This marker expression expands to an empty string.) + To make the sub-make run in compat mode, add -B to its invocation. + To make the sub-make independent from the parent make, unset the MAKEFLAGS environment variable in the target's commands. + in make[2] in directory "<curdir>" +indirect-comment: mode=compat +indirect-silent-comment: mode=parallel +indirect-expr-empty: mode=parallel +exit status 0 diff --git a/unit-tests/opt-jobs-internal.mk b/unit-tests/opt-jobs-internal.mk index 44755a797751..13db820f86c1 100644 --- a/unit-tests/opt-jobs-internal.mk +++ b/unit-tests/opt-jobs-internal.mk @@ -1,9 +1,65 @@ -# $NetBSD: opt-jobs-internal.mk,v 1.3 2022/01/23 16:09:38 rillig Exp $ +# $NetBSD: opt-jobs-internal.mk,v 1.6 2025/05/23 21:05:56 rillig Exp $ # -# Tests for the (intentionally undocumented) -J command line option. -# -# Only test the error handling here, the happy path is covered in other tests -# as a side effect. +# Tests for the (intentionally undocumented) internal -J command line option. + +# This test expects +.MAKE.ALWAYS_PASS_JOB_QUEUE= no + +all: .PHONY + @${MAKE} -f ${MAKEFILE} -j1 direct + @${MAKE} -f ${MAKEFILE} -j1 direct-syntax + @${MAKE} -f ${MAKEFILE} -j1 direct-open + @${MAKE} -f ${MAKEFILE} -j1 indirect-open + @${MAKE} -f ${MAKEFILE} -j1 indirect-expr + @${MAKE} -f ${MAKEFILE} -j1 indirect-comment + @${MAKE} -f ${MAKEFILE} -j1 indirect-silent-comment + @${MAKE} -f ${MAKEFILE} -j1 indirect-expr-empty + +detect-mode: .PHONY + @mode=parallel + @echo ${HEADING}: mode=$${mode:-compat} + +# expect: direct: mode=parallel +direct: .PHONY + @mode=parallel + @echo ${.TARGET}: mode=$${mode:-compat} + +# expect: make: error: invalid internal option "-J garbage" in "<curdir>" +direct-syntax: .PHONY + @${MAKE} -f ${MAKEFILE} -J garbage unexpected-target || : + +# expect: direct-open: mode=compat +direct-open: .PHONY + @${MAKE} -f ${MAKEFILE} -J 31,32 detect-mode HEADING=${.TARGET} + +# expect: indirect-open: mode=compat +indirect-open: .PHONY + @${MAKE:U} -f ${MAKEFILE} detect-mode HEADING=${.TARGET} + +# When a command in its unexpanded form contains the expression "${MAKE}" +# without any modifiers, the file descriptors get passed around. +# expect: indirect-expr: mode=parallel +indirect-expr: .PHONY + @${MAKE} -f ${MAKEFILE} detect-mode HEADING=${.TARGET} + +# The "# make" comment starts directly after the leading tab and is thus not +# considered a shell command line. No file descriptors are passed around. +# expect: indirect-comment: mode=compat +indirect-comment: .PHONY + # make + @${MAKE:U} -f ${MAKEFILE} detect-mode HEADING=${.TARGET} + +# When the "# make" comment is prefixed with "@", it becomes a shell command. +# As that shell command contains the plain word "make", the file descriptors +# get passed around. +# expect: indirect-silent-comment: mode=parallel +indirect-silent-comment: .PHONY + @# make + @${MAKE:U} -f ${MAKEFILE} detect-mode HEADING=${.TARGET} -# expect: make: internal error -- J option malformed (garbage) -.MAKEFLAGS: -Jgarbage +# When a command in its unexpanded form contains the plain word "make", the +# file descriptors get passed around. +# expect: indirect-expr-empty: mode=parallel +indirect-expr-empty: .PHONY + @${:D make} + @${MAKE:U} -f ${MAKEFILE} detect-mode HEADING=${.TARGET} diff --git a/unit-tests/opt-jobs.mk b/unit-tests/opt-jobs.mk index ce84e47e91e1..818a4844e94b 100644 --- a/unit-tests/opt-jobs.mk +++ b/unit-tests/opt-jobs.mk @@ -1,4 +1,4 @@ -# $NetBSD: opt-jobs.mk,v 1.5 2023/09/10 16:25:32 sjg Exp $ +# $NetBSD: opt-jobs.mk,v 1.7 2025/05/20 17:56:40 sjg Exp $ # # Tests for the -j command line option, which creates the targets in parallel. diff --git a/unit-tests/opt-touch-jobs.mk b/unit-tests/opt-touch-jobs.mk index 6005ab49d125..8c9c25c59015 100644 --- a/unit-tests/opt-touch-jobs.mk +++ b/unit-tests/opt-touch-jobs.mk @@ -1,4 +1,4 @@ -# $NetBSD: opt-touch-jobs.mk,v 1.2 2021/01/30 12:14:08 rillig Exp $ +# $NetBSD: opt-touch-jobs.mk,v 1.3 2025/05/18 06:24:27 rillig Exp $ # # Tests for the -t command line option in jobs mode. @@ -27,7 +27,7 @@ opt-touch-use: .USE # Even though it is listed last, in the output it appears first. # This is because it is the only node that actually needs to be run. # The "is up to date" of the other nodes happens after all jobs have -# finished, by Make_Run > MakePrintStatusList > MakePrintStatus. +# finished, by Make_MakeParallel > MakePrintStatusList > MakePrintStatus. opt-touch-make: .MAKE : Making $@. diff --git a/unit-tests/opt-tracefile.exp b/unit-tests/opt-tracefile.exp index 0e815606d34f..202c3c1afe49 100644 --- a/unit-tests/opt-tracefile.exp +++ b/unit-tests/opt-tracefile.exp @@ -1,12 +1,12 @@ Making dependency1 from <nothing>. Making dependency2 from <nothing>. Making trace from dependency1 dependency2. -0 BEG -1 JOB -1 DON -1 JOB -1 DON -1 JOB -1 DON -0 END +<timestamp> 0 BEG <make-pid> <curdir> +<timestamp> 1 JOB <make-pid> <curdir> dependency1 <job-pid> --- OP_DEPENDS|OP_PHONY|OP_HAS_COMMANDS|OP_DEPS_FOUND|OP_MARK +<timestamp> 1 DON <make-pid> <curdir> dependency1 <job-pid> --- OP_DEPENDS|OP_PHONY|OP_HAS_COMMANDS|OP_DEPS_FOUND|OP_MARK +<timestamp> 1 JOB <make-pid> <curdir> dependency2 <job-pid> --- OP_DEPENDS|OP_PHONY|OP_HAS_COMMANDS|OP_DEPS_FOUND|OP_MARK +<timestamp> 1 DON <make-pid> <curdir> dependency2 <job-pid> --- OP_DEPENDS|OP_PHONY|OP_HAS_COMMANDS|OP_DEPS_FOUND|OP_MARK +<timestamp> 1 JOB <make-pid> <curdir> trace <job-pid> --- OP_DEPENDS|OP_PHONY|OP_HAS_COMMANDS|OP_DEPS_FOUND +<timestamp> 1 DON <make-pid> <curdir> trace <job-pid> --- OP_DEPENDS|OP_PHONY|OP_HAS_COMMANDS|OP_DEPS_FOUND +<timestamp> 0 END <make-pid> <curdir> exit status 0 diff --git a/unit-tests/opt-tracefile.mk b/unit-tests/opt-tracefile.mk index 291824680606..d94c8045c224 100644 --- a/unit-tests/opt-tracefile.mk +++ b/unit-tests/opt-tracefile.mk @@ -1,4 +1,4 @@ -# $NetBSD: opt-tracefile.mk,v 1.5 2021/12/06 22:35:20 rillig Exp $ +# $NetBSD: opt-tracefile.mk,v 1.6 2025/05/09 18:38:40 rillig Exp $ # # Tests for the command line option '-T', which in jobs mode appends a trace # record to a trace log whenever a job is started or completed. @@ -6,8 +6,7 @@ all: .PHONY @rm -f opt-tracefile.log @${MAKE} -f ${MAKEFILE} -j1 -Topt-tracefile.log trace - # Remove timestamps, process IDs and directory paths. - @awk '{ print $$2, $$3 }' opt-tracefile.log + @awk '{ $$1 = "<timestamp>"; $$4 = "<make-pid>"; if (NF >= 7) $$7 = "<job-pid>"; print }' opt-tracefile.log @rm opt-tracefile.log trace dependency1 dependency2: .PHONY diff --git a/unit-tests/sh-errctl.exp b/unit-tests/sh-errctl.exp index 8e6bc3c82125..8419d215fe38 100644 --- a/unit-tests/sh-errctl.exp +++ b/unit-tests/sh-errctl.exp @@ -1,6 +1,6 @@ job_pipe -1 -1, maxjobs 1, tokens 1, compat 0 -Job_TokenWithdraw(<pid>): aborting 0, running 0 -(<pid>) withdrew token +TokenPool_Take: pid <pid>, aborting NONE, running 0 +TokenPool_Take: pid <pid> took a token # echo off echo silent # echo on @@ -16,12 +16,12 @@ set -e echo always Running all Command: <shell> -JobExec(all): pid <pid> added to jobs table -job table @ job started -job 0, status 3, flags ---, pid <pid> +JobExec: target all, pid <pid> added to jobs table +job started, job table: +job 0, status running, flags ---, pid <pid> silent ignerr always -Job_TokenWithdraw(<pid>): aborting 0, running 0 -(<pid>) withdrew token +TokenPool_Take: pid <pid>, aborting NONE, running 0 +TokenPool_Take: pid <pid> took a token exit status 0 diff --git a/unit-tests/shell-csh.mk b/unit-tests/shell-csh.mk index 21517d5dbd78..cfe28199dbd2 100644 --- a/unit-tests/shell-csh.mk +++ b/unit-tests/shell-csh.mk @@ -1,4 +1,4 @@ -# $NetBSD: shell-csh.mk,v 1.9 2024/05/25 15:37:17 rillig Exp $ +# $NetBSD: shell-csh.mk,v 1.10 2025/06/05 21:56:54 rillig Exp $ # # Tests for using a C shell for running the commands. @@ -33,7 +33,9 @@ all: -echo ignore errors # In the C shell, "unset verbose" is set as the noPrint command. - # Therefore it is filtered from the output, rather naively. + # Therefore, it is filtered from the output, rather naively. +# FIXME: Don't assume a newline character in PrintFilteredOutput. +# expect: They chatted in the sy. @echo 'They chatted in the sunset verbosely.' .else @sed '$$d' ${MAKEFILE:.mk=.exp} # This is cheated. diff --git a/unit-tests/shell-ksh.exp b/unit-tests/shell-ksh.exp index 0bf83203a23a..fcb9fb888d21 100644 --- a/unit-tests/shell-ksh.exp +++ b/unit-tests/shell-ksh.exp @@ -1,4 +1,9 @@ -: normal -: always -: ignore errors +echo normal +normal +hidden +echo always +always +echo ignore errors +ignore errors +The "is filtered out. exit status 0 diff --git a/unit-tests/shell-ksh.mk b/unit-tests/shell-ksh.mk index 3acf98cdb5d1..676c8e2d47d9 100644 --- a/unit-tests/shell-ksh.mk +++ b/unit-tests/shell-ksh.mk @@ -1,11 +1,39 @@ -# $NetBSD: shell-ksh.mk,v 1.1 2020/10/03 14:39:36 rillig Exp $ +# $NetBSD: shell-ksh.mk,v 1.2 2025/06/05 21:56:54 rillig Exp $ # -# Tests for using a korn shell for running the commands. +# Tests for using a Korn shell for running the commands. -.SHELL: name="ksh" path="ksh" +KSH!= which ksh 2> /dev/null || true + +# The shell path must be an absolute path. +# This is only obvious in parallel mode since in compat mode, +# simple commands are executed via execvp directly. +.if ${KSH} != "" +.SHELL: name="ksh" path="${KSH}" +.endif + +# In parallel mode, the shell->noPrint command is filtered from +# the output, rather naively (in PrintOutput). +.MAKEFLAGS: -j1 all: - : normal - @: hidden - +: always - -: ignore errors +.if ${KSH} != "" + # This command is both printed and executed. + echo normal + + # This command is only executed. + @echo hidden + + # This command is both printed and executed. + +echo always + + # This command is both printed and executed. + -echo ignore errors + + # In the Korn shell, "set +v" is set as the noPrint command. + # Therefore, it is filtered from the output, rather naively. +# FIXME: Don't assume a newline character in PrintFilteredOutput. +# expect: The "is filtered out. + @echo 'The "set +v" is filtered out.' +.else + @sed '$$d' ${MAKEFILE:.mk=.exp} # This is cheated. +.endif diff --git a/unit-tests/suff-main-several.exp b/unit-tests/suff-main-several.exp index c06fb0cf88b9..bb1fc91ea21d 100644 --- a/unit-tests/suff-main-several.exp +++ b/unit-tests/suff-main-several.exp @@ -63,7 +63,7 @@ Target "next-main" depends on "suff-main-several.{2,3,4}" # suff-main-several.{2,3,4}, unmade, type none, flags none Parsing suff-main-several.mk:42: .MAKEFLAGS: -d0 -dg1 ParseDependency(.MAKEFLAGS: -d0 -dg1) -#*** Input graph: +#*** Begin input graph for pass 1 in <curdir>: # .1.2, unmade, type OP_TRANSFORM, flags none # .1.3, unmade, type OP_TRANSFORM, flags none # .1.4, unmade, type OP_TRANSFORM, flags none @@ -131,6 +131,7 @@ MFLAGS = -r -k -d mps -d 0 -d g1 # From: # Search Path: #*** Transformations: +#*** End input graph for pass 1 in <curdir>: make: don't know how to make suff-main-several.2 (continuing) make: don't know how to make suff-main-several.3 (continuing) make: don't know how to make suff-main-several.4 (continuing) diff --git a/unit-tests/suff-transform-debug.exp b/unit-tests/suff-transform-debug.exp index 2e88db58bc8c..ad0584f204d1 100644 --- a/unit-tests/suff-transform-debug.exp +++ b/unit-tests/suff-transform-debug.exp @@ -1,4 +1,4 @@ -#*** Input graph: +#*** Begin input graph for pass 1 in <curdir>: # all, unmade, type OP_DEPENDS, flags none @@ -59,4 +59,5 @@ MFLAGS = -r -k -d g1 .cpp.a : : Making ${.TARGET} from impsrc ${.IMPSRC} allsrc ${.ALLSRC}. +#*** End input graph for pass 1 in <curdir>: exit status 0 diff --git a/unit-tests/var-op-expand.exp b/unit-tests/var-op-expand.exp index 88d2333a3f6f..105d2f50acc8 100644 --- a/unit-tests/var-op-expand.exp +++ b/unit-tests/var-op-expand.exp @@ -6,6 +6,18 @@ make: var-op-expand.mk:283: Unknown modifier ":s,value,replaced," while evaluating variable "later" with value "lowercase-value" while evaluating variable "indirect" with value "${later:s,value,replaced,} ok ${later:value=sysv}" make: var-op-expand.mk:285: warning: XXX Neither branch should be taken. +make: var-op-expand.mk:297: Bad condition + while evaluating condition " < 0 " +make: var-op-expand.mk:297: Unknown modifier ":Z1" + while parsing "${:Z1}:${:Z2}}" + while evaluating then-branch of condition " < 0 " +make: var-op-expand.mk:297: Unknown modifier ":Z2" + while parsing "${:Z2}}" + while evaluating else-branch of condition " < 0 " +make: var-op-expand.mk:297: Unknown modifier ":Z1" + while evaluating "${:Z1}:${:Z2}}" with value "" +make: var-op-expand.mk:297: Unknown modifier ":Z2" + while evaluating "${:Z2}}" with value "" make: Fatal errors encountered -- cannot continue -make: stopped making "all" in unit-tests +make: stopped in unit-tests exit status 1 diff --git a/unit-tests/var-op-expand.mk b/unit-tests/var-op-expand.mk index 863ed19d4348..6a49648f0618 100644 --- a/unit-tests/var-op-expand.mk +++ b/unit-tests/var-op-expand.mk @@ -1,4 +1,4 @@ -# $NetBSD: var-op-expand.mk,v 1.23 2025/03/29 19:08:52 rillig Exp $ +# $NetBSD: var-op-expand.mk,v 1.24 2025/04/30 06:01:07 rillig Exp $ # # Tests for the := variable assignment operator, which expands its # right-hand side. @@ -288,5 +288,10 @@ later= lowercase-value .endif -all: - @:; +# FIXME: The expression is evaluated twice, for no obvious reason. +# expect+5: Bad condition +# expect+4: Unknown modifier ":Z1" +# expect+3: Unknown modifier ":Z2" +# expect+2: Unknown modifier ":Z1" +# expect+1: Unknown modifier ":Z2" +_:= ${ < 0 :?${:Z1}:${:Z2}} diff --git a/unit-tests/var-recursive.exp b/unit-tests/var-recursive.exp index 75701e4c8c4f..97568873bf1b 100644 --- a/unit-tests/var-recursive.exp +++ b/unit-tests/var-recursive.exp @@ -1,20 +1,20 @@ make: var-recursive.mk:11: Variable DIRECT is recursive. while evaluating variable "DIRECT" with value "${DIRECT}" - in directory <curdir> + in make[1] in directory "<curdir>" make: var-recursive.mk:11: <> make: var-recursive.mk:19: Variable INDIRECT1 is recursive. while evaluating variable "INDIRECT2" with value "${INDIRECT1}" while evaluating variable "INDIRECT1" with value "${INDIRECT2}" - in directory <curdir> + in make[1] in directory "<curdir>" make: var-recursive.mk:19: <> make: var-recursive.mk:26: <ok> make: var-recursive.mk:34: Variable MODIFIERS is recursive. while evaluating variable "MODIFIERS" with value "${MODIFIERS:Mpattern}" - in directory <curdir> + in make[1] in directory "<curdir>" make: var-recursive.mk:34: <Mpattern}> make: var-recursive.mk:43: Variable V is recursive. while evaluating variable "V" with value "$V" - in directory <curdir> + in make[1] in directory "<curdir>" make: var-recursive.mk:43: <> make: Fatal errors encountered -- cannot continue make: stopped making "loadtime" in unit-tests @@ -24,5 +24,6 @@ make: Variable VAR is recursive. while evaluating variable "VAR" with value "${VAR}" in command ": recursive-line-before <${VAR}> recursive-line-after" in target "runtime" + in make[1] in directory "<curdir>" sub-exit status 2 exit status 0 diff --git a/unit-tests/varmod-assign.exp b/unit-tests/varmod-assign.exp index a4b689ab9fd4..ae7f6787d124 100644 --- a/unit-tests/varmod-assign.exp +++ b/unit-tests/varmod-assign.exp @@ -61,6 +61,9 @@ make: Unfinished modifier after "value # missing closing brace", expecting "}" in target "mod-assign-parse-3" ok=word make: warning: Command " echo word; (exit 13) " exited with status 13 + while evaluating variable "SH_ERR" with value "previous" + in command "@${SH_ERR::!= echo word; (exit 13) } echo err=${SH_ERR}" + in target "mod-assign-shell-error" err=previous Command: TARGET_CMD_VAR = cmd-value Global: TARGET_GLOBAL_VAR = global-value diff --git a/unit-tests/varmod-ifelse.exp b/unit-tests/varmod-ifelse.exp index d92134a83c84..039f0fd0b9a7 100644 --- a/unit-tests/varmod-ifelse.exp +++ b/unit-tests/varmod-ifelse.exp @@ -57,6 +57,15 @@ make: varmod-ifelse.mk:314: Unknown modifier ":X-then" make: varmod-ifelse.mk:314: Unknown modifier ":X-else" while parsing "${:X-else}}" while evaluating else-branch of condition "1" +make: varmod-ifelse.mk:322: Bad condition + while evaluating condition " < 0 " +make: varmod-ifelse.mk:322: Unknown modifier ":Z1" + while parsing "${:Z1}:${:Z2}}>" + while evaluating then-branch of condition " < 0 " +make: varmod-ifelse.mk:322: Unknown modifier ":Z2" + while parsing "${:Z2}}>" + while evaluating else-branch of condition " < 0 " +make: varmod-ifelse.mk:322: <> make: Fatal errors encountered -- cannot continue make: stopped in unit-tests exit status 1 diff --git a/unit-tests/varmod-ifelse.mk b/unit-tests/varmod-ifelse.mk index c316da51c3a9..986524330a97 100644 --- a/unit-tests/varmod-ifelse.mk +++ b/unit-tests/varmod-ifelse.mk @@ -1,4 +1,4 @@ -# $NetBSD: varmod-ifelse.mk,v 1.37 2025/04/04 18:57:01 rillig Exp $ +# $NetBSD: varmod-ifelse.mk,v 1.39 2025/04/30 06:01:07 rillig Exp $ # # Tests for the ${cond:?then:else} variable modifier, which evaluates either # the then-expression or the else-expression, depending on the condition. @@ -313,3 +313,10 @@ BOTH= <${YES}> <${NO}> # expect+1: Unknown modifier ":X-else" .if ${1:?${:X-then}:${:X-else}} .endif + + +# expect+4: Bad condition +# expect+3: Unknown modifier ":Z1" +# expect+2: Unknown modifier ":Z2" +# expect+1: <> +.info <${ < 0 :?${:Z1}:${:Z2}}> diff --git a/unit-tests/varmod-mtime.exp b/unit-tests/varmod-mtime.exp index 538294b2cc7f..9960dd877768 100644 --- a/unit-tests/varmod-mtime.exp +++ b/unit-tests/varmod-mtime.exp @@ -1,8 +1,8 @@ make: varmod-mtime.mk:46: Invalid argument '123x' for modifier ':mtime' while evaluating variable "no/such/file" with value "no/such/file" -make: varmod-mtime.mk:68: Cannot determine mtime for 'no/such/file1': <ENOENT> +make: varmod-mtime.mk:68: Cannot determine mtime for "no/such/file1": <ENOENT> while evaluating variable "no/such/file1 no/such/file2" with value "no/such/file1 no/such/file2" -make: varmod-mtime.mk:68: Cannot determine mtime for 'no/such/file2': <ENOENT> +make: varmod-mtime.mk:68: Cannot determine mtime for "no/such/file2": <ENOENT> while evaluating variable "no/such/file1 no/such/file2" with value "no/such/file1 no/such/file2" make: varmod-mtime.mk:78: Invalid argument 'errorhandler-no' for modifier ':mtime' while evaluating variable "MAKEFILE" with value "varmod-mtime.mk" diff --git a/unit-tests/varmod-mtime.mk b/unit-tests/varmod-mtime.mk index 67bca7c9d557..ec33cd698b41 100644 --- a/unit-tests/varmod-mtime.mk +++ b/unit-tests/varmod-mtime.mk @@ -1,4 +1,4 @@ -# $NetBSD: varmod-mtime.mk,v 1.15 2025/03/29 19:08:52 rillig Exp $ +# $NetBSD: varmod-mtime.mk,v 1.16 2025/06/12 18:51:05 rillig Exp $ # # Tests for the ':mtime' variable modifier, which maps each word of the # expression to that file's modification time. @@ -63,8 +63,8 @@ _!= rm -f ${COOKIE} # If the optional argument of the ':mtime' modifier is the word 'error', the # modifier fails with an error message, once for each affected file. # -# expect+2: Cannot determine mtime for 'no/such/file1': <ENOENT> -# expect+1: Cannot determine mtime for 'no/such/file2': <ENOENT> +# expect+2: Cannot determine mtime for "no/such/file1": <ENOENT> +# expect+1: Cannot determine mtime for "no/such/file2": <ENOENT> .if ${no/such/file1 no/such/file2:L:mtime=error} . error .else diff --git a/unit-tests/varname-dot-make-level.exp b/unit-tests/varname-dot-make-level.exp index 63dcf58c6569..1209cf3b4af8 100644 --- a/unit-tests/varname-dot-make-level.exp +++ b/unit-tests/varname-dot-make-level.exp @@ -5,6 +5,7 @@ level 3: variable 2, env 3 ok : set-env-different make: Cannot override read-only global variable ".MAKE.LEVEL.ENV" with a command line variable + in make[1] in directory "<curdir>" make: Fatal errors encountered -- cannot continue make: stopped making "ok" in unit-tests set-env-different: exit 1 diff --git a/unit-tests/varname-dot-makeflags.mk b/unit-tests/varname-dot-makeflags.mk index ffb09decb70e..87e4e4443cd2 100644 --- a/unit-tests/varname-dot-makeflags.mk +++ b/unit-tests/varname-dot-makeflags.mk @@ -1,4 +1,4 @@ -# $NetBSD: varname-dot-makeflags.mk,v 1.8 2023/06/01 20:56:35 rillig Exp $ +# $NetBSD: varname-dot-makeflags.mk,v 1.11 2025/05/20 17:56:40 sjg Exp $ # # Tests for the special .MAKEFLAGS variable, which collects almost all # command line arguments and passes them on to any child processes via diff --git a/unit-tests/varname-dot-newline.exp b/unit-tests/varname-dot-newline.exp index 6358d076ed5f..12114c9f24fd 100644 --- a/unit-tests/varname-dot-newline.exp +++ b/unit-tests/varname-dot-newline.exp @@ -1,9 +1,9 @@ make: varname-dot-newline.mk:28: Cannot overwrite ".newline" as it is read-only - in directory <curdir> + in make[1] in directory "<curdir>" make: varname-dot-newline.mk:30: Cannot append to ".newline" as it is read-only - in directory <curdir> + in make[1] in directory "<curdir>" make: varname-dot-newline.mk:32: Cannot delete ".newline" as it is read-only - in directory <curdir> + in make[1] in directory "<curdir>" make: Fatal errors encountered -- cannot continue make: stopped making "try-to-modify" in unit-tests first diff --git a/unit-tests/varname-make_stack_trace.exp b/unit-tests/varname-make_stack_trace.exp new file mode 100644 index 000000000000..c0f46cc5aa1e --- /dev/null +++ b/unit-tests/varname-make_stack_trace.exp @@ -0,0 +1,40 @@ +make: Unknown modifier ":Z" + while evaluating "${:Z}" with value "" + in command "@echo ${:Z}" + in target "provoke-error" + in make[2] in directory "<curdir>" +*** Error code 2 (continuing) + +Stop. +make: stopped making "disabled-compat" in unit-tests +make: Unknown modifier ":Z" + while evaluating "${:Z}" with value "" + in command "@echo ${:Z}" + in target "provoke-error" + in make[2] in directory "<curdir>" +*** [disabled-parallel] Error code 2 + +make: stopped making "disabled-parallel" in unit-tests +make: Unknown modifier ":Z" + while evaluating "${:Z}" with value "" + in command "@echo ${:Z}" + in target "provoke-error" + in make[2] in directory "<curdir>" + in command "make -f varname-make_stack_trace.mk provoke-error" + in target "enabled-compat" + in make[1] in directory "<curdir>" +*** Error code 2 (continuing) + +Stop. +make: stopped making "enabled-compat" in unit-tests +make: Unknown modifier ":Z" + while evaluating "${:Z}" with value "" + in command "@echo ${:Z}" + in target "provoke-error" + in make[2] in directory "<curdir>" + in target "enabled-parallel" + in make[1] in directory "<curdir>" +*** [enabled-parallel] Error code 2 + +make: stopped making "enabled-parallel" in unit-tests +exit status 0 diff --git a/unit-tests/varname-make_stack_trace.mk b/unit-tests/varname-make_stack_trace.mk new file mode 100644 index 000000000000..cba02559bafe --- /dev/null +++ b/unit-tests/varname-make_stack_trace.mk @@ -0,0 +1,37 @@ +# $NetBSD: varname-make_stack_trace.mk,v 1.1 2025/06/13 03:51:18 rillig Exp $ +# +# Tests for the MAKE_STACK_TRACE environment variable, which controls whether +# to print inter-process stack traces that are useful to narrow down where an +# erroneous expression comes from. +# +# While inter-process stack traces are useful to narrow down errors, they are +# disabled by default since the stack trace is stored in an environment +# variable and a stack trace can grow large depending on the shell commands in +# the sub-make processes. The space used for the stack traces would compete +# with the space for the command line arguments, and long command lines are +# already written to a temporary file by Cmd_Exec to not overwhelm this space. + +all: .PHONY + @${MAKE} -f ${MAKEFILE} disabled-compat || : + @${MAKE} -f ${MAKEFILE} -j1 disabled-parallel || : + @MAKE_STACK_TRACE=yes ${MAKE} -f ${MAKEFILE} enabled-compat || : + @MAKE_STACK_TRACE=yes ${MAKE} -f ${MAKEFILE} -j1 enabled-parallel || : + +# expect-not: in target "disabled-compat" +disabled-compat: .PHONY + @${MAKE} -f ${MAKEFILE} provoke-error + +# expect-not: in target "disabled-parallel" +disabled-parallel: .PHONY + @${MAKE} -f ${MAKEFILE} provoke-error + +# expect: in target "enabled-compat" +enabled-compat: .PHONY + @${MAKE} -f ${MAKEFILE} provoke-error + +# expect: in target "enabled-parallel" +enabled-parallel: .PHONY + @${MAKE} -f ${MAKEFILE} provoke-error + +provoke-error: .PHONY + @echo ${:Z} diff --git a/unit-tests/varname.exp b/unit-tests/varname.exp index 4886477c6125..d173d2025e9f 100644 --- a/unit-tests/varname.exp +++ b/unit-tests/varname.exp @@ -5,17 +5,26 @@ Var_Parse: ${VARNAME} (eval) Global: VAR((( = 3 open parentheses Var_Parse: ${VAR(((}}}}" != "3 open parentheses}}}" (eval) Global: .ALLTARGETS = VAR(((=) -make: varname.mk:32: Missing ')' in archive specification -make: varname.mk:32: Error in archive specification: "VAR" +make: varname.mk:31: Missing ')' in archive specification +make: varname.mk:31: Error in archive specification: "VAR" Var_Parse: ${:UVAR\(\(\(}= try2 (eval) Evaluating modifier ${:U...} on value "" (eval, undefined) Result of ${:UVAR\(\(\(} is "VAR\(\(\(" (eval, defined) Global: .ALLTARGETS = VAR(((=) VAR\(\(\(= -make: varname.mk:38: Invalid line '${:UVAR\(\(\(}= try2', expanded to 'VAR\(\(\(= try2' +make: varname.mk:37: Invalid line '${:UVAR\(\(\(}= try2', expanded to 'VAR\(\(\(= try2' Var_Parse: ${VARNAME} (eval) Global: VAR((( = try3 Global: .MAKEFLAGS = -r -k -d v -d Global: .MAKEFLAGS = -r -k -d v -d 0 +make: varname.mk:98: warning: Invalid character " " in variable name "if ,yes,no" +make: varname.mk:113: warning: Invalid character " " in variable name "if ,yes,no" +make: varname.mk:120: warning: Invalid character " " in variable name "if ,," +make: varname.mk:128: Unknown modifier ":yes,answer" + while evaluating variable "if ,answer" with value "" + while evaluating variable "GNU_MAKE_IF_MODIFIER" with value "$(if ${HAVE_STRLEN},answer:yes,answer:no)" +make: varname.mk:138: warning: Invalid character "\x09" in variable name "a b" +make: varname.mk:138: Variable "a b" is undefined +make: varname.mk:144: Variable "ÄÖÜ" is undefined make: Fatal errors encountered -- cannot continue -make: stopped making "all" in unit-tests +make: stopped in unit-tests exit status 1 diff --git a/unit-tests/varname.mk b/unit-tests/varname.mk index d2c36afb3aa2..ae18819aa19e 100644 --- a/unit-tests/varname.mk +++ b/unit-tests/varname.mk @@ -1,12 +1,11 @@ -# $NetBSD: varname.mk,v 1.16 2025/01/11 20:16:40 rillig Exp $ +# $NetBSD: varname.mk,v 1.17 2025/06/12 04:33:00 rillig Exp $ # -# Tests for special variables, such as .MAKE or .PARSEDIR. -# And for variable names in general. +# Tests for variable names. .MAKEFLAGS: -dv -# In variable names, braces are allowed, but they must be balanced. -# Parentheses and braces may be mixed. +# In a variable assignment, braces are allowed in the variable name, but they +# must be balanced. Parentheses and braces may be mixed. VAR{{{}}}= 3 braces .if "${VAR{{{}}}}" != "3 braces" . error @@ -86,4 +85,61 @@ ASDZguv.param= once . error .endif -all: + +# Warn about expressions in the style of GNU make, as these would silently +# expand to an empty string instead. +# +# https://pubs.opengroup.org/onlinepubs/9799919799/utilities/make.html says: +# a macro name shall not contain an <equals-sign>, <blank>, or control +# character. +# +GNU_MAKE_IF= $(if ${HAVE_STRLEN},yes,no) +# expect+1: warning: Invalid character " " in variable name "if ,yes,no" +.if ${GNU_MAKE_IF} != "" +. error +.endif +# +# This requirement needs to be ignored for expressions with a ":L" or ":?:" +# modifier, as these modifiers rely on arbitrary characters in the expression +# name. +.if ${"left" == "right":?equal:unequal} != "unequal" +. error +.endif +# +# In fact, this requirement is ignored for any expression that has a modifier. +# In this indirect case, though, the expression with the space in the name is +# a nested expression, so the ":U" modifier doesn't affect the warning. +# expect+1: warning: Invalid character " " in variable name "if ,yes,no" +.if ${GNU_MAKE_IF:Ufallback} != "" +. error +.endif +# +# A modifier in a nested expression does not affect the warning. +GNU_MAKE_IF_EXPR= $(if ${HAVE_STRLEN},${HEADERS:.h=.c},) +# expect+1: warning: Invalid character " " in variable name "if ,," +.if ${GNU_MAKE_IF_EXPR} != "" +. error +.endif +# +# When the GNU make expression contains a colon, chances are good that the +# colon is interpreted as an unknown modifier. +GNU_MAKE_IF_MODIFIER= $(if ${HAVE_STRLEN},answer:yes,answer:no) +# expect+1: Unknown modifier ":yes,answer" +.if ${GNU_MAKE_IF_MODIFIER} != "no)" +. error +.endif +# +# If the variable name contains a non-printable character, the warning +# contains the numeric character value instead, to prevent control sequences +# in the output. +CONTROL_CHARACTER= ${:U a b:ts\t} +# expect+2: warning: Invalid character "\x09" in variable name "a b" +# expect+1: Variable "a b" is undefined +.if ${${CONTROL_CHARACTER}} != "" +.endif +# +# For now, only whitespace generates a warning, non-ASCII characters don't. +UMLAUT= ÄÖÜ +# expect+1: Variable "ÄÖÜ" is undefined +.if ${${UMLAUT}} != "" +.endif diff --git a/unit-tests/varparse-errors.mk b/unit-tests/varparse-errors.mk index 921f2229f102..2d1142fbb65c 100644 --- a/unit-tests/varparse-errors.mk +++ b/unit-tests/varparse-errors.mk @@ -1,4 +1,4 @@ -# $NetBSD: varparse-errors.mk,v 1.24 2025/03/30 09:51:51 rillig Exp $ +# $NetBSD: varparse-errors.mk,v 1.25 2025/05/03 08:18:33 rillig Exp $ # Tests for parsing and evaluating all kinds of expressions. # @@ -122,6 +122,6 @@ UNCLOSED:= ${:U:localtime _!= echo '.info $${VALUE}' > varparse-errors.tmp VALUE= ${INDIRECT} INDIRECT= ${:Z} -# The "${.OBJDIR}/" is necessary to skip the directory cache. +# The "${.OBJDIR}/" is necessary to bypass the directory cache. .include "${.OBJDIR}/varparse-errors.tmp" _!= rm -f varparse-errors.tmp @@ -1,4 +1,4 @@ -/* $NetBSD: var.c,v 1.1159 2025/04/04 18:57:01 rillig Exp $ */ +/* $NetBSD: var.c,v 1.1168 2025/06/13 18:31:08 rillig Exp $ */ /* * Copyright (c) 1988, 1989, 1990, 1993 @@ -143,7 +143,7 @@ #endif /* "@(#)var.c 8.3 (Berkeley) 3/19/94" */ -MAKE_RCSID("$NetBSD: var.c,v 1.1159 2025/04/04 18:57:01 rillig Exp $"); +MAKE_RCSID("$NetBSD: var.c,v 1.1168 2025/06/13 18:31:08 rillig Exp $"); /* * Variables are defined using one of the VAR=value assignments. Their @@ -271,6 +271,7 @@ typedef struct SepBuf { } SepBuf; typedef enum { + VSK_MAKEFLAGS, VSK_TARGET, VSK_COMMAND, VSK_VARNAME, @@ -378,7 +379,13 @@ EvalStack_Push(EvalStackElementKind kind, const char *str, const FStr *value) evalStack.len++; } -static void +void +EvalStack_PushMakeflags(const char *makeflags) +{ + EvalStack_Push(VSK_MAKEFLAGS, makeflags, NULL); +} + +void EvalStack_Pop(void) { assert(evalStack.len > 0); @@ -386,12 +393,13 @@ EvalStack_Pop(void) } bool -EvalStack_PrintDetails(void) +EvalStack_Details(Buffer *buf) { size_t i; for (i = evalStack.len; i > 0; i--) { static const char descr[][42] = { + "while evaluating MAKEFLAGS", "in target", "in command", "while evaluating variable", @@ -408,9 +416,15 @@ EvalStack_PrintDetails(void) && (kind == VSK_VARNAME || kind == VSK_EXPR) ? elem->value->str : NULL; - debug_printf("\t%s \"%s%s%s\"\n", descr[kind], elem->str, - value != NULL ? "\" with value \"" : "", - value != NULL ? value : ""); + Buf_AddStr(buf, "\t"); + Buf_AddStr(buf, descr[kind]); + Buf_AddStr(buf, " \""); + Buf_AddStr(buf, elem->str); + if (value != NULL) { + Buf_AddStr(buf, "\" with value \""); + Buf_AddStr(buf, value); + } + Buf_AddStr(buf, "\"\n"); } return evalStack.len > 0; } @@ -466,7 +480,7 @@ CanonicalVarname(Substring name) } static Var * -GNode_FindVar(GNode *scope, Substring varname, unsigned int hash) +GNode_FindVar(GNode *scope, Substring varname, unsigned hash) { return HashTable_FindValueBySubstringHash(&scope->vars, varname, hash); } @@ -488,7 +502,7 @@ static Var * VarFindSubstring(Substring name, GNode *scope, bool elsewhere) { Var *var; - unsigned int nameHash; + unsigned nameHash; /* Replace '.TARGET' with '@', likewise for other local variables. */ name = CanonicalVarname(name); @@ -1600,7 +1614,7 @@ static void RegexReplaceBackref(char ref, SepBuf *buf, const char *wp, const regmatch_t *m, size_t nsub) { - unsigned int n = (unsigned)ref - '0'; + unsigned n = (unsigned)ref - '0'; if (n >= nsub) Parse_Error(PARSE_FATAL, "No subexpression \\%u", n); @@ -2868,7 +2882,7 @@ ModifyWord_Mtime(Substring word, SepBuf *buf, void *data) if (stat(word.start, &st) < 0) { if (args->error) { Parse_Error(PARSE_FATAL, - "Cannot determine mtime for '%s': %s", + "Cannot determine mtime for \"%s\": %s", word.start, strerror(errno)); args->rc = AMR_CLEANUP; return; @@ -3481,7 +3495,7 @@ ApplyModifier_IfElse(const char **pp, ModChain *ch) VarEvalMode then_emode = VARE_PARSE; VarEvalMode else_emode = VARE_PARSE; - int parseErrorsBefore = parseErrors, parseErrorsAfter = parseErrors; + int parseErrorsBefore = parseErrors; CondResult cond_rc = CR_TRUE; /* anything other than CR_ERROR */ if (Expr_ShouldEval(expr)) { @@ -3489,9 +3503,10 @@ ApplyModifier_IfElse(const char **pp, ModChain *ch) cond_rc = Cond_EvalCondition(expr->name); if (cond_rc == CR_TRUE) then_emode = expr->emode; - if (cond_rc == CR_FALSE) + else if (cond_rc == CR_FALSE) else_emode = expr->emode; - parseErrorsAfter = parseErrors; + else if (parseErrors == parseErrorsBefore) + Parse_Error(PARSE_FATAL, "Bad condition"); } evalStack.elems[evalStack.len - 1].kind = VSK_COND_THEN; @@ -3510,9 +3525,6 @@ ApplyModifier_IfElse(const char **pp, ModChain *ch) (*pp)--; /* Go back to the ch->endc. */ if (cond_rc == CR_ERROR) { - evalStack.elems[evalStack.len - 1].kind = VSK_COND; - if (parseErrorsAfter == parseErrorsBefore) - Parse_Error(PARSE_FATAL, "Bad condition"); LazyBuf_Done(&thenBuf); LazyBuf_Done(&elseBuf); return AMR_CLEANUP; @@ -3578,9 +3590,9 @@ found_op: /* Take a guess at where the modifier ends. */ Parse_Error(PARSE_FATAL, "Invalid attempt to assign \"%.*s\" to variable \"\" " - "via modifier \"::%.*s\"", + "via modifier \":%.*s\"", (int)strcspn(value, ":)}"), value, - (int)(value - op), op); + (int)(value - mod), mod); return AMR_CLEANUP; } @@ -3693,15 +3705,9 @@ ApplyModifier_Unique(const char **pp, ModChain *ch) if (words.len > 1) { size_t di, si; - - di = 0; - for (si = 1; si < words.len; si++) { - if (!Substring_Eq(words.words[si], words.words[di])) { - di++; - if (di != si) - words.words[di] = words.words[si]; - } - } + for (di = 0, si = 1; si < words.len; si++) + if (!Substring_Eq(words.words[di], words.words[si])) + words.words[++di] = words.words[si]; words.len = di + 1; } @@ -4358,6 +4364,25 @@ EvalUndefined(bool dynamic, const char *start, const char *p, ? var_Error : varUndefined); } +static void +CheckVarname(Substring name) +{ + const char *p; + + for (p = name.start; p < name.end; p++) { + if (ch_isspace(*p)) + break; + } + if (p < name.end) { + Parse_Error(PARSE_WARNING, + ch_isprint(*p) + ? "Invalid character \"%c\" in variable name \"%.*s\"" + : "Invalid character \"\\x%02x\" in variable name \"%.*s\"", + (int)(*p), + (int)Substring_Length(name), name.start); + } +} + /* * Parse a long variable name enclosed in braces or parentheses such as $(VAR) * or ${VAR}, up to the closing brace or parenthesis, or in the case of @@ -4435,6 +4460,7 @@ ParseVarnameLong( (scope == SCOPE_CMDLINE || scope == SCOPE_GLOBAL); if (!haveModifier) { + CheckVarname(name); p++; /* skip endc */ *out_false_pp = p; *out_false_val = EvalUndefined(dynamic, start, p, @@ -4674,6 +4700,8 @@ Var_Parse(const char **pp, GNode *scope, VarEvalMode emode) Expr_SetValue(&expr, value); } + EvalStack_Pop(); + if (v->shortLived) { if (expr.value.str == v->val.data) { /* move ownership */ @@ -4683,7 +4711,6 @@ Var_Parse(const char **pp, GNode *scope, VarEvalMode emode) VarFreeShortLived(v); } - EvalStack_Pop(); return expr.value; } @@ -4788,6 +4815,29 @@ Var_SubstInTarget(const char *str, GNode *scope) } void +Var_ExportStackTrace(const char *target, const char *cmd) +{ + char *stackTrace; + + if (GetParentStackTrace() == NULL) + return; + + if (target != NULL) + EvalStack_Push(VSK_TARGET, target, NULL); + if (cmd != NULL) + EvalStack_Push(VSK_COMMAND, cmd, NULL); + + stackTrace = GetStackTrace(true); + (void)setenv("MAKE_STACK_TRACE", stackTrace, 1); + free(stackTrace); + + if (cmd != NULL) + EvalStack_Pop(); + if (target != NULL) + EvalStack_Pop(); +} + +void Var_Expand(FStr *str, GNode *scope, VarEvalMode emode) { char *expanded; |