From 136a613cce6e119ba5e0a66767c041ce58f7e8c5 Mon Sep 17 00:00:00 2001 From: Guillaume Girol Date: Sun, 25 Jul 2021 12:00:00 +0000 Subject: [PATCH 001/117] nix-channel: use nix-env -i --remove-all to upgrade Fixes #4110 --- src/nix-channel/nix-channel.cc | 54 ++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 3272c6125..eae15885a 100755 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -88,15 +88,6 @@ static void update(const StringSet & channelNames) for (const auto & channel : channels) { auto name = channel.first; auto url = channel.second; - if (!(channelNames.empty() || channelNames.count(name))) - continue; - - // We want to download the url to a file to see if it's a tarball while also checking if we - // got redirected in the process, so that we can grab the various parts of a nix channel - // definition from a consistent location if the redirect changes mid-download. - auto result = fetchers::downloadFile(store, url, std::string(baseNameOf(url)), false); - auto filename = store->toRealPath(result.storePath); - url = result.effectiveUrl; // If the URL contains a version number, append it to the name // attribute (so that "nix-env -q" on the channels profile @@ -109,30 +100,43 @@ static void update(const StringSet & channelNames) std::string extraAttrs; - bool unpacked = false; - if (std::regex_search(filename, std::regex("\\.tar\\.(gz|bz2|xz)$"))) { - runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import " + unpackChannelPath + - "{ name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; }" }); - unpacked = true; - } + if (!(channelNames.empty() || channelNames.count(name))) { + // no need to update this channel, reuse the existing store path + Path symlink = profile + "/" + name; + Path storepath = dirOf(readLink(symlink)); + exprs.push_back("f: rec { name = \"" + cname + "\"; type = \"derivation\"; outputs = [\"out\"]; system = \"builtin\"; outPath = builtins.storePath \"" + storepath + "\"; out = { inherit outPath; };}"); + } else { + // We want to download the url to a file to see if it's a tarball while also checking if we + // got redirected in the process, so that we can grab the various parts of a nix channel + // definition from a consistent location if the redirect changes mid-download. + auto result = fetchers::downloadFile(store, url, std::string(baseNameOf(url)), false); + auto filename = store->toRealPath(result.storePath); + url = result.effectiveUrl; - if (!unpacked) { - // Download the channel tarball. - try { - filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz", false).storePath); - } catch (FileTransferError & e) { - filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2", false).storePath); + bool unpacked = false; + if (std::regex_search(filename, std::regex("\\.tar\\.(gz|bz2|xz)$"))) { + runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import " + unpackChannelPath + + "{ name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; }" }); + unpacked = true; } - } - // Regardless of where it came from, add the expression representing this channel to accumulated expression - exprs.push_back("f: f { name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; " + extraAttrs + " }"); + if (!unpacked) { + // Download the channel tarball. + try { + filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz", false).storePath); + } catch (FileTransferError & e) { + filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2", false).storePath); + } + } + // Regardless of where it came from, add the expression representing this channel to accumulated expression + exprs.push_back("f: f { name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; " + extraAttrs + " }"); + } } // Unpack the channel tarballs into the Nix store and install them // into the channels profile. std::cerr << "unpacking channels...\n"; - Strings envArgs{ "--profile", profile, "--file", unpackChannelPath, "--install", "--from-expression" }; + Strings envArgs{ "--profile", profile, "--file", unpackChannelPath, "--install", "--remove-all", "--from-expression" }; for (auto & expr : exprs) envArgs.push_back(std::move(expr)); envArgs.push_back("--quiet"); From 8438114399ce025b6977215dbaedc05697a6d958 Mon Sep 17 00:00:00 2001 From: Magic_RB Date: Mon, 3 May 2021 09:54:31 +0200 Subject: [PATCH 002/117] Add ignored_acls setting Signed-off-by: Magic_RB --- src/libstore/globals.hh | 9 +++++++++ src/libstore/local-store.cc | 4 +--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 8784d5faf..7e01b4960 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -805,6 +805,15 @@ public: may be useful in certain scenarios (e.g. to spin up containers or set up userspace network interfaces in tests). )"}; + + Setting ignoredAcls{ + this, {"security.selinux"}, "ignored-acls", + R"( + A list of ACLs that should be ignored, normally Nix attempts to + remove all ACLs from files and directories in the Nix store, but + some ACLs like `security.selinux` or `system.nfs4_acl` can't be + removed even by root. Therefore it's best to just ignore them. + )"}; #endif Setting hashedMirrors{ diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 747eb205e..c6f774bc2 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -583,9 +583,7 @@ static void canonicalisePathMetaData_(const Path & path, uid_t fromUid, InodesSe throw SysError("querying extended attributes of '%s'", path); for (auto & eaName: tokenizeString(std::string(eaBuf.data(), eaSize), std::string("\000", 1))) { - /* Ignore SELinux security labels since these cannot be - removed even by root. */ - if (eaName == "security.selinux") continue; + if (settings.ignoredAcls.get().count(eaName)) continue; if (lremovexattr(path.c_str(), eaName.c_str()) == -1) throw SysError("removing extended attribute '%s' from '%s'", eaName, path); } From 374fe49ff78c13457c6cfe396f9ed0cb986c903b Mon Sep 17 00:00:00 2001 From: Michael Bishop Date: Sun, 19 Sep 2021 23:07:10 -0300 Subject: [PATCH 003/117] set the PER_LINUX32 personality flag, when building for armv6l-linux or armv7l-linux this prevents 32bit builds from detecting a 64bit kernel and picking the wrong target --- src/libstore/build/local-derivation-goal.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 518edae9c..21d7d7408 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1778,6 +1778,10 @@ void LocalDerivationGoal::runChild() if (personality(PER_LINUX32) == -1) throw SysError("cannot set i686-linux personality"); } + if (drv->platform == "armv7l-linux" || drv->platform == "armv6l-linux") { + if (personality(PER_LINUX32) == -1) + throw SysError("cannot set 32bit linux personality"); + } /* Impersonate a Linux 2.6 machine to get some determinism in builds that depend on the kernel version. */ From 9d67332e4b8d75a101b8cf6e1d7e4dc8e99e964a Mon Sep 17 00:00:00 2001 From: Geoff Reedy Date: Wed, 22 Sep 2021 20:30:31 -0600 Subject: [PATCH 004/117] Better eval error locations for interpolation and + Previously, type or coercion errors for string interpolation, path interpolation, and plus expressions were always reported at the beginning of the outer expression. This leads to confusing evaluation error messages making it hard to accurately diagnose and then fix the error. For example, errors were reported as follows. ``` cannot coerce an integer to a string 1| let foo = 7; in "bar" + foo | ^ cannot add a string to an integer 1| let foo = "bar"; in 4 + foo | ^ cannot coerce an integer to a string 1| let foo = 7; in "x${foo}" | ^ ``` This commit changes the ExprConcatStrings expression vector to store a sequence of expressions *and* their expansion locations so that error locations can be reported accurately. For interpolation, the error is reported at the beginning of the entire `${foo}`, not at the beginning of `foo` because I thought this was slightly clearer. The previous errors are now reported as: ``` cannot coerce an integer to a string 1| let foo = 7; in "bar" + foo | ^ cannot add a string to an integer 1| let foo = "bar"; in 4 + foo | ^ cannot coerce an integer to a string 1| let foo = 7; in "x${foo}" | ^ ``` The error is reported at this kind of precise location even for multi-line indented strings. This probably helps with at least some of the cases mentioned in #561 --- src/libexpr/eval.cc | 8 ++++---- src/libexpr/nixexpr.cc | 4 ++-- src/libexpr/nixexpr.hh | 4 ++-- src/libexpr/parser.y | 38 +++++++++++++++++++------------------- 4 files changed, 27 insertions(+), 27 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index bc41a2cd9..3f18ad1ed 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1577,7 +1577,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) bool first = !forceString; ValueType firstType = nString; - for (auto & i : *es) { + for (auto & [i_pos, i] : *es) { Value vTmp; i->eval(state, env, vTmp); @@ -1598,19 +1598,19 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) nf = n; nf += vTmp.fpoint; } else - throwEvalError(pos, "cannot add %1% to an integer", showType(vTmp)); + throwEvalError(i_pos, "cannot add %1% to an integer", showType(vTmp)); } else if (firstType == nFloat) { if (vTmp.type() == nInt) { nf += vTmp.integer; } else if (vTmp.type() == nFloat) { nf += vTmp.fpoint; } else - throwEvalError(pos, "cannot add %1% to a float", showType(vTmp)); + throwEvalError(i_pos, "cannot add %1% to a float", showType(vTmp)); } else /* skip canonization of first path, which would only be not canonized in the first place if it's coming from a ./${foo} type path */ - s << state.coerceToString(pos, vTmp, context, false, firstType == nString, !first); + s << state.coerceToString(i_pos, vTmp, context, false, firstType == nString, !first); first = false; } diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 492b819e7..15d963d55 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -181,7 +181,7 @@ void ExprConcatStrings::show(std::ostream & str) const str << "("; for (auto & i : *es) { if (first) first = false; else str << " + "; - str << *i; + str << i.second; } str << ")"; } @@ -413,7 +413,7 @@ void ExprOpNot::bindVars(const StaticEnv & env) void ExprConcatStrings::bindVars(const StaticEnv & env) { for (auto & i : *es) - i->bindVars(env); + i.second->bindVars(env); } void ExprPos::bindVars(const StaticEnv & env) diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 51a14cd59..d3f7f5bd6 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -321,8 +321,8 @@ struct ExprConcatStrings : Expr { Pos pos; bool forceString; - vector * es; - ExprConcatStrings(const Pos & pos, bool forceString, vector * es) + vector > * es; + ExprConcatStrings(const Pos & pos, bool forceString, vector > * es) : pos(pos), forceString(forceString), es(es) { }; COMMON_METHODS }; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index e3749783a..c21d7b033 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -154,7 +154,7 @@ static void addFormal(const Pos & pos, Formals * formals, const Formal & formal) } -static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector & es) +static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector > & es) { if (es.empty()) return new ExprString(symbols.create("")); @@ -164,7 +164,7 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector(i); if (!e) { /* Anti-quotations end the current start-of-line whitespace. */ @@ -194,12 +194,12 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector * es2 = new vector; + vector > * es2 = new vector >; atStartOfLine = true; size_t curDropped = 0; size_t n = es.size(); - for (vector::iterator i = es.begin(); i != es.end(); ++i, --n) { - ExprIndStr * e = dynamic_cast(*i); + for (vector >::iterator i = es.begin(); i != es.end(); ++i, --n) { + ExprIndStr * e = dynamic_cast(i->second); if (!e) { atStartOfLine = false; curDropped = 0; @@ -236,11 +236,11 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vectorpush_back(new ExprString(symbols.create(s2))); + es2->emplace_back(i->first, new ExprString(symbols.create(s2))); } /* If this is a single string, then don't do a concatenation. */ - return es2->size() == 1 && dynamic_cast((*es2)[0]) ? (*es2)[0] : new ExprConcatStrings(pos, true, es2); + return es2->size() == 1 && dynamic_cast((*es2)[0].second) ? (*es2)[0].second : new ExprConcatStrings(pos, true, es2); } @@ -279,7 +279,7 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err char * path; char * uri; std::vector * attrNames; - std::vector * string_parts; + std::vector > * string_parts; } %type start expr expr_function expr_if expr_op @@ -366,7 +366,7 @@ expr_op | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(CUR_POS, $1, $3); } | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, *$3); } | expr_op '+' expr_op - { $$ = new ExprConcatStrings(CUR_POS, false, new vector({$1, $3})); } + { $$ = new ExprConcatStrings(CUR_POS, false, new vector >({{makeCurPos(@1, data), $1}, {makeCurPos(@3, data), $3}})); } | expr_op '-' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__sub")), $1), $3); } | expr_op '*' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__mul")), $1), $3); } | expr_op '/' expr_op { $$ = new ExprApp(CUR_POS, new ExprApp(new ExprVar(data->symbols.create("__div")), $1), $3); } @@ -407,7 +407,7 @@ expr_simple } | path_start PATH_END { $$ = $1; } | path_start string_parts_interpolated PATH_END { - $2->insert($2->begin(), $1); + $2->insert($2->begin(), {makeCurPos(@1, data), $1}); $$ = new ExprConcatStrings(CUR_POS, false, $2); } | SPATH { @@ -445,13 +445,13 @@ string_parts ; string_parts_interpolated - : string_parts_interpolated STR { $$ = $1; $1->push_back($2); } - | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->push_back($3); } - | DOLLAR_CURLY expr '}' { $$ = new vector; $$->push_back($2); } + : string_parts_interpolated STR { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $2); } + | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $3); } + | DOLLAR_CURLY expr '}' { $$ = new vector >; $$->emplace_back(makeCurPos(@1, data), $2); } | STR DOLLAR_CURLY expr '}' { - $$ = new vector; - $$->push_back($1); - $$->push_back($3); + $$ = new vector >; + $$->emplace_back(makeCurPos(@1, data), $1); + $$->emplace_back(makeCurPos(@2, data), $3); } ; @@ -470,9 +470,9 @@ path_start ; ind_string_parts - : ind_string_parts IND_STR { $$ = $1; $1->push_back($2); } - | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->push_back($3); } - | { $$ = new vector; } + : ind_string_parts IND_STR { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $2); } + | ind_string_parts DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(makeCurPos(@2, data), $3); } + | { $$ = new vector >; } ; binds From d1da45855c0f7d71bb13928f8028937d3fa062d9 Mon Sep 17 00:00:00 2001 From: Arthur Gautier Date: Wed, 13 Oct 2021 17:10:35 +0000 Subject: [PATCH 005/117] preloadNSS: Drop the dns query workaround We can actually just load nss ourselves and call in nss to configure it and we don't need to run a dummy query entirely to have nss load nss_dns as a side-effect. Signed-off-by: Arthur Gautier --- src/libmain/shared.cc | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 85f9f0d58..93775bed1 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -15,9 +15,9 @@ #include #include #include -#include -#include -#include +#include +#include +#include #include @@ -121,21 +121,8 @@ static void preloadNSS() { been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to load its lookup libraries in the parent before any child gets a chance to. */ std::call_once(dns_resolve_flag, []() { - struct addrinfo *res = NULL; - - /* nss will only force the "local" (not through nscd) dns resolution if its on the LOCALDOMAIN. - We need the resolution to be done locally, as nscd socket will not be accessible in the - sandbox. */ - char * previous_env = getenv("LOCALDOMAIN"); - setenv("LOCALDOMAIN", "invalid", 1); - if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) == 0) { - if (res) freeaddrinfo(res); - } - if (previous_env) { - setenv("LOCALDOMAIN", previous_env, 1); - } else { - unsetenv("LOCALDOMAIN"); - } + dlopen (LIBNSS_DNS_SO, RTLD_NOW); + __nss_configure_lookup ("hosts", "dns"); }); } From 85717eff15fadd625014ed57de80f59304fa67db Mon Sep 17 00:00:00 2001 From: Arthur Gautier Date: Thu, 14 Oct 2021 21:43:07 +0000 Subject: [PATCH 006/117] preloadNSS: detect glibc Signed-off-by: Arthur Gautier --- src/libmain/shared.cc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 93775bed1..1fb2d62d3 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -15,9 +15,12 @@ #include #include #include +#include +#ifdef __GLIBC__ #include #include #include +#endif #include @@ -121,8 +124,10 @@ static void preloadNSS() { been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to load its lookup libraries in the parent before any child gets a chance to. */ std::call_once(dns_resolve_flag, []() { +#ifdef __GLIBC__ dlopen (LIBNSS_DNS_SO, RTLD_NOW); __nss_configure_lookup ("hosts", "dns"); +#endif }); } From ca8989daf36629ce47cf35d6e56f92d01f3172ec Mon Sep 17 00:00:00 2001 From: Arthur Gautier Date: Thu, 14 Oct 2021 21:54:29 +0000 Subject: [PATCH 007/117] preloadNSS: warn if unable to open nss backend Signed-off-by: Arthur Gautier --- src/libmain/shared.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 1fb2d62d3..da3bc3bb5 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -125,7 +125,9 @@ static void preloadNSS() { load its lookup libraries in the parent before any child gets a chance to. */ std::call_once(dns_resolve_flag, []() { #ifdef __GLIBC__ - dlopen (LIBNSS_DNS_SO, RTLD_NOW); + if (dlopen (LIBNSS_DNS_SO, RTLD_NOW) == NULL) { + printMsg(Verbosity::lvlWarn, fmt("Unable to load nss_dns backend")); + } __nss_configure_lookup ("hosts", "dns"); #endif }); From c345a4a1e8ad2c5274b06a72337b91a6d11c7f8a Mon Sep 17 00:00:00 2001 From: Arthur Gautier Date: Fri, 15 Oct 2021 07:33:49 +0000 Subject: [PATCH 008/117] fixup macos build Signed-off-by: Arthur Gautier --- src/libmain/shared.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index da3bc3bb5..5eb7871f3 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -15,7 +15,9 @@ #include #include #include +#ifdef __linux__ #include +#endif #ifdef __GLIBC__ #include #include From fa4abe46e2cedfacc90c82177671a3000b229f28 Mon Sep 17 00:00:00 2001 From: Arthur Gautier Date: Sat, 16 Oct 2021 01:39:14 +0000 Subject: [PATCH 009/117] preloadNSS: document the preload mechanism Signed-off-by: Arthur Gautier --- src/libmain/shared.cc | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 5eb7871f3..b6bfea8cb 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -127,6 +127,24 @@ static void preloadNSS() { load its lookup libraries in the parent before any child gets a chance to. */ std::call_once(dns_resolve_flag, []() { #ifdef __GLIBC__ + /* On linux, glibc will run every lookup through the nss layer. + * That means every lookup goes, by default, through nscd, which acts as a local + * cache. + * Because we run builds in a sandbox, we also remove access to nscd otherwise + * lookups would leak into the sandbox. + * + * But now we have a new problem, we need to make sure the nss_dns backend that + * does the dns lookups when nscd is not available is loaded or available. + * + * We can't make it available without leaking nix's environment, so instead we'll + * load the backend, and configure nss so it does not try to run dns lookups + * through nscd. + * + * This is technically only used for builtins:fetch* functions so we only care + * about dns. + * + * All other platforms are unaffected. + */ if (dlopen (LIBNSS_DNS_SO, RTLD_NOW) == NULL) { printMsg(Verbosity::lvlWarn, fmt("Unable to load nss_dns backend")); } From a2473823d748b0249804d393c739622146954d02 Mon Sep 17 00:00:00 2001 From: figsoda Date: Wed, 27 Oct 2021 14:18:30 -0400 Subject: [PATCH 010/117] run: use pname as a fallback for main program --- src/nix/app.cc | 3 +++ src/nix/run.md | 7 ++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/nix/app.cc b/src/nix/app.cc index 9719a65dd..2fcf4752c 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -83,11 +83,14 @@ UnresolvedApp Installable::toApp(EvalState & state) auto outPath = cursor->getAttr(state.sOutPath)->getString(); auto outputName = cursor->getAttr(state.sOutputName)->getString(); auto name = cursor->getAttr(state.sName)->getString(); + auto aPname = cursor->maybeGetAttr("pname"); auto aMeta = cursor->maybeGetAttr("meta"); auto aMainProgram = aMeta ? aMeta->maybeGetAttr("mainProgram") : nullptr; auto mainProgram = aMainProgram ? aMainProgram->getString() + : aPname + ? aPname->getString() : DrvName(name).name; auto program = outPath + "/bin/" + mainProgram; return UnresolvedApp { App { diff --git a/src/nix/run.md b/src/nix/run.md index a76750376..697db9a96 100644 --- a/src/nix/run.md +++ b/src/nix/run.md @@ -44,9 +44,10 @@ program specified by the app definition. If *installable* evaluates to a derivation, it will try to execute the program `/bin/`, where *out* is the primary output store path of the derivation and *name* is the `meta.mainProgram` attribute -of the derivation if it exists, and otherwise the name part of the -value of the `name` attribute of the derivation (e.g. if `name` is set -to `hello-1.10`, it will run `$out/bin/hello`). +of the derivation if it exists, and otherwise the `pname` attribute of +the derivation or the name part of the value of the `name` attribute of +the derivation (e.g. if `name` is set to `hello-1.10`, it will run +`$out/bin/hello`). # Flake output attributes From 69431edbc174c7644f7143ba5c81681b7ece186e Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Mon, 8 Nov 2021 17:03:09 -0600 Subject: [PATCH 011/117] installer: clarify starting assumption task We had a macOS user present in Matrix with some confusion because the lack of a clear task statement here made them think the error meant that a problem had occurred during the preceding task in a macOS install: "Fixing any leftover Nix volume state" --- scripts/install-multi-user.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 7d1cb8c5a..c91dd404a 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -377,6 +377,11 @@ cure_artifacts() { } validate_starting_assumptions() { + task "Checking for artifacts of previous installs" + cat < /dev/null >&2; then warning < Date: Sat, 9 Oct 2021 18:47:05 +0100 Subject: [PATCH 012/117] ca-specific-schema.sql: add index on RealisationsRefs(referrer) and (outputPath) For a typical desktop system (~2K packages) we can easily get 100K entries in RealisationsRefs. Without indices query for RealisationsRefs requires linear scan. RealisationsRefs(referrer) -------------------------- Inefficiency is seen as a 100% CPU load of nix-daemon for the following scenario: $ nix edit -f . bash # add unused environment variable, like FOO="1" # populate RealisationsRefs, build fresh system $ nix build -f nixos system --arg config '{ contentAddressedByDefault = true; }' $ nix edit -f . bash # add unused environment variable, like FOO="2" $ time nix build -f nixos system --arg config '{ contentAddressedByDefault = true; }' In this case `bash `will be rebuilt a few times and then rest of CPU time is spent on scanning RealisationsRefs table (about 5 CPU-minutes on my machine). Before the change: $ time nix build -f nixos system ... # step 4 above real 34m3,613s user 0m5,232s sys 0m0,758s Of all this time about 29.5 minutes are taken by nix-daemon's CPU time. After the change: $ time nix build -f nixos system ... # step 4 above real 4m50,061s user 0m5,038s sys 0m0,677s Of all this time about 1 minute is taken by nix-daemon's CPU time. Most of the time is spent polling for non-existent realisations on cache-nixos.org. Realisations(outputPath) ------------------------ After running CA system for two weeks I got ~1M entries in Realisations table. `nix-collect-garbage` became very slow (seemingly 100 path deletions per second). It happens due to a slow cascading delete from Realisations triggered by deletion from ValidPaths. The fix is to add an index on primary key from ValidPaths(id) that triggers cascading deletions. Before the change: $ time nix-collect-garbage -d --max-freed 100G real 23m32.411s user 17m49.679s sys 4m50.609s Most of time was spent in re-scanning Realisations table on each path deletion. After the change: $ time nix-collect-garbage -d --max-freed 100G real 8m43.226s user 6m16.317s sys 1m40.188s Time is spent scanning sqlite indices and in kernel when unlinking directories. --- src/libstore/ca-specific-schema.sql | 5 +++++ src/libstore/local-store.cc | 13 ++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/libstore/ca-specific-schema.sql b/src/libstore/ca-specific-schema.sql index 08af0cc1f..64cc97fde 100644 --- a/src/libstore/ca-specific-schema.sql +++ b/src/libstore/ca-specific-schema.sql @@ -19,3 +19,8 @@ create table if not exists RealisationsRefs ( foreign key (referrer) references Realisations(id) on delete cascade, foreign key (realisationReference) references Realisations(id) on delete restrict ); + +-- used by QueryRealisationReferences +create index if not exists IndexRealisationsRefs on RealisationsRefs(referrer); +-- used by cascade deletion when ValidPaths is deleted +create index if not exists IndexRealisationsRefsOnOutputPath on Realisations(outputPath); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index eecd407f5..fcddd1f8e 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -79,7 +79,7 @@ int getSchema(Path schemaPath) void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) { - const int nixCASchemaVersion = 2; + const int nixCASchemaVersion = 3; int curCASchema = getSchema(schemaPath); if (curCASchema != nixCASchemaVersion) { if (curCASchema > nixCASchemaVersion) { @@ -130,6 +130,17 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) txn.commit(); } + if (curCASchema < 3) { + SQLiteTxn txn(db); + // Apply new indices added in this schema update. + db.exec(R"( + -- used by QueryRealisationReferences + create index if not exists IndexRealisationsRefs on RealisationsRefs(referrer); + -- used by cascade deletion when ValidPaths is deleted + create index if not exists IndexRealisationsRefsOnOutputPath on Realisations(outputPath); + )"); + txn.commit(); + } writeFile(schemaPath, fmt("%d", nixCASchemaVersion)); lockFile(lockFd.get(), ltRead, true); } From 4b28798bfcb659bf83557a33546a03115f8bfed2 Mon Sep 17 00:00:00 2001 From: Alex Shabalin Date: Thu, 18 Nov 2021 14:32:52 +0100 Subject: [PATCH 013/117] Offer suggestions for nix-env -i Closes https://github.com/NixOS/nix/issues/972 --- src/libstore/names.cc | 2 +- src/libstore/names.hh | 2 +- src/nix-env/nix-env.cc | 174 ++++++++++++++++++++++++++--------------- 3 files changed, 113 insertions(+), 65 deletions(-) diff --git a/src/libstore/names.cc b/src/libstore/names.cc index ce808accc..54c95055d 100644 --- a/src/libstore/names.cc +++ b/src/libstore/names.cc @@ -42,7 +42,7 @@ DrvName::~DrvName() { } -bool DrvName::matches(DrvName & n) +bool DrvName::matches(const DrvName & n) { if (name != "*") { if (!regex) { diff --git a/src/libstore/names.hh b/src/libstore/names.hh index bc62aac93..3f861bc44 100644 --- a/src/libstore/names.hh +++ b/src/libstore/names.hh @@ -19,7 +19,7 @@ struct DrvName DrvName(std::string_view s); ~DrvName(); - bool matches(DrvName & n); + bool matches(const DrvName & n); private: std::unique_ptr regex; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 4056d973d..1c78b5307 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -224,6 +224,91 @@ static void checkSelectorUse(DrvNames & selectors) } +namespace { + +std::set searchByPrefix(const DrvInfos & allElems, std::string_view prefix) { + constexpr std::size_t maxResults = 3; + std::set result; + for (const auto & drvInfo : allElems) { + const auto drvName = DrvName { drvInfo.queryName() }; + if (hasPrefix(drvName.name, prefix)) { + result.emplace(drvName.name); + + if (result.size() >= maxResults) { + break; + } + } + } + return result; +} + +struct Match +{ + DrvInfo drvInfo; + std::size_t index; + + Match(DrvInfo drvInfo_, std::size_t index_) + : drvInfo{std::move(drvInfo_)} + , index{index_} + {} +}; + +/* If a selector matches multiple derivations + with the same name, pick the one matching the current + system. If there are still multiple derivations, pick the + one with the highest priority. If there are still multiple + derivations, pick the one with the highest version. + Finally, if there are still multiple derivations, + arbitrarily pick the first one. */ +std::vector pickNewestOnly(EvalState & state, std::vector matches) { + /* Map from package names to derivations. */ + std::map newest; + StringSet multiple; + + for (auto & match : matches) { + auto & oneDrv = match.drvInfo; + + const auto drvName = DrvName { oneDrv.queryName() }; + long comparison = 1; + + const auto itOther = newest.find(drvName.name); + + if (itOther != newest.end()) { + auto & newestDrv = itOther->second.drvInfo; + + comparison = + oneDrv.querySystem() == newestDrv.querySystem() ? 0 : + oneDrv.querySystem() == settings.thisSystem ? 1 : + newestDrv.querySystem() == settings.thisSystem ? -1 : 0; + if (comparison == 0) + comparison = comparePriorities(state, oneDrv, newestDrv); + if (comparison == 0) + comparison = compareVersions(drvName.version, DrvName { newestDrv.queryName() }.version); + } + + if (comparison > 0) { + newest.erase(drvName.name); + newest.emplace(drvName.name, match); + multiple.erase(drvName.fullName); + } else if (comparison == 0) { + multiple.insert(drvName.fullName); + } + } + + matches.clear(); + for (auto & [name, match] : newest) { + if (multiple.find(name) != multiple.end()) + printInfo( + "warning: there are multiple derivations named '%1%'; using the first one", + name); + matches.push_back(match); + } + + return matches; +} + +} // end namespace + static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems, const Strings & args, bool newestOnly) { @@ -232,79 +317,42 @@ static DrvInfos filterBySelector(EvalState & state, const DrvInfos & allElems, selectors.emplace_back("*"); DrvInfos elems; - set done; + std::set done; - for (auto & i : selectors) { - typedef list > Matches; - Matches matches; - unsigned int n = 0; - for (DrvInfos::const_iterator j = allElems.begin(); - j != allElems.end(); ++j, ++n) - { - DrvName drvName(j->queryName()); - if (i.matches(drvName)) { - i.hits++; - matches.push_back(std::pair(*j, n)); + for (auto & selector : selectors) { + std::vector matches; + for (const auto & [index, drvInfo] : enumerate(allElems)) { + const auto drvName = DrvName { drvInfo.queryName() }; + if (selector.matches(drvName)) { + ++selector.hits; + matches.emplace_back(drvInfo, index); } } - /* If `newestOnly', if a selector matches multiple derivations - with the same name, pick the one matching the current - system. If there are still multiple derivations, pick the - one with the highest priority. If there are still multiple - derivations, pick the one with the highest version. - Finally, if there are still multiple derivations, - arbitrarily pick the first one. */ if (newestOnly) { - - /* Map from package names to derivations. */ - typedef map > Newest; - Newest newest; - StringSet multiple; - - for (auto & j : matches) { - DrvName drvName(j.first.queryName()); - long d = 1; - - Newest::iterator k = newest.find(drvName.name); - - if (k != newest.end()) { - d = j.first.querySystem() == k->second.first.querySystem() ? 0 : - j.first.querySystem() == settings.thisSystem ? 1 : - k->second.first.querySystem() == settings.thisSystem ? -1 : 0; - if (d == 0) - d = comparePriorities(state, j.first, k->second.first); - if (d == 0) - d = compareVersions(drvName.version, DrvName(k->second.first.queryName()).version); - } - - if (d > 0) { - newest.erase(drvName.name); - newest.insert(Newest::value_type(drvName.name, j)); - multiple.erase(j.first.queryName()); - } else if (d == 0) { - multiple.insert(j.first.queryName()); - } - } - - matches.clear(); - for (auto & j : newest) { - if (multiple.find(j.second.first.queryName()) != multiple.end()) - printInfo( - "warning: there are multiple derivations named '%1%'; using the first one", - j.second.first.queryName()); - matches.push_back(j.second); - } + matches = pickNewestOnly(state, std::move(matches)); } /* Insert only those elements in the final list that we haven't inserted before. */ - for (auto & j : matches) - if (done.insert(j.second).second) - elems.push_back(j.first); - } + for (auto & match : matches) + if (done.insert(match.index).second) + elems.push_back(match.drvInfo); - checkSelectorUse(selectors); + if (selector.hits == 0 && selector.fullName != "*") { + const auto prefixHits = searchByPrefix(allElems, selector.name); + + if (prefixHits.empty()) { + throw Error("selector '%1%' matches no derivations", selector.fullName); + } else { + std::string suggestionMessage = ", maybe you meant:"; + for (const auto & drvName : prefixHits) { + suggestionMessage += fmt("\n%s", drvName); + } + throw Error("selector '%1%' matches no derivations" + suggestionMessage, selector.fullName); + } + } + } return elems; } From 86b79628073154b853b2e57360c062654826f5ee Mon Sep 17 00:00:00 2001 From: Alex Shabalin Date: Fri, 19 Nov 2021 16:29:55 +0100 Subject: [PATCH 014/117] Use warn to print a warning --- src/nix-env/nix-env.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 1c78b5307..490f450df 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -298,8 +298,8 @@ std::vector pickNewestOnly(EvalState & state, std::vector matches) matches.clear(); for (auto & [name, match] : newest) { if (multiple.find(name) != multiple.end()) - printInfo( - "warning: there are multiple derivations named '%1%'; using the first one", + warn( + "there are multiple derivations named '%1%'; using the first one", name); matches.push_back(match); } From d41af23a6c879fe2b5f88e40e0ad87db099a23cf Mon Sep 17 00:00:00 2001 From: Farid Zakaria Date: Thu, 18 Nov 2021 14:54:50 -0800 Subject: [PATCH 015/117] Fix heap use after free in progress-bar.cc Fix some heap-use-after-free in progress-bar.cc These are somewhat tricky failures here due to temporary variable creation and string_view --- src/libmain/progress-bar.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 63955eed1..f4306ab91 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -11,7 +11,7 @@ namespace nix { -static std::string getS(const std::vector & fields, size_t n) +static std::string_view getS(const std::vector & fields, size_t n) { assert(n < fields.size()); assert(fields[n].type == Logger::Field::tString); From 286eb81143908a49e96fa800e01b302d8f4b7370 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Fri, 19 Nov 2021 02:28:20 +0000 Subject: [PATCH 016/117] "recursive-nix" system feature only with experimental feature --- src/libstore/globals.cc | 2 +- src/libstore/store-api.cc | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 9f1a88130..81ca9cc0f 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -122,7 +122,7 @@ StringSet Settings::getDefaultSystemFeatures() /* For backwards compatibility, accept some "features" that are used in Nixpkgs to route builds to certain machines but don't actually require anything special on the machines. */ - StringSet features{"nixos-test", "benchmark", "big-parallel", "recursive-nix"}; + StringSet features{"nixos-test", "benchmark", "big-parallel"}; #if __linux__ if (access("/dev/kvm", R_OK | W_OK) == 0) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c88dfe179..71350906e 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -355,8 +355,13 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, StringSet StoreConfig::getDefaultSystemFeatures() { auto res = settings.systemFeatures.get(); + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) res.insert("ca-derivations"); + + if (settings.isExperimentalFeatureEnabled(Xp::RecursiveNix)) + res.insert("recursive-nix"); + return res; } From c47027f3a139669dfb607c22b153564ff53d111c Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 25 Nov 2021 14:48:01 +0100 Subject: [PATCH 017/117] Fix the error when accessing a forbidden path in pure eval MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If we’re in pure eval mode, then tell that in the error message rather than (wrongly) speaking about restricted mode. Fix https://github.com/NixOS/nix/issues/5611 --- src/libexpr/eval.cc | 8 ++++++-- tests/pure-eval.sh | 5 ++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 97fc04711..1fd609bd4 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -519,8 +519,12 @@ Path EvalState::checkSourcePath(const Path & path_) } } - if (!found) - throw RestrictedPathError("access to absolute path '%1%' is forbidden in restricted mode", abspath); + if (!found) { + auto modeInformation = evalSettings.pureEval + ? "in pure eval mode (use '--impure' to override)" + : "in restricted mode"; + throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", abspath, modeInformation); + } /* Resolve symlinks. */ debug(format("checking access to '%s'") % abspath); diff --git a/tests/pure-eval.sh b/tests/pure-eval.sh index c994fbb98..cb4b5c5fc 100644 --- a/tests/pure-eval.sh +++ b/tests/pure-eval.sh @@ -6,7 +6,10 @@ nix eval --expr 'assert 1 + 2 == 3; true' [[ $(nix eval --impure --expr 'builtins.readFile ./pure-eval.sh') =~ clearStore ]] -(! nix eval --expr 'builtins.readFile ./pure-eval.sh') +missingImpureErrorMsg=$(! nix eval --expr 'builtins.readFile ./pure-eval.sh' 2>&1) + +echo "$missingImpureErrorMsg" | grep -q -- --impure || \ + fail "The error message should mention the “--impure” flag to unblock users" (! nix eval --expr builtins.currentTime) (! nix eval --expr builtins.currentSystem) From b6c8e57056f81fa3c2827a7fdc6f335ec54727bd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Nov 2021 20:21:34 +0100 Subject: [PATCH 018/117] Support range-based for loop over list values --- src/libexpr/eval.cc | 18 ++++---- src/libexpr/flake/flake.cc | 3 +- src/libexpr/get-drvs.cc | 22 ++++----- src/libexpr/primops.cc | 82 ++++++++++++++++------------------ src/libexpr/primops/context.cc | 7 ++- src/libexpr/value-to-json.cc | 4 +- src/libexpr/value-to-xml.cc | 4 +- src/libexpr/value.hh | 30 +++++++++++++ src/nix-env/nix-env.cc | 6 +-- src/nix/repl.cc | 6 +-- 10 files changed, 103 insertions(+), 79 deletions(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 97fc04711..5adac87cd 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -119,8 +119,8 @@ void printValue(std::ostream & str, std::set & active, const Valu case tList2: case tListN: str << "[ "; - for (unsigned int n = 0; n < v.listSize(); ++n) { - printValue(str, active, *v.listElems()[n]); + for (auto v2 : v.listItems()) { + printValue(str, active, *v2); str << " "; } str << "]"; @@ -1151,8 +1151,8 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v) void ExprList::eval(EvalState & state, Env & env, Value & v) { state.mkList(v, elems.size()); - for (size_t n = 0; n < elems.size(); ++n) - v.listElems()[n] = elems[n]->maybeThunk(state, env); + for (auto [n, v2] : enumerate(v.listItems())) + const_cast(v2) = elems[n]->maybeThunk(state, env); } @@ -1732,8 +1732,8 @@ void EvalState::forceValueDeep(Value & v) } else if (v.isList()) { - for (size_t n = 0; n < v.listSize(); ++n) - recurse(*v.listElems()[n]); + for (auto v2 : v.listItems()) + recurse(*v2); } }; @@ -1917,12 +1917,12 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, if (v.isList()) { string result; - for (size_t n = 0; n < v.listSize(); ++n) { - result += coerceToString(pos, *v.listElems()[n], + for (auto [n, v2] : enumerate(v.listItems())) { + result += coerceToString(pos, *v2, context, coerceMore, copyToStore); if (n < v.listSize() - 1 /* !!! not quite correct */ - && (!v.listElems()[n]->isList() || v.listElems()[n]->listSize() != 0)) + && (!v2->isList() || v2->listSize() != 0)) result += " "; } return result; diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index f5be67d67..06136579e 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -257,8 +257,7 @@ static Flake getFlake( flake.config.settings.insert({setting.name, state.forceBool(*setting.value, *setting.pos)}); else if (setting.value->type() == nList) { std::vector ss; - for (unsigned int n = 0; n < setting.value->listSize(); ++n) { - auto elem = setting.value->listElems()[n]; + for (auto elem : setting.value->listItems()) { if (elem->type() != nString) throw TypeError("list element in flake configuration setting '%s' is %s while a string is expected", setting.name, showType(*setting.value)); diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index f774e6493..ed4c47fbb 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -102,9 +102,9 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) state->forceList(*i->value, *i->pos); /* For each output... */ - for (unsigned int j = 0; j < i->value->listSize(); ++j) { + for (auto elem : i->value->listItems()) { /* Evaluate the corresponding set. */ - string name = state->forceStringNoCtx(*i->value->listElems()[j], *i->pos); + string name = state->forceStringNoCtx(*elem, *i->pos); Bindings::iterator out = attrs->find(state->symbols.create(name)); if (out == attrs->end()) continue; // FIXME: throw error? state->forceAttrs(*out->value); @@ -128,9 +128,9 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) /* ^ this shows during `nix-env -i` right under the bad derivation */ if (!outTI->isList()) throw errMsg; Outputs result; - for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize(); ++i) { - if ((*i)->type() != nString) throw errMsg; - auto out = outputs.find((*i)->string.s); + for (auto elem : outTI->listItems()) { + if (elem->type() != nString) throw errMsg; + auto out = outputs.find(elem->string.s); if (out == outputs.end()) throw errMsg; result.insert(*out); } @@ -174,8 +174,8 @@ bool DrvInfo::checkMeta(Value & v) { state->forceValue(v); if (v.type() == nList) { - for (unsigned int n = 0; n < v.listSize(); ++n) - if (!checkMeta(*v.listElems()[n])) return false; + for (auto elem : v.listItems()) + if (!checkMeta(*elem)) return false; return true; } else if (v.type() == nAttrs) { @@ -364,10 +364,10 @@ static void getDerivations(EvalState & state, Value & vIn, } else if (v.type() == nList) { - for (unsigned int n = 0; n < v.listSize(); ++n) { - string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str()); - if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures)) - getDerivations(state, *v.listElems()[n], pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); + for (auto [n, elem] : enumerate(v.listItems())) { + string pathPrefix2 = addToPath(pathPrefix, fmt("%d", n)); + if (getDerivation(state, *elem, pathPrefix2, drvs, done, ignoreAssertionFailures)) + getDerivations(state, *elem, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); } } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 8cbeaa520..c0d59da8c 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -335,9 +335,8 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v) PathSet context; auto program = state.coerceToString(pos, *elems[0], context, false, false); Strings commandArgs; - for (unsigned int i = 1; i < args[0]->listSize(); ++i) { + for (unsigned int i = 1; i < args[0]->listSize(); ++i) commandArgs.emplace_back(state.coerceToString(pos, *elems[i], context, false, false)); - } try { state.realiseContext(context); } catch (InvalidPathError & e) { @@ -616,8 +615,8 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar state.forceList(*startSet->value, pos); ValueList workSet; - for (unsigned int n = 0; n < startSet->value->listSize(); ++n) - workSet.push_back(startSet->value->listElems()[n]); + for (auto elem : startSet->value->listItems()) + workSet.push_back(elem); /* Get the operator. */ Bindings::iterator op = getAttr( @@ -662,9 +661,9 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar state.forceList(call, pos); /* Add the values returned by the operator to the work set. */ - for (unsigned int n = 0; n < call.listSize(); ++n) { - state.forceValue(*call.listElems()[n], pos); - workSet.push_back(call.listElems()[n]); + for (auto elem : call.listItems()) { + state.forceValue(*elem, pos); + workSet.push_back(elem); } } @@ -1013,8 +1012,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * command-line arguments to the builder. */ else if (i->name == state.sArgs) { state.forceList(*i->value, pos); - for (unsigned int n = 0; n < i->value->listSize(); ++n) { - string s = state.coerceToString(posDrvName, *i->value->listElems()[n], context, true); + for (auto elem : i->value->listItems()) { + string s = state.coerceToString(posDrvName, *elem, context, true); drv.args.push_back(s); } } @@ -1044,8 +1043,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * /* Require ‘outputs’ to be a list of strings. */ state.forceList(*i->value, posDrvName); Strings ss; - for (unsigned int n = 0; n < i->value->listSize(); ++n) - ss.emplace_back(state.forceStringNoCtx(*i->value->listElems()[n], posDrvName)); + for (auto elem : i->value->listItems()) + ss.emplace_back(state.forceStringNoCtx(*elem, posDrvName)); handleOutputs(ss); } @@ -1460,20 +1459,19 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va SearchPath searchPath; - for (unsigned int n = 0; n < args[0]->listSize(); ++n) { - Value & v2(*args[0]->listElems()[n]); - state.forceAttrs(v2, pos); + for (auto v2 : args[0]->listItems()) { + state.forceAttrs(*v2, pos); string prefix; - Bindings::iterator i = v2.attrs->find(state.symbols.create("prefix")); - if (i != v2.attrs->end()) + Bindings::iterator i = v2->attrs->find(state.symbols.create("prefix")); + if (i != v2->attrs->end()) prefix = state.forceStringNoCtx(*i->value, pos); i = getAttr( state, "findFile", "path", - v2.attrs, + v2->attrs, pos ); @@ -2239,9 +2237,9 @@ static void prim_removeAttrs(EvalState & state, const Pos & pos, Value * * args, /* Get the attribute names to be removed. */ std::set names; - for (unsigned int i = 0; i < args[1]->listSize(); ++i) { - state.forceStringNoCtx(*args[1]->listElems()[i], pos); - names.insert(state.symbols.create(args[1]->listElems()[i]->string.s)); + for (auto elem : args[1]->listItems()) { + state.forceStringNoCtx(*elem, pos); + names.insert(state.symbols.create(elem->string.s)); } /* Copy all attributes not in that set. Note that we don't need @@ -2249,7 +2247,7 @@ static void prim_removeAttrs(EvalState & state, const Pos & pos, Value * * args, vector. */ state.mkAttrs(v, args[0]->attrs->size()); for (auto & i : *args[0]->attrs) { - if (names.find(i.name) == names.end()) + if (!names.count(i.name)) v.attrs->push_back(i); } } @@ -2283,15 +2281,14 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args, std::set seen; - for (unsigned int i = 0; i < args[0]->listSize(); ++i) { - Value & v2(*args[0]->listElems()[i]); - state.forceAttrs(v2, pos); + for (auto v2 : args[0]->listItems()) { + state.forceAttrs(*v2, pos); Bindings::iterator j = getAttr( state, "listToAttrs", state.sName, - v2.attrs, + v2->attrs, pos ); @@ -2303,7 +2300,7 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args, state, "listToAttrs", state.sValue, - v2.attrs, + v2->attrs, pos ); v.attrs->push_back(Attr(sym, j2->value, j2->pos)); @@ -2370,11 +2367,10 @@ static void prim_catAttrs(EvalState & state, const Pos & pos, Value * * args, Va Value * res[args[1]->listSize()]; unsigned int found = 0; - for (unsigned int n = 0; n < args[1]->listSize(); ++n) { - Value & v2(*args[1]->listElems()[n]); - state.forceAttrs(v2, pos); - Bindings::iterator i = v2.attrs->find(attrName); - if (i != v2.attrs->end()) + for (auto v2 : args[1]->listItems()) { + state.forceAttrs(*v2, pos); + Bindings::iterator i = v2->attrs->find(attrName); + if (i != v2->attrs->end()) res[found++] = i->value; } @@ -2649,8 +2645,8 @@ static void prim_elem(EvalState & state, const Pos & pos, Value * * args, Value { bool res = false; state.forceList(*args[1], pos); - for (unsigned int n = 0; n < args[1]->listSize(); ++n) - if (state.eqValues(*args[0], *args[1]->listElems()[n])) { + for (auto elem : args[1]->listItems()) + if (state.eqValues(*args[0], *elem)) { res = true; break; } @@ -2709,8 +2705,8 @@ static void prim_foldlStrict(EvalState & state, const Pos & pos, Value * * args, if (args[2]->listSize()) { Value * vCur = args[1]; - for (unsigned int n = 0; n < args[2]->listSize(); ++n) { - Value * vs []{vCur, args[2]->listElems()[n]}; + for (auto [n, elem] : enumerate(args[2]->listItems())) { + Value * vs []{vCur, elem}; vCur = n == args[2]->listSize() - 1 ? &v : state.allocValue(); state.callFunction(*args[0], 2, vs, *vCur, pos); } @@ -2740,8 +2736,8 @@ static void anyOrAll(bool any, EvalState & state, const Pos & pos, Value * * arg state.forceList(*args[1], pos); Value vTmp; - for (unsigned int n = 0; n < args[1]->listSize(); ++n) { - state.callFunction(*args[0], *args[1]->listElems()[n], vTmp, pos); + for (auto elem : args[1]->listItems()) { + state.callFunction(*args[0], *elem, vTmp, pos); bool res = state.forceBool(vTmp, pos); if (res == any) { mkBool(v, any); @@ -3470,9 +3466,9 @@ static void prim_concatStringsSep(EvalState & state, const Pos & pos, Value * * res.reserve((args[1]->listSize() + 32) * sep.size()); bool first = true; - for (unsigned int n = 0; n < args[1]->listSize(); ++n) { + for (auto elem : args[1]->listItems()) { if (first) first = false; else res += sep; - res += state.coerceToString(pos, *args[1]->listElems()[n], context); + res += state.coerceToString(pos, *elem, context); } mkString(v, res, context); @@ -3501,14 +3497,14 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar vector from; from.reserve(args[0]->listSize()); - for (unsigned int n = 0; n < args[0]->listSize(); ++n) - from.push_back(state.forceString(*args[0]->listElems()[n], pos)); + for (auto elem : args[0]->listItems()) + from.push_back(state.forceString(*elem, pos)); vector> to; to.reserve(args[1]->listSize()); - for (unsigned int n = 0; n < args[1]->listSize(); ++n) { + for (auto elem : args[1]->listItems()) { PathSet ctx; - auto s = state.forceString(*args[1]->listElems()[n], ctx, pos); + auto s = state.forceString(*elem, ctx, pos); to.push_back(std::make_pair(std::move(s), std::move(ctx))); } diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 31cf812b4..20545afd0 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -118,9 +118,8 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args, auto & outputsVal = *state.allocAttr(infoVal, state.sOutputs); state.mkList(outputsVal, info.second.outputs.size()); size_t i = 0; - for (const auto & output : info.second.outputs) { + for (const auto & output : info.second.outputs) mkString(*(outputsVal.listElems()[i++] = state.allocValue()), output); - } } infoVal.attrs->sort(); } @@ -181,8 +180,8 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg .errPos = *i.pos }); } - for (unsigned int n = 0; n < iter->value->listSize(); ++n) { - auto name = state.forceStringNoCtx(*iter->value->listElems()[n], *iter->pos); + for (auto elem : iter->value->listItems()) { + auto name = state.forceStringNoCtx(*elem, *iter->pos); context.insert("!" + name + "!" + string(i.name)); } } diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 4d642c720..517da4c01 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -63,9 +63,9 @@ void printValueAsJSON(EvalState & state, bool strict, case nList: { auto list(out.list()); - for (unsigned int n = 0; n < v.listSize(); ++n) { + for (auto elem : v.listItems()) { auto placeholder(list.placeholder()); - printValueAsJSON(state, strict, *v.listElems()[n], pos, placeholder, context); + printValueAsJSON(state, strict, *elem, pos, placeholder, context); } break; } diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 54268ece0..a875f82d7 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -122,8 +122,8 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, case nList: { XMLOpenElement _(doc, "list"); - for (unsigned int n = 0; n < v.listSize(); ++n) - printValueAsXML(state, strict, location, *v.listElems()[n], doc, context, drvsSeen, pos); + for (auto v2 : v.listItems()) + printValueAsXML(state, strict, location, *v2, doc, context, drvsSeen, pos); break; } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 3bb97b3c2..4b43e47ae 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -1,5 +1,7 @@ #pragma once +#include + #include "symbol-table.hh" #if HAVE_BOEHMGC @@ -350,6 +352,34 @@ public: bool isTrivial() const; std::vector> getContext(); + + auto listItems() + { + struct ListIterable + { + typedef Value * const * iterator; + iterator _begin, _end; + iterator begin() const { return _begin; } + iterator end() const { return _end; } + }; + assert(isList()); + auto begin = listElems(); + return ListIterable { begin, begin + listSize() }; + } + + auto listItems() const + { + struct ConstListIterable + { + typedef const Value * const * iterator; + iterator _begin, _end; + iterator begin() const { return _begin; } + iterator end() const { return _end; } + }; + assert(isList()); + auto begin = listElems(); + return ConstListIterable { begin, begin + listSize() }; + } }; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 4056d973d..0dd0e34a0 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1149,10 +1149,10 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) } else if (v->type() == nList) { attrs2["type"] = "strings"; XMLOpenElement m(xml, "meta", attrs2); - for (unsigned int j = 0; j < v->listSize(); ++j) { - if (v->listElems()[j]->type() != nString) continue; + for (auto elem : v->listItems()) { + if (elem->type() != nString) continue; XMLAttrs attrs3; - attrs3["value"] = v->listElems()[j]->string.s; + attrs3["value"] = elem->string.s; xml.writeEmptyElement("string", attrs3); } } else if (v->type() == nAttrs) { diff --git a/src/nix/repl.cc b/src/nix/repl.cc index fd86174f2..41283c5f2 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -771,12 +771,12 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m str << "[ "; if (maxDepth > 0) - for (unsigned int n = 0; n < v.listSize(); ++n) { - if (seen.find(v.listElems()[n]) != seen.end()) + for (auto elem : v.listItems()) { + if (seen.count(elem)) str << "«repeated»"; else try { - printValue(str, *v.listElems()[n], maxDepth - 1, seen); + printValue(str, *elem, maxDepth - 1, seen); } catch (AssertionError & e) { str << ANSI_RED "«error: " << e.msg() << "»" ANSI_NORMAL; } From 986906e687cc4d3b456592f72a0081c4a9adb1db Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 25 Nov 2021 17:04:31 +0100 Subject: [PATCH 019/117] Update names --- doc/manual/src/release-notes/rl-2.4.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/release-notes/rl-2.4.md b/doc/manual/src/release-notes/rl-2.4.md index 70b715053..0f632f100 100644 --- a/doc/manual/src/release-notes/rl-2.4.md +++ b/doc/manual/src/release-notes/rl-2.4.md @@ -395,6 +395,7 @@ dramforever, Dustin DeWeese, edef, Eelco Dolstra, +Ellie Hermaszewska, Emilio Karakey, Emily, Eric Culp, @@ -405,7 +406,7 @@ Federico Pellegrin, Finn Behrens, Florian Franzen, Félix Baylac-Jacqué, -Gabriel Gonzalez, +Gabriella Gonzalez, Geoff Reedy, Georges Dubus, Graham Christensen, @@ -428,7 +429,6 @@ Jaroslavas Pocepko, Jarrett Keifer, Jeremy Schlatter, Joachim Breitner, -Joe Hermaszewski, Joe Pea, John Ericson, Jonathan Ringer, From 05081bedc156c8e6556b66fe2470fec14e074caf Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 26 Nov 2021 09:56:48 +0100 Subject: [PATCH 020/117] Make `nix-shell` work when nixpkgs is content-addressed Fix #5259 --- src/nix-build/nix-build.cc | 9 ++++++-- tests/ca-shell.nix | 1 + tests/nix-shell.sh | 42 ++++++++++++++++---------------------- tests/shell.nix | 4 ++++ 4 files changed, 30 insertions(+), 26 deletions(-) create mode 100644 tests/ca-shell.nix diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 75576ef8a..e2325c91f 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -359,6 +359,7 @@ static void main_nix_build(int argc, char * * argv) is not set, then build bashInteractive from . */ auto shell = getEnv("NIX_BUILD_SHELL"); + std::optional shellDrv; if (!shell) { @@ -375,8 +376,7 @@ static void main_nix_build(int argc, char * * argv) auto bashDrv = store->parseStorePath(drv->queryDrvPath()); pathsToBuild.push_back({bashDrv}); pathsToCopy.insert(bashDrv); - - shell = drv->queryOutPath() + "/bin/bash"; + shellDrv = bashDrv; } catch (Error & e) { logError(e.info()); @@ -402,6 +402,11 @@ static void main_nix_build(int argc, char * * argv) if (dryRun) return; + if (shellDrv) { + auto shellDrvOutputs = store->queryPartialDerivationOutputMap(shellDrv.value()); + shell = store->printStorePath(shellDrvOutputs.at("out").value()) + "/bin/bash"; + } + if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) { auto resolvedDrv = drv.tryResolve(*store); assert(resolvedDrv && "Successfully resolved the derivation"); diff --git a/tests/ca-shell.nix b/tests/ca-shell.nix new file mode 100644 index 000000000..ad2ab6aff --- /dev/null +++ b/tests/ca-shell.nix @@ -0,0 +1 @@ +{ ... }@args: import ./shell.nix (args // { contentAddressed = true; }) diff --git a/tests/nix-shell.sh b/tests/nix-shell.sh index a31d35887..3241d7a0f 100644 --- a/tests/nix-shell.sh +++ b/tests/nix-shell.sh @@ -3,59 +3,53 @@ source common.sh clearStore if [[ -n ${CONTENT_ADDRESSED:-} ]]; then - nix-shell () { - command nix-shell --arg contentAddressed true "$@" - } - - nix_develop() { - nix develop --arg contentAddressed true "$@" - } + shellDotNix="$PWD/ca-shell.nix" else - nix_develop() { - nix develop "$@" - } + shellDotNix="$PWD/shell.nix" fi +export NIX_PATH=nixpkgs="$shellDotNix" + # Test nix-shell -A export IMPURE_VAR=foo export SELECTED_IMPURE_VAR=baz -export NIX_BUILD_SHELL=$SHELL -output=$(nix-shell --pure shell.nix -A shellDrv --run \ + +output=$(nix-shell --pure "$shellDotNix" -A shellDrv --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"') [ "$output" = " - foo - bar - true" ] # Test --keep -output=$(nix-shell --pure --keep SELECTED_IMPURE_VAR shell.nix -A shellDrv --run \ +output=$(nix-shell --pure --keep SELECTED_IMPURE_VAR "$shellDotNix" -A shellDrv --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $SELECTED_IMPURE_VAR"') [ "$output" = " - foo - bar - baz" ] # Test nix-shell on a .drv -[[ $(nix-shell --pure $(nix-instantiate shell.nix -A shellDrv) --run \ +[[ $(nix-shell --pure $(nix-instantiate "$shellDotNix" -A shellDrv) --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"') = " - foo - bar - false" ]] -[[ $(nix-shell --pure $(nix-instantiate shell.nix -A shellDrv) --run \ +[[ $(nix-shell --pure $(nix-instantiate "$shellDotNix" -A shellDrv) --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"') = " - foo - bar - false" ]] # Test nix-shell on a .drv symlink # Legacy: absolute path and .drv extension required -nix-instantiate shell.nix -A shellDrv --add-root $TEST_ROOT/shell.drv +nix-instantiate "$shellDotNix" -A shellDrv --add-root $TEST_ROOT/shell.drv [[ $(nix-shell --pure $TEST_ROOT/shell.drv --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] # New behaviour: just needs to resolve to a derivation in the store -nix-instantiate shell.nix -A shellDrv --add-root $TEST_ROOT/shell +nix-instantiate "$shellDotNix" -A shellDrv --add-root $TEST_ROOT/shell [[ $(nix-shell --pure $TEST_ROOT/shell --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] # Test nix-shell -p -output=$(NIX_PATH=nixpkgs=shell.nix nix-shell --pure -p foo bar --run 'echo "$(foo) $(bar)"') +output=$(NIX_PATH=nixpkgs="$shellDotNix" nix-shell --pure -p foo bar --run 'echo "$(foo) $(bar)"') [ "$output" = "foo bar" ] # Test nix-shell -p --arg x y -output=$(NIX_PATH=nixpkgs=shell.nix nix-shell --pure -p foo --argstr fooContents baz --run 'echo "$(foo)"') +output=$(NIX_PATH=nixpkgs="$shellDotNix" nix-shell --pure -p foo --argstr fooContents baz --run 'echo "$(foo)"') [ "$output" = "baz" ] # Test nix-shell shebang mode @@ -91,18 +85,18 @@ output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.rb abc ruby) [ "$output" = '-e load(ARGV.shift) -- '"$TEST_ROOT"'/spaced \'\''"shell.shebang.rb abc ruby' ] # Test 'nix develop'. -nix_develop -f shell.nix shellDrv -c bash -c '[[ -n $stdenv ]]' +nix develop -f "$shellDotNix" shellDrv -c bash -c '[[ -n $stdenv ]]' # Ensure `nix develop -c` preserves stdin -echo foo | nix develop -f shell.nix shellDrv -c cat | grep -q foo +echo foo | nix develop -f "$shellDotNix" shellDrv -c cat | grep -q foo # Ensure `nix develop -c` actually executes the command if stdout isn't a terminal -nix_develop -f shell.nix shellDrv -c echo foo |& grep -q foo +nix develop -f "$shellDotNix" shellDrv -c echo foo |& grep -q foo # Test 'nix print-dev-env'. -[[ $(nix print-dev-env -f shell.nix shellDrv --json | jq -r .variables.arr1.value[2]) = '3 4' ]] +[[ $(nix print-dev-env -f "$shellDotNix" shellDrv --json | jq -r .variables.arr1.value[2]) = '3 4' ]] -source <(nix print-dev-env -f shell.nix shellDrv) +source <(nix print-dev-env -f "$shellDotNix" shellDrv) [[ -n $stdenv ]] [[ ${arr1[2]} = "3 4" ]] [[ ${arr2[1]} = $'\n' ]] diff --git a/tests/shell.nix b/tests/shell.nix index 4912d295a..92d94fbc2 100644 --- a/tests/shell.nix +++ b/tests/shell.nix @@ -74,6 +74,10 @@ let pkgs = rec { ''; bash = shell; + bashInteractive = runCommand "bash" {} '' + mkdir -p $out/bin + ln -s ${shell} $out/bin/bash + ''; # ruby "interpreter" that outputs "$@" ruby = runCommand "ruby" {} '' From e588f4c6558cb486aa065e17a124f272ccc1fcdb Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 8 Jul 2021 17:01:51 +0200 Subject: [PATCH 021/117] Add a matrix of stdenvs to the flake For a (currently hardcoded and limited) list of stdenvs, make `.#$nix-${stdenvName}` correspond to a Nix built with the corresponding stdenv. For example, `.#nix-${clang11Stdenv}` is Nix built with clang11. Likewise, `devShells.x86_64-linux.clang11StdenvPackages` is a development shell for Nix with clang11, that can be used with ```shell nix develop .#clang11StdenvPackages ``` Fix #4129 /cc @pamplemousse --- flake.nix | 80 ++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 55 insertions(+), 25 deletions(-) diff --git a/flake.nix b/flake.nix index ff152ebd6..9a5d188ae 100644 --- a/flake.nix +++ b/flake.nix @@ -22,15 +22,36 @@ crossSystems = [ "armv6l-linux" "armv7l-linux" ]; + stdenvs = [ "gccStdenv" "clangStdenv" "clang11Stdenv" "stdenv" ]; + forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system); + forAllSystemsAndStdenvs = f: forAllSystems (system: + nixpkgs.lib.listToAttrs + (map + (n: + nixpkgs.lib.nameValuePair "${n}Packages" ( + f system n + )) stdenvs + ) + ); + + forAllStdenvs = stdenvs: f: nixpkgs.lib.genAttrs stdenvs (stdenv: f stdenv); # Memoize nixpkgs for different platforms for efficiency. - nixpkgsFor = forAllSystems (system: - import nixpkgs { - inherit system; - overlays = [ self.overlay ]; - } - ); + nixpkgsFor = + let stdenvsPackages = forAllSystemsAndStdenvs + (system: stdenv: + import nixpkgs { + inherit system; + overlays = [ + (overlayFor (p: p.${stdenv})) + ]; + } + ); + in + # Add the `stdenvPackages` at toplevel, both because these are the ones + # we want most of the time and for backwards compatibility + forAllSystems (system: stdenvsPackages.${system} // stdenvsPackages.${system}.stdenvPackages); commonDeps = pkgs: with pkgs; rec { # Use "busybox-sandbox-shell" if present, @@ -255,18 +276,15 @@ $(cat ${installerClosureInfo}/store-paths) ''; - in { - - # A Nixpkgs overlay that overrides the 'nix' and - # 'nix.perl-bindings' packages. - overlay = final: prev: { - + overlayFor = getStdenv: final: prev: + let currentStdenv = getStdenv final; in + { nixStable = prev.nix; # Forward from the previous stage as we don’t want it to pick the lowdown override nixUnstable = prev.nixUnstable; - nix = with final; with commonDeps pkgs; stdenv.mkDerivation { + nix = with final; with commonDeps pkgs; currentStdenv.mkDerivation { name = "nix-${version}"; inherit version; @@ -288,9 +306,9 @@ mkdir -p $out/lib cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib rm -f $out/lib/*.a - ${lib.optionalString stdenv.isLinux '' + ${lib.optionalString currentStdenv.isLinux '' chmod u+w $out/lib/*.so.* - patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* + patchelf --set-rpath $out/lib:${currentStdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* ''} ''; @@ -317,7 +335,7 @@ strictDeps = true; - passthru.perl-bindings = with final; stdenv.mkDerivation { + passthru.perl-bindings = with final; currentStdenv.mkDerivation { name = "nix-perl-${version}"; src = self; @@ -336,8 +354,8 @@ pkgs.perl boost ] - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium - ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security; + ++ lib.optional (currentStdenv.isLinux || currentStdenv.isDarwin) libsodium + ++ lib.optional currentStdenv.isDarwin darwin.apple_sdk.frameworks.Security; configureFlags = '' --with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix} @@ -351,7 +369,7 @@ }; - lowdown-nix = with final; stdenv.mkDerivation rec { + lowdown-nix = with final; currentStdenv.mkDerivation rec { name = "lowdown-0.9.0"; src = lowdown-src; @@ -361,15 +379,20 @@ nativeBuildInputs = [ buildPackages.which ]; configurePhase = '' - ${if (stdenv.isDarwin && stdenv.isAarch64) then "echo \"HAVE_SANDBOX_INIT=false\" > configure.local" else ""} + ${if (currentStdenv.isDarwin && currentStdenv.isAarch64) then "echo \"HAVE_SANDBOX_INIT=false\" > configure.local" else ""} ./configure \ PREFIX=${placeholder "dev"} \ BINDIR=${placeholder "bin"}/bin - ''; + ''; }; - }; + in { + + # A Nixpkgs overlay that overrides the 'nix' and + # 'nix.perl-bindings' packages. + overlay = overlayFor (p: p.stdenv); + hydraJobs = { # Binary package for various platforms. @@ -610,15 +633,22 @@ doInstallCheck = true; installCheckFlags = "sysconfdir=$(out)/etc"; }; - }) crossSystems))); + }) crossSystems)) // (builtins.listToAttrs (map (stdenvName: + nixpkgsFor.${system}.lib.nameValuePair + "nix-${stdenvName}" + nixpkgsFor.${system}."${stdenvName}Packages".nix + ) stdenvs)) + ); defaultPackage = forAllSystems (system: self.packages.${system}.nix); - devShell = forAllSystems (system: + devShell = forAllSystems (system: self.devShells.${system}.stdenvPackages); + + devShells = forAllSystemsAndStdenvs (system: stdenv: with nixpkgsFor.${system}; with commonDeps pkgs; - stdenv.mkDerivation { + nixpkgsFor.${system}.${stdenv}.mkDerivation { name = "nix"; outputs = [ "out" "dev" "doc" ]; From 288c25257074e70e69a5d51f63dc783695ff4175 Mon Sep 17 00:00:00 2001 From: Pamplemousse Date: Thu, 8 Jul 2021 09:13:55 -0700 Subject: [PATCH 022/117] Documentation: alternative `stdenv` for hacking Signed-off-by: Pamplemousse --- doc/manual/src/contributing/hacking.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 2a1e55e5b..90a8f1f94 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -35,6 +35,25 @@ variables are set up so that those dependencies can be found: $ nix-shell ``` +or if you have a flake-enabled nix: + +```console +$ nix develop +``` + +To get a shell with a different compilation environment (e.g. stdenv, +gccStdenv, clangStdenv, clang11Stdenv): + +```console +$ nix-shell -A devShells.x86_64-linux.clang11StdenvPackages +``` + +or if you have a flake-enabled nix: + +```console +$ nix develop .#clang11StdenvPackages +``` + To build Nix itself in this shell: ```console From 6d166d19a6c1ffc8e651b7045ca1a8a7fff42e57 Mon Sep 17 00:00:00 2001 From: Alex Shabalin Date: Fri, 26 Nov 2021 11:03:48 +0100 Subject: [PATCH 023/117] Catch flake-related exception type in REPL Closes https://github.com/NixOS/nix/issues/5656 --- src/nix/repl.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 41283c5f2..22a57de60 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -356,6 +356,8 @@ StringSet NixRepl::completePrefix(string prefix) // Quietly ignore evaluation errors. } catch (UndefinedVarError & e) { // Quietly ignore undefined variable errors. + } catch (BadURL & e) { + // Quietly ignore BadURL flake-related errors. } } From 3ef66cd23a81ae809bee8c27c0d65c3699be5c66 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Nov 2021 13:10:28 +0100 Subject: [PATCH 024/117] nix repl: Reset the terminal on exceptional exits --- src/nix/repl.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 22a57de60..2649eb0bd 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -279,6 +279,7 @@ bool NixRepl::getLine(string & input, const std::string &prompt) }; setupSignals(); + Finally resetTerminal([&]() { rl_deprep_terminal(); }); char * s = readline(prompt.c_str()); Finally doFree([&]() { free(s); }); restoreSignals(); From 152e3cda0c3e2309b37593b06f9e113ec44f506a Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Fri, 26 Nov 2021 10:55:43 -0500 Subject: [PATCH 025/117] reproducibility: determinstic man page output for cores --- doc/manual/local.mk | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index e43d9f2fb..6b232a736 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -12,11 +12,13 @@ man-pages := $(foreach n, \ clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8 # Provide a dummy environment for nix, so that it will not access files outside the macOS sandbox. +# Set cores to 0 because otherwise nix show-config resolves the cores based on the current machine dummy-env = env -i \ HOME=/dummy \ NIX_CONF_DIR=/dummy \ NIX_SSL_CERT_FILE=/dummy/no-ca-bundle.crt \ - NIX_STATE_DIR=/dummy + NIX_STATE_DIR=/dummy \ + NIX_CONFIG='cores = 0' nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix/corepkgs=corepkgs --store dummy:// --impure --raw From 3a0277305a5d86f29e5bdc1c425c45e4bd7dbe71 Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 26 Nov 2021 16:56:25 +0100 Subject: [PATCH 026/117] =?UTF-8?q?Don=E2=80=99t=20try=20to=20complete=20f?= =?UTF-8?q?lakes=20is=20the=20feature=20isn=E2=80=99t=20enabled?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix #5661 --- src/libcmd/installables.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 5758b52ad..ef200b1d2 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -291,6 +291,9 @@ void completeFlakeRefWithFragment( void completeFlakeRef(ref store, std::string_view prefix) { + if (!settings.isExperimentalFeatureEnabled(Xp::Flakes)) + return; + if (prefix == "") completions->add("."); From 1d5a881da574fc4883e4950c82f520fcf59d583a Mon Sep 17 00:00:00 2001 From: Aleksey Kladov Date: Sat, 27 Nov 2021 23:29:48 +0300 Subject: [PATCH 027/117] Fix typo --- src/nix/registry.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nix/registry.md b/src/nix/registry.md index a1674bd2e..d5c9ef442 100644 --- a/src/nix/registry.md +++ b/src/nix/registry.md @@ -2,7 +2,7 @@ R""( # Description -`nix flake` provides subcommands for managing *flake +`nix registry` provides subcommands for managing *flake registries*. Flake registries are a convenience feature that allows you to refer to flakes using symbolic identifiers such as `nixpkgs`, rather than full URLs such as `git://github.com/NixOS/nixpkgs`. You From 33a227503adfaa570174c48d8c893f968ed8337f Mon Sep 17 00:00:00 2001 From: Alexander Sosedkin Date: Sat, 27 Nov 2021 23:02:04 +0100 Subject: [PATCH 028/117] Document libsodium, which is now mandatory, as a dependency --- configure.ac | 2 +- doc/manual/src/installation/prerequisites-source.md | 5 +++++ perl/configure.ac | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/configure.ac b/configure.ac index 65478ecc5..bca650005 100644 --- a/configure.ac +++ b/configure.ac @@ -188,7 +188,7 @@ PKG_CHECK_MODULES([EDITLINE], [libeditline], [CXXFLAGS="$EDITLINE_CFLAGS $CXXFLA [AC_MSG_ERROR([Nix requires libeditline; it was not found via pkg-config, but via its header, but required functions do not work. Maybe it is too old? >= 1.14 is required.])]) ]) -# Look for libsodium, an optional dependency. +# Look for libsodium. PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"]) # Look for libbrotli{enc,dec}. diff --git a/doc/manual/src/installation/prerequisites-source.md b/doc/manual/src/installation/prerequisites-source.md index 0323a4f55..b3753b53b 100644 --- a/doc/manual/src/installation/prerequisites-source.md +++ b/doc/manual/src/installation/prerequisites-source.md @@ -44,6 +44,11 @@ obtained from the its repository . + - The `libsodium` library for verifying cryptographic signatures + of contents fetched from binary caches. + It can be obtained from the official web site + . + - Recent versions of Bison and Flex to build the parser. (This is because Nix needs GLR support in Bison and reentrancy support in Flex.) For Bison, you need version 2.6, which can be obtained from diff --git a/perl/configure.ac b/perl/configure.ac index eb65ac17b..a02cb06c9 100644 --- a/perl/configure.ac +++ b/perl/configure.ac @@ -41,7 +41,7 @@ perlarchname=$($perl -e 'use Config; print $Config{archname};') AC_SUBST(perllibdir, [${libdir}/perl5/site_perl/$perlversion/$perlarchname]) AC_MSG_RESULT($perllibdir) -# Look for libsodium, an optional dependency. +# Look for libsodium. PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"]) # Check for the required Perl dependencies (DBI and DBD::SQLite). From b73a1c06386f43b0b0d7ce99448b8c55f4c5156c Mon Sep 17 00:00:00 2001 From: Alexander Sosedkin Date: Sun, 28 Nov 2021 00:47:25 +0100 Subject: [PATCH 029/117] Document libcpuid dependency --- doc/manual/src/installation/prerequisites-source.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/manual/src/installation/prerequisites-source.md b/doc/manual/src/installation/prerequisites-source.md index 0323a4f55..37e0b5a10 100644 --- a/doc/manual/src/installation/prerequisites-source.md +++ b/doc/manual/src/installation/prerequisites-source.md @@ -58,3 +58,9 @@ `--disable-seccomp-sandboxing` option to the `configure` script (Not recommended unless your system doesn't support `libseccomp`). To get the library, visit . + + - On 64-bit x86 machines only, `libcpuid` library + is used to determine which microarchitecture levels are supported + (e.g., as whether to have `x86_64-v2-linux` among additional system types). + The library is available from its homepage + . From a9bd06d0ea2c7af9fa33763bdca174db76400ee0 Mon Sep 17 00:00:00 2001 From: Alexander Sosedkin Date: Sun, 28 Nov 2021 00:48:04 +0100 Subject: [PATCH 030/117] Make libcpuid dependency optional with --disable-cpuid --- configure.ac | 13 ++++++++++--- doc/manual/src/installation/prerequisites-source.md | 2 ++ tests/local.mk | 5 ++++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/configure.ac b/configure.ac index 65478ecc5..4108eb0a8 100644 --- a/configure.ac +++ b/configure.ac @@ -195,10 +195,17 @@ PKG_CHECK_MODULES([SODIUM], [libsodium], [CXXFLAGS="$SODIUM_CFLAGS $CXXFLAGS"]) PKG_CHECK_MODULES([LIBBROTLI], [libbrotlienc libbrotlidec], [CXXFLAGS="$LIBBROTLI_CFLAGS $CXXFLAGS"]) # Look for libcpuid. +have_libcpuid= if test "$machine_name" = "x86_64"; then - PKG_CHECK_MODULES([LIBCPUID], [libcpuid], [CXXFLAGS="$LIBCPUID_CFLAGS $CXXFLAGS"]) - have_libcpuid=1 - AC_DEFINE([HAVE_LIBCPUID], [1], [Use libcpuid]) + AC_ARG_ENABLE([cpuid], + AS_HELP_STRING([--disable-cpuid], [Do not determine microarchitecture levels with libcpuid (relevant to x86_64 only)])) + if test "x$enable_cpuid" != "xno"; then + PKG_CHECK_MODULES([LIBCPUID], [libcpuid], + [CXXFLAGS="$LIBCPUID_CFLAGS $CXXFLAGS" + have_libcpuid=1 + AC_DEFINE([HAVE_LIBCPUID], [1], [Use libcpuid])] + ) + fi fi AC_SUBST(HAVE_LIBCPUID, [$have_libcpuid]) diff --git a/doc/manual/src/installation/prerequisites-source.md b/doc/manual/src/installation/prerequisites-source.md index 37e0b5a10..1b03bacd3 100644 --- a/doc/manual/src/installation/prerequisites-source.md +++ b/doc/manual/src/installation/prerequisites-source.md @@ -64,3 +64,5 @@ (e.g., as whether to have `x86_64-v2-linux` among additional system types). The library is available from its homepage . + This is an optional dependency and can be disabled + by providing a `--disable-cpuid` to the `configure` script. diff --git a/tests/local.mk b/tests/local.mk index 6f38853bc..936b72c2a 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -48,7 +48,6 @@ nix_tests = \ flakes.sh \ flake-local-settings.sh \ build.sh \ - compute-levels.sh \ repl.sh ca/repl.sh \ ca/build.sh \ ca/build-with-garbage-path.sh \ @@ -63,6 +62,10 @@ nix_tests = \ eval-store.sh # parallel.sh +ifeq ($(HAVE_LIBCPUID), 1) + nix_tests += compute-levels.sh +endif + install-tests += $(foreach x, $(nix_tests), tests/$(x)) tests-environment = NIX_REMOTE= $(bash) -e From 90d8178009381fe7ad6f0380127df2bc5a0d1dc9 Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Sat, 6 Nov 2021 19:50:27 +0100 Subject: [PATCH 031/117] Don't move the arguments of the primOp Moving arguments of the primOp into the registration structure makes it impossible to initialize a second EvalState with the correct primOp registration. It will end up registering all those "RegisterPrimOp"'s with an arity of zero on all but the 2nd instance of the EvalState. Not moving the memory will add a tiny bit of memory overhead during the eval since we need a copy of all the argument lists of all the primOp's. The overhead shouldn't be too bad as it is static (based on the amonut of registered operations) and only occurs once during the interpreter startup. --- src/libexpr/primops.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index c0d59da8c..d1f4d9009 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -3732,7 +3732,7 @@ void EvalState::createBaseEnv() .fun = primOp.fun, .arity = std::max(primOp.args.size(), primOp.arity), .name = symbols.create(primOp.name), - .args = std::move(primOp.args), + .args = primOp.args, .doc = primOp.doc, }); From 6e0cbc666b60515b5e201dd28855f5fe1de9a107 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 29 Nov 2021 11:20:50 +0100 Subject: [PATCH 032/117] createTempFile(): Mark file as CLOEEXEC Fixes #5674. --- src/libutil/util.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index defb77a10..1b6467eb2 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -512,6 +512,7 @@ std::pair createTempFile(const Path & prefix) AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); if (!fd) throw SysError("creating temporary file '%s'", tmpl); + closeOnExec(fd.get()); return {std::move(fd), tmpl}; } From bd628cf3da14b7d3fe0ba00932eb26d545b4b893 Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Mon, 22 Nov 2021 13:29:49 +0100 Subject: [PATCH 033/117] flakes: fix boolean and int nixConfig values Some type confusion was causing ints to be pointers, and bools to be ints. Fixes #5621 --- src/libexpr/flake/config.cc | 8 ++++---- src/libexpr/flake/flake.cc | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/libexpr/flake/config.cc b/src/libexpr/flake/config.cc index c03f4106c..7ecd61816 100644 --- a/src/libexpr/flake/config.cc +++ b/src/libexpr/flake/config.cc @@ -38,11 +38,11 @@ void ConfigFile::apply() // FIXME: Move into libutil/config.cc. std::string valueS; - if (auto s = std::get_if(&value)) + if (auto* s = std::get_if(&value)) valueS = *s; - else if (auto n = std::get_if(&value)) - valueS = fmt("%d", n); - else if (auto b = std::get_if>(&value)) + else if (auto* n = std::get_if(&value)) + valueS = fmt("%d", *n); + else if (auto* b = std::get_if>(&value)) valueS = b->t ? "true" : "false"; else if (auto ss = std::get_if>(&value)) valueS = concatStringsSep(" ", *ss); // FIXME: evil diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index f5be67d67..732198031 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -254,7 +254,7 @@ static Flake getFlake( else if (setting.value->type() == nInt) flake.config.settings.insert({setting.name, state.forceInt(*setting.value, *setting.pos)}); else if (setting.value->type() == nBool) - flake.config.settings.insert({setting.name, state.forceBool(*setting.value, *setting.pos)}); + flake.config.settings.insert({setting.name, Explicit { state.forceBool(*setting.value, *setting.pos) }}); else if (setting.value->type() == nList) { std::vector ss; for (unsigned int n = 0; n < setting.value->listSize(); ++n) { From a473e85c808120def9a645235e1e6bfa746753b5 Mon Sep 17 00:00:00 2001 From: Eli Flanagan Date: Tue, 30 Nov 2021 11:13:46 -0500 Subject: [PATCH 034/117] docs: document set theory symbols These symbols confused me so I wanted to gloss them to help future users. You can see the context here: https://discourse.nixos.org/t/flakes-what-does-mean-big-bundle-of-questions --- doc/manual/src/glossary.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index bb350d9de..92192e4cb 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -47,7 +47,7 @@ the store object at `P` contains the path `Q` somewhere. The *references* of a store path are the set of store paths to which it has a reference. - + A derivation can reference other derivations and sources (but not output paths), whereas an output path only references other output paths. @@ -66,7 +66,7 @@ is necessary to deploy whole closures, since otherwise at runtime files could be missing. The command `nix-store -qR` prints out closures of store paths. - + As an example, if the store object at path `P` contains a reference to path `Q`, then `Q` is in the closure of `P`. Further, if `Q` references `R` then `R` is also in the closure of `P`. @@ -98,3 +98,7 @@ store. It can contain regular files, directories and symbolic links. NARs are generated and unpacked using `nix-store --dump` and `nix-store --restore`. + - `∅` \ + The empty set symbol. In the context of package history, this denotes a package was not present prior to installation. + - `ε` \ + The epsilon symbol. In the context of a package, this means the version is empty. More precisely, the derivation does not have a version attribute. From 5f64b69d23bfee70d222671a7d10f2efe22b99c6 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 30 Nov 2021 17:32:40 +0100 Subject: [PATCH 035/117] Add a github cron to check the hydra status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a regular github action that will check the status of the latest hydra evaluation. Things aren’t ideal right now because this job will only notify “the user who last modified the cron syntax in the workflow file” (so myself atm). But at least that’ll give a notification for failing hydra jobs --- .github/workflows/hydra_status.yml | 16 ++++++++++++++++ scripts/check-hydra-status.sh | 28 ++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 .github/workflows/hydra_status.yml create mode 100644 scripts/check-hydra-status.sh diff --git a/.github/workflows/hydra_status.yml b/.github/workflows/hydra_status.yml new file mode 100644 index 000000000..b97076bd7 --- /dev/null +++ b/.github/workflows/hydra_status.yml @@ -0,0 +1,16 @@ +name: Hydra status +on: + schedule: + - cron: "12,42 * * * *" + workflow_dispatch: +jobs: + check_hydra_status: + name: Check Hydra status + if: github.repository_owner == 'NixOS' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2.4.0 + with: + fetch-depth: 0 + - run: bash scripts/check-hydra-status.sh + diff --git a/scripts/check-hydra-status.sh b/scripts/check-hydra-status.sh new file mode 100644 index 000000000..c1d2d7c40 --- /dev/null +++ b/scripts/check-hydra-status.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +set -euo pipefail +# set -x + + +# mapfile BUILDS_FOR_LATEST_EVAL < <( +# curl -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \ +# jq -r '.evals[0].builds[] | @sh') +BUILDS_FOR_LATEST_EVAL=$( +curl -sS -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \ + jq -r '.evals[0].builds[]') + +someBuildFailed=0 + +for buildId in $BUILDS_FOR_LATEST_EVAL; do + buildInfo=$(curl -sS -H 'Accept: application/json' "https://hydra.nixos.org/build/$buildId") + + buildStatus=$(echo "$buildInfo" | \ + jq -r '.buildstatus') + + if [[ "$buildStatus" -ne 0 ]]; then + someBuildFailed=1 + echo "Job “$(echo "$buildInfo" | jq -r '.job')” failed on hydra" + fi +done + +exit "$someBuildFailed" From 092c375cda9efc24f9a3d6b56d818792406b232b Mon Sep 17 00:00:00 2001 From: Eli Flanagan Date: Tue, 30 Nov 2021 13:19:42 -0500 Subject: [PATCH 036/117] Update doc/manual/src/glossary.md Co-authored-by: Eelco Dolstra --- doc/manual/src/glossary.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 92192e4cb..71ff13275 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -99,6 +99,6 @@ links. NARs are generated and unpacked using `nix-store --dump` and `nix-store --restore`. - `∅` \ - The empty set symbol. In the context of package history, this denotes a package was not present prior to installation. + The empty set symbol. In the context of profile history, this denotes a package is not present in a particular version of the profile. - `ε` \ The epsilon symbol. In the context of a package, this means the version is empty. More precisely, the derivation does not have a version attribute. From 1f15441103caab1b41317a6107fc2560eb109a4a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 30 Nov 2021 20:23:13 +0000 Subject: [PATCH 037/117] Tidy up the logging Use the macros more, so we properly skip work when the log level excludes. Also log the daemon operation number on the daemon side. --- src/libstore/daemon.cc | 4 +++- src/libstore/store-api.cc | 2 +- src/libutil/logging.hh | 5 +++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 2eb566080..58c567b05 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -951,7 +951,7 @@ void processConnection( Finally finally([&]() { _isInterrupted = false; - prevLogger->log(lvlDebug, fmt("%d operations", opCount)); + printMsgUsing(prevLogger, lvlDebug, "%d operations", opCount); }); if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) { @@ -984,6 +984,8 @@ void processConnection( break; } + printMsgUsing(prevLogger, lvlDebug, "received daemon op %d", op); + opCount++; try { diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 71350906e..aab4ce94c 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1079,7 +1079,7 @@ std::map copyPaths( nrFailed++; if (!settings.keepGoing) throw e; - logger->log(lvlError, fmt("could not copy %s: %s", dstStore.printStorePath(storePath), e.what())); + printMsg(lvlError, "could not copy %s: %s", dstStore.printStorePath(storePath), e.what()); showProgress(); return; } diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 96ad69790..ce9c3dfed 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -189,13 +189,14 @@ extern Verbosity verbosity; /* suppress msgs > this */ /* Print a string message if the current log level is at least the specified level. Note that this has to be implemented as a macro to ensure that the arguments are evaluated lazily. */ -#define printMsg(level, args...) \ +#define printMsgUsing(loggerParam, level, args...) \ do { \ auto __lvl = level; \ if (__lvl <= nix::verbosity) { \ - logger->log(__lvl, fmt(args)); \ + loggerParam->log(__lvl, fmt(args)); \ } \ } while (0) +#define printMsg(level, args...) printMsgUsing(logger, level, args) #define printError(args...) printMsg(lvlError, args) #define notice(args...) printMsg(lvlNotice, args) From 70a717f7a8feb15d9daf1215e305162b999b6175 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 30 Nov 2021 20:53:10 +0000 Subject: [PATCH 038/117] Nix daemon stream old wopAddToStore No more buffering in string. --- src/libstore/daemon.cc | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 2eb566080..773794ffc 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -431,25 +431,23 @@ static void performOp(TunnelLogger * logger, ref store, hashAlgo = parseHashType(hashAlgoRaw); } - StringSink saved; - TeeSource savedNARSource(from, saved); - RetrieveRegularNARSink savedRegular { saved }; + auto dumpSource = sinkToSource([&](Sink & saved) { + TeeSource savedNARSource(from, saved); + RetrieveRegularNARSink savedRegular { saved }; - if (method == FileIngestionMethod::Recursive) { - /* Get the entire NAR dump from the client and save it to - a string so that we can pass it to - addToStoreFromDump(). */ - ParseSink sink; /* null sink; just parse the NAR */ - parseDump(sink, savedNARSource); - } else - parseDump(savedRegular, from); + if (method == FileIngestionMethod::Recursive) { + /* Get the entire NAR dump from the client and save it to + a string so that we can pass it to + addToStoreFromDump(). */ + ParseSink sink; /* null sink; just parse the NAR */ + parseDump(sink, savedNARSource); + } else + parseDump(savedRegular, from); + if (!savedRegular.regular) throw Error("regular file expected"); + }); logger->startWork(); - if (!savedRegular.regular) throw Error("regular file expected"); - - // FIXME: try to stream directly from `from`. - StringSource dumpSource { *saved.s }; - auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo); + auto path = store->addToStoreFromDump(*dumpSource, baseName, method, hashAlgo); logger->stopWork(); to << store->printStorePath(path); From 95157b4e664dc6f4dece15beac6fa1143086323e Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 30 Nov 2021 21:02:45 +0000 Subject: [PATCH 039/117] Push wopAddToStore old style stream adapters into smaller scopes This doesn't fix the bug, but makes the code less difficult to read. Also improve the comments, now that it is clear what part is needed in each code path. --- src/libstore/daemon.cc | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 773794ffc..dc4889dfd 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -432,19 +432,26 @@ static void performOp(TunnelLogger * logger, ref store, } auto dumpSource = sinkToSource([&](Sink & saved) { - TeeSource savedNARSource(from, saved); - RetrieveRegularNARSink savedRegular { saved }; - if (method == FileIngestionMethod::Recursive) { - /* Get the entire NAR dump from the client and save it to - a string so that we can pass it to - addToStoreFromDump(). */ + /* We parse the NAR dump through into `saved` unmodified, + so why all this extra work? We still parse the NAR so + that we aren't sending arbitrary data to `saved` + unwittingly`, and we know when the NAR ends so we don't + consume the rest of `from` and can't parse another + command. (We don't trust `addToStoreFromDump` to not + eagerly consume the entire stream it's given, past the + length of the Nar. */ + TeeSource savedNARSource(from, saved); ParseSink sink; /* null sink; just parse the NAR */ parseDump(sink, savedNARSource); - } else + } else { + /* Incrementally parse the NAR file, stripping the + metadata, and streaming the sole file we expect into + `saved`. */ + RetrieveRegularNARSink savedRegular { saved }; parseDump(savedRegular, from); - - if (!savedRegular.regular) throw Error("regular file expected"); + if (!savedRegular.regular) throw Error("regular file expected"); + } }); logger->startWork(); auto path = store->addToStoreFromDump(*dumpSource, baseName, method, hashAlgo); From 2f5c913d4a21c60799290d6efe769d188c57b906 Mon Sep 17 00:00:00 2001 From: John Ericson Date: Tue, 30 Nov 2021 21:48:43 +0000 Subject: [PATCH 040/117] Fix #5299 No matter what, we need to resize the buffer to not have any scratch space after we do the `read`. In the end of file case, `got` will be 0 from it's initial value. Before, we forgot to resize in the EOF case with the break. Yes, we know we didn't recieve any data in that case, but we still have the scatch space to undo. Co-Authored-By: Will Fancher --- src/libstore/local-store.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 64019314f..3a1688272 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -8,6 +8,7 @@ #include "references.hh" #include "callback.hh" #include "topo-sort.hh" +#include "finally.hh" #include #include @@ -1333,13 +1334,15 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name, auto want = std::min(chunkSize, settings.narBufferSize - oldSize); dump.resize(oldSize + want); auto got = 0; + Finally cleanup([&]() { + dump.resize(oldSize + got); + }); try { got = source.read(dump.data() + oldSize, want); } catch (EndOfFile &) { inMemory = true; break; } - dump.resize(oldSize + got); } std::unique_ptr delTempDir; From 21520297dad8a198004e4551808a4818a8de7ddd Mon Sep 17 00:00:00 2001 From: Arthur Gautier Date: Wed, 1 Dec 2021 16:08:23 +0100 Subject: [PATCH 041/117] reproducibility: hide non-reproducible settings from manual Because the manual is generated from default values which are themselves generated from various sources (cpuid, bios settings (kvm), number of cores). This commit hides non-reproducible settings from the manual output. --- doc/manual/generate-options.nix | 24 +++++++++++++----------- src/libstore/globals.hh | 17 ++++++++++------- src/libutil/abstract-setting-to-json.hh | 1 + src/libutil/config.hh | 10 +++++++--- src/libutil/tests/config.cc | 2 +- 5 files changed, 32 insertions(+), 22 deletions(-) diff --git a/doc/manual/generate-options.nix b/doc/manual/generate-options.nix index 3c31a4eec..9a77f4d36 100644 --- a/doc/manual/generate-options.nix +++ b/doc/manual/generate-options.nix @@ -8,17 +8,19 @@ concatStrings (map let option = options.${name}; in " - `${name}` \n\n" + concatStrings (map (s: " ${s}\n") (splitLines option.description)) + "\n\n" - + " **Default:** " + ( - if option.value == "" || option.value == [] - then "*empty*" - else if isBool option.value - then (if option.value then "`true`" else "`false`") - else - # n.b. a StringMap value type is specified as a string, but - # this shows the value type. The empty stringmap is "null" in - # JSON, but that converts to "{ }" here. - (if isAttrs option.value then "`\"\"`" - else "`" + toString option.value + "`")) + "\n\n" + + (if option.documentDefault + then " **Default:** " + ( + if option.value == "" || option.value == [] + then "*empty*" + else if isBool option.value + then (if option.value then "`true`" else "`false`") + else + # n.b. a StringMap value type is specified as a string, but + # this shows the value type. The empty stringmap is "null" in + # JSON, but that converts to "{ }" here. + (if isAttrs option.value then "`\"\"`" + else "`" + toString option.value + "`")) + "\n\n" + else " **Default:** *machine-specific*") + (if option.aliases != [] then " **Deprecated alias:** " + (concatStringsSep ", " (map (s: "`${s}`") option.aliases)) + "\n\n" else "") diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index a50eb6803..e9eeffe8c 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -21,7 +21,7 @@ struct MaxBuildJobsSetting : public BaseSetting const std::string & name, const std::string & description, const std::set & aliases = {}) - : BaseSetting(def, name, description, aliases) + : BaseSetting(def, true, name, description, aliases) { options->addSetting(this); } @@ -38,7 +38,7 @@ struct PluginFilesSetting : public BaseSetting const std::string & name, const std::string & description, const std::set & aliases = {}) - : BaseSetting(def, name, description, aliases) + : BaseSetting(def, true, name, description, aliases) { options->addSetting(this); } @@ -130,7 +130,9 @@ public: {"build-max-jobs"}}; Setting buildCores{ - this, getDefaultCores(), "cores", + this, + getDefaultCores(), + "cores", R"( Sets the value of the `NIX_BUILD_CORES` environment variable in the invocation of builders. Builders can use this variable at their @@ -141,7 +143,7 @@ public: command line switch and defaults to `1`. The value `0` means that the builder should use all available CPU cores in the system. )", - {"build-cores"}}; + {"build-cores"}, false}; /* Read-only mode. Don't copy stuff to the store, don't change the database. */ @@ -583,10 +585,11 @@ public: platform and generate incompatible code, so you may wish to cross-check the results of using this option against proper natively-built versions of your derivations. - )"}; + )", {}, false}; Setting systemFeatures{ - this, getDefaultSystemFeatures(), + this, + getDefaultSystemFeatures(), "system-features", R"( A set of system “features” supported by this machine, e.g. `kvm`. @@ -602,7 +605,7 @@ public: This setting by default includes `kvm` if `/dev/kvm` is accessible, and the pseudo-features `nixos-test`, `benchmark` and `big-parallel` that are used in Nixpkgs to route builds to specific machines. - )"}; + )", {}, false}; Setting substituters{ this, diff --git a/src/libutil/abstract-setting-to-json.hh b/src/libutil/abstract-setting-to-json.hh index b3fbc84f7..2d82b54e7 100644 --- a/src/libutil/abstract-setting-to-json.hh +++ b/src/libutil/abstract-setting-to-json.hh @@ -10,6 +10,7 @@ std::map BaseSetting::toJSONObject() auto obj = AbstractSetting::toJSONObject(); obj.emplace("value", value); obj.emplace("defaultValue", defaultValue); + obj.emplace("documentDefault", documentDefault); return obj; } } diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 736810bf3..79ec0f9cf 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -232,16 +232,19 @@ protected: T value; const T defaultValue; + const bool documentDefault; public: BaseSetting(const T & def, + const bool documentDefault, const std::string & name, const std::string & description, const std::set & aliases = {}) : AbstractSetting(name, description, aliases) , value(def) , defaultValue(def) + , documentDefault(documentDefault) { } operator const T &() const { return value; } @@ -288,8 +291,9 @@ public: const T & def, const std::string & name, const std::string & description, - const std::set & aliases = {}) - : BaseSetting(def, name, description, aliases) + const std::set & aliases = {}, + const bool documentDefault = true) + : BaseSetting(def, documentDefault, name, description, aliases) { options->addSetting(this); } @@ -311,7 +315,7 @@ public: const std::string & name, const std::string & description, const std::set & aliases = {}) - : BaseSetting(def, name, description, aliases) + : BaseSetting(def, true, name, description, aliases) , allowEmpty(allowEmpty) { options->addSetting(this); diff --git a/src/libutil/tests/config.cc b/src/libutil/tests/config.cc index 0ebdaf3db..8be6730dd 100644 --- a/src/libutil/tests/config.cc +++ b/src/libutil/tests/config.cc @@ -161,7 +161,7 @@ namespace nix { Setting setting{&config, "", "name-of-the-setting", "description"}; setting.assign("value"); - ASSERT_EQ(config.toJSON().dump(), R"#({"name-of-the-setting":{"aliases":[],"defaultValue":"","description":"description\n","value":"value"}})#"); + ASSERT_EQ(config.toJSON().dump(), R"#({"name-of-the-setting":{"aliases":[],"defaultValue":"","description":"description\n","documentDefault":true,"value":"value"}})#"); } TEST(Config, setSettingAlias) { From c32a5f4d380daafc33d129a097a494e1af8d9600 Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Wed, 1 Dec 2021 22:06:15 +0000 Subject: [PATCH 042/117] src/libutil/util.hh: fix build on gcc-11 Due to missing declaration the build fails as: src/libutil/util.hh:350:24: error: no match for 'operator||' (operand types are 'std::atomic' and 'bool') 350 | if (_isInterrupted || (interruptCheck && interruptCheck())) | ~~~~~~~~~~~~~~ ^~ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ | | | | std::atomic bool --- src/libutil/util.hh | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 0bdb37a79..bc96bfed1 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -11,6 +11,7 @@ #include #include +#include #include #include #include From 74898117360733bfc7ee57d1180a13f354c2ebbf Mon Sep 17 00:00:00 2001 From: David Arnold Date: Wed, 1 Dec 2021 18:36:19 -0500 Subject: [PATCH 043/117] Fix parent path check boundary - Previous to this commit the boundary was exclusive of the top level flake. - This is wrong since the top level flake is still a valid relative reference. - Now, the check boundary is inclusive of the top level flake. Signed-off-by: Timothy DeHerrera --- src/libfetchers/path.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index fb5702c4c..07e543c53 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -97,7 +97,7 @@ struct PathInputScheme : InputScheme // for security, ensure that if the parent is a store path, it's inside it if (store->isInStore(parent)) { auto storePath = store->printStorePath(store->toStorePath(parent).first); - if (!isInDir(absPath, storePath)) + if (!isDirOrInDir(absPath, storePath)) throw BadStorePath("relative path '%s' points outside of its parent's store path '%s'", path, storePath); } } else From 3bf7a868eefa5388400a63efd5c5b92b9177ff56 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Wed, 1 Dec 2021 18:38:32 -0600 Subject: [PATCH 044/117] darwin-install: nail down diskutil Same purpose as de9efa3b79af7886fcf2a67b6ce97d4f96a57421 For some unclear reason, we get occasional reports from people who do not have /usr/sbin on their PATH that the installer fails. It's a standard part of the PATH, so I have no clue what they're doing to remove it--but it's also fairly cheap to avoid. --- scripts/install-darwin-multi-user.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh index 96eba8310..afaa6783b 100644 --- a/scripts/install-darwin-multi-user.sh +++ b/scripts/install-darwin-multi-user.sh @@ -218,7 +218,7 @@ EOF setup_darwin_volume fi - if [ "$(diskutil info -plist /nix | xmllint --xpath "(/plist/dict/key[text()='GlobalPermissionsEnabled'])/following-sibling::*[1]" -)" = "" ]; then - failure "This script needs a /nix volume with global permissions! This may require running sudo diskutil enableOwnership /nix." + if [ "$(/usr/sbin/diskutil info -plist /nix | xmllint --xpath "(/plist/dict/key[text()='GlobalPermissionsEnabled'])/following-sibling::*[1]" -)" = "" ]; then + failure "This script needs a /nix volume with global permissions! This may require running sudo /usr/sbin/diskutil enableOwnership /nix." fi } From 8cbf862e6f6e8250d59491fdb88dfc11a805a970 Mon Sep 17 00:00:00 2001 From: Alexander Bantyev Date: Thu, 2 Dec 2021 11:35:50 +0300 Subject: [PATCH 045/117] Flakes: computeLocks: pass correct LockParent when reusing oldLock Previously, when we were attempting to reuse the old lockfile information in the computeLocks function, we have passed the parent of the current input to the next computeLocks call. This was incorrect, since the follows are resolved relative to the parent. This caused issues when we tried to reuse oldLock but couldn't for some reason (read: mustRefetch is true), in that case the follows were resolved incorrectly. Fix this by passing the correct parent, and adding some tests to prevent this particular regression from happening again. Closes https://github.com/NixOS/nix/issues/5697 --- src/libexpr/flake/flake.cc | 7 ++++++- tests/flakes.sh | 12 ++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 06136579e..33d253eee 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -480,11 +480,16 @@ LockedFlake lockFlake( } } + LockParent newParent { + .path = inputPath, + .absolute = false + }; + computeLocks( mustRefetch ? getFlake(state, oldLock->lockedRef, false, flakeCache).inputs : fakeInputs, - childNode, inputPath, oldLock, parent, parentPath); + childNode, inputPath, oldLock, newParent, parentPath); } else { /* We need to create a new lock file entry. So fetch diff --git a/tests/flakes.sh b/tests/flakes.sh index 9e10322b9..20966ab2a 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -722,6 +722,7 @@ cat > $flakeFollowsB/flake.nix < $flakeFollowsC/flake.nix < Date: Thu, 2 Dec 2021 17:46:44 +0100 Subject: [PATCH 046/117] Introduce builtins.groupBy primop This function is very useful in nixpkgs, but its implementation in Nix itself is rather slow due to it requiring a lot of attribute set and list appends. --- doc/manual/src/release-notes/rl-next.md | 3 ++ src/libexpr/primops.cc | 50 +++++++++++++++++++++++++ src/libexpr/value.hh | 2 + 3 files changed, 55 insertions(+) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 26c7d2cce..a6b22dfa7 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -5,3 +5,6 @@ * `nix develop` now has a flag `--unpack` to run `unpackPhase`. * Lists can now be compared lexicographically using the `<` operator. + +* New built-in function: `builtins.groupBy`, with the same functionality as + Nixpkgs' `lib.groupBy`, but faster. diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index d1f4d9009..66af373d7 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -2928,6 +2928,56 @@ static RegisterPrimOp primop_partition({ .fun = prim_partition, }); +static void prim_groupBy(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + state.forceFunction(*args[0], pos); + state.forceList(*args[1], pos); + + ValueVectorMap attrs; + + for (auto vElem : args[1]->listItems()) { + Value res; + state.callFunction(*args[0], *vElem, res, pos); + string name = state.forceStringNoCtx(res, pos); + Symbol sym = state.symbols.create(name); + auto vector = attrs.try_emplace(sym, ValueVector()).first; + vector->second.push_back(vElem); + } + + state.mkAttrs(v, attrs.size()); + + for (auto & i : attrs) { + Value * list = state.allocAttr(v, i.first); + auto size = i.second.size(); + state.mkList(*list, size); + memcpy(list->listElems(), i.second.data(), sizeof(Value *) * size); + } +} + +static RegisterPrimOp primop_groupBy({ + .name = "__groupBy", + .args = {"f", "list"}, + .doc = R"( + Groups elements of *list* together by the string returned from the + function *f* called on each element. It returns an attribute set + where each attribute value contains the elements of *list* that are + mapped to the same corresponding attribute name returned by *f*. + + For example, + + ```nix + builtins.groupBy (builtins.substring 0 1) ["foo" "bar" "baz"] + ``` + + evaluates to + + ```nix + { b = [ "bar" "baz" ]; f = [ "foo" ]; } + ``` + )", + .fun = prim_groupBy, +}); + static void prim_concatMap(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceFunction(*args[0], pos); diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 4b43e47ae..6b4f3c0ae 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -425,9 +425,11 @@ void mkPath(Value & v, const char * s); #if HAVE_BOEHMGC typedef std::vector > ValueVector; typedef std::map, traceable_allocator > > ValueMap; +typedef std::map, traceable_allocator > > ValueVectorMap; #else typedef std::vector ValueVector; typedef std::map ValueMap; +typedef std::map ValueVectorMap; #endif From 69b919887550483925b5a30388c13a63dabcca1a Mon Sep 17 00:00:00 2001 From: David Purdum Date: Thu, 2 Dec 2021 23:38:33 -0500 Subject: [PATCH 047/117] Fix docker command `docker -ti run nixos` does not run on docker version 20.10.7 (my machine). This fixes it to read `docker run -ti nixos`. --- doc/manual/src/installation/installing-docker.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/installation/installing-docker.md b/doc/manual/src/installation/installing-docker.md index 3d2255b7a..ee7614bdb 100644 --- a/doc/manual/src/installation/installing-docker.md +++ b/doc/manual/src/installation/installing-docker.md @@ -3,7 +3,7 @@ To run the latest stable release of Nix with Docker run the following command: ```console -$ docker -ti run nixos/nix +$ docker run -ti nixos/nix Unable to find image 'nixos/nix:latest' locally latest: Pulling from nixos/nix 5843afab3874: Pull complete From ad9322a48fce955fb30b111b0fb5c3b8007a6e2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Fri, 3 Dec 2021 18:36:33 +0100 Subject: [PATCH 048/117] update flake.nix to 21.11 --- flake.lock | 8 ++++---- flake.nix | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index 861af1c54..5fb200248 100644 --- a/flake.lock +++ b/flake.lock @@ -18,16 +18,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1632864508, - "narHash": "sha256-d127FIvGR41XbVRDPVvozUPQ/uRHbHwvfyKHwEt5xFM=", + "lastModified": 1638501222, + "narHash": "sha256-tgOZW/rLcB4P7iffx88K3iF70YH+44PtV6kVg5YGIg0=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "82891b5e2c2359d7e58d08849e4c89511ab94234", + "rev": "45c527ae2de6f2f36d5790273343f06cd14bc0d1", "type": "github" }, "original": { "id": "nixpkgs", - "ref": "nixos-21.05-small", + "ref": "nixos-21.11-small", "type": "indirect" } }, diff --git a/flake.nix b/flake.nix index 9a5d188ae..942388ab4 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,7 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "nixpkgs/nixos-21.05-small"; + inputs.nixpkgs.url = "nixpkgs/nixos-21.11-small"; inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; outputs = { self, nixpkgs, lowdown-src }: From 52f5fa948a4784b6a9b707770f4beee6a8674dee Mon Sep 17 00:00:00 2001 From: Connor Baker Date: Fri, 3 Dec 2021 22:16:02 -0500 Subject: [PATCH 049/117] Add support for .tar.gz shorthand (.tgz) to TarballInputScheme --- src/libfetchers/tarball.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 031ccc5f7..69d020748 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -176,6 +176,7 @@ struct TarballInputScheme : InputScheme if (!hasSuffix(url.path, ".zip") && !hasSuffix(url.path, ".tar") + && !hasSuffix(url.path, ".tgz") && !hasSuffix(url.path, ".tar.gz") && !hasSuffix(url.path, ".tar.xz") && !hasSuffix(url.path, ".tar.bz2") From d9b3adca756bdc10c298439f4502626bbcefcc2a Mon Sep 17 00:00:00 2001 From: Zhong Jianxin Date: Sat, 4 Dec 2021 20:46:29 +0800 Subject: [PATCH 050/117] launchd: Set NumberOfFiles to 4096 The default maxfiles on macOS 11 and macOS 12 is 256, which is too low for nix to work: ``` $ launchctl limit maxfiles maxfiles 256 unlimited ``` Set NumberOfFiles of nix-daemon to 4096 to avoid `Too many open files` error. --- misc/launchd/org.nixos.nix-daemon.plist.in | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in index f1b439840..da1970f69 100644 --- a/misc/launchd/org.nixos.nix-daemon.plist.in +++ b/misc/launchd/org.nixos.nix-daemon.plist.in @@ -25,5 +25,10 @@ /var/log/nix-daemon.log StandardOutPath /dev/null + SoftResourceLimits + + NumberOfFiles + 4096 + From 80934561114df14137ef07b6c07e17dff3a95544 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Sat, 4 Dec 2021 23:07:17 -0600 Subject: [PATCH 051/117] document some darwin mount settings --- .../src/installation/installing-binary.md | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 96fa34635..4367654a2 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -119,6 +119,30 @@ this to run the installer, but it may help if you run into trouble: - update `/etc/synthetic.conf` to direct macOS to create a "synthetic" empty root directory to mount your volume - specify mount options for the volume in `/etc/fstab` + - `rw`: read-write + - `noauto`: prevent the system from auto-mounting the volume (so the + LaunchDaemon mentioned below can control mounting it, and to avoid + masking problems with that mounting service). + - `nobrowse`: prevent the Nix Store volume from showing up on your + desktop; also keeps Spotlight from spending resources to index + this volume + - if you have FileVault enabled - generate an encryption password - put it in your system Keychain From fa58bff1ab9aacffd50f8200d72843af7abd827f Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 6 Dec 2021 11:45:18 +0100 Subject: [PATCH 052/117] Fallback when the daemon is too old for CA derivations Fix #5712 --- src/libstore/remote-store.cc | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index a627e9cf1..c3d23affb 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -684,6 +684,14 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id, Callback> callback) noexcept { auto conn(getConnection()); + + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 27) { + warn("The daemon is too old for content-addressed derivations. I’ll do what I can"); + try { + callback(nullptr); + } catch (...) { return callback.rethrow(); } + } + conn->to << wopQueryRealisation; conn->to << id.to_string(); conn.processStderr(); From d1aaa7ef71713b6693ad3ddf8704ce62bab82095 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 6 Dec 2021 13:33:39 +0100 Subject: [PATCH 053/117] Change version to X.Y.Z Fixes #5732. --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index 68151b2e1..fad066f80 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.5 \ No newline at end of file +2.5.0 \ No newline at end of file From 0a2fa2d68460eca946c91b72cd264725e5cda7db Mon Sep 17 00:00:00 2001 From: Alexander Bantyev Date: Fri, 26 Nov 2021 17:38:46 +0300 Subject: [PATCH 054/117] RunPager: restore stdout upon pager exit Before this change, stdout was closed after the pager exits. This is fine for non-interactive commands where we want to exit right after the pager exits anyways, but for interactive things (e.g. nix repl) this breaks the output after we quit the pager. Keep the initial stdout fd as part of RunPager, and restore it in RunPager::~RunPager using dup2. --- src/libmain/shared.cc | 4 ++-- src/libmain/shared.hh | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index b6bfea8cb..4404e0195 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -427,7 +427,7 @@ RunPager::RunPager() }); pid.setKillSignal(SIGINT); - + stdout = fcntl(STDOUT_FILENO, F_DUPFD_CLOEXEC, 0); if (dup2(toPager.writeSide.get(), STDOUT_FILENO) == -1) throw SysError("dupping stdout"); } @@ -438,7 +438,7 @@ RunPager::~RunPager() try { if (pid != -1) { std::cout.flush(); - close(STDOUT_FILENO); + dup2(stdout, STDOUT_FILENO); pid.wait(); } } catch (...) { diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 05277d90a..ed012959b 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -88,6 +88,7 @@ public: private: Pid pid; + int stdout; }; extern volatile ::sig_atomic_t blockInt; From 33926ed1e75c06cbf70a920e462841cc9daa0520 Mon Sep 17 00:00:00 2001 From: Alexander Bantyev Date: Fri, 26 Nov 2021 18:03:07 +0300 Subject: [PATCH 055/117] nix repl: add :log Add a :log command that shows logs for a derivation. Closes https://github.com/NixOS/nix/issues/3504 Co-authored-by: Taeer Bar-Yam --- doc/manual/src/release-notes/rl-next.md | 2 ++ src/nix/repl.cc | 26 +++++++++++++++++++++++-- src/nix/repl.md | 7 +++++-- tests/repl.sh | 13 +++++++++++++ tests/simple-failing.builder.sh | 2 ++ tests/simple-failing.nix | 8 ++++++++ 6 files changed, 54 insertions(+), 4 deletions(-) create mode 100644 tests/simple-failing.builder.sh create mode 100644 tests/simple-failing.nix diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index a6b22dfa7..cf2da9aa8 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -8,3 +8,5 @@ * New built-in function: `builtins.groupBy`, with the same functionality as Nixpkgs' `lib.groupBy`, but faster. + +* `nix repl` now has a `:log` command. diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 2649eb0bd..42143871f 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -430,7 +430,8 @@ bool NixRepl::processLine(string line) << " :s Build dependencies of derivation, then start nix-shell\n" << " :t Describe result of evaluation\n" << " :u Build derivation, then start nix-shell\n" - << " :doc Show documentation of a builtin function\n"; + << " :doc Show documentation of a builtin function\n" + << " :log Show logs for a derivation\n"; } else if (command == ":a" || command == ":add") { @@ -500,7 +501,7 @@ bool NixRepl::processLine(string line) runNix("nix-shell", {state->store->printStorePath(drvPath)}); } - else if (command == ":b" || command == ":i" || command == ":s") { + else if (command == ":b" || command == ":i" || command == ":s" || command == ":log") { Value v; evalString(arg, v); StorePath drvPath = getDerivationPath(v); @@ -514,6 +515,27 @@ bool NixRepl::processLine(string line) logger->cout(" %s -> %s", outputName, state->store->printStorePath(outputPath)); } else if (command == ":i") { runNix("nix-env", {"-i", drvPathRaw}); + } else if (command == ":log") { + settings.readOnlyMode = true; + Finally roModeReset([&]() { + settings.readOnlyMode = false; + }); + auto subs = getDefaultSubstituters(); + + subs.push_front(state->store); + + bool foundLog = false; + RunPager pager; + for (auto & sub : subs) { + auto log = sub->getBuildLog(drvPath); + if (log) { + printInfo("got build log for '%s' from '%s'", drvPathRaw, sub->getUri()); + logger->writeToStdout(*log); + foundLog = true; + break; + } + } + if (!foundLog) throw Error("build log of '%s' is not available", drvPathRaw); } else { runNix("nix-shell", {drvPathRaw}); } diff --git a/src/nix/repl.md b/src/nix/repl.md index bba60f871..9b6f2bee3 100644 --- a/src/nix/repl.md +++ b/src/nix/repl.md @@ -35,14 +35,17 @@ R""( nix-repl> emacs.drvPath "/nix/store/lp0sjrhgg03y2n0l10n70rg0k7hhyz0l-emacs-27.1.drv" - nix-repl> drv = runCommand "hello" { buildInputs = [ hello ]; } "hello > $out" + nix-repl> drv = runCommand "hello" { buildInputs = [ hello ]; } "hello; hello > $out" - nix-repl> :b x + nix-repl> :b drv this derivation produced the following outputs: out -> /nix/store/0njwbgwmkwls0w5dv9mpc1pq5fj39q0l-hello nix-repl> builtins.readFile drv "Hello, world!\n" + + nix-repl> :log drv + Hello, world! ``` # Description diff --git a/tests/repl.sh b/tests/repl.sh index d360821f2..f592822bc 100644 --- a/tests/repl.sh +++ b/tests/repl.sh @@ -3,6 +3,13 @@ source common.sh replCmds=" simple = import ./simple.nix :b simple +:log simple +" + +replFailingCmds=" +failing = import ./simple-failing.nix +:b failing +:log failing " testRepl () { @@ -12,6 +19,12 @@ testRepl () { local outPath=$(echo "$replOutput" |& grep -o -E "$NIX_STORE_DIR/\w*-simple") nix path-info "${nixArgs[@]}" "$outPath" + # simple.nix prints a PATH during build + echo "$replOutput" | grep -qs 'PATH=' || fail "nix repl :log doesn't output logs" + local replOutput="$(nix repl "${nixArgs[@]}" <<< "$replFailingCmds")" + echo "$replOutput" + echo "$replOutput" | grep -qs 'This should fail' \ + || fail "nix repl :log doesn't output logs for a failed derivation" } # Simple test, try building a drv diff --git a/tests/simple-failing.builder.sh b/tests/simple-failing.builder.sh new file mode 100644 index 000000000..6285488ca --- /dev/null +++ b/tests/simple-failing.builder.sh @@ -0,0 +1,2 @@ +echo "This should fail" +exit 1 diff --git a/tests/simple-failing.nix b/tests/simple-failing.nix new file mode 100644 index 000000000..1e3a080ed --- /dev/null +++ b/tests/simple-failing.nix @@ -0,0 +1,8 @@ +with import ./config.nix; + +mkDerivation { + name = "simple-failing"; + builder = ./simple-failing.builder.sh; + PATH = ""; + goodPath = path; +} From ae21aab456f4e07fa621d87487c40381e45947dc Mon Sep 17 00:00:00 2001 From: Jan Tojnar Date: Mon, 6 Dec 2021 16:42:57 +0100 Subject: [PATCH 056/117] Update manual links Fixes: https://github.com/NixOS/nixos-homepage/issues/762 --- scripts/install-nix-from-closure.sh | 6 +++--- src/libstore/build/derivation-goal.cc | 2 +- src/libstore/build/worker.cc | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/install-nix-from-closure.sh b/scripts/install-nix-from-closure.sh index b5e2fea83..d543b4463 100644 --- a/scripts/install-nix-from-closure.sh +++ b/scripts/install-nix-from-closure.sh @@ -38,7 +38,7 @@ fi # Determine if we could use the multi-user installer or not if [ "$(uname -s)" = "Linux" ]; then - echo "Note: a multi-user installation is possible. See https://nixos.org/nix/manual/#sect-multi-user-installation" >&2 + echo "Note: a multi-user installation is possible. See https://nixos.org/manual/nix/stable/installation/installing-binary.html#multi-user-installation" >&2 fi case "$(uname -s)" in @@ -98,7 +98,7 @@ while [ $# -gt 0 ]; do echo " providing multi-user support and better isolation for local builds." echo " Both for security and reproducibility, this method is recommended if" echo " supported on your platform." - echo " See https://nixos.org/nix/manual/#sect-multi-user-installation" + echo " See https://nixos.org/manual/nix/stable/installation/installing-binary.html#multi-user-installation" echo "" echo " --no-daemon: Simple, single-user installation that does not require root and is" echo " trivial to uninstall." @@ -144,7 +144,7 @@ if ! [ -e "$dest" ]; then fi if ! [ -w "$dest" ]; then - echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see https://nixos.org/nix/manual/#ssec-multi-user. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2 + echo "$0: directory $dest exists, but is not writable by you. This could indicate that another user has already performed a single-user installation of Nix on this system. If you wish to enable multi-user support see https://nixos.org/manual/nix/stable/installation/multi-user.html. If you wish to continue with a single-user install for $USER please run 'chown -R $USER $dest' as root." >&2 exit 1 fi diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index b924d23b2..60945403e 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -655,7 +655,7 @@ void DerivationGoal::tryLocalBuild() { throw Error( "unable to build with a primary store that isn't a local store; " "either pass a different '--store' or enable remote builds." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); + "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html"); } diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 55afb5cca..f11c5ce68 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -281,11 +281,11 @@ void Worker::run(const Goals & _topGoals) if (getMachines().empty()) throw Error("unable to start any build; either increase '--max-jobs' " "or enable remote builds." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); + "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html"); else throw Error("unable to start any build; remote machines may not have " "all required system features." - "\nhttps://nixos.org/nix/manual/#chap-distributed-builds"); + "\nhttps://nixos.org/manual/nix/stable/advanced-topics/distributed-builds.html"); } assert(!awake.empty()); From 853ef1304c379f03ef33f40bc68225d528cc6105 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Dec 2021 21:45:09 +0100 Subject: [PATCH 057/117] Inline file --- tests/simple-failing.builder.sh | 2 -- tests/simple-failing.nix | 6 +++++- 2 files changed, 5 insertions(+), 3 deletions(-) delete mode 100644 tests/simple-failing.builder.sh diff --git a/tests/simple-failing.builder.sh b/tests/simple-failing.builder.sh deleted file mode 100644 index 6285488ca..000000000 --- a/tests/simple-failing.builder.sh +++ /dev/null @@ -1,2 +0,0 @@ -echo "This should fail" -exit 1 diff --git a/tests/simple-failing.nix b/tests/simple-failing.nix index 1e3a080ed..d176c9c51 100644 --- a/tests/simple-failing.nix +++ b/tests/simple-failing.nix @@ -2,7 +2,11 @@ with import ./config.nix; mkDerivation { name = "simple-failing"; - builder = ./simple-failing.builder.sh; + builder = builtins.toFile "builder.sh" + '' + echo "This should fail" + exit 1 + ''; PATH = ""; goodPath = path; } From 92ff5b4254b1a38a9dba4529d3dd4e9b11ea9333 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 7 Dec 2021 22:22:02 +0100 Subject: [PATCH 058/117] Tweak warning --- src/libstore/remote-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index c3d23affb..7f7e973e9 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -686,7 +686,7 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id, auto conn(getConnection()); if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 27) { - warn("The daemon is too old for content-addressed derivations. I’ll do what I can"); + warn("the daemon is too old to support content-addressed derivations, please upgrade it to 2.4"); try { callback(nullptr); } catch (...) { return callback.rethrow(); } From b6d08a9e3ff76a38636377abe383b179b38e37b5 Mon Sep 17 00:00:00 2001 From: "Travis A. Everett" Date: Wed, 3 Nov 2021 23:17:48 -0500 Subject: [PATCH 059/117] darwin-install: fix break from bad vim plugins --- scripts/create-darwin-volume.sh | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/scripts/create-darwin-volume.sh b/scripts/create-darwin-volume.sh index b52232dd3..7e1a38cbd 100755 --- a/scripts/create-darwin-volume.sh +++ b/scripts/create-darwin-volume.sh @@ -440,7 +440,22 @@ add_nix_vol_fstab_line() { # shellcheck disable=SC1003,SC2026 local escaped_mountpoint="${NIX_ROOT/ /'\\\'040}" shift - EDITOR="/usr/bin/ex" _sudo "to add nix to fstab" "$@" < "$SCRATCH/ex_cleanroom_wrapper" <&2 - _sudo "to install the Nix volume mounter" /usr/bin/ex "$NIX_VOLUME_MOUNTD_DEST" < Date: Thu, 9 Dec 2021 16:35:27 +0100 Subject: [PATCH 060/117] Properly take the last assignment in the REPL When a variable is assigned in the REPL, make sure to remove any possible reference to the old one so that we correctly pick the new one afterwards Fix #5706 --- src/nix/repl.cc | 2 ++ tests/repl.sh | 1 + 2 files changed, 3 insertions(+) diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 42143871f..f453343f3 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -672,6 +672,8 @@ void NixRepl::addVarToScope(const Symbol & name, Value & v) { if (displ >= envSize) throw Error("environment full; cannot add more variables"); + if (auto oldVar = staticEnv.find(name); oldVar != staticEnv.vars.end()) + staticEnv.vars.erase(oldVar); staticEnv.vars.emplace_back(name, displ); staticEnv.sort(); env->values[displ++] = &v; diff --git a/tests/repl.sh b/tests/repl.sh index f592822bc..995db869c 100644 --- a/tests/repl.sh +++ b/tests/repl.sh @@ -1,6 +1,7 @@ source common.sh replCmds=" +simple = 1 simple = import ./simple.nix :b simple :log simple From 0bd93ed7478a827ea81e71dbce113cd7e073d8cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amon=20Stopin=C5=A1ek?= Date: Thu, 9 Dec 2021 16:44:24 +0000 Subject: [PATCH 061/117] Fix link to nixos channel The old link returned 404 errror. --- doc/manual/src/package-management/basic-package-mgmt.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/package-management/basic-package-mgmt.md b/doc/manual/src/package-management/basic-package-mgmt.md index 50c6d3c2d..5f1d7a89c 100644 --- a/doc/manual/src/package-management/basic-package-mgmt.md +++ b/doc/manual/src/package-management/basic-package-mgmt.md @@ -40,7 +40,7 @@ $ nix-channel --update > > On NixOS, you’re automatically subscribed to a NixOS channel > corresponding to your NixOS major release (e.g. -> ). A NixOS channel is identical +> ). A NixOS channel is identical > to the Nixpkgs channel, except that it contains only Linux binaries > and is updated only if a set of regression tests succeed. From 5d64c33fa534773cbebfec3eee59d7b0ac74e59c Mon Sep 17 00:00:00 2001 From: Tom Bereknyei Date: Thu, 9 Dec 2021 14:27:05 -0500 Subject: [PATCH 062/117] add pull_request_template.md --- .github/PULL_REQUEST_TEMPLATE/pull_request_template.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .github/PULL_REQUEST_TEMPLATE/pull_request_template.md diff --git a/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md b/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md new file mode 100644 index 000000000..537aa0909 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md @@ -0,0 +1,7 @@ +**Release Notes** +Please include relevant [release notes](https://github.com/NixOS/nix/blob/master/doc/manual/src/release-notes/rl-next.md) as needed. + + +**Testing** + +If this issue is a regression or something that should block release, please consider including a test either in the [testsuite](https://github.com/NixOS/nix/tree/master/tests) or as a [hydraJob]( https://github.com/NixOS/nix/blob/master/flake.nix#L396) so that it can be part of the [automatic checks](https://hydra.nixos.org/jobset/nix/master). From 4194b682500ee1e40b575e9729a8ddcbb6ac1518 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Janne=20He=C3=9F?= Date: Fri, 10 Dec 2021 12:58:12 +0100 Subject: [PATCH 063/117] doc: Fix command for distributed builds We also need to build impurely or the lookup will not work. --- doc/manual/src/advanced-topics/distributed-builds.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/advanced-topics/distributed-builds.md b/doc/manual/src/advanced-topics/distributed-builds.md index 580b36736..c4c60db15 100644 --- a/doc/manual/src/advanced-topics/distributed-builds.md +++ b/doc/manual/src/advanced-topics/distributed-builds.md @@ -53,8 +53,8 @@ example, the following command allows you to build a derivation for $ uname Linux -$ nix build \ - '(with import { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \ +$ nix build --impure \ + --expr '(with import { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \ --builders 'ssh://mac x86_64-darwin' [1/0/1 built, 0.0 MiB DL] building foo on ssh://mac From 20b1290103f23e40614518291b8d4847e37fea05 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 10 Dec 2021 15:12:55 +0100 Subject: [PATCH 064/117] Ignore system.nfs4_acl Closes #1584. --- src/libstore/globals.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 2f9e8c6e8..433deaf0f 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -799,7 +799,7 @@ public: )"}; Setting ignoredAcls{ - this, {"security.selinux"}, "ignored-acls", + this, {"security.selinux", "system.nfs4_acl"}, "ignored-acls", R"( A list of ACLs that should be ignored, normally Nix attempts to remove all ACLs from files and directories in the Nix store, but From be1055f2cc8841954521bf30a6f886cf3c807a28 Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Fri, 10 Dec 2021 17:00:32 +0100 Subject: [PATCH 065/117] extract_archive: use copy_pathname instead of set_pathname. Libarchive documentation mentions that archive_entry_set_pathname expects us to keep the passed string alive, which we don't. --- src/libutil/tarfile.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index 50e691a3d..934b70162 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -93,7 +93,7 @@ static void extract_archive(TarArchive & archive, const Path & destDir) else archive.check(r); - archive_entry_set_pathname(entry, + archive_entry_copy_pathname(entry, (destDir + "/" + name).c_str()); archive.check(archive_read_extract(archive.archive, entry, flags)); From 0e0de90b35b61edcfae3030c150b9d7e42bf39fe Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Fri, 10 Dec 2021 17:03:51 +0100 Subject: [PATCH 066/117] extract_archive: fix "Hard-link target '...'" error Fixes #5741 --- src/libutil/tarfile.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index 934b70162..790bc943a 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -96,6 +96,13 @@ static void extract_archive(TarArchive & archive, const Path & destDir) archive_entry_copy_pathname(entry, (destDir + "/" + name).c_str()); + // Patch hardlink path + const char *original_hardlink = archive_entry_hardlink(entry); + if (original_hardlink) { + archive_entry_copy_hardlink(entry, + (destDir + "/" + original_hardlink).c_str()); + } + archive.check(archive_read_extract(archive.archive, entry, flags)); } From 33e96820d52dcfea387214f84ff2271959b3467b Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Sat, 11 Dec 2021 16:02:08 +0100 Subject: [PATCH 067/117] EvalCommand::getEvalState: use gc tracable allocator for EvalState --- src/libcmd/command.cc | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index fd3edfc46..429cd32cc 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -73,8 +73,13 @@ ref EvalCommand::getEvalStore() ref EvalCommand::getEvalState() { - if (!evalState) - evalState = std::make_shared(searchPath, getEvalStore(), getStore()); + if (!evalState) evalState = +#if HAVE_BOEHMGC + std::allocate_shared(traceable_allocator(), +#else + std::make_shared( +#endif + searchPath, getEvalStore(), getStore()); return ref(evalState); } From 3f417c8d1c3fc05af16766f21f13f2ecc15192a0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 29 Nov 2021 12:42:01 +0100 Subject: [PATCH 068/117] Add release notes entry for #5149 --- doc/manual/src/release-notes/rl-next.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index cf2da9aa8..61c189d22 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,5 +1,9 @@ # Release 2.5 (2021-XX-XX) +* The garbage collector no longer blocks new builds, so the message + `waiting for the big garbage collector lock...` is a thing of the + past. + * Binary cache stores now have a setting `compression-level`. * `nix develop` now has a flag `--unpack` to run `unpackPhase`. From f133001dc8c849053b8af9933902d11f274426bd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Dec 2021 16:48:44 +0100 Subject: [PATCH 069/117] Move rl-next.md to rl-2.5.md --- doc/manual/src/SUMMARY.md.in | 1 + doc/manual/src/release-notes/rl-2.5.md | 16 ++++++++++++++++ doc/manual/src/release-notes/rl-next.md | 17 +---------------- 3 files changed, 18 insertions(+), 16 deletions(-) create mode 100644 doc/manual/src/release-notes/rl-2.5.md diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 8d9b061ba..2876be52a 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -72,6 +72,7 @@ - [CLI guideline](contributing/cli-guideline.md) - [Release Notes](release-notes/release-notes.md) - [Release X.Y (202?-??-??)](release-notes/rl-next.md) + - [Release 2.5 (2021-12-13)](release-notes/rl-2.5.md) - [Release 2.4 (2021-11-01)](release-notes/rl-2.4.md) - [Release 2.3 (2019-09-04)](release-notes/rl-2.3.md) - [Release 2.2 (2019-01-11)](release-notes/rl-2.2.md) diff --git a/doc/manual/src/release-notes/rl-2.5.md b/doc/manual/src/release-notes/rl-2.5.md new file mode 100644 index 000000000..61c189d22 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.5.md @@ -0,0 +1,16 @@ +# Release 2.5 (2021-XX-XX) + +* The garbage collector no longer blocks new builds, so the message + `waiting for the big garbage collector lock...` is a thing of the + past. + +* Binary cache stores now have a setting `compression-level`. + +* `nix develop` now has a flag `--unpack` to run `unpackPhase`. + +* Lists can now be compared lexicographically using the `<` operator. + +* New built-in function: `builtins.groupBy`, with the same functionality as + Nixpkgs' `lib.groupBy`, but faster. + +* `nix repl` now has a `:log` command. diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index 61c189d22..c869b5e2f 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1,16 +1 @@ -# Release 2.5 (2021-XX-XX) - -* The garbage collector no longer blocks new builds, so the message - `waiting for the big garbage collector lock...` is a thing of the - past. - -* Binary cache stores now have a setting `compression-level`. - -* `nix develop` now has a flag `--unpack` to run `unpackPhase`. - -* Lists can now be compared lexicographically using the `<` operator. - -* New built-in function: `builtins.groupBy`, with the same functionality as - Nixpkgs' `lib.groupBy`, but faster. - -* `nix repl` now has a `:log` command. +# Release X.Y (202?-??-??) From 55dbb7f1ccc0e991df6b09af0a16ce9246ac3f19 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 13 Dec 2021 16:56:44 +0100 Subject: [PATCH 070/117] More properly track the status of CA builds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make the build of unresolved derivations return the same status as the resolved one, except in the case of an `AlreadyValid` in which case it will return `ResolvesToAlreadyValid` to mean that the outputs of the unresolved derivation weren’t known, but the resolved one is. --- src/libstore/build/derivation-goal.cc | 26 +++++++++++++++++--------- src/libstore/build/derivation-goal.hh | 4 ++-- src/libstore/store-api.hh | 3 ++- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 60945403e..a713d7222 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -464,7 +464,6 @@ void DerivationGoal::inputsRealised() Derivation drvResolved { *std::move(attempt) }; auto pathResolved = writeDerivation(worker.store, drvResolved); - resolvedDrv = drvResolved; auto msg = fmt("Resolved derivation: '%s' -> '%s'", worker.store.printStorePath(drvPath), @@ -475,9 +474,9 @@ void DerivationGoal::inputsRealised() worker.store.printStorePath(pathResolved), }); - auto resolvedGoal = worker.makeDerivationGoal( + resolvedDrvGoal = worker.makeDerivationGoal( pathResolved, wantedOutputs, buildMode); - addWaitee(resolvedGoal); + addWaitee(resolvedDrvGoal); state = &DerivationGoal::resolvedFinished; return; @@ -938,16 +937,17 @@ void DerivationGoal::buildDone() } void DerivationGoal::resolvedFinished() { - assert(resolvedDrv); + assert(resolvedDrvGoal); + auto resolvedDrv = *resolvedDrvGoal->drv; - auto resolvedHashes = staticOutputHashes(worker.store, *resolvedDrv); + auto resolvedHashes = staticOutputHashes(worker.store, resolvedDrv); StorePathSet outputPaths; // `wantedOutputs` might be empty, which means “all the outputs” auto realWantedOutputs = wantedOutputs; if (realWantedOutputs.empty()) - realWantedOutputs = resolvedDrv->outputNames(); + realWantedOutputs = resolvedDrv.outputNames(); for (auto & wantedOutput : realWantedOutputs) { assert(initialOutputs.count(wantedOutput) != 0); @@ -979,9 +979,17 @@ void DerivationGoal::resolvedFinished() { outputPaths ); - // This is potentially a bit fishy in terms of error reporting. Not sure - // how to do it in a cleaner way - amDone(nrFailed == 0 ? ecSuccess : ecFailed, ex); + auto status = [&]() { + auto resolvedResult = resolvedDrvGoal->getResult(); + switch (resolvedResult.status) { + case BuildResult::AlreadyValid: + return BuildResult::ResolvesToAlreadyValid; + default: + return resolvedResult.status; + } + }(); + + done(status); } HookReply DerivationGoal::tryBuildHook() diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh index 704b77caf..e112542c7 100644 --- a/src/libstore/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -50,8 +50,8 @@ struct DerivationGoal : public Goal /* The path of the derivation. */ StorePath drvPath; - /* The path of the corresponding resolved derivation */ - std::optional resolvedDrv; + /* The goal for the corresponding resolved derivation */ + std::shared_ptr resolvedDrvGoal; /* The specific outputs that we need to build. Empty means all of them. */ diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index aa44651d4..3a385b293 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -151,6 +151,7 @@ struct BuildResult DependencyFailed, LogLimitExceeded, NotDeterministic, + ResolvesToAlreadyValid, } status = MiscFailure; std::string errorMsg; @@ -170,7 +171,7 @@ struct BuildResult time_t startTime = 0, stopTime = 0; bool success() { - return status == Built || status == Substituted || status == AlreadyValid; + return status == Built || status == Substituted || status == AlreadyValid || status == ResolvesToAlreadyValid; } }; From 2eec2f765a86b8954f3a74ff148bc70a2d32be27 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 13 Dec 2021 16:58:43 +0100 Subject: [PATCH 071/117] Add a crude tracing mechansim for the build results MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a `_NIX_TRACE_BUILT_OUTPUTS` environment variable that can be set to a filename in which the result of each build will be logged. This is intentionally crude and undocumented as it’s only meant to be a temporary thing to assess the usefulness of CA derivations. Any other use would need a cleaner re-implementation first. --- src/libstore/build/derivation-goal.cc | 8 ++++++++ src/libstore/store-api.hh | 23 +++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index a713d7222..c6ab5c3d1 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -1337,6 +1338,13 @@ void DerivationGoal::done(BuildResult::Status status, std::optional ex) } worker.updateProgress(); + + auto traceBuiltOutputsFile = getEnv("_NIX_TRACE_BUILT_OUTPUTS").value_or(""); + if (traceBuiltOutputsFile != "") { + std::fstream fs; + fs.open(traceBuiltOutputsFile, std::fstream::out); + fs << worker.store.printStorePath(drvPath) << "\t" << result.toString() << std::endl; + } } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 3a385b293..3dd446f23 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -155,6 +155,29 @@ struct BuildResult } status = MiscFailure; std::string errorMsg; + std::string toString() const { + auto strStatus = [&]() { + switch (status) { + case Built: return "Built"; + case Substituted: return "Substituted"; + case AlreadyValid: return "AlreadyValid"; + case PermanentFailure: return "PermanentFailure"; + case InputRejected: return "InputRejected"; + case OutputRejected: return "OutputRejected"; + case TransientFailure: return "TransientFailure"; + case CachedFailure: return "CachedFailure"; + case TimedOut: return "TimedOut"; + case MiscFailure: return "MiscFailure"; + case DependencyFailed: return "DependencyFailed"; + case LogLimitExceeded: return "LogLimitExceeded"; + case NotDeterministic: return "NotDeterministic"; + case ResolvesToAlreadyValid: return "ResolvesToAlreadyValid"; + default: return "Unknown"; + }; + }(); + return strStatus + ((errorMsg == "") ? "" : " : " + errorMsg); + } + /* How many times this build was performed. */ unsigned int timesBuilt = 0; From 6d30f9e6fea7d451033653f8f167aef58f7f5347 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Dec 2021 17:16:46 +0100 Subject: [PATCH 072/117] Explicitly make GC roots client sockets blocking On macOS / BSD, these sockets inherit the non-blocking flag of the server soocket, which is not what we want. https://github.com/dotnet/runtime/issues/25069 https://bugs.python.org/issue7995 https://hydra.nixos.org/build/161439304 --- src/libstore/gc.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 7a414da6b..ee123d84e 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -535,6 +535,12 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } }); + /* On macOS, accepted sockets inherit the + non-blocking flag from the server socket, so + explicitly make it blocking. */ + if (fcntl(fdServer.get(), F_SETFL, fcntl(fdServer.get(), F_GETFL) & ~O_NONBLOCK) == -1) + abort(); + while (true) { try { auto path = readLine(fdClient.get()); From 05fcc681ac07c4ea04b365e6b03a76f3e6c54c94 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Dec 2021 16:14:57 +0100 Subject: [PATCH 073/117] Retry on ECONNREFUSED https://hydra.nixos.org/build/161439235 --- src/libstore/gc.cc | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index ee123d84e..fa68cd8f3 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -126,7 +126,17 @@ void LocalStore::addTempRoot(const StorePath & path) auto socketPath = stateDir.get() + gcSocketPath; debug("connecting to '%s'", socketPath); state->fdRootsSocket = createUnixDomainSocket(); - nix::connect(state->fdRootsSocket.get(), socketPath); + try { + nix::connect(state->fdRootsSocket.get(), socketPath); + } catch (SysError & e) { + /* The garbage collector may have exited, so we need to + restart. */ + if (e.errNo == ECONNREFUSED) { + debug("GC socket connection refused"); + state->fdRootsSocket.close(); + goto restart; + } + } } try { From df1edd143fcb2532e32bbcb28f774ced1402cda4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Dec 2021 16:24:25 +0100 Subject: [PATCH 074/117] gc-non-blocking.sh: Don't hang indefinitely if the GC roots server crashes --- tests/gc-non-blocking.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/gc-non-blocking.sh b/tests/gc-non-blocking.sh index 8b21c6f1c..0d781485d 100644 --- a/tests/gc-non-blocking.sh +++ b/tests/gc-non-blocking.sh @@ -19,7 +19,7 @@ pid=$! sleep 2 -outPath=$(nix-build -o "$TEST_ROOT/result" -E " +outPath=$(nix-build --max-silent-time 60 -o "$TEST_ROOT/result" -E " with import ./config.nix; mkDerivation { name = \"non-blocking\"; From d17d46cfc2e38efb63565270a1fba7b03b37de16 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Dec 2021 17:12:32 +0100 Subject: [PATCH 075/117] More debug info --- src/libstore/gc.cc | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index fa68cd8f3..e35199b3d 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -533,6 +533,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) AutoCloseFD fdClient = accept(fdServer.get(), nullptr, nullptr); if (!fdClient) continue; + debug("GC roots server accepted new client"); + /* Process the connection in a separate thread. */ auto fdClient_ = fdClient.get(); std::thread clientThread([&, fdClient = std::move(fdClient)]() { @@ -575,7 +577,10 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } else printError("received garbage instead of a root from client"); writeFull(fdClient.get(), "1", false); - } catch (Error &) { break; } + } catch (Error & e) { + debug("reading GC root from client: %s", e.msg()); + break; + } } }); From 2567b74c66ed52f752578373d4ec81ac2f09d6f0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Dec 2021 19:14:32 +0100 Subject: [PATCH 076/117] nix run: Tweak docs --- src/nix/run.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/nix/run.md b/src/nix/run.md index 697db9a96..cd3b978c0 100644 --- a/src/nix/run.md +++ b/src/nix/run.md @@ -43,11 +43,15 @@ program specified by the app definition. If *installable* evaluates to a derivation, it will try to execute the program `/bin/`, where *out* is the primary output store -path of the derivation and *name* is the `meta.mainProgram` attribute -of the derivation if it exists, and otherwise the `pname` attribute of -the derivation or the name part of the value of the `name` attribute of -the derivation (e.g. if `name` is set to `hello-1.10`, it will run -`$out/bin/hello`). +path of the derivation, and *name* is the first of the following that +exists: + +* The `meta.mainProgram` attribute of the derivation. +* The `pname` attribute of the derivation. +* The name part of the value of the `name` attribute of the derivation. + +For instance, if `name` is set to `hello-1.10`, `nix run` will run +`$out/bin/hello`. # Flake output attributes From e2e5f3a78f31e4e72170ebfe48cbbb082860cc24 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Dec 2021 19:37:30 +0100 Subject: [PATCH 077/117] Respect /etc/hosts https://hydra.nixos.org/build/161439329 --- src/libmain/shared.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 4404e0195..f605184bb 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -145,10 +145,10 @@ static void preloadNSS() { * * All other platforms are unaffected. */ - if (dlopen (LIBNSS_DNS_SO, RTLD_NOW) == NULL) { - printMsg(Verbosity::lvlWarn, fmt("Unable to load nss_dns backend")); - } - __nss_configure_lookup ("hosts", "dns"); + if (!dlopen(LIBNSS_DNS_SO, RTLD_NOW)) + warn("unable to load nss_dns backend"); + // FIXME: get hosts entry from nsswitch.conf. + __nss_configure_lookup("hosts", "files dns"); #endif }); } From 18cb094aab2e50c9b90e484592212bfce026b640 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Dec 2021 19:57:06 +0100 Subject: [PATCH 078/117] rl-2.5.md: Set date --- doc/manual/src/release-notes/rl-2.5.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/release-notes/rl-2.5.md b/doc/manual/src/release-notes/rl-2.5.md index 61c189d22..dd6fd3b0f 100644 --- a/doc/manual/src/release-notes/rl-2.5.md +++ b/doc/manual/src/release-notes/rl-2.5.md @@ -1,4 +1,4 @@ -# Release 2.5 (2021-XX-XX) +# Release 2.5 (2021-12-13) * The garbage collector no longer blocks new builds, so the message `waiting for the big garbage collector lock...` is a thing of the From 46d86e06ba54dc708fa8fd7d0109845fa2ac402e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Dec 2021 20:28:53 +0100 Subject: [PATCH 079/117] Simplify --- src/libstore/build/local-derivation-goal.cc | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index edfa46052..66d950b42 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1779,15 +1779,14 @@ void LocalDerivationGoal::runChild() i686-linux build on an x86_64-linux machine. */ struct utsname utsbuf; uname(&utsbuf); - if (drv->platform == "i686-linux" && - (settings.thisSystem == "x86_64-linux" || - (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) { + if ((drv->platform == "i686-linux" + && (settings.thisSystem == "x86_64-linux" + || (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) + || drv->platform == "armv7l-linux" + || drv->platform == "armv6l-linux") + { if (personality(PER_LINUX32) == -1) - throw SysError("cannot set i686-linux personality"); - } - if (drv->platform == "armv7l-linux" || drv->platform == "armv6l-linux") { - if (personality(PER_LINUX32) == -1) - throw SysError("cannot set 32bit linux personality"); + throw SysError("cannot set 32-bit personality"); } /* Impersonate a Linux 2.6 machine to get some determinism in From 32a62b0d25116ffe0d66ecaf6a479fbc915d196e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Dec 2021 20:54:43 +0100 Subject: [PATCH 080/117] Re-allow inputs.x.url = "/path/to/file" --- src/libexpr/flake/flake.cc | 8 ++-- src/libexpr/flake/flakeref.cc | 77 +++++++++++++++++++---------------- src/libexpr/flake/flakeref.hh | 10 ++++- tests/flakes.sh | 10 +++++ 4 files changed, 65 insertions(+), 40 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 33d253eee..f598400fc 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -155,7 +155,7 @@ static FlakeInput parseFlakeInput(EvalState & state, if (!attrs.empty()) throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, pos); if (url) - input.ref = parseFlakeRef(*url, baseDir, true); + input.ref = parseFlakeRef(*url, baseDir, true, input.isFlake); } if (!input.follows && !input.ref) @@ -194,8 +194,8 @@ static Flake getFlake( state, originalRef, allowLookup, flakeCache); // Guard against symlink attacks. - auto flakeDir = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir); - auto flakeFile = canonPath(flakeDir + "/flake.nix"); + auto flakeDir = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir, true); + auto flakeFile = canonPath(flakeDir + "/flake.nix", true); if (!isInDir(flakeFile, sourceInfo.actualPath)) throw Error("'flake.nix' file of flake '%s' escapes from '%s'", lockedRef, state.store->printStorePath(sourceInfo.storePath)); @@ -570,7 +570,7 @@ LockedFlake lockFlake( }; // Bring in the current ref for relative path resolution if we have it - auto parentPath = canonPath(flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir); + auto parentPath = canonPath(flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir, true); computeLocks( flake.inputs, newLockFile.root, {}, diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc index 29128d789..c3b74e0fe 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libexpr/flake/flakeref.cc @@ -48,9 +48,12 @@ FlakeRef FlakeRef::resolve(ref store) const } FlakeRef parseFlakeRef( - const std::string & url, const std::optional & baseDir, bool allowMissing) + const std::string & url, + const std::optional & baseDir, + bool allowMissing, + bool isFlake) { - auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing); + auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing, isFlake); if (fragment != "") throw Error("unexpected fragment '%s' in flake reference '%s'", fragment, url); return flakeRef; @@ -67,7 +70,10 @@ std::optional maybeParseFlakeRef( } std::pair parseFlakeRefWithFragment( - const std::string & url, const std::optional & baseDir, bool allowMissing) + const std::string & url, + const std::optional & baseDir, + bool allowMissing, + bool isFlake) { using namespace fetchers; @@ -112,46 +118,49 @@ std::pair parseFlakeRefWithFragment( to 'baseDir'). If so, search upward to the root of the repo (i.e. the directory containing .git). */ - path = absPath(path, baseDir, true); + path = absPath(path, baseDir); - if (!S_ISDIR(lstat(path).st_mode)) - throw BadURL("path '%s' is not a flake (because it's not a directory)", path); + if (isFlake) { - if (!allowMissing && !pathExists(path + "/flake.nix")) - throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path); + if (!S_ISDIR(lstat(path).st_mode)) + throw BadURL("path '%s' is not a flake (because it's not a directory)", path); - auto flakeRoot = path; - std::string subdir; + if (!allowMissing && !pathExists(path + "/flake.nix")) + throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path); - while (flakeRoot != "/") { - if (pathExists(flakeRoot + "/.git")) { - auto base = std::string("git+file://") + flakeRoot; + auto flakeRoot = path; + std::string subdir; - auto parsedURL = ParsedURL{ - .url = base, // FIXME - .base = base, - .scheme = "git+file", - .authority = "", - .path = flakeRoot, - .query = decodeQuery(match[2]), - }; + while (flakeRoot != "/") { + if (pathExists(flakeRoot + "/.git")) { + auto base = std::string("git+file://") + flakeRoot; - if (subdir != "") { - if (parsedURL.query.count("dir")) - throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url); - parsedURL.query.insert_or_assign("dir", subdir); + auto parsedURL = ParsedURL{ + .url = base, // FIXME + .base = base, + .scheme = "git+file", + .authority = "", + .path = flakeRoot, + .query = decodeQuery(match[2]), + }; + + if (subdir != "") { + if (parsedURL.query.count("dir")) + throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url); + parsedURL.query.insert_or_assign("dir", subdir); + } + + if (pathExists(flakeRoot + "/.git/shallow")) + parsedURL.query.insert_or_assign("shallow", "1"); + + return std::make_pair( + FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")), + fragment); } - if (pathExists(flakeRoot + "/.git/shallow")) - parsedURL.query.insert_or_assign("shallow", "1"); - - return std::make_pair( - FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")), - fragment); + subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir); + flakeRoot = dirOf(flakeRoot); } - - subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir); - flakeRoot = dirOf(flakeRoot); } } else { diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh index 0292eb210..1fddfd9a0 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libexpr/flake/flakeref.hh @@ -62,13 +62,19 @@ struct FlakeRef std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef); FlakeRef parseFlakeRef( - const std::string & url, const std::optional & baseDir = {}, bool allowMissing = false); + const std::string & url, + const std::optional & baseDir = {}, + bool allowMissing = false, + bool isFlake = true); std::optional maybeParseFlake( const std::string & url, const std::optional & baseDir = {}); std::pair parseFlakeRefWithFragment( - const std::string & url, const std::optional & baseDir = {}, bool allowMissing = false); + const std::string & url, + const std::optional & baseDir = {}, + bool allowMissing = false, + bool isFlake = true); std::optional> maybeParseFlakeRefWithFragment( const std::string & url, const std::optional & baseDir = {}); diff --git a/tests/flakes.sh b/tests/flakes.sh index 20966ab2a..942d07010 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -249,6 +249,14 @@ cat > $flake3Dir/flake.nix < $flake3Dir/flake.nix < \$out + [[ \$(cat \${inputs.nonFlake}/README.md) = \$(cat \${inputs.nonFlakeFile}) ]] + [[ \${inputs.nonFlakeFile} = \${inputs.nonFlakeFile2} ]] ''; }; }; From 95bd5da341aec4b99f4719d1b1ab2343ee14fad7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Dec 2021 21:08:42 +0100 Subject: [PATCH 081/117] queryMissing(): Use a bigger thread pool This is a temporary hack until we make queryMissing() async. Closes #5324. --- src/libstore/misc.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 32786e963..6409874ff 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -7,6 +7,7 @@ #include "topo-sort.hh" #include "callback.hh" #include "closure.hh" +#include "filetransfer.hh" namespace nix { @@ -100,7 +101,8 @@ void Store::queryMissing(const std::vector & targets, downloadSize_ = narSize_ = 0; - ThreadPool pool; + // FIXME: make async. + ThreadPool pool(fileTransferSettings.httpConnections); struct State { From bcd4d2e4c6c8ec03ec8cf5d3558975c9152314d5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 14 Dec 2021 00:01:42 +0100 Subject: [PATCH 082/117] Bump version --- .version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.version b/.version index fad066f80..914ec9671 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.5.0 \ No newline at end of file +2.6.0 \ No newline at end of file From bb8a53ab08c43425c6073aff5fe34bd5bc93dd13 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 14 Dec 2021 09:15:24 +0100 Subject: [PATCH 083/117] Add a test for boolean config options in flakes --- tests/flake-local-settings.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/flake-local-settings.sh b/tests/flake-local-settings.sh index 09f6b4ca8..98ad60174 100644 --- a/tests/flake-local-settings.sh +++ b/tests/flake-local-settings.sh @@ -18,6 +18,7 @@ chmod +x echoing-post-hook.sh cat < flake.nix { nixConfig.post-build-hook = "$PWD/echoing-post-hook.sh"; + nixConfig.allow-dirty = false; # See #5621 outputs = a: { defaultPackage.$system = import ./simple.nix; From 1b6adfe18f17c90a56ab172e031d040815b3e1f3 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 15 Dec 2021 13:49:05 +0100 Subject: [PATCH 084/117] Revert "update flake.nix to 21.11" This reverts commit ad9322a48fce955fb30b111b0fb5c3b8007a6e2e because it breaks hydraJobs.buildStatic.* and the evaluation of some tests. --- flake.lock | 8 ++++---- flake.nix | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/flake.lock b/flake.lock index 5fb200248..861af1c54 100644 --- a/flake.lock +++ b/flake.lock @@ -18,16 +18,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1638501222, - "narHash": "sha256-tgOZW/rLcB4P7iffx88K3iF70YH+44PtV6kVg5YGIg0=", + "lastModified": 1632864508, + "narHash": "sha256-d127FIvGR41XbVRDPVvozUPQ/uRHbHwvfyKHwEt5xFM=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "45c527ae2de6f2f36d5790273343f06cd14bc0d1", + "rev": "82891b5e2c2359d7e58d08849e4c89511ab94234", "type": "github" }, "original": { "id": "nixpkgs", - "ref": "nixos-21.11-small", + "ref": "nixos-21.05-small", "type": "indirect" } }, diff --git a/flake.nix b/flake.nix index 942388ab4..9a5d188ae 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,7 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "nixpkgs/nixos-21.11-small"; + inputs.nixpkgs.url = "nixpkgs/nixos-21.05-small"; inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; outputs = { self, nixpkgs, lowdown-src }: From f66923efdefa5160222b6d6ef0dd9fa822242116 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 15 Dec 2021 19:13:06 +0100 Subject: [PATCH 085/117] pkgconfig -> pkg-config --- flake.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/flake.nix b/flake.nix index 9a5d188ae..806ed45de 100644 --- a/flake.nix +++ b/flake.nix @@ -96,7 +96,7 @@ buildPackages.mdbook buildPackages.autoconf-archive buildPackages.autoreconfHook - buildPackages.pkgconfig + buildPackages.pkg-config # Tests buildPackages.git @@ -343,7 +343,7 @@ nativeBuildInputs = [ buildPackages.autoconf-archive buildPackages.autoreconfHook - buildPackages.pkgconfig + buildPackages.pkg-config ]; buildInputs = From 11ee875a6deaba021e7ea4e3f6c3563e3fab1425 Mon Sep 17 00:00:00 2001 From: jesse Date: Wed, 15 Dec 2021 10:53:03 -0800 Subject: [PATCH 086/117] List aarch64 as supported macOS hardware Resolves #5767 --- doc/manual/src/installation/supported-platforms.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/manual/src/installation/supported-platforms.md b/doc/manual/src/installation/supported-platforms.md index 8ef1f0e78..8ca3ce8d4 100644 --- a/doc/manual/src/installation/supported-platforms.md +++ b/doc/manual/src/installation/supported-platforms.md @@ -4,4 +4,4 @@ Nix is currently supported on the following platforms: - Linux (i686, x86\_64, aarch64). - - macOS (x86\_64). + - macOS (x86\_64, aarch64). From c3f9344cae1921e4ddf1568469d063c2bba787dc Mon Sep 17 00:00:00 2001 From: Noah Snelson Date: Wed, 15 Dec 2021 23:40:37 -0800 Subject: [PATCH 087/117] Fix typo in `src/nix/shell.md` Changed `If not command` to `If no command`, also specified that the default shell launched in `nix shell` can be specified with `$SHELL`. --- src/nix/shell.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/nix/shell.md b/src/nix/shell.md index 2a379e03f..90b81fb2f 100644 --- a/src/nix/shell.md +++ b/src/nix/shell.md @@ -41,8 +41,8 @@ R""( # Description -`nix shell` runs a command in an environment in which the `$PATH` -variable provides the specified *installables*. If not command is -specified, it starts the default shell of your user account. +`nix shell` runs a command in an environment in which the `$PATH` variable +provides the specified *installables*. If no command is specified, it starts the +default shell of your user account specified by `$SHELL`. )"" From 6942ee8a84f695332149f9a14046b51f2cb6462c Mon Sep 17 00:00:00 2001 From: John Axel Eriksson Date: Thu, 16 Dec 2021 11:29:05 +0100 Subject: [PATCH 088/117] docker: fix image so that nix profile works nix profile will otherwise throw this error: error: path '/nix/var/nix/profiles/default/manifest.nix' is not in the Nix store That's not entirely true since manifest.nix is within a directory in the nix store but nix profile seems to require the manifest.nix itself to be a store path. --- docker.nix | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docker.nix b/docker.nix index 2a13c23fb..ffb553f78 100644 --- a/docker.nix +++ b/docker.nix @@ -137,11 +137,8 @@ let name = "root-profile-env"; paths = defaultPkgs; }; - profile = pkgs.buildPackages.runCommand "user-environment" { } '' - mkdir $out - cp -a ${rootEnv}/* $out/ - - cat > $out/manifest.nix < $out < Date: Thu, 16 Dec 2021 19:59:13 +0100 Subject: [PATCH 089/117] Add a test case for builtins.groupBy --- tests/lang/eval-okay-groupBy.exp | 1 + tests/lang/eval-okay-groupBy.nix | 5 +++++ 2 files changed, 6 insertions(+) create mode 100644 tests/lang/eval-okay-groupBy.exp create mode 100644 tests/lang/eval-okay-groupBy.nix diff --git a/tests/lang/eval-okay-groupBy.exp b/tests/lang/eval-okay-groupBy.exp new file mode 100644 index 000000000..bfca5652a --- /dev/null +++ b/tests/lang/eval-okay-groupBy.exp @@ -0,0 +1 @@ +{ "1" = [ 9 ]; "2" = [ 8 ]; "3" = [ 13 29 ]; "4" = [ 3 4 10 11 17 18 ]; "5" = [ 0 23 26 28 ]; "6" = [ 1 12 21 27 30 ]; "7" = [ 7 22 ]; "8" = [ 14 ]; "9" = [ 19 ]; b = [ 16 25 ]; c = [ 24 ]; d = [ 2 ]; e = [ 5 6 15 31 ]; f = [ 20 ]; } diff --git a/tests/lang/eval-okay-groupBy.nix b/tests/lang/eval-okay-groupBy.nix new file mode 100644 index 000000000..862d89dbd --- /dev/null +++ b/tests/lang/eval-okay-groupBy.nix @@ -0,0 +1,5 @@ +with import ./lib.nix; + +builtins.groupBy (n: + builtins.substring 0 1 (builtins.hashString "sha256" (toString n)) +) (range 0 31) From c260640dec6e35c714b666a1e7adede5aab6972a Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Dec 2021 21:48:38 +0100 Subject: [PATCH 090/117] Fix docker instructions --- doc/manual/src/installation/installing-docker.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/manual/src/installation/installing-docker.md b/doc/manual/src/installation/installing-docker.md index ee7614bdb..9d6d8f2d9 100644 --- a/doc/manual/src/installation/installing-docker.md +++ b/doc/manual/src/installation/installing-docker.md @@ -16,7 +16,7 @@ nix (Nix) 2.3.12 35ca4ada6e96:/# exit ``` -# What is included in Nix' Docker image? +# What is included in Nix's Docker image? The official Docker image is created using `pkgs.dockerTools.buildLayeredImage` (and not with `Dockerfile` as it is usual with Docker images). You can still @@ -54,6 +54,6 @@ You can also build a Docker image from source yourself: ```console $ nix build ./\#hydraJobs.dockerImage.x86_64-linux -$ docker load -i ./result +$ docker load -i ./result/image.tar.gz $ docker run -ti nix:2.5pre20211105 ``` From ec8f24ed3a3115cba85f908515c423112e5b13e0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 16 Dec 2021 21:26:22 +0100 Subject: [PATCH 091/117] Ignore EPERM when unsharing FS state On Docker (but not podman), unshare(CLONE_FS) fails with EPERM. So let's ignore it and hope nothing bad happens. Attempted fix for #5777. --- src/libstore/filetransfer.cc | 8 +------- src/libutil/util.cc | 8 ++++++++ src/libutil/util.hh | 5 +++++ 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 4621a8217..17980ab22 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -544,13 +544,7 @@ struct curlFileTransfer : public FileTransfer stopWorkerThread(); }); -#ifdef __linux__ - /* Cause this thread to not share any FS attributes with the main thread, - because this causes setns() in restoreMountNamespace() to fail. - Ideally, this would happen in the std::thread() constructor. */ - if (unshare(CLONE_FS) != 0) - throw SysError("unsharing filesystem state in download thread"); -#endif + unshareFilesystem(); std::map> items; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 1b6467eb2..43fea1b1e 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1660,6 +1660,14 @@ void restoreMountNamespace() #endif } +void unshareFilesystem() +{ +#ifdef __linux__ + if (unshare(CLONE_FS) != 0 && errno != EPERM) + throw SysError("unsharing filesystem state in download thread"); +#endif +} + void restoreProcessContext(bool restoreMounts) { restoreSignals(); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index bc96bfed1..4cc043a84 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -311,6 +311,11 @@ void saveMountNamespace(); if saveMountNamespace() was never called. */ void restoreMountNamespace(); +/* Cause this thread to not share any FS attributes with the main + thread, because this causes setns() in restoreMountNamespace() to + fail. */ +void unshareFilesystem(); + class ExecError : public Error { From 6f1b3d711e6174286ae8ecb83c5d8a348e9fa2c3 Mon Sep 17 00:00:00 2001 From: regnat Date: Fri, 17 Dec 2021 09:14:28 +0100 Subject: [PATCH 092/117] Test the update of multiple channels MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make sure that `nix-channel --update` won’t accidentaly remove one --- tests/nix-channel.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/nix-channel.sh b/tests/nix-channel.sh index 63c0f97ba..54b8f5979 100644 --- a/tests/nix-channel.sh +++ b/tests/nix-channel.sh @@ -35,16 +35,14 @@ grep -q 'item.*attrPath="foo".*name="dependencies-top"' $TEST_ROOT/meta.xml nix-env -i dependencies-top [ -e $TEST_HOME/.nix-profile/foobar ] -clearProfiles -rm -f $TEST_HOME/.nix-channels - # Test updating from a tarball -nix-channel --add file://$TEST_ROOT/foo/nixexprs.tar.bz2 foo +nix-channel --add file://$TEST_ROOT/foo/nixexprs.tar.bz2 bar nix-channel --update # Do a query. nix-env -qa \* --meta --xml --out-path > $TEST_ROOT/meta.xml grep -q 'meta.*description.*Random test package' $TEST_ROOT/meta.xml +grep -q 'item.*attrPath="bar".*name="dependencies-top"' $TEST_ROOT/meta.xml grep -q 'item.*attrPath="foo".*name="dependencies-top"' $TEST_ROOT/meta.xml # Do an install. From c3f68b5db7b9d4b963102a2abc20e20445a8bec5 Mon Sep 17 00:00:00 2001 From: Nicolas Mattia Date: Fri, 17 Dec 2021 22:03:33 +0100 Subject: [PATCH 093/117] Replace cpptoml with toml11 --- src/cpptoml/LICENSE | 18 - src/cpptoml/cpptoml.h | 3668 --------------------------- src/toml11/LICENSE | 21 + src/toml11/README.md | 1966 ++++++++++++++ src/toml11/toml.hpp | 46 + src/toml11/toml/color.hpp | 64 + src/toml11/toml/combinator.hpp | 306 +++ src/toml11/toml/comments.hpp | 472 ++++ src/toml11/toml/datetime.hpp | 631 +++++ src/toml11/toml/exception.hpp | 65 + src/toml11/toml/from.hpp | 19 + src/toml11/toml/get.hpp | 1117 ++++++++ src/toml11/toml/into.hpp | 19 + src/toml11/toml/lexer.hpp | 293 +++ src/toml11/toml/literal.hpp | 113 + src/toml11/toml/macros.hpp | 121 + src/toml11/toml/parser.hpp | 2364 +++++++++++++++++ src/toml11/toml/region.hpp | 417 +++ src/toml11/toml/result.hpp | 717 ++++++ src/toml11/toml/serializer.hpp | 922 +++++++ src/toml11/toml/source_location.hpp | 233 ++ src/toml11/toml/storage.hpp | 43 + src/toml11/toml/string.hpp | 225 ++ src/toml11/toml/traits.hpp | 327 +++ src/toml11/toml/types.hpp | 173 ++ src/toml11/toml/utility.hpp | 149 ++ src/toml11/toml/value.hpp | 2035 +++++++++++++++ 27 files changed, 12858 insertions(+), 3686 deletions(-) delete mode 100644 src/cpptoml/LICENSE delete mode 100644 src/cpptoml/cpptoml.h create mode 100644 src/toml11/LICENSE create mode 100644 src/toml11/README.md create mode 100644 src/toml11/toml.hpp create mode 100644 src/toml11/toml/color.hpp create mode 100644 src/toml11/toml/combinator.hpp create mode 100644 src/toml11/toml/comments.hpp create mode 100644 src/toml11/toml/datetime.hpp create mode 100644 src/toml11/toml/exception.hpp create mode 100644 src/toml11/toml/from.hpp create mode 100644 src/toml11/toml/get.hpp create mode 100644 src/toml11/toml/into.hpp create mode 100644 src/toml11/toml/lexer.hpp create mode 100644 src/toml11/toml/literal.hpp create mode 100644 src/toml11/toml/macros.hpp create mode 100644 src/toml11/toml/parser.hpp create mode 100644 src/toml11/toml/region.hpp create mode 100644 src/toml11/toml/result.hpp create mode 100644 src/toml11/toml/serializer.hpp create mode 100644 src/toml11/toml/source_location.hpp create mode 100644 src/toml11/toml/storage.hpp create mode 100644 src/toml11/toml/string.hpp create mode 100644 src/toml11/toml/traits.hpp create mode 100644 src/toml11/toml/types.hpp create mode 100644 src/toml11/toml/utility.hpp create mode 100644 src/toml11/toml/value.hpp diff --git a/src/cpptoml/LICENSE b/src/cpptoml/LICENSE deleted file mode 100644 index 8802c4fa5..000000000 --- a/src/cpptoml/LICENSE +++ /dev/null @@ -1,18 +0,0 @@ -Copyright (c) 2014 Chase Geigle - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/src/cpptoml/cpptoml.h b/src/cpptoml/cpptoml.h deleted file mode 100644 index 5a00da3b4..000000000 --- a/src/cpptoml/cpptoml.h +++ /dev/null @@ -1,3668 +0,0 @@ -/** - * @file cpptoml.h - * @author Chase Geigle - * @date May 2013 - */ - -#ifndef CPPTOML_H -#define CPPTOML_H - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#if __cplusplus > 201103L -#define CPPTOML_DEPRECATED(reason) [[deprecated(reason)]] -#elif defined(__clang__) -#define CPPTOML_DEPRECATED(reason) __attribute__((deprecated(reason))) -#elif defined(__GNUG__) -#define CPPTOML_DEPRECATED(reason) __attribute__((deprecated)) -#elif defined(_MSC_VER) -#if _MSC_VER < 1910 -#define CPPTOML_DEPRECATED(reason) __declspec(deprecated) -#else -#define CPPTOML_DEPRECATED(reason) [[deprecated(reason)]] -#endif -#endif - -namespace cpptoml -{ -class writer; // forward declaration -class base; // forward declaration -#if defined(CPPTOML_USE_MAP) -// a std::map will ensure that entries a sorted, albeit at a slight -// performance penalty relative to the (default) unordered_map -using string_to_base_map = std::map>; -#else -// by default an unordered_map is used for best performance as the -// toml specification does not require entries to be sorted -using string_to_base_map - = std::unordered_map>; -#endif - -// if defined, `base` will retain type information in form of an enum class -// such that static_cast can be used instead of dynamic_cast -// #define CPPTOML_NO_RTTI - -template -class option -{ - public: - option() : empty_{true} - { - // nothing - } - - option(T value) : empty_{false}, value_(std::move(value)) - { - // nothing - } - - explicit operator bool() const - { - return !empty_; - } - - const T& operator*() const - { - return value_; - } - - const T* operator->() const - { - return &value_; - } - - template - T value_or(U&& alternative) const - { - if (!empty_) - return value_; - return static_cast(std::forward(alternative)); - } - - private: - bool empty_; - T value_; -}; - -struct local_date -{ - int year = 0; - int month = 0; - int day = 0; -}; - -struct local_time -{ - int hour = 0; - int minute = 0; - int second = 0; - int microsecond = 0; -}; - -struct zone_offset -{ - int hour_offset = 0; - int minute_offset = 0; -}; - -struct local_datetime : local_date, local_time -{ -}; - -struct offset_datetime : local_datetime, zone_offset -{ - static inline struct offset_datetime from_zoned(const struct tm& t) - { - offset_datetime dt; - dt.year = t.tm_year + 1900; - dt.month = t.tm_mon + 1; - dt.day = t.tm_mday; - dt.hour = t.tm_hour; - dt.minute = t.tm_min; - dt.second = t.tm_sec; - - char buf[16]; - strftime(buf, 16, "%z", &t); - - int offset = std::stoi(buf); - dt.hour_offset = offset / 100; - dt.minute_offset = offset % 100; - return dt; - } - - CPPTOML_DEPRECATED("from_local has been renamed to from_zoned") - static inline struct offset_datetime from_local(const struct tm& t) - { - return from_zoned(t); - } - - static inline struct offset_datetime from_utc(const struct tm& t) - { - offset_datetime dt; - dt.year = t.tm_year + 1900; - dt.month = t.tm_mon + 1; - dt.day = t.tm_mday; - dt.hour = t.tm_hour; - dt.minute = t.tm_min; - dt.second = t.tm_sec; - return dt; - } -}; - -CPPTOML_DEPRECATED("datetime has been renamed to offset_datetime") -typedef offset_datetime datetime; - -class fill_guard -{ - public: - fill_guard(std::ostream& os) : os_(os), fill_{os.fill()} - { - // nothing - } - - ~fill_guard() - { - os_.fill(fill_); - } - - private: - std::ostream& os_; - std::ostream::char_type fill_; -}; - -inline std::ostream& operator<<(std::ostream& os, const local_date& dt) -{ - fill_guard g{os}; - os.fill('0'); - - using std::setw; - os << setw(4) << dt.year << "-" << setw(2) << dt.month << "-" << setw(2) - << dt.day; - - return os; -} - -inline std::ostream& operator<<(std::ostream& os, const local_time& ltime) -{ - fill_guard g{os}; - os.fill('0'); - - using std::setw; - os << setw(2) << ltime.hour << ":" << setw(2) << ltime.minute << ":" - << setw(2) << ltime.second; - - if (ltime.microsecond > 0) - { - os << "."; - int power = 100000; - for (int curr_us = ltime.microsecond; curr_us; power /= 10) - { - auto num = curr_us / power; - os << num; - curr_us -= num * power; - } - } - - return os; -} - -inline std::ostream& operator<<(std::ostream& os, const zone_offset& zo) -{ - fill_guard g{os}; - os.fill('0'); - - using std::setw; - - if (zo.hour_offset != 0 || zo.minute_offset != 0) - { - if (zo.hour_offset > 0) - { - os << "+"; - } - else - { - os << "-"; - } - os << setw(2) << std::abs(zo.hour_offset) << ":" << setw(2) - << std::abs(zo.minute_offset); - } - else - { - os << "Z"; - } - - return os; -} - -inline std::ostream& operator<<(std::ostream& os, const local_datetime& dt) -{ - return os << static_cast(dt) << "T" - << static_cast(dt); -} - -inline std::ostream& operator<<(std::ostream& os, const offset_datetime& dt) -{ - return os << static_cast(dt) - << static_cast(dt); -} - -template -struct is_one_of; - -template -struct is_one_of : std::is_same -{ -}; - -template -struct is_one_of -{ - const static bool value - = std::is_same::value || is_one_of::value; -}; - -template -class value; - -template -struct valid_value - : is_one_of -{ -}; - -template -struct value_traits; - -template -struct valid_value_or_string_convertible -{ - - const static bool value = valid_value::type>::value - || std::is_convertible::value; -}; - -template -struct value_traits::value>::type> -{ - using value_type = typename std::conditional< - valid_value::type>::value, - typename std::decay::type, std::string>::type; - - using type = value; - - static value_type construct(T&& val) - { - return value_type(val); - } -}; - -template -struct value_traits< - T, - typename std::enable_if< - !valid_value_or_string_convertible::value - && std::is_floating_point::type>::value>::type> -{ - using value_type = typename std::decay::type; - - using type = value; - - static value_type construct(T&& val) - { - return value_type(val); - } -}; - -template -struct value_traits< - T, typename std::enable_if< - !valid_value_or_string_convertible::value - && !std::is_floating_point::type>::value - && std::is_signed::type>::value>::type> -{ - using value_type = int64_t; - - using type = value; - - static value_type construct(T&& val) - { - if (val < (std::numeric_limits::min)()) - throw std::underflow_error{"constructed value cannot be " - "represented by a 64-bit signed " - "integer"}; - - if (val > (std::numeric_limits::max)()) - throw std::overflow_error{"constructed value cannot be represented " - "by a 64-bit signed integer"}; - - return static_cast(val); - } -}; - -template -struct value_traits< - T, typename std::enable_if< - !valid_value_or_string_convertible::value - && std::is_unsigned::type>::value>::type> -{ - using value_type = int64_t; - - using type = value; - - static value_type construct(T&& val) - { - if (val > static_cast((std::numeric_limits::max)())) - throw std::overflow_error{"constructed value cannot be represented " - "by a 64-bit signed integer"}; - - return static_cast(val); - } -}; - -class array; -class table; -class table_array; - -template -struct array_of_trait -{ - using return_type = option>; -}; - -template <> -struct array_of_trait -{ - using return_type = option>>; -}; - -template -inline std::shared_ptr::type> make_value(T&& val); -inline std::shared_ptr make_array(); - -namespace detail -{ -template -inline std::shared_ptr make_element(); -} - -inline std::shared_ptr make_table(); -inline std::shared_ptr make_table_array(bool is_inline = false); - -#if defined(CPPTOML_NO_RTTI) -/// Base type used to store underlying data type explicitly if RTTI is disabled -enum class base_type -{ - NONE, - STRING, - LOCAL_TIME, - LOCAL_DATE, - LOCAL_DATETIME, - OFFSET_DATETIME, - INT, - FLOAT, - BOOL, - TABLE, - ARRAY, - TABLE_ARRAY -}; - -/// Type traits class to convert C++ types to enum member -template -struct base_type_traits; - -template <> -struct base_type_traits -{ - static const base_type type = base_type::STRING; -}; - -template <> -struct base_type_traits -{ - static const base_type type = base_type::LOCAL_TIME; -}; - -template <> -struct base_type_traits -{ - static const base_type type = base_type::LOCAL_DATE; -}; - -template <> -struct base_type_traits -{ - static const base_type type = base_type::LOCAL_DATETIME; -}; - -template <> -struct base_type_traits -{ - static const base_type type = base_type::OFFSET_DATETIME; -}; - -template <> -struct base_type_traits -{ - static const base_type type = base_type::INT; -}; - -template <> -struct base_type_traits -{ - static const base_type type = base_type::FLOAT; -}; - -template <> -struct base_type_traits -{ - static const base_type type = base_type::BOOL; -}; - -template <> -struct base_type_traits
-{ - static const base_type type = base_type::TABLE; -}; - -template <> -struct base_type_traits -{ - static const base_type type = base_type::ARRAY; -}; - -template <> -struct base_type_traits -{ - static const base_type type = base_type::TABLE_ARRAY; -}; -#endif - -/** - * A generic base TOML value used for type erasure. - */ -class base : public std::enable_shared_from_this -{ - public: - virtual ~base() = default; - - virtual std::shared_ptr clone() const = 0; - - /** - * Determines if the given TOML element is a value. - */ - virtual bool is_value() const - { - return false; - } - - /** - * Determines if the given TOML element is a table. - */ - virtual bool is_table() const - { - return false; - } - - /** - * Converts the TOML element into a table. - */ - std::shared_ptr
as_table() - { - if (is_table()) - return std::static_pointer_cast
(shared_from_this()); - return nullptr; - } - /** - * Determines if the TOML element is an array of "leaf" elements. - */ - virtual bool is_array() const - { - return false; - } - - /** - * Converts the TOML element to an array. - */ - std::shared_ptr as_array() - { - if (is_array()) - return std::static_pointer_cast(shared_from_this()); - return nullptr; - } - - /** - * Determines if the given TOML element is an array of tables. - */ - virtual bool is_table_array() const - { - return false; - } - - /** - * Converts the TOML element into a table array. - */ - std::shared_ptr as_table_array() - { - if (is_table_array()) - return std::static_pointer_cast(shared_from_this()); - return nullptr; - } - - /** - * Attempts to coerce the TOML element into a concrete TOML value - * of type T. - */ - template - std::shared_ptr> as(); - - template - std::shared_ptr> as() const; - - template - void accept(Visitor&& visitor, Args&&... args) const; - -#if defined(CPPTOML_NO_RTTI) - base_type type() const - { - return type_; - } - - protected: - base(const base_type t) : type_(t) - { - // nothing - } - - private: - const base_type type_ = base_type::NONE; - -#else - protected: - base() - { - // nothing - } -#endif -}; - -/** - * A concrete TOML value representing the "leaves" of the "tree". - */ -template -class value : public base -{ - struct make_shared_enabler - { - // nothing; this is a private key accessible only to friends - }; - - template - friend std::shared_ptr::type> - cpptoml::make_value(U&& val); - - public: - static_assert(valid_value::value, "invalid value type"); - - std::shared_ptr clone() const override; - - value(const make_shared_enabler&, const T& val) : value(val) - { - // nothing; note that users cannot actually invoke this function - // because they lack access to the make_shared_enabler. - } - - bool is_value() const override - { - return true; - } - - /** - * Gets the data associated with this value. - */ - T& get() - { - return data_; - } - - /** - * Gets the data associated with this value. Const version. - */ - const T& get() const - { - return data_; - } - - private: - T data_; - - /** - * Constructs a value from the given data. - */ -#if defined(CPPTOML_NO_RTTI) - value(const T& val) : base(base_type_traits::type), data_(val) - { - } -#else - value(const T& val) : data_(val) - { - } -#endif - - value(const value& val) = delete; - value& operator=(const value& val) = delete; -}; - -template -std::shared_ptr::type> make_value(T&& val) -{ - using value_type = typename value_traits::type; - using enabler = typename value_type::make_shared_enabler; - return std::make_shared( - enabler{}, value_traits::construct(std::forward(val))); -} - -template -inline std::shared_ptr> base::as() -{ -#if defined(CPPTOML_NO_RTTI) - if (type() == base_type_traits::type) - return std::static_pointer_cast>(shared_from_this()); - else - return nullptr; -#else - return std::dynamic_pointer_cast>(shared_from_this()); -#endif -} - -// special case value to allow getting an integer parameter as a -// double value -template <> -inline std::shared_ptr> base::as() -{ -#if defined(CPPTOML_NO_RTTI) - if (type() == base_type::FLOAT) - return std::static_pointer_cast>(shared_from_this()); - - if (type() == base_type::INT) - { - auto v = std::static_pointer_cast>(shared_from_this()); - return make_value(static_cast(v->get())); - } -#else - if (auto v = std::dynamic_pointer_cast>(shared_from_this())) - return v; - - if (auto v = std::dynamic_pointer_cast>(shared_from_this())) - return make_value(static_cast(v->get())); -#endif - - return nullptr; -} - -template -inline std::shared_ptr> base::as() const -{ -#if defined(CPPTOML_NO_RTTI) - if (type() == base_type_traits::type) - return std::static_pointer_cast>(shared_from_this()); - else - return nullptr; -#else - return std::dynamic_pointer_cast>(shared_from_this()); -#endif -} - -// special case value to allow getting an integer parameter as a -// double value -template <> -inline std::shared_ptr> base::as() const -{ -#if defined(CPPTOML_NO_RTTI) - if (type() == base_type::FLOAT) - return std::static_pointer_cast>( - shared_from_this()); - - if (type() == base_type::INT) - { - auto v = as(); - // the below has to be a non-const value due to a bug in - // libc++: https://llvm.org/bugs/show_bug.cgi?id=18843 - return make_value(static_cast(v->get())); - } -#else - if (auto v - = std::dynamic_pointer_cast>(shared_from_this())) - return v; - - if (auto v = as()) - { - // the below has to be a non-const value due to a bug in - // libc++: https://llvm.org/bugs/show_bug.cgi?id=18843 - return make_value(static_cast(v->get())); - } -#endif - - return nullptr; -} - -/** - * Exception class for array insertion errors. - */ -class array_exception : public std::runtime_error -{ - public: - array_exception(const std::string& err) : std::runtime_error{err} - { - } -}; - -class array : public base -{ - public: - friend std::shared_ptr make_array(); - - std::shared_ptr clone() const override; - - virtual bool is_array() const override - { - return true; - } - - using size_type = std::size_t; - - /** - * arrays can be iterated over - */ - using iterator = std::vector>::iterator; - - /** - * arrays can be iterated over. Const version. - */ - using const_iterator = std::vector>::const_iterator; - - iterator begin() - { - return values_.begin(); - } - - const_iterator begin() const - { - return values_.begin(); - } - - iterator end() - { - return values_.end(); - } - - const_iterator end() const - { - return values_.end(); - } - - /** - * Obtains the array (vector) of base values. - */ - std::vector>& get() - { - return values_; - } - - /** - * Obtains the array (vector) of base values. Const version. - */ - const std::vector>& get() const - { - return values_; - } - - std::shared_ptr at(size_t idx) const - { - return values_.at(idx); - } - - /** - * Obtains an array of values. Note that elements may be - * nullptr if they cannot be converted to a value. - */ - template - std::vector>> array_of() const - { - std::vector>> result(values_.size()); - - std::transform(values_.begin(), values_.end(), result.begin(), - [&](std::shared_ptr v) { return v->as(); }); - - return result; - } - - /** - * Obtains a option>. The option will be empty if the array - * contains values that are not of type T. - */ - template - inline typename array_of_trait::return_type get_array_of() const - { - std::vector result; - result.reserve(values_.size()); - - for (const auto& val : values_) - { - if (auto v = val->as()) - result.push_back(v->get()); - else - return {}; - } - - return {std::move(result)}; - } - - /** - * Obtains an array of arrays. Note that elements may be nullptr - * if they cannot be converted to a array. - */ - std::vector> nested_array() const - { - std::vector> result(values_.size()); - - std::transform(values_.begin(), values_.end(), result.begin(), - [&](std::shared_ptr v) -> std::shared_ptr { - if (v->is_array()) - return std::static_pointer_cast(v); - return std::shared_ptr{}; - }); - - return result; - } - - /** - * Add a value to the end of the array - */ - template - void push_back(const std::shared_ptr>& val) - { - if (values_.empty() || values_[0]->as()) - { - values_.push_back(val); - } - else - { - throw array_exception{"Arrays must be homogenous."}; - } - } - - /** - * Add an array to the end of the array - */ - void push_back(const std::shared_ptr& val) - { - if (values_.empty() || values_[0]->is_array()) - { - values_.push_back(val); - } - else - { - throw array_exception{"Arrays must be homogenous."}; - } - } - - /** - * Convenience function for adding a simple element to the end - * of the array. - */ - template - void push_back(T&& val, typename value_traits::type* = 0) - { - push_back(make_value(std::forward(val))); - } - - /** - * Insert a value into the array - */ - template - iterator insert(iterator position, const std::shared_ptr>& value) - { - if (values_.empty() || values_[0]->as()) - { - return values_.insert(position, value); - } - else - { - throw array_exception{"Arrays must be homogenous."}; - } - } - - /** - * Insert an array into the array - */ - iterator insert(iterator position, const std::shared_ptr& value) - { - if (values_.empty() || values_[0]->is_array()) - { - return values_.insert(position, value); - } - else - { - throw array_exception{"Arrays must be homogenous."}; - } - } - - /** - * Convenience function for inserting a simple element in the array - */ - template - iterator insert(iterator position, T&& val, - typename value_traits::type* = 0) - { - return insert(position, make_value(std::forward(val))); - } - - /** - * Erase an element from the array - */ - iterator erase(iterator position) - { - return values_.erase(position); - } - - /** - * Clear the array - */ - void clear() - { - values_.clear(); - } - - /** - * Reserve space for n values. - */ - void reserve(size_type n) - { - values_.reserve(n); - } - - private: -#if defined(CPPTOML_NO_RTTI) - array() : base(base_type::ARRAY) - { - // empty - } -#else - array() = default; -#endif - - template - array(InputIterator begin, InputIterator end) : values_{begin, end} - { - // nothing - } - - array(const array& obj) = delete; - array& operator=(const array& obj) = delete; - - std::vector> values_; -}; - -inline std::shared_ptr make_array() -{ - struct make_shared_enabler : public array - { - make_shared_enabler() - { - // nothing - } - }; - - return std::make_shared(); -} - -namespace detail -{ -template <> -inline std::shared_ptr make_element() -{ - return make_array(); -} -} // namespace detail - -/** - * Obtains a option>. The option will be empty if the array - * contains values that are not of type T. - */ -template <> -inline typename array_of_trait::return_type -array::get_array_of() const -{ - std::vector> result; - result.reserve(values_.size()); - - for (const auto& val : values_) - { - if (auto v = val->as_array()) - result.push_back(v); - else - return {}; - } - - return {std::move(result)}; -} - -class table; - -class table_array : public base -{ - friend class table; - friend std::shared_ptr make_table_array(bool); - - public: - std::shared_ptr clone() const override; - - using size_type = std::size_t; - - /** - * arrays can be iterated over - */ - using iterator = std::vector>::iterator; - - /** - * arrays can be iterated over. Const version. - */ - using const_iterator = std::vector>::const_iterator; - - iterator begin() - { - return array_.begin(); - } - - const_iterator begin() const - { - return array_.begin(); - } - - iterator end() - { - return array_.end(); - } - - const_iterator end() const - { - return array_.end(); - } - - virtual bool is_table_array() const override - { - return true; - } - - std::vector>& get() - { - return array_; - } - - const std::vector>& get() const - { - return array_; - } - - /** - * Add a table to the end of the array - */ - void push_back(const std::shared_ptr
& val) - { - array_.push_back(val); - } - - /** - * Insert a table into the array - */ - iterator insert(iterator position, const std::shared_ptr
& value) - { - return array_.insert(position, value); - } - - /** - * Erase an element from the array - */ - iterator erase(iterator position) - { - return array_.erase(position); - } - - /** - * Clear the array - */ - void clear() - { - array_.clear(); - } - - /** - * Reserve space for n tables. - */ - void reserve(size_type n) - { - array_.reserve(n); - } - - /** - * Whether or not the table array is declared inline. This mostly - * matters for parsing, where statically defined arrays cannot be - * appended to using the array-of-table syntax. - */ - bool is_inline() const - { - return is_inline_; - } - - private: -#if defined(CPPTOML_NO_RTTI) - table_array(bool is_inline = false) - : base(base_type::TABLE_ARRAY), is_inline_(is_inline) - { - // nothing - } -#else - table_array(bool is_inline = false) : is_inline_(is_inline) - { - // nothing - } -#endif - - table_array(const table_array& obj) = delete; - table_array& operator=(const table_array& rhs) = delete; - - std::vector> array_; - const bool is_inline_ = false; -}; - -inline std::shared_ptr make_table_array(bool is_inline) -{ - struct make_shared_enabler : public table_array - { - make_shared_enabler(bool mse_is_inline) : table_array(mse_is_inline) - { - // nothing - } - }; - - return std::make_shared(is_inline); -} - -namespace detail -{ -template <> -inline std::shared_ptr make_element() -{ - return make_table_array(true); -} -} // namespace detail - -// The below are overloads for fetching specific value types out of a value -// where special casting behavior (like bounds checking) is desired - -template -typename std::enable_if::value - && std::is_signed::value, - option>::type -get_impl(const std::shared_ptr& elem) -{ - if (auto v = elem->as()) - { - if (v->get() < (std::numeric_limits::min)()) - throw std::underflow_error{ - "T cannot represent the value requested in get"}; - - if (v->get() > (std::numeric_limits::max)()) - throw std::overflow_error{ - "T cannot represent the value requested in get"}; - - return {static_cast(v->get())}; - } - else - { - return {}; - } -} - -template -typename std::enable_if::value - && std::is_unsigned::value, - option>::type -get_impl(const std::shared_ptr& elem) -{ - if (auto v = elem->as()) - { - if (v->get() < 0) - throw std::underflow_error{"T cannot store negative value in get"}; - - if (static_cast(v->get()) > (std::numeric_limits::max)()) - throw std::overflow_error{ - "T cannot represent the value requested in get"}; - - return {static_cast(v->get())}; - } - else - { - return {}; - } -} - -template -typename std::enable_if::value - || std::is_same::value, - option>::type -get_impl(const std::shared_ptr& elem) -{ - if (auto v = elem->as()) - { - return {v->get()}; - } - else - { - return {}; - } -} - -/** - * Represents a TOML keytable. - */ -class table : public base -{ - public: - friend class table_array; - friend std::shared_ptr
make_table(); - - std::shared_ptr clone() const override; - - /** - * tables can be iterated over. - */ - using iterator = string_to_base_map::iterator; - - /** - * tables can be iterated over. Const version. - */ - using const_iterator = string_to_base_map::const_iterator; - - iterator begin() - { - return map_.begin(); - } - - const_iterator begin() const - { - return map_.begin(); - } - - iterator end() - { - return map_.end(); - } - - const_iterator end() const - { - return map_.end(); - } - - bool is_table() const override - { - return true; - } - - bool empty() const - { - return map_.empty(); - } - - /** - * Determines if this key table contains the given key. - */ - bool contains(const std::string& key) const - { - return map_.find(key) != map_.end(); - } - - /** - * Determines if this key table contains the given key. Will - * resolve "qualified keys". Qualified keys are the full access - * path separated with dots like "grandparent.parent.child". - */ - bool contains_qualified(const std::string& key) const - { - return resolve_qualified(key); - } - - /** - * Obtains the base for a given key. - * @throw std::out_of_range if the key does not exist - */ - std::shared_ptr get(const std::string& key) const - { - return map_.at(key); - } - - /** - * Obtains the base for a given key. Will resolve "qualified - * keys". Qualified keys are the full access path separated with - * dots like "grandparent.parent.child". - * - * @throw std::out_of_range if the key does not exist - */ - std::shared_ptr get_qualified(const std::string& key) const - { - std::shared_ptr p; - resolve_qualified(key, &p); - return p; - } - - /** - * Obtains a table for a given key, if possible. - */ - std::shared_ptr
get_table(const std::string& key) const - { - if (contains(key) && get(key)->is_table()) - return std::static_pointer_cast
(get(key)); - return nullptr; - } - - /** - * Obtains a table for a given key, if possible. Will resolve - * "qualified keys". - */ - std::shared_ptr
get_table_qualified(const std::string& key) const - { - if (contains_qualified(key) && get_qualified(key)->is_table()) - return std::static_pointer_cast
(get_qualified(key)); - return nullptr; - } - - /** - * Obtains an array for a given key. - */ - std::shared_ptr get_array(const std::string& key) const - { - if (!contains(key)) - return nullptr; - return get(key)->as_array(); - } - - /** - * Obtains an array for a given key. Will resolve "qualified keys". - */ - std::shared_ptr get_array_qualified(const std::string& key) const - { - if (!contains_qualified(key)) - return nullptr; - return get_qualified(key)->as_array(); - } - - /** - * Obtains a table_array for a given key, if possible. - */ - std::shared_ptr get_table_array(const std::string& key) const - { - if (!contains(key)) - return nullptr; - return get(key)->as_table_array(); - } - - /** - * Obtains a table_array for a given key, if possible. Will resolve - * "qualified keys". - */ - std::shared_ptr - get_table_array_qualified(const std::string& key) const - { - if (!contains_qualified(key)) - return nullptr; - return get_qualified(key)->as_table_array(); - } - - /** - * Helper function that attempts to get a value corresponding - * to the template parameter from a given key. - */ - template - option get_as(const std::string& key) const - { - try - { - return get_impl(get(key)); - } - catch (const std::out_of_range&) - { - return {}; - } - } - - /** - * Helper function that attempts to get a value corresponding - * to the template parameter from a given key. Will resolve "qualified - * keys". - */ - template - option get_qualified_as(const std::string& key) const - { - try - { - return get_impl(get_qualified(key)); - } - catch (const std::out_of_range&) - { - return {}; - } - } - - /** - * Helper function that attempts to get an array of values of a given - * type corresponding to the template parameter for a given key. - * - * If the key doesn't exist, doesn't exist as an array type, or one or - * more keys inside the array type are not of type T, an empty option - * is returned. Otherwise, an option containing a vector of the values - * is returned. - */ - template - inline typename array_of_trait::return_type - get_array_of(const std::string& key) const - { - if (auto v = get_array(key)) - { - std::vector result; - result.reserve(v->get().size()); - - for (const auto& b : v->get()) - { - if (auto val = b->as()) - result.push_back(val->get()); - else - return {}; - } - return {std::move(result)}; - } - - return {}; - } - - /** - * Helper function that attempts to get an array of values of a given - * type corresponding to the template parameter for a given key. Will - * resolve "qualified keys". - * - * If the key doesn't exist, doesn't exist as an array type, or one or - * more keys inside the array type are not of type T, an empty option - * is returned. Otherwise, an option containing a vector of the values - * is returned. - */ - template - inline typename array_of_trait::return_type - get_qualified_array_of(const std::string& key) const - { - if (auto v = get_array_qualified(key)) - { - std::vector result; - result.reserve(v->get().size()); - - for (const auto& b : v->get()) - { - if (auto val = b->as()) - result.push_back(val->get()); - else - return {}; - } - return {std::move(result)}; - } - - return {}; - } - - /** - * Adds an element to the keytable. - */ - void insert(const std::string& key, const std::shared_ptr& value) - { - map_[key] = value; - } - - /** - * Convenience shorthand for adding a simple element to the - * keytable. - */ - template - void insert(const std::string& key, T&& val, - typename value_traits::type* = 0) - { - insert(key, make_value(std::forward(val))); - } - - /** - * Removes an element from the table. - */ - void erase(const std::string& key) - { - map_.erase(key); - } - - private: -#if defined(CPPTOML_NO_RTTI) - table() : base(base_type::TABLE) - { - // nothing - } -#else - table() - { - // nothing - } -#endif - - table(const table& obj) = delete; - table& operator=(const table& rhs) = delete; - - std::vector split(const std::string& value, - char separator) const - { - std::vector result; - std::string::size_type p = 0; - std::string::size_type q; - while ((q = value.find(separator, p)) != std::string::npos) - { - result.emplace_back(value, p, q - p); - p = q + 1; - } - result.emplace_back(value, p); - return result; - } - - // If output parameter p is specified, fill it with the pointer to the - // specified entry and throw std::out_of_range if it couldn't be found. - // - // Otherwise, just return true if the entry could be found or false - // otherwise and do not throw. - bool resolve_qualified(const std::string& key, - std::shared_ptr* p = nullptr) const - { - auto parts = split(key, '.'); - auto last_key = parts.back(); - parts.pop_back(); - - auto cur_table = this; - for (const auto& part : parts) - { - cur_table = cur_table->get_table(part).get(); - if (!cur_table) - { - if (!p) - return false; - - throw std::out_of_range{key + " is not a valid key"}; - } - } - - if (!p) - return cur_table->map_.count(last_key) != 0; - - *p = cur_table->map_.at(last_key); - return true; - } - - string_to_base_map map_; -}; - -/** - * Helper function that attempts to get an array of arrays for a given - * key. - * - * If the key doesn't exist, doesn't exist as an array type, or one or - * more keys inside the array type are not of type T, an empty option - * is returned. Otherwise, an option containing a vector of the values - * is returned. - */ -template <> -inline typename array_of_trait::return_type -table::get_array_of(const std::string& key) const -{ - if (auto v = get_array(key)) - { - std::vector> result; - result.reserve(v->get().size()); - - for (const auto& b : v->get()) - { - if (auto val = b->as_array()) - result.push_back(val); - else - return {}; - } - - return {std::move(result)}; - } - - return {}; -} - -/** - * Helper function that attempts to get an array of arrays for a given - * key. Will resolve "qualified keys". - * - * If the key doesn't exist, doesn't exist as an array type, or one or - * more keys inside the array type are not of type T, an empty option - * is returned. Otherwise, an option containing a vector of the values - * is returned. - */ -template <> -inline typename array_of_trait::return_type -table::get_qualified_array_of(const std::string& key) const -{ - if (auto v = get_array_qualified(key)) - { - std::vector> result; - result.reserve(v->get().size()); - - for (const auto& b : v->get()) - { - if (auto val = b->as_array()) - result.push_back(val); - else - return {}; - } - - return {std::move(result)}; - } - - return {}; -} - -std::shared_ptr
make_table() -{ - struct make_shared_enabler : public table - { - make_shared_enabler() - { - // nothing - } - }; - - return std::make_shared(); -} - -namespace detail -{ -template <> -inline std::shared_ptr
make_element
() -{ - return make_table(); -} -} // namespace detail - -template -std::shared_ptr value::clone() const -{ - return make_value(data_); -} - -inline std::shared_ptr array::clone() const -{ - auto result = make_array(); - result->reserve(values_.size()); - for (const auto& ptr : values_) - result->values_.push_back(ptr->clone()); - return result; -} - -inline std::shared_ptr table_array::clone() const -{ - auto result = make_table_array(is_inline()); - result->reserve(array_.size()); - for (const auto& ptr : array_) - result->array_.push_back(ptr->clone()->as_table()); - return result; -} - -inline std::shared_ptr table::clone() const -{ - auto result = make_table(); - for (const auto& pr : map_) - result->insert(pr.first, pr.second->clone()); - return result; -} - -/** - * Exception class for all TOML parsing errors. - */ -class parse_exception : public std::runtime_error -{ - public: - parse_exception(const std::string& err) : std::runtime_error{err} - { - } - - parse_exception(const std::string& err, std::size_t line_number) - : std::runtime_error{err + " at line " + std::to_string(line_number)} - { - } -}; - -inline bool is_number(char c) -{ - return c >= '0' && c <= '9'; -} - -inline bool is_hex(char c) -{ - return is_number(c) || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); -} - -/** - * Helper object for consuming expected characters. - */ -template -class consumer -{ - public: - consumer(std::string::iterator& it, const std::string::iterator& end, - OnError&& on_error) - : it_(it), end_(end), on_error_(std::forward(on_error)) - { - // nothing - } - - void operator()(char c) - { - if (it_ == end_ || *it_ != c) - on_error_(); - ++it_; - } - - template - void operator()(const char (&str)[N]) - { - std::for_each(std::begin(str), std::end(str) - 1, - [&](char c) { (*this)(c); }); - } - - void eat_or(char a, char b) - { - if (it_ == end_ || (*it_ != a && *it_ != b)) - on_error_(); - ++it_; - } - - int eat_digits(int len) - { - int val = 0; - for (int i = 0; i < len; ++i) - { - if (!is_number(*it_) || it_ == end_) - on_error_(); - val = 10 * val + (*it_++ - '0'); - } - return val; - } - - void error() - { - on_error_(); - } - - private: - std::string::iterator& it_; - const std::string::iterator& end_; - OnError on_error_; -}; - -template -consumer make_consumer(std::string::iterator& it, - const std::string::iterator& end, - OnError&& on_error) -{ - return consumer(it, end, std::forward(on_error)); -} - -// replacement for std::getline to handle incorrectly line-ended files -// https://stackoverflow.com/questions/6089231/getting-std-ifstream-to-handle-lf-cr-and-crlf -namespace detail -{ -inline std::istream& getline(std::istream& input, std::string& line) -{ - line.clear(); - - std::istream::sentry sentry{input, true}; - auto sb = input.rdbuf(); - - while (true) - { - auto c = sb->sbumpc(); - if (c == '\r') - { - if (sb->sgetc() == '\n') - c = sb->sbumpc(); - } - - if (c == '\n') - return input; - - if (c == std::istream::traits_type::eof()) - { - if (line.empty()) - input.setstate(std::ios::eofbit); - return input; - } - - line.push_back(static_cast(c)); - } -} -} // namespace detail - -/** - * The parser class. - */ -class parser -{ - public: - /** - * Parsers are constructed from streams. - */ - parser(std::istream& stream) : input_(stream) - { - // nothing - } - - parser& operator=(const parser& parser) = delete; - - /** - * Parses the stream this parser was created on until EOF. - * @throw parse_exception if there are errors in parsing - */ - std::shared_ptr
parse() - { - std::shared_ptr
root = make_table(); - - table* curr_table = root.get(); - - while (detail::getline(input_, line_)) - { - line_number_++; - auto it = line_.begin(); - auto end = line_.end(); - consume_whitespace(it, end); - if (it == end || *it == '#') - continue; - if (*it == '[') - { - curr_table = root.get(); - parse_table(it, end, curr_table); - } - else - { - parse_key_value(it, end, curr_table); - consume_whitespace(it, end); - eol_or_comment(it, end); - } - } - return root; - } - - private: -#if defined _MSC_VER - __declspec(noreturn) -#elif defined __GNUC__ - __attribute__((noreturn)) -#endif - void throw_parse_exception(const std::string& err) - { - throw parse_exception{err, line_number_}; - } - - void parse_table(std::string::iterator& it, - const std::string::iterator& end, table*& curr_table) - { - // remove the beginning keytable marker - ++it; - if (it == end) - throw_parse_exception("Unexpected end of table"); - if (*it == '[') - parse_table_array(it, end, curr_table); - else - parse_single_table(it, end, curr_table); - } - - void parse_single_table(std::string::iterator& it, - const std::string::iterator& end, - table*& curr_table) - { - if (it == end || *it == ']') - throw_parse_exception("Table name cannot be empty"); - - std::string full_table_name; - bool inserted = false; - - auto key_end = [](char c) { return c == ']'; }; - - auto key_part_handler = [&](const std::string& part) { - if (part.empty()) - throw_parse_exception("Empty component of table name"); - - if (!full_table_name.empty()) - full_table_name += '.'; - full_table_name += part; - - if (curr_table->contains(part)) - { -#if !defined(__PGI) - auto b = curr_table->get(part); -#else - // Workaround for PGI compiler - std::shared_ptr b = curr_table->get(part); -#endif - if (b->is_table()) - curr_table = static_cast(b.get()); - else if (b->is_table_array()) - curr_table = std::static_pointer_cast(b) - ->get() - .back() - .get(); - else - throw_parse_exception("Key " + full_table_name - + "already exists as a value"); - } - else - { - inserted = true; - curr_table->insert(part, make_table()); - curr_table = static_cast(curr_table->get(part).get()); - } - }; - - key_part_handler(parse_key(it, end, key_end, key_part_handler)); - - if (it == end) - throw_parse_exception( - "Unterminated table declaration; did you forget a ']'?"); - - if (*it != ']') - { - std::string errmsg{"Unexpected character in table definition: "}; - errmsg += '"'; - errmsg += *it; - errmsg += '"'; - throw_parse_exception(errmsg); - } - - // table already existed - if (!inserted) - { - auto is_value - = [](const std::pair&>& p) { - return p.second->is_value(); - }; - - // if there are any values, we can't add values to this table - // since it has already been defined. If there aren't any - // values, then it was implicitly created by something like - // [a.b] - if (curr_table->empty() - || std::any_of(curr_table->begin(), curr_table->end(), - is_value)) - { - throw_parse_exception("Redefinition of table " - + full_table_name); - } - } - - ++it; - consume_whitespace(it, end); - eol_or_comment(it, end); - } - - void parse_table_array(std::string::iterator& it, - const std::string::iterator& end, table*& curr_table) - { - ++it; - if (it == end || *it == ']') - throw_parse_exception("Table array name cannot be empty"); - - auto key_end = [](char c) { return c == ']'; }; - - std::string full_ta_name; - auto key_part_handler = [&](const std::string& part) { - if (part.empty()) - throw_parse_exception("Empty component of table array name"); - - if (!full_ta_name.empty()) - full_ta_name += '.'; - full_ta_name += part; - - if (curr_table->contains(part)) - { -#if !defined(__PGI) - auto b = curr_table->get(part); -#else - // Workaround for PGI compiler - std::shared_ptr b = curr_table->get(part); -#endif - - // if this is the end of the table array name, add an - // element to the table array that we just looked up, - // provided it was not declared inline - if (it != end && *it == ']') - { - if (!b->is_table_array()) - { - throw_parse_exception("Key " + full_ta_name - + " is not a table array"); - } - - auto v = b->as_table_array(); - - if (v->is_inline()) - { - throw_parse_exception("Static array " + full_ta_name - + " cannot be appended to"); - } - - v->get().push_back(make_table()); - curr_table = v->get().back().get(); - } - // otherwise, just keep traversing down the key name - else - { - if (b->is_table()) - curr_table = static_cast(b.get()); - else if (b->is_table_array()) - curr_table = std::static_pointer_cast(b) - ->get() - .back() - .get(); - else - throw_parse_exception("Key " + full_ta_name - + " already exists as a value"); - } - } - else - { - // if this is the end of the table array name, add a new - // table array and a new table inside that array for us to - // add keys to next - if (it != end && *it == ']') - { - curr_table->insert(part, make_table_array()); - auto arr = std::static_pointer_cast( - curr_table->get(part)); - arr->get().push_back(make_table()); - curr_table = arr->get().back().get(); - } - // otherwise, create the implicitly defined table and move - // down to it - else - { - curr_table->insert(part, make_table()); - curr_table - = static_cast(curr_table->get(part).get()); - } - } - }; - - key_part_handler(parse_key(it, end, key_end, key_part_handler)); - - // consume the last "]]" - auto eat = make_consumer(it, end, [this]() { - throw_parse_exception("Unterminated table array name"); - }); - eat(']'); - eat(']'); - - consume_whitespace(it, end); - eol_or_comment(it, end); - } - - void parse_key_value(std::string::iterator& it, std::string::iterator& end, - table* curr_table) - { - auto key_end = [](char c) { return c == '='; }; - - auto key_part_handler = [&](const std::string& part) { - // two cases: this key part exists already, in which case it must - // be a table, or it doesn't exist in which case we must create - // an implicitly defined table - if (curr_table->contains(part)) - { - auto val = curr_table->get(part); - if (val->is_table()) - { - curr_table = static_cast(val.get()); - } - else - { - throw_parse_exception("Key " + part - + " already exists as a value"); - } - } - else - { - auto newtable = make_table(); - curr_table->insert(part, newtable); - curr_table = newtable.get(); - } - }; - - auto key = parse_key(it, end, key_end, key_part_handler); - - if (curr_table->contains(key)) - throw_parse_exception("Key " + key + " already present"); - if (it == end || *it != '=') - throw_parse_exception("Value must follow after a '='"); - ++it; - consume_whitespace(it, end); - curr_table->insert(key, parse_value(it, end)); - consume_whitespace(it, end); - } - - template - std::string - parse_key(std::string::iterator& it, const std::string::iterator& end, - KeyEndFinder&& key_end, KeyPartHandler&& key_part_handler) - { - // parse the key as a series of one or more simple-keys joined with '.' - while (it != end && !key_end(*it)) - { - auto part = parse_simple_key(it, end); - consume_whitespace(it, end); - - if (it == end || key_end(*it)) - { - return part; - } - - if (*it != '.') - { - std::string errmsg{"Unexpected character in key: "}; - errmsg += '"'; - errmsg += *it; - errmsg += '"'; - throw_parse_exception(errmsg); - } - - key_part_handler(part); - - // consume the dot - ++it; - } - - throw_parse_exception("Unexpected end of key"); - } - - std::string parse_simple_key(std::string::iterator& it, - const std::string::iterator& end) - { - consume_whitespace(it, end); - - if (it == end) - throw_parse_exception("Unexpected end of key (blank key?)"); - - if (*it == '"' || *it == '\'') - { - return string_literal(it, end, *it); - } - else - { - auto bke = std::find_if(it, end, [](char c) { - return c == '.' || c == '=' || c == ']'; - }); - return parse_bare_key(it, bke); - } - } - - std::string parse_bare_key(std::string::iterator& it, - const std::string::iterator& end) - { - if (it == end) - { - throw_parse_exception("Bare key missing name"); - } - - auto key_end = end; - --key_end; - consume_backwards_whitespace(key_end, it); - ++key_end; - std::string key{it, key_end}; - - if (std::find(it, key_end, '#') != key_end) - { - throw_parse_exception("Bare key " + key + " cannot contain #"); - } - - if (std::find_if(it, key_end, - [](char c) { return c == ' ' || c == '\t'; }) - != key_end) - { - throw_parse_exception("Bare key " + key - + " cannot contain whitespace"); - } - - if (std::find_if(it, key_end, - [](char c) { return c == '[' || c == ']'; }) - != key_end) - { - throw_parse_exception("Bare key " + key - + " cannot contain '[' or ']'"); - } - - it = end; - return key; - } - - enum class parse_type - { - STRING = 1, - LOCAL_TIME, - LOCAL_DATE, - LOCAL_DATETIME, - OFFSET_DATETIME, - INT, - FLOAT, - BOOL, - ARRAY, - INLINE_TABLE - }; - - std::shared_ptr parse_value(std::string::iterator& it, - std::string::iterator& end) - { - parse_type type = determine_value_type(it, end); - switch (type) - { - case parse_type::STRING: - return parse_string(it, end); - case parse_type::LOCAL_TIME: - return parse_time(it, end); - case parse_type::LOCAL_DATE: - case parse_type::LOCAL_DATETIME: - case parse_type::OFFSET_DATETIME: - return parse_date(it, end); - case parse_type::INT: - case parse_type::FLOAT: - return parse_number(it, end); - case parse_type::BOOL: - return parse_bool(it, end); - case parse_type::ARRAY: - return parse_array(it, end); - case parse_type::INLINE_TABLE: - return parse_inline_table(it, end); - default: - throw_parse_exception("Failed to parse value"); - } - } - - parse_type determine_value_type(const std::string::iterator& it, - const std::string::iterator& end) - { - if (it == end) - { - throw_parse_exception("Failed to parse value type"); - } - if (*it == '"' || *it == '\'') - { - return parse_type::STRING; - } - else if (is_time(it, end)) - { - return parse_type::LOCAL_TIME; - } - else if (auto dtype = date_type(it, end)) - { - return *dtype; - } - else if (is_number(*it) || *it == '-' || *it == '+' - || (*it == 'i' && it + 1 != end && it[1] == 'n' - && it + 2 != end && it[2] == 'f') - || (*it == 'n' && it + 1 != end && it[1] == 'a' - && it + 2 != end && it[2] == 'n')) - { - return determine_number_type(it, end); - } - else if (*it == 't' || *it == 'f') - { - return parse_type::BOOL; - } - else if (*it == '[') - { - return parse_type::ARRAY; - } - else if (*it == '{') - { - return parse_type::INLINE_TABLE; - } - throw_parse_exception("Failed to parse value type"); - } - - parse_type determine_number_type(const std::string::iterator& it, - const std::string::iterator& end) - { - // determine if we are an integer or a float - auto check_it = it; - if (*check_it == '-' || *check_it == '+') - ++check_it; - - if (check_it == end) - throw_parse_exception("Malformed number"); - - if (*check_it == 'i' || *check_it == 'n') - return parse_type::FLOAT; - - while (check_it != end && is_number(*check_it)) - ++check_it; - if (check_it != end && *check_it == '.') - { - ++check_it; - while (check_it != end && is_number(*check_it)) - ++check_it; - return parse_type::FLOAT; - } - else - { - return parse_type::INT; - } - } - - std::shared_ptr> parse_string(std::string::iterator& it, - std::string::iterator& end) - { - auto delim = *it; - assert(delim == '"' || delim == '\''); - - // end is non-const here because we have to be able to potentially - // parse multiple lines in a string, not just one - auto check_it = it; - ++check_it; - if (check_it != end && *check_it == delim) - { - ++check_it; - if (check_it != end && *check_it == delim) - { - it = ++check_it; - return parse_multiline_string(it, end, delim); - } - } - return make_value(string_literal(it, end, delim)); - } - - std::shared_ptr> - parse_multiline_string(std::string::iterator& it, - std::string::iterator& end, char delim) - { - std::stringstream ss; - - auto is_ws = [](char c) { return c == ' ' || c == '\t'; }; - - bool consuming = false; - std::shared_ptr> ret; - - auto handle_line = [&](std::string::iterator& local_it, - std::string::iterator& local_end) { - if (consuming) - { - local_it = std::find_if_not(local_it, local_end, is_ws); - - // whole line is whitespace - if (local_it == local_end) - return; - } - - consuming = false; - - while (local_it != local_end) - { - // handle escaped characters - if (delim == '"' && *local_it == '\\') - { - auto check = local_it; - // check if this is an actual escape sequence or a - // whitespace escaping backslash - ++check; - consume_whitespace(check, local_end); - if (check == local_end) - { - consuming = true; - break; - } - - ss << parse_escape_code(local_it, local_end); - continue; - } - - // if we can end the string - if (std::distance(local_it, local_end) >= 3) - { - auto check = local_it; - // check for """ - if (*check++ == delim && *check++ == delim - && *check++ == delim) - { - local_it = check; - ret = make_value(ss.str()); - break; - } - } - - ss << *local_it++; - } - }; - - // handle the remainder of the current line - handle_line(it, end); - if (ret) - return ret; - - // start eating lines - while (detail::getline(input_, line_)) - { - ++line_number_; - - it = line_.begin(); - end = line_.end(); - - handle_line(it, end); - - if (ret) - return ret; - - if (!consuming) - ss << std::endl; - } - - throw_parse_exception("Unterminated multi-line basic string"); - } - - std::string string_literal(std::string::iterator& it, - const std::string::iterator& end, char delim) - { - ++it; - std::string val; - while (it != end) - { - // handle escaped characters - if (delim == '"' && *it == '\\') - { - val += parse_escape_code(it, end); - } - else if (*it == delim) - { - ++it; - consume_whitespace(it, end); - return val; - } - else - { - val += *it++; - } - } - throw_parse_exception("Unterminated string literal"); - } - - std::string parse_escape_code(std::string::iterator& it, - const std::string::iterator& end) - { - ++it; - if (it == end) - throw_parse_exception("Invalid escape sequence"); - char value; - if (*it == 'b') - { - value = '\b'; - } - else if (*it == 't') - { - value = '\t'; - } - else if (*it == 'n') - { - value = '\n'; - } - else if (*it == 'f') - { - value = '\f'; - } - else if (*it == 'r') - { - value = '\r'; - } - else if (*it == '"') - { - value = '"'; - } - else if (*it == '\\') - { - value = '\\'; - } - else if (*it == 'u' || *it == 'U') - { - return parse_unicode(it, end); - } - else - { - throw_parse_exception("Invalid escape sequence"); - } - ++it; - return std::string(1, value); - } - - std::string parse_unicode(std::string::iterator& it, - const std::string::iterator& end) - { - bool large = *it++ == 'U'; - auto codepoint = parse_hex(it, end, large ? 0x10000000 : 0x1000); - - if ((codepoint > 0xd7ff && codepoint < 0xe000) || codepoint > 0x10ffff) - { - throw_parse_exception( - "Unicode escape sequence is not a Unicode scalar value"); - } - - std::string result; - // See Table 3-6 of the Unicode standard - if (codepoint <= 0x7f) - { - // 1-byte codepoints: 00000000 0xxxxxxx - // repr: 0xxxxxxx - result += static_cast(codepoint & 0x7f); - } - else if (codepoint <= 0x7ff) - { - // 2-byte codepoints: 00000yyy yyxxxxxx - // repr: 110yyyyy 10xxxxxx - // - // 0x1f = 00011111 - // 0xc0 = 11000000 - // - result += static_cast(0xc0 | ((codepoint >> 6) & 0x1f)); - // - // 0x80 = 10000000 - // 0x3f = 00111111 - // - result += static_cast(0x80 | (codepoint & 0x3f)); - } - else if (codepoint <= 0xffff) - { - // 3-byte codepoints: zzzzyyyy yyxxxxxx - // repr: 1110zzzz 10yyyyyy 10xxxxxx - // - // 0xe0 = 11100000 - // 0x0f = 00001111 - // - result += static_cast(0xe0 | ((codepoint >> 12) & 0x0f)); - result += static_cast(0x80 | ((codepoint >> 6) & 0x1f)); - result += static_cast(0x80 | (codepoint & 0x3f)); - } - else - { - // 4-byte codepoints: 000uuuuu zzzzyyyy yyxxxxxx - // repr: 11110uuu 10uuzzzz 10yyyyyy 10xxxxxx - // - // 0xf0 = 11110000 - // 0x07 = 00000111 - // - result += static_cast(0xf0 | ((codepoint >> 18) & 0x07)); - result += static_cast(0x80 | ((codepoint >> 12) & 0x3f)); - result += static_cast(0x80 | ((codepoint >> 6) & 0x3f)); - result += static_cast(0x80 | (codepoint & 0x3f)); - } - return result; - } - - uint32_t parse_hex(std::string::iterator& it, - const std::string::iterator& end, uint32_t place) - { - uint32_t value = 0; - while (place > 0) - { - if (it == end) - throw_parse_exception("Unexpected end of unicode sequence"); - - if (!is_hex(*it)) - throw_parse_exception("Invalid unicode escape sequence"); - - value += place * hex_to_digit(*it++); - place /= 16; - } - return value; - } - - uint32_t hex_to_digit(char c) - { - if (is_number(c)) - return static_cast(c - '0'); - return 10 - + static_cast(c - - ((c >= 'a' && c <= 'f') ? 'a' : 'A')); - } - - std::shared_ptr parse_number(std::string::iterator& it, - const std::string::iterator& end) - { - auto check_it = it; - auto check_end = find_end_of_number(it, end); - - auto eat_sign = [&]() { - if (check_it != end && (*check_it == '-' || *check_it == '+')) - ++check_it; - }; - - auto check_no_leading_zero = [&]() { - if (check_it != end && *check_it == '0' && check_it + 1 != check_end - && check_it[1] != '.') - { - throw_parse_exception("Numbers may not have leading zeros"); - } - }; - - auto eat_digits = [&](bool (*check_char)(char)) { - auto beg = check_it; - while (check_it != end && check_char(*check_it)) - { - ++check_it; - if (check_it != end && *check_it == '_') - { - ++check_it; - if (check_it == end || !check_char(*check_it)) - throw_parse_exception("Malformed number"); - } - } - - if (check_it == beg) - throw_parse_exception("Malformed number"); - }; - - auto eat_hex = [&]() { eat_digits(&is_hex); }; - - auto eat_numbers = [&]() { eat_digits(&is_number); }; - - if (check_it != end && *check_it == '0' && check_it + 1 != check_end - && (check_it[1] == 'x' || check_it[1] == 'o' || check_it[1] == 'b')) - { - ++check_it; - char base = *check_it; - ++check_it; - if (base == 'x') - { - eat_hex(); - return parse_int(it, check_it, 16); - } - else if (base == 'o') - { - auto start = check_it; - eat_numbers(); - auto val = parse_int(start, check_it, 8, "0"); - it = start; - return val; - } - else // if (base == 'b') - { - auto start = check_it; - eat_numbers(); - auto val = parse_int(start, check_it, 2); - it = start; - return val; - } - } - - eat_sign(); - check_no_leading_zero(); - - if (check_it != end && check_it + 1 != end && check_it + 2 != end) - { - if (check_it[0] == 'i' && check_it[1] == 'n' && check_it[2] == 'f') - { - auto val = std::numeric_limits::infinity(); - if (*it == '-') - val = -val; - it = check_it + 3; - return make_value(val); - } - else if (check_it[0] == 'n' && check_it[1] == 'a' - && check_it[2] == 'n') - { - auto val = std::numeric_limits::quiet_NaN(); - if (*it == '-') - val = -val; - it = check_it + 3; - return make_value(val); - } - } - - eat_numbers(); - - if (check_it != end - && (*check_it == '.' || *check_it == 'e' || *check_it == 'E')) - { - bool is_exp = *check_it == 'e' || *check_it == 'E'; - - ++check_it; - if (check_it == end) - throw_parse_exception("Floats must have trailing digits"); - - auto eat_exp = [&]() { - eat_sign(); - check_no_leading_zero(); - eat_numbers(); - }; - - if (is_exp) - eat_exp(); - else - eat_numbers(); - - if (!is_exp && check_it != end - && (*check_it == 'e' || *check_it == 'E')) - { - ++check_it; - eat_exp(); - } - - return parse_float(it, check_it); - } - else - { - return parse_int(it, check_it); - } - } - - std::shared_ptr> parse_int(std::string::iterator& it, - const std::string::iterator& end, - int base = 10, - const char* prefix = "") - { - std::string v{it, end}; - v = prefix + v; - v.erase(std::remove(v.begin(), v.end(), '_'), v.end()); - it = end; - try - { - return make_value(std::stoll(v, nullptr, base)); - } - catch (const std::invalid_argument& ex) - { - throw_parse_exception("Malformed number (invalid argument: " - + std::string{ex.what()} + ")"); - } - catch (const std::out_of_range& ex) - { - throw_parse_exception("Malformed number (out of range: " - + std::string{ex.what()} + ")"); - } - } - - std::shared_ptr> parse_float(std::string::iterator& it, - const std::string::iterator& end) - { - std::string v{it, end}; - v.erase(std::remove(v.begin(), v.end(), '_'), v.end()); - it = end; - char decimal_point = std::localeconv()->decimal_point[0]; - std::replace(v.begin(), v.end(), '.', decimal_point); - try - { - return make_value(std::stod(v)); - } - catch (const std::invalid_argument& ex) - { - throw_parse_exception("Malformed number (invalid argument: " - + std::string{ex.what()} + ")"); - } - catch (const std::out_of_range& ex) - { - throw_parse_exception("Malformed number (out of range: " - + std::string{ex.what()} + ")"); - } - } - - std::shared_ptr> parse_bool(std::string::iterator& it, - const std::string::iterator& end) - { - auto eat = make_consumer(it, end, [this]() { - throw_parse_exception("Attempted to parse invalid boolean value"); - }); - - if (*it == 't') - { - eat("true"); - return make_value(true); - } - else if (*it == 'f') - { - eat("false"); - return make_value(false); - } - - eat.error(); - return nullptr; - } - - std::string::iterator find_end_of_number(std::string::iterator it, - std::string::iterator end) - { - auto ret = std::find_if(it, end, [](char c) { - return !is_number(c) && c != '_' && c != '.' && c != 'e' && c != 'E' - && c != '-' && c != '+' && c != 'x' && c != 'o' && c != 'b'; - }); - if (ret != end && ret + 1 != end && ret + 2 != end) - { - if ((ret[0] == 'i' && ret[1] == 'n' && ret[2] == 'f') - || (ret[0] == 'n' && ret[1] == 'a' && ret[2] == 'n')) - { - ret = ret + 3; - } - } - return ret; - } - - std::string::iterator find_end_of_date(std::string::iterator it, - std::string::iterator end) - { - auto end_of_date = std::find_if(it, end, [](char c) { - return !is_number(c) && c != '-'; - }); - if (end_of_date != end && *end_of_date == ' ' && end_of_date + 1 != end - && is_number(end_of_date[1])) - end_of_date++; - return std::find_if(end_of_date, end, [](char c) { - return !is_number(c) && c != 'T' && c != 'Z' && c != ':' - && c != '-' && c != '+' && c != '.'; - }); - } - - std::string::iterator find_end_of_time(std::string::iterator it, - std::string::iterator end) - { - return std::find_if(it, end, [](char c) { - return !is_number(c) && c != ':' && c != '.'; - }); - } - - local_time read_time(std::string::iterator& it, - const std::string::iterator& end) - { - auto time_end = find_end_of_time(it, end); - - auto eat = make_consumer( - it, time_end, [&]() { throw_parse_exception("Malformed time"); }); - - local_time ltime; - - ltime.hour = eat.eat_digits(2); - eat(':'); - ltime.minute = eat.eat_digits(2); - eat(':'); - ltime.second = eat.eat_digits(2); - - int power = 100000; - if (it != time_end && *it == '.') - { - ++it; - while (it != time_end && is_number(*it)) - { - ltime.microsecond += power * (*it++ - '0'); - power /= 10; - } - } - - if (it != time_end) - throw_parse_exception("Malformed time"); - - return ltime; - } - - std::shared_ptr> - parse_time(std::string::iterator& it, const std::string::iterator& end) - { - return make_value(read_time(it, end)); - } - - std::shared_ptr parse_date(std::string::iterator& it, - const std::string::iterator& end) - { - auto date_end = find_end_of_date(it, end); - - auto eat = make_consumer( - it, date_end, [&]() { throw_parse_exception("Malformed date"); }); - - local_date ldate; - ldate.year = eat.eat_digits(4); - eat('-'); - ldate.month = eat.eat_digits(2); - eat('-'); - ldate.day = eat.eat_digits(2); - - if (it == date_end) - return make_value(ldate); - - eat.eat_or('T', ' '); - - local_datetime ldt; - static_cast(ldt) = ldate; - static_cast(ldt) = read_time(it, date_end); - - if (it == date_end) - return make_value(ldt); - - offset_datetime dt; - static_cast(dt) = ldt; - - int hoff = 0; - int moff = 0; - if (*it == '+' || *it == '-') - { - auto plus = *it == '+'; - ++it; - - hoff = eat.eat_digits(2); - dt.hour_offset = (plus) ? hoff : -hoff; - eat(':'); - moff = eat.eat_digits(2); - dt.minute_offset = (plus) ? moff : -moff; - } - else if (*it == 'Z') - { - ++it; - } - - if (it != date_end) - throw_parse_exception("Malformed date"); - - return make_value(dt); - } - - std::shared_ptr parse_array(std::string::iterator& it, - std::string::iterator& end) - { - // this gets ugly because of the "homogeneity" restriction: - // arrays can either be of only one type, or contain arrays - // (each of those arrays could be of different types, though) - // - // because of the latter portion, we don't really have a choice - // but to represent them as arrays of base values... - ++it; - - // ugh---have to read the first value to determine array type... - skip_whitespace_and_comments(it, end); - - // edge case---empty array - if (*it == ']') - { - ++it; - return make_array(); - } - - auto val_end = std::find_if( - it, end, [](char c) { return c == ',' || c == ']' || c == '#'; }); - parse_type type = determine_value_type(it, val_end); - switch (type) - { - case parse_type::STRING: - return parse_value_array(it, end); - case parse_type::LOCAL_TIME: - return parse_value_array(it, end); - case parse_type::LOCAL_DATE: - return parse_value_array(it, end); - case parse_type::LOCAL_DATETIME: - return parse_value_array(it, end); - case parse_type::OFFSET_DATETIME: - return parse_value_array(it, end); - case parse_type::INT: - return parse_value_array(it, end); - case parse_type::FLOAT: - return parse_value_array(it, end); - case parse_type::BOOL: - return parse_value_array(it, end); - case parse_type::ARRAY: - return parse_object_array(&parser::parse_array, '[', it, - end); - case parse_type::INLINE_TABLE: - return parse_object_array( - &parser::parse_inline_table, '{', it, end); - default: - throw_parse_exception("Unable to parse array"); - } - } - - template - std::shared_ptr parse_value_array(std::string::iterator& it, - std::string::iterator& end) - { - auto arr = make_array(); - while (it != end && *it != ']') - { - auto val = parse_value(it, end); - if (auto v = val->as()) - arr->get().push_back(val); - else - throw_parse_exception("Arrays must be homogeneous"); - skip_whitespace_and_comments(it, end); - if (*it != ',') - break; - ++it; - skip_whitespace_and_comments(it, end); - } - if (it != end) - ++it; - return arr; - } - - template - std::shared_ptr parse_object_array(Function&& fun, char delim, - std::string::iterator& it, - std::string::iterator& end) - { - auto arr = detail::make_element(); - - while (it != end && *it != ']') - { - if (*it != delim) - throw_parse_exception("Unexpected character in array"); - - arr->get().push_back(((*this).*fun)(it, end)); - skip_whitespace_and_comments(it, end); - - if (it == end || *it != ',') - break; - - ++it; - skip_whitespace_and_comments(it, end); - } - - if (it == end || *it != ']') - throw_parse_exception("Unterminated array"); - - ++it; - return arr; - } - - std::shared_ptr
parse_inline_table(std::string::iterator& it, - std::string::iterator& end) - { - auto tbl = make_table(); - do - { - ++it; - if (it == end) - throw_parse_exception("Unterminated inline table"); - - consume_whitespace(it, end); - if (it != end && *it != '}') - { - parse_key_value(it, end, tbl.get()); - consume_whitespace(it, end); - } - } while (*it == ','); - - if (it == end || *it != '}') - throw_parse_exception("Unterminated inline table"); - - ++it; - consume_whitespace(it, end); - - return tbl; - } - - void skip_whitespace_and_comments(std::string::iterator& start, - std::string::iterator& end) - { - consume_whitespace(start, end); - while (start == end || *start == '#') - { - if (!detail::getline(input_, line_)) - throw_parse_exception("Unclosed array"); - line_number_++; - start = line_.begin(); - end = line_.end(); - consume_whitespace(start, end); - } - } - - void consume_whitespace(std::string::iterator& it, - const std::string::iterator& end) - { - while (it != end && (*it == ' ' || *it == '\t')) - ++it; - } - - void consume_backwards_whitespace(std::string::iterator& back, - const std::string::iterator& front) - { - while (back != front && (*back == ' ' || *back == '\t')) - --back; - } - - void eol_or_comment(const std::string::iterator& it, - const std::string::iterator& end) - { - if (it != end && *it != '#') - throw_parse_exception("Unidentified trailing character '" - + std::string{*it} - + "'---did you forget a '#'?"); - } - - bool is_time(const std::string::iterator& it, - const std::string::iterator& end) - { - auto time_end = find_end_of_time(it, end); - auto len = std::distance(it, time_end); - - if (len < 8) - return false; - - if (it[2] != ':' || it[5] != ':') - return false; - - if (len > 8) - return it[8] == '.' && len > 9; - - return true; - } - - option date_type(const std::string::iterator& it, - const std::string::iterator& end) - { - auto date_end = find_end_of_date(it, end); - auto len = std::distance(it, date_end); - - if (len < 10) - return {}; - - if (it[4] != '-' || it[7] != '-') - return {}; - - if (len >= 19 && (it[10] == 'T' || it[10] == ' ') - && is_time(it + 11, date_end)) - { - // datetime type - auto time_end = find_end_of_time(it + 11, date_end); - if (time_end == date_end) - return {parse_type::LOCAL_DATETIME}; - else - return {parse_type::OFFSET_DATETIME}; - } - else if (len == 10) - { - // just a regular date - return {parse_type::LOCAL_DATE}; - } - - return {}; - } - - std::istream& input_; - std::string line_; - std::size_t line_number_ = 0; -}; - -/** - * Utility function to parse a file as a TOML file. Returns the root table. - * Throws a parse_exception if the file cannot be opened. - */ -inline std::shared_ptr
parse_file(const std::string& filename) -{ -#if defined(BOOST_NOWIDE_FSTREAM_INCLUDED_HPP) - boost::nowide::ifstream file{filename.c_str()}; -#elif defined(NOWIDE_FSTREAM_INCLUDED_HPP) - nowide::ifstream file{filename.c_str()}; -#else - std::ifstream file{filename}; -#endif - if (!file.is_open()) - throw parse_exception{filename + " could not be opened for parsing"}; - parser p{file}; - return p.parse(); -} - -template -struct value_accept; - -template <> -struct value_accept<> -{ - template - static void accept(const base&, Visitor&&, Args&&...) - { - // nothing - } -}; - -template -struct value_accept -{ - template - static void accept(const base& b, Visitor&& visitor, Args&&... args) - { - if (auto v = b.as()) - { - visitor.visit(*v, std::forward(args)...); - } - else - { - value_accept::accept(b, std::forward(visitor), - std::forward(args)...); - } - } -}; - -/** - * base implementation of accept() that calls visitor.visit() on the concrete - * class. - */ -template -void base::accept(Visitor&& visitor, Args&&... args) const -{ - if (is_value()) - { - using value_acceptor - = value_accept; - value_acceptor::accept(*this, std::forward(visitor), - std::forward(args)...); - } - else if (is_table()) - { - visitor.visit(static_cast(*this), - std::forward(args)...); - } - else if (is_array()) - { - visitor.visit(static_cast(*this), - std::forward(args)...); - } - else if (is_table_array()) - { - visitor.visit(static_cast(*this), - std::forward(args)...); - } -} - -/** - * Writer that can be passed to accept() functions of cpptoml objects and - * will output valid TOML to a stream. - */ -class toml_writer -{ - public: - /** - * Construct a toml_writer that will write to the given stream - */ - toml_writer(std::ostream& s, const std::string& indent_space = "\t") - : stream_(s), indent_(indent_space), has_naked_endline_(false) - { - // nothing - } - - public: - /** - * Output a base value of the TOML tree. - */ - template - void visit(const value& v, bool = false) - { - write(v); - } - - /** - * Output a table element of the TOML tree - */ - void visit(const table& t, bool in_array = false) - { - write_table_header(in_array); - std::vector values; - std::vector tables; - - for (const auto& i : t) - { - if (i.second->is_table() || i.second->is_table_array()) - { - tables.push_back(i.first); - } - else - { - values.push_back(i.first); - } - } - - for (unsigned int i = 0; i < values.size(); ++i) - { - path_.push_back(values[i]); - - if (i > 0) - endline(); - - write_table_item_header(*t.get(values[i])); - t.get(values[i])->accept(*this, false); - path_.pop_back(); - } - - for (unsigned int i = 0; i < tables.size(); ++i) - { - path_.push_back(tables[i]); - - if (values.size() > 0 || i > 0) - endline(); - - write_table_item_header(*t.get(tables[i])); - t.get(tables[i])->accept(*this, false); - path_.pop_back(); - } - - endline(); - } - - /** - * Output an array element of the TOML tree - */ - void visit(const array& a, bool = false) - { - write("["); - - for (unsigned int i = 0; i < a.get().size(); ++i) - { - if (i > 0) - write(", "); - - if (a.get()[i]->is_array()) - { - a.get()[i]->as_array()->accept(*this, true); - } - else - { - a.get()[i]->accept(*this, true); - } - } - - write("]"); - } - - /** - * Output a table_array element of the TOML tree - */ - void visit(const table_array& t, bool = false) - { - for (unsigned int j = 0; j < t.get().size(); ++j) - { - if (j > 0) - endline(); - - t.get()[j]->accept(*this, true); - } - - endline(); - } - - /** - * Escape a string for output. - */ - static std::string escape_string(const std::string& str) - { - std::string res; - for (auto it = str.begin(); it != str.end(); ++it) - { - if (*it == '\b') - { - res += "\\b"; - } - else if (*it == '\t') - { - res += "\\t"; - } - else if (*it == '\n') - { - res += "\\n"; - } - else if (*it == '\f') - { - res += "\\f"; - } - else if (*it == '\r') - { - res += "\\r"; - } - else if (*it == '"') - { - res += "\\\""; - } - else if (*it == '\\') - { - res += "\\\\"; - } - else if (static_cast(*it) <= UINT32_C(0x001f)) - { - res += "\\u"; - std::stringstream ss; - ss << std::hex << static_cast(*it); - res += ss.str(); - } - else - { - res += *it; - } - } - return res; - } - - protected: - /** - * Write out a string. - */ - void write(const value& v) - { - write("\""); - write(escape_string(v.get())); - write("\""); - } - - /** - * Write out a double. - */ - void write(const value& v) - { - std::stringstream ss; - ss << std::showpoint - << std::setprecision(std::numeric_limits::max_digits10) - << v.get(); - - auto double_str = ss.str(); - auto pos = double_str.find("e0"); - if (pos != std::string::npos) - double_str.replace(pos, 2, "e"); - pos = double_str.find("e-0"); - if (pos != std::string::npos) - double_str.replace(pos, 3, "e-"); - - stream_ << double_str; - has_naked_endline_ = false; - } - - /** - * Write out an integer, local_date, local_time, local_datetime, or - * offset_datetime. - */ - template - typename std::enable_if< - is_one_of::value>::type - write(const value& v) - { - write(v.get()); - } - - /** - * Write out a boolean. - */ - void write(const value& v) - { - write((v.get() ? "true" : "false")); - } - - /** - * Write out the header of a table. - */ - void write_table_header(bool in_array = false) - { - if (!path_.empty()) - { - indent(); - - write("["); - - if (in_array) - { - write("["); - } - - for (unsigned int i = 0; i < path_.size(); ++i) - { - if (i > 0) - { - write("."); - } - - if (path_[i].find_first_not_of("ABCDEFGHIJKLMNOPQRSTUVWXYZabcde" - "fghijklmnopqrstuvwxyz0123456789" - "_-") - == std::string::npos) - { - write(path_[i]); - } - else - { - write("\""); - write(escape_string(path_[i])); - write("\""); - } - } - - if (in_array) - { - write("]"); - } - - write("]"); - endline(); - } - } - - /** - * Write out the identifier for an item in a table. - */ - void write_table_item_header(const base& b) - { - if (!b.is_table() && !b.is_table_array()) - { - indent(); - - if (path_.back().find_first_not_of("ABCDEFGHIJKLMNOPQRSTUVWXYZabcde" - "fghijklmnopqrstuvwxyz0123456789" - "_-") - == std::string::npos) - { - write(path_.back()); - } - else - { - write("\""); - write(escape_string(path_.back())); - write("\""); - } - - write(" = "); - } - } - - private: - /** - * Indent the proper number of tabs given the size of - * the path. - */ - void indent() - { - for (std::size_t i = 1; i < path_.size(); ++i) - write(indent_); - } - - /** - * Write a value out to the stream. - */ - template - void write(const T& v) - { - stream_ << v; - has_naked_endline_ = false; - } - - /** - * Write an endline out to the stream - */ - void endline() - { - if (!has_naked_endline_) - { - stream_ << "\n"; - has_naked_endline_ = true; - } - } - - private: - std::ostream& stream_; - const std::string indent_; - std::vector path_; - bool has_naked_endline_; -}; - -inline std::ostream& operator<<(std::ostream& stream, const base& b) -{ - toml_writer writer{stream}; - b.accept(writer); - return stream; -} - -template -std::ostream& operator<<(std::ostream& stream, const value& v) -{ - toml_writer writer{stream}; - v.accept(writer); - return stream; -} - -inline std::ostream& operator<<(std::ostream& stream, const table& t) -{ - toml_writer writer{stream}; - t.accept(writer); - return stream; -} - -inline std::ostream& operator<<(std::ostream& stream, const table_array& t) -{ - toml_writer writer{stream}; - t.accept(writer); - return stream; -} - -inline std::ostream& operator<<(std::ostream& stream, const array& a) -{ - toml_writer writer{stream}; - a.accept(writer); - return stream; -} -} // namespace cpptoml -#endif // CPPTOML_H diff --git a/src/toml11/LICENSE b/src/toml11/LICENSE new file mode 100644 index 000000000..f55c511d6 --- /dev/null +++ b/src/toml11/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2017 Toru Niina + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/src/toml11/README.md b/src/toml11/README.md new file mode 100644 index 000000000..62b586305 --- /dev/null +++ b/src/toml11/README.md @@ -0,0 +1,1966 @@ +toml11 +====== + +[![Build Status on GitHub Actions](https://github.com/ToruNiina/toml11/workflows/build/badge.svg)](https://github.com/ToruNiina/toml11/actions) +[![Build Status on TravisCI](https://travis-ci.org/ToruNiina/toml11.svg?branch=master)](https://travis-ci.org/ToruNiina/toml11) +[![Build status on Appveyor](https://ci.appveyor.com/api/projects/status/m2n08a926asvg5mg/branch/master?svg=true)](https://ci.appveyor.com/project/ToruNiina/toml11/branch/master) +[![Build status on CircleCI](https://circleci.com/gh/ToruNiina/toml11/tree/master.svg?style=svg)](https://circleci.com/gh/ToruNiina/toml11/tree/master) +[![Version](https://img.shields.io/github/release/ToruNiina/toml11.svg?style=flat)](https://github.com/ToruNiina/toml11/releases) +[![License](https://img.shields.io/github/license/ToruNiina/toml11.svg?style=flat)](LICENSE) +[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.1209136.svg)](https://doi.org/10.5281/zenodo.1209136) + +toml11 is a C++11 (or later) header-only toml parser/encoder depending only on C++ standard library. + +- It is compatible to the latest version of [TOML v1.0.0](https://toml.io/en/v1.0.0). +- It is one of the most TOML standard compliant libraries, tested with [the language agnostic test suite for TOML parsers by BurntSushi](https://github.com/BurntSushi/toml-test). +- It shows highly informative error messages. You can see the error messages about invalid files at [CircleCI](https://circleci.com/gh/ToruNiina/toml11). +- It has configurable container. You can use any random-access containers and key-value maps as backend containers. +- It optionally preserves comments without any overhead. +- It has configurable serializer that supports comments, inline tables, literal strings and multiline strings. +- It supports user-defined type conversion from/into toml values. +- It correctly handles UTF-8 sequences, with or without BOM, both on posix and Windows. + +## Example + +```cpp +#include +#include + +int main() +{ + // ```toml + // title = "an example toml file" + // nums = [3, 1, 4, 1, 5] + // ``` + auto data = toml::parse("example.toml"); + + // find a value with the specified type from a table + std::string title = toml::find(data, "title"); + + // convert the whole array into any container automatically + std::vector nums = toml::find>(data, "nums"); + + // access with STL-like manner + if(!data.contains("foo")) + { + data["foo"] = "bar"; + } + + // pass a fallback + std::string name = toml::find_or(data, "name", "not found"); + + // width-dependent formatting + std::cout << std::setw(80) << data << std::endl; + + return 0; +} +``` + +## Table of Contents + +- [Integration](#integration) +- [Decoding a toml file](#decoding-a-toml-file) + - [In the case of syntax error](#in-the-case-of-syntax-error) + - [Invalid UTF-8 Codepoints](#invalid-utf-8-codepoints) +- [Finding a toml value](#finding-a-toml-value) + - [Finding a value in a table](#finding-a-value-in-a-table) + - [In case of error](#in-case-of-error) + - [Dotted keys](#dotted-keys) +- [Casting a toml value](#casting-a-toml-value) +- [Checking value type](#checking-value-type) +- [More about conversion](#more-about-conversion) + - [Converting an array](#converting-an-array) + - [Converting a table](#converting-a-table) + - [Getting an array of tables](#getting-an-array-of-tables) + - [Cost of conversion](#cost-of-conversion) + - [Converting datetime and its variants](#converting-datetime-and-its-variants) +- [Getting with a fallback](#getting-with-a-fallback) +- [Expecting conversion](#expecting-conversion) +- [Visiting a toml::value](#visiting-a-tomlvalue) +- [Constructing a toml::value](#constructing-a-tomlvalue) +- [Preserving Comments](#preserving-comments) +- [Customizing containers](#customizing-containers) +- [TOML literal](#toml-literal) +- [Conversion between toml value and arbitrary types](#conversion-between-toml-value-and-arbitrary-types) +- [Formatting user-defined error messages](#formatting-user-defined-error-messages) +- [Obtaining location information](#obtaining-location-information) +- [Exceptions](#exceptions) +- [Colorize Error Messages](#colorize-error-messages) +- [Serializing TOML data](#serializing-toml-data) +- [Underlying types](#underlying-types) +- [Unreleased TOML features](#unreleased-toml-features) +- [Breaking Changes from v2](#breaking-changes-from-v2) +- [Running Tests](#running-tests) +- [Contributors](#contributors) +- [Licensing Terms](#licensing-terms) + +## Integration + +Just include the file after adding it to the include path. + +```cpp +#include // that's all! now you can use it. +#include + +int main() +{ + const auto data = toml::parse("example.toml"); + const auto title = toml::find(data, "title"); + std::cout << "the title is " << title << std::endl; + return 0; +} +``` + +The convenient way is to add this repository as a git-submodule or to install +it in your system by CMake. + +Note for MSVC: We recommend to set `/Zc:__cplusplus` to detect C++ version correctly. + +## Decoding a toml file + +To parse a toml file, the only thing you have to do is +to pass a filename to the `toml::parse` function. + +```cpp +const std::string fname("sample.toml"); +const toml::value data = toml::parse(fname); +``` + +As required by the TOML specification, the top-level value is always a table. +You can find a value inside it, cast it into a table explicitly, and insert it as a value into other `toml::value`. + +If it encounters an error while opening a file, it will throw `std::runtime_error`. + +You can also pass a `std::istream` to the `toml::parse` function. +To show a filename in an error message, however, it is recommended to pass the +filename with the stream. + +```cpp +std::ifstream ifs("sample.toml", std::ios_base::binary); +assert(ifs.good()); +const auto data = toml::parse(ifs, /*optional -> */ "sample.toml"); +``` + +**Note**: When you are **on Windows, open a file in binary mode**. +If a file is opened in text-mode, CRLF ("\r\n") will automatically be +converted to LF ("\n") and this causes inconsistency between file size +and the contents that would be read. This causes weird error. + +### In the case of syntax error + +If there is a syntax error in a toml file, `toml::parse` will throw +`toml::syntax_error` that inherits `std::exception`. + +toml11 has clean and informative error messages inspired by Rust and +it looks like the following. + +```console +terminate called after throwing an instance of 'toml::syntax_error' + what(): [error] toml::parse_table: invalid line format # error description + --> example.toml # file name + 3 | a = 42 = true # line num and content + | ^------ expected newline, but got '='. # error reason +``` + +If you (mistakenly) duplicate tables and got an error, it is helpful to see +where they are. toml11 shows both at the same time like the following. + +```console +terminate called after throwing an instance of 'toml::syntax_error' + what(): [error] toml::insert_value: table ("table") already exists. + --> duplicate-table.toml + 1 | [table] + | ~~~~~~~ table already exists here + ... + 3 | [table] + | ~~~~~~~ table defined twice +``` + +When toml11 encounters a malformed value, it tries to detect what type it is. +Then it shows hints to fix the format. An error message while reading one of +the malformed files in [the language agnostic test suite](https://github.com/BurntSushi/toml-test). +is shown below. + +```console +what(): [error] bad time: should be HH:MM:SS.subsec + --> ./datetime-malformed-no-secs.toml + 1 | no-secs = 1987-07-05T17:45Z + | ^------- HH:MM:SS.subsec + | +Hint: pass: 1979-05-27T07:32:00, 1979-05-27 07:32:00.999999 +Hint: fail: 1979-05-27T7:32:00, 1979-05-27 17:32 +``` + +You can find other examples in a job named `output_result` on +[CircleCI](https://circleci.com/gh/ToruNiina/toml11). + +Since the error message generation is generally a difficult task, the current +status is not ideal. If you encounter a weird error message, please let us know +and contribute to improve the quality! + +### Invalid UTF-8 codepoints + +It throws `syntax_error` if a value of an escape sequence +representing unicode character is not a valid UTF-8 codepoint. + +```console + what(): [error] toml::read_utf8_codepoint: input codepoint is too large. + --> utf8.toml + 1 | exceeds_unicode = "\U0011FFFF example" + | ^--------- should be in [0x00..0x10FFFF] +``` + +## Finding a toml value + +After parsing successfully, you can obtain the values from the result of +`toml::parse` using `toml::find` function. + +```toml +# sample.toml +answer = 42 +pi = 3.14 +numbers = [1,2,3] +time = 1979-05-27T07:32:00Z +``` + +``` cpp +const auto data = toml::parse("sample.toml"); +const auto answer = toml::find(data, "answer"); +const auto pi = toml::find(data, "pi"); +const auto numbers = toml::find>(data, "numbers"); +const auto timepoint = toml::find(data, "time"); +``` + +By default, `toml::find` returns a `toml::value`. + +```cpp +const toml::value& answer = toml::find(data, "answer"); +``` + +When you pass an exact TOML type that does not require type conversion, +`toml::find` returns a reference without copying the value. + +```cpp +const auto data = toml::parse("sample.toml"); +const auto& answer = toml::find(data, "answer"); +``` + +If the specified type requires conversion, you can't take a reference to the value. +See also [underlying types](#underlying-types). + +**NOTE**: For some technical reason, automatic conversion between `integer` and +`floating` is not supported. If you want to get a floating value even if a value +has integer value, you need to convert it manually after obtaining a value, +like the following. + +```cpp +const auto vx = toml::find(data, "x"); +double x = vx.is_floating() ? vx.as_floating(std::nothrow) : + static_cast(vx.as_integer()); // it throws if vx is neither + // floating nor integer. +``` + +### Finding a value in a table + +There are several way to get a value defined in a table. +First, you can get a table as a normal value and find a value from the table. + +```toml +[fruit] +name = "apple" +[fruit.physical] +color = "red" +shape = "round" +``` + +``` cpp +const auto data = toml::parse("fruit.toml"); +const auto& fruit = toml::find(data, "fruit"); +const auto name = toml::find(fruit, "name"); + +const auto& physical = toml::find(fruit, "physical"); +const auto color = toml::find(physical, "color"); +const auto shape = toml::find(physical, "shape"); +``` + +Here, variable `fruit` is a `toml::value` and can be used as the first argument +of `toml::find`. + +Second, you can pass as many arguments as the number of subtables to `toml::find`. + +```cpp +const auto data = toml::parse("fruit.toml"); +const auto color = toml::find(data, "fruit", "physical", "color"); +const auto shape = toml::find(data, "fruit", "physical", "shape"); +``` + +### Finding a value in an array + +You can find n-th value in an array by `toml::find`. + +```toml +values = ["foo", "bar", "baz"] +``` + +``` cpp +const auto data = toml::parse("sample.toml"); +const auto values = toml::find(data, "values"); +const auto bar = toml::find(values, 1); +``` + +`toml::find` can also search array recursively. + +```cpp +const auto data = toml::parse("fruit.toml"); +const auto bar = toml::find(data, "values", 1); +``` + +Before calling `toml::find`, you can check if a value corresponding to a key +exists. You can use both `bool toml::value::contains(const key&) const` and +`std::size_t toml::value::count(const key&) const`. Those behaves like the +`std::map::contains` and `std::map::count`. + +```cpp +const auto data = toml::parse("fruit.toml"); +if(data.contains("fruit") && data.at("fruit").count("physical") != 0) +{ + // ... +} +``` + +### In case of error + +If the value does not exist, `toml::find` throws `std::out_of_range` with the +location of the table. + +```console +terminate called after throwing an instance of 'std::out_of_range' + what(): [error] key "answer" not found + --> example.toml + 6 | [tab] + | ~~~~~ in this table +``` + +---- + +If the specified type differs from the actual value contained, it throws +`toml::type_error` that inherits `std::exception`. + +Similar to the case of syntax error, toml11 also displays clean error messages. +The error message when you choose `int` to get `string` value would be like this. + +```console +terminate called after throwing an instance of 'toml::type_error' + what(): [error] toml::value bad_cast to integer + --> example.toml + 3 | title = "TOML Example" + | ~~~~~~~~~~~~~~ the actual type is string +``` + +**NOTE**: In order to show this kind of error message, all the toml values have +a pointer to represent its range in a file. The entire contents of a file is +shared by `toml::value`s and remains on the heap memory. It is recommended to +destruct all the `toml::value` classes after configuring your application +if you have a large TOML file compared to the memory resource. + +### Dotted keys + +TOML v0.5.0 has a new feature named "dotted keys". +You can chain keys to represent the structure of the data. + +```toml +physical.color = "orange" +physical.shape = "round" +``` + +This is equivalent to the following. + +```toml +[physical] +color = "orange" +shape = "round" +``` + +You can get both of the above tables with the same c++ code. + +```cpp +const auto physical = toml::find(data, "physical"); +const auto color = toml::find(physical, "color"); +``` + +The following code does not work for the above toml file. + +```cpp +// XXX this does not work! +const auto color = toml::find(data, "physical.color"); +``` + +The above code works with the following toml file. + +```toml +"physical.color" = "orange" +# equivalent to {"physical.color": "orange"}, +# NOT {"physical": {"color": "orange"}}. +``` + + +## Casting a toml value + +### `toml::get` + +`toml::parse` returns `toml::value`. `toml::value` is a union type that can +contain one of the following types. + +- `toml::boolean` (`bool`) +- `toml::integer` (`std::int64_t`) +- `toml::floating` (`double`) +- `toml::string` (a type convertible to std::string) +- `toml::local_date` +- `toml::local_time` +- `toml::local_datetime` +- `toml::offset_datetime` +- `toml::array` (by default, `std::vector`) + - It depends. See [customizing containers](#customizing-containers) for detail. +- `toml::table` (by default, `std::unordered_map`) + - It depends. See [customizing containers](#customizing-containers) for detail. + +To get a value inside, you can use `toml::get()`. The usage is the same as +`toml::find` (actually, `toml::find` internally uses `toml::get` after casting +a value to `toml::table`). + +``` cpp +const toml::value data = toml::parse("sample.toml"); +const toml::value answer_ = toml::get(data).at("answer"); +const std::int64_t answer = toml::get(answer_); +``` + +When you pass an exact TOML type that does not require type conversion, +`toml::get` returns a reference through which you can modify the content +(if the `toml::value` is `const`, it returns `const` reference). + +```cpp +toml::value data = toml::parse("sample.toml"); +toml::value answer_ = toml::get(data).at("answer"); +toml::integer& answer = toml::get(answer_); +answer = 6 * 9; // write to data.answer. now `answer_` contains 54. +``` + +If the specified type requires conversion, you can't take a reference to the value. +See also [underlying types](#underlying-types). + +It also throws a `toml::type_error` if the type differs. + +### `as_xxx` + +You can also use a member function to cast a value. + +```cpp +const std::int64_t answer = data.as_table().at("answer").as_integer(); +``` + +It also throws a `toml::type_error` if the type differs. If you are sure that +the value `v` contains a value of the specified type, you can suppress checking +by passing `std::nothrow`. + +```cpp +const auto& answer = data.as_table().at("answer"); +if(answer.is_integer() && answer.as_integer(std::nothrow) == 42) +{ + std::cout << "value is 42" << std::endl; +} +``` + +If `std::nothrow` is passed, the functions are marked as noexcept. + +By casting a `toml::value` into an array or a table, you can iterate over the +elements. + +```cpp +const auto data = toml::parse("example.toml"); +std::cout << "keys in the top-level table are the following: \n"; +for(const auto& [k, v] : data.as_table()) +{ + std::cout << k << '\n'; +} + +const auto& fruits = toml::find(data, "fruits"); +for(const auto& v : fruits.as_array()) +{ + std::cout << toml::find(v, "name") << '\n'; +} +``` + +The full list of the functions is below. + +```cpp +namespace toml { +class value { + // ... + const boolean& as_boolean() const&; + const integer& as_integer() const&; + const floating& as_floating() const&; + const string& as_string() const&; + const offset_datetime& as_offset_datetime() const&; + const local_datetime& as_local_datetime() const&; + const local_date& as_local_date() const&; + const local_time& as_local_time() const&; + const array& as_array() const&; + const table& as_table() const&; + // -------------------------------------------------------- + // non-const version + boolean& as_boolean() &; + // ditto... + // -------------------------------------------------------- + // rvalue version + boolean&& as_boolean() &&; + // ditto... + + // -------------------------------------------------------- + // noexcept versions ... + const boolean& as_boolean(const std::nothrow_t&) const& noexcept; + boolean& as_boolean(const std::nothrow_t&) & noexcept; + boolean&& as_boolean(const std::nothrow_t&) && noexcept; + // ditto... +}; +} // toml +``` + +### `at()` + +You can access to the element of a table and an array by `toml::basic_value::at`. + +```cpp +const toml::value v{1,2,3,4,5}; +std::cout << v.at(2).as_integer() << std::endl; // 3 + +const toml::value v{{"foo", 42}, {"bar", 3.14}}; +std::cout << v.at("foo").as_integer() << std::endl; // 42 +``` + +If an invalid key (integer for a table, string for an array), it throws +`toml::type_error` for the conversion. If the provided key is out-of-range, +it throws `std::out_of_range`. + +Note that, although `std::string` has `at()` member function, `toml::value::at` +throws if the contained type is a string. Because `std::string` does not +contain `toml::value`. + +### `operator[]` + +You can also access to the element of a table and an array by +`toml::basic_value::operator[]`. + +```cpp +const toml::value v{1,2,3,4,5}; +std::cout << v[2].as_integer() << std::endl; // 3 + +const toml::value v{{"foo", 42}, {"bar", 3.14}}; +std::cout << v["foo"].as_integer() << std::endl; // 42 +``` + +When you access to a `toml::value` that is not initialized yet via +`operator[](const std::string&)`, the `toml::value` will be a table, +just like the `std::map`. + +```cpp +toml::value v; // not initialized as a table. +v["foo"] = 42; // OK. `v` will be a table. +``` + +Contrary, if you access to a `toml::value` that contains an array via `operator[]`, +it does not check anything. It converts `toml::value` without type check and then +access to the n-th element without boundary check, just like the `std::vector::operator[]`. + +```cpp +toml::value v; // not initialized as an array +v[2] = 42; // error! UB +``` + +Please make sure that the `toml::value` has an array inside when you access to +its element via `operator[]`. + +## Checking value type + +You can check the type of a value by `is_xxx` function. + +```cpp +const toml::value v = /* ... */; +if(v.is_integer()) +{ + std::cout << "value is an integer" << std::endl; +} +``` + +The complete list of the functions is below. + +```cpp +namespace toml { +class value { + // ... + bool is_boolean() const noexcept; + bool is_integer() const noexcept; + bool is_floating() const noexcept; + bool is_string() const noexcept; + bool is_offset_datetime() const noexcept; + bool is_local_datetime() const noexcept; + bool is_local_date() const noexcept; + bool is_local_time() const noexcept; + bool is_array() const noexcept; + bool is_table() const noexcept; + bool is_uninitialized() const noexcept; + // ... +}; +} // toml +``` + +Also, you can get `enum class value_t` from `toml::value::type()`. + +```cpp +switch(data.at("something").type()) +{ + case toml::value_t::integer: /*do some stuff*/ ; break; + case toml::value_t::floating: /*do some stuff*/ ; break; + case toml::value_t::string : /*do some stuff*/ ; break; + default : throw std::runtime_error( + "unexpected type : " + toml::stringize(data.at("something").type())); +} +``` + +The complete list of the `enum`s can be found in the section +[underlying types](#underlying-types). + +The `enum`s can be used as a parameter of `toml::value::is` function like the following. + +```cpp +toml::value v = /* ... */; +if(v.is(toml::value_t::boolean)) // ... +``` + +## More about conversion + +Since `toml::find` internally uses `toml::get`, all the following examples work +with both `toml::get` and `toml::find`. + +### Converting an array + +You can get any kind of `container` class from a `toml::array` +except for `map`-like classes. + +``` cpp +// # sample.toml +// numbers = [1,2,3] + +const auto numbers = toml::find(data, "numbers"); + +const auto vc = toml::get >(numbers); +const auto ls = toml::get >(numbers); +const auto dq = toml::get >(numbers); +const auto ar = toml::get>(numbers); +// if the size of data.at("numbers") is larger than that of std::array, +// it will throw toml::type_error because std::array is not resizable. +``` + +Surprisingly, you can convert `toml::array` into `std::pair` and `std::tuple`. + +```cpp +// numbers = [1,2,3] +const auto tp = toml::get>(numbers); +``` + +This functionality is helpful when you have a toml file like the following. + +```toml +array_of_arrays = [[1, 2, 3], ["foo", "bar", "baz"]] # toml allows this +``` + +What is the corresponding C++ type? +Obviously, it is a `std::pair` of `std::vector`s. + +```cpp +const auto array_of_arrays = toml::find(data, "array_of_arrays"); +const auto aofa = toml::get< + std::pair, std::vector> + >(array_of_arrays); +``` + +If you don't know the type of the elements, you can use `toml::array`, +which is a `std::vector` of `toml::value`, instead. + +```cpp +const auto a_of_a = toml::get(array_of_arrays); +const auto first = toml::get>(a_of_a.at(0)); +``` + +You can change the implementation of `toml::array` with `std::deque` or some +other array-like container. See [Customizing containers](#customizing-containers) +for detail. + +### Converting a table + +When all the values of the table have the same type, toml11 allows you to +convert a `toml::table` to a `map` that contains the convertible type. + +```toml +[tab] +key1 = "foo" # all the values are +key2 = "bar" # toml String +``` + +```cpp +const auto data = toml::parse("sample.toml"); +const auto tab = toml::find>(data, "tab"); +std::cout << tab["key1"] << std::endl; // foo +std::cout << tab["key2"] << std::endl; // bar +``` + +But since `toml::table` is just an alias of `std::unordered_map`, +normally you don't need to convert it because it has all the functionalities that +`std::unordered_map` has (e.g. `operator[]`, `count`, and `find`). In most cases +`toml::table` is sufficient. + +```cpp +toml::table tab = toml::get(data); +if(data.count("title") != 0) +{ + data["title"] = std::string("TOML example"); +} +``` + +You can change the implementation of `toml::table` with `std::map` or some +other map-like container. See [Customizing containers](#customizing-containers) +for detail. + +### Getting an array of tables + +An array of tables is just an array of tables. +You can get it in completely the same way as the other arrays and tables. + +```toml +# sample.toml +array_of_inline_tables = [{key = "value1"}, {key = "value2"}, {key = "value3"}] + +[[array_of_tables]] +key = "value4" +[[array_of_tables]] +key = "value5" +[[array_of_tables]] +key = "value6" +``` + +```cpp +const auto data = toml::parse("sample.toml"); +const auto aot1 = toml::find>(data, "array_of_inline_tables"); +const auto aot2 = toml::find>(data, "array_of_tables"); +``` + +### Cost of conversion + +Although conversion through `toml::(get|find)` is convenient, it has additional +copy-cost because it copies data contained in `toml::value` to the +user-specified type. Of course in some cases this overhead is not ignorable. + +```cpp +// the following code constructs a std::vector. +// it requires heap allocation for vector and element conversion. +const auto array = toml::find>(data, "foo"); +``` + +By passing the exact types, `toml::get` returns reference that has no overhead. + +``` cpp +const auto& tab = toml::find(data, "tab"); +const auto& numbers = toml::find(data, "numbers"); +``` + +Also, `as_xxx` are zero-overhead because they always return a reference. + +``` cpp +const auto& tab = toml::find(data, "tab" ).as_table(); +const auto& numbers = toml::find(data, "numbers").as_array(); +``` + +In this case you need to call `toml::get` each time you access to +the element of `toml::array` because `toml::array` is an array of `toml::value`. + +```cpp +const auto& num0 = toml::get(numbers.at(0)); +const auto& num1 = toml::get(numbers.at(1)); +const auto& num2 = toml::get(numbers.at(2)); +``` + +### Converting datetime and its variants + +TOML v0.5.0 has 4 different datetime objects, `local_date`, `local_time`, +`local_datetime`, and `offset_datetime`. + +Since `local_date`, `local_datetime`, and `offset_datetime` represent a time +point, you can convert them to `std::chrono::system_clock::time_point`. + +Contrary, `local_time` does not represents a time point because they lack a +date information, but it can be converted to `std::chrono::duration` that +represents a duration from the beginning of the day, `00:00:00.000`. + +```toml +# sample.toml +date = 2018-12-23 +time = 12:30:00 +l_dt = 2018-12-23T12:30:00 +o_dt = 2018-12-23T12:30:00+09:30 +``` + +```cpp +const auto data = toml::parse("sample.toml"); + +const auto date = toml::get(data.at("date")); +const auto l_dt = toml::get(data.at("l_dt")); +const auto o_dt = toml::get(data.at("o_dt")); + +const auto time = toml::get(data.at("time")); // 12 * 60 + 30 min +``` + +`local_date` and `local_datetime` are assumed to be in the local timezone when +they are converted into `time_point`. On the other hand, `offset_datetime` only +uses the offset part of the data and it does not take local timezone into account. + +To contain datetime data, toml11 defines its own datetime types. +For more detail, you can see the definitions in [toml/datetime.hpp](toml/datetime.hpp). + +## Getting with a fallback + +`toml::find_or` returns a default value if the value is not found or has a +different type. + +```cpp +const auto data = toml::parse("example.toml"); +const auto num = toml::find_or(data, "num", 42); +``` + +It works recursively if you pass several keys for subtables. +In that case, the last argument is considered to be the optional value. +All other arguments between `toml::value` and the optinoal value are considered as keys. + +```cpp +// [fruit.physical] +// color = "red" +auto data = toml::parse("fruit.toml"); +auto color = toml::find_or(data, "fruit", "physical", "color", "red"); +// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^ +// arguments optional value +``` + +Also, `toml::get_or` returns a default value if `toml::get` failed. + +```cpp +toml::value v("foo"); // v contains String +const int value = toml::get_or(v, 42); // conversion fails. it returns 42. +``` + +These functions automatically deduce what type you want to get +from the default value you passed. + +To get a reference through this function, take care about the default value. + +```cpp +toml::value v("foo"); // v contains String +toml::integer& i = toml::get_or(v, 42); // does not work because binding `42` + // to `integer&` is invalid +toml::integer opt = 42; +toml::integer& i = toml::get_or(v, opt); // this works. +``` + +## Expecting conversion + +By using `toml::expect`, you will get your expected value or an error message +without throwing `toml::type_error`. + +```cpp +const auto value = toml::expect(data.at("title")); +if(value.is_ok()) { + std::cout << value.unwrap() << std::endl; +} else { + std::cout << value.unwrap_err() << std::endl; +} +``` + +Also, you can pass a function object to modify the expected value. + +```cpp +const auto value = toml::expect(data.at("number")) + .map(// function that receives expected type (here, int) + [](const int number) -> double { + return number * 1.5 + 1.0; + }).unwrap_or(/*default value =*/ 3.14); +``` + +## Visiting a toml::value + +toml11 provides `toml::visit` to apply a function to `toml::value` in the +same way as `std::variant`. + +```cpp +const toml::value v(3.14); +toml::visit([](const auto& val) -> void { + std::cout << val << std::endl; + }, v); +``` + +The function object that would be passed to `toml::visit` must be able to +receive all the possible TOML types. Also, the result types should be the same +each other. + +## Constructing a toml::value + +`toml::value` can be constructed in various ways. + +```cpp +toml::value v(true); // boolean +toml::value v(42); // integer +toml::value v(3.14); // floating +toml::value v("foobar"); // string +toml::value v(toml::local_date(2019, toml::month_t::Apr, 1)); // date +toml::value v{1, 2, 3, 4, 5}; // array +toml::value v{{"foo", 42}, {"bar", 3.14}, {"baz", "qux"}}; // table +``` + +When constructing a string, you can choose to use either literal or basic string. +By default, it will be a basic string. + +```cpp +toml::value v("foobar", toml::string_t::basic ); +toml::value v("foobar", toml::string_t::literal); +``` + +Datetime objects can be constructed from `std::tm` and +`std::chrono::system_clock::time_point`. But you need to specify what type +you use to avoid ambiguity. + +```cpp +const auto now = std::chrono::system_clock::now(); +toml::value v(toml::local_date(now)); +toml::value v(toml::local_datetime(now)); +toml::value v(toml::offset_datetime(now)); +``` + +Since local time is not equivalent to a time point, because it lacks date +information, it will be constructed from `std::chrono::duration`. + +```cpp +toml::value v(toml::local_time(std::chrono::hours(10))); +``` + +You can construct an array object not only from `initializer_list`, but also +from STL containers. In that case, the element type must be convertible to +`toml::value`. + +```cpp +std::vector vec{1,2,3,4,5}; +toml::value v(vec); +``` + +When you construct an array value, all the elements of `initializer_list` +must be convertible into `toml::value`. + +If a `toml::value` has an array, you can `push_back` an element in it. + +```cpp +toml::value v{1,2,3,4,5}; +v.push_back(6); +``` + +`emplace_back` also works. + +## Preserving comments + +toml11 v3 or later allows you yo choose whether comments are preserved or not via template parameter + +```cpp +const auto data1 = toml::parse("example.toml"); +const auto data2 = toml::parse("example.toml"); +``` + +or macro definition. + +```cpp +#define TOML11_PRESERVE_COMMENTS_BY_DEFAULT +#include +``` + +This feature is controlled by template parameter in `toml::basic_value<...>`. +`toml::value` is an alias of `toml::basic_value<...>`. + +If template parameter is explicitly specified, the return value of `toml::parse` +will be `toml::basic_value`. +If the macro is defined, the alias `toml::value` will be +`toml::basic_value`. + +Comments related to a value can be obtained by `toml::value::comments()`. +The return value has the same interface as `std::vector`. + +```cpp +const auto& com = v.comments(); +for(const auto& c : com) +{ + std::cout << c << std::endl; +} +``` + +Comments just before and just after (within the same line) a value are kept in a value. + +```toml +# this is a comment for v1. +v1 = "foo" + +v2 = "bar" # this is a comment for v2. +# Note that this comment is NOT a comment for v2. + +# this comment is not related to any value +# because there are empty lines between v3. +# this comment will be ignored even if you set `preserve_comments`. + +# this is a comment for v3 +# this is also a comment for v3. +v3 = "baz" # ditto. +``` + +Each comment line becomes one element of a `std::vector`. + +Hash signs will be removed, but spaces after hash sign will not be removed. + +```cpp +v1.comments().at(0) == " this is a comment for v1."s; + +v2.comments().at(1) == " this is a comment for v1."s; + +v3.comments().at(0) == " this is a comment for v3."s; +v3.comments().at(1) == " this is also a comment for v3."s; +v3.comments().at(2) == " ditto."s; +``` + +Note that a comment just after an opening brace of an array will not be a +comment for the array. + +```toml +# this is a comment for a. +a = [ # this is not a comment for a. this will be ignored. + 1, 2, 3, + # this is a comment for `42`. + 42, # this is also a comment for `42`. + 5 +] # this is a comment for a. +``` + +You can also append and modify comments. +The interfaces are the same as `std::vector`. + +```cpp +toml::basic_value v(42); +v.comments().push_back(" add this comment."); +// # add this comment. +// i = 42 +``` + +Also, you can pass a `std::vector` when constructing a +`toml::basic_value`. + +```cpp +std::vector comments{"comment 1", "comment 2"}; +const toml::basic_value v1(42, std::move(comments)); +const toml::basic_value v2(42, {"comment 1", "comment 2"}); +``` + +When `toml::discard_comments` is chosen, comments will not be contained in a value. +`value::comments()` will always be kept empty. +All the modification on comments would be ignored. +All the element access in a `discard_comments` causes the same error as accessing +an element of an empty `std::vector`. + +The comments will also be serialized. If comments exist, those comments will be +added just before the values. + +__NOTE__: Result types from `toml::parse(...)` and +`toml::parse(...)` are different. + +## Customizing containers + +Actually, `toml::basic_value` has 3 template arguments. + +```cpp +template class Table = std::unordered_map, + template class Array = std::vector> +class basic_value; +``` + +This enables you to change the containers used inside. E.g. you can use +`std::map` to contain a table object instead of `std::unordered_map`. +And also can use `std::deque` as a array object instead of `std::vector`. + +You can set these parameters while calling `toml::parse` function. + +```cpp +const auto data = toml::parse< + toml::preserve_comments, std::map, std::deque + >("example.toml"); +``` + +Needless to say, the result types from `toml::parse(...)` and +`toml::parse(...)` are different (unless you specify the same +types as default). + +Note that, since `toml::table` and `toml::array` is an alias for a table and an +array of a default `toml::value`, so it is different from the types actually +contained in a `toml::basic_value` when you customize containers. +To get the actual type in a generic way, use +`typename toml::basic_type::table_type` and +`typename toml::basic_type::array_type`. + +## TOML literal + +toml11 supports `"..."_toml` literal. +It accept both a bare value and a file content. + +```cpp +using namespace toml::literals::toml_literals; + +// `_toml` can convert a bare value without key +const toml::value v = u8"0xDEADBEEF"_toml; +// v is an Integer value containing 0xDEADBEEF. + +// raw string literal (`R"(...)"` is useful for this purpose) +const toml::value t = u8R"( + title = "this is TOML literal" + [table] + key = "value" +)"_toml; +// the literal will be parsed and the result will be contained in t +``` + +The literal function is defined in the same way as the standard library literals +such as `std::literals::string_literals::operator""s`. + +```cpp +namespace toml +{ +inline namespace literals +{ +inline namespace toml_literals +{ +toml::value operator"" _toml(const char* str, std::size_t len); +} // toml_literals +} // literals +} // toml +``` + +Access to the operator can be gained with `using namespace toml::literals;`, +`using namespace toml::toml_literals`, and `using namespace toml::literals::toml_literals`. + +Note that a key that is composed only of digits is allowed in TOML. +And, unlike the file parser, toml-literal allows a bare value without a key. +Thus it is difficult to distinguish arrays having integers and definitions of +tables that are named as digits. +Currently, literal `[1]` becomes a table named "1". +To ensure a literal to be considered as an array with one element, you need to +add a comma after the first element (like `[1,]`). + +```cpp +"[1,2,3]"_toml; // This is an array +"[table]"_toml; // This is a table that has an empty table named "table" inside. +"[[1,2,3]]"_toml; // This is an array of arrays +"[[table]]"_toml; // This is a table that has an array of tables inside. + +"[[1]]"_toml; // This literal is ambiguous. + // Currently, it becomes a table that has array of table "1". +"1 = [{}]"_toml; // This is a table that has an array of table named 1. +"[[1,]]"_toml; // This is an array of arrays. +"[[1],]"_toml; // ditto. +``` + +NOTE: `_toml` literal returns a `toml::value` that does not have comments. + +## Conversion between toml value and arbitrary types + +You can also use `toml::get` and other related functions with the types +you defined after you implement a way to convert it. + +```cpp +namespace ext +{ +struct foo +{ + int a; + double b; + std::string c; +}; +} // ext + +const auto data = toml::parse("example.toml"); + +// to do this +const foo f = toml::find(data, "foo"); +``` + +There are 3 ways to use `toml::get` with the types that you defined. + +The first one is to implement `from_toml(const toml::value&)` member function. + +```cpp +namespace ext +{ +struct foo +{ + int a; + double b; + std::string c; + + void from_toml(const toml::value& v) + { + this->a = toml::find(v, "a"); + this->b = toml::find(v, "b"); + this->c = toml::find(v, "c"); + return; + } +}; +} // ext +``` + +In this way, because `toml::get` first constructs `foo` without arguments, +the type should be default-constructible. + +The second is to implement `constructor(const toml::value&)`. + +```cpp +namespace ext +{ +struct foo +{ + explicit foo(const toml::value& v) + : a(toml::find(v, "a")), b(toml::find(v, "b")), + c(toml::find(v, "c")) + {} + + int a; + double b; + std::string c; +}; +} // ext +``` + +Note that implicit default constructor declaration will be suppressed +when a constructor is defined. If you want to use the struct (here, `foo`) +in a container (e.g. `std::vector`), you may need to define default +constructor explicitly. + +The third is to implement specialization of `toml::from` for your type. + +```cpp +namespace ext +{ +struct foo +{ + int a; + double b; + std::string c; +}; +} // ext + +namespace toml +{ +template<> +struct from +{ + static ext::foo from_toml(const value& v) + { + ext::foo f; + f.a = find(v, "a"); + f.b = find(v, "b"); + f.c = find(v, "c"); + return f; + } +}; +} // toml +``` + +In this way, since the conversion function is defined outside of the class, +you can add conversion between `toml::value` and classes defined in another library. + +In some cases, a class has a templatized constructor that takes a template, `T`. +It confuses `toml::get/find` because it makes the class "constructible" from +`toml::value`. To avoid this problem, `toml::from` and `from_toml` always +precede constructor. It makes easier to implement conversion between +`toml::value` and types defined in other libraries because it skips constructor. + +But, importantly, you cannot define `toml::from` and `T.from_toml` at the same +time because it causes ambiguity in the overload resolution of `toml::get` and `toml::find`. + +So the precedence is `toml::from` == `T.from_toml()` > `T(toml::value)`. + +If you want to convert any versions of `toml::basic_value`, +you need to templatize the conversion function as follows. + +```cpp +struct foo +{ + template class M, template class A> + void from_toml(const toml::basic_value& v) + { + this->a = toml::find(v, "a"); + this->b = toml::find(v, "b"); + this->c = toml::find(v, "c"); + return; + } +}; +// or +namespace toml +{ +template<> +struct from +{ + template class M, template class A> + static ext::foo from_toml(const basic_value& v) + { + ext::foo f; + f.a = find(v, "a"); + f.b = find(v, "b"); + f.c = find(v, "c"); + return f; + } +}; +} // toml +``` + +---- + +The opposite direction is also supported in a similar way. You can directly +pass your type to `toml::value`'s constructor by introducing `into_toml` or +`toml::into`. + +```cpp +namespace ext +{ +struct foo +{ + int a; + double b; + std::string c; + + toml::value into_toml() const // you need to mark it const. + { + return toml::value{{"a", this->a}, {"b", this->b}, {"c", this->c}}; + } +}; +} // ext + +ext::foo f{42, 3.14, "foobar"}; +toml::value v(f); +``` + +The definition of `toml::into` is similar to `toml::from`. + +```cpp +namespace ext +{ +struct foo +{ + int a; + double b; + std::string c; +}; +} // ext + +namespace toml +{ +template<> +struct into +{ + static toml::value into_toml(const ext::foo& f) + { + return toml::value{{"a", f.a}, {"b", f.b}, {"c", f.c}}; + } +}; +} // toml + +ext::foo f{42, 3.14, "foobar"}; +toml::value v(f); +``` + +Any type that can be converted to `toml::value`, e.g. `int`, `toml::table` and +`toml::array` are okay to return from `into_toml`. + +You can also return a custom `toml::basic_value` from `toml::into`. + +```cpp +namespace toml +{ +template<> +struct into +{ + static toml::basic_value into_toml(const ext::foo& f) + { + toml::basic_value v{{"a", f.a}, {"b", f.b}, {"c", f.c}}; + v.comments().push_back(" comment"); + return v; + } +}; +} // toml +``` + +But note that, if this `basic_value` would be assigned into other `toml::value` +that discards `comments`, the comments would be dropped. + +### Macro to automatically define conversion functions + +There is a helper macro that automatically generates conversion functions `from` and `into` for a simple struct. + +```cpp +namespace foo +{ +struct Foo +{ + std::string s; + double d; + int i; +}; +} // foo + +TOML11_DEFINE_CONVERSION_NON_INTRUSIVE(foo::Foo, s, d, i) + +int main() +{ + const auto file = toml::parse("example.toml"); + auto f = toml::find(file, "foo"); +} +``` + +And then you can use `toml::find(file, "foo");` + +**Note** that, because of a slight difference in implementation of preprocessor between gcc/clang and MSVC, [you need to define `/Zc:preprocessor`](https://github.com/ToruNiina/toml11/issues/139#issuecomment-803683682) to use it in MSVC (Thank you @glebm !). + +## Formatting user-defined error messages + +When you encounter an error after you read the toml value, you may want to +show the error with the value. + +toml11 provides you a function that formats user-defined error message with +related values. With a code like the following, + +```cpp +const auto value = toml::find(data, "num"); +if(value < 0) +{ + std::cerr << toml::format_error("[error] value should be positive", + data.at("num"), "positive number required") + << std::endl; +} +``` + +you will get an error message like this. + +```console +[error] value should be positive + --> example.toml + 3 | num = -42 + | ~~~ positive number required +``` + +When you pass two values to `toml::format_error`, + +```cpp +const auto min = toml::find(range, "min"); +const auto max = toml::find(range, "max"); +if(max < min) +{ + std::cerr << toml::format_error("[error] max should be larger than min", + data.at("min"), "minimum number here", + data.at("max"), "maximum number here"); + << std::endl; +} +``` + +you will get an error message like this. + +```console +[error] max should be larger than min + --> example.toml + 3 | min = 54 + | ~~ minimum number here + ... + 4 | max = 42 + | ~~ maximum number here +``` + +You can print hints at the end of the message. + +```cpp +std::vector hints; +hints.push_back("positive number means n >= 0."); +hints.push_back("negative number is not positive."); +std::cerr << toml::format_error("[error] value should be positive", + data.at("num"), "positive number required", hints) + << std::endl; +``` + +```console +[error] value should be positive + --> example.toml + 2 | num = 42 + | ~~ positive number required + | +Hint: positive number means n >= 0. +Hint: negative number is not positive. +``` + +## Obtaining location information + +You can also format error messages in your own way by using `source_location`. + +```cpp +struct source_location +{ + std::uint_least32_t line() const noexcept; + std::uint_least32_t column() const noexcept; + std::uint_least32_t region() const noexcept; + std::string const& file_name() const noexcept; + std::string const& line_str() const noexcept; +}; +// +-- line() +--- length of the region (here, region() == 9) +// v .---+---. +// 12 | value = "foo bar" <- line_str() returns the line itself. +// ^-------- column() points here +``` + +You can get this by +```cpp +const toml::value v = /*...*/; +const toml::source_location loc = v.location(); +``` + +## Exceptions + +The following `exception` classes inherits `toml::exception` that inherits +`std::exception`. + +```cpp +namespace toml { +struct exception : public std::exception {/**/}; +struct syntax_error : public toml::exception {/**/}; +struct type_error : public toml::exception {/**/}; +struct internal_error : public toml::exception {/**/}; +} // toml +``` + +`toml::exception` has `toml::exception::location()` member function that returns +`toml::source_location`, in addition to `what()`. + +```cpp +namespace toml { +struct exception : public std::exception +{ + // ... + source_location const& location() const noexcept; +}; +} // toml +``` + +It represents where the error occurs. + +`syntax_error` will be thrown from `toml::parse` and `_toml` literal. +`type_error` will be thrown from `toml::get/find`, `toml::value::as_xxx()`, and +other functions that takes a content inside of `toml::value`. + +Note that, currently, from `toml::value::at()` and `toml::find(value, key)` +may throw an `std::out_of_range` that does not inherits `toml::exception`. + +Also, in some cases, most likely in the file open error, it will throw an +`std::runtime_error`. + +## Colorize Error Messages + +By defining `TOML11_COLORIZE_ERROR_MESSAGE`, the error messages from +`toml::parse` and `toml::find|get` will be colorized. By default, this feature +is turned off. + +With the following toml file taken from `toml-lang/toml/tests/hard_example.toml`, + +```toml +[error] +array = [ + "This might most likely happen in multiline arrays", + Like here, + "or here, + and here" + ] End of array comment, forgot the # +``` + +the error message would be like this. + +![error-message-1](https://github.com/ToruNiina/toml11/blob/misc/misc/toml11-err-msg-1.png) + +With the following, + +```toml +[error] +# array = [ +# "This might most likely happen in multiline arrays", +# Like here, +# "or here, +# and here" +# ] End of array comment, forgot the # +number = 3.14 pi <--again forgot the # +``` + +the error message would be like this. + +![error-message-2](https://github.com/ToruNiina/toml11/blob/misc/misc/toml11-err-msg-2.png) + +The message would be messy when it is written to a file, not a terminal because +it uses [ANSI escape code](https://en.wikipedia.org/wiki/ANSI_escape_code). + +Without `TOML11_COLORIZE_ERROR_MESSAGE`, you can still colorize user-defined +error message by passing `true` to the `toml::format_error` function. +If you define `TOML11_COLORIZE_ERROR_MESSAGE`, the value is `true` by default. +If not, the default value would be `false`. + +```cpp +std::cerr << toml::format_error("[error] value should be positive", + data.at("num"), "positive number required", + hints, /*colorize = */ true) << std::endl; +``` + +Note: It colorize `[error]` in red. That means that it detects `[error]` prefix +at the front of the error message. If there is no `[error]` prefix, +`format_error` adds it to the error message. + +## Serializing TOML data + +toml11 enables you to serialize data into toml format. + +```cpp +const toml::value data{{"foo", 42}, {"bar", "baz"}}; +std::cout << data << std::endl; +// bar = "baz" +// foo = 42 +``` + +toml11 automatically makes a small table and small array inline. +You can specify the width to make them inline by `std::setw` for streams. + +```cpp +const toml::value data{ + {"qux", {{"foo", 42}, {"bar", "baz"}}}, + {"quux", {"small", "array", "of", "strings"}}, + {"foobar", {"this", "array", "of", "strings", "is", "too", "long", + "to", "print", "into", "single", "line", "isn't", "it?"}}, +}; + +// the threshold becomes 80. +std::cout << std::setw(80) << data << std::endl; +// foobar = [ +// "this","array","of","strings","is","too","long","to","print","into", +// "single","line","isn't","it?", +// ] +// quux = ["small","array","of","strings"] +// qux = {bar="baz",foo=42} + + +// the width is 0. nothing become inline. +std::cout << std::setw(0) << data << std::endl; +// foobar = [ +// "this", +// ... (snip) +// "it?", +// ] +// quux = [ +// "small", +// "array", +// "of", +// "strings", +// ] +// [qux] +// bar = "baz" +// foo = 42 +``` + +It is recommended to set width before printing data. Some I/O functions changes +width to 0, and it makes all the stuff (including `toml::array`) multiline. +The resulting files becomes too long. + +To control the precision of floating point numbers, you need to pass +`std::setprecision` to stream. + +```cpp +const toml::value data{ + {"pi", 3.141592653589793}, + {"e", 2.718281828459045} +}; +std::cout << std::setprecision(17) << data << std::endl; +// e = 2.7182818284590451 +// pi = 3.1415926535897931 +std::cout << std::setprecision( 7) << data << std::endl; +// e = 2.718282 +// pi = 3.141593 +``` + +There is another way to format toml values, `toml::format()`. +It returns `std::string` that represents a value. + +```cpp +const toml::value v{{"a", 42}}; +const std::string fmt = toml::format(v); +// a = 42 +``` + +Note that since `toml::format` formats a value, the resulting string may lack +the key value. + +```cpp +const toml::value v{3.14}; +const std::string fmt = toml::format(v); +// 3.14 +``` + +To control the width and precision, `toml::format` receives optional second and +third arguments to set them. By default, the width is 80 and the precision is +`std::numeric_limits::max_digit10`. + +```cpp +const auto serial = toml::format(data, /*width = */ 0, /*prec = */ 17); +``` + +When you pass a comment-preserving-value, the comment will also be serialized. +An array or a table containing a value that has a comment would not be inlined. + +## Underlying types + +The toml types (can be used as `toml::*` in this library) and corresponding `enum` names are listed in the table below. + +| TOML type | underlying c++ type | enum class | +| -------------- | ---------------------------------- | -------------------------------- | +| Boolean | `bool` | `toml::value_t::boolean` | +| Integer | `std::int64_t` | `toml::value_t::integer` | +| Float | `double` | `toml::value_t::floating` | +| String | `toml::string` | `toml::value_t::string` | +| LocalDate | `toml::local_date` | `toml::value_t::local_date` | +| LocalTime | `toml::local_time` | `toml::value_t::local_time` | +| LocalDatetime | `toml::local_datetime` | `toml::value_t::local_datetime` | +| OffsetDatetime | `toml::offset_datetime` | `toml::value_t::offset_datetime` | +| Array | `array-like` | `toml::value_t::array` | +| Table | `map-like` | `toml::value_t::table` | + +`array-like` and `map-like` are the STL containers that works like a `std::vector` and +`std::unordered_map`, respectively. By default, `std::vector` and `std::unordered_map` +are used. See [Customizing containers](#customizing-containers) for detail. + +`toml::string` is effectively the same as `std::string` but has an additional +flag that represents a kind of a string, `string_t::basic` and `string_t::literal`. +Although `std::string` is not an exact toml type, still you can get a reference +that points to internal `std::string` by using `toml::get()` for convenience. +The most important difference between `std::string` and `toml::string` is that +`toml::string` will be formatted as a TOML string when outputted with `ostream`. +This feature is introduced to make it easy to write a custom serializer. + +`Datetime` variants are `struct` that are defined in this library. +Because `std::chrono::system_clock::time_point` is a __time point__, +not capable of representing a Local Time independent from a specific day. + +## Unreleased TOML features + +Since TOML v1.0.0-rc.1 has been released, those features are now activated by +default. We no longer need to define `TOML11_USE_UNRELEASED_FEATURES`. + +- Leading zeroes in exponent parts of floats are permitted. + - e.g. `1.0e+01`, `5e+05` + - [toml-lang/toml/PR/656](https://github.com/toml-lang/toml/pull/656) +- Allow raw tab characters in basic strings and multi-line basic strings. + - [toml-lang/toml/PR/627](https://github.com/toml-lang/toml/pull/627) +- Allow heterogeneous arrays + - [toml-lang/toml/PR/676](https://github.com/toml-lang/toml/pull/676) + +## Note about heterogeneous arrays + +Although `toml::parse` allows heterogeneous arrays, constructor of `toml::value` +does not. Here the reason is explained. + +```cpp +// this won't be compiled +toml::value v{ + "foo", 3.14, 42, {1,2,3,4,5}, {{"key", "value"}} +} +``` + +There is a workaround for this. By explicitly converting values into +`toml::value`, you can initialize `toml::value` with a heterogeneous array. +Also, you can first initialize a `toml::value` with an array and then +`push_back` into it. + +```cpp +// OK! +toml::value v{ + toml::value("foo"), toml::value(3.14), toml::value(42), + toml::value{1,2,3,4,5}, toml::value{{"key", "value"}} +} + +// OK! +toml::value v(toml::array{}); +v.push_back("foo"); +v.push_back(3.14); + +// OK! +toml::array a; +a.push_back("foo"); +a.push_back(3.14); +toml::value v(std::move(a)); +``` + +The reason why the first example is not allowed is the following. +Let's assume that you are initializing a `toml::value` with a table. + +```cpp + // # expecting TOML table. +toml::value v{ // [v] + {"answer", 42}, // answer = 42 + {"pi", 3.14}, // pi = 3.14 + {"foo", "bar"} // foo = "bar" +}; +``` + +This is indistinguishable from a (heterogeneous) TOML array definition. + +```toml +v = [ + ["answer", 42], + ["pi", 3.14], + ["foo", "bar"], +] +``` + +This means that the above C++ code makes constructor's overload resolution +ambiguous. So a constructor that allows both "table as an initializer-list" and +"heterogeneous array as an initializer-list" cannot be implemented. + +Thus, although it is painful, we need to explicitly cast values into +`toml::value` when you initialize heterogeneous array in a C++ code. + +```cpp +toml::value v{ + toml::value("foo"), toml::value(3.14), toml::value(42), + toml::value{1,2,3,4,5}, toml::value{{"key", "value"}} +}; +``` + +## Breaking Changes from v2 + +Although toml11 is relatively new library (it's three years old now), it had +some confusing and inconvenient user-interfaces because of historical reasons. + +Between v2 and v3, those interfaces are rearranged. + +- `toml::parse` now returns a `toml::value`, not `toml::table`. +- `toml::value` is now an alias of `toml::basic_value`. + - See [Customizing containers](#customizing-containers) for detail. +- The elements of `toml::value_t` are renamed as `snake_case`. + - See [Underlying types](#underlying-types) for detail. +- Supports for the CamelCaseNames are dropped. + - See [Underlying types](#underlying-types) for detail. +- `(is|as)_float` has been removed to make the function names consistent with others. + - Since `float` is a keyword, toml11 named a float type as `toml::floating`. + - Also a `value_t` corresponds to `toml::floating` is named `value_t::floating`. + - So `(is|as)_floating` is introduced and `is_float` has been removed. + - See [Casting a toml::value](#casting-a-tomlvalue) and [Checking value type](#checking-value-type) for detail. +- An overload of `toml::find` for `toml::table` has been dropped. Use `toml::value` version instead. + - Because type conversion between a table and a value causes ambiguity while overload resolution + - Since `toml::parse` now returns a `toml::value`, this feature becomes less important. + - Also because `toml::table` is a normal STL container, implementing utility function is easy. + - See [Finding a toml::value](#finding-a-toml-value) for detail. +- An overload of `operator<<` and `toml::format` for `toml::table`s are dropped. + - Use `toml::value` instead. + - See [Serializing TOML data](#serializing-toml-data) for detail. +- Interface around comments. + - See [Preserving Comments](#preserving-comments) for detail. +- An ancient `from_toml/into_toml` has been removed. Use arbitrary type conversion support. + - See [Conversion between toml value and arbitrary types](#conversion-between-toml-value-and-arbitrary-types) for detail. + +Such a big change will not happen in the coming years. + +## Running Tests + +After cloning this repository, run the following command (thank you @jwillikers +for automating test set fetching!). + +```sh +$ mkdir build +$ cd build +$ cmake .. -Dtoml11_BUILD_TEST=ON +$ make +$ make test +``` + +To run the language agnostic test suite, you need to compile +`tests/check_toml_test.cpp` and pass it to the tester. + +## Contributors + +I appreciate the help of the contributors who introduced the great feature to this library. + +- Guillaume Fraux (@Luthaf) + - Windows support and CI on Appvayor + - Intel Compiler support +- Quentin Khan (@xaxousis) + - Found & Fixed a bug around ODR + - Improved error messages for invalid keys to show the location where the parser fails +- Petr Beneš (@wbenny) + - Fixed warnings on MSVC +- Ivan Shynkarenka (@chronoxor) + - Fixed Visual Studio 2019 warnings +- @khoitd1997 + - Fixed warnings while type conversion +- @KerstinKeller + - Added installation script to CMake +- J.C. Moyer (@jcmoyer) + - Fixed an example code in the documentation +- Jt Freeman (@blockparty-sh) + - Fixed feature test macro around `localtime_s` + - Suppress warnings in Debug mode +- OGAWA Kenichi (@kenichiice) + - Suppress warnings on intel compiler +- Jordan Williams (@jwillikers) + - Fixed clang range-loop-analysis warnings + - Fixed feature test macro to suppress -Wundef + - Use cache variables in CMakeLists.txt + - Automate test set fetching, update and refactor CMakeLists.txt +- Scott McCaskill + - Parse 9 digits (nanoseconds) of fractional seconds in a `local_time` +- Shu Wang (@halfelf) + - fix "Finding a value in an array" example in README +- @maass-tv and @SeverinLeonhardt + - Fix MSVC warning C4866 +- OGAWA KenIchi (@kenichiice) + - Fix include path in README +- Mohammed Alyousef (@MoAlyousef) + - Made testing optional in CMake +- Ivan Shynkarenka (@chronoxor) + - Fix compilation error in `` with MinGW +- Alex Merry (@amerry) + - Add missing include files +- sneakypete81 (@sneakypete81) + - Fix typo in error message +- Oliver Kahrmann (@founderio) + - Fix missing filename in error message if parsed file is empty +- Karl Nilsson (@karl-nilsson) + - Fix many spelling errors +- ohdarling88 (@ohdarling) + - Fix a bug in a constructor of serializer +- estshorter (@estshorter) + - Fix MSVC warning C26478 +- Philip Top (@phlptp) + - Improve checking standard library feature availability check +- Louis Marascio (@marascio) + - Fix free-nonheap-object warning + + +## Licensing terms + +This product is licensed under the terms of the [MIT License](LICENSE). + +- Copyright (c) 2017-2021 Toru Niina + +All rights reserved. diff --git a/src/toml11/toml.hpp b/src/toml11/toml.hpp new file mode 100644 index 000000000..f34cfccca --- /dev/null +++ b/src/toml11/toml.hpp @@ -0,0 +1,46 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2017 Toru Niina + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#ifndef TOML_FOR_MODERN_CPP +#define TOML_FOR_MODERN_CPP + +#ifndef __cplusplus +# error "__cplusplus is not defined" +#endif + +#if __cplusplus < 201103L && _MSC_VER < 1900 +# error "toml11 requires C++11 or later." +#endif + +#define TOML11_VERSION_MAJOR 3 +#define TOML11_VERSION_MINOR 7 +#define TOML11_VERSION_PATCH 0 + +#include "toml/parser.hpp" +#include "toml/literal.hpp" +#include "toml/serializer.hpp" +#include "toml/get.hpp" +#include "toml/macros.hpp" + +#endif// TOML_FOR_MODERN_CPP diff --git a/src/toml11/toml/color.hpp b/src/toml11/toml/color.hpp new file mode 100644 index 000000000..4cb572cb0 --- /dev/null +++ b/src/toml11/toml/color.hpp @@ -0,0 +1,64 @@ +#ifndef TOML11_COLOR_HPP +#define TOML11_COLOR_HPP +#include +#include + +#ifdef TOML11_COLORIZE_ERROR_MESSAGE +#define TOML11_ERROR_MESSAGE_COLORIZED true +#else +#define TOML11_ERROR_MESSAGE_COLORIZED false +#endif + +namespace toml +{ + +// put ANSI escape sequence to ostream +namespace color_ansi +{ +namespace detail +{ +inline int colorize_index() +{ + static const int index = std::ios_base::xalloc(); + return index; +} +} // detail + +inline std::ostream& colorize(std::ostream& os) +{ + // by default, it is zero. + os.iword(detail::colorize_index()) = 1; + return os; +} +inline std::ostream& nocolorize(std::ostream& os) +{ + os.iword(detail::colorize_index()) = 0; + return os; +} +inline std::ostream& reset (std::ostream& os) +{if(os.iword(detail::colorize_index()) == 1) {os << "\033[00m";} return os;} +inline std::ostream& bold (std::ostream& os) +{if(os.iword(detail::colorize_index()) == 1) {os << "\033[01m";} return os;} +inline std::ostream& grey (std::ostream& os) +{if(os.iword(detail::colorize_index()) == 1) {os << "\033[30m";} return os;} +inline std::ostream& red (std::ostream& os) +{if(os.iword(detail::colorize_index()) == 1) {os << "\033[31m";} return os;} +inline std::ostream& green (std::ostream& os) +{if(os.iword(detail::colorize_index()) == 1) {os << "\033[32m";} return os;} +inline std::ostream& yellow (std::ostream& os) +{if(os.iword(detail::colorize_index()) == 1) {os << "\033[33m";} return os;} +inline std::ostream& blue (std::ostream& os) +{if(os.iword(detail::colorize_index()) == 1) {os << "\033[34m";} return os;} +inline std::ostream& magenta(std::ostream& os) +{if(os.iword(detail::colorize_index()) == 1) {os << "\033[35m";} return os;} +inline std::ostream& cyan (std::ostream& os) +{if(os.iword(detail::colorize_index()) == 1) {os << "\033[36m";} return os;} +inline std::ostream& white (std::ostream& os) +{if(os.iword(detail::colorize_index()) == 1) {os << "\033[37m";} return os;} +} // color_ansi + +// ANSI escape sequence is the only and default colorization method currently +namespace color = color_ansi; + +} // toml +#endif// TOML11_COLOR_HPP diff --git a/src/toml11/toml/combinator.hpp b/src/toml11/toml/combinator.hpp new file mode 100644 index 000000000..33ecca1eb --- /dev/null +++ b/src/toml11/toml/combinator.hpp @@ -0,0 +1,306 @@ +// Copyright Toru Niina 2017. +// Distributed under the MIT License. +#ifndef TOML11_COMBINATOR_HPP +#define TOML11_COMBINATOR_HPP +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "region.hpp" +#include "result.hpp" +#include "traits.hpp" +#include "utility.hpp" + +// they scans characters and returns region if it matches to the condition. +// when they fail, it does not change the location. +// in lexer.hpp, these are used. + +namespace toml +{ +namespace detail +{ + +// to output character as an error message. +inline std::string show_char(const char c) +{ + // It suppresses an error that occurs only in Debug mode of MSVC++ on Windows. + // I'm not completely sure but they check the value of char to be in the + // range [0, 256) and some of the COMPLETELY VALID utf-8 character sometimes + // has negative value (if char has sign). So here it re-interprets c as + // unsigned char through pointer. In general, converting pointer to a + // pointer that has different type cause UB, but `(signed|unsigned)?char` + // are one of the exceptions. Converting pointer only to char and std::byte + // (c++17) are valid. + if(std::isgraph(*reinterpret_cast(std::addressof(c)))) + { + return std::string(1, c); + } + else + { + std::array buf; + buf.fill('\0'); + const auto r = std::snprintf( + buf.data(), buf.size(), "0x%02x", static_cast(c) & 0xFF); + (void) r; // Unused variable warning + assert(r == static_cast(buf.size()) - 1); + return std::string(buf.data()); + } +} + +template +struct character +{ + static constexpr char target = C; + + static result + invoke(location& loc) + { + if(loc.iter() == loc.end()) {return none();} + const auto first = loc.iter(); + + const char c = *(loc.iter()); + if(c != target) + { + return none(); + } + loc.advance(); // update location + + return ok(region(loc, first, loc.iter())); + } +}; +template +constexpr char character::target; + +// closed interval [Low, Up]. both Low and Up are included. +template +struct in_range +{ + // assuming ascii part of UTF-8... + static_assert(Low <= Up, "lower bound should be less than upper bound."); + + static constexpr char upper = Up; + static constexpr char lower = Low; + + static result + invoke(location& loc) + { + if(loc.iter() == loc.end()) {return none();} + const auto first = loc.iter(); + + const char c = *(loc.iter()); + if(c < lower || upper < c) + { + return none(); + } + + loc.advance(); + return ok(region(loc, first, loc.iter())); + } +}; +template constexpr char in_range::upper; +template constexpr char in_range::lower; + +// keep iterator if `Combinator` matches. otherwise, increment `iter` by 1 char. +// for detecting invalid characters, like control sequences in toml string. +template +struct exclude +{ + static result + invoke(location& loc) + { + if(loc.iter() == loc.end()) {return none();} + auto first = loc.iter(); + + auto rslt = Combinator::invoke(loc); + if(rslt.is_ok()) + { + loc.reset(first); + return none(); + } + loc.reset(std::next(first)); // XXX maybe loc.advance() is okay but... + return ok(region(loc, first, loc.iter())); + } +}; + +// increment `iter`, if matches. otherwise, just return empty string. +template +struct maybe +{ + static result + invoke(location& loc) + { + const auto rslt = Combinator::invoke(loc); + if(rslt.is_ok()) + { + return rslt; + } + return ok(region(loc)); + } +}; + +template +struct sequence; + +template +struct sequence +{ + static result + invoke(location& loc) + { + const auto first = loc.iter(); + auto rslt = Head::invoke(loc); + if(rslt.is_err()) + { + loc.reset(first); + return none(); + } + return sequence::invoke(loc, std::move(rslt.unwrap()), first); + } + + // called from the above function only, recursively. + template + static result + invoke(location& loc, region reg, Iterator first) + { + const auto rslt = Head::invoke(loc); + if(rslt.is_err()) + { + loc.reset(first); + return none(); + } + reg += rslt.unwrap(); // concat regions + return sequence::invoke(loc, std::move(reg), first); + } +}; + +template +struct sequence +{ + // would be called from sequence::invoke only. + template + static result + invoke(location& loc, region reg, Iterator first) + { + const auto rslt = Head::invoke(loc); + if(rslt.is_err()) + { + loc.reset(first); + return none(); + } + reg += rslt.unwrap(); // concat regions + return ok(reg); + } +}; + +template +struct either; + +template +struct either +{ + static result + invoke(location& loc) + { + const auto rslt = Head::invoke(loc); + if(rslt.is_ok()) {return rslt;} + return either::invoke(loc); + } +}; +template +struct either +{ + static result + invoke(location& loc) + { + return Head::invoke(loc); + } +}; + +template +struct repeat; + +template struct exactly{}; +template struct at_least{}; +struct unlimited{}; + +template +struct repeat> +{ + static result + invoke(location& loc) + { + region retval(loc); + const auto first = loc.iter(); + for(std::size_t i=0; i +struct repeat> +{ + static result + invoke(location& loc) + { + region retval(loc); + + const auto first = loc.iter(); + for(std::size_t i=0; i +struct repeat +{ + static result + invoke(location& loc) + { + region retval(loc); + while(true) + { + auto rslt = T::invoke(loc); + if(rslt.is_err()) + { + return ok(std::move(retval)); + } + retval += rslt.unwrap(); + } + } +}; + +} // detail +} // toml +#endif// TOML11_COMBINATOR_HPP diff --git a/src/toml11/toml/comments.hpp b/src/toml11/toml/comments.hpp new file mode 100644 index 000000000..ec2504117 --- /dev/null +++ b/src/toml11/toml/comments.hpp @@ -0,0 +1,472 @@ +// Copyright Toru Niina 2019. +// Distributed under the MIT License. +#ifndef TOML11_COMMENTS_HPP +#define TOML11_COMMENTS_HPP +#include +#include +#include +#include +#include +#include +#include + +#ifdef TOML11_PRESERVE_COMMENTS_BY_DEFAULT +# define TOML11_DEFAULT_COMMENT_STRATEGY ::toml::preserve_comments +#else +# define TOML11_DEFAULT_COMMENT_STRATEGY ::toml::discard_comments +#endif + +// This file provides mainly two classes, `preserve_comments` and `discard_comments`. +// Those two are a container that have the same interface as `std::vector` +// but bahaves in the opposite way. `preserve_comments` is just the same as +// `std::vector` and each `std::string` corresponds to a comment line. +// Conversely, `discard_comments` discards all the strings and ignores everything +// assigned in it. `discard_comments` is always empty and you will encounter an +// error whenever you access to the element. +namespace toml +{ +struct discard_comments; // forward decl + +// use it in the following way +// +// const toml::basic_value data = +// toml::parse("example.toml"); +// +// the interface is almost the same as std::vector. +struct preserve_comments +{ + // `container_type` is not provided in discard_comments. + // do not use this inner-type in a generic code. + using container_type = std::vector; + + using size_type = container_type::size_type; + using difference_type = container_type::difference_type; + using value_type = container_type::value_type; + using reference = container_type::reference; + using const_reference = container_type::const_reference; + using pointer = container_type::pointer; + using const_pointer = container_type::const_pointer; + using iterator = container_type::iterator; + using const_iterator = container_type::const_iterator; + using reverse_iterator = container_type::reverse_iterator; + using const_reverse_iterator = container_type::const_reverse_iterator; + + preserve_comments() = default; + ~preserve_comments() = default; + preserve_comments(preserve_comments const&) = default; + preserve_comments(preserve_comments &&) = default; + preserve_comments& operator=(preserve_comments const&) = default; + preserve_comments& operator=(preserve_comments &&) = default; + + explicit preserve_comments(const std::vector& c): comments(c){} + explicit preserve_comments(std::vector&& c) + : comments(std::move(c)) + {} + preserve_comments& operator=(const std::vector& c) + { + comments = c; + return *this; + } + preserve_comments& operator=(std::vector&& c) + { + comments = std::move(c); + return *this; + } + + explicit preserve_comments(const discard_comments&) {} + + explicit preserve_comments(size_type n): comments(n) {} + preserve_comments(size_type n, const std::string& x): comments(n, x) {} + preserve_comments(std::initializer_list x): comments(x) {} + template + preserve_comments(InputIterator first, InputIterator last) + : comments(first, last) + {} + + template + void assign(InputIterator first, InputIterator last) {comments.assign(first, last);} + void assign(std::initializer_list ini) {comments.assign(ini);} + void assign(size_type n, const std::string& val) {comments.assign(n, val);} + + // Related to the issue #97. + // + // It is known that `std::vector::insert` and `std::vector::erase` in + // the standard library implementation included in GCC 4.8.5 takes + // `std::vector::iterator` instead of `std::vector::const_iterator`. + // Because of the const-correctness, we cannot convert a `const_iterator` to + // an `iterator`. It causes compilation error in GCC 4.8.5. +#if defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__) && !defined(__clang__) +# if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) <= 40805 +# define TOML11_WORKAROUND_GCC_4_8_X_STANDARD_LIBRARY_IMPLEMENTATION +# endif +#endif + +#ifdef TOML11_WORKAROUND_GCC_4_8_X_STANDARD_LIBRARY_IMPLEMENTATION + iterator insert(iterator p, const std::string& x) + { + return comments.insert(p, x); + } + iterator insert(iterator p, std::string&& x) + { + return comments.insert(p, std::move(x)); + } + void insert(iterator p, size_type n, const std::string& x) + { + return comments.insert(p, n, x); + } + template + void insert(iterator p, InputIterator first, InputIterator last) + { + return comments.insert(p, first, last); + } + void insert(iterator p, std::initializer_list ini) + { + return comments.insert(p, ini); + } + + template + iterator emplace(iterator p, Ts&& ... args) + { + return comments.emplace(p, std::forward(args)...); + } + + iterator erase(iterator pos) {return comments.erase(pos);} + iterator erase(iterator first, iterator last) + { + return comments.erase(first, last); + } +#else + iterator insert(const_iterator p, const std::string& x) + { + return comments.insert(p, x); + } + iterator insert(const_iterator p, std::string&& x) + { + return comments.insert(p, std::move(x)); + } + iterator insert(const_iterator p, size_type n, const std::string& x) + { + return comments.insert(p, n, x); + } + template + iterator insert(const_iterator p, InputIterator first, InputIterator last) + { + return comments.insert(p, first, last); + } + iterator insert(const_iterator p, std::initializer_list ini) + { + return comments.insert(p, ini); + } + + template + iterator emplace(const_iterator p, Ts&& ... args) + { + return comments.emplace(p, std::forward(args)...); + } + + iterator erase(const_iterator pos) {return comments.erase(pos);} + iterator erase(const_iterator first, const_iterator last) + { + return comments.erase(first, last); + } +#endif + + void swap(preserve_comments& other) {comments.swap(other.comments);} + + void push_back(const std::string& v) {comments.push_back(v);} + void push_back(std::string&& v) {comments.push_back(std::move(v));} + void pop_back() {comments.pop_back();} + + template + void emplace_back(Ts&& ... args) {comments.emplace_back(std::forward(args)...);} + + void clear() {comments.clear();} + + size_type size() const noexcept {return comments.size();} + size_type max_size() const noexcept {return comments.max_size();} + size_type capacity() const noexcept {return comments.capacity();} + bool empty() const noexcept {return comments.empty();} + + void reserve(size_type n) {comments.reserve(n);} + void resize(size_type n) {comments.resize(n);} + void resize(size_type n, const std::string& c) {comments.resize(n, c);} + void shrink_to_fit() {comments.shrink_to_fit();} + + reference operator[](const size_type n) noexcept {return comments[n];} + const_reference operator[](const size_type n) const noexcept {return comments[n];} + reference at(const size_type n) {return comments.at(n);} + const_reference at(const size_type n) const {return comments.at(n);} + reference front() noexcept {return comments.front();} + const_reference front() const noexcept {return comments.front();} + reference back() noexcept {return comments.back();} + const_reference back() const noexcept {return comments.back();} + + pointer data() noexcept {return comments.data();} + const_pointer data() const noexcept {return comments.data();} + + iterator begin() noexcept {return comments.begin();} + iterator end() noexcept {return comments.end();} + const_iterator begin() const noexcept {return comments.begin();} + const_iterator end() const noexcept {return comments.end();} + const_iterator cbegin() const noexcept {return comments.cbegin();} + const_iterator cend() const noexcept {return comments.cend();} + + reverse_iterator rbegin() noexcept {return comments.rbegin();} + reverse_iterator rend() noexcept {return comments.rend();} + const_reverse_iterator rbegin() const noexcept {return comments.rbegin();} + const_reverse_iterator rend() const noexcept {return comments.rend();} + const_reverse_iterator crbegin() const noexcept {return comments.crbegin();} + const_reverse_iterator crend() const noexcept {return comments.crend();} + + friend bool operator==(const preserve_comments&, const preserve_comments&); + friend bool operator!=(const preserve_comments&, const preserve_comments&); + friend bool operator< (const preserve_comments&, const preserve_comments&); + friend bool operator<=(const preserve_comments&, const preserve_comments&); + friend bool operator> (const preserve_comments&, const preserve_comments&); + friend bool operator>=(const preserve_comments&, const preserve_comments&); + + friend void swap(preserve_comments&, std::vector&); + friend void swap(std::vector&, preserve_comments&); + + private: + + container_type comments; +}; + +inline bool operator==(const preserve_comments& lhs, const preserve_comments& rhs) {return lhs.comments == rhs.comments;} +inline bool operator!=(const preserve_comments& lhs, const preserve_comments& rhs) {return lhs.comments != rhs.comments;} +inline bool operator< (const preserve_comments& lhs, const preserve_comments& rhs) {return lhs.comments < rhs.comments;} +inline bool operator<=(const preserve_comments& lhs, const preserve_comments& rhs) {return lhs.comments <= rhs.comments;} +inline bool operator> (const preserve_comments& lhs, const preserve_comments& rhs) {return lhs.comments > rhs.comments;} +inline bool operator>=(const preserve_comments& lhs, const preserve_comments& rhs) {return lhs.comments >= rhs.comments;} + +inline void swap(preserve_comments& lhs, preserve_comments& rhs) +{ + lhs.swap(rhs); + return; +} +inline void swap(preserve_comments& lhs, std::vector& rhs) +{ + lhs.comments.swap(rhs); + return; +} +inline void swap(std::vector& lhs, preserve_comments& rhs) +{ + lhs.swap(rhs.comments); + return; +} + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const preserve_comments& com) +{ + for(const auto& c : com) + { + os << '#' << c << '\n'; + } + return os; +} + +namespace detail +{ + +// To provide the same interface with `preserve_comments`, `discard_comments` +// should have an iterator. But it does not contain anything, so we need to +// add an iterator that points nothing. +// +// It always points null, so DO NOT unwrap this iterator. It always crashes +// your program. +template +struct empty_iterator +{ + using value_type = T; + using reference_type = typename std::conditional::type; + using pointer_type = typename std::conditional::type; + using difference_type = std::ptrdiff_t; + using iterator_category = std::random_access_iterator_tag; + + empty_iterator() = default; + ~empty_iterator() = default; + empty_iterator(empty_iterator const&) = default; + empty_iterator(empty_iterator &&) = default; + empty_iterator& operator=(empty_iterator const&) = default; + empty_iterator& operator=(empty_iterator &&) = default; + + // DO NOT call these operators. + reference_type operator*() const noexcept {std::terminate();} + pointer_type operator->() const noexcept {return nullptr;} + reference_type operator[](difference_type) const noexcept {return this->operator*();} + + // These operators do nothing. + empty_iterator& operator++() noexcept {return *this;} + empty_iterator operator++(int) noexcept {return *this;} + empty_iterator& operator--() noexcept {return *this;} + empty_iterator operator--(int) noexcept {return *this;} + + empty_iterator& operator+=(difference_type) noexcept {return *this;} + empty_iterator& operator-=(difference_type) noexcept {return *this;} + + empty_iterator operator+(difference_type) const noexcept {return *this;} + empty_iterator operator-(difference_type) const noexcept {return *this;} +}; + +template +bool operator==(const empty_iterator&, const empty_iterator&) noexcept {return true;} +template +bool operator!=(const empty_iterator&, const empty_iterator&) noexcept {return false;} +template +bool operator< (const empty_iterator&, const empty_iterator&) noexcept {return false;} +template +bool operator<=(const empty_iterator&, const empty_iterator&) noexcept {return true;} +template +bool operator> (const empty_iterator&, const empty_iterator&) noexcept {return false;} +template +bool operator>=(const empty_iterator&, const empty_iterator&) noexcept {return true;} + +template +typename empty_iterator::difference_type +operator-(const empty_iterator&, const empty_iterator&) noexcept {return 0;} + +template +empty_iterator +operator+(typename empty_iterator::difference_type, const empty_iterator& rhs) noexcept {return rhs;} +template +empty_iterator +operator+(const empty_iterator& lhs, typename empty_iterator::difference_type) noexcept {return lhs;} + +} // detail + +// The default comment type. It discards all the comments. It requires only one +// byte to contain, so the memory footprint is smaller than preserve_comments. +// +// It just ignores `push_back`, `insert`, `erase`, and any other modifications. +// IT always returns size() == 0, the iterator taken by `begin()` is always the +// same as that of `end()`, and accessing through `operator[]` or iterators +// always causes a segmentation fault. DO NOT access to the element of this. +// +// Why this is chose as the default type is because the last version (2.x.y) +// does not contain any comments in a value. To minimize the impact on the +// efficiency, this is chosen as a default. +// +// To reduce the memory footprint, later we can try empty base optimization (EBO). +struct discard_comments +{ + using size_type = std::size_t; + using difference_type = std::ptrdiff_t; + using value_type = std::string; + using reference = std::string&; + using const_reference = std::string const&; + using pointer = std::string*; + using const_pointer = std::string const*; + using iterator = detail::empty_iterator; + using const_iterator = detail::empty_iterator; + using reverse_iterator = detail::empty_iterator; + using const_reverse_iterator = detail::empty_iterator; + + discard_comments() = default; + ~discard_comments() = default; + discard_comments(discard_comments const&) = default; + discard_comments(discard_comments &&) = default; + discard_comments& operator=(discard_comments const&) = default; + discard_comments& operator=(discard_comments &&) = default; + + explicit discard_comments(const std::vector&) noexcept {} + explicit discard_comments(std::vector&&) noexcept {} + discard_comments& operator=(const std::vector&) noexcept {return *this;} + discard_comments& operator=(std::vector&&) noexcept {return *this;} + + explicit discard_comments(const preserve_comments&) noexcept {} + + explicit discard_comments(size_type) noexcept {} + discard_comments(size_type, const std::string&) noexcept {} + discard_comments(std::initializer_list) noexcept {} + template + discard_comments(InputIterator, InputIterator) noexcept {} + + template + void assign(InputIterator, InputIterator) noexcept {} + void assign(std::initializer_list) noexcept {} + void assign(size_type, const std::string&) noexcept {} + + iterator insert(const_iterator, const std::string&) {return iterator{};} + iterator insert(const_iterator, std::string&&) {return iterator{};} + iterator insert(const_iterator, size_type, const std::string&) {return iterator{};} + template + iterator insert(const_iterator, InputIterator, InputIterator) {return iterator{};} + iterator insert(const_iterator, std::initializer_list) {return iterator{};} + + template + iterator emplace(const_iterator, Ts&& ...) {return iterator{};} + iterator erase(const_iterator) {return iterator{};} + iterator erase(const_iterator, const_iterator) {return iterator{};} + + void swap(discard_comments&) {return;} + + void push_back(const std::string&) {return;} + void push_back(std::string&& ) {return;} + void pop_back() {return;} + + template + void emplace_back(Ts&& ...) {return;} + + void clear() {return;} + + size_type size() const noexcept {return 0;} + size_type max_size() const noexcept {return 0;} + size_type capacity() const noexcept {return 0;} + bool empty() const noexcept {return true;} + + void reserve(size_type) {return;} + void resize(size_type) {return;} + void resize(size_type, const std::string&) {return;} + void shrink_to_fit() {return;} + + // DO NOT access to the element of this container. This container is always + // empty, so accessing through operator[], front/back, data causes address + // error. + + reference operator[](const size_type) noexcept {return *data();} + const_reference operator[](const size_type) const noexcept {return *data();} + reference at(const size_type) {throw std::out_of_range("toml::discard_comment is always empty.");} + const_reference at(const size_type) const {throw std::out_of_range("toml::discard_comment is always empty.");} + reference front() noexcept {return *data();} + const_reference front() const noexcept {return *data();} + reference back() noexcept {return *data();} + const_reference back() const noexcept {return *data();} + + pointer data() noexcept {return nullptr;} + const_pointer data() const noexcept {return nullptr;} + + iterator begin() noexcept {return iterator{};} + iterator end() noexcept {return iterator{};} + const_iterator begin() const noexcept {return const_iterator{};} + const_iterator end() const noexcept {return const_iterator{};} + const_iterator cbegin() const noexcept {return const_iterator{};} + const_iterator cend() const noexcept {return const_iterator{};} + + reverse_iterator rbegin() noexcept {return iterator{};} + reverse_iterator rend() noexcept {return iterator{};} + const_reverse_iterator rbegin() const noexcept {return const_iterator{};} + const_reverse_iterator rend() const noexcept {return const_iterator{};} + const_reverse_iterator crbegin() const noexcept {return const_iterator{};} + const_reverse_iterator crend() const noexcept {return const_iterator{};} +}; + +inline bool operator==(const discard_comments&, const discard_comments&) noexcept {return true;} +inline bool operator!=(const discard_comments&, const discard_comments&) noexcept {return false;} +inline bool operator< (const discard_comments&, const discard_comments&) noexcept {return false;} +inline bool operator<=(const discard_comments&, const discard_comments&) noexcept {return true;} +inline bool operator> (const discard_comments&, const discard_comments&) noexcept {return false;} +inline bool operator>=(const discard_comments&, const discard_comments&) noexcept {return true;} + +inline void swap(const discard_comments&, const discard_comments&) noexcept {return;} + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const discard_comments&) +{ + return os; +} + +} // toml11 +#endif// TOML11_COMMENTS_HPP diff --git a/src/toml11/toml/datetime.hpp b/src/toml11/toml/datetime.hpp new file mode 100644 index 000000000..d8127c150 --- /dev/null +++ b/src/toml11/toml/datetime.hpp @@ -0,0 +1,631 @@ +// Copyright Toru Niina 2017. +// Distributed under the MIT License. +#ifndef TOML11_DATETIME_HPP +#define TOML11_DATETIME_HPP +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace toml +{ + +// To avoid non-threadsafe std::localtime. In C11 (not C++11!), localtime_s is +// provided in the absolutely same purpose, but C++11 is actually not compatible +// with C11. We need to dispatch the function depending on the OS. +namespace detail +{ +// TODO: find more sophisticated way to handle this +#if (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 1) || defined(_XOPEN_SOURCE) || defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || defined(_POSIX_SOURCE) +inline std::tm localtime_s(const std::time_t* src) +{ + std::tm dst; + const auto result = ::localtime_r(src, &dst); + if (!result) { throw std::runtime_error("localtime_r failed."); } + return dst; +} +inline std::tm gmtime_s(const std::time_t* src) +{ + std::tm dst; + const auto result = ::gmtime_r(src, &dst); + if (!result) { throw std::runtime_error("gmtime_r failed."); } + return dst; +} +#elif defined(_MSC_VER) +inline std::tm localtime_s(const std::time_t* src) +{ + std::tm dst; + const auto result = ::localtime_s(&dst, src); + if (result) { throw std::runtime_error("localtime_s failed."); } + return dst; +} +inline std::tm gmtime_s(const std::time_t* src) +{ + std::tm dst; + const auto result = ::gmtime_s(&dst, src); + if (result) { throw std::runtime_error("gmtime_s failed."); } + return dst; +} +#else // fallback. not threadsafe +inline std::tm localtime_s(const std::time_t* src) +{ + const auto result = std::localtime(src); + if (!result) { throw std::runtime_error("localtime failed."); } + return *result; +} +inline std::tm gmtime_s(const std::time_t* src) +{ + const auto result = std::gmtime(src); + if (!result) { throw std::runtime_error("gmtime failed."); } + return *result; +} +#endif +} // detail + +enum class month_t : std::uint8_t +{ + Jan = 0, + Feb = 1, + Mar = 2, + Apr = 3, + May = 4, + Jun = 5, + Jul = 6, + Aug = 7, + Sep = 8, + Oct = 9, + Nov = 10, + Dec = 11 +}; + +struct local_date +{ + std::int16_t year; // A.D. (like, 2018) + std::uint8_t month; // [0, 11] + std::uint8_t day; // [1, 31] + + local_date(int y, month_t m, int d) + : year (static_cast(y)), + month(static_cast(m)), + day (static_cast(d)) + {} + + explicit local_date(const std::tm& t) + : year (static_cast(t.tm_year + 1900)), + month(static_cast(t.tm_mon)), + day (static_cast(t.tm_mday)) + {} + + explicit local_date(const std::chrono::system_clock::time_point& tp) + { + const auto t = std::chrono::system_clock::to_time_t(tp); + const auto time = detail::localtime_s(&t); + *this = local_date(time); + } + + explicit local_date(const std::time_t t) + : local_date(std::chrono::system_clock::from_time_t(t)) + {} + + operator std::chrono::system_clock::time_point() const + { + // std::mktime returns date as local time zone. no conversion needed + std::tm t; + t.tm_sec = 0; + t.tm_min = 0; + t.tm_hour = 0; + t.tm_mday = static_cast(this->day); + t.tm_mon = static_cast(this->month); + t.tm_year = static_cast(this->year) - 1900; + t.tm_wday = 0; // the value will be ignored + t.tm_yday = 0; // the value will be ignored + t.tm_isdst = -1; + return std::chrono::system_clock::from_time_t(std::mktime(&t)); + } + + operator std::time_t() const + { + return std::chrono::system_clock::to_time_t( + std::chrono::system_clock::time_point(*this)); + } + + local_date() = default; + ~local_date() = default; + local_date(local_date const&) = default; + local_date(local_date&&) = default; + local_date& operator=(local_date const&) = default; + local_date& operator=(local_date&&) = default; +}; + +inline bool operator==(const local_date& lhs, const local_date& rhs) +{ + return std::make_tuple(lhs.year, lhs.month, lhs.day) == + std::make_tuple(rhs.year, rhs.month, rhs.day); +} +inline bool operator!=(const local_date& lhs, const local_date& rhs) +{ + return !(lhs == rhs); +} +inline bool operator< (const local_date& lhs, const local_date& rhs) +{ + return std::make_tuple(lhs.year, lhs.month, lhs.day) < + std::make_tuple(rhs.year, rhs.month, rhs.day); +} +inline bool operator<=(const local_date& lhs, const local_date& rhs) +{ + return (lhs < rhs) || (lhs == rhs); +} +inline bool operator> (const local_date& lhs, const local_date& rhs) +{ + return !(lhs <= rhs); +} +inline bool operator>=(const local_date& lhs, const local_date& rhs) +{ + return !(lhs < rhs); +} + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const local_date& date) +{ + os << std::setfill('0') << std::setw(4) << static_cast(date.year ) << '-'; + os << std::setfill('0') << std::setw(2) << static_cast(date.month) + 1 << '-'; + os << std::setfill('0') << std::setw(2) << static_cast(date.day ) ; + return os; +} + +struct local_time +{ + std::uint8_t hour; // [0, 23] + std::uint8_t minute; // [0, 59] + std::uint8_t second; // [0, 60] + std::uint16_t millisecond; // [0, 999] + std::uint16_t microsecond; // [0, 999] + std::uint16_t nanosecond; // [0, 999] + + local_time(int h, int m, int s, + int ms = 0, int us = 0, int ns = 0) + : hour (static_cast(h)), + minute(static_cast(m)), + second(static_cast(s)), + millisecond(static_cast(ms)), + microsecond(static_cast(us)), + nanosecond (static_cast(ns)) + {} + + explicit local_time(const std::tm& t) + : hour (static_cast(t.tm_hour)), + minute(static_cast(t.tm_min)), + second(static_cast(t.tm_sec)), + millisecond(0), microsecond(0), nanosecond(0) + {} + + template + explicit local_time(const std::chrono::duration& t) + { + const auto h = std::chrono::duration_cast(t); + this->hour = static_cast(h.count()); + const auto t2 = t - h; + const auto m = std::chrono::duration_cast(t2); + this->minute = static_cast(m.count()); + const auto t3 = t2 - m; + const auto s = std::chrono::duration_cast(t3); + this->second = static_cast(s.count()); + const auto t4 = t3 - s; + const auto ms = std::chrono::duration_cast(t4); + this->millisecond = static_cast(ms.count()); + const auto t5 = t4 - ms; + const auto us = std::chrono::duration_cast(t5); + this->microsecond = static_cast(us.count()); + const auto t6 = t5 - us; + const auto ns = std::chrono::duration_cast(t6); + this->nanosecond = static_cast(ns.count()); + } + + operator std::chrono::nanoseconds() const + { + return std::chrono::nanoseconds (this->nanosecond) + + std::chrono::microseconds(this->microsecond) + + std::chrono::milliseconds(this->millisecond) + + std::chrono::seconds(this->second) + + std::chrono::minutes(this->minute) + + std::chrono::hours(this->hour); + } + + local_time() = default; + ~local_time() = default; + local_time(local_time const&) = default; + local_time(local_time&&) = default; + local_time& operator=(local_time const&) = default; + local_time& operator=(local_time&&) = default; +}; + +inline bool operator==(const local_time& lhs, const local_time& rhs) +{ + return std::make_tuple(lhs.hour, lhs.minute, lhs.second, lhs.millisecond, lhs.microsecond, lhs.nanosecond) == + std::make_tuple(rhs.hour, rhs.minute, rhs.second, rhs.millisecond, rhs.microsecond, rhs.nanosecond); +} +inline bool operator!=(const local_time& lhs, const local_time& rhs) +{ + return !(lhs == rhs); +} +inline bool operator< (const local_time& lhs, const local_time& rhs) +{ + return std::make_tuple(lhs.hour, lhs.minute, lhs.second, lhs.millisecond, lhs.microsecond, lhs.nanosecond) < + std::make_tuple(rhs.hour, rhs.minute, rhs.second, rhs.millisecond, rhs.microsecond, rhs.nanosecond); +} +inline bool operator<=(const local_time& lhs, const local_time& rhs) +{ + return (lhs < rhs) || (lhs == rhs); +} +inline bool operator> (const local_time& lhs, const local_time& rhs) +{ + return !(lhs <= rhs); +} +inline bool operator>=(const local_time& lhs, const local_time& rhs) +{ + return !(lhs < rhs); +} + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const local_time& time) +{ + os << std::setfill('0') << std::setw(2) << static_cast(time.hour ) << ':'; + os << std::setfill('0') << std::setw(2) << static_cast(time.minute) << ':'; + os << std::setfill('0') << std::setw(2) << static_cast(time.second); + if(time.millisecond != 0 || time.microsecond != 0 || time.nanosecond != 0) + { + os << '.'; + os << std::setfill('0') << std::setw(3) << static_cast(time.millisecond); + if(time.microsecond != 0 || time.nanosecond != 0) + { + os << std::setfill('0') << std::setw(3) << static_cast(time.microsecond); + if(time.nanosecond != 0) + { + os << std::setfill('0') << std::setw(3) << static_cast(time.nanosecond); + } + } + } + return os; +} + +struct time_offset +{ + std::int8_t hour; // [-12, 12] + std::int8_t minute; // [-59, 59] + + time_offset(int h, int m) + : hour (static_cast(h)), + minute(static_cast(m)) + {} + + operator std::chrono::minutes() const + { + return std::chrono::minutes(this->minute) + + std::chrono::hours(this->hour); + } + + time_offset() = default; + ~time_offset() = default; + time_offset(time_offset const&) = default; + time_offset(time_offset&&) = default; + time_offset& operator=(time_offset const&) = default; + time_offset& operator=(time_offset&&) = default; +}; + +inline bool operator==(const time_offset& lhs, const time_offset& rhs) +{ + return std::make_tuple(lhs.hour, lhs.minute) == + std::make_tuple(rhs.hour, rhs.minute); +} +inline bool operator!=(const time_offset& lhs, const time_offset& rhs) +{ + return !(lhs == rhs); +} +inline bool operator< (const time_offset& lhs, const time_offset& rhs) +{ + return std::make_tuple(lhs.hour, lhs.minute) < + std::make_tuple(rhs.hour, rhs.minute); +} +inline bool operator<=(const time_offset& lhs, const time_offset& rhs) +{ + return (lhs < rhs) || (lhs == rhs); +} +inline bool operator> (const time_offset& lhs, const time_offset& rhs) +{ + return !(lhs <= rhs); +} +inline bool operator>=(const time_offset& lhs, const time_offset& rhs) +{ + return !(lhs < rhs); +} + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const time_offset& offset) +{ + if(offset.hour == 0 && offset.minute == 0) + { + os << 'Z'; + return os; + } + int minute = static_cast(offset.hour) * 60 + offset.minute; + if(minute < 0){os << '-'; minute = std::abs(minute);} else {os << '+';} + os << std::setfill('0') << std::setw(2) << minute / 60 << ':'; + os << std::setfill('0') << std::setw(2) << minute % 60; + return os; +} + +struct local_datetime +{ + local_date date; + local_time time; + + local_datetime(local_date d, local_time t): date(d), time(t) {} + + explicit local_datetime(const std::tm& t): date(t), time(t){} + + explicit local_datetime(const std::chrono::system_clock::time_point& tp) + { + const auto t = std::chrono::system_clock::to_time_t(tp); + std::tm ltime = detail::localtime_s(&t); + + this->date = local_date(ltime); + this->time = local_time(ltime); + + // std::tm lacks subsecond information, so diff between tp and tm + // can be used to get millisecond & microsecond information. + const auto t_diff = tp - + std::chrono::system_clock::from_time_t(std::mktime(<ime)); + this->time.millisecond = static_cast( + std::chrono::duration_cast(t_diff).count()); + this->time.microsecond = static_cast( + std::chrono::duration_cast(t_diff).count()); + this->time.nanosecond = static_cast( + std::chrono::duration_cast(t_diff).count()); + } + + explicit local_datetime(const std::time_t t) + : local_datetime(std::chrono::system_clock::from_time_t(t)) + {} + + operator std::chrono::system_clock::time_point() const + { + using internal_duration = + typename std::chrono::system_clock::time_point::duration; + + // Normally DST begins at A.M. 3 or 4. If we re-use conversion operator + // of local_date and local_time independently, the conversion fails if + // it is the day when DST begins or ends. Since local_date considers the + // time is 00:00 A.M. and local_time does not consider DST because it + // does not have any date information. We need to consider both date and + // time information at the same time to convert it correctly. + + std::tm t; + t.tm_sec = static_cast(this->time.second); + t.tm_min = static_cast(this->time.minute); + t.tm_hour = static_cast(this->time.hour); + t.tm_mday = static_cast(this->date.day); + t.tm_mon = static_cast(this->date.month); + t.tm_year = static_cast(this->date.year) - 1900; + t.tm_wday = 0; // the value will be ignored + t.tm_yday = 0; // the value will be ignored + t.tm_isdst = -1; + + // std::mktime returns date as local time zone. no conversion needed + auto dt = std::chrono::system_clock::from_time_t(std::mktime(&t)); + dt += std::chrono::duration_cast( + std::chrono::milliseconds(this->time.millisecond) + + std::chrono::microseconds(this->time.microsecond) + + std::chrono::nanoseconds (this->time.nanosecond)); + return dt; + } + + operator std::time_t() const + { + return std::chrono::system_clock::to_time_t( + std::chrono::system_clock::time_point(*this)); + } + + local_datetime() = default; + ~local_datetime() = default; + local_datetime(local_datetime const&) = default; + local_datetime(local_datetime&&) = default; + local_datetime& operator=(local_datetime const&) = default; + local_datetime& operator=(local_datetime&&) = default; +}; + +inline bool operator==(const local_datetime& lhs, const local_datetime& rhs) +{ + return std::make_tuple(lhs.date, lhs.time) == + std::make_tuple(rhs.date, rhs.time); +} +inline bool operator!=(const local_datetime& lhs, const local_datetime& rhs) +{ + return !(lhs == rhs); +} +inline bool operator< (const local_datetime& lhs, const local_datetime& rhs) +{ + return std::make_tuple(lhs.date, lhs.time) < + std::make_tuple(rhs.date, rhs.time); +} +inline bool operator<=(const local_datetime& lhs, const local_datetime& rhs) +{ + return (lhs < rhs) || (lhs == rhs); +} +inline bool operator> (const local_datetime& lhs, const local_datetime& rhs) +{ + return !(lhs <= rhs); +} +inline bool operator>=(const local_datetime& lhs, const local_datetime& rhs) +{ + return !(lhs < rhs); +} + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const local_datetime& dt) +{ + os << dt.date << 'T' << dt.time; + return os; +} + +struct offset_datetime +{ + local_date date; + local_time time; + time_offset offset; + + offset_datetime(local_date d, local_time t, time_offset o) + : date(d), time(t), offset(o) + {} + offset_datetime(const local_datetime& dt, time_offset o) + : date(dt.date), time(dt.time), offset(o) + {} + explicit offset_datetime(const local_datetime& ld) + : date(ld.date), time(ld.time), offset(get_local_offset(nullptr)) + // use the current local timezone offset + {} + explicit offset_datetime(const std::chrono::system_clock::time_point& tp) + : offset(0, 0) // use gmtime + { + const auto timet = std::chrono::system_clock::to_time_t(tp); + const auto tm = detail::gmtime_s(&timet); + this->date = local_date(tm); + this->time = local_time(tm); + } + explicit offset_datetime(const std::time_t& t) + : offset(0, 0) // use gmtime + { + const auto tm = detail::gmtime_s(&t); + this->date = local_date(tm); + this->time = local_time(tm); + } + explicit offset_datetime(const std::tm& t) + : offset(0, 0) // assume gmtime + { + this->date = local_date(t); + this->time = local_time(t); + } + + operator std::chrono::system_clock::time_point() const + { + // get date-time + using internal_duration = + typename std::chrono::system_clock::time_point::duration; + + // first, convert it to local date-time information in the same way as + // local_datetime does. later we will use time_t to adjust time offset. + std::tm t; + t.tm_sec = static_cast(this->time.second); + t.tm_min = static_cast(this->time.minute); + t.tm_hour = static_cast(this->time.hour); + t.tm_mday = static_cast(this->date.day); + t.tm_mon = static_cast(this->date.month); + t.tm_year = static_cast(this->date.year) - 1900; + t.tm_wday = 0; // the value will be ignored + t.tm_yday = 0; // the value will be ignored + t.tm_isdst = -1; + const std::time_t tp_loc = std::mktime(std::addressof(t)); + + auto tp = std::chrono::system_clock::from_time_t(tp_loc); + tp += std::chrono::duration_cast( + std::chrono::milliseconds(this->time.millisecond) + + std::chrono::microseconds(this->time.microsecond) + + std::chrono::nanoseconds (this->time.nanosecond)); + + // Since mktime uses local time zone, it should be corrected. + // `12:00:00+09:00` means `03:00:00Z`. So mktime returns `03:00:00Z` if + // we are in `+09:00` timezone. To represent `12:00:00Z` there, we need + // to add `+09:00` to `03:00:00Z`. + // Here, it uses the time_t converted from date-time info to handle + // daylight saving time. + const auto ofs = get_local_offset(std::addressof(tp_loc)); + tp += std::chrono::hours (ofs.hour); + tp += std::chrono::minutes(ofs.minute); + + // We got `12:00:00Z` by correcting local timezone applied by mktime. + // Then we will apply the offset. Let's say `12:00:00-08:00` is given. + // And now, we have `12:00:00Z`. `12:00:00-08:00` means `20:00:00Z`. + // So we need to subtract the offset. + tp -= std::chrono::minutes(this->offset); + return tp; + } + + operator std::time_t() const + { + return std::chrono::system_clock::to_time_t( + std::chrono::system_clock::time_point(*this)); + } + + offset_datetime() = default; + ~offset_datetime() = default; + offset_datetime(offset_datetime const&) = default; + offset_datetime(offset_datetime&&) = default; + offset_datetime& operator=(offset_datetime const&) = default; + offset_datetime& operator=(offset_datetime&&) = default; + + private: + + static time_offset get_local_offset(const std::time_t* tp) + { + // get local timezone with the same date-time information as mktime + const auto t = detail::localtime_s(tp); + + std::array buf; + const auto result = std::strftime(buf.data(), 6, "%z", &t); // +hhmm\0 + if(result != 5) + { + throw std::runtime_error("toml::offset_datetime: cannot obtain " + "timezone information of current env"); + } + const int ofs = std::atoi(buf.data()); + const int ofs_h = ofs / 100; + const int ofs_m = ofs - (ofs_h * 100); + return time_offset(ofs_h, ofs_m); + } +}; + +inline bool operator==(const offset_datetime& lhs, const offset_datetime& rhs) +{ + return std::make_tuple(lhs.date, lhs.time, lhs.offset) == + std::make_tuple(rhs.date, rhs.time, rhs.offset); +} +inline bool operator!=(const offset_datetime& lhs, const offset_datetime& rhs) +{ + return !(lhs == rhs); +} +inline bool operator< (const offset_datetime& lhs, const offset_datetime& rhs) +{ + return std::make_tuple(lhs.date, lhs.time, lhs.offset) < + std::make_tuple(rhs.date, rhs.time, rhs.offset); +} +inline bool operator<=(const offset_datetime& lhs, const offset_datetime& rhs) +{ + return (lhs < rhs) || (lhs == rhs); +} +inline bool operator> (const offset_datetime& lhs, const offset_datetime& rhs) +{ + return !(lhs <= rhs); +} +inline bool operator>=(const offset_datetime& lhs, const offset_datetime& rhs) +{ + return !(lhs < rhs); +} + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const offset_datetime& dt) +{ + os << dt.date << 'T' << dt.time << dt.offset; + return os; +} + +}//toml +#endif// TOML11_DATETIME diff --git a/src/toml11/toml/exception.hpp b/src/toml11/toml/exception.hpp new file mode 100644 index 000000000..c64651d0a --- /dev/null +++ b/src/toml11/toml/exception.hpp @@ -0,0 +1,65 @@ +// Copyright Toru Niina 2017. +// Distributed under the MIT License. +#ifndef TOML11_EXCEPTION_HPP +#define TOML11_EXCEPTION_HPP +#include +#include + +#include "source_location.hpp" + +namespace toml +{ + +struct exception : public std::exception +{ + public: + explicit exception(const source_location& loc): loc_(loc) {} + virtual ~exception() noexcept override = default; + virtual const char* what() const noexcept override {return "";} + virtual source_location const& location() const noexcept {return loc_;} + + protected: + source_location loc_; +}; + +struct syntax_error : public toml::exception +{ + public: + explicit syntax_error(const std::string& what_arg, const source_location& loc) + : exception(loc), what_(what_arg) + {} + virtual ~syntax_error() noexcept override = default; + virtual const char* what() const noexcept override {return what_.c_str();} + + protected: + std::string what_; +}; + +struct type_error : public toml::exception +{ + public: + explicit type_error(const std::string& what_arg, const source_location& loc) + : exception(loc), what_(what_arg) + {} + virtual ~type_error() noexcept override = default; + virtual const char* what() const noexcept override {return what_.c_str();} + + protected: + std::string what_; +}; + +struct internal_error : public toml::exception +{ + public: + explicit internal_error(const std::string& what_arg, const source_location& loc) + : exception(loc), what_(what_arg) + {} + virtual ~internal_error() noexcept override = default; + virtual const char* what() const noexcept override {return what_.c_str();} + + protected: + std::string what_; +}; + +} // toml +#endif // TOML_EXCEPTION diff --git a/src/toml11/toml/from.hpp b/src/toml11/toml/from.hpp new file mode 100644 index 000000000..10815caf5 --- /dev/null +++ b/src/toml11/toml/from.hpp @@ -0,0 +1,19 @@ +// Copyright Toru Niina 2019. +// Distributed under the MIT License. +#ifndef TOML11_FROM_HPP +#define TOML11_FROM_HPP + +namespace toml +{ + +template +struct from; +// { +// static T from_toml(const toml::value& v) +// { +// // User-defined conversions ... +// } +// }; + +} // toml +#endif // TOML11_FROM_HPP diff --git a/src/toml11/toml/get.hpp b/src/toml11/toml/get.hpp new file mode 100644 index 000000000..d7fdf553b --- /dev/null +++ b/src/toml11/toml/get.hpp @@ -0,0 +1,1117 @@ +// Copyright Toru Niina 2017. +// Distributed under the MIT License. +#ifndef TOML11_GET_HPP +#define TOML11_GET_HPP +#include + +#include "from.hpp" +#include "result.hpp" +#include "value.hpp" + +namespace toml +{ + +// ============================================================================ +// exact toml::* type + +template class M, template class V> +detail::enable_if_t>::value, T> & +get(basic_value& v) +{ + return v.template cast>::value>(); +} + +template class M, template class V> +detail::enable_if_t>::value, T> const& +get(const basic_value& v) +{ + return v.template cast>::value>(); +} + +template class M, template class V> +detail::enable_if_t>::value, T> +get(basic_value&& v) +{ + return T(std::move(v).template cast>::value>()); +} + +// ============================================================================ +// T == toml::value; identity transformation. + +template class M, template class V> +inline detail::enable_if_t>::value, T>& +get(basic_value& v) +{ + return v; +} + +template class M, template class V> +inline detail::enable_if_t>::value, T> const& +get(const basic_value& v) +{ + return v; +} + +template class M, template class V> +inline detail::enable_if_t>::value, T> +get(basic_value&& v) +{ + return basic_value(std::move(v)); +} + +// ============================================================================ +// T == toml::basic_value; basic_value -> basic_value + +template class M, template class V> +inline detail::enable_if_t, + detail::negation>> + >::value, T> +get(const basic_value& v) +{ + return T(v); +} + +// ============================================================================ +// integer convertible from toml::Integer + +template class M, template class V> +inline detail::enable_if_t, // T is integral + detail::negation>, // but not bool + detail::negation< // but not toml::integer + detail::is_exact_toml_type>> + >::value, T> +get(const basic_value& v) +{ + return static_cast(v.as_integer()); +} + +// ============================================================================ +// floating point convertible from toml::Float + +template class M, template class V> +inline detail::enable_if_t, // T is floating_point + detail::negation< // but not toml::floating + detail::is_exact_toml_type>> + >::value, T> +get(const basic_value& v) +{ + return static_cast(v.as_floating()); +} + +// ============================================================================ +// std::string; toml uses its own toml::string, but it should be convertible to +// std::string seamlessly + +template class M, template class V> +inline detail::enable_if_t::value, std::string>& +get(basic_value& v) +{ + return v.as_string().str; +} + +template class M, template class V> +inline detail::enable_if_t::value, std::string> const& +get(const basic_value& v) +{ + return v.as_string().str; +} + +template class M, template class V> +inline detail::enable_if_t::value, std::string> +get(basic_value&& v) +{ + return std::string(std::move(v.as_string().str)); +} + +// ============================================================================ +// std::string_view + +#if defined(TOML11_USING_STRING_VIEW) && TOML11_USING_STRING_VIEW>0 +template class M, template class V> +inline detail::enable_if_t::value, std::string_view> +get(const basic_value& v) +{ + return std::string_view(v.as_string().str); +} +#endif + +// ============================================================================ +// std::chrono::duration from toml::local_time. + +template class M, template class V> +inline detail::enable_if_t::value, T> +get(const basic_value& v) +{ + return std::chrono::duration_cast( + std::chrono::nanoseconds(v.as_local_time())); +} + +// ============================================================================ +// std::chrono::system_clock::time_point from toml::datetime variants + +template class M, template class V> +inline detail::enable_if_t< + std::is_same::value, T> +get(const basic_value& v) +{ + switch(v.type()) + { + case value_t::local_date: + { + return std::chrono::system_clock::time_point(v.as_local_date()); + } + case value_t::local_datetime: + { + return std::chrono::system_clock::time_point(v.as_local_datetime()); + } + case value_t::offset_datetime: + { + return std::chrono::system_clock::time_point(v.as_offset_datetime()); + } + default: + { + throw type_error(detail::format_underline("toml::value: " + "bad_cast to std::chrono::system_clock::time_point", { + {v.location(), concat_to_string("the actual type is ", v.type())} + }), v.location()); + } + } +} + +// ============================================================================ +// forward declaration to use this recursively. ignore this and go ahead. + +// array-like type with push_back(value) method +template class M, template class V> +detail::enable_if_t, // T is a container + detail::has_push_back_method, // T::push_back(value) works + detail::negation< // but not toml::array + detail::is_exact_toml_type>> + >::value, T> +get(const basic_value&); + +// array-like type without push_back(value) method +template class M, template class V> +detail::enable_if_t, // T is a container + detail::negation>, // w/o push_back(...) + detail::negation< // not toml::array + detail::is_exact_toml_type>> + >::value, T> +get(const basic_value&); + +// std::pair +template class M, template class V> +detail::enable_if_t::value, T> +get(const basic_value&); + +// std::tuple +template class M, template class V> +detail::enable_if_t::value, T> +get(const basic_value&); + +// map-like classes +template class M, template class V> +detail::enable_if_t, // T is map + detail::negation< // but not toml::table + detail::is_exact_toml_type>> + >::value, T> +get(const basic_value&); + +// T.from_toml(v) +template class M, template class V> +detail::enable_if_t>>, + detail::has_from_toml_method, // but has from_toml(toml::value) + std::is_default_constructible // and default constructible + >::value, T> +get(const basic_value&); + +// toml::from::from_toml(v) +template class M, template class V> +detail::enable_if_t::value, T> +get(const basic_value&); + +// T(const toml::value&) and T is not toml::basic_value, +// and it does not have `from` nor `from_toml`. +template class M, template class V> +detail::enable_if_t>, + std::is_constructible&>, + detail::negation>, + detail::negation> + >::value, T> +get(const basic_value&); + +// ============================================================================ +// array-like types; most likely STL container, like std::vector, etc. + +template class M, template class V> +detail::enable_if_t, // T is a container + detail::has_push_back_method, // container.push_back(elem) works + detail::negation< // but not toml::array + detail::is_exact_toml_type>> + >::value, T> +get(const basic_value& v) +{ + using value_type = typename T::value_type; + const auto& ary = v.as_array(); + + T container; + try_reserve(container, ary.size()); + + for(const auto& elem : ary) + { + container.push_back(get(elem)); + } + return container; +} + +// ============================================================================ +// std::forward_list does not have push_back, insert, or emplace. +// It has insert_after, emplace_after, push_front. + +template class M, template class V> +detail::enable_if_t::value, T> +get(const basic_value& v) +{ + using value_type = typename T::value_type; + T container; + for(const auto& elem : v.as_array()) + { + container.push_front(get(elem)); + } + container.reverse(); + return container; +} + +// ============================================================================ +// array-like types, without push_back(). most likely [std|boost]::array. + +template class M, template class V> +detail::enable_if_t, // T is a container + detail::negation>, // w/o push_back + detail::negation< // T is not toml::array + detail::is_exact_toml_type>> + >::value, T> +get(const basic_value& v) +{ + using value_type = typename T::value_type; + const auto& ar = v.as_array(); + + T container; + if(ar.size() != container.size()) + { + throw std::out_of_range(detail::format_underline(concat_to_string( + "toml::get: specified container size is ", container.size(), + " but there are ", ar.size(), " elements in toml array."), { + {v.location(), "here"} + })); + } + for(std::size_t i=0; i(ar[i]); + } + return container; +} + +// ============================================================================ +// std::pair. + +template class M, template class V> +detail::enable_if_t::value, T> +get(const basic_value& v) +{ + using first_type = typename T::first_type; + using second_type = typename T::second_type; + + const auto& ar = v.as_array(); + if(ar.size() != 2) + { + throw std::out_of_range(detail::format_underline(concat_to_string( + "toml::get: specified std::pair but there are ", ar.size(), + " elements in toml array."), {{v.location(), "here"}})); + } + return std::make_pair(::toml::get(ar.at(0)), + ::toml::get(ar.at(1))); +} + +// ============================================================================ +// std::tuple. + +namespace detail +{ +template +T get_tuple_impl(const Array& a, index_sequence) +{ + return std::make_tuple( + ::toml::get::type>(a.at(I))...); +} +} // detail + +template class M, template class V> +detail::enable_if_t::value, T> +get(const basic_value& v) +{ + const auto& ar = v.as_array(); + if(ar.size() != std::tuple_size::value) + { + throw std::out_of_range(detail::format_underline(concat_to_string( + "toml::get: specified std::tuple with ", + std::tuple_size::value, " elements, but there are ", ar.size(), + " elements in toml array."), {{v.location(), "here"}})); + } + return detail::get_tuple_impl(ar, + detail::make_index_sequence::value>{}); +} + +// ============================================================================ +// map-like types; most likely STL map, like std::map or std::unordered_map. + +template class M, template class V> +detail::enable_if_t, // T is map + detail::negation< // but not toml::array + detail::is_exact_toml_type>> + >::value, T> +get(const basic_value& v) +{ + using key_type = typename T::key_type; + using mapped_type = typename T::mapped_type; + static_assert(std::is_convertible::value, + "toml::get only supports map type of which key_type is " + "convertible from std::string."); + T map; + for(const auto& kv : v.as_table()) + { + map.emplace(key_type(kv.first), get(kv.second)); + } + return map; +} + +// ============================================================================ +// user-defined, but compatible types. + +template class M, template class V> +detail::enable_if_t>>, + detail::has_from_toml_method, // but has from_toml(toml::value) memfn + std::is_default_constructible // and default constructible + >::value, T> +get(const basic_value& v) +{ + T ud; + ud.from_toml(v); + return ud; +} +template class M, template class V> +detail::enable_if_t::value, T> +get(const basic_value& v) +{ + return ::toml::from::from_toml(v); +} + +template class M, template class V> +detail::enable_if_t>, // T is not a toml::value + std::is_constructible&>, // T is constructible from toml::value + detail::negation>, // and T does not have T.from_toml(v); + detail::negation> // and T does not have toml::from{}; + >::value, T> +get(const basic_value& v) +{ + return T(v); +} + +// ============================================================================ +// find + +// ---------------------------------------------------------------------------- +// these overloads do not require to set T. and returns value itself. +template class M, template class V> +basic_value const& find(const basic_value& v, const key& ky) +{ + const auto& tab = v.as_table(); + if(tab.count(ky) == 0) + { + detail::throw_key_not_found_error(v, ky); + } + return tab.at(ky); +} +template class M, template class V> +basic_value& find(basic_value& v, const key& ky) +{ + auto& tab = v.as_table(); + if(tab.count(ky) == 0) + { + detail::throw_key_not_found_error(v, ky); + } + return tab.at(ky); +} +template class M, template class V> +basic_value find(basic_value&& v, const key& ky) +{ + typename basic_value::table_type tab = std::move(v).as_table(); + if(tab.count(ky) == 0) + { + detail::throw_key_not_found_error(v, ky); + } + return basic_value(std::move(tab.at(ky))); +} + +// ---------------------------------------------------------------------------- +// find(value, idx) +template class M, template class V> +basic_value const& +find(const basic_value& v, const std::size_t idx) +{ + const auto& ary = v.as_array(); + if(ary.size() <= idx) + { + throw std::out_of_range(detail::format_underline(concat_to_string( + "index ", idx, " is out of range"), {{v.location(), "in this array"}})); + } + return ary.at(idx); +} +template class M, template class V> +basic_value& find(basic_value& v, const std::size_t idx) +{ + auto& ary = v.as_array(); + if(ary.size() <= idx) + { + throw std::out_of_range(detail::format_underline(concat_to_string( + "index ", idx, " is out of range"), {{v.location(), "in this array"}})); + } + return ary.at(idx); +} +template class M, template class V> +basic_value find(basic_value&& v, const std::size_t idx) +{ + auto& ary = v.as_array(); + if(ary.size() <= idx) + { + throw std::out_of_range(detail::format_underline(concat_to_string( + "index ", idx, " is out of range"), {{v.location(), "in this array"}})); + } + return basic_value(std::move(ary.at(idx))); +} + +// ---------------------------------------------------------------------------- +// find(value, key); + +template class M, template class V> +decltype(::toml::get(std::declval const&>())) +find(const basic_value& v, const key& ky) +{ + const auto& tab = v.as_table(); + if(tab.count(ky) == 0) + { + detail::throw_key_not_found_error(v, ky); + } + return ::toml::get(tab.at(ky)); +} + +template class M, template class V> +decltype(::toml::get(std::declval&>())) +find(basic_value& v, const key& ky) +{ + auto& tab = v.as_table(); + if(tab.count(ky) == 0) + { + detail::throw_key_not_found_error(v, ky); + } + return ::toml::get(tab.at(ky)); +} + +template class M, template class V> +decltype(::toml::get(std::declval&&>())) +find(basic_value&& v, const key& ky) +{ + typename basic_value::table_type tab = std::move(v).as_table(); + if(tab.count(ky) == 0) + { + detail::throw_key_not_found_error(v, ky); + } + return ::toml::get(std::move(tab.at(ky))); +} + +// ---------------------------------------------------------------------------- +// find(value, idx) +template class M, template class V> +decltype(::toml::get(std::declval const&>())) +find(const basic_value& v, const std::size_t idx) +{ + const auto& ary = v.as_array(); + if(ary.size() <= idx) + { + throw std::out_of_range(detail::format_underline(concat_to_string( + "index ", idx, " is out of range"), {{v.location(), "in this array"}})); + } + return ::toml::get(ary.at(idx)); +} +template class M, template class V> +decltype(::toml::get(std::declval&>())) +find(basic_value& v, const std::size_t idx) +{ + auto& ary = v.as_array(); + if(ary.size() <= idx) + { + throw std::out_of_range(detail::format_underline(concat_to_string( + "index ", idx, " is out of range"), {{v.location(), "in this array"}})); + } + return ::toml::get(ary.at(idx)); +} +template class M, template class V> +decltype(::toml::get(std::declval&&>())) +find(basic_value&& v, const std::size_t idx) +{ + typename basic_value::array_type ary = std::move(v).as_array(); + if(ary.size() <= idx) + { + throw std::out_of_range(detail::format_underline(concat_to_string( + "index ", idx, " is out of range"), {{v.location(), "in this array"}})); + } + return ::toml::get(std::move(ary.at(idx))); +} + +// -------------------------------------------------------------------------- +// toml::find(toml::value, toml::key, Ts&& ... keys) + +namespace detail +{ +// It suppresses warnings by -Wsign-conversion. Let's say we have the following +// code. +// ```cpp +// const auto x = toml::find(data, "array", 0); +// ``` +// Here, the type of literal number `0` is `int`. `int` is a signed integer. +// `toml::find` takes `std::size_t` as an index. So it causes implicit sign +// conversion and `-Wsign-conversion` warns about it. Using `0u` instead of `0` +// suppresses the warning, but it makes user code messy. +// To suppress this warning, we need to be aware of type conversion caused +// by `toml::find(v, key1, key2, ... keys)`. But the thing is that the types of +// keys can be any combination of {string-like, size_t-like}. Of course we can't +// write down all the combinations. Thus we need to use some function that +// recognize the type of argument and cast it into `std::string` or +// `std::size_t` depending on the context. +// `key_cast` does the job. It has 2 overloads. One is invoked when the +// argument type is an integer and cast the argument into `std::size_t`. The +// other is invoked when the argument type is not an integer, possibly one of +// std::string, const char[N] or const char*, and construct std::string from +// the argument. +// `toml::find(v, k1, k2, ... ks)` uses `key_cast` before passing `ks` to +// `toml::find(v, k)` to suppress -Wsign-conversion. + +template +enable_if_t>, + negation, bool>>>::value, std::size_t> +key_cast(T&& v) noexcept +{ + return std::size_t(v); +} +template +enable_if_t>, + negation, bool>>>>::value, std::string> +key_cast(T&& v) noexcept +{ + return std::string(std::forward(v)); +} +} // detail + +template class M, template class V, + typename Key1, typename Key2, typename ... Keys> +const basic_value& +find(const basic_value& v, Key1&& k1, Key2&& k2, Keys&& ... keys) +{ + return ::toml::find(::toml::find(v, detail::key_cast(k1)), + detail::key_cast(k2), std::forward(keys)...); +} +template class M, template class V, + typename Key1, typename Key2, typename ... Keys> +basic_value& +find(basic_value& v, Key1&& k1, Key2&& k2, Keys&& ... keys) +{ + return ::toml::find(::toml::find(v, detail::key_cast(k1)), + detail::key_cast(k2), std::forward(keys)...); +} +template class M, template class V, + typename Key1, typename Key2, typename ... Keys> +basic_value +find(basic_value&& v, Key1&& k1, Key2&& k2, Keys&& ... keys) +{ + return ::toml::find(::toml::find(std::move(v), std::forward(k1)), + detail::key_cast(k2), std::forward(keys)...); +} + +template class M, template class V, + typename Key1, typename Key2, typename ... Keys> +decltype(::toml::get(std::declval&>())) +find(const basic_value& v, Key1&& k1, Key2&& k2, Keys&& ... keys) +{ + return ::toml::find(::toml::find(v, detail::key_cast(k1)), + detail::key_cast(k2), std::forward(keys)...); +} +template class M, template class V, + typename Key1, typename Key2, typename ... Keys> +decltype(::toml::get(std::declval&>())) +find(basic_value& v, Key1&& k1, Key2&& k2, Keys&& ... keys) +{ + return ::toml::find(::toml::find(v, detail::key_cast(k1)), + detail::key_cast(k2), std::forward(keys)...); +} +template class M, template class V, + typename Key1, typename Key2, typename ... Keys> +decltype(::toml::get(std::declval&&>())) +find(basic_value&& v, Key1&& k1, Key2&& k2, Keys&& ... keys) +{ + return ::toml::find(::toml::find(std::move(v), detail::key_cast(k1)), + detail::key_cast(k2), std::forward(keys)...); +} + +// ============================================================================ +// get_or(value, fallback) + +template class M, template class V> +basic_value const& +get_or(const basic_value& v, const basic_value&) +{ + return v; +} +template class M, template class V> +basic_value& +get_or(basic_value& v, basic_value&) +{ + return v; +} +template class M, template class V> +basic_value +get_or(basic_value&& v, basic_value&&) +{ + return v; +} + +// ---------------------------------------------------------------------------- +// specialization for the exact toml types (return type becomes lvalue ref) + +template class M, template class V> +detail::enable_if_t< + detail::is_exact_toml_type>::value, T> const& +get_or(const basic_value& v, const T& opt) +{ + try + { + return get>(v); + } + catch(...) + { + return opt; + } +} +template class M, template class V> +detail::enable_if_t< + detail::is_exact_toml_type>::value, T>& +get_or(basic_value& v, T& opt) +{ + try + { + return get>(v); + } + catch(...) + { + return opt; + } +} +template class M, template class V> +detail::enable_if_t, + basic_value>::value, detail::remove_cvref_t> +get_or(basic_value&& v, T&& opt) +{ + try + { + return get>(std::move(v)); + } + catch(...) + { + return detail::remove_cvref_t(std::forward(opt)); + } +} + +// ---------------------------------------------------------------------------- +// specialization for std::string (return type becomes lvalue ref) + +template class M, template class V> +detail::enable_if_t, std::string>::value, + std::string> const& +get_or(const basic_value& v, const T& opt) +{ + try + { + return v.as_string().str; + } + catch(...) + { + return opt; + } +} +template class M, template class V> +detail::enable_if_t::value, std::string>& +get_or(basic_value& v, T& opt) +{ + try + { + return v.as_string().str; + } + catch(...) + { + return opt; + } +} +template class M, template class V> +detail::enable_if_t< + std::is_same, std::string>::value, std::string> +get_or(basic_value&& v, T&& opt) +{ + try + { + return std::move(v.as_string().str); + } + catch(...) + { + return std::string(std::forward(opt)); + } +} + +// ---------------------------------------------------------------------------- +// specialization for string literal + +template class M, template class V> +detail::enable_if_t::type>::value, std::string> +get_or(const basic_value& v, T&& opt) +{ + try + { + return std::move(v.as_string().str); + } + catch(...) + { + return std::string(std::forward(opt)); + } +} + +// ---------------------------------------------------------------------------- +// others (require type conversion and return type cannot be lvalue reference) + +template class M, template class V> +detail::enable_if_t, + basic_value>>, + detail::negation>>, + detail::negation::type>> + >::value, detail::remove_cvref_t> +get_or(const basic_value& v, T&& opt) +{ + try + { + return get>(v); + } + catch(...) + { + return detail::remove_cvref_t(std::forward(opt)); + } +} + +// =========================================================================== +// find_or(value, key, fallback) + +template class M, template class V> +basic_value const& +find_or(const basic_value& v, const key& ky, + const basic_value& opt) +{ + if(!v.is_table()) {return opt;} + const auto& tab = v.as_table(); + if(tab.count(ky) == 0) {return opt;} + return tab.at(ky); +} + +template class M, template class V> +basic_value& +find_or(basic_value& v, const toml::key& ky, basic_value& opt) +{ + if(!v.is_table()) {return opt;} + auto& tab = v.as_table(); + if(tab.count(ky) == 0) {return opt;} + return tab.at(ky); +} + +template class M, template class V> +basic_value +find_or(basic_value&& v, const toml::key& ky, basic_value&& opt) +{ + if(!v.is_table()) {return opt;} + auto tab = std::move(v).as_table(); + if(tab.count(ky) == 0) {return opt;} + return basic_value(std::move(tab.at(ky))); +} + +// --------------------------------------------------------------------------- +// exact types (return type can be a reference) +template class M, template class V> +detail::enable_if_t< + detail::is_exact_toml_type>::value, T> const& +find_or(const basic_value& v, const key& ky, const T& opt) +{ + if(!v.is_table()) {return opt;} + const auto& tab = v.as_table(); + if(tab.count(ky) == 0) {return opt;} + return get_or(tab.at(ky), opt); +} + +template class M, template class V> +detail::enable_if_t< + detail::is_exact_toml_type>::value, T>& +find_or(basic_value& v, const toml::key& ky, T& opt) +{ + if(!v.is_table()) {return opt;} + auto& tab = v.as_table(); + if(tab.count(ky) == 0) {return opt;} + return get_or(tab.at(ky), opt); +} + +template class M, template class V> +detail::enable_if_t< + detail::is_exact_toml_type>::value, + detail::remove_cvref_t> +find_or(basic_value&& v, const toml::key& ky, T&& opt) +{ + if(!v.is_table()) {return std::forward(opt);} + auto tab = std::move(v).as_table(); + if(tab.count(ky) == 0) {return std::forward(opt);} + return get_or(std::move(tab.at(ky)), std::forward(opt)); +} + +// --------------------------------------------------------------------------- +// std::string (return type can be a reference) + +template class M, template class V> +detail::enable_if_t::value, std::string> const& +find_or(const basic_value& v, const key& ky, const T& opt) +{ + if(!v.is_table()) {return opt;} + const auto& tab = v.as_table(); + if(tab.count(ky) == 0) {return opt;} + return get_or(tab.at(ky), opt); +} +template class M, template class V> +detail::enable_if_t::value, std::string>& +find_or(basic_value& v, const toml::key& ky, T& opt) +{ + if(!v.is_table()) {return opt;} + auto& tab = v.as_table(); + if(tab.count(ky) == 0) {return opt;} + return get_or(tab.at(ky), opt); +} +template class M, template class V> +detail::enable_if_t::value, std::string> +find_or(basic_value&& v, const toml::key& ky, T&& opt) +{ + if(!v.is_table()) {return std::forward(opt);} + auto tab = std::move(v).as_table(); + if(tab.count(ky) == 0) {return std::forward(opt);} + return get_or(std::move(tab.at(ky)), std::forward(opt)); +} + +// --------------------------------------------------------------------------- +// string literal (deduced as std::string) +template class M, template class V> +detail::enable_if_t< + detail::is_string_literal::type>::value, + std::string> +find_or(const basic_value& v, const toml::key& ky, T&& opt) +{ + if(!v.is_table()) {return std::string(opt);} + const auto& tab = v.as_table(); + if(tab.count(ky) == 0) {return std::string(opt);} + return get_or(tab.at(ky), std::forward(opt)); +} + +// --------------------------------------------------------------------------- +// others (require type conversion and return type cannot be lvalue reference) +template class M, template class V> +detail::enable_if_t, basic_value>>, + // T is not std::string + detail::negation>>, + // T is not a string literal + detail::negation::type>> + >::value, detail::remove_cvref_t> +find_or(const basic_value& v, const toml::key& ky, T&& opt) +{ + if(!v.is_table()) {return std::forward(opt);} + const auto& tab = v.as_table(); + if(tab.count(ky) == 0) {return std::forward(opt);} + return get_or(tab.at(ky), std::forward(opt)); +} + +// --------------------------------------------------------------------------- +// recursive find-or with type deduction (find_or(value, keys, opt)) + +template 1), std::nullptr_t> = nullptr> + // here we need to add SFINAE in the template parameter to avoid + // infinite recursion in type deduction on gcc +auto find_or(Value&& v, const toml::key& ky, Ks&& ... keys) + -> decltype(find_or(std::forward(v), ky, detail::last_one(std::forward(keys)...))) +{ + if(!v.is_table()) + { + return detail::last_one(std::forward(keys)...); + } + auto&& tab = std::forward(v).as_table(); + if(tab.count(ky) == 0) + { + return detail::last_one(std::forward(keys)...); + } + return find_or(std::forward(tab).at(ky), std::forward(keys)...); +} + +// --------------------------------------------------------------------------- +// recursive find_or with explicit type specialization, find_or(value, keys...) + +template 1), std::nullptr_t> = nullptr> + // here we need to add SFINAE in the template parameter to avoid + // infinite recursion in type deduction on gcc +auto find_or(Value&& v, const toml::key& ky, Ks&& ... keys) + -> decltype(find_or(std::forward(v), ky, detail::last_one(std::forward(keys)...))) +{ + if(!v.is_table()) + { + return detail::last_one(std::forward(keys)...); + } + auto&& tab = std::forward(v).as_table(); + if(tab.count(ky) == 0) + { + return detail::last_one(std::forward(keys)...); + } + return find_or(std::forward(tab).at(ky), std::forward(keys)...); +} + +// ============================================================================ +// expect + +template class M, template class V> +result expect(const basic_value& v) noexcept +{ + try + { + return ok(get(v)); + } + catch(const std::exception& e) + { + return err(e.what()); + } +} +template class M, template class V> +result +expect(const basic_value& v, const toml::key& k) noexcept +{ + try + { + return ok(find(v, k)); + } + catch(const std::exception& e) + { + return err(e.what()); + } +} + +} // toml +#endif// TOML11_GET diff --git a/src/toml11/toml/into.hpp b/src/toml11/toml/into.hpp new file mode 100644 index 000000000..74495560e --- /dev/null +++ b/src/toml11/toml/into.hpp @@ -0,0 +1,19 @@ +// Copyright Toru Niina 2019. +// Distributed under the MIT License. +#ifndef TOML11_INTO_HPP +#define TOML11_INTO_HPP + +namespace toml +{ + +template +struct into; +// { +// static toml::value into_toml(const T& user_defined_type) +// { +// // User-defined conversions ... +// } +// }; + +} // toml +#endif // TOML11_INTO_HPP diff --git a/src/toml11/toml/lexer.hpp b/src/toml11/toml/lexer.hpp new file mode 100644 index 000000000..ea5050b8d --- /dev/null +++ b/src/toml11/toml/lexer.hpp @@ -0,0 +1,293 @@ +// Copyright Toru Niina 2017. +// Distributed under the MIT License. +#ifndef TOML11_LEXER_HPP +#define TOML11_LEXER_HPP +#include +#include +#include +#include + +#include "combinator.hpp" + +namespace toml +{ +namespace detail +{ + +// these scans contents from current location in a container of char +// and extract a region that matches their own pattern. +// to see the implementation of each component, see combinator.hpp. + +using lex_wschar = either, character<'\t'>>; +using lex_ws = repeat>; +using lex_newline = either, + sequence, character<'\n'>>>; +using lex_lower = in_range<'a', 'z'>; +using lex_upper = in_range<'A', 'Z'>; +using lex_alpha = either; +using lex_digit = in_range<'0', '9'>; +using lex_nonzero = in_range<'1', '9'>; +using lex_oct_dig = in_range<'0', '7'>; +using lex_bin_dig = in_range<'0', '1'>; +using lex_hex_dig = either, in_range<'a', 'f'>>; + +using lex_hex_prefix = sequence, character<'x'>>; +using lex_oct_prefix = sequence, character<'o'>>; +using lex_bin_prefix = sequence, character<'b'>>; +using lex_underscore = character<'_'>; +using lex_plus = character<'+'>; +using lex_minus = character<'-'>; +using lex_sign = either; + +// digit | nonzero 1*(digit | _ digit) +using lex_unsigned_dec_int = either>, at_least<1>>>, + lex_digit>; +// (+|-)? unsigned_dec_int +using lex_dec_int = sequence, lex_unsigned_dec_int>; + +// hex_prefix hex_dig *(hex_dig | _ hex_dig) +using lex_hex_int = sequence>, unlimited>>>; +// oct_prefix oct_dig *(oct_dig | _ oct_dig) +using lex_oct_int = sequence>, unlimited>>>; +// bin_prefix bin_dig *(bin_dig | _ bin_dig) +using lex_bin_int = sequence>, unlimited>>>; + +// (dec_int | hex_int | oct_int | bin_int) +using lex_integer = either; + +// =========================================================================== + +using lex_inf = sequence, character<'n'>, character<'f'>>; +using lex_nan = sequence, character<'a'>, character<'n'>>; +using lex_special_float = sequence, either>; + +using lex_zero_prefixable_int = sequence>, unlimited>>; + +using lex_fractional_part = sequence, lex_zero_prefixable_int>; + +using lex_exponent_part = sequence, character<'E'>>, + maybe, lex_zero_prefixable_int>; + +using lex_float = either>>>>; + +// =========================================================================== + +using lex_true = sequence, character<'r'>, + character<'u'>, character<'e'>>; +using lex_false = sequence, character<'a'>, character<'l'>, + character<'s'>, character<'e'>>; +using lex_boolean = either; + +// =========================================================================== + +using lex_date_fullyear = repeat>; +using lex_date_month = repeat>; +using lex_date_mday = repeat>; +using lex_time_delim = either, character<'t'>, character<' '>>; +using lex_time_hour = repeat>; +using lex_time_minute = repeat>; +using lex_time_second = repeat>; +using lex_time_secfrac = sequence, + repeat>>; + +using lex_time_numoffset = sequence, character<'-'>>, + sequence, + lex_time_minute>>; +using lex_time_offset = either, character<'z'>, + lex_time_numoffset>; + +using lex_partial_time = sequence, + lex_time_minute, character<':'>, + lex_time_second, maybe>; +using lex_full_date = sequence, + lex_date_month, character<'-'>, + lex_date_mday>; +using lex_full_time = sequence; + +using lex_offset_date_time = sequence; +using lex_local_date_time = sequence; +using lex_local_date = lex_full_date; +using lex_local_time = lex_partial_time; + +// =========================================================================== + +using lex_quotation_mark = character<'"'>; +using lex_basic_unescaped = exclude, // 0x09 (tab) is allowed + in_range<0x0A, 0x1F>, + character<0x22>, character<0x5C>, + character<0x7F>>>; + +using lex_escape = character<'\\'>; +using lex_escape_unicode_short = sequence, + repeat>>; +using lex_escape_unicode_long = sequence, + repeat>>; +using lex_escape_seq_char = either, character<'\\'>, + character<'b'>, character<'f'>, + character<'n'>, character<'r'>, + character<'t'>, + lex_escape_unicode_short, + lex_escape_unicode_long + >; +using lex_escaped = sequence; +using lex_basic_char = either; +using lex_basic_string = sequence, + lex_quotation_mark>; + +// After toml post-v0.5.0, it is explicitly clarified how quotes in ml-strings +// are allowed to be used. +// After this, the following strings are *explicitly* allowed. +// - One or two `"`s in a multi-line basic string is allowed wherever it is. +// - Three consecutive `"`s in a multi-line basic string is considered as a delimiter. +// - One or two `"`s can appear just before or after the delimiter. +// ```toml +// str4 = """Here are two quotation marks: "". Simple enough.""" +// str5 = """Here are three quotation marks: ""\".""" +// str6 = """Here are fifteen quotation marks: ""\"""\"""\"""\"""\".""" +// str7 = """"This," she said, "is just a pointless statement."""" +// ``` +// In the current implementation (v3.3.0), it is difficult to parse `str7` in +// the above example. It is difficult to recognize `"` at the end of string body +// collectly. It will be misunderstood as a `"""` delimiter and an additional, +// invalid `"`. Like this: +// ```console +// what(): [error] toml::parse_table: invalid line format +// --> hoge.toml +// | +// 13 | str7 = """"This," she said, "is just a pointless statement."""" +// | ^- expected newline, but got '"'. +// ``` +// As a quick workaround for this problem, `lex_ml_basic_string_delim` was +// split into two, `lex_ml_basic_string_open` and `lex_ml_basic_string_close`. +// `lex_ml_basic_string_open` allows only `"""`. `_close` allows 3-5 `"`s. +// In parse_ml_basic_string() function, the trailing `"`s will be attached to +// the string body. +// +using lex_ml_basic_string_delim = repeat>; +using lex_ml_basic_string_open = lex_ml_basic_string_delim; +using lex_ml_basic_string_close = sequence< + repeat>, + maybe, maybe + >; + +using lex_ml_basic_unescaped = exclude, // 0x09 is tab + in_range<0x0A, 0x1F>, + character<0x5C>, // backslash + character<0x7F>, // DEL + lex_ml_basic_string_delim>>; + +using lex_ml_basic_escaped_newline = sequence< + lex_escape, maybe, lex_newline, + repeat, unlimited>>; + +using lex_ml_basic_char = either; +using lex_ml_basic_body = repeat, + unlimited>; +using lex_ml_basic_string = sequence; + +using lex_literal_char = exclude, in_range<0x0A, 0x1F>, + character<0x7F>, character<0x27>>>; +using lex_apostrophe = character<'\''>; +using lex_literal_string = sequence, + lex_apostrophe>; + +// the same reason as above. +using lex_ml_literal_string_delim = repeat>; +using lex_ml_literal_string_open = lex_ml_literal_string_delim; +using lex_ml_literal_string_close = sequence< + repeat>, + maybe, maybe + >; + +using lex_ml_literal_char = exclude, + in_range<0x0A, 0x1F>, + character<0x7F>, + lex_ml_literal_string_delim>>; +using lex_ml_literal_body = repeat, + unlimited>; +using lex_ml_literal_string = sequence; + +using lex_string = either; + +// =========================================================================== +using lex_dot_sep = sequence, character<'.'>, maybe>; + +using lex_unquoted_key = repeat, character<'_'>>, + at_least<1>>; +using lex_quoted_key = either; +using lex_simple_key = either; +using lex_dotted_key = sequence, + at_least<1> + > + >; +using lex_key = either; + +using lex_keyval_sep = sequence, + character<'='>, + maybe>; + +using lex_std_table_open = character<'['>; +using lex_std_table_close = character<']'>; +using lex_std_table = sequence, + lex_key, + maybe, + lex_std_table_close>; + +using lex_array_table_open = sequence; +using lex_array_table_close = sequence; +using lex_array_table = sequence, + lex_key, + maybe, + lex_array_table_close>; + +using lex_utf8_1byte = in_range<0x00, 0x7F>; +using lex_utf8_2byte = sequence< + in_range(0xC2), static_cast(0xDF)>, + in_range(0x80), static_cast(0xBF)> + >; +using lex_utf8_3byte = sequence(0xE0)>, in_range(0xA0), static_cast(0xBF)>>, + sequence(0xE1), static_cast(0xEC)>, in_range(0x80), static_cast(0xBF)>>, + sequence(0xED)>, in_range(0x80), static_cast(0x9F)>>, + sequence(0xEE), static_cast(0xEF)>, in_range(0x80), static_cast(0xBF)>> + >, in_range(0x80), static_cast(0xBF)>>; +using lex_utf8_4byte = sequence(0xF0)>, in_range(0x90), static_cast(0xBF)>>, + sequence(0xF1), static_cast(0xF3)>, in_range(0x80), static_cast(0xBF)>>, + sequence(0xF4)>, in_range(0x80), static_cast(0x8F)>> + >, in_range(0x80), static_cast(0xBF)>, + in_range(0x80), static_cast(0xBF)>>; +using lex_utf8_code = either< + lex_utf8_1byte, + lex_utf8_2byte, + lex_utf8_3byte, + lex_utf8_4byte + >; + +using lex_comment_start_symbol = character<'#'>; +using lex_non_eol_ascii = either, in_range<0x20, 0x7E>>; +using lex_comment = sequence, unlimited>>; + +} // detail +} // toml +#endif // TOML_LEXER_HPP diff --git a/src/toml11/toml/literal.hpp b/src/toml11/toml/literal.hpp new file mode 100644 index 000000000..04fbbc13e --- /dev/null +++ b/src/toml11/toml/literal.hpp @@ -0,0 +1,113 @@ +// Copyright Toru Niina 2019. +// Distributed under the MIT License. +#ifndef TOML11_LITERAL_HPP +#define TOML11_LITERAL_HPP +#include "parser.hpp" + +namespace toml +{ +inline namespace literals +{ +inline namespace toml_literals +{ + +// implementation +inline ::toml::basic_value +literal_internal_impl(::toml::detail::location loc) +{ + using value_type = ::toml::basic_value< + TOML11_DEFAULT_COMMENT_STRATEGY, std::unordered_map, std::vector>; + // if there are some comments or empty lines, skip them. + using skip_line = ::toml::detail::repeat, + ::toml::detail::maybe<::toml::detail::lex_comment>, + ::toml::detail::lex_newline + >, ::toml::detail::at_least<1>>; + skip_line::invoke(loc); + + // if there are some whitespaces before a value, skip them. + using skip_ws = ::toml::detail::repeat< + ::toml::detail::lex_ws, ::toml::detail::at_least<1>>; + skip_ws::invoke(loc); + + // to distinguish arrays and tables, first check it is a table or not. + // + // "[1,2,3]"_toml; // this is an array + // "[table]"_toml; // a table that has an empty table named "table" inside. + // "[[1,2,3]]"_toml; // this is an array of arrays + // "[[table]]"_toml; // this is a table that has an array of tables inside. + // + // "[[1]]"_toml; // this can be both... (currently it becomes a table) + // "1 = [{}]"_toml; // this is a table that has an array of table named 1. + // "[[1,]]"_toml; // this is an array of arrays. + // "[[1],]"_toml; // this also. + + const auto the_front = loc.iter(); + + const bool is_table_key = ::toml::detail::lex_std_table::invoke(loc); + loc.reset(the_front); + + const bool is_aots_key = ::toml::detail::lex_array_table::invoke(loc); + loc.reset(the_front); + + // If it is neither a table-key or a array-of-table-key, it may be a value. + if(!is_table_key && !is_aots_key) + { + if(auto data = ::toml::detail::parse_value(loc)) + { + return data.unwrap(); + } + } + + // Note that still it can be a table, because the literal might be something + // like the following. + // ```cpp + // R"( // c++11 raw string literals + // key = "value" + // int = 42 + // )"_toml; + // ``` + // It is a valid toml file. + // It should be parsed as if we parse a file with this content. + + if(auto data = ::toml::detail::parse_toml_file(loc)) + { + return data.unwrap(); + } + else // none of them. + { + throw ::toml::syntax_error(data.unwrap_err(), source_location(loc)); + } + +} + +inline ::toml::basic_value +operator"" _toml(const char* str, std::size_t len) +{ + ::toml::detail::location loc( + std::string("TOML literal encoded in a C++ code"), + std::vector(str, str + len)); + // literal length does not include the null character at the end. + return literal_internal_impl(std::move(loc)); +} + +// value of __cplusplus in C++2a/20 mode is not fixed yet along compilers. +// So here we use the feature test macro for `char8_t` itself. +#if defined(__cpp_char8_t) && __cpp_char8_t >= 201811L +// value of u8"" literal has been changed from char to char8_t and char8_t is +// NOT compatible to char +inline ::toml::basic_value +operator"" _toml(const char8_t* str, std::size_t len) +{ + ::toml::detail::location loc( + std::string("TOML literal encoded in a C++ code"), + std::vector(reinterpret_cast(str), + reinterpret_cast(str) + len)); + return literal_internal_impl(std::move(loc)); +} +#endif + +} // toml_literals +} // literals +} // toml +#endif//TOML11_LITERAL_HPP diff --git a/src/toml11/toml/macros.hpp b/src/toml11/toml/macros.hpp new file mode 100644 index 000000000..e8f91aecd --- /dev/null +++ b/src/toml11/toml/macros.hpp @@ -0,0 +1,121 @@ +#ifndef TOML11_MACROS_HPP +#define TOML11_MACROS_HPP + +#define TOML11_STRINGIZE_AUX(x) #x +#define TOML11_STRINGIZE(x) TOML11_STRINGIZE_AUX(x) + +#define TOML11_CONCATENATE_AUX(x, y) x##y +#define TOML11_CONCATENATE(x, y) TOML11_CONCATENATE_AUX(x, y) + +// ============================================================================ +// TOML11_DEFINE_CONVERSION_NON_INTRUSIVE + +#ifndef TOML11_WITHOUT_DEFINE_NON_INTRUSIVE + +// ---------------------------------------------------------------------------- +// TOML11_ARGS_SIZE + +#define TOML11_INDEX_RSEQ() \ + 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, \ + 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 +#define TOML11_ARGS_SIZE_IMPL(\ + ARG1, ARG2, ARG3, ARG4, ARG5, ARG6, ARG7, ARG8, ARG9, ARG10, \ + ARG11, ARG12, ARG13, ARG14, ARG15, ARG16, ARG17, ARG18, ARG19, ARG20, \ + ARG21, ARG22, ARG23, ARG24, ARG25, ARG26, ARG27, ARG28, ARG29, ARG30, \ + ARG31, ARG32, N, ...) N +#define TOML11_ARGS_SIZE_AUX(...) TOML11_ARGS_SIZE_IMPL(__VA_ARGS__) +#define TOML11_ARGS_SIZE(...) TOML11_ARGS_SIZE_AUX(__VA_ARGS__, TOML11_INDEX_RSEQ()) + +// ---------------------------------------------------------------------------- +// TOML11_FOR_EACH_VA_ARGS + +#define TOML11_FOR_EACH_VA_ARGS_AUX_1( FUNCTOR, ARG1 ) FUNCTOR(ARG1) +#define TOML11_FOR_EACH_VA_ARGS_AUX_2( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_1( FUNCTOR, __VA_ARGS__) +#define TOML11_FOR_EACH_VA_ARGS_AUX_3( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_2( FUNCTOR, __VA_ARGS__) +#define TOML11_FOR_EACH_VA_ARGS_AUX_4( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_3( FUNCTOR, __VA_ARGS__) +#define TOML11_FOR_EACH_VA_ARGS_AUX_5( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_4( FUNCTOR, __VA_ARGS__) +#define TOML11_FOR_EACH_VA_ARGS_AUX_6( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_5( FUNCTOR, __VA_ARGS__) +#define TOML11_FOR_EACH_VA_ARGS_AUX_7( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_6( FUNCTOR, __VA_ARGS__) +#define TOML11_FOR_EACH_VA_ARGS_AUX_8( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_7( FUNCTOR, __VA_ARGS__) +#define TOML11_FOR_EACH_VA_ARGS_AUX_9( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_8( FUNCTOR, __VA_ARGS__) +#define TOML11_FOR_EACH_VA_ARGS_AUX_10(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_9( FUNCTOR, __VA_ARGS__) +#define TOML11_FOR_EACH_VA_ARGS_AUX_11(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_10(FUNCTOR, __VA_ARGS__) +#define TOML11_FOR_EACH_VA_ARGS_AUX_12(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_11(FUNCTOR, __VA_ARGS__) +#define TOML11_FOR_EACH_VA_ARGS_AUX_13(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_12(FUNCTOR, __VA_ARGS__) +#define TOML11_FOR_EACH_VA_ARGS_AUX_14(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_13(FUNCTOR, __VA_ARGS__) +#define TOML11_FOR_EACH_VA_ARGS_AUX_15(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_14(FUNCTOR, __VA_ARGS__) +#define TOML11_FOR_EACH_VA_ARGS_AUX_16(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_15(FUNCTOR, __VA_ARGS__) +#define TOML11_FOR_EACH_VA_ARGS_AUX_17(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_16(FUNCTOR, __VA_ARGS__) +#define TOML11_FOR_EACH_VA_ARGS_AUX_18(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_17(FUNCTOR, __VA_ARGS__) +#define TOML11_FOR_EACH_VA_ARGS_AUX_19(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_18(FUNCTOR, __VA_ARGS__) +#define TOML11_FOR_EACH_VA_ARGS_AUX_20(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_19(FUNCTOR, __VA_ARGS__) +#define TOML11_FOR_EACH_VA_ARGS_AUX_21(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_20(FUNCTOR, __VA_ARGS__) +#define TOML11_FOR_EACH_VA_ARGS_AUX_22(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_21(FUNCTOR, __VA_ARGS__) +#define TOML11_FOR_EACH_VA_ARGS_AUX_23(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_22(FUNCTOR, __VA_ARGS__) +#define TOML11_FOR_EACH_VA_ARGS_AUX_24(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_23(FUNCTOR, __VA_ARGS__) +#define TOML11_FOR_EACH_VA_ARGS_AUX_25(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_24(FUNCTOR, __VA_ARGS__) +#define TOML11_FOR_EACH_VA_ARGS_AUX_26(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_25(FUNCTOR, __VA_ARGS__) +#define TOML11_FOR_EACH_VA_ARGS_AUX_27(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_26(FUNCTOR, __VA_ARGS__) +#define TOML11_FOR_EACH_VA_ARGS_AUX_28(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_27(FUNCTOR, __VA_ARGS__) +#define TOML11_FOR_EACH_VA_ARGS_AUX_29(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_28(FUNCTOR, __VA_ARGS__) +#define TOML11_FOR_EACH_VA_ARGS_AUX_30(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_29(FUNCTOR, __VA_ARGS__) +#define TOML11_FOR_EACH_VA_ARGS_AUX_31(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_30(FUNCTOR, __VA_ARGS__) +#define TOML11_FOR_EACH_VA_ARGS_AUX_32(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_31(FUNCTOR, __VA_ARGS__) + +#define TOML11_FOR_EACH_VA_ARGS(FUNCTOR, ...)\ + TOML11_CONCATENATE(TOML11_FOR_EACH_VA_ARGS_AUX_, TOML11_ARGS_SIZE(__VA_ARGS__))(FUNCTOR, __VA_ARGS__) + +// ---------------------------------------------------------------------------- +// TOML11_DEFINE_CONVERSION_NON_INTRUSIVE + +// use it in the following way. +// ```cpp +// namespace foo +// { +// struct Foo +// { +// std::string s; +// double d; +// int i; +// }; +// } // foo +// +// TOML11_DEFINE_CONVERSION_NON_INTRUSIVE(foo::Foo, s, d, i) +// ``` +// And then you can use `toml::find(file, "foo");` +// +#define TOML11_FIND_MEMBER_VARIABLE_FROM_VALUE(VAR_NAME)\ + obj.VAR_NAME = toml::find(v, TOML11_STRINGIZE(VAR_NAME)); + +#define TOML11_ASSIGN_MEMBER_VARIABLE_TO_VALUE(VAR_NAME)\ + v[TOML11_STRINGIZE(VAR_NAME)] = obj.VAR_NAME; + +#define TOML11_DEFINE_CONVERSION_NON_INTRUSIVE(NAME, ...)\ + namespace toml { \ + template<> \ + struct from \ + { \ + template class T, \ + template class A> \ + static NAME from_toml(const basic_value& v) \ + { \ + NAME obj; \ + TOML11_FOR_EACH_VA_ARGS(TOML11_FIND_MEMBER_VARIABLE_FROM_VALUE, __VA_ARGS__) \ + return obj; \ + } \ + }; \ + template<> \ + struct into \ + { \ + static value into_toml(const NAME& obj) \ + { \ + ::toml::value v = ::toml::table{}; \ + TOML11_FOR_EACH_VA_ARGS(TOML11_ASSIGN_MEMBER_VARIABLE_TO_VALUE, __VA_ARGS__) \ + return v; \ + } \ + }; \ + } /* toml */ + +#endif// TOML11_WITHOUT_DEFINE_NON_INTRUSIVE + +#endif// TOML11_MACROS_HPP diff --git a/src/toml11/toml/parser.hpp b/src/toml11/toml/parser.hpp new file mode 100644 index 000000000..e31179918 --- /dev/null +++ b/src/toml11/toml/parser.hpp @@ -0,0 +1,2364 @@ +// Copyright Toru Niina 2017. +// Distributed under the MIT License. +#ifndef TOML11_PARSER_HPP +#define TOML11_PARSER_HPP +#include +#include +#include + +#include "combinator.hpp" +#include "lexer.hpp" +#include "region.hpp" +#include "result.hpp" +#include "types.hpp" +#include "value.hpp" + +#ifndef TOML11_DISABLE_STD_FILESYSTEM +#ifdef __cpp_lib_filesystem +#if __has_include() +#define TOML11_HAS_STD_FILESYSTEM +#include +#endif // has_include() +#endif // __cpp_lib_filesystem +#endif // TOML11_DISABLE_STD_FILESYSTEM + +namespace toml +{ +namespace detail +{ + +inline result, std::string> +parse_boolean(location& loc) +{ + const auto first = loc.iter(); + if(const auto token = lex_boolean::invoke(loc)) + { + const auto reg = token.unwrap(); + if (reg.str() == "true") {return ok(std::make_pair(true, reg));} + else if(reg.str() == "false") {return ok(std::make_pair(false, reg));} + else // internal error. + { + throw internal_error(format_underline( + "toml::parse_boolean: internal error", + {{source_location(reg), "invalid token"}}), + source_location(reg)); + } + } + loc.reset(first); //rollback + return err(format_underline("toml::parse_boolean: ", + {{source_location(loc), "the next token is not a boolean"}})); +} + +inline result, std::string> +parse_binary_integer(location& loc) +{ + const auto first = loc.iter(); + if(const auto token = lex_bin_int::invoke(loc)) + { + auto str = token.unwrap().str(); + assert(str.size() > 2); // minimum -> 0b1 + integer retval(0), base(1); + for(auto i(str.rbegin()), e(str.rend() - 2); i!=e; ++i) + { + if (*i == '1'){retval += base; base *= 2;} + else if(*i == '0'){base *= 2;} + else if(*i == '_'){/* do nothing. */} + else // internal error. + { + throw internal_error(format_underline( + "toml::parse_integer: internal error", + {{source_location(token.unwrap()), "invalid token"}}), + source_location(loc)); + } + } + return ok(std::make_pair(retval, token.unwrap())); + } + loc.reset(first); + return err(format_underline("toml::parse_binary_integer:", + {{source_location(loc), "the next token is not an integer"}})); +} + +inline result, std::string> +parse_octal_integer(location& loc) +{ + const auto first = loc.iter(); + if(const auto token = lex_oct_int::invoke(loc)) + { + auto str = token.unwrap().str(); + str.erase(std::remove(str.begin(), str.end(), '_'), str.end()); + str.erase(str.begin()); str.erase(str.begin()); // remove `0o` prefix + + std::istringstream iss(str); + integer retval(0); + iss >> std::oct >> retval; + return ok(std::make_pair(retval, token.unwrap())); + } + loc.reset(first); + return err(format_underline("toml::parse_octal_integer:", + {{source_location(loc), "the next token is not an integer"}})); +} + +inline result, std::string> +parse_hexadecimal_integer(location& loc) +{ + const auto first = loc.iter(); + if(const auto token = lex_hex_int::invoke(loc)) + { + auto str = token.unwrap().str(); + str.erase(std::remove(str.begin(), str.end(), '_'), str.end()); + str.erase(str.begin()); str.erase(str.begin()); // remove `0x` prefix + + std::istringstream iss(str); + integer retval(0); + iss >> std::hex >> retval; + return ok(std::make_pair(retval, token.unwrap())); + } + loc.reset(first); + return err(format_underline("toml::parse_hexadecimal_integer", + {{source_location(loc), "the next token is not an integer"}})); +} + +inline result, std::string> +parse_integer(location& loc) +{ + const auto first = loc.iter(); + if(first != loc.end() && *first == '0') + { + const auto second = std::next(first); + if(second == loc.end()) // the token is just zero. + { + loc.advance(); + return ok(std::make_pair(0, region(loc, first, second))); + } + + if(*second == 'b') {return parse_binary_integer (loc);} // 0b1100 + if(*second == 'o') {return parse_octal_integer (loc);} // 0o775 + if(*second == 'x') {return parse_hexadecimal_integer(loc);} // 0xC0FFEE + + if(std::isdigit(*second)) + { + return err(format_underline("toml::parse_integer: " + "leading zero in an Integer is not allowed.", + {{source_location(loc), "leading zero"}})); + } + else if(std::isalpha(*second)) + { + return err(format_underline("toml::parse_integer: " + "unknown integer prefix appeared.", + {{source_location(loc), "none of 0x, 0o, 0b"}})); + } + } + + if(const auto token = lex_dec_int::invoke(loc)) + { + auto str = token.unwrap().str(); + str.erase(std::remove(str.begin(), str.end(), '_'), str.end()); + + std::istringstream iss(str); + integer retval(0); + iss >> retval; + return ok(std::make_pair(retval, token.unwrap())); + } + loc.reset(first); + return err(format_underline("toml::parse_integer: ", + {{source_location(loc), "the next token is not an integer"}})); +} + +inline result, std::string> +parse_floating(location& loc) +{ + const auto first = loc.iter(); + if(const auto token = lex_float::invoke(loc)) + { + auto str = token.unwrap().str(); + if(str == "inf" || str == "+inf") + { + if(std::numeric_limits::has_infinity) + { + return ok(std::make_pair( + std::numeric_limits::infinity(), token.unwrap())); + } + else + { + throw std::domain_error("toml::parse_floating: inf value found" + " but the current environment does not support inf. Please" + " make sure that the floating-point implementation conforms" + " IEEE 754/ISO 60559 international standard."); + } + } + else if(str == "-inf") + { + if(std::numeric_limits::has_infinity) + { + return ok(std::make_pair( + -std::numeric_limits::infinity(), token.unwrap())); + } + else + { + throw std::domain_error("toml::parse_floating: inf value found" + " but the current environment does not support inf. Please" + " make sure that the floating-point implementation conforms" + " IEEE 754/ISO 60559 international standard."); + } + } + else if(str == "nan" || str == "+nan") + { + if(std::numeric_limits::has_quiet_NaN) + { + return ok(std::make_pair( + std::numeric_limits::quiet_NaN(), token.unwrap())); + } + else if(std::numeric_limits::has_signaling_NaN) + { + return ok(std::make_pair( + std::numeric_limits::signaling_NaN(), token.unwrap())); + } + else + { + throw std::domain_error("toml::parse_floating: NaN value found" + " but the current environment does not support NaN. Please" + " make sure that the floating-point implementation conforms" + " IEEE 754/ISO 60559 international standard."); + } + } + else if(str == "-nan") + { + if(std::numeric_limits::has_quiet_NaN) + { + return ok(std::make_pair( + -std::numeric_limits::quiet_NaN(), token.unwrap())); + } + else if(std::numeric_limits::has_signaling_NaN) + { + return ok(std::make_pair( + -std::numeric_limits::signaling_NaN(), token.unwrap())); + } + else + { + throw std::domain_error("toml::parse_floating: NaN value found" + " but the current environment does not support NaN. Please" + " make sure that the floating-point implementation conforms" + " IEEE 754/ISO 60559 international standard."); + } + } + str.erase(std::remove(str.begin(), str.end(), '_'), str.end()); + std::istringstream iss(str); + floating v(0.0); + iss >> v; + return ok(std::make_pair(v, token.unwrap())); + } + loc.reset(first); + return err(format_underline("toml::parse_floating: ", + {{source_location(loc), "the next token is not a float"}})); +} + +inline std::string read_utf8_codepoint(const region& reg, const location& loc) +{ + const auto str = reg.str().substr(1); + std::uint_least32_t codepoint; + std::istringstream iss(str); + iss >> std::hex >> codepoint; + + const auto to_char = [](const std::uint_least32_t i) noexcept -> char { + const auto uc = static_cast(i); + return *reinterpret_cast(std::addressof(uc)); + }; + + std::string character; + if(codepoint < 0x80) // U+0000 ... U+0079 ; just an ASCII. + { + character += static_cast(codepoint); + } + else if(codepoint < 0x800) //U+0080 ... U+07FF + { + // 110yyyyx 10xxxxxx; 0x3f == 0b0011'1111 + character += to_char(0xC0| codepoint >> 6); + character += to_char(0x80|(codepoint & 0x3F)); + } + else if(codepoint < 0x10000) // U+0800...U+FFFF + { + if(0xD800 <= codepoint && codepoint <= 0xDFFF) + { + throw syntax_error(format_underline( + "toml::read_utf8_codepoint: codepoints in the range " + "[0xD800, 0xDFFF] are not valid UTF-8.", {{ + source_location(loc), "not a valid UTF-8 codepoint" + }}), source_location(loc)); + } + assert(codepoint < 0xD800 || 0xDFFF < codepoint); + // 1110yyyy 10yxxxxx 10xxxxxx + character += to_char(0xE0| codepoint >> 12); + character += to_char(0x80|(codepoint >> 6 & 0x3F)); + character += to_char(0x80|(codepoint & 0x3F)); + } + else if(codepoint < 0x110000) // U+010000 ... U+10FFFF + { + // 11110yyy 10yyxxxx 10xxxxxx 10xxxxxx + character += to_char(0xF0| codepoint >> 18); + character += to_char(0x80|(codepoint >> 12 & 0x3F)); + character += to_char(0x80|(codepoint >> 6 & 0x3F)); + character += to_char(0x80|(codepoint & 0x3F)); + } + else // out of UTF-8 region + { + throw syntax_error(format_underline("toml::read_utf8_codepoint:" + " input codepoint is too large.", + {{source_location(loc), "should be in [0x00..0x10FFFF]"}}), + source_location(loc)); + } + return character; +} + +inline result parse_escape_sequence(location& loc) +{ + const auto first = loc.iter(); + if(first == loc.end() || *first != '\\') + { + return err(format_underline("toml::parse_escape_sequence: ", {{ + source_location(loc), "the next token is not a backslash \"\\\""}})); + } + loc.advance(); + switch(*loc.iter()) + { + case '\\':{loc.advance(); return ok(std::string("\\"));} + case '"' :{loc.advance(); return ok(std::string("\""));} + case 'b' :{loc.advance(); return ok(std::string("\b"));} + case 't' :{loc.advance(); return ok(std::string("\t"));} + case 'n' :{loc.advance(); return ok(std::string("\n"));} + case 'f' :{loc.advance(); return ok(std::string("\f"));} + case 'r' :{loc.advance(); return ok(std::string("\r"));} + case 'u' : + { + if(const auto token = lex_escape_unicode_short::invoke(loc)) + { + return ok(read_utf8_codepoint(token.unwrap(), loc)); + } + else + { + return err(format_underline("parse_escape_sequence: " + "invalid token found in UTF-8 codepoint uXXXX.", + {{source_location(loc), "here"}})); + } + } + case 'U': + { + if(const auto token = lex_escape_unicode_long::invoke(loc)) + { + return ok(read_utf8_codepoint(token.unwrap(), loc)); + } + else + { + return err(format_underline("parse_escape_sequence: " + "invalid token found in UTF-8 codepoint Uxxxxxxxx", + {{source_location(loc), "here"}})); + } + } + } + + const auto msg = format_underline("parse_escape_sequence: " + "unknown escape sequence appeared.", {{source_location(loc), + "escape sequence is one of \\, \", b, t, n, f, r, uxxxx, Uxxxxxxxx"}}, + /* Hints = */{"if you want to write backslash as just one backslash, " + "use literal string like: regex = '<\\i\\c*\\s*>'"}); + loc.reset(first); + return err(msg); +} + +inline std::ptrdiff_t check_utf8_validity(const std::string& reg) +{ + location loc("tmp", reg); + const auto u8 = repeat::invoke(loc); + if(!u8 || loc.iter() != loc.end()) + { + const auto error_location = std::distance(loc.begin(), loc.iter()); + assert(0 <= error_location); + return error_location; + } + return -1; +} + +inline result, std::string> +parse_ml_basic_string(location& loc) +{ + const auto first = loc.iter(); + if(const auto token = lex_ml_basic_string::invoke(loc)) + { + auto inner_loc = loc; + inner_loc.reset(first); + + std::string retval; + retval.reserve(token.unwrap().size()); + + auto delim = lex_ml_basic_string_open::invoke(inner_loc); + if(!delim) + { + throw internal_error(format_underline( + "parse_ml_basic_string: invalid token", + {{source_location(inner_loc), "should be \"\"\""}}), + source_location(inner_loc)); + } + // immediate newline is ignored (if exists) + /* discard return value */ lex_newline::invoke(inner_loc); + + delim = none(); + while(!delim) + { + using lex_unescaped_seq = repeat< + either, unlimited>; + if(auto unescaped = lex_unescaped_seq::invoke(inner_loc)) + { + retval += unescaped.unwrap().str(); + } + if(auto escaped = parse_escape_sequence(inner_loc)) + { + retval += escaped.unwrap(); + } + if(auto esc_nl = lex_ml_basic_escaped_newline::invoke(inner_loc)) + { + // ignore newline after escape until next non-ws char + } + if(inner_loc.iter() == inner_loc.end()) + { + throw internal_error(format_underline( + "parse_ml_basic_string: unexpected end of region", + {{source_location(inner_loc), "not sufficient token"}}), + source_location(inner_loc)); + } + delim = lex_ml_basic_string_close::invoke(inner_loc); + } + // `lex_ml_basic_string_close` allows 3 to 5 `"`s to allow 1 or 2 `"`s + // at just before the delimiter. Here, we need to attach `"`s at the + // end of the string body, if it exists. + // For detail, see the definition of `lex_ml_basic_string_close`. + assert(std::all_of(delim.unwrap().first(), delim.unwrap().last(), + [](const char c) noexcept {return c == '\"';})); + switch(delim.unwrap().size()) + { + case 3: {break;} + case 4: {retval += "\""; break;} + case 5: {retval += "\"\""; break;} + default: + { + throw internal_error(format_underline( + "parse_ml_basic_string: closing delimiter has invalid length", + {{source_location(inner_loc), "end of this"}}), + source_location(inner_loc)); + } + } + + const auto err_loc = check_utf8_validity(token.unwrap().str()); + if(err_loc == -1) + { + return ok(std::make_pair(toml::string(retval), token.unwrap())); + } + else + { + inner_loc.reset(first); + inner_loc.advance(err_loc); + throw syntax_error(format_underline( + "parse_ml_basic_string: invalid utf8 sequence found", + {{source_location(inner_loc), "here"}}), + source_location(inner_loc)); + } + } + else + { + loc.reset(first); + return err(format_underline("toml::parse_ml_basic_string: " + "the next token is not a valid multiline string", + {{source_location(loc), "here"}})); + } +} + +inline result, std::string> +parse_basic_string(location& loc) +{ + const auto first = loc.iter(); + if(const auto token = lex_basic_string::invoke(loc)) + { + auto inner_loc = loc; + inner_loc.reset(first); + + auto quot = lex_quotation_mark::invoke(inner_loc); + if(!quot) + { + throw internal_error(format_underline("parse_basic_string: " + "invalid token", {{source_location(inner_loc), "should be \""}}), + source_location(inner_loc)); + } + + std::string retval; + retval.reserve(token.unwrap().size()); + + quot = none(); + while(!quot) + { + using lex_unescaped_seq = repeat; + if(auto unescaped = lex_unescaped_seq::invoke(inner_loc)) + { + retval += unescaped.unwrap().str(); + } + if(auto escaped = parse_escape_sequence(inner_loc)) + { + retval += escaped.unwrap(); + } + if(inner_loc.iter() == inner_loc.end()) + { + throw internal_error(format_underline( + "parse_basic_string: unexpected end of region", + {{source_location(inner_loc), "not sufficient token"}}), + source_location(inner_loc)); + } + quot = lex_quotation_mark::invoke(inner_loc); + } + + const auto err_loc = check_utf8_validity(token.unwrap().str()); + if(err_loc == -1) + { + return ok(std::make_pair(toml::string(retval), token.unwrap())); + } + else + { + inner_loc.reset(first); + inner_loc.advance(err_loc); + throw syntax_error(format_underline( + "parse_ml_basic_string: invalid utf8 sequence found", + {{source_location(inner_loc), "here"}}), + source_location(inner_loc)); + } + } + else + { + loc.reset(first); // rollback + return err(format_underline("toml::parse_basic_string: " + "the next token is not a valid string", + {{source_location(loc), "here"}})); + } +} + +inline result, std::string> +parse_ml_literal_string(location& loc) +{ + const auto first = loc.iter(); + if(const auto token = lex_ml_literal_string::invoke(loc)) + { + location inner_loc(loc.name(), token.unwrap().str()); + + const auto open = lex_ml_literal_string_open::invoke(inner_loc); + if(!open) + { + throw internal_error(format_underline( + "parse_ml_literal_string: invalid token", + {{source_location(inner_loc), "should be '''"}}), + source_location(inner_loc)); + } + // immediate newline is ignored (if exists) + /* discard return value */ lex_newline::invoke(inner_loc); + + const auto body = lex_ml_literal_body::invoke(inner_loc); + + const auto close = lex_ml_literal_string_close::invoke(inner_loc); + if(!close) + { + throw internal_error(format_underline( + "parse_ml_literal_string: invalid token", + {{source_location(inner_loc), "should be '''"}}), + source_location(inner_loc)); + } + // `lex_ml_literal_string_close` allows 3 to 5 `'`s to allow 1 or 2 `'`s + // at just before the delimiter. Here, we need to attach `'`s at the + // end of the string body, if it exists. + // For detail, see the definition of `lex_ml_basic_string_close`. + + std::string retval = body.unwrap().str(); + assert(std::all_of(close.unwrap().first(), close.unwrap().last(), + [](const char c) noexcept {return c == '\'';})); + switch(close.unwrap().size()) + { + case 3: {break;} + case 4: {retval += "'"; break;} + case 5: {retval += "''"; break;} + default: + { + throw internal_error(format_underline( + "parse_ml_literal_string: closing delimiter has invalid length", + {{source_location(inner_loc), "end of this"}}), + source_location(inner_loc)); + } + } + + const auto err_loc = check_utf8_validity(token.unwrap().str()); + if(err_loc == -1) + { + return ok(std::make_pair(toml::string(retval, toml::string_t::literal), + token.unwrap())); + } + else + { + inner_loc.reset(first); + inner_loc.advance(err_loc); + throw syntax_error(format_underline( + "parse_ml_basic_string: invalid utf8 sequence found", + {{source_location(inner_loc), "here"}}), + source_location(inner_loc)); + } + } + else + { + loc.reset(first); // rollback + return err(format_underline("toml::parse_ml_literal_string: " + "the next token is not a valid multiline literal string", + {{source_location(loc), "here"}})); + } +} + +inline result, std::string> +parse_literal_string(location& loc) +{ + const auto first = loc.iter(); + if(const auto token = lex_literal_string::invoke(loc)) + { + location inner_loc(loc.name(), token.unwrap().str()); + + const auto open = lex_apostrophe::invoke(inner_loc); + if(!open) + { + throw internal_error(format_underline( + "parse_literal_string: invalid token", + {{source_location(inner_loc), "should be '"}}), + source_location(inner_loc)); + } + + const auto body = repeat::invoke(inner_loc); + + const auto close = lex_apostrophe::invoke(inner_loc); + if(!close) + { + throw internal_error(format_underline( + "parse_literal_string: invalid token", + {{source_location(inner_loc), "should be '"}}), + source_location(inner_loc)); + } + + const auto err_loc = check_utf8_validity(token.unwrap().str()); + if(err_loc == -1) + { + return ok(std::make_pair( + toml::string(body.unwrap().str(), toml::string_t::literal), + token.unwrap())); + } + else + { + inner_loc.reset(first); + inner_loc.advance(err_loc); + throw syntax_error(format_underline( + "parse_ml_basic_string: invalid utf8 sequence found", + {{source_location(inner_loc), "here"}}), + source_location(inner_loc)); + } + } + else + { + loc.reset(first); // rollback + return err(format_underline("toml::parse_literal_string: " + "the next token is not a valid literal string", + {{source_location(loc), "here"}})); + } +} + +inline result, std::string> +parse_string(location& loc) +{ + if(loc.iter() != loc.end() && *(loc.iter()) == '"') + { + if(loc.iter() + 1 != loc.end() && *(loc.iter() + 1) == '"' && + loc.iter() + 2 != loc.end() && *(loc.iter() + 2) == '"') + { + return parse_ml_basic_string(loc); + } + else + { + return parse_basic_string(loc); + } + } + else if(loc.iter() != loc.end() && *(loc.iter()) == '\'') + { + if(loc.iter() + 1 != loc.end() && *(loc.iter() + 1) == '\'' && + loc.iter() + 2 != loc.end() && *(loc.iter() + 2) == '\'') + { + return parse_ml_literal_string(loc); + } + else + { + return parse_literal_string(loc); + } + } + return err(format_underline("toml::parse_string: ", + {{source_location(loc), "the next token is not a string"}})); +} + +inline result, std::string> +parse_local_date(location& loc) +{ + const auto first = loc.iter(); + if(const auto token = lex_local_date::invoke(loc)) + { + location inner_loc(loc.name(), token.unwrap().str()); + + const auto y = lex_date_fullyear::invoke(inner_loc); + if(!y || inner_loc.iter() == inner_loc.end() || *inner_loc.iter() != '-') + { + throw internal_error(format_underline( + "toml::parse_inner_local_date: invalid year format", + {{source_location(inner_loc), "should be `-`"}}), + source_location(inner_loc)); + } + inner_loc.advance(); + const auto m = lex_date_month::invoke(inner_loc); + if(!m || inner_loc.iter() == inner_loc.end() || *inner_loc.iter() != '-') + { + throw internal_error(format_underline( + "toml::parse_local_date: invalid month format", + {{source_location(inner_loc), "should be `-`"}}), + source_location(inner_loc)); + } + inner_loc.advance(); + const auto d = lex_date_mday::invoke(inner_loc); + if(!d) + { + throw internal_error(format_underline( + "toml::parse_local_date: invalid day format", + {{source_location(inner_loc), "here"}}), + source_location(inner_loc)); + } + + const auto year = static_cast(from_string(y.unwrap().str(), 0)); + const auto month = static_cast(from_string(m.unwrap().str(), 0)); + const auto day = static_cast(from_string(d.unwrap().str(), 0)); + + // We briefly check whether the input date is valid or not. But here, we + // only check if the RFC3339 compliance. + // Actually there are several special date that does not exist, + // because of historical reasons, such as 1582/10/5-1582/10/14 (only in + // several countries). But here, we do not care about such a complicated + // rule. It makes the code complicated and there is only low probability + // that such a specific date is needed in practice. If someone need to + // validate date accurately, that means that the one need a specialized + // library for their purpose in a different layer. + { + const bool is_leap = (year % 4 == 0) && ((year % 100 != 0) || (year % 400 == 0)); + const auto max_day = (month == 2) ? (is_leap ? 29 : 28) : + ((month == 4 || month == 6 || month == 9 || month == 11) ? 30 : 31); + + if((month < 1 || 12 < month) || (day < 1 || max_day < day)) + { + throw syntax_error(format_underline("toml::parse_date: " + "invalid date: it does not conform RFC3339.", {{ + source_location(loc), "month should be 01-12, day should be" + " 01-28,29,30,31, depending on month/year." + }}), source_location(inner_loc)); + } + } + return ok(std::make_pair(local_date(year, static_cast(month - 1), day), + token.unwrap())); + } + else + { + loc.reset(first); + return err(format_underline("toml::parse_local_date: ", + {{source_location(loc), "the next token is not a local_date"}})); + } +} + +inline result, std::string> +parse_local_time(location& loc) +{ + const auto first = loc.iter(); + if(const auto token = lex_local_time::invoke(loc)) + { + location inner_loc(loc.name(), token.unwrap().str()); + + const auto h = lex_time_hour::invoke(inner_loc); + if(!h || inner_loc.iter() == inner_loc.end() || *inner_loc.iter() != ':') + { + throw internal_error(format_underline( + "toml::parse_local_time: invalid year format", + {{source_location(inner_loc), "should be `:`"}}), + source_location(inner_loc)); + } + inner_loc.advance(); + const auto m = lex_time_minute::invoke(inner_loc); + if(!m || inner_loc.iter() == inner_loc.end() || *inner_loc.iter() != ':') + { + throw internal_error(format_underline( + "toml::parse_local_time: invalid month format", + {{source_location(inner_loc), "should be `:`"}}), + source_location(inner_loc)); + } + inner_loc.advance(); + const auto s = lex_time_second::invoke(inner_loc); + if(!s) + { + throw internal_error(format_underline( + "toml::parse_local_time: invalid second format", + {{source_location(inner_loc), "here"}}), + source_location(inner_loc)); + } + + const int hour = from_string(h.unwrap().str(), 0); + const int minute = from_string(m.unwrap().str(), 0); + const int second = from_string(s.unwrap().str(), 0); + + if((hour < 0 || 23 < hour) || (minute < 0 || 59 < minute) || + (second < 0 || 60 < second)) // it may be leap second + { + throw syntax_error(format_underline("toml::parse_time: " + "invalid time: it does not conform RFC3339.", {{ + source_location(loc), "hour should be 00-23, minute should be" + " 00-59, second should be 00-60 (depending on the leap" + " second rules.)"}}), source_location(inner_loc)); + } + + local_time time(hour, minute, second, 0, 0); + + const auto before_secfrac = inner_loc.iter(); + if(const auto secfrac = lex_time_secfrac::invoke(inner_loc)) + { + auto sf = secfrac.unwrap().str(); + sf.erase(sf.begin()); // sf.front() == '.' + switch(sf.size() % 3) + { + case 2: sf += '0'; break; + case 1: sf += "00"; break; + case 0: break; + default: break; + } + if(sf.size() >= 9) + { + time.millisecond = from_string(sf.substr(0, 3), 0u); + time.microsecond = from_string(sf.substr(3, 3), 0u); + time.nanosecond = from_string(sf.substr(6, 3), 0u); + } + else if(sf.size() >= 6) + { + time.millisecond = from_string(sf.substr(0, 3), 0u); + time.microsecond = from_string(sf.substr(3, 3), 0u); + } + else if(sf.size() >= 3) + { + time.millisecond = from_string(sf, 0u); + time.microsecond = 0u; + } + } + else + { + if(before_secfrac != inner_loc.iter()) + { + throw internal_error(format_underline( + "toml::parse_local_time: invalid subsecond format", + {{source_location(inner_loc), "here"}}), + source_location(inner_loc)); + } + } + return ok(std::make_pair(time, token.unwrap())); + } + else + { + loc.reset(first); + return err(format_underline("toml::parse_local_time: ", + {{source_location(loc), "the next token is not a local_time"}})); + } +} + +inline result, std::string> +parse_local_datetime(location& loc) +{ + const auto first = loc.iter(); + if(const auto token = lex_local_date_time::invoke(loc)) + { + location inner_loc(loc.name(), token.unwrap().str()); + const auto date = parse_local_date(inner_loc); + if(!date || inner_loc.iter() == inner_loc.end()) + { + throw internal_error(format_underline( + "toml::parse_local_datetime: invalid datetime format", + {{source_location(inner_loc), "date, not datetime"}}), + source_location(inner_loc)); + } + const char delim = *(inner_loc.iter()); + if(delim != 'T' && delim != 't' && delim != ' ') + { + throw internal_error(format_underline( + "toml::parse_local_datetime: invalid datetime format", + {{source_location(inner_loc), "should be `T` or ` ` (space)"}}), + source_location(inner_loc)); + } + inner_loc.advance(); + const auto time = parse_local_time(inner_loc); + if(!time) + { + throw internal_error(format_underline( + "toml::parse_local_datetime: invalid datetime format", + {{source_location(inner_loc), "invalid time format"}}), + source_location(inner_loc)); + } + return ok(std::make_pair( + local_datetime(date.unwrap().first, time.unwrap().first), + token.unwrap())); + } + else + { + loc.reset(first); + return err(format_underline("toml::parse_local_datetime: ", + {{source_location(loc), "the next token is not a local_datetime"}})); + } +} + +inline result, std::string> +parse_offset_datetime(location& loc) +{ + const auto first = loc.iter(); + if(const auto token = lex_offset_date_time::invoke(loc)) + { + location inner_loc(loc.name(), token.unwrap().str()); + const auto datetime = parse_local_datetime(inner_loc); + if(!datetime || inner_loc.iter() == inner_loc.end()) + { + throw internal_error(format_underline( + "toml::parse_offset_datetime: invalid datetime format", + {{source_location(inner_loc), "date, not datetime"}}), + source_location(inner_loc)); + } + time_offset offset(0, 0); + if(const auto ofs = lex_time_numoffset::invoke(inner_loc)) + { + const auto str = ofs.unwrap().str(); + + const auto hour = from_string(str.substr(1,2), 0); + const auto minute = from_string(str.substr(4,2), 0); + + if((hour < 0 || 23 < hour) || (minute < 0 || 59 < minute)) + { + throw syntax_error(format_underline("toml::parse_offset_datetime: " + "invalid offset: it does not conform RFC3339.", {{ + source_location(loc), "month should be 01-12, day should be" + " 01-28,29,30,31, depending on month/year." + }}), source_location(inner_loc)); + } + + if(str.front() == '+') + { + offset = time_offset(hour, minute); + } + else + { + offset = time_offset(-hour, -minute); + } + } + else if(*inner_loc.iter() != 'Z' && *inner_loc.iter() != 'z') + { + throw internal_error(format_underline( + "toml::parse_offset_datetime: invalid datetime format", + {{source_location(inner_loc), "should be `Z` or `+HH:MM`"}}), + source_location(inner_loc)); + } + return ok(std::make_pair(offset_datetime(datetime.unwrap().first, offset), + token.unwrap())); + } + else + { + loc.reset(first); + return err(format_underline("toml::parse_offset_datetime: ", + {{source_location(loc), "the next token is not a offset_datetime"}})); + } +} + +inline result, std::string> +parse_simple_key(location& loc) +{ + if(const auto bstr = parse_basic_string(loc)) + { + return ok(std::make_pair(bstr.unwrap().first.str, bstr.unwrap().second)); + } + if(const auto lstr = parse_literal_string(loc)) + { + return ok(std::make_pair(lstr.unwrap().first.str, lstr.unwrap().second)); + } + if(const auto bare = lex_unquoted_key::invoke(loc)) + { + const auto reg = bare.unwrap(); + return ok(std::make_pair(reg.str(), reg)); + } + return err(format_underline("toml::parse_simple_key: ", + {{source_location(loc), "the next token is not a simple key"}})); +} + +// dotted key become vector of keys +inline result, region>, std::string> +parse_key(location& loc) +{ + const auto first = loc.iter(); + // dotted key -> `foo.bar.baz` where several single keys are chained by + // dots. Whitespaces between keys and dots are allowed. + if(const auto token = lex_dotted_key::invoke(loc)) + { + const auto reg = token.unwrap(); + location inner_loc(loc.name(), reg.str()); + std::vector keys; + + while(inner_loc.iter() != inner_loc.end()) + { + lex_ws::invoke(inner_loc); + if(const auto k = parse_simple_key(inner_loc)) + { + keys.push_back(k.unwrap().first); + } + else + { + throw internal_error(format_underline( + "toml::detail::parse_key: dotted key contains invalid key", + {{source_location(inner_loc), k.unwrap_err()}}), + source_location(inner_loc)); + } + + lex_ws::invoke(inner_loc); + if(inner_loc.iter() == inner_loc.end()) + { + break; + } + else if(*inner_loc.iter() == '.') + { + inner_loc.advance(); // to skip `.` + } + else + { + throw internal_error(format_underline("toml::parse_key: " + "dotted key contains invalid key ", + {{source_location(inner_loc), "should be `.`"}}), + source_location(inner_loc)); + } + } + return ok(std::make_pair(keys, reg)); + } + loc.reset(first); + + // simple_key: a single (basic_string|literal_string|bare key) + if(const auto smpl = parse_simple_key(loc)) + { + return ok(std::make_pair(std::vector(1, smpl.unwrap().first), + smpl.unwrap().second)); + } + return err(format_underline("toml::parse_key: an invalid key appeared.", + {{source_location(loc), "is not a valid key"}}, { + "bare keys : non-empty strings composed only of [A-Za-z0-9_-].", + "quoted keys: same as \"basic strings\" or 'literal strings'.", + "dotted keys: sequence of bare or quoted keys joined with a dot." + })); +} + +// forward-decl to implement parse_array and parse_table +template +result parse_value(location&); + +template +result, std::string> +parse_array(location& loc) +{ + using value_type = Value; + using array_type = typename value_type::array_type; + + const auto first = loc.iter(); + if(loc.iter() == loc.end()) + { + return err("toml::parse_array: input is empty"); + } + if(*loc.iter() != '[') + { + return err("toml::parse_array: token is not an array"); + } + loc.advance(); + + using lex_ws_comment_newline = repeat< + either, unlimited>; + + array_type retval; + while(loc.iter() != loc.end()) + { + lex_ws_comment_newline::invoke(loc); // skip + + if(loc.iter() != loc.end() && *loc.iter() == ']') + { + loc.advance(); // skip ']' + return ok(std::make_pair(retval, + region(loc, first, loc.iter()))); + } + + if(auto val = parse_value(loc)) + { + // After TOML v1.0.0-rc.1, array becomes to be able to have values + // with different types. So here we will omit this by default. + // + // But some of the test-suite checks if the parser accepts a hetero- + // geneous arrays, so we keep this for a while. +#ifdef TOML11_DISALLOW_HETEROGENEOUS_ARRAYS + if(!retval.empty() && retval.front().type() != val.as_ok().type()) + { + auto array_start_loc = loc; + array_start_loc.reset(first); + + throw syntax_error(format_underline("toml::parse_array: " + "type of elements should be the same each other.", { + {source_location(array_start_loc), "array starts here"}, + { + retval.front().location(), + "value has type " + stringize(retval.front().type()) + }, + { + val.unwrap().location(), + "value has different type, " + stringize(val.unwrap().type()) + } + }), source_location(loc)); + } +#endif + retval.push_back(std::move(val.unwrap())); + } + else + { + auto array_start_loc = loc; + array_start_loc.reset(first); + + throw syntax_error(format_underline("toml::parse_array: " + "value having invalid format appeared in an array", { + {source_location(array_start_loc), "array starts here"}, + {source_location(loc), "it is not a valid value."} + }), source_location(loc)); + } + + using lex_array_separator = sequence, character<','>>; + const auto sp = lex_array_separator::invoke(loc); + if(!sp) + { + lex_ws_comment_newline::invoke(loc); + if(loc.iter() != loc.end() && *loc.iter() == ']') + { + loc.advance(); // skip ']' + return ok(std::make_pair(retval, + region(loc, first, loc.iter()))); + } + else + { + auto array_start_loc = loc; + array_start_loc.reset(first); + + throw syntax_error(format_underline("toml::parse_array:" + " missing array separator `,` after a value", { + {source_location(array_start_loc), "array starts here"}, + {source_location(loc), "should be `,`"} + }), source_location(loc)); + } + } + } + loc.reset(first); + throw syntax_error(format_underline("toml::parse_array: " + "array did not closed by `]`", + {{source_location(loc), "should be closed"}}), + source_location(loc)); +} + +template +result, region>, Value>, std::string> +parse_key_value_pair(location& loc) +{ + using value_type = Value; + + const auto first = loc.iter(); + auto key_reg = parse_key(loc); + if(!key_reg) + { + std::string msg = std::move(key_reg.unwrap_err()); + // if the next token is keyvalue-separator, it means that there are no + // key. then we need to show error as "empty key is not allowed". + if(const auto keyval_sep = lex_keyval_sep::invoke(loc)) + { + loc.reset(first); + msg = format_underline("toml::parse_key_value_pair: " + "empty key is not allowed.", + {{source_location(loc), "key expected before '='"}}); + } + return err(std::move(msg)); + } + + const auto kvsp = lex_keyval_sep::invoke(loc); + if(!kvsp) + { + std::string msg; + // if the line contains '=' after the invalid sequence, possibly the + // error is in the key (like, invalid character in bare key). + const auto line_end = std::find(loc.iter(), loc.end(), '\n'); + if(std::find(loc.iter(), line_end, '=') != line_end) + { + msg = format_underline("toml::parse_key_value_pair: " + "invalid format for key", + {{source_location(loc), "invalid character in key"}}, + {"Did you forget '.' to separate dotted-key?", + "Allowed characters for bare key are [0-9a-zA-Z_-]."}); + } + else // if not, the error is lack of key-value separator. + { + msg = format_underline("toml::parse_key_value_pair: " + "missing key-value separator `=`", + {{source_location(loc), "should be `=`"}}); + } + loc.reset(first); + return err(std::move(msg)); + } + + const auto after_kvsp = loc.iter(); // err msg + auto val = parse_value(loc); + if(!val) + { + std::string msg; + loc.reset(after_kvsp); + // check there is something not a comment/whitespace after `=` + if(sequence, maybe, lex_newline>::invoke(loc)) + { + loc.reset(after_kvsp); + msg = format_underline("toml::parse_key_value_pair: " + "missing value after key-value separator '='", + {{source_location(loc), "expected value, but got nothing"}}); + } + else // there is something not a comment/whitespace, so invalid format. + { + msg = std::move(val.unwrap_err()); + } + loc.reset(first); + return err(msg); + } + return ok(std::make_pair(std::move(key_reg.unwrap()), + std::move(val.unwrap()))); +} + +// for error messages. +template +std::string format_dotted_keys(InputIterator first, const InputIterator last) +{ + static_assert(std::is_same::value_type>::value,""); + + std::string retval(*first++); + for(; first != last; ++first) + { + retval += '.'; + retval += *first; + } + return retval; +} + +// forward decl for is_valid_forward_table_definition +result, region>, std::string> +parse_table_key(location& loc); +template +result, std::string> +parse_inline_table(location& loc); + +// The following toml file is allowed. +// ```toml +// [a.b.c] # here, table `a` has element `b`. +// foo = "bar" +// [a] # merge a = {baz = "qux"} to a = {b = {...}} +// baz = "qux" +// ``` +// But the following is not allowed. +// ```toml +// [a] +// b.c.foo = "bar" +// [a] # error! the same table [a] defined! +// baz = "qux" +// ``` +// The following is neither allowed. +// ```toml +// a = { b.c.foo = "bar"} +// [a] # error! the same table [a] defined! +// baz = "qux" +// ``` +// Here, it parses region of `tab->at(k)` as a table key and check the depth +// of the key. If the key region points deeper node, it would be allowed. +// Otherwise, the key points the same node. It would be rejected. +template +bool is_valid_forward_table_definition(const Value& fwd, const Value& inserting, + Iterator key_first, Iterator key_curr, Iterator key_last) +{ + // ------------------------------------------------------------------------ + // check type of the value to be inserted/merged + + std::string inserting_reg = ""; + if(const auto ptr = detail::get_region(inserting)) + { + inserting_reg = ptr->str(); + } + location inserting_def("internal", std::move(inserting_reg)); + if(const auto inlinetable = parse_inline_table(inserting_def)) + { + // check if we are overwriting existing table. + // ```toml + // # NG + // a.b = 42 + // a = {d = 3.14} + // ``` + // Inserting an inline table to a existing super-table is not allowed in + // any case. If we found it, we can reject it without further checking. + return false; + } + + // ------------------------------------------------------------------------ + // check table defined before + + std::string internal = ""; + if(const auto ptr = detail::get_region(fwd)) + { + internal = ptr->str(); + } + location def("internal", std::move(internal)); + if(const auto tabkeys = parse_table_key(def)) // [table.key] + { + // table keys always contains all the nodes from the root. + const auto& tks = tabkeys.unwrap().first; + if(std::size_t(std::distance(key_first, key_last)) == tks.size() && + std::equal(tks.begin(), tks.end(), key_first)) + { + // the keys are equivalent. it is not allowed. + return false; + } + // the keys are not equivalent. it is allowed. + return true; + } + if(const auto dotkeys = parse_key(def)) + { + // consider the following case. + // [a] + // b.c = {d = 42} + // [a.b.c] + // e = 2.71 + // this defines the table [a.b.c] twice. no? + + // a dotted key starts from the node representing a table in which the + // dotted key belongs to. + const auto& dks = dotkeys.unwrap().first; + if(std::size_t(std::distance(key_curr, key_last)) == dks.size() && + std::equal(dks.begin(), dks.end(), key_curr)) + { + // the keys are equivalent. it is not allowed. + return false; + } + // the keys are not equivalent. it is allowed. + return true; + } + return false; +} + +template +result +insert_nested_key(typename Value::table_type& root, const Value& v, + InputIterator iter, const InputIterator last, + region key_reg, + const bool is_array_of_table = false) +{ + static_assert(std::is_same::value_type>::value,""); + + using value_type = Value; + using table_type = typename value_type::table_type; + using array_type = typename value_type::array_type; + + const auto first = iter; + assert(iter != last); + + table_type* tab = std::addressof(root); + for(; iter != last; ++iter) // search recursively + { + const key& k = *iter; + if(std::next(iter) == last) // k is the last key + { + // XXX if the value is array-of-tables, there can be several + // tables that are in the same array. in that case, we need to + // find the last element and insert it to there. + if(is_array_of_table) + { + if(tab->count(k) == 1) // there is already an array of table + { + if(tab->at(k).is_table()) + { + // show special err msg for conflicting table + throw syntax_error(format_underline(concat_to_string( + "toml::insert_value: array of table (\"", + format_dotted_keys(first, last), + "\") cannot be defined"), { + {tab->at(k).location(), "table already defined"}, + {v.location(), "this conflicts with the previous table"} + }), v.location()); + } + else if(!(tab->at(k).is_array())) + { + throw syntax_error(format_underline(concat_to_string( + "toml::insert_value: array of table (\"", + format_dotted_keys(first, last), "\") collides with" + " existing value"), { + {tab->at(k).location(), + concat_to_string("this ", tab->at(k).type(), + " value already exists")}, + {v.location(), + "while inserting this array-of-tables"} + }), v.location()); + } + // the above if-else-if checks tab->at(k) is an array + auto& a = tab->at(k).as_array(); + // If table element is defined as [[array_of_tables]], it + // cannot be an empty array. If an array of tables is + // defined as `aot = []`, it cannot be appended. + if(a.empty() || !(a.front().is_table())) + { + throw syntax_error(format_underline(concat_to_string( + "toml::insert_value: array of table (\"", + format_dotted_keys(first, last), "\") collides with" + " existing value"), { + {tab->at(k).location(), + concat_to_string("this ", tab->at(k).type(), + " value already exists")}, + {v.location(), + "while inserting this array-of-tables"} + }), v.location()); + } + // avoid conflicting array of table like the following. + // ```toml + // a = [{b = 42}] # define a as an array of *inline* tables + // [[a]] # a is an array of *multi-line* tables + // b = 54 + // ``` + // Here, from the type information, these cannot be detected + // because inline table is also a table. + // But toml v0.5.0 explicitly says it is invalid. The above + // array-of-tables has a static size and appending to the + // array is invalid. + // In this library, multi-line table value has a region + // that points to the key of the table (e.g. [[a]]). By + // comparing the first two letters in key, we can detect + // the array-of-table is inline or multiline. + if(const auto ptr = detail::get_region(a.front())) + { + if(ptr->str().substr(0,2) != "[[") + { + throw syntax_error(format_underline(concat_to_string( + "toml::insert_value: array of table (\"", + format_dotted_keys(first, last), "\") collides " + "with existing array-of-tables"), { + {tab->at(k).location(), + concat_to_string("this ", tab->at(k).type(), + " value has static size")}, + {v.location(), + "appending it to the statically sized array"} + }), v.location()); + } + } + a.push_back(v); + return ok(true); + } + else // if not, we need to create the array of table + { + // XXX: Consider the following array of tables. + // ```toml + // # This is a comment. + // [[aot]] + // foo = "bar" + // ``` + // Here, the comment is for `aot`. But here, actually two + // values are defined. An array that contains tables, named + // `aot`, and the 0th element of the `aot`, `{foo = "bar"}`. + // Those two are different from each other. But both of them + // points to the same portion of the TOML file, `[[aot]]`, + // so `key_reg.comments()` returns `# This is a comment`. + // If it is assigned as a comment of `aot` defined here, the + // comment will be duplicated. Both the `aot` itself and + // the 0-th element will have the same comment. This causes + // "duplication of the same comments" bug when the data is + // serialized. + // Next, consider the following. + // ```toml + // # comment 1 + // aot = [ + // # comment 2 + // {foo = "bar"}, + // ] + // ``` + // In this case, we can distinguish those two comments. So + // here we need to add "comment 1" to the `aot` and + // "comment 2" to the 0th element of that. + // To distinguish those two, we check the key region. + std::vector comments{/* empty by default */}; + if(key_reg.str().substr(0, 2) != "[[") + { + comments = key_reg.comments(); + } + value_type aot(array_type(1, v), key_reg, std::move(comments)); + tab->insert(std::make_pair(k, aot)); + return ok(true); + } + } // end if(array of table) + + if(tab->count(k) == 1) + { + if(tab->at(k).is_table() && v.is_table()) + { + if(!is_valid_forward_table_definition( + tab->at(k), v, first, iter, last)) + { + throw syntax_error(format_underline(concat_to_string( + "toml::insert_value: table (\"", + format_dotted_keys(first, last), + "\") already exists."), { + {tab->at(k).location(), "table already exists here"}, + {v.location(), "table defined twice"} + }), v.location()); + } + // to allow the following toml file. + // [a.b.c] + // d = 42 + // [a] + // e = 2.71 + auto& t = tab->at(k).as_table(); + for(const auto& kv : v.as_table()) + { + if(tab->at(k).contains(kv.first)) + { + throw syntax_error(format_underline(concat_to_string( + "toml::insert_value: value (\"", + format_dotted_keys(first, last), + "\") already exists."), { + {t.at(kv.first).location(), "already exists here"}, + {v.location(), "this defined twice"} + }), v.location()); + } + t[kv.first] = kv.second; + } + detail::change_region(tab->at(k), key_reg); + return ok(true); + } + else if(v.is_table() && + tab->at(k).is_array() && + tab->at(k).as_array().size() > 0 && + tab->at(k).as_array().front().is_table()) + { + throw syntax_error(format_underline(concat_to_string( + "toml::insert_value: array of tables (\"", + format_dotted_keys(first, last), "\") already exists."), { + {tab->at(k).location(), "array of tables defined here"}, + {v.location(), "table conflicts with the previous array of table"} + }), v.location()); + } + else + { + throw syntax_error(format_underline(concat_to_string( + "toml::insert_value: value (\"", + format_dotted_keys(first, last), "\") already exists."), { + {tab->at(k).location(), "value already exists here"}, + {v.location(), "value defined twice"} + }), v.location()); + } + } + tab->insert(std::make_pair(k, v)); + return ok(true); + } + else // k is not the last one, we should insert recursively + { + // if there is no corresponding value, insert it first. + // related: you don't need to write + // # [x] + // # [x.y] + // to write + // [x.y.z] + if(tab->count(k) == 0) + { + // a table that is defined implicitly doesn't have any comments. + (*tab)[k] = value_type(table_type{}, key_reg, {/*no comment*/}); + } + + // type checking... + if(tab->at(k).is_table()) + { + // According to toml-lang/toml:36d3091b3 "Clarify that inline + // tables are immutable", check if it adds key-value pair to an + // inline table. + if(const auto* ptr = get_region(tab->at(k))) + { + // here, if the value is a (multi-line) table, the region + // should be something like `[table-name]`. + if(ptr->front() == '{') + { + throw syntax_error(format_underline(concat_to_string( + "toml::insert_value: inserting to an inline table (", + format_dotted_keys(first, std::next(iter)), + ") but inline tables are immutable"), { + {tab->at(k).location(), "inline tables are immutable"}, + {v.location(), "inserting this"} + }), v.location()); + } + } + tab = std::addressof((*tab)[k].as_table()); + } + else if(tab->at(k).is_array()) // inserting to array-of-tables? + { + auto& a = (*tab)[k].as_array(); + if(!a.back().is_table()) + { + throw syntax_error(format_underline(concat_to_string( + "toml::insert_value: target (", + format_dotted_keys(first, std::next(iter)), + ") is neither table nor an array of tables"), { + {a.back().location(), concat_to_string( + "actual type is ", a.back().type())}, + {v.location(), "inserting this"} + }), v.location()); + } + tab = std::addressof(a.back().as_table()); + } + else + { + throw syntax_error(format_underline(concat_to_string( + "toml::insert_value: target (", + format_dotted_keys(first, std::next(iter)), + ") is neither table nor an array of tables"), { + {tab->at(k).location(), concat_to_string( + "actual type is ", tab->at(k).type())}, + {v.location(), "inserting this"} + }), v.location()); + } + } + } + return err(std::string("toml::detail::insert_nested_key: never reach here")); +} + +template +result, std::string> +parse_inline_table(location& loc) +{ + using value_type = Value; + using table_type = typename value_type::table_type; + + const auto first = loc.iter(); + table_type retval; + if(!(loc.iter() != loc.end() && *loc.iter() == '{')) + { + return err(format_underline("toml::parse_inline_table: ", + {{source_location(loc), "the next token is not an inline table"}})); + } + loc.advance(); + + // check if the inline table is an empty table = { } + maybe::invoke(loc); + if(loc.iter() != loc.end() && *loc.iter() == '}') + { + loc.advance(); // skip `}` + return ok(std::make_pair(retval, region(loc, first, loc.iter()))); + } + + // it starts from "{". it should be formatted as inline-table + while(loc.iter() != loc.end()) + { + const auto kv_r = parse_key_value_pair(loc); + if(!kv_r) + { + return err(kv_r.unwrap_err()); + } + + const auto& kvpair = kv_r.unwrap(); + const std::vector& keys = kvpair.first.first; + const auto& key_reg = kvpair.first.second; + const value_type& val = kvpair.second; + + const auto inserted = + insert_nested_key(retval, val, keys.begin(), keys.end(), key_reg); + if(!inserted) + { + throw internal_error("toml::parse_inline_table: " + "failed to insert value into table: " + inserted.unwrap_err(), + source_location(loc)); + } + + using lex_table_separator = sequence, character<','>>; + const auto sp = lex_table_separator::invoke(loc); + + if(!sp) + { + maybe::invoke(loc); + + if(loc.iter() == loc.end()) + { + throw syntax_error(format_underline( + "toml::parse_inline_table: missing table separator `}` ", + {{source_location(loc), "should be `}`"}}), + source_location(loc)); + } + else if(*loc.iter() == '}') + { + loc.advance(); // skip `}` + return ok(std::make_pair( + retval, region(loc, first, loc.iter()))); + } + else if(*loc.iter() == '#' || *loc.iter() == '\r' || *loc.iter() == '\n') + { + throw syntax_error(format_underline( + "toml::parse_inline_table: missing curly brace `}`", + {{source_location(loc), "should be `}`"}}), + source_location(loc)); + } + else + { + throw syntax_error(format_underline( + "toml::parse_inline_table: missing table separator `,` ", + {{source_location(loc), "should be `,`"}}), + source_location(loc)); + } + } + else // `,` is found + { + maybe::invoke(loc); + if(loc.iter() != loc.end() && *loc.iter() == '}') + { + throw syntax_error(format_underline( + "toml::parse_inline_table: trailing comma is not allowed in" + " an inline table", + {{source_location(loc), "should be `}`"}}), + source_location(loc)); + } + } + } + loc.reset(first); + throw syntax_error(format_underline("toml::parse_inline_table: " + "inline table did not closed by `}`", + {{source_location(loc), "should be closed"}}), + source_location(loc)); +} + +inline result guess_number_type(const location& l) +{ + // This function tries to find some (common) mistakes by checking characters + // that follows the last character of a value. But it is often difficult + // because some non-newline characters can appear after a value. E.g. + // spaces, tabs, commas (in an array or inline table), closing brackets + // (of an array or inline table), comment-sign (#). Since this function + // does not parse further, those characters are always allowed to be there. + location loc = l; + + if(lex_offset_date_time::invoke(loc)) {return ok(value_t::offset_datetime);} + loc.reset(l.iter()); + + if(lex_local_date_time::invoke(loc)) + { + // bad offset may appear after this. + if(loc.iter() != loc.end() && (*loc.iter() == '+' || *loc.iter() == '-' + || *loc.iter() == 'Z' || *loc.iter() == 'z')) + { + return err(format_underline("bad offset: should be [+-]HH:MM or Z", + {{source_location(loc), "[+-]HH:MM or Z"}}, + {"pass: +09:00, -05:30", "fail: +9:00, -5:30"})); + } + return ok(value_t::local_datetime); + } + loc.reset(l.iter()); + + if(lex_local_date::invoke(loc)) + { + // bad time may appear after this. + // A space is allowed as a delimiter between local time. But there are + // both cases in which a space becomes valid or invalid. + // - invalid: 2019-06-16 7:00:00 + // - valid : 2019-06-16 07:00:00 + if(loc.iter() != loc.end()) + { + const auto c = *loc.iter(); + if(c == 'T' || c == 't') + { + return err(format_underline("bad time: should be HH:MM:SS.subsec", + {{source_location(loc), "HH:MM:SS.subsec"}}, + {"pass: 1979-05-27T07:32:00, 1979-05-27 07:32:00.999999", + "fail: 1979-05-27T7:32:00, 1979-05-27 17:32"})); + } + if('0' <= c && c <= '9') + { + return err(format_underline("bad time: missing T", + {{source_location(loc), "T or space required here"}}, + {"pass: 1979-05-27T07:32:00, 1979-05-27 07:32:00.999999", + "fail: 1979-05-27T7:32:00, 1979-05-27 7:32"})); + } + if(c == ' ' && std::next(loc.iter()) != loc.end() && + ('0' <= *std::next(loc.iter()) && *std::next(loc.iter())<= '9')) + { + loc.advance(); + return err(format_underline("bad time: should be HH:MM:SS.subsec", + {{source_location(loc), "HH:MM:SS.subsec"}}, + {"pass: 1979-05-27T07:32:00, 1979-05-27 07:32:00.999999", + "fail: 1979-05-27T7:32:00, 1979-05-27 7:32"})); + } + } + return ok(value_t::local_date); + } + loc.reset(l.iter()); + + if(lex_local_time::invoke(loc)) {return ok(value_t::local_time);} + loc.reset(l.iter()); + + if(lex_float::invoke(loc)) + { + if(loc.iter() != loc.end() && *loc.iter() == '_') + { + return err(format_underline("bad float: `_` should be surrounded by digits", + {{source_location(loc), "here"}}, + {"pass: +1.0, -2e-2, 3.141_592_653_589, inf, nan", + "fail: .0, 1., _1.0, 1.0_, 1_.0, 1.0__0"})); + } + return ok(value_t::floating); + } + loc.reset(l.iter()); + + if(lex_integer::invoke(loc)) + { + if(loc.iter() != loc.end()) + { + const auto c = *loc.iter(); + if(c == '_') + { + return err(format_underline("bad integer: `_` should be surrounded by digits", + {{source_location(loc), "here"}}, + {"pass: -42, 1_000, 1_2_3_4_5, 0xC0FFEE, 0b0010, 0o755", + "fail: 1__000, 0123"})); + } + if('0' <= c && c <= '9') + { + // leading zero. point '0' + loc.retrace(); + return err(format_underline("bad integer: leading zero", + {{source_location(loc), "here"}}, + {"pass: -42, 1_000, 1_2_3_4_5, 0xC0FFEE, 0b0010, 0o755", + "fail: 1__000, 0123"})); + } + if(c == ':' || c == '-') + { + return err(format_underline("bad datetime: invalid format", + {{source_location(loc), "here"}}, + {"pass: 1979-05-27T07:32:00-07:00, 1979-05-27 07:32:00.999999Z", + "fail: 1979-05-27T7:32:00-7:00, 1979-05-27 7:32-00:30"})); + } + if(c == '.' || c == 'e' || c == 'E') + { + return err(format_underline("bad float: invalid format", + {{source_location(loc), "here"}}, + {"pass: +1.0, -2e-2, 3.141_592_653_589, inf, nan", + "fail: .0, 1., _1.0, 1.0_, 1_.0, 1.0__0"})); + } + } + return ok(value_t::integer); + } + if(loc.iter() != loc.end() && *loc.iter() == '.') + { + return err(format_underline("bad float: invalid format", + {{source_location(loc), "integer part required before this"}}, + {"pass: +1.0, -2e-2, 3.141_592_653_589, inf, nan", + "fail: .0, 1., _1.0, 1.0_, 1_.0, 1.0__0"})); + } + if(loc.iter() != loc.end() && *loc.iter() == '_') + { + return err(format_underline("bad number: `_` should be surrounded by digits", + {{source_location(loc), "`_` is not surrounded by digits"}}, + {"pass: -42, 1_000, 1_2_3_4_5, 0xC0FFEE, 0b0010, 0o755", + "fail: 1__000, 0123"})); + } + return err(format_underline("bad format: unknown value appeared", + {{source_location(loc), "here"}})); +} + +inline result guess_value_type(const location& loc) +{ + switch(*loc.iter()) + { + case '"' : {return ok(value_t::string); } + case '\'': {return ok(value_t::string); } + case 't' : {return ok(value_t::boolean); } + case 'f' : {return ok(value_t::boolean); } + case '[' : {return ok(value_t::array); } + case '{' : {return ok(value_t::table); } + case 'i' : {return ok(value_t::floating);} // inf. + case 'n' : {return ok(value_t::floating);} // nan. + default : {return guess_number_type(loc);} + } +} + +template +result +parse_value_helper(result, std::string> rslt) +{ + if(rslt.is_ok()) + { + auto comments = rslt.as_ok().second.comments(); + return ok(Value(std::move(rslt.as_ok()), std::move(comments))); + } + else + { + return err(std::move(rslt.as_err())); + } +} + +template +result parse_value(location& loc) +{ + const auto first = loc.iter(); + if(first == loc.end()) + { + return err(format_underline("toml::parse_value: input is empty", + {{source_location(loc), ""}})); + } + + const auto type = guess_value_type(loc); + if(!type) + { + return err(type.unwrap_err()); + } + + switch(type.unwrap()) + { + case value_t::boolean : {return parse_value_helper(parse_boolean(loc) );} + case value_t::integer : {return parse_value_helper(parse_integer(loc) );} + case value_t::floating : {return parse_value_helper(parse_floating(loc) );} + case value_t::string : {return parse_value_helper(parse_string(loc) );} + case value_t::offset_datetime: {return parse_value_helper(parse_offset_datetime(loc) );} + case value_t::local_datetime : {return parse_value_helper(parse_local_datetime(loc) );} + case value_t::local_date : {return parse_value_helper(parse_local_date(loc) );} + case value_t::local_time : {return parse_value_helper(parse_local_time(loc) );} + case value_t::array : {return parse_value_helper(parse_array(loc) );} + case value_t::table : {return parse_value_helper(parse_inline_table(loc));} + default: + { + const auto msg = format_underline("toml::parse_value: " + "unknown token appeared", {{source_location(loc), "unknown"}}); + loc.reset(first); + return err(msg); + } + } +} + +inline result, region>, std::string> +parse_table_key(location& loc) +{ + if(auto token = lex_std_table::invoke(loc)) + { + location inner_loc(loc.name(), token.unwrap().str()); + + const auto open = lex_std_table_open::invoke(inner_loc); + if(!open || inner_loc.iter() == inner_loc.end()) + { + throw internal_error(format_underline( + "toml::parse_table_key: no `[`", + {{source_location(inner_loc), "should be `[`"}}), + source_location(inner_loc)); + } + // to skip [ a . b . c ] + // ^----------- this whitespace + lex_ws::invoke(inner_loc); + const auto keys = parse_key(inner_loc); + if(!keys) + { + throw internal_error(format_underline( + "toml::parse_table_key: invalid key", + {{source_location(inner_loc), "not key"}}), + source_location(inner_loc)); + } + // to skip [ a . b . c ] + // ^-- this whitespace + lex_ws::invoke(inner_loc); + const auto close = lex_std_table_close::invoke(inner_loc); + if(!close) + { + throw internal_error(format_underline( + "toml::parse_table_key: no `]`", + {{source_location(inner_loc), "should be `]`"}}), + source_location(inner_loc)); + } + + // after [table.key], newline or EOF(empty table) required. + if(loc.iter() != loc.end()) + { + using lex_newline_after_table_key = + sequence, maybe, lex_newline>; + const auto nl = lex_newline_after_table_key::invoke(loc); + if(!nl) + { + throw syntax_error(format_underline( + "toml::parse_table_key: newline required after [table.key]", + {{source_location(loc), "expected newline"}}), + source_location(loc)); + } + } + return ok(std::make_pair(keys.unwrap().first, token.unwrap())); + } + else + { + return err(format_underline("toml::parse_table_key: " + "not a valid table key", {{source_location(loc), "here"}})); + } +} + +inline result, region>, std::string> +parse_array_table_key(location& loc) +{ + if(auto token = lex_array_table::invoke(loc)) + { + location inner_loc(loc.name(), token.unwrap().str()); + + const auto open = lex_array_table_open::invoke(inner_loc); + if(!open || inner_loc.iter() == inner_loc.end()) + { + throw internal_error(format_underline( + "toml::parse_array_table_key: no `[[`", + {{source_location(inner_loc), "should be `[[`"}}), + source_location(inner_loc)); + } + lex_ws::invoke(inner_loc); + const auto keys = parse_key(inner_loc); + if(!keys) + { + throw internal_error(format_underline( + "toml::parse_array_table_key: invalid key", + {{source_location(inner_loc), "not a key"}}), + source_location(inner_loc)); + } + lex_ws::invoke(inner_loc); + const auto close = lex_array_table_close::invoke(inner_loc); + if(!close) + { + throw internal_error(format_underline( + "toml::parse_table_key: no `]]`", + {{source_location(inner_loc), "should be `]]`"}}), + source_location(inner_loc)); + } + + // after [[table.key]], newline or EOF(empty table) required. + if(loc.iter() != loc.end()) + { + using lex_newline_after_table_key = + sequence, maybe, lex_newline>; + const auto nl = lex_newline_after_table_key::invoke(loc); + if(!nl) + { + throw syntax_error(format_underline("toml::" + "parse_array_table_key: newline required after [[table.key]]", + {{source_location(loc), "expected newline"}}), + source_location(loc)); + } + } + return ok(std::make_pair(keys.unwrap().first, token.unwrap())); + } + else + { + return err(format_underline("toml::parse_array_table_key: " + "not a valid table key", {{source_location(loc), "here"}})); + } +} + +// parse table body (key-value pairs until the iter hits the next [tablekey]) +template +result +parse_ml_table(location& loc) +{ + using value_type = Value; + using table_type = typename value_type::table_type; + + const auto first = loc.iter(); + if(first == loc.end()) + { + return ok(table_type{}); + } + + // XXX at lest one newline is needed. + using skip_line = repeat< + sequence, maybe, lex_newline>, at_least<1>>; + skip_line::invoke(loc); + lex_ws::invoke(loc); + + table_type tab; + while(loc.iter() != loc.end()) + { + lex_ws::invoke(loc); + const auto before = loc.iter(); + if(const auto tmp = parse_array_table_key(loc)) // next table found + { + loc.reset(before); + return ok(tab); + } + if(const auto tmp = parse_table_key(loc)) // next table found + { + loc.reset(before); + return ok(tab); + } + + if(const auto kv = parse_key_value_pair(loc)) + { + const auto& kvpair = kv.unwrap(); + const std::vector& keys = kvpair.first.first; + const auto& key_reg = kvpair.first.second; + const value_type& val = kvpair.second; + const auto inserted = + insert_nested_key(tab, val, keys.begin(), keys.end(), key_reg); + if(!inserted) + { + return err(inserted.unwrap_err()); + } + } + else + { + return err(kv.unwrap_err()); + } + + // comment lines are skipped by the above function call. + // However, since the `skip_line` requires at least 1 newline, it fails + // if the file ends with ws and/or comment without newline. + // `skip_line` matches `ws? + comment? + newline`, not `ws` or `comment` + // itself. To skip the last ws and/or comment, call lexers. + // It does not matter if these fails, so the return value is discarded. + lex_ws::invoke(loc); + lex_comment::invoke(loc); + + // skip_line is (whitespace? comment? newline)_{1,}. multiple empty lines + // and comments after the last key-value pairs are allowed. + const auto newline = skip_line::invoke(loc); + if(!newline && loc.iter() != loc.end()) + { + const auto before2 = loc.iter(); + lex_ws::invoke(loc); // skip whitespace + const auto msg = format_underline("toml::parse_table: " + "invalid line format", {{source_location(loc), concat_to_string( + "expected newline, but got '", show_char(*loc.iter()), "'.")}}); + loc.reset(before2); + return err(msg); + } + + // the skip_lines only matches with lines that includes newline. + // to skip the last line that includes comment and/or whitespace + // but no newline, call them one more time. + lex_ws::invoke(loc); + lex_comment::invoke(loc); + } + return ok(tab); +} + +template +result parse_toml_file(location& loc) +{ + using value_type = Value; + using table_type = typename value_type::table_type; + + const auto first = loc.iter(); + if(first == loc.end()) + { + // For empty files, return an empty table with an empty region (zero-length). + // Without the region, error messages would miss the filename. + return ok(value_type(table_type{}, region(loc, first, first), {})); + } + + // put the first line as a region of a file + // Here first != loc.end(), so taking std::next is okay + const region file(loc, first, std::next(loc.iter())); + + // The first successive comments that are separated from the first value + // by an empty line are for a file itself. + // ```toml + // # this is a comment for a file. + // + // key = "the first value" + // ``` + // ```toml + // # this is a comment for "the first value". + // key = "the first value" + // ``` + std::vector comments; + using lex_first_comments = sequence< + repeat, lex_comment, lex_newline>, at_least<1>>, + sequence, lex_newline> + >; + if(const auto token = lex_first_comments::invoke(loc)) + { + location inner_loc(loc.name(), token.unwrap().str()); + while(inner_loc.iter() != inner_loc.end()) + { + maybe::invoke(inner_loc); // remove ws if exists + if(lex_newline::invoke(inner_loc)) + { + assert(inner_loc.iter() == inner_loc.end()); + break; // empty line found. + } + auto com = lex_comment::invoke(inner_loc).unwrap().str(); + com.erase(com.begin()); // remove # sign + comments.push_back(std::move(com)); + lex_newline::invoke(inner_loc); + } + } + + table_type data; + // root object is also a table, but without [tablename] + if(const auto tab = parse_ml_table(loc)) + { + data = std::move(tab.unwrap()); + } + else // failed (empty table is regarded as success in parse_ml_table) + { + return err(tab.unwrap_err()); + } + while(loc.iter() != loc.end()) + { + // here, the region of [table] is regarded as the table-key because + // the table body is normally too big and it is not so informative + // if the first key-value pair of the table is shown in the error + // message. + if(const auto tabkey = parse_array_table_key(loc)) + { + const auto tab = parse_ml_table(loc); + if(!tab){return err(tab.unwrap_err());} + + const auto& tk = tabkey.unwrap(); + const auto& keys = tk.first; + const auto& reg = tk.second; + + const auto inserted = insert_nested_key(data, + value_type(tab.unwrap(), reg, reg.comments()), + keys.begin(), keys.end(), reg, + /*is_array_of_table=*/ true); + if(!inserted) {return err(inserted.unwrap_err());} + + continue; + } + if(const auto tabkey = parse_table_key(loc)) + { + const auto tab = parse_ml_table(loc); + if(!tab){return err(tab.unwrap_err());} + + const auto& tk = tabkey.unwrap(); + const auto& keys = tk.first; + const auto& reg = tk.second; + + const auto inserted = insert_nested_key(data, + value_type(tab.unwrap(), reg, reg.comments()), + keys.begin(), keys.end(), reg); + if(!inserted) {return err(inserted.unwrap_err());} + + continue; + } + return err(format_underline("toml::parse_toml_file: " + "unknown line appeared", {{source_location(loc), "unknown format"}})); + } + + return ok(Value(std::move(data), file, comments)); +} + +} // detail + +template class Table = std::unordered_map, + template class Array = std::vector> +basic_value +parse(std::istream& is, const std::string& fname = "unknown file") +{ + using value_type = basic_value; + + const auto beg = is.tellg(); + is.seekg(0, std::ios::end); + const auto end = is.tellg(); + const auto fsize = end - beg; + is.seekg(beg); + + // read whole file as a sequence of char + assert(fsize >= 0); + std::vector letters(static_cast(fsize)); + is.read(letters.data(), fsize); + + // append LF. + // Although TOML does not require LF at the EOF, to make parsing logic + // simpler, we "normalize" the content by adding LF if it does not exist. + // It also checks if the last char is CR, to avoid changing the meaning. + // This is not the *best* way to deal with the last character, but is a + // simple and quick fix. + if(!letters.empty() && letters.back() != '\n' && letters.back() != '\r') + { + letters.push_back('\n'); + } + + detail::location loc(std::move(fname), std::move(letters)); + + // skip BOM if exists. + // XXX component of BOM (like 0xEF) exceeds the representable range of + // signed char, so on some (actually, most) of the environment, these cannot + // be compared to char. However, since we are always out of luck, we need to + // check our chars are equivalent to BOM. To do this, first we need to + // convert char to unsigned char to guarantee the comparability. + if(loc.source()->size() >= 3) + { + std::array BOM; + std::memcpy(BOM.data(), loc.source()->data(), 3); + if(BOM[0] == 0xEF && BOM[1] == 0xBB && BOM[2] == 0xBF) + { + loc.advance(3); // BOM found. skip. + } + } + + const auto data = detail::parse_toml_file(loc); + if(!data) + { + throw syntax_error(data.unwrap_err(), source_location(loc)); + } + return data.unwrap(); +} + +template class Table = std::unordered_map, + template class Array = std::vector> +basic_value parse(const std::string& fname) +{ + std::ifstream ifs(fname.c_str(), std::ios_base::binary); + if(!ifs.good()) + { + throw std::runtime_error("toml::parse: file open error -> " + fname); + } + return parse(ifs, fname); +} + +#ifdef TOML11_HAS_STD_FILESYSTEM +// This function just forwards `parse("filename.toml")` to std::string version +// to avoid the ambiguity in overload resolution. +// +// Both std::string and std::filesystem::path are convertible from const char*. +// Without this, both parse(std::string) and parse(std::filesystem::path) +// matches to parse("filename.toml"). This breaks the existing code. +// +// This function exactly matches to the invocation with c-string. +// So this function is preferred than others and the ambiguity disappears. +template class Table = std::unordered_map, + template class Array = std::vector> +basic_value parse(const char* fname) +{ + return parse(std::string(fname)); +} + +template class Table = std::unordered_map, + template class Array = std::vector> +basic_value parse(const std::filesystem::path& fpath) +{ + std::ifstream ifs(fpath, std::ios_base::binary); + if(!ifs.good()) + { + throw std::runtime_error("toml::parse: file open error -> " + + fpath.string()); + } + return parse(ifs, fpath.string()); +} +#endif // TOML11_HAS_STD_FILESYSTEM + +} // toml +#endif// TOML11_PARSER_HPP diff --git a/src/toml11/toml/region.hpp b/src/toml11/toml/region.hpp new file mode 100644 index 000000000..2e01e51d0 --- /dev/null +++ b/src/toml11/toml/region.hpp @@ -0,0 +1,417 @@ +// Copyright Toru Niina 2017. +// Distributed under the MIT License. +#ifndef TOML11_REGION_HPP +#define TOML11_REGION_HPP +#include +#include +#include +#include +#include +#include +#include +#include "color.hpp" + +namespace toml +{ +namespace detail +{ + +// helper function to avoid std::string(0, 'c') or std::string(iter, iter) +template +std::string make_string(Iterator first, Iterator last) +{ + if(first == last) {return "";} + return std::string(first, last); +} +inline std::string make_string(std::size_t len, char c) +{ + if(len == 0) {return "";} + return std::string(len, c); +} + +// region_base is a base class of location and region that are defined below. +// it will be used to generate better error messages. +struct region_base +{ + region_base() = default; + virtual ~region_base() = default; + region_base(const region_base&) = default; + region_base(region_base&& ) = default; + region_base& operator=(const region_base&) = default; + region_base& operator=(region_base&& ) = default; + + virtual bool is_ok() const noexcept {return false;} + virtual char front() const noexcept {return '\0';} + + virtual std::string str() const {return std::string("unknown region");} + virtual std::string name() const {return std::string("unknown file");} + virtual std::string line() const {return std::string("unknown line");} + virtual std::string line_num() const {return std::string("?");} + + // length of the region + virtual std::size_t size() const noexcept {return 0;} + // number of characters in the line before the region + virtual std::size_t before() const noexcept {return 0;} + // number of characters in the line after the region + virtual std::size_t after() const noexcept {return 0;} + + virtual std::vector comments() const {return {};} + // ```toml + // # comment_before + // key = "value" # comment_inline + // ``` +}; + +// location represents a position in a container, which contains a file content. +// it can be considered as a region that contains only one character. +// +// it contains pointer to the file content and iterator that points the current +// location. +struct location final : public region_base +{ + using const_iterator = typename std::vector::const_iterator; + using difference_type = typename const_iterator::difference_type; + using source_ptr = std::shared_ptr>; + + location(std::string source_name, std::vector cont) + : source_(std::make_shared>(std::move(cont))), + line_number_(1), source_name_(std::move(source_name)), iter_(source_->cbegin()) + {} + location(std::string source_name, const std::string& cont) + : source_(std::make_shared>(cont.begin(), cont.end())), + line_number_(1), source_name_(std::move(source_name)), iter_(source_->cbegin()) + {} + + location(const location&) = default; + location(location&&) = default; + location& operator=(const location&) = default; + location& operator=(location&&) = default; + ~location() = default; + + bool is_ok() const noexcept override {return static_cast(source_);} + char front() const noexcept override {return *iter_;} + + // this const prohibits codes like `++(loc.iter())`. + const const_iterator iter() const noexcept {return iter_;} + + const_iterator begin() const noexcept {return source_->cbegin();} + const_iterator end() const noexcept {return source_->cend();} + + // XXX `location::line_num()` used to be implemented using `std::count` to + // count a number of '\n'. But with a long toml file (typically, 10k lines), + // it becomes intolerably slow because each time it generates error messages, + // it counts '\n' from thousands of characters. To workaround it, I decided + // to introduce `location::line_number_` member variable and synchronize it + // to the location changes the point to look. So an overload of `iter()` + // which returns mutable reference is removed and `advance()`, `retrace()` + // and `reset()` is added. + void advance(difference_type n = 1) noexcept + { + this->line_number_ += static_cast( + std::count(this->iter_, std::next(this->iter_, n), '\n')); + this->iter_ += n; + return; + } + void retrace(difference_type n = 1) noexcept + { + this->line_number_ -= static_cast( + std::count(std::prev(this->iter_, n), this->iter_, '\n')); + this->iter_ -= n; + return; + } + void reset(const_iterator rollback) noexcept + { + // since c++11, std::distance works in both ways for random-access + // iterators and returns a negative value if `first > last`. + if(0 <= std::distance(rollback, this->iter_)) // rollback < iter + { + this->line_number_ -= static_cast( + std::count(rollback, this->iter_, '\n')); + } + else // iter < rollback [[unlikely]] + { + this->line_number_ += static_cast( + std::count(this->iter_, rollback, '\n')); + } + this->iter_ = rollback; + return; + } + + std::string str() const override {return make_string(1, *this->iter());} + std::string name() const override {return source_name_;} + + std::string line_num() const override + { + return std::to_string(this->line_number_); + } + + std::string line() const override + { + return make_string(this->line_begin(), this->line_end()); + } + + const_iterator line_begin() const noexcept + { + using reverse_iterator = std::reverse_iterator; + return std::find(reverse_iterator(this->iter()), + reverse_iterator(this->begin()), '\n').base(); + } + const_iterator line_end() const noexcept + { + return std::find(this->iter(), this->end(), '\n'); + } + + // location is always points a character. so the size is 1. + std::size_t size() const noexcept override + { + return 1u; + } + std::size_t before() const noexcept override + { + const auto sz = std::distance(this->line_begin(), this->iter()); + assert(sz >= 0); + return static_cast(sz); + } + std::size_t after() const noexcept override + { + const auto sz = std::distance(this->iter(), this->line_end()); + assert(sz >= 0); + return static_cast(sz); + } + + source_ptr const& source() const& noexcept {return source_;} + source_ptr&& source() && noexcept {return std::move(source_);} + + private: + + source_ptr source_; + std::size_t line_number_; + std::string source_name_; + const_iterator iter_; +}; + +// region represents a range in a container, which contains a file content. +// +// it contains pointer to the file content and iterator that points the first +// and last location. +struct region final : public region_base +{ + using const_iterator = typename std::vector::const_iterator; + using source_ptr = std::shared_ptr>; + + // delete default constructor. source_ never be null. + region() = delete; + + explicit region(const location& loc) + : source_(loc.source()), source_name_(loc.name()), + first_(loc.iter()), last_(loc.iter()) + {} + explicit region(location&& loc) + : source_(loc.source()), source_name_(loc.name()), + first_(loc.iter()), last_(loc.iter()) + {} + + region(const location& loc, const_iterator f, const_iterator l) + : source_(loc.source()), source_name_(loc.name()), first_(f), last_(l) + {} + region(location&& loc, const_iterator f, const_iterator l) + : source_(loc.source()), source_name_(loc.name()), first_(f), last_(l) + {} + + region(const region&) = default; + region(region&&) = default; + region& operator=(const region&) = default; + region& operator=(region&&) = default; + ~region() = default; + + region& operator+=(const region& other) + { + // different regions cannot be concatenated + assert(this->begin() == other.begin() && this->end() == other.end() && + this->last_ == other.first_); + + this->last_ = other.last_; + return *this; + } + + bool is_ok() const noexcept override {return static_cast(source_);} + char front() const noexcept override {return *first_;} + + std::string str() const override {return make_string(first_, last_);} + std::string line() const override + { + if(this->contain_newline()) + { + return make_string(this->line_begin(), + std::find(this->line_begin(), this->last(), '\n')); + } + return make_string(this->line_begin(), this->line_end()); + } + std::string line_num() const override + { + return std::to_string(1 + std::count(this->begin(), this->first(), '\n')); + } + + std::size_t size() const noexcept override + { + const auto sz = std::distance(first_, last_); + assert(sz >= 0); + return static_cast(sz); + } + std::size_t before() const noexcept override + { + const auto sz = std::distance(this->line_begin(), this->first()); + assert(sz >= 0); + return static_cast(sz); + } + std::size_t after() const noexcept override + { + const auto sz = std::distance(this->last(), this->line_end()); + assert(sz >= 0); + return static_cast(sz); + } + + bool contain_newline() const noexcept + { + return std::find(this->first(), this->last(), '\n') != this->last(); + } + + const_iterator line_begin() const noexcept + { + using reverse_iterator = std::reverse_iterator; + return std::find(reverse_iterator(this->first()), + reverse_iterator(this->begin()), '\n').base(); + } + const_iterator line_end() const noexcept + { + return std::find(this->last(), this->end(), '\n'); + } + + const_iterator begin() const noexcept {return source_->cbegin();} + const_iterator end() const noexcept {return source_->cend();} + const_iterator first() const noexcept {return first_;} + const_iterator last() const noexcept {return last_;} + + source_ptr const& source() const& noexcept {return source_;} + source_ptr&& source() && noexcept {return std::move(source_);} + + std::string name() const override {return source_name_;} + + std::vector comments() const override + { + // assuming the current region (`*this`) points a value. + // ```toml + // a = "value" + // ^^^^^^^- this region + // ``` + using rev_iter = std::reverse_iterator; + + std::vector com{}; + { + // find comments just before the current region. + // ```toml + // # this should be collected. + // # this also. + // a = value # not this. + // ``` + + // # this is a comment for `a`, not array elements. + // a = [1, 2, 3, 4, 5] + if(this->first() == std::find_if(this->line_begin(), this->first(), + [](const char c) noexcept -> bool {return c == '[' || c == '{';})) + { + auto iter = this->line_begin(); // points the first character + while(iter != this->begin()) + { + iter = std::prev(iter); + + // range [line_start, iter) represents the previous line + const auto line_start = std::find( + rev_iter(iter), rev_iter(this->begin()), '\n').base(); + const auto comment_found = std::find(line_start, iter, '#'); + if(comment_found == iter) + { + break; // comment not found. + } + + // exclude the following case. + // > a = "foo" # comment // <-- this is not a comment for b but a. + // > b = "current value" + if(std::all_of(line_start, comment_found, + [](const char c) noexcept -> bool { + return c == ' ' || c == '\t'; + })) + { + // unwrap the first '#' by std::next. + auto s = make_string(std::next(comment_found), iter); + if(!s.empty() && s.back() == '\r') {s.pop_back();} + com.push_back(std::move(s)); + } + else + { + break; + } + iter = line_start; + } + } + } + + if(com.size() > 1) + { + std::reverse(com.begin(), com.end()); + } + + { + // find comments just after the current region. + // ```toml + // # not this. + // a = value # this one. + // a = [ # not this (technically difficult) + // + // ] # and this. + // ``` + // The reason why it's difficult is that it requires parsing in the + // following case. + // ```toml + // a = [ 10 # this comment is for `10`. not for `a` but `a[0]`. + // # ... + // ] # this is apparently a comment for a. + // + // b = [ + // 3.14 ] # there is no way to add a comment to `3.14` currently. + // + // c = [ + // 3.14 # do this if you need a comment here. + // ] + // ``` + const auto comment_found = + std::find(this->last(), this->line_end(), '#'); + if(comment_found != this->line_end()) // '#' found + { + // table = {key = "value"} # what is this for? + // the above comment is not for "value", but {key="value"}. + if(comment_found == std::find_if(this->last(), comment_found, + [](const char c) noexcept -> bool { + return !(c == ' ' || c == '\t' || c == ','); + })) + { + // unwrap the first '#' by std::next. + auto s = make_string(std::next(comment_found), this->line_end()); + if(!s.empty() && s.back() == '\r') {s.pop_back();} + com.push_back(std::move(s)); + } + } + } + return com; + } + + private: + + source_ptr source_; + std::string source_name_; + const_iterator first_, last_; +}; + +} // detail +} // toml +#endif// TOML11_REGION_H diff --git a/src/toml11/toml/result.hpp b/src/toml11/toml/result.hpp new file mode 100644 index 000000000..77cd46c64 --- /dev/null +++ b/src/toml11/toml/result.hpp @@ -0,0 +1,717 @@ +// Copyright Toru Niina 2017. +// Distributed under the MIT License. +#ifndef TOML11_RESULT_HPP +#define TOML11_RESULT_HPP +#include "traits.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace toml +{ + +template +struct success +{ + using value_type = T; + value_type value; + + explicit success(const value_type& v) + noexcept(std::is_nothrow_copy_constructible::value) + : value(v) + {} + explicit success(value_type&& v) + noexcept(std::is_nothrow_move_constructible::value) + : value(std::move(v)) + {} + + template + explicit success(U&& v): value(std::forward(v)) {} + + template + explicit success(const success& v): value(v.value) {} + template + explicit success(success&& v): value(std::move(v.value)) {} + + ~success() = default; + success(const success&) = default; + success(success&&) = default; + success& operator=(const success&) = default; + success& operator=(success&&) = default; +}; + +template +struct failure +{ + using value_type = T; + value_type value; + + explicit failure(const value_type& v) + noexcept(std::is_nothrow_copy_constructible::value) + : value(v) + {} + explicit failure(value_type&& v) + noexcept(std::is_nothrow_move_constructible::value) + : value(std::move(v)) + {} + + template + explicit failure(U&& v): value(std::forward(v)) {} + + template + explicit failure(const failure& v): value(v.value) {} + template + explicit failure(failure&& v): value(std::move(v.value)) {} + + ~failure() = default; + failure(const failure&) = default; + failure(failure&&) = default; + failure& operator=(const failure&) = default; + failure& operator=(failure&&) = default; +}; + +template +success::type>::type> +ok(T&& v) +{ + return success< + typename std::remove_cv::type>::type + >(std::forward(v)); +} +template +failure::type>::type> +err(T&& v) +{ + return failure< + typename std::remove_cv::type>::type + >(std::forward(v)); +} + +inline success ok(const char* literal) +{ + return success(std::string(literal)); +} +inline failure err(const char* literal) +{ + return failure(std::string(literal)); +} + + +template +struct result +{ + using value_type = T; + using error_type = E; + using success_type = success; + using failure_type = failure; + + result(const success_type& s): is_ok_(true) + { + auto tmp = ::new(std::addressof(this->succ)) success_type(s); + assert(tmp == std::addressof(this->succ)); + (void)tmp; + } + result(const failure_type& f): is_ok_(false) + { + auto tmp = ::new(std::addressof(this->fail)) failure_type(f); + assert(tmp == std::addressof(this->fail)); + (void)tmp; + } + result(success_type&& s): is_ok_(true) + { + auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(s)); + assert(tmp == std::addressof(this->succ)); + (void)tmp; + } + result(failure_type&& f): is_ok_(false) + { + auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(f)); + assert(tmp == std::addressof(this->fail)); + (void)tmp; + } + + template + result(const success& s): is_ok_(true) + { + auto tmp = ::new(std::addressof(this->succ)) success_type(s.value); + assert(tmp == std::addressof(this->succ)); + (void)tmp; + } + template + result(const failure& f): is_ok_(false) + { + auto tmp = ::new(std::addressof(this->fail)) failure_type(f.value); + assert(tmp == std::addressof(this->fail)); + (void)tmp; + } + template + result(success&& s): is_ok_(true) + { + auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(s.value)); + assert(tmp == std::addressof(this->succ)); + (void)tmp; + } + template + result(failure&& f): is_ok_(false) + { + auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(f.value)); + assert(tmp == std::addressof(this->fail)); + (void)tmp; + } + + result& operator=(const success_type& s) + { + this->cleanup(); + this->is_ok_ = true; + auto tmp = ::new(std::addressof(this->succ)) success_type(s); + assert(tmp == std::addressof(this->succ)); + (void)tmp; + return *this; + } + result& operator=(const failure_type& f) + { + this->cleanup(); + this->is_ok_ = false; + auto tmp = ::new(std::addressof(this->fail)) failure_type(f); + assert(tmp == std::addressof(this->fail)); + (void)tmp; + return *this; + } + result& operator=(success_type&& s) + { + this->cleanup(); + this->is_ok_ = true; + auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(s)); + assert(tmp == std::addressof(this->succ)); + (void)tmp; + return *this; + } + result& operator=(failure_type&& f) + { + this->cleanup(); + this->is_ok_ = false; + auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(f)); + assert(tmp == std::addressof(this->fail)); + (void)tmp; + return *this; + } + + template + result& operator=(const success& s) + { + this->cleanup(); + this->is_ok_ = true; + auto tmp = ::new(std::addressof(this->succ)) success_type(s.value); + assert(tmp == std::addressof(this->succ)); + (void)tmp; + return *this; + } + template + result& operator=(const failure& f) + { + this->cleanup(); + this->is_ok_ = false; + auto tmp = ::new(std::addressof(this->fail)) failure_type(f.value); + assert(tmp == std::addressof(this->fail)); + (void)tmp; + return *this; + } + template + result& operator=(success&& s) + { + this->cleanup(); + this->is_ok_ = true; + auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(s.value)); + assert(tmp == std::addressof(this->succ)); + (void)tmp; + return *this; + } + template + result& operator=(failure&& f) + { + this->cleanup(); + this->is_ok_ = false; + auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(f.value)); + assert(tmp == std::addressof(this->fail)); + (void)tmp; + return *this; + } + + ~result() noexcept {this->cleanup();} + + result(const result& other): is_ok_(other.is_ok()) + { + if(other.is_ok()) + { + auto tmp = ::new(std::addressof(this->succ)) success_type(other.as_ok()); + assert(tmp == std::addressof(this->succ)); + (void)tmp; + } + else + { + auto tmp = ::new(std::addressof(this->fail)) failure_type(other.as_err()); + assert(tmp == std::addressof(this->fail)); + (void)tmp; + } + } + result(result&& other): is_ok_(other.is_ok()) + { + if(other.is_ok()) + { + auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(other.as_ok())); + assert(tmp == std::addressof(this->succ)); + (void)tmp; + } + else + { + auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(other.as_err())); + assert(tmp == std::addressof(this->fail)); + (void)tmp; + } + } + + template + result(const result& other): is_ok_(other.is_ok()) + { + if(other.is_ok()) + { + auto tmp = ::new(std::addressof(this->succ)) success_type(other.as_ok()); + assert(tmp == std::addressof(this->succ)); + (void)tmp; + } + else + { + auto tmp = ::new(std::addressof(this->fail)) failure_type(other.as_err()); + assert(tmp == std::addressof(this->fail)); + (void)tmp; + } + } + template + result(result&& other): is_ok_(other.is_ok()) + { + if(other.is_ok()) + { + auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(other.as_ok())); + assert(tmp == std::addressof(this->succ)); + (void)tmp; + } + else + { + auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(other.as_err())); + assert(tmp == std::addressof(this->fail)); + (void)tmp; + } + } + + result& operator=(const result& other) + { + this->cleanup(); + if(other.is_ok()) + { + auto tmp = ::new(std::addressof(this->succ)) success_type(other.as_ok()); + assert(tmp == std::addressof(this->succ)); + (void)tmp; + } + else + { + auto tmp = ::new(std::addressof(this->fail)) failure_type(other.as_err()); + assert(tmp == std::addressof(this->fail)); + (void)tmp; + } + is_ok_ = other.is_ok(); + return *this; + } + result& operator=(result&& other) + { + this->cleanup(); + if(other.is_ok()) + { + auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(other.as_ok())); + assert(tmp == std::addressof(this->succ)); + (void)tmp; + } + else + { + auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(other.as_err())); + assert(tmp == std::addressof(this->fail)); + (void)tmp; + } + is_ok_ = other.is_ok(); + return *this; + } + + template + result& operator=(const result& other) + { + this->cleanup(); + if(other.is_ok()) + { + auto tmp = ::new(std::addressof(this->succ)) success_type(other.as_ok()); + assert(tmp == std::addressof(this->succ)); + (void)tmp; + } + else + { + auto tmp = ::new(std::addressof(this->fail)) failure_type(other.as_err()); + assert(tmp == std::addressof(this->fail)); + (void)tmp; + } + is_ok_ = other.is_ok(); + return *this; + } + template + result& operator=(result&& other) + { + this->cleanup(); + if(other.is_ok()) + { + auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(other.as_ok())); + assert(tmp == std::addressof(this->succ)); + (void)tmp; + } + else + { + auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(other.as_err())); + assert(tmp == std::addressof(this->fail)); + (void)tmp; + } + is_ok_ = other.is_ok(); + return *this; + } + + bool is_ok() const noexcept {return is_ok_;} + bool is_err() const noexcept {return !is_ok_;} + + operator bool() const noexcept {return is_ok_;} + + value_type& unwrap() & + { + if(is_err()) + { + throw std::runtime_error("toml::result: bad unwrap: " + + format_error(this->as_err())); + } + return this->succ.value; + } + value_type const& unwrap() const& + { + if(is_err()) + { + throw std::runtime_error("toml::result: bad unwrap: " + + format_error(this->as_err())); + } + return this->succ.value; + } + value_type&& unwrap() && + { + if(is_err()) + { + throw std::runtime_error("toml::result: bad unwrap: " + + format_error(this->as_err())); + } + return std::move(this->succ.value); + } + + value_type& unwrap_or(value_type& opt) & + { + if(is_err()) {return opt;} + return this->succ.value; + } + value_type const& unwrap_or(value_type const& opt) const& + { + if(is_err()) {return opt;} + return this->succ.value; + } + value_type unwrap_or(value_type opt) && + { + if(is_err()) {return opt;} + return this->succ.value; + } + + error_type& unwrap_err() & + { + if(is_ok()) {throw std::runtime_error("toml::result: bad unwrap_err");} + return this->fail.value; + } + error_type const& unwrap_err() const& + { + if(is_ok()) {throw std::runtime_error("toml::result: bad unwrap_err");} + return this->fail.value; + } + error_type&& unwrap_err() && + { + if(is_ok()) {throw std::runtime_error("toml::result: bad unwrap_err");} + return std::move(this->fail.value); + } + + value_type& as_ok() & noexcept {return this->succ.value;} + value_type const& as_ok() const& noexcept {return this->succ.value;} + value_type&& as_ok() && noexcept {return std::move(this->succ.value);} + + error_type& as_err() & noexcept {return this->fail.value;} + error_type const& as_err() const& noexcept {return this->fail.value;} + error_type&& as_err() && noexcept {return std::move(this->fail.value);} + + + // prerequisities + // F: T -> U + // retval: result + template + result, error_type> + map(F&& f) & + { + if(this->is_ok()){return ok(f(this->as_ok()));} + return err(this->as_err()); + } + template + result, error_type> + map(F&& f) const& + { + if(this->is_ok()){return ok(f(this->as_ok()));} + return err(this->as_err()); + } + template + result, error_type> + map(F&& f) && + { + if(this->is_ok()){return ok(f(std::move(this->as_ok())));} + return err(std::move(this->as_err())); + } + + // prerequisities + // F: E -> F + // retval: result + template + result> + map_err(F&& f) & + { + if(this->is_err()){return err(f(this->as_err()));} + return ok(this->as_ok()); + } + template + result> + map_err(F&& f) const& + { + if(this->is_err()){return err(f(this->as_err()));} + return ok(this->as_ok()); + } + template + result> + map_err(F&& f) && + { + if(this->is_err()){return err(f(std::move(this->as_err())));} + return ok(std::move(this->as_ok())); + } + + // prerequisities + // F: T -> U + // retval: U + template + detail::return_type_of_t + map_or_else(F&& f, U&& opt) & + { + if(this->is_err()){return std::forward(opt);} + return f(this->as_ok()); + } + template + detail::return_type_of_t + map_or_else(F&& f, U&& opt) const& + { + if(this->is_err()){return std::forward(opt);} + return f(this->as_ok()); + } + template + detail::return_type_of_t + map_or_else(F&& f, U&& opt) && + { + if(this->is_err()){return std::forward(opt);} + return f(std::move(this->as_ok())); + } + + // prerequisities + // F: E -> U + // retval: U + template + detail::return_type_of_t + map_err_or_else(F&& f, U&& opt) & + { + if(this->is_ok()){return std::forward(opt);} + return f(this->as_err()); + } + template + detail::return_type_of_t + map_err_or_else(F&& f, U&& opt) const& + { + if(this->is_ok()){return std::forward(opt);} + return f(this->as_err()); + } + template + detail::return_type_of_t + map_err_or_else(F&& f, U&& opt) && + { + if(this->is_ok()){return std::forward(opt);} + return f(std::move(this->as_err())); + } + + // prerequisities: + // F: func T -> U + // toml::err(error_type) should be convertible to U. + // normally, type U is another result and E is convertible to F + template + detail::return_type_of_t + and_then(F&& f) & + { + if(this->is_ok()){return f(this->as_ok());} + return err(this->as_err()); + } + template + detail::return_type_of_t + and_then(F&& f) const& + { + if(this->is_ok()){return f(this->as_ok());} + return err(this->as_err()); + } + template + detail::return_type_of_t + and_then(F&& f) && + { + if(this->is_ok()){return f(std::move(this->as_ok()));} + return err(std::move(this->as_err())); + } + + // prerequisities: + // F: func E -> U + // toml::ok(value_type) should be convertible to U. + // normally, type U is another result and T is convertible to S + template + detail::return_type_of_t + or_else(F&& f) & + { + if(this->is_err()){return f(this->as_err());} + return ok(this->as_ok()); + } + template + detail::return_type_of_t + or_else(F&& f) const& + { + if(this->is_err()){return f(this->as_err());} + return ok(this->as_ok()); + } + template + detail::return_type_of_t + or_else(F&& f) && + { + if(this->is_err()){return f(std::move(this->as_err()));} + return ok(std::move(this->as_ok())); + } + + // if *this is error, returns *this. otherwise, returns other. + result and_other(const result& other) const& + { + return this->is_err() ? *this : other; + } + result and_other(result&& other) && + { + return this->is_err() ? std::move(*this) : std::move(other); + } + + // if *this is okay, returns *this. otherwise, returns other. + result or_other(const result& other) const& + { + return this->is_ok() ? *this : other; + } + result or_other(result&& other) && + { + return this->is_ok() ? std::move(*this) : std::move(other); + } + + void swap(result& other) + { + result tmp(std::move(*this)); + *this = std::move(other); + other = std::move(tmp); + return ; + } + + private: + + static std::string format_error(std::exception const& excpt) + { + return std::string(excpt.what()); + } + template::value, std::nullptr_t>::type = nullptr> + static std::string format_error(U const& others) + { + std::ostringstream oss; oss << others; + return oss.str(); + } + + void cleanup() noexcept + { + if(this->is_ok_) {this->succ.~success_type();} + else {this->fail.~failure_type();} + return; + } + + private: + + bool is_ok_; + union + { + success_type succ; + failure_type fail; + }; +}; + +template +void swap(result& lhs, result& rhs) +{ + lhs.swap(rhs); + return; +} + +// this might be confusing because it eagerly evaluated, while in the other +// cases operator && and || are short-circuited. +// +// template +// inline result +// operator&&(const result& lhs, const result& rhs) noexcept +// { +// return lhs.is_ok() ? rhs : lhs; +// } +// +// template +// inline result +// operator||(const result& lhs, const result& rhs) noexcept +// { +// return lhs.is_ok() ? lhs : rhs; +// } + +// ---------------------------------------------------------------------------- +// re-use result as a optional with none_t + +namespace detail +{ +struct none_t {}; +inline bool operator==(const none_t&, const none_t&) noexcept {return true;} +inline bool operator!=(const none_t&, const none_t&) noexcept {return false;} +inline bool operator< (const none_t&, const none_t&) noexcept {return false;} +inline bool operator<=(const none_t&, const none_t&) noexcept {return true;} +inline bool operator> (const none_t&, const none_t&) noexcept {return false;} +inline bool operator>=(const none_t&, const none_t&) noexcept {return true;} +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const none_t&) +{ + os << "none"; + return os; +} +inline failure none() noexcept {return failure{none_t{}};} +} // detail +} // toml11 +#endif// TOML11_RESULT_H diff --git a/src/toml11/toml/serializer.hpp b/src/toml11/toml/serializer.hpp new file mode 100644 index 000000000..88ae775a8 --- /dev/null +++ b/src/toml11/toml/serializer.hpp @@ -0,0 +1,922 @@ +// Copyright Toru Niina 2019. +// Distributed under the MIT License. +#ifndef TOML11_SERIALIZER_HPP +#define TOML11_SERIALIZER_HPP +#include +#include + +#include + +#include "lexer.hpp" +#include "value.hpp" + +namespace toml +{ + +// This function serialize a key. It checks a string is a bare key and +// escapes special characters if the string is not compatible to a bare key. +// ```cpp +// std::string k("non.bare.key"); // the key itself includes `.`s. +// std::string formatted = toml::format_key(k); +// assert(formatted == "\"non.bare.key\""); +// ``` +// +// This function is exposed to make it easy to write a user-defined serializer. +// Since toml restricts characters available in a bare key, generally a string +// should be escaped. But checking whether a string needs to be surrounded by +// a `"` and escaping some special character is boring. +template +std::basic_string +format_key(const std::basic_string& k) +{ + if(k.empty()) + { + return std::string("\"\""); + } + + // check the key can be a bare (unquoted) key + detail::location loc(k, std::vector(k.begin(), k.end())); + detail::lex_unquoted_key::invoke(loc); + if(loc.iter() == loc.end()) + { + return k; // all the tokens are consumed. the key is unquoted-key. + } + + //if it includes special characters, then format it in a "quoted" key. + std::basic_string serialized("\""); + for(const char c : k) + { + switch(c) + { + case '\\': {serialized += "\\\\"; break;} + case '\"': {serialized += "\\\""; break;} + case '\b': {serialized += "\\b"; break;} + case '\t': {serialized += "\\t"; break;} + case '\f': {serialized += "\\f"; break;} + case '\n': {serialized += "\\n"; break;} + case '\r': {serialized += "\\r"; break;} + default : {serialized += c; break;} + } + } + serialized += "\""; + return serialized; +} + +template +std::basic_string +format_keys(const std::vector>& keys) +{ + if(keys.empty()) + { + return std::string("\"\""); + } + + std::basic_string serialized; + for(const auto& ky : keys) + { + serialized += format_key(ky); + serialized += charT('.'); + } + serialized.pop_back(); // remove the last dot '.' + return serialized; +} + +template +struct serializer +{ + static_assert(detail::is_basic_value::value, + "toml::serializer is for toml::value and its variants, " + "toml::basic_value<...>."); + + using value_type = Value; + using key_type = typename value_type::key_type ; + using comment_type = typename value_type::comment_type ; + using boolean_type = typename value_type::boolean_type ; + using integer_type = typename value_type::integer_type ; + using floating_type = typename value_type::floating_type ; + using string_type = typename value_type::string_type ; + using local_time_type = typename value_type::local_time_type ; + using local_date_type = typename value_type::local_date_type ; + using local_datetime_type = typename value_type::local_datetime_type ; + using offset_datetime_type = typename value_type::offset_datetime_type; + using array_type = typename value_type::array_type ; + using table_type = typename value_type::table_type ; + + serializer(const std::size_t w = 80u, + const int float_prec = std::numeric_limits::max_digits10, + const bool can_be_inlined = false, + const bool no_comment = false, + std::vector ks = {}, + const bool value_has_comment = false) + : can_be_inlined_(can_be_inlined), no_comment_(no_comment), + value_has_comment_(value_has_comment && !no_comment), + float_prec_(float_prec), width_(w), keys_(std::move(ks)) + {} + ~serializer() = default; + + std::string operator()(const boolean_type& b) const + { + return b ? "true" : "false"; + } + std::string operator()(const integer_type i) const + { + return std::to_string(i); + } + std::string operator()(const floating_type f) const + { + if(std::isnan(f)) + { + if(std::signbit(f)) + { + return std::string("-nan"); + } + else + { + return std::string("nan"); + } + } + else if(!std::isfinite(f)) + { + if(std::signbit(f)) + { + return std::string("-inf"); + } + else + { + return std::string("inf"); + } + } + + const auto fmt = "%.*g"; + const auto bsz = std::snprintf(nullptr, 0, fmt, this->float_prec_, f); + // +1 for null character(\0) + std::vector buf(static_cast(bsz + 1), '\0'); + std::snprintf(buf.data(), buf.size(), fmt, this->float_prec_, f); + + std::string token(buf.begin(), std::prev(buf.end())); + if(!token.empty() && token.back() == '.') // 1. => 1.0 + { + token += '0'; + } + + const auto e = std::find_if( + token.cbegin(), token.cend(), [](const char c) noexcept -> bool { + return c == 'e' || c == 'E'; + }); + const auto has_exponent = (token.cend() != e); + const auto has_fraction = (token.cend() != std::find( + token.cbegin(), token.cend(), '.')); + + if(!has_exponent && !has_fraction) + { + // the resulting value does not have any float specific part! + token += ".0"; + } + return token; + } + std::string operator()(const string_type& s) const + { + if(s.kind == string_t::basic) + { + if((std::find(s.str.cbegin(), s.str.cend(), '\n') != s.str.cend() || + std::find(s.str.cbegin(), s.str.cend(), '\"') != s.str.cend()) && + this->width_ != (std::numeric_limits::max)()) + { + // if linefeed or double-quote is contained, + // make it multiline basic string. + const auto escaped = this->escape_ml_basic_string(s.str); + std::string open("\"\"\""); + std::string close("\"\"\""); + if(escaped.find('\n') != std::string::npos || + this->width_ < escaped.size() + 6) + { + // if the string body contains newline or is enough long, + // add newlines after and before delimiters. + open += "\n"; + close = std::string("\\\n") + close; + } + return open + escaped + close; + } + + // no linefeed. try to make it oneline-string. + std::string oneline = this->escape_basic_string(s.str); + if(oneline.size() + 2 < width_ || width_ < 2) + { + const std::string quote("\""); + return quote + oneline + quote; + } + + // the line is too long compared to the specified width. + // split it into multiple lines. + std::string token("\"\"\"\n"); + while(!oneline.empty()) + { + if(oneline.size() < width_) + { + token += oneline; + oneline.clear(); + } + else if(oneline.at(width_-2) == '\\') + { + token += oneline.substr(0, width_-2); + token += "\\\n"; + oneline.erase(0, width_-2); + } + else + { + token += oneline.substr(0, width_-1); + token += "\\\n"; + oneline.erase(0, width_-1); + } + } + return token + std::string("\\\n\"\"\""); + } + else // the string `s` is literal-string. + { + if(std::find(s.str.cbegin(), s.str.cend(), '\n') != s.str.cend() || + std::find(s.str.cbegin(), s.str.cend(), '\'') != s.str.cend() ) + { + std::string open("'''"); + if(this->width_ + 6 < s.str.size()) + { + open += '\n'; // the first newline is ignored by TOML spec + } + const std::string close("'''"); + return open + s.str + close; + } + else + { + const std::string quote("'"); + return quote + s.str + quote; + } + } + } + + std::string operator()(const local_date_type& d) const + { + std::ostringstream oss; + oss << d; + return oss.str(); + } + std::string operator()(const local_time_type& t) const + { + std::ostringstream oss; + oss << t; + return oss.str(); + } + std::string operator()(const local_datetime_type& dt) const + { + std::ostringstream oss; + oss << dt; + return oss.str(); + } + std::string operator()(const offset_datetime_type& odt) const + { + std::ostringstream oss; + oss << odt; + return oss.str(); + } + + std::string operator()(const array_type& v) const + { + if(v.empty()) + { + return std::string("[]"); + } + if(this->is_array_of_tables(v)) + { + return make_array_of_tables(v); + } + + // not an array of tables. normal array. + // first, try to make it inline if none of the elements have a comment. + if( ! this->has_comment_inside(v)) + { + const auto inl = this->make_inline_array(v); + if(inl.size() < this->width_ && + std::find(inl.cbegin(), inl.cend(), '\n') == inl.cend()) + { + return inl; + } + } + + // if the length exceeds this->width_, print multiline array. + // key = [ + // # ... + // 42, + // ... + // ] + std::string token; + std::string current_line; + token += "[\n"; + for(const auto& item : v) + { + if( ! item.comments().empty() && !no_comment_) + { + // if comment exists, the element must be the only element in the line. + // e.g. the following is not allowed. + // ```toml + // array = [ + // # comment for what? + // 1, 2, 3, 4, 5 + // ] + // ``` + if(!current_line.empty()) + { + if(current_line.back() != '\n') + { + current_line += '\n'; + } + token += current_line; + current_line.clear(); + } + for(const auto& c : item.comments()) + { + token += '#'; + token += c; + token += '\n'; + } + token += toml::visit(*this, item); + if(!token.empty() && token.back() == '\n') {token.pop_back();} + token += ",\n"; + continue; + } + std::string next_elem; + if(item.is_table()) + { + serializer ser(*this); + ser.can_be_inlined_ = true; + ser.width_ = (std::numeric_limits::max)(); + next_elem += toml::visit(ser, item); + } + else + { + next_elem += toml::visit(*this, item); + } + + // comma before newline. + if(!next_elem.empty() && next_elem.back() == '\n') {next_elem.pop_back();} + + // if current line does not exceeds the width limit, continue. + if(current_line.size() + next_elem.size() + 1 < this->width_) + { + current_line += next_elem; + current_line += ','; + } + else if(current_line.empty()) + { + // if current line was empty, force put the next_elem because + // next_elem is not splittable + token += next_elem; + token += ",\n"; + // current_line is kept empty + } + else // reset current_line + { + assert(current_line.back() == ','); + token += current_line; + token += '\n'; + current_line = next_elem; + current_line += ','; + } + } + if(!current_line.empty()) + { + if(!current_line.empty() && current_line.back() != '\n') + { + current_line += '\n'; + } + token += current_line; + } + token += "]\n"; + return token; + } + + // templatize for any table-like container + std::string operator()(const table_type& v) const + { + // if an element has a comment, then it can't be inlined. + // table = {# how can we write a comment for this? key = "value"} + if(this->can_be_inlined_ && !(this->has_comment_inside(v))) + { + std::string token; + if(!this->keys_.empty()) + { + token += format_key(this->keys_.back()); + token += " = "; + } + token += this->make_inline_table(v); + if(token.size() < this->width_ && + token.end() == std::find(token.begin(), token.end(), '\n')) + { + return token; + } + } + + std::string token; + if(!keys_.empty()) + { + token += '['; + token += format_keys(keys_); + token += "]\n"; + } + token += this->make_multiline_table(v); + return token; + } + + private: + + std::string escape_basic_string(const std::string& s) const + { + //XXX assuming `s` is a valid utf-8 sequence. + std::string retval; + for(const char c : s) + { + switch(c) + { + case '\\': {retval += "\\\\"; break;} + case '\"': {retval += "\\\""; break;} + case '\b': {retval += "\\b"; break;} + case '\t': {retval += "\\t"; break;} + case '\f': {retval += "\\f"; break;} + case '\n': {retval += "\\n"; break;} + case '\r': {retval += "\\r"; break;} + default : + { + if((0x00 <= c && c <= 0x08) || (0x0A <= c && c <= 0x1F) || c == 0x7F) + { + retval += "\\u00"; + retval += char(48 + (c / 16)); + retval += char((c % 16 < 10 ? 48 : 55) + (c % 16)); + } + else + { + retval += c; + } + } + } + } + return retval; + } + + std::string escape_ml_basic_string(const std::string& s) const + { + std::string retval; + for(auto i=s.cbegin(), e=s.cend(); i!=e; ++i) + { + switch(*i) + { + case '\\': {retval += "\\\\"; break;} + // One or two consecutive "s are allowed. + // Later we will check there are no three consecutive "s. + // case '\"': {retval += "\\\""; break;} + case '\b': {retval += "\\b"; break;} + case '\t': {retval += "\\t"; break;} + case '\f': {retval += "\\f"; break;} + case '\n': {retval += "\n"; break;} + case '\r': + { + if(std::next(i) != e && *std::next(i) == '\n') + { + retval += "\r\n"; + ++i; + } + else + { + retval += "\\r"; + } + break; + } + default : + { + const auto c = *i; + if((0x00 <= c && c <= 0x08) || (0x0A <= c && c <= 0x1F) || c == 0x7F) + { + retval += "\\u00"; + retval += char(48 + (c / 16)); + retval += char((c % 16 < 10 ? 48 : 55) + (c % 16)); + } + else + { + retval += c; + } + } + + } + } + // Only 1 or 2 consecutive `"`s are allowed in multiline basic string. + // 3 consecutive `"`s are considered as a closing delimiter. + // We need to check if there are 3 or more consecutive `"`s and insert + // backslash to break them down into several short `"`s like the `str6` + // in the following example. + // ```toml + // str4 = """Here are two quotation marks: "". Simple enough.""" + // # str5 = """Here are three quotation marks: """.""" # INVALID + // str5 = """Here are three quotation marks: ""\".""" + // str6 = """Here are fifteen quotation marks: ""\"""\"""\"""\"""\".""" + // ``` + auto found_3_quotes = retval.find("\"\"\""); + while(found_3_quotes != std::string::npos) + { + retval.replace(found_3_quotes, 3, "\"\"\\\""); + found_3_quotes = retval.find("\"\"\""); + } + return retval; + } + + // if an element of a table or an array has a comment, it cannot be inlined. + bool has_comment_inside(const array_type& a) const noexcept + { + // if no_comment is set, comments would not be written. + if(this->no_comment_) {return false;} + + for(const auto& v : a) + { + if(!v.comments().empty()) {return true;} + } + return false; + } + bool has_comment_inside(const table_type& t) const noexcept + { + // if no_comment is set, comments would not be written. + if(this->no_comment_) {return false;} + + for(const auto& kv : t) + { + if(!kv.second.comments().empty()) {return true;} + } + return false; + } + + std::string make_inline_array(const array_type& v) const + { + assert(!has_comment_inside(v)); + std::string token; + token += '['; + bool is_first = true; + for(const auto& item : v) + { + if(is_first) {is_first = false;} else {token += ',';} + token += visit(serializer( + (std::numeric_limits::max)(), this->float_prec_, + /* inlined */ true, /*no comment*/ false, /*keys*/ {}, + /*has_comment*/ !item.comments().empty()), item); + } + token += ']'; + return token; + } + + std::string make_inline_table(const table_type& v) const + { + assert(!has_comment_inside(v)); + assert(this->can_be_inlined_); + std::string token; + token += '{'; + bool is_first = true; + for(const auto& kv : v) + { + // in inline tables, trailing comma is not allowed (toml-lang #569). + if(is_first) {is_first = false;} else {token += ',';} + token += format_key(kv.first); + token += '='; + token += visit(serializer( + (std::numeric_limits::max)(), this->float_prec_, + /* inlined */ true, /*no comment*/ false, /*keys*/ {}, + /*has_comment*/ !kv.second.comments().empty()), kv.second); + } + token += '}'; + return token; + } + + std::string make_multiline_table(const table_type& v) const + { + std::string token; + + // print non-table elements first. + // ```toml + // [foo] # a table we're writing now here + // key = "value" # <- non-table element, "key" + // # ... + // [foo.bar] # <- table element, "bar" + // ``` + // because after printing [foo.bar], the remaining non-table values will + // be assigned into [foo.bar], not [foo]. Those values should be printed + // earlier. + for(const auto& kv : v) + { + if(kv.second.is_table() || is_array_of_tables(kv.second)) + { + continue; + } + + token += write_comments(kv.second); + + const auto key_and_sep = format_key(kv.first) + " = "; + const auto residual_width = (this->width_ > key_and_sep.size()) ? + this->width_ - key_and_sep.size() : 0; + token += key_and_sep; + token += visit(serializer(residual_width, this->float_prec_, + /*can be inlined*/ true, /*no comment*/ false, /*keys*/ {}, + /*has_comment*/ !kv.second.comments().empty()), kv.second); + + if(token.back() != '\n') + { + token += '\n'; + } + } + + // normal tables / array of tables + + // after multiline table appeared, the other tables cannot be inline + // because the table would be assigned into the table. + // [foo] + // ... + // bar = {...} # <- bar will be a member of [foo]. + bool multiline_table_printed = false; + for(const auto& kv : v) + { + if(!kv.second.is_table() && !is_array_of_tables(kv.second)) + { + continue; // other stuff are already serialized. skip them. + } + + std::vector ks(this->keys_); + ks.push_back(kv.first); + + auto tmp = visit(serializer(this->width_, this->float_prec_, + !multiline_table_printed, this->no_comment_, ks, + /*has_comment*/ !kv.second.comments().empty()), kv.second); + + // If it is the first time to print a multi-line table, it would be + // helpful to separate normal key-value pair and subtables by a + // newline. + // (this checks if the current key-value pair contains newlines. + // but it is not perfect because multi-line string can also contain + // a newline. in such a case, an empty line will be written) TODO + if((!multiline_table_printed) && + std::find(tmp.cbegin(), tmp.cend(), '\n') != tmp.cend()) + { + multiline_table_printed = true; + token += '\n'; // separate key-value pairs and subtables + + token += write_comments(kv.second); + token += tmp; + + // care about recursive tables (all tables in each level prints + // newline and there will be a full of newlines) + if(tmp.substr(tmp.size() - 2, 2) != "\n\n" && + tmp.substr(tmp.size() - 4, 4) != "\r\n\r\n" ) + { + token += '\n'; + } + } + else + { + token += write_comments(kv.second); + token += tmp; + token += '\n'; + } + } + return token; + } + + std::string make_array_of_tables(const array_type& v) const + { + // if it's not inlined, we need to add `[[table.key]]`. + // but if it can be inlined, we can format it as the following. + // ``` + // table.key = [ + // {...}, + // # comment + // {...}, + // ] + // ``` + // This function checks if inlinization is possible or not, and then + // format the array-of-tables in a proper way. + // + // Note about comments: + // + // If the array itself has a comment (value_has_comment_ == true), we + // should try to make it inline. + // ```toml + // # comment about array + // array = [ + // # comment about table element + // {of = "table"} + // ] + // ``` + // If it is formatted as a multiline table, the two comments becomes + // indistinguishable. + // ```toml + // # comment about array + // # comment about table element + // [[array]] + // of = "table" + // ``` + // So we need to try to make it inline, and it force-inlines regardless + // of the line width limit. + // It may fail if the element of a table has comment. In that case, + // the array-of-tables will be formatted as a multiline table. + if(this->can_be_inlined_ || this->value_has_comment_) + { + std::string token; + if(!keys_.empty()) + { + token += format_key(keys_.back()); + token += " = "; + } + + bool failed = false; + token += "[\n"; + for(const auto& item : v) + { + // if an element of the table has a comment, the table + // cannot be inlined. + if(this->has_comment_inside(item.as_table())) + { + failed = true; + break; + } + // write comments for the table itself + token += write_comments(item); + + const auto t = this->make_inline_table(item.as_table()); + + if(t.size() + 1 > width_ || // +1 for the last comma {...}, + std::find(t.cbegin(), t.cend(), '\n') != t.cend()) + { + // if the value itself has a comment, ignore the line width limit + if( ! this->value_has_comment_) + { + failed = true; + break; + } + } + token += t; + token += ",\n"; + } + + if( ! failed) + { + token += "]\n"; + return token; + } + // if failed, serialize them as [[array.of.tables]]. + } + + std::string token; + for(const auto& item : v) + { + token += write_comments(item); + token += "[["; + token += format_keys(keys_); + token += "]]\n"; + token += this->make_multiline_table(item.as_table()); + } + return token; + } + + std::string write_comments(const value_type& v) const + { + std::string retval; + if(this->no_comment_) {return retval;} + + for(const auto& c : v.comments()) + { + retval += '#'; + retval += c; + retval += '\n'; + } + return retval; + } + + bool is_array_of_tables(const value_type& v) const + { + if(!v.is_array() || v.as_array().empty()) {return false;} + return is_array_of_tables(v.as_array()); + } + bool is_array_of_tables(const array_type& v) const + { + // Since TOML v0.5.0, heterogeneous arrays are allowed. So we need to + // check all the element in an array to check if the array is an array + // of tables. + return std::all_of(v.begin(), v.end(), [](const value_type& elem) { + return elem.is_table(); + }); + } + + private: + + bool can_be_inlined_; + bool no_comment_; + bool value_has_comment_; + int float_prec_; + std::size_t width_; + std::vector keys_; +}; + +template class M, template class V> +std::string +format(const basic_value& v, std::size_t w = 80u, + int fprec = std::numeric_limits::max_digits10, + bool no_comment = false, bool force_inline = false) +{ + using value_type = basic_value; + // if value is a table, it is considered to be a root object. + // the root object can't be an inline table. + if(v.is_table()) + { + std::ostringstream oss; + if(!v.comments().empty()) + { + oss << v.comments(); + oss << '\n'; // to split the file comment from the first element + } + const auto serialized = visit(serializer(w, fprec, false, no_comment), v); + oss << serialized; + return oss.str(); + } + return visit(serializer(w, fprec, force_inline), v); +} + +namespace detail +{ +template +int comment_index(std::basic_ostream&) +{ + static const int index = std::ios_base::xalloc(); + return index; +} +} // detail + +template +std::basic_ostream& +nocomment(std::basic_ostream& os) +{ + // by default, it is zero. and by default, it shows comments. + os.iword(detail::comment_index(os)) = 1; + return os; +} + +template +std::basic_ostream& +showcomment(std::basic_ostream& os) +{ + // by default, it is zero. and by default, it shows comments. + os.iword(detail::comment_index(os)) = 0; + return os; +} + +template class M, template class V> +std::basic_ostream& +operator<<(std::basic_ostream& os, const basic_value& v) +{ + using value_type = basic_value; + + // get status of std::setw(). + const auto w = static_cast(os.width()); + const int fprec = static_cast(os.precision()); + os.width(0); + + // by default, iword is initialized by 0. And by default, toml11 outputs + // comments. So `0` means showcomment. 1 means nocommnet. + const bool no_comment = (1 == os.iword(detail::comment_index(os))); + + if(!no_comment && v.is_table() && !v.comments().empty()) + { + os << v.comments(); + os << '\n'; // to split the file comment from the first element + } + // the root object can't be an inline table. so pass `false`. + const auto serialized = visit(serializer(w, fprec, no_comment, false), v); + os << serialized; + + // if v is a non-table value, and has only one comment, then + // put a comment just after a value. in the following way. + // + // ```toml + // key = "value" # comment. + // ``` + // + // Since the top-level toml object is a table, one who want to put a + // non-table toml value must use this in a following way. + // + // ```cpp + // toml::value v; + // std::cout << "user-defined-key = " << v << std::endl; + // ``` + // + // In this case, it is impossible to put comments before key-value pair. + // The only way to preserve comments is to put all of them after a value. + if(!no_comment && !v.is_table() && !v.comments().empty()) + { + os << " #"; + for(const auto& c : v.comments()) {os << c;} + } + return os; +} + +} // toml +#endif// TOML11_SERIALIZER_HPP diff --git a/src/toml11/toml/source_location.hpp b/src/toml11/toml/source_location.hpp new file mode 100644 index 000000000..fa175b5b4 --- /dev/null +++ b/src/toml11/toml/source_location.hpp @@ -0,0 +1,233 @@ +// Copyright Toru Niina 2019. +// Distributed under the MIT License. +#ifndef TOML11_SOURCE_LOCATION_HPP +#define TOML11_SOURCE_LOCATION_HPP +#include +#include + +#include "region.hpp" + +namespace toml +{ + +// A struct to contain location in a toml file. +// The interface imitates std::experimental::source_location, +// but not completely the same. +// +// It would be constructed by toml::value. It can be used to generate +// user-defined error messages. +// +// - std::uint_least32_t line() const noexcept +// - returns the line number where the region is on. +// - std::uint_least32_t column() const noexcept +// - returns the column number where the region starts. +// - std::uint_least32_t region() const noexcept +// - returns the size of the region. +// +// +-- line() +-- region of interest (region() == 9) +// v .---+---. +// 12 | value = "foo bar" +// ^ +// +-- column() +// +// - std::string const& file_name() const noexcept; +// - name of the file. +// - std::string const& line_str() const noexcept; +// - the whole line that contains the region of interest. +// +struct source_location +{ + public: + + source_location() + : line_num_(1), column_num_(1), region_size_(1), + file_name_("unknown file"), line_str_("") + {} + + explicit source_location(const detail::region_base* reg) + : line_num_(1), column_num_(1), region_size_(1), + file_name_("unknown file"), line_str_("") + { + if(reg) + { + if(reg->line_num() != detail::region_base().line_num()) + { + line_num_ = static_cast( + std::stoul(reg->line_num())); + } + column_num_ = static_cast(reg->before() + 1); + region_size_ = static_cast(reg->size()); + file_name_ = reg->name(); + line_str_ = reg->line(); + } + } + + explicit source_location(const detail::region& reg) + : line_num_(static_cast(std::stoul(reg.line_num()))), + column_num_(static_cast(reg.before() + 1)), + region_size_(static_cast(reg.size())), + file_name_(reg.name()), + line_str_ (reg.line()) + {} + explicit source_location(const detail::location& loc) + : line_num_(static_cast(std::stoul(loc.line_num()))), + column_num_(static_cast(loc.before() + 1)), + region_size_(static_cast(loc.size())), + file_name_(loc.name()), + line_str_ (loc.line()) + {} + + ~source_location() = default; + source_location(source_location const&) = default; + source_location(source_location &&) = default; + source_location& operator=(source_location const&) = default; + source_location& operator=(source_location &&) = default; + + std::uint_least32_t line() const noexcept {return line_num_;} + std::uint_least32_t column() const noexcept {return column_num_;} + std::uint_least32_t region() const noexcept {return region_size_;} + + std::string const& file_name() const noexcept {return file_name_;} + std::string const& line_str() const noexcept {return line_str_;} + + private: + + std::uint_least32_t line_num_; + std::uint_least32_t column_num_; + std::uint_least32_t region_size_; + std::string file_name_; + std::string line_str_; +}; + +namespace detail +{ + +// internal error message generation. +inline std::string format_underline(const std::string& message, + const std::vector>& loc_com, + const std::vector& helps = {}, + const bool colorize = TOML11_ERROR_MESSAGE_COLORIZED) +{ + std::size_t line_num_width = 0; + for(const auto& lc : loc_com) + { + std::uint_least32_t line = lc.first.line(); + std::size_t digit = 0; + while(line != 0) + { + line /= 10; + digit += 1; + } + line_num_width = (std::max)(line_num_width, digit); + } + // 1 is the minimum width + line_num_width = std::max(line_num_width, 1); + + std::ostringstream retval; + + if(colorize) + { + retval << color::colorize; // turn on ANSI color + } + + // XXX + // Here, before `colorize` support, it does not output `[error]` prefix + // automatically. So some user may output it manually and this change may + // duplicate the prefix. To avoid it, check the first 7 characters and + // if it is "[error]", it removes that part from the message shown. + if(message.size() > 7 && message.substr(0, 7) == "[error]") + { + retval << color::bold << color::red << "[error]" << color::reset + << color::bold << message.substr(7) << color::reset << '\n'; + } + else + { + retval << color::bold << color::red << "[error] " << color::reset + << color::bold << message << color::reset << '\n'; + } + + const auto format_one_location = [line_num_width] + (std::ostringstream& oss, + const source_location& loc, const std::string& comment) -> void + { + oss << ' ' << color::bold << color::blue + << std::setw(static_cast(line_num_width)) + << std::right << loc.line() << " | " << color::reset + << loc.line_str() << '\n'; + + oss << make_string(line_num_width + 1, ' ') + << color::bold << color::blue << " | " << color::reset + << make_string(loc.column()-1 /*1-origin*/, ' '); + + if(loc.region() == 1) + { + // invalid + // ^------ + oss << color::bold << color::red << "^---" << color::reset; + } + else + { + // invalid + // ~~~~~~~ + const auto underline_len = (std::min)( + static_cast(loc.region()), loc.line_str().size()); + oss << color::bold << color::red + << make_string(underline_len, '~') << color::reset; + } + oss << ' '; + oss << comment; + return; + }; + + assert(!loc_com.empty()); + + // --> example.toml + // | + retval << color::bold << color::blue << " --> " << color::reset + << loc_com.front().first.file_name() << '\n'; + retval << make_string(line_num_width + 1, ' ') + << color::bold << color::blue << " |\n" << color::reset; + // 1 | key value + // | ^--- missing = + format_one_location(retval, loc_com.front().first, loc_com.front().second); + + // process the rest of the locations + for(std::size_t i=1; i filename.toml" again + { + retval << color::bold << color::blue << " --> " << color::reset + << curr.first.file_name() << '\n'; + retval << make_string(line_num_width + 1, ' ') + << color::bold << color::blue << " |\n" << color::reset; + } + + format_one_location(retval, curr.first, curr.second); + } + + if(!helps.empty()) + { + retval << '\n'; + retval << make_string(line_num_width + 1, ' '); + retval << color::bold << color::blue << " |" << color::reset; + for(const auto& help : helps) + { + retval << color::bold << "\nHint: " << color::reset; + retval << help; + } + } + return retval.str(); +} + +} // detail +} // toml +#endif// TOML11_SOURCE_LOCATION_HPP diff --git a/src/toml11/toml/storage.hpp b/src/toml11/toml/storage.hpp new file mode 100644 index 000000000..202f9035f --- /dev/null +++ b/src/toml11/toml/storage.hpp @@ -0,0 +1,43 @@ +// Copyright Toru Niina 2017. +// Distributed under the MIT License. +#ifndef TOML11_STORAGE_HPP +#define TOML11_STORAGE_HPP +#include "utility.hpp" + +namespace toml +{ +namespace detail +{ + +// this contains pointer and deep-copy the content if copied. +// to avoid recursive pointer. +template +struct storage +{ + using value_type = T; + + explicit storage(value_type const& v): ptr(toml::make_unique(v)) {} + explicit storage(value_type&& v): ptr(toml::make_unique(std::move(v))) {} + ~storage() = default; + storage(const storage& rhs): ptr(toml::make_unique(*rhs.ptr)) {} + storage& operator=(const storage& rhs) + { + this->ptr = toml::make_unique(*rhs.ptr); + return *this; + } + storage(storage&&) = default; + storage& operator=(storage&&) = default; + + bool is_ok() const noexcept {return static_cast(ptr);} + + value_type& value() & noexcept {return *ptr;} + value_type const& value() const& noexcept {return *ptr;} + value_type&& value() && noexcept {return std::move(*ptr);} + + private: + std::unique_ptr ptr; +}; + +} // detail +} // toml +#endif// TOML11_STORAGE_HPP diff --git a/src/toml11/toml/string.hpp b/src/toml11/toml/string.hpp new file mode 100644 index 000000000..5136d8c56 --- /dev/null +++ b/src/toml11/toml/string.hpp @@ -0,0 +1,225 @@ +// Copyright Toru Niina 2017. +// Distributed under the MIT License. +#ifndef TOML11_STRING_HPP +#define TOML11_STRING_HPP +#include + +#include +#include + +#if __cplusplus >= 201703L +#if __has_include() +#define TOML11_USING_STRING_VIEW 1 +#include +#endif +#endif + +namespace toml +{ + +enum class string_t : std::uint8_t +{ + basic = 0, + literal = 1, +}; + +struct string +{ + string() = default; + ~string() = default; + string(const string& s) = default; + string(string&& s) = default; + string& operator=(const string& s) = default; + string& operator=(string&& s) = default; + + string(const std::string& s): kind(string_t::basic), str(s){} + string(const std::string& s, string_t k): kind(k), str(s){} + string(const char* s): kind(string_t::basic), str(s){} + string(const char* s, string_t k): kind(k), str(s){} + + string(std::string&& s): kind(string_t::basic), str(std::move(s)){} + string(std::string&& s, string_t k): kind(k), str(std::move(s)){} + + string& operator=(const std::string& s) + {kind = string_t::basic; str = s; return *this;} + string& operator=(std::string&& s) + {kind = string_t::basic; str = std::move(s); return *this;} + + operator std::string& () & noexcept {return str;} + operator std::string const& () const& noexcept {return str;} + operator std::string&& () && noexcept {return std::move(str);} + + string& operator+=(const char* rhs) {str += rhs; return *this;} + string& operator+=(const char rhs) {str += rhs; return *this;} + string& operator+=(const std::string& rhs) {str += rhs; return *this;} + string& operator+=(const string& rhs) {str += rhs.str; return *this;} + +#if defined(TOML11_USING_STRING_VIEW) && TOML11_USING_STRING_VIEW>0 + explicit string(std::string_view s): kind(string_t::basic), str(s){} + string(std::string_view s, string_t k): kind(k), str(s){} + + string& operator=(std::string_view s) + {kind = string_t::basic; str = s; return *this;} + + explicit operator std::string_view() const noexcept + {return std::string_view(str);} + + string& operator+=(const std::string_view& rhs) {str += rhs; return *this;} +#endif + + string_t kind; + std::string str; +}; + +inline bool operator==(const string& lhs, const string& rhs) +{ + return lhs.kind == rhs.kind && lhs.str == rhs.str; +} +inline bool operator!=(const string& lhs, const string& rhs) +{ + return !(lhs == rhs); +} +inline bool operator<(const string& lhs, const string& rhs) +{ + return (lhs.kind == rhs.kind) ? (lhs.str < rhs.str) : (lhs.kind < rhs.kind); +} +inline bool operator>(const string& lhs, const string& rhs) +{ + return rhs < lhs; +} +inline bool operator<=(const string& lhs, const string& rhs) +{ + return !(rhs < lhs); +} +inline bool operator>=(const string& lhs, const string& rhs) +{ + return !(lhs < rhs); +} + +inline bool +operator==(const string& lhs, const std::string& rhs) {return lhs.str == rhs;} +inline bool +operator!=(const string& lhs, const std::string& rhs) {return lhs.str != rhs;} +inline bool +operator< (const string& lhs, const std::string& rhs) {return lhs.str < rhs;} +inline bool +operator> (const string& lhs, const std::string& rhs) {return lhs.str > rhs;} +inline bool +operator<=(const string& lhs, const std::string& rhs) {return lhs.str <= rhs;} +inline bool +operator>=(const string& lhs, const std::string& rhs) {return lhs.str >= rhs;} + +inline bool +operator==(const std::string& lhs, const string& rhs) {return lhs == rhs.str;} +inline bool +operator!=(const std::string& lhs, const string& rhs) {return lhs != rhs.str;} +inline bool +operator< (const std::string& lhs, const string& rhs) {return lhs < rhs.str;} +inline bool +operator> (const std::string& lhs, const string& rhs) {return lhs > rhs.str;} +inline bool +operator<=(const std::string& lhs, const string& rhs) {return lhs <= rhs.str;} +inline bool +operator>=(const std::string& lhs, const string& rhs) {return lhs >= rhs.str;} + +inline bool +operator==(const string& lhs, const char* rhs) {return lhs.str == std::string(rhs);} +inline bool +operator!=(const string& lhs, const char* rhs) {return lhs.str != std::string(rhs);} +inline bool +operator< (const string& lhs, const char* rhs) {return lhs.str < std::string(rhs);} +inline bool +operator> (const string& lhs, const char* rhs) {return lhs.str > std::string(rhs);} +inline bool +operator<=(const string& lhs, const char* rhs) {return lhs.str <= std::string(rhs);} +inline bool +operator>=(const string& lhs, const char* rhs) {return lhs.str >= std::string(rhs);} + +inline bool +operator==(const char* lhs, const string& rhs) {return std::string(lhs) == rhs.str;} +inline bool +operator!=(const char* lhs, const string& rhs) {return std::string(lhs) != rhs.str;} +inline bool +operator< (const char* lhs, const string& rhs) {return std::string(lhs) < rhs.str;} +inline bool +operator> (const char* lhs, const string& rhs) {return std::string(lhs) > rhs.str;} +inline bool +operator<=(const char* lhs, const string& rhs) {return std::string(lhs) <= rhs.str;} +inline bool +operator>=(const char* lhs, const string& rhs) {return std::string(lhs) >= rhs.str;} + +template +std::basic_ostream& +operator<<(std::basic_ostream& os, const string& s) +{ + if(s.kind == string_t::basic) + { + if(std::find(s.str.cbegin(), s.str.cend(), '\n') != s.str.cend()) + { + // it contains newline. make it multiline string. + os << "\"\"\"\n"; + for(auto i=s.str.cbegin(), e=s.str.cend(); i!=e; ++i) + { + switch(*i) + { + case '\\': {os << "\\\\"; break;} + case '\"': {os << "\\\""; break;} + case '\b': {os << "\\b"; break;} + case '\t': {os << "\\t"; break;} + case '\f': {os << "\\f"; break;} + case '\n': {os << '\n'; break;} + case '\r': + { + // since it is a multiline string, + // CRLF is not needed to be escaped. + if(std::next(i) != e && *std::next(i) == '\n') + { + os << "\r\n"; + ++i; + } + else + { + os << "\\r"; + } + break; + } + default: {os << *i; break;} + } + } + os << "\\\n\"\"\""; + return os; + } + // no newline. make it inline. + os << "\""; + for(const auto c : s.str) + { + switch(c) + { + case '\\': {os << "\\\\"; break;} + case '\"': {os << "\\\""; break;} + case '\b': {os << "\\b"; break;} + case '\t': {os << "\\t"; break;} + case '\f': {os << "\\f"; break;} + case '\n': {os << "\\n"; break;} + case '\r': {os << "\\r"; break;} + default : {os << c; break;} + } + } + os << "\""; + return os; + } + // the string `s` is literal-string. + if(std::find(s.str.cbegin(), s.str.cend(), '\n') != s.str.cend() || + std::find(s.str.cbegin(), s.str.cend(), '\'') != s.str.cend() ) + { + // contains newline or single quote. make it multiline. + os << "'''\n" << s.str << "'''"; + return os; + } + // normal literal string + os << '\'' << s.str << '\''; + return os; +} + +} // toml +#endif// TOML11_STRING_H diff --git a/src/toml11/toml/traits.hpp b/src/toml11/toml/traits.hpp new file mode 100644 index 000000000..5495c93b2 --- /dev/null +++ b/src/toml11/toml/traits.hpp @@ -0,0 +1,327 @@ +// Copyright Toru Niina 2017. +// Distributed under the MIT License. +#ifndef TOML11_TRAITS_HPP +#define TOML11_TRAITS_HPP + +#include "from.hpp" +#include "into.hpp" + +#include +#include +#include +#include +#include +#include + +#if __cplusplus >= 201703L +#if __has_include() +#include +#endif // has_include() +#endif // cplusplus >= C++17 + +namespace toml +{ +template class T, template class A> +class basic_value; + +namespace detail +{ +// --------------------------------------------------------------------------- +// check whether type T is a kind of container/map class + +struct has_iterator_impl +{ + template static std::true_type check(typename T::iterator*); + template static std::false_type check(...); +}; +struct has_value_type_impl +{ + template static std::true_type check(typename T::value_type*); + template static std::false_type check(...); +}; +struct has_key_type_impl +{ + template static std::true_type check(typename T::key_type*); + template static std::false_type check(...); +}; +struct has_mapped_type_impl +{ + template static std::true_type check(typename T::mapped_type*); + template static std::false_type check(...); +}; +struct has_reserve_method_impl +{ + template static std::false_type check(...); + template static std::true_type check( + decltype(std::declval().reserve(std::declval()))*); +}; +struct has_push_back_method_impl +{ + template static std::false_type check(...); + template static std::true_type check( + decltype(std::declval().push_back(std::declval()))*); +}; +struct is_comparable_impl +{ + template static std::false_type check(...); + template static std::true_type check( + decltype(std::declval() < std::declval())*); +}; + +struct has_from_toml_method_impl +{ + template class Tb, template class A> + static std::true_type check( + decltype(std::declval().from_toml( + std::declval<::toml::basic_value>()))*); + + template class Tb, template class A> + static std::false_type check(...); +}; +struct has_into_toml_method_impl +{ + template + static std::true_type check(decltype(std::declval().into_toml())*); + template + static std::false_type check(...); +}; + +struct has_specialized_from_impl +{ + template + static std::false_type check(...); + template)> + static std::true_type check(::toml::from*); +}; +struct has_specialized_into_impl +{ + template + static std::false_type check(...); + template)> + static std::true_type check(::toml::from*); +}; + + +/// Intel C++ compiler can not use decltype in parent class declaration, here +/// is a hack to work around it. https://stackoverflow.com/a/23953090/4692076 +#ifdef __INTEL_COMPILER +#define decltype(...) std::enable_if::type +#endif + +template +struct has_iterator : decltype(has_iterator_impl::check(nullptr)){}; +template +struct has_value_type : decltype(has_value_type_impl::check(nullptr)){}; +template +struct has_key_type : decltype(has_key_type_impl::check(nullptr)){}; +template +struct has_mapped_type : decltype(has_mapped_type_impl::check(nullptr)){}; +template +struct has_reserve_method : decltype(has_reserve_method_impl::check(nullptr)){}; +template +struct has_push_back_method : decltype(has_push_back_method_impl::check(nullptr)){}; +template +struct is_comparable : decltype(is_comparable_impl::check(nullptr)){}; + +template class Tb, template class A> +struct has_from_toml_method +: decltype(has_from_toml_method_impl::check(nullptr)){}; + +template +struct has_into_toml_method +: decltype(has_into_toml_method_impl::check(nullptr)){}; + +template +struct has_specialized_from : decltype(has_specialized_from_impl::check(nullptr)){}; +template +struct has_specialized_into : decltype(has_specialized_into_impl::check(nullptr)){}; + +#ifdef __INTEL_COMPILER +#undef decltype +#endif + +// --------------------------------------------------------------------------- +// C++17 and/or/not + +#if __cplusplus >= 201703L + +using std::conjunction; +using std::disjunction; +using std::negation; + +#else + +template struct conjunction : std::true_type{}; +template struct conjunction : T{}; +template +struct conjunction : + std::conditional(T::value), conjunction, T>::type +{}; + +template struct disjunction : std::false_type{}; +template struct disjunction : T {}; +template +struct disjunction : + std::conditional(T::value), T, disjunction>::type +{}; + +template +struct negation : std::integral_constant(T::value)>{}; + +#endif + +// --------------------------------------------------------------------------- +// type checkers + +template struct is_std_pair : std::false_type{}; +template +struct is_std_pair> : std::true_type{}; + +template struct is_std_tuple : std::false_type{}; +template +struct is_std_tuple> : std::true_type{}; + +template struct is_std_forward_list : std::false_type{}; +template +struct is_std_forward_list> : std::true_type{}; + +template struct is_chrono_duration: std::false_type{}; +template +struct is_chrono_duration>: std::true_type{}; + +template +struct is_map : conjunction< // map satisfies all the following conditions + has_iterator, // has T::iterator + has_value_type, // has T::value_type + has_key_type, // has T::key_type + has_mapped_type // has T::mapped_type + >{}; +template struct is_map : is_map{}; +template struct is_map : is_map{}; +template struct is_map : is_map{}; +template struct is_map : is_map{}; + +template +struct is_container : conjunction< + negation>, // not a map + negation>, // not a std::string +#if __cplusplus >= 201703L +#if __has_include() + negation>, // not a std::string_view +#endif // has_include() +#endif + has_iterator, // has T::iterator + has_value_type // has T::value_type + >{}; +template struct is_container : is_container{}; +template struct is_container : is_container{}; +template struct is_container : is_container{}; +template struct is_container : is_container{}; + +template +struct is_basic_value: std::false_type{}; +template struct is_basic_value : is_basic_value{}; +template struct is_basic_value : is_basic_value{}; +template struct is_basic_value : is_basic_value{}; +template struct is_basic_value : is_basic_value{}; +template class M, template class V> +struct is_basic_value<::toml::basic_value>: std::true_type{}; + +// --------------------------------------------------------------------------- +// C++14 index_sequence + +#if __cplusplus >= 201402L + +using std::index_sequence; +using std::make_index_sequence; + +#else + +template struct index_sequence{}; + +template struct push_back_index_sequence{}; +template +struct push_back_index_sequence, N> +{ + typedef index_sequence type; +}; + +template +struct index_sequence_maker +{ + typedef typename push_back_index_sequence< + typename index_sequence_maker::type, N>::type type; +}; +template<> +struct index_sequence_maker<0> +{ + typedef index_sequence<0> type; +}; +template +using make_index_sequence = typename index_sequence_maker::type; + +#endif // __cplusplus >= 2014 + +// --------------------------------------------------------------------------- +// C++14 enable_if_t + +#if __cplusplus >= 201402L + +using std::enable_if_t; + +#else + +template +using enable_if_t = typename std::enable_if::type; + +#endif // __cplusplus >= 2014 + +// --------------------------------------------------------------------------- +// return_type_of_t + +#if __cplusplus >= 201703L && defined(__cpp_lib_is_invocable) && __cpp_lib_is_invocable>=201703 + +template +using return_type_of_t = std::invoke_result_t; + +#else +// result_of is deprecated after C++17 +template +using return_type_of_t = typename std::result_of::type; + +#endif + +// --------------------------------------------------------------------------- +// is_string_literal +// +// to use this, pass `typename remove_reference::type` to T. + +template +struct is_string_literal: +disjunction< + std::is_same, + conjunction< + std::is_array, + std::is_same::type> + > + >{}; + +// --------------------------------------------------------------------------- +// C++20 remove_cvref_t + +template +struct remove_cvref +{ + using type = typename std::remove_cv< + typename std::remove_reference::type>::type; +}; + +template +using remove_cvref_t = typename remove_cvref::type; + +}// detail +}//toml +#endif // TOML_TRAITS diff --git a/src/toml11/toml/types.hpp b/src/toml11/toml/types.hpp new file mode 100644 index 000000000..1e420e7fd --- /dev/null +++ b/src/toml11/toml/types.hpp @@ -0,0 +1,173 @@ +// Copyright Toru Niina 2017. +// Distributed under the MIT License. +#ifndef TOML11_TYPES_HPP +#define TOML11_TYPES_HPP +#include +#include + +#include "comments.hpp" +#include "datetime.hpp" +#include "string.hpp" +#include "traits.hpp" + +namespace toml +{ + +template class Table, // map-like class + template class Array> // vector-like class +class basic_value; + +using character = char; +using key = std::string; + +#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ <= 4 +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wshadow" +#endif + +using boolean = bool; +using integer = std::int64_t; +using floating = double; // "float" is a keyword, cannot use it here. +// the following stuffs are structs defined here, so aliases are not needed. +// - string +// - offset_datetime +// - offset_datetime +// - local_datetime +// - local_date +// - local_time + +#if defined(__GNUC__) && !defined(__clang__) +# pragma GCC diagnostic pop +#endif + +// default toml::value and default array/table. these are defined after defining +// basic_value itself. +// using value = basic_value; +// using array = typename value::array_type; +// using table = typename value::table_type; + +// to avoid warnings about `value_t::integer` is "shadowing" toml::integer in +// GCC -Wshadow=global. +#if defined(__GNUC__) && !defined(__clang__) +# pragma GCC diagnostic push +# if 7 <= __GNUC__ +# pragma GCC diagnostic ignored "-Wshadow=global" +# else // gcc-6 or older +# pragma GCC diagnostic ignored "-Wshadow" +# endif +#endif +enum class value_t : std::uint8_t +{ + empty = 0, + boolean = 1, + integer = 2, + floating = 3, + string = 4, + offset_datetime = 5, + local_datetime = 6, + local_date = 7, + local_time = 8, + array = 9, + table = 10, +}; +#if defined(__GNUC__) && !defined(__clang__) +# pragma GCC diagnostic pop +#endif + +template +inline std::basic_ostream& +operator<<(std::basic_ostream& os, value_t t) +{ + switch(t) + { + case value_t::boolean : os << "boolean"; return os; + case value_t::integer : os << "integer"; return os; + case value_t::floating : os << "floating"; return os; + case value_t::string : os << "string"; return os; + case value_t::offset_datetime : os << "offset_datetime"; return os; + case value_t::local_datetime : os << "local_datetime"; return os; + case value_t::local_date : os << "local_date"; return os; + case value_t::local_time : os << "local_time"; return os; + case value_t::array : os << "array"; return os; + case value_t::table : os << "table"; return os; + case value_t::empty : os << "empty"; return os; + default : os << "unknown"; return os; + } +} + +template, + typename alloc = std::allocator> +inline std::basic_string stringize(value_t t) +{ + std::basic_ostringstream oss; + oss << t; + return oss.str(); +} + +namespace detail +{ + +// helper to define a type that represents a value_t value. +template +using value_t_constant = std::integral_constant; + +// meta-function that convertes from value_t to the exact toml type that corresponds to. +// It takes toml::basic_value type because array and table types depend on it. +template struct enum_to_type {using type = void ;}; +template struct enum_to_type{using type = void ;}; +template struct enum_to_type{using type = boolean ;}; +template struct enum_to_type{using type = integer ;}; +template struct enum_to_type{using type = floating ;}; +template struct enum_to_type{using type = string ;}; +template struct enum_to_type{using type = offset_datetime ;}; +template struct enum_to_type{using type = local_datetime ;}; +template struct enum_to_type{using type = local_date ;}; +template struct enum_to_type{using type = local_time ;}; +template struct enum_to_type{using type = typename Value::array_type;}; +template struct enum_to_type{using type = typename Value::table_type;}; + +// meta-function that converts from an exact toml type to the enum that corresponds to. +template +struct type_to_enum : std::conditional< + std::is_same::value, // if T == array_type, + value_t_constant, // then value_t::array + typename std::conditional< // else... + std::is_same::value, // if T == table_type + value_t_constant, // then value_t::table + value_t_constant // else value_t::empty + >::type + >::type {}; +template struct type_to_enum: value_t_constant {}; +template struct type_to_enum: value_t_constant {}; +template struct type_to_enum: value_t_constant {}; +template struct type_to_enum: value_t_constant {}; +template struct type_to_enum: value_t_constant {}; +template struct type_to_enum: value_t_constant {}; +template struct type_to_enum: value_t_constant {}; +template struct type_to_enum: value_t_constant {}; + +// meta-function that checks the type T is the same as one of the toml::* types. +template +struct is_exact_toml_type : disjunction< + std::is_same, + std::is_same, + std::is_same, + std::is_same, + std::is_same, + std::is_same, + std::is_same, + std::is_same, + std::is_same, + std::is_same + >{}; +template struct is_exact_toml_type : is_exact_toml_type{}; +template struct is_exact_toml_type : is_exact_toml_type{}; +template struct is_exact_toml_type : is_exact_toml_type{}; +template struct is_exact_toml_type: is_exact_toml_type{}; + +} // detail +} // toml + +#endif// TOML11_TYPES_H diff --git a/src/toml11/toml/utility.hpp b/src/toml11/toml/utility.hpp new file mode 100644 index 000000000..4a6b4309d --- /dev/null +++ b/src/toml11/toml/utility.hpp @@ -0,0 +1,149 @@ +// Copyright Toru Niina 2017. +// Distributed under the MIT License. +#ifndef TOML11_UTILITY_HPP +#define TOML11_UTILITY_HPP +#include +#include +#include + +#include "traits.hpp" + +#if __cplusplus >= 201402L +# define TOML11_MARK_AS_DEPRECATED(msg) [[deprecated(msg)]] +#elif defined(__GNUC__) +# define TOML11_MARK_AS_DEPRECATED(msg) __attribute__((deprecated(msg))) +#elif defined(_MSC_VER) +# define TOML11_MARK_AS_DEPRECATED(msg) __declspec(deprecated(msg)) +#else +# define TOML11_MARK_AS_DEPRECATED +#endif + +namespace toml +{ + +#if __cplusplus >= 201402L + +using std::make_unique; + +#else + +template +inline std::unique_ptr make_unique(Ts&& ... args) +{ + return std::unique_ptr(new T(std::forward(args)...)); +} + +#endif // __cplusplus >= 2014 + +namespace detail +{ +template +void try_reserve_impl(Container& container, std::size_t N, std::true_type) +{ + container.reserve(N); + return; +} +template +void try_reserve_impl(Container&, std::size_t, std::false_type) noexcept +{ + return; +} +} // detail + +template +void try_reserve(Container& container, std::size_t N) +{ + if(N <= container.size()) {return;} + detail::try_reserve_impl(container, N, detail::has_reserve_method{}); + return; +} + +namespace detail +{ +inline std::string concat_to_string_impl(std::ostringstream& oss) +{ + return oss.str(); +} +template +std::string concat_to_string_impl(std::ostringstream& oss, T&& head, Ts&& ... tail) +{ + oss << std::forward(head); + return concat_to_string_impl(oss, std::forward(tail) ... ); +} +} // detail + +template +std::string concat_to_string(Ts&& ... args) +{ + std::ostringstream oss; + oss << std::boolalpha << std::fixed; + return detail::concat_to_string_impl(oss, std::forward(args) ...); +} + +template +T from_string(const std::string& str, T opt) +{ + T v(opt); + std::istringstream iss(str); + iss >> v; + return v; +} + +namespace detail +{ +#if __cplusplus >= 201402L +template +decltype(auto) last_one(T&& tail) noexcept +{ + return std::forward(tail); +} + +template +decltype(auto) last_one(T&& /*head*/, Ts&& ... tail) noexcept +{ + return last_one(std::forward(tail)...); +} +#else // C++11 +// The following code +// ```cpp +// 1 | template +// 2 | auto last_one(T&& /*head*/, Ts&& ... tail) +// 3 | -> decltype(last_one(std::forward(tail)...)) +// 4 | { +// 5 | return last_one(std::forward(tail)...); +// 6 | } +// ``` +// does not work because the function `last_one(...)` is not yet defined at +// line #3, so `decltype()` cannot deduce the type returned from `last_one`. +// So we need to determine return type in a different way, like a meta func. + +template +struct last_one_in_pack +{ + using type = typename last_one_in_pack::type; +}; +template +struct last_one_in_pack +{ + using type = T; +}; +template +using last_one_in_pack_t = typename last_one_in_pack::type; + +template +T&& last_one(T&& tail) noexcept +{ + return std::forward(tail); +} +template +enable_if_t<(sizeof...(Ts) > 0), last_one_in_pack_t> +last_one(T&& /*head*/, Ts&& ... tail) +{ + return last_one(std::forward(tail)...); +} + +#endif +} // detail + +}// toml +#endif // TOML11_UTILITY diff --git a/src/toml11/toml/value.hpp b/src/toml11/toml/value.hpp new file mode 100644 index 000000000..1b43db8d4 --- /dev/null +++ b/src/toml11/toml/value.hpp @@ -0,0 +1,2035 @@ +// Copyright Toru Niina 2017. +// Distributed under the MIT License. +#ifndef TOML11_VALUE_HPP +#define TOML11_VALUE_HPP +#include + +#include "comments.hpp" +#include "exception.hpp" +#include "into.hpp" +#include "region.hpp" +#include "source_location.hpp" +#include "storage.hpp" +#include "traits.hpp" +#include "types.hpp" +#include "utility.hpp" + +namespace toml +{ + +namespace detail +{ + +// to show error messages. not recommended for users. +template +inline region_base const* get_region(const Value& v) +{ + return v.region_info_.get(); +} + +template +void change_region(Value& v, region reg) +{ + v.region_info_ = std::make_shared(std::move(reg)); + return; +} + +template +[[noreturn]] inline void +throw_bad_cast(const std::string& funcname, value_t actual, const Value& v) +{ + throw type_error(detail::format_underline( + concat_to_string(funcname, "bad_cast to ", Expected), { + {v.location(), concat_to_string("the actual type is ", actual)} + }), v.location()); +} + +// Throw `out_of_range` from `toml::value::at()` and `toml::find()` +// after generating an error message. +// +// The implementation is a bit complicated and there are many edge-cases. +// If you are not interested in the error message generation, just skip this. +template +[[noreturn]] void +throw_key_not_found_error(const Value& v, const key& ky) +{ + // The top-level table has its region at the first character of the file. + // That means that, in the case when a key is not found in the top-level + // table, the error message points to the first character. If the file has + // its first table at the first line, the error message would be like this. + // ```console + // [error] key "a" not found + // --> example.toml + // | + // 1 | [table] + // | ^------ in this table + // ``` + // It actually points to the top-level table at the first character, + // not `[table]`. But it is too confusing. To avoid the confusion, the error + // message should explicitly say "key not found in the top-level table", + // or "the parsed file is empty" if there is no content at all (0 bytes in file). + const auto loc = v.location(); + if(loc.line() == 1 && loc.region() == 0) + { + // First line with a zero-length region means "empty file". + // The region will be generated at `parse_toml_file` function + // if the file contains no bytes. + throw std::out_of_range(format_underline(concat_to_string( + "key \"", ky, "\" not found in the top-level table"), { + {loc, "the parsed file is empty"} + })); + } + else if(loc.line() == 1 && loc.region() == 1) + { + // Here it assumes that top-level table starts at the first character. + // The region corresponds to the top-level table will be generated at + // `parse_toml_file` function. + // It also assumes that the top-level table size is just one and + // the line number is `1`. It is always satisfied. And those conditions + // are satisfied only if the table is the top-level table. + // + // 1. one-character dot-key at the first line + // ```toml + // a.b = "c" + // ``` + // toml11 counts whole key as the table key. Here, `a.b` is the region + // of the table "a". It could be counter intuitive, but it works. + // The size of the region is 3, not 1. The above example is the shortest + // dot-key example. The size cannot be 1. + // + // 2. one-character inline-table at the first line + // ```toml + // a = {b = "c"} + // ``` + // toml11 considers the inline table body as the table region. Here, + // `{b = "c"}` is the region of the table "a". The size of the region + // is 9, not 1. The shotest inline table still has two characters, `{` + // and `}`. The size cannot be 1. + // + // 3. one-character table declaration at the first line + // ```toml + // [a] + // ``` + // toml11 considers the whole table key as the table region. Here, + // `[a]` is the table region. The size is 3, not 1. + // + throw std::out_of_range(format_underline(concat_to_string( + "key \"", ky, "\" not found in the top-level table"), { + {loc, "the top-level table starts here"} + })); + } + else + { + // normal table. + throw std::out_of_range(format_underline(concat_to_string( + "key \"", ky, "\" not found"), { {loc, "in this table"} })); + } +} + +// switch by `value_t` at the compile time. +template +struct switch_cast {}; +#define TOML11_GENERATE_SWITCH_CASTER(TYPE) \ + template<> \ + struct switch_cast \ + { \ + template \ + static typename Value::TYPE##_type& invoke(Value& v) \ + { \ + return v.as_##TYPE(); \ + } \ + template \ + static typename Value::TYPE##_type const& invoke(const Value& v) \ + { \ + return v.as_##TYPE(); \ + } \ + template \ + static typename Value::TYPE##_type&& invoke(Value&& v) \ + { \ + return std::move(v).as_##TYPE(); \ + } \ + }; \ + /**/ +TOML11_GENERATE_SWITCH_CASTER(boolean) +TOML11_GENERATE_SWITCH_CASTER(integer) +TOML11_GENERATE_SWITCH_CASTER(floating) +TOML11_GENERATE_SWITCH_CASTER(string) +TOML11_GENERATE_SWITCH_CASTER(offset_datetime) +TOML11_GENERATE_SWITCH_CASTER(local_datetime) +TOML11_GENERATE_SWITCH_CASTER(local_date) +TOML11_GENERATE_SWITCH_CASTER(local_time) +TOML11_GENERATE_SWITCH_CASTER(array) +TOML11_GENERATE_SWITCH_CASTER(table) + +#undef TOML11_GENERATE_SWITCH_CASTER + +}// detail + +template class Table = std::unordered_map, + template class Array = std::vector> +class basic_value +{ + template + static void assigner(T& dst, U&& v) + { + const auto tmp = ::new(std::addressof(dst)) T(std::forward(v)); + assert(tmp == std::addressof(dst)); + (void)tmp; + } + + using region_base = detail::region_base; + + template class T, + template class A> + friend class basic_value; + + public: + + using comment_type = Comment; + using key_type = ::toml::key; + using value_type = basic_value; + using boolean_type = ::toml::boolean; + using integer_type = ::toml::integer; + using floating_type = ::toml::floating; + using string_type = ::toml::string; + using local_time_type = ::toml::local_time; + using local_date_type = ::toml::local_date; + using local_datetime_type = ::toml::local_datetime; + using offset_datetime_type = ::toml::offset_datetime; + using array_type = Array; + using table_type = Table; + + public: + + basic_value() noexcept + : type_(value_t::empty), + region_info_(std::make_shared(region_base{})) + {} + ~basic_value() noexcept {this->cleanup();} + + basic_value(const basic_value& v) + : type_(v.type()), region_info_(v.region_info_), comments_(v.comments_) + { + switch(v.type()) + { + case value_t::boolean : assigner(boolean_ , v.boolean_ ); break; + case value_t::integer : assigner(integer_ , v.integer_ ); break; + case value_t::floating : assigner(floating_ , v.floating_ ); break; + case value_t::string : assigner(string_ , v.string_ ); break; + case value_t::offset_datetime: assigner(offset_datetime_, v.offset_datetime_); break; + case value_t::local_datetime : assigner(local_datetime_ , v.local_datetime_ ); break; + case value_t::local_date : assigner(local_date_ , v.local_date_ ); break; + case value_t::local_time : assigner(local_time_ , v.local_time_ ); break; + case value_t::array : assigner(array_ , v.array_ ); break; + case value_t::table : assigner(table_ , v.table_ ); break; + default: break; + } + } + basic_value(basic_value&& v) + : type_(v.type()), region_info_(std::move(v.region_info_)), + comments_(std::move(v.comments_)) + { + switch(this->type_) // here this->type_ is already initialized + { + case value_t::boolean : assigner(boolean_ , std::move(v.boolean_ )); break; + case value_t::integer : assigner(integer_ , std::move(v.integer_ )); break; + case value_t::floating : assigner(floating_ , std::move(v.floating_ )); break; + case value_t::string : assigner(string_ , std::move(v.string_ )); break; + case value_t::offset_datetime: assigner(offset_datetime_, std::move(v.offset_datetime_)); break; + case value_t::local_datetime : assigner(local_datetime_ , std::move(v.local_datetime_ )); break; + case value_t::local_date : assigner(local_date_ , std::move(v.local_date_ )); break; + case value_t::local_time : assigner(local_time_ , std::move(v.local_time_ )); break; + case value_t::array : assigner(array_ , std::move(v.array_ )); break; + case value_t::table : assigner(table_ , std::move(v.table_ )); break; + default: break; + } + } + basic_value& operator=(const basic_value& v) + { + this->cleanup(); + this->region_info_ = v.region_info_; + this->comments_ = v.comments_; + this->type_ = v.type(); + switch(this->type_) + { + case value_t::boolean : assigner(boolean_ , v.boolean_ ); break; + case value_t::integer : assigner(integer_ , v.integer_ ); break; + case value_t::floating : assigner(floating_ , v.floating_ ); break; + case value_t::string : assigner(string_ , v.string_ ); break; + case value_t::offset_datetime: assigner(offset_datetime_, v.offset_datetime_); break; + case value_t::local_datetime : assigner(local_datetime_ , v.local_datetime_ ); break; + case value_t::local_date : assigner(local_date_ , v.local_date_ ); break; + case value_t::local_time : assigner(local_time_ , v.local_time_ ); break; + case value_t::array : assigner(array_ , v.array_ ); break; + case value_t::table : assigner(table_ , v.table_ ); break; + default: break; + } + return *this; + } + basic_value& operator=(basic_value&& v) + { + this->cleanup(); + this->region_info_ = std::move(v.region_info_); + this->comments_ = std::move(v.comments_); + this->type_ = v.type(); + switch(this->type_) + { + case value_t::boolean : assigner(boolean_ , std::move(v.boolean_ )); break; + case value_t::integer : assigner(integer_ , std::move(v.integer_ )); break; + case value_t::floating : assigner(floating_ , std::move(v.floating_ )); break; + case value_t::string : assigner(string_ , std::move(v.string_ )); break; + case value_t::offset_datetime: assigner(offset_datetime_, std::move(v.offset_datetime_)); break; + case value_t::local_datetime : assigner(local_datetime_ , std::move(v.local_datetime_ )); break; + case value_t::local_date : assigner(local_date_ , std::move(v.local_date_ )); break; + case value_t::local_time : assigner(local_time_ , std::move(v.local_time_ )); break; + case value_t::array : assigner(array_ , std::move(v.array_ )); break; + case value_t::table : assigner(table_ , std::move(v.table_ )); break; + default: break; + } + return *this; + } + + // overwrite comments ---------------------------------------------------- + + basic_value(const basic_value& v, std::vector com) + : type_(v.type()), region_info_(v.region_info_), + comments_(std::move(com)) + { + switch(v.type()) + { + case value_t::boolean : assigner(boolean_ , v.boolean_ ); break; + case value_t::integer : assigner(integer_ , v.integer_ ); break; + case value_t::floating : assigner(floating_ , v.floating_ ); break; + case value_t::string : assigner(string_ , v.string_ ); break; + case value_t::offset_datetime: assigner(offset_datetime_, v.offset_datetime_); break; + case value_t::local_datetime : assigner(local_datetime_ , v.local_datetime_ ); break; + case value_t::local_date : assigner(local_date_ , v.local_date_ ); break; + case value_t::local_time : assigner(local_time_ , v.local_time_ ); break; + case value_t::array : assigner(array_ , v.array_ ); break; + case value_t::table : assigner(table_ , v.table_ ); break; + default: break; + } + } + + basic_value(basic_value&& v, std::vector com) + : type_(v.type()), region_info_(std::move(v.region_info_)), + comments_(std::move(com)) + { + switch(this->type_) // here this->type_ is already initialized + { + case value_t::boolean : assigner(boolean_ , std::move(v.boolean_ )); break; + case value_t::integer : assigner(integer_ , std::move(v.integer_ )); break; + case value_t::floating : assigner(floating_ , std::move(v.floating_ )); break; + case value_t::string : assigner(string_ , std::move(v.string_ )); break; + case value_t::offset_datetime: assigner(offset_datetime_, std::move(v.offset_datetime_)); break; + case value_t::local_datetime : assigner(local_datetime_ , std::move(v.local_datetime_ )); break; + case value_t::local_date : assigner(local_date_ , std::move(v.local_date_ )); break; + case value_t::local_time : assigner(local_time_ , std::move(v.local_time_ )); break; + case value_t::array : assigner(array_ , std::move(v.array_ )); break; + case value_t::table : assigner(table_ , std::move(v.table_ )); break; + default: break; + } + } + + // ----------------------------------------------------------------------- + // conversion between different basic_values. + template class T, + template class A> + basic_value(const basic_value& v) + : type_(v.type()), region_info_(v.region_info_), comments_(v.comments()) + { + switch(v.type()) + { + case value_t::boolean : assigner(boolean_ , v.boolean_ ); break; + case value_t::integer : assigner(integer_ , v.integer_ ); break; + case value_t::floating : assigner(floating_ , v.floating_ ); break; + case value_t::string : assigner(string_ , v.string_ ); break; + case value_t::offset_datetime: assigner(offset_datetime_, v.offset_datetime_); break; + case value_t::local_datetime : assigner(local_datetime_ , v.local_datetime_ ); break; + case value_t::local_date : assigner(local_date_ , v.local_date_ ); break; + case value_t::local_time : assigner(local_time_ , v.local_time_ ); break; + case value_t::array : + { + array_type tmp(v.as_array(std::nothrow).begin(), + v.as_array(std::nothrow).end()); + assigner(array_, std::move(tmp)); + break; + } + case value_t::table : + { + table_type tmp(v.as_table(std::nothrow).begin(), + v.as_table(std::nothrow).end()); + assigner(table_, std::move(tmp)); + break; + } + default: break; + } + } + template class T, + template class A> + basic_value(const basic_value& v, std::vector com) + : type_(v.type()), region_info_(v.region_info_), + comments_(std::move(com)) + { + switch(v.type()) + { + case value_t::boolean : assigner(boolean_ , v.boolean_ ); break; + case value_t::integer : assigner(integer_ , v.integer_ ); break; + case value_t::floating : assigner(floating_ , v.floating_ ); break; + case value_t::string : assigner(string_ , v.string_ ); break; + case value_t::offset_datetime: assigner(offset_datetime_, v.offset_datetime_); break; + case value_t::local_datetime : assigner(local_datetime_ , v.local_datetime_ ); break; + case value_t::local_date : assigner(local_date_ , v.local_date_ ); break; + case value_t::local_time : assigner(local_time_ , v.local_time_ ); break; + case value_t::array : + { + array_type tmp(v.as_array(std::nothrow).begin(), + v.as_array(std::nothrow).end()); + assigner(array_, std::move(tmp)); + break; + } + case value_t::table : + { + table_type tmp(v.as_table(std::nothrow).begin(), + v.as_table(std::nothrow).end()); + assigner(table_, std::move(tmp)); + break; + } + default: break; + } + } + template class T, + template class A> + basic_value& operator=(const basic_value& v) + { + this->region_info_ = v.region_info_; + this->comments_ = comment_type(v.comments()); + this->type_ = v.type(); + switch(v.type()) + { + case value_t::boolean : assigner(boolean_ , v.boolean_ ); break; + case value_t::integer : assigner(integer_ , v.integer_ ); break; + case value_t::floating : assigner(floating_ , v.floating_ ); break; + case value_t::string : assigner(string_ , v.string_ ); break; + case value_t::offset_datetime: assigner(offset_datetime_, v.offset_datetime_); break; + case value_t::local_datetime : assigner(local_datetime_ , v.local_datetime_ ); break; + case value_t::local_date : assigner(local_date_ , v.local_date_ ); break; + case value_t::local_time : assigner(local_time_ , v.local_time_ ); break; + case value_t::array : + { + array_type tmp(v.as_array(std::nothrow).begin(), + v.as_array(std::nothrow).end()); + assigner(array_, std::move(tmp)); + break; + } + case value_t::table : + { + table_type tmp(v.as_table(std::nothrow).begin(), + v.as_table(std::nothrow).end()); + assigner(table_, std::move(tmp)); + break; + } + default: break; + } + return *this; + } + + // boolean ============================================================== + + basic_value(boolean b) + : type_(value_t::boolean), + region_info_(std::make_shared(region_base{})) + { + assigner(this->boolean_, b); + } + basic_value& operator=(boolean b) + { + this->cleanup(); + this->type_ = value_t::boolean; + this->region_info_ = std::make_shared(region_base{}); + assigner(this->boolean_, b); + return *this; + } + basic_value(boolean b, std::vector com) + : type_(value_t::boolean), + region_info_(std::make_shared(region_base{})), + comments_(std::move(com)) + { + assigner(this->boolean_, b); + } + + // integer ============================================================== + + template, detail::negation>>::value, + std::nullptr_t>::type = nullptr> + basic_value(T i) + : type_(value_t::integer), + region_info_(std::make_shared(region_base{})) + { + assigner(this->integer_, static_cast(i)); + } + + template, detail::negation>>::value, + std::nullptr_t>::type = nullptr> + basic_value& operator=(T i) + { + this->cleanup(); + this->type_ = value_t::integer; + this->region_info_ = std::make_shared(region_base{}); + assigner(this->integer_, static_cast(i)); + return *this; + } + + template, detail::negation>>::value, + std::nullptr_t>::type = nullptr> + basic_value(T i, std::vector com) + : type_(value_t::integer), + region_info_(std::make_shared(region_base{})), + comments_(std::move(com)) + { + assigner(this->integer_, static_cast(i)); + } + + // floating ============================================================= + + template::value, std::nullptr_t>::type = nullptr> + basic_value(T f) + : type_(value_t::floating), + region_info_(std::make_shared(region_base{})) + { + assigner(this->floating_, static_cast(f)); + } + + + template::value, std::nullptr_t>::type = nullptr> + basic_value& operator=(T f) + { + this->cleanup(); + this->type_ = value_t::floating; + this->region_info_ = std::make_shared(region_base{}); + assigner(this->floating_, static_cast(f)); + return *this; + } + + template::value, std::nullptr_t>::type = nullptr> + basic_value(T f, std::vector com) + : type_(value_t::floating), + region_info_(std::make_shared(region_base{})), + comments_(std::move(com)) + { + assigner(this->floating_, f); + } + + // string =============================================================== + + basic_value(toml::string s) + : type_(value_t::string), + region_info_(std::make_shared(region_base{})) + { + assigner(this->string_, std::move(s)); + } + basic_value& operator=(toml::string s) + { + this->cleanup(); + this->type_ = value_t::string ; + this->region_info_ = std::make_shared(region_base{}); + assigner(this->string_, s); + return *this; + } + basic_value(toml::string s, std::vector com) + : type_(value_t::string), + region_info_(std::make_shared(region_base{})), + comments_(std::move(com)) + { + assigner(this->string_, std::move(s)); + } + + basic_value(std::string s) + : type_(value_t::string), + region_info_(std::make_shared(region_base{})) + { + assigner(this->string_, toml::string(std::move(s))); + } + basic_value& operator=(std::string s) + { + this->cleanup(); + this->type_ = value_t::string ; + this->region_info_ = std::make_shared(region_base{}); + assigner(this->string_, toml::string(std::move(s))); + return *this; + } + basic_value(std::string s, string_t kind) + : type_(value_t::string), + region_info_(std::make_shared(region_base{})) + { + assigner(this->string_, toml::string(std::move(s), kind)); + } + basic_value(std::string s, std::vector com) + : type_(value_t::string), + region_info_(std::make_shared(region_base{})), + comments_(std::move(com)) + { + assigner(this->string_, toml::string(std::move(s))); + } + basic_value(std::string s, string_t kind, std::vector com) + : type_(value_t::string), + region_info_(std::make_shared(region_base{})), + comments_(std::move(com)) + { + assigner(this->string_, toml::string(std::move(s), kind)); + } + + basic_value(const char* s) + : type_(value_t::string), + region_info_(std::make_shared(region_base{})) + { + assigner(this->string_, toml::string(std::string(s))); + } + basic_value& operator=(const char* s) + { + this->cleanup(); + this->type_ = value_t::string ; + this->region_info_ = std::make_shared(region_base{}); + assigner(this->string_, toml::string(std::string(s))); + return *this; + } + basic_value(const char* s, string_t kind) + : type_(value_t::string), + region_info_(std::make_shared(region_base{})) + { + assigner(this->string_, toml::string(std::string(s), kind)); + } + basic_value(const char* s, std::vector com) + : type_(value_t::string), + region_info_(std::make_shared(region_base{})), + comments_(std::move(com)) + { + assigner(this->string_, toml::string(std::string(s))); + } + basic_value(const char* s, string_t kind, std::vector com) + : type_(value_t::string), + region_info_(std::make_shared(region_base{})), + comments_(std::move(com)) + { + assigner(this->string_, toml::string(std::string(s), kind)); + } + +#if defined(TOML11_USING_STRING_VIEW) && TOML11_USING_STRING_VIEW>0 + basic_value(std::string_view s) + : type_(value_t::string), + region_info_(std::make_shared(region_base{})) + { + assigner(this->string_, toml::string(s)); + } + basic_value& operator=(std::string_view s) + { + this->cleanup(); + this->type_ = value_t::string ; + this->region_info_ = std::make_shared(region_base{}); + assigner(this->string_, toml::string(s)); + return *this; + } + basic_value(std::string_view s, std::vector com) + : type_(value_t::string), + region_info_(std::make_shared(region_base{})), + comments_(std::move(com)) + { + assigner(this->string_, toml::string(s)); + } + basic_value(std::string_view s, string_t kind) + : type_(value_t::string), + region_info_(std::make_shared(region_base{})) + { + assigner(this->string_, toml::string(s, kind)); + } + basic_value(std::string_view s, string_t kind, std::vector com) + : type_(value_t::string), + region_info_(std::make_shared(region_base{})), + comments_(std::move(com)) + { + assigner(this->string_, toml::string(s, kind)); + } +#endif + + // local date =========================================================== + + basic_value(const local_date& ld) + : type_(value_t::local_date), + region_info_(std::make_shared(region_base{})) + { + assigner(this->local_date_, ld); + } + basic_value& operator=(const local_date& ld) + { + this->cleanup(); + this->type_ = value_t::local_date; + this->region_info_ = std::make_shared(region_base{}); + assigner(this->local_date_, ld); + return *this; + } + basic_value(const local_date& ld, std::vector com) + : type_(value_t::local_date), + region_info_(std::make_shared(region_base{})), + comments_(std::move(com)) + { + assigner(this->local_date_, ld); + } + + // local time =========================================================== + + basic_value(const local_time& lt) + : type_(value_t::local_time), + region_info_(std::make_shared(region_base{})) + { + assigner(this->local_time_, lt); + } + basic_value(const local_time& lt, std::vector com) + : type_(value_t::local_time), + region_info_(std::make_shared(region_base{})), + comments_(std::move(com)) + { + assigner(this->local_time_, lt); + } + basic_value& operator=(const local_time& lt) + { + this->cleanup(); + this->type_ = value_t::local_time; + this->region_info_ = std::make_shared(region_base{}); + assigner(this->local_time_, lt); + return *this; + } + + template + basic_value(const std::chrono::duration& dur) + : type_(value_t::local_time), + region_info_(std::make_shared(region_base{})) + { + assigner(this->local_time_, local_time(dur)); + } + template + basic_value(const std::chrono::duration& dur, + std::vector com) + : type_(value_t::local_time), + region_info_(std::make_shared(region_base{})), + comments_(std::move(com)) + { + assigner(this->local_time_, local_time(dur)); + } + template + basic_value& operator=(const std::chrono::duration& dur) + { + this->cleanup(); + this->type_ = value_t::local_time; + this->region_info_ = std::make_shared(region_base{}); + assigner(this->local_time_, local_time(dur)); + return *this; + } + + // local datetime ======================================================= + + basic_value(const local_datetime& ldt) + : type_(value_t::local_datetime), + region_info_(std::make_shared(region_base{})) + { + assigner(this->local_datetime_, ldt); + } + basic_value(const local_datetime& ldt, std::vector com) + : type_(value_t::local_datetime), + region_info_(std::make_shared(region_base{})), + comments_(std::move(com)) + { + assigner(this->local_datetime_, ldt); + } + basic_value& operator=(const local_datetime& ldt) + { + this->cleanup(); + this->type_ = value_t::local_datetime; + this->region_info_ = std::make_shared(region_base{}); + assigner(this->local_datetime_, ldt); + return *this; + } + + // offset datetime ====================================================== + + basic_value(const offset_datetime& odt) + : type_(value_t::offset_datetime), + region_info_(std::make_shared(region_base{})) + { + assigner(this->offset_datetime_, odt); + } + basic_value(const offset_datetime& odt, std::vector com) + : type_(value_t::offset_datetime), + region_info_(std::make_shared(region_base{})), + comments_(std::move(com)) + { + assigner(this->offset_datetime_, odt); + } + basic_value& operator=(const offset_datetime& odt) + { + this->cleanup(); + this->type_ = value_t::offset_datetime; + this->region_info_ = std::make_shared(region_base{}); + assigner(this->offset_datetime_, odt); + return *this; + } + basic_value(const std::chrono::system_clock::time_point& tp) + : type_(value_t::offset_datetime), + region_info_(std::make_shared(region_base{})) + { + assigner(this->offset_datetime_, offset_datetime(tp)); + } + basic_value(const std::chrono::system_clock::time_point& tp, + std::vector com) + : type_(value_t::offset_datetime), + region_info_(std::make_shared(region_base{})), + comments_(std::move(com)) + { + assigner(this->offset_datetime_, offset_datetime(tp)); + } + basic_value& operator=(const std::chrono::system_clock::time_point& tp) + { + this->cleanup(); + this->type_ = value_t::offset_datetime; + this->region_info_ = std::make_shared(region_base{}); + assigner(this->offset_datetime_, offset_datetime(tp)); + return *this; + } + + // array ================================================================ + + basic_value(const array_type& ary) + : type_(value_t::array), + region_info_(std::make_shared(region_base{})) + { + assigner(this->array_, ary); + } + basic_value(const array_type& ary, std::vector com) + : type_(value_t::array), + region_info_(std::make_shared(region_base{})), + comments_(std::move(com)) + { + assigner(this->array_, ary); + } + basic_value& operator=(const array_type& ary) + { + this->cleanup(); + this->type_ = value_t::array ; + this->region_info_ = std::make_shared(region_base{}); + assigner(this->array_, ary); + return *this; + } + + // array (initializer_list) ---------------------------------------------- + + template::value, + std::nullptr_t>::type = nullptr> + basic_value(std::initializer_list list) + : type_(value_t::array), + region_info_(std::make_shared(region_base{})) + { + array_type ary(list.begin(), list.end()); + assigner(this->array_, std::move(ary)); + } + template::value, + std::nullptr_t>::type = nullptr> + basic_value(std::initializer_list list, std::vector com) + : type_(value_t::array), + region_info_(std::make_shared(region_base{})), + comments_(std::move(com)) + { + array_type ary(list.begin(), list.end()); + assigner(this->array_, std::move(ary)); + } + template::value, + std::nullptr_t>::type = nullptr> + basic_value& operator=(std::initializer_list list) + { + this->cleanup(); + this->type_ = value_t::array; + this->region_info_ = std::make_shared(region_base{}); + + array_type ary(list.begin(), list.end()); + assigner(this->array_, std::move(ary)); + return *this; + } + + // array (STL Containers) ------------------------------------------------ + + template>, + detail::is_container + >::value, std::nullptr_t>::type = nullptr> + basic_value(const T& list) + : type_(value_t::array), + region_info_(std::make_shared(region_base{})) + { + static_assert(std::is_convertible::value, + "elements of a container should be convertible to toml::value"); + + array_type ary(list.size()); + std::copy(list.begin(), list.end(), ary.begin()); + assigner(this->array_, std::move(ary)); + } + template>, + detail::is_container + >::value, std::nullptr_t>::type = nullptr> + basic_value(const T& list, std::vector com) + : type_(value_t::array), + region_info_(std::make_shared(region_base{})), + comments_(std::move(com)) + { + static_assert(std::is_convertible::value, + "elements of a container should be convertible to toml::value"); + + array_type ary(list.size()); + std::copy(list.begin(), list.end(), ary.begin()); + assigner(this->array_, std::move(ary)); + } + template>, + detail::is_container + >::value, std::nullptr_t>::type = nullptr> + basic_value& operator=(const T& list) + { + static_assert(std::is_convertible::value, + "elements of a container should be convertible to toml::value"); + + this->cleanup(); + this->type_ = value_t::array; + this->region_info_ = std::make_shared(region_base{}); + + array_type ary(list.size()); + std::copy(list.begin(), list.end(), ary.begin()); + assigner(this->array_, std::move(ary)); + return *this; + } + + // table ================================================================ + + basic_value(const table_type& tab) + : type_(value_t::table), + region_info_(std::make_shared(region_base{})) + { + assigner(this->table_, tab); + } + basic_value(const table_type& tab, std::vector com) + : type_(value_t::table), + region_info_(std::make_shared(region_base{})), + comments_(std::move(com)) + { + assigner(this->table_, tab); + } + basic_value& operator=(const table_type& tab) + { + this->cleanup(); + this->type_ = value_t::table; + this->region_info_ = std::make_shared(region_base{}); + assigner(this->table_, tab); + return *this; + } + + // initializer-list ------------------------------------------------------ + + basic_value(std::initializer_list> list) + : type_(value_t::table), + region_info_(std::make_shared(region_base{})) + { + table_type tab; + for(const auto& elem : list) {tab[elem.first] = elem.second;} + assigner(this->table_, std::move(tab)); + } + + basic_value(std::initializer_list> list, + std::vector com) + : type_(value_t::table), + region_info_(std::make_shared(region_base{})), + comments_(std::move(com)) + { + table_type tab; + for(const auto& elem : list) {tab[elem.first] = elem.second;} + assigner(this->table_, std::move(tab)); + } + basic_value& operator=(std::initializer_list> list) + { + this->cleanup(); + this->type_ = value_t::table; + this->region_info_ = std::make_shared(region_base{}); + + table_type tab; + for(const auto& elem : list) {tab[elem.first] = elem.second;} + assigner(this->table_, std::move(tab)); + return *this; + } + + // other table-like ----------------------------------------------------- + + template>, + detail::is_map + >::value, std::nullptr_t>::type = nullptr> + basic_value(const Map& mp) + : type_(value_t::table), + region_info_(std::make_shared(region_base{})) + { + table_type tab; + for(const auto& elem : mp) {tab[elem.first] = elem.second;} + assigner(this->table_, std::move(tab)); + } + template>, + detail::is_map + >::value, std::nullptr_t>::type = nullptr> + basic_value(const Map& mp, std::vector com) + : type_(value_t::table), + region_info_(std::make_shared(region_base{})), + comments_(std::move(com)) + { + table_type tab; + for(const auto& elem : mp) {tab[elem.first] = elem.second;} + assigner(this->table_, std::move(tab)); + } + template>, + detail::is_map + >::value, std::nullptr_t>::type = nullptr> + basic_value& operator=(const Map& mp) + { + this->cleanup(); + this->type_ = value_t::table; + this->region_info_ = std::make_shared(region_base{}); + + table_type tab; + for(const auto& elem : mp) {tab[elem.first] = elem.second;} + assigner(this->table_, std::move(tab)); + return *this; + } + + // user-defined ========================================================= + + // convert using into_toml() method ------------------------------------- + + template::value, std::nullptr_t>::type = nullptr> + basic_value(const T& ud): basic_value(ud.into_toml()) {} + + template::value, std::nullptr_t>::type = nullptr> + basic_value(const T& ud, std::vector com) + : basic_value(ud.into_toml(), std::move(com)) + {} + template::value, std::nullptr_t>::type = nullptr> + basic_value& operator=(const T& ud) + { + *this = ud.into_toml(); + return *this; + } + + // convert using into struct ----------------------------------------- + + template)> + basic_value(const T& ud): basic_value(::toml::into::into_toml(ud)) {} + template)> + basic_value(const T& ud, std::vector com) + : basic_value(::toml::into::into_toml(ud), std::move(com)) + {} + template)> + basic_value& operator=(const T& ud) + { + *this = ::toml::into::into_toml(ud); + return *this; + } + + // for internal use ------------------------------------------------------ + // + // Those constructors take detail::region that contains parse result. + + basic_value(boolean b, detail::region reg, std::vector cm) + : type_(value_t::boolean), + region_info_(std::make_shared(std::move(reg))), + comments_(std::move(cm)) + { + assigner(this->boolean_, b); + } + template, detail::negation> + >::value, std::nullptr_t>::type = nullptr> + basic_value(T i, detail::region reg, std::vector cm) + : type_(value_t::integer), + region_info_(std::make_shared(std::move(reg))), + comments_(std::move(cm)) + { + assigner(this->integer_, static_cast(i)); + } + template::value, std::nullptr_t>::type = nullptr> + basic_value(T f, detail::region reg, std::vector cm) + : type_(value_t::floating), + region_info_(std::make_shared(std::move(reg))), + comments_(std::move(cm)) + { + assigner(this->floating_, static_cast(f)); + } + basic_value(toml::string s, detail::region reg, + std::vector cm) + : type_(value_t::string), + region_info_(std::make_shared(std::move(reg))), + comments_(std::move(cm)) + { + assigner(this->string_, std::move(s)); + } + basic_value(const local_date& ld, detail::region reg, + std::vector cm) + : type_(value_t::local_date), + region_info_(std::make_shared(std::move(reg))), + comments_(std::move(cm)) + { + assigner(this->local_date_, ld); + } + basic_value(const local_time& lt, detail::region reg, + std::vector cm) + : type_(value_t::local_time), + region_info_(std::make_shared(std::move(reg))), + comments_(std::move(cm)) + { + assigner(this->local_time_, lt); + } + basic_value(const local_datetime& ldt, detail::region reg, + std::vector cm) + : type_(value_t::local_datetime), + region_info_(std::make_shared(std::move(reg))), + comments_(std::move(cm)) + { + assigner(this->local_datetime_, ldt); + } + basic_value(const offset_datetime& odt, detail::region reg, + std::vector cm) + : type_(value_t::offset_datetime), + region_info_(std::make_shared(std::move(reg))), + comments_(std::move(cm)) + { + assigner(this->offset_datetime_, odt); + } + basic_value(const array_type& ary, detail::region reg, + std::vector cm) + : type_(value_t::array), + region_info_(std::make_shared(std::move(reg))), + comments_(std::move(cm)) + { + assigner(this->array_, ary); + } + basic_value(const table_type& tab, detail::region reg, + std::vector cm) + : type_(value_t::table), + region_info_(std::make_shared(std::move(reg))), + comments_(std::move(cm)) + { + assigner(this->table_, tab); + } + + template::value, + std::nullptr_t>::type = nullptr> + basic_value(std::pair parse_result, std::vector com) + : basic_value(std::move(parse_result.first), + std::move(parse_result.second), + std::move(com)) + {} + + // type checking and casting ============================================ + + template::value, + std::nullptr_t>::type = nullptr> + bool is() const noexcept + { + return detail::type_to_enum::value == this->type_; + } + bool is(value_t t) const noexcept {return t == this->type_;} + + bool is_uninitialized() const noexcept {return this->is(value_t::empty );} + bool is_boolean() const noexcept {return this->is(value_t::boolean );} + bool is_integer() const noexcept {return this->is(value_t::integer );} + bool is_floating() const noexcept {return this->is(value_t::floating );} + bool is_string() const noexcept {return this->is(value_t::string );} + bool is_offset_datetime() const noexcept {return this->is(value_t::offset_datetime);} + bool is_local_datetime() const noexcept {return this->is(value_t::local_datetime );} + bool is_local_date() const noexcept {return this->is(value_t::local_date );} + bool is_local_time() const noexcept {return this->is(value_t::local_time );} + bool is_array() const noexcept {return this->is(value_t::array );} + bool is_table() const noexcept {return this->is(value_t::table );} + + value_t type() const noexcept {return type_;} + + template + typename detail::enum_to_type::type& cast() & + { + if(this->type_ != T) + { + detail::throw_bad_cast("toml::value::cast: ", this->type_, *this); + } + return detail::switch_cast::invoke(*this); + } + template + typename detail::enum_to_type::type const& cast() const& + { + if(this->type_ != T) + { + detail::throw_bad_cast("toml::value::cast: ", this->type_, *this); + } + return detail::switch_cast::invoke(*this); + } + template + typename detail::enum_to_type::type&& cast() && + { + if(this->type_ != T) + { + detail::throw_bad_cast("toml::value::cast: ", this->type_, *this); + } + return detail::switch_cast::invoke(std::move(*this)); + } + + // ------------------------------------------------------------------------ + // nothrow version + + boolean const& as_boolean (const std::nothrow_t&) const& noexcept {return this->boolean_;} + integer const& as_integer (const std::nothrow_t&) const& noexcept {return this->integer_;} + floating const& as_floating (const std::nothrow_t&) const& noexcept {return this->floating_;} + string const& as_string (const std::nothrow_t&) const& noexcept {return this->string_;} + offset_datetime const& as_offset_datetime(const std::nothrow_t&) const& noexcept {return this->offset_datetime_;} + local_datetime const& as_local_datetime (const std::nothrow_t&) const& noexcept {return this->local_datetime_;} + local_date const& as_local_date (const std::nothrow_t&) const& noexcept {return this->local_date_;} + local_time const& as_local_time (const std::nothrow_t&) const& noexcept {return this->local_time_;} + array_type const& as_array (const std::nothrow_t&) const& noexcept {return this->array_.value();} + table_type const& as_table (const std::nothrow_t&) const& noexcept {return this->table_.value();} + + boolean & as_boolean (const std::nothrow_t&) & noexcept {return this->boolean_;} + integer & as_integer (const std::nothrow_t&) & noexcept {return this->integer_;} + floating & as_floating (const std::nothrow_t&) & noexcept {return this->floating_;} + string & as_string (const std::nothrow_t&) & noexcept {return this->string_;} + offset_datetime& as_offset_datetime(const std::nothrow_t&) & noexcept {return this->offset_datetime_;} + local_datetime & as_local_datetime (const std::nothrow_t&) & noexcept {return this->local_datetime_;} + local_date & as_local_date (const std::nothrow_t&) & noexcept {return this->local_date_;} + local_time & as_local_time (const std::nothrow_t&) & noexcept {return this->local_time_;} + array_type & as_array (const std::nothrow_t&) & noexcept {return this->array_.value();} + table_type & as_table (const std::nothrow_t&) & noexcept {return this->table_.value();} + + boolean && as_boolean (const std::nothrow_t&) && noexcept {return std::move(this->boolean_);} + integer && as_integer (const std::nothrow_t&) && noexcept {return std::move(this->integer_);} + floating && as_floating (const std::nothrow_t&) && noexcept {return std::move(this->floating_);} + string && as_string (const std::nothrow_t&) && noexcept {return std::move(this->string_);} + offset_datetime&& as_offset_datetime(const std::nothrow_t&) && noexcept {return std::move(this->offset_datetime_);} + local_datetime && as_local_datetime (const std::nothrow_t&) && noexcept {return std::move(this->local_datetime_);} + local_date && as_local_date (const std::nothrow_t&) && noexcept {return std::move(this->local_date_);} + local_time && as_local_time (const std::nothrow_t&) && noexcept {return std::move(this->local_time_);} + array_type && as_array (const std::nothrow_t&) && noexcept {return std::move(this->array_.value());} + table_type && as_table (const std::nothrow_t&) && noexcept {return std::move(this->table_.value());} + + // ======================================================================== + // throw version + // ------------------------------------------------------------------------ + // const reference {{{ + + boolean const& as_boolean() const& + { + if(this->type_ != value_t::boolean) + { + detail::throw_bad_cast( + "toml::value::as_boolean(): ", this->type_, *this); + } + return this->boolean_; + } + integer const& as_integer() const& + { + if(this->type_ != value_t::integer) + { + detail::throw_bad_cast( + "toml::value::as_integer(): ", this->type_, *this); + } + return this->integer_; + } + floating const& as_floating() const& + { + if(this->type_ != value_t::floating) + { + detail::throw_bad_cast( + "toml::value::as_floating(): ", this->type_, *this); + } + return this->floating_; + } + string const& as_string() const& + { + if(this->type_ != value_t::string) + { + detail::throw_bad_cast( + "toml::value::as_string(): ", this->type_, *this); + } + return this->string_; + } + offset_datetime const& as_offset_datetime() const& + { + if(this->type_ != value_t::offset_datetime) + { + detail::throw_bad_cast( + "toml::value::as_offset_datetime(): ", this->type_, *this); + } + return this->offset_datetime_; + } + local_datetime const& as_local_datetime() const& + { + if(this->type_ != value_t::local_datetime) + { + detail::throw_bad_cast( + "toml::value::as_local_datetime(): ", this->type_, *this); + } + return this->local_datetime_; + } + local_date const& as_local_date() const& + { + if(this->type_ != value_t::local_date) + { + detail::throw_bad_cast( + "toml::value::as_local_date(): ", this->type_, *this); + } + return this->local_date_; + } + local_time const& as_local_time() const& + { + if(this->type_ != value_t::local_time) + { + detail::throw_bad_cast( + "toml::value::as_local_time(): ", this->type_, *this); + } + return this->local_time_; + } + array_type const& as_array() const& + { + if(this->type_ != value_t::array) + { + detail::throw_bad_cast( + "toml::value::as_array(): ", this->type_, *this); + } + return this->array_.value(); + } + table_type const& as_table() const& + { + if(this->type_ != value_t::table) + { + detail::throw_bad_cast( + "toml::value::as_table(): ", this->type_, *this); + } + return this->table_.value(); + } + // }}} + // ------------------------------------------------------------------------ + // nonconst reference {{{ + + boolean & as_boolean() & + { + if(this->type_ != value_t::boolean) + { + detail::throw_bad_cast( + "toml::value::as_boolean(): ", this->type_, *this); + } + return this->boolean_; + } + integer & as_integer() & + { + if(this->type_ != value_t::integer) + { + detail::throw_bad_cast( + "toml::value::as_integer(): ", this->type_, *this); + } + return this->integer_; + } + floating & as_floating() & + { + if(this->type_ != value_t::floating) + { + detail::throw_bad_cast( + "toml::value::as_floating(): ", this->type_, *this); + } + return this->floating_; + } + string & as_string() & + { + if(this->type_ != value_t::string) + { + detail::throw_bad_cast( + "toml::value::as_string(): ", this->type_, *this); + } + return this->string_; + } + offset_datetime & as_offset_datetime() & + { + if(this->type_ != value_t::offset_datetime) + { + detail::throw_bad_cast( + "toml::value::as_offset_datetime(): ", this->type_, *this); + } + return this->offset_datetime_; + } + local_datetime & as_local_datetime() & + { + if(this->type_ != value_t::local_datetime) + { + detail::throw_bad_cast( + "toml::value::as_local_datetime(): ", this->type_, *this); + } + return this->local_datetime_; + } + local_date & as_local_date() & + { + if(this->type_ != value_t::local_date) + { + detail::throw_bad_cast( + "toml::value::as_local_date(): ", this->type_, *this); + } + return this->local_date_; + } + local_time & as_local_time() & + { + if(this->type_ != value_t::local_time) + { + detail::throw_bad_cast( + "toml::value::as_local_time(): ", this->type_, *this); + } + return this->local_time_; + } + array_type & as_array() & + { + if(this->type_ != value_t::array) + { + detail::throw_bad_cast( + "toml::value::as_array(): ", this->type_, *this); + } + return this->array_.value(); + } + table_type & as_table() & + { + if(this->type_ != value_t::table) + { + detail::throw_bad_cast( + "toml::value::as_table(): ", this->type_, *this); + } + return this->table_.value(); + } + + // }}} + // ------------------------------------------------------------------------ + // rvalue reference {{{ + + boolean && as_boolean() && + { + if(this->type_ != value_t::boolean) + { + detail::throw_bad_cast( + "toml::value::as_boolean(): ", this->type_, *this); + } + return std::move(this->boolean_); + } + integer && as_integer() && + { + if(this->type_ != value_t::integer) + { + detail::throw_bad_cast( + "toml::value::as_integer(): ", this->type_, *this); + } + return std::move(this->integer_); + } + floating && as_floating() && + { + if(this->type_ != value_t::floating) + { + detail::throw_bad_cast( + "toml::value::as_floating(): ", this->type_, *this); + } + return std::move(this->floating_); + } + string && as_string() && + { + if(this->type_ != value_t::string) + { + detail::throw_bad_cast( + "toml::value::as_string(): ", this->type_, *this); + } + return std::move(this->string_); + } + offset_datetime && as_offset_datetime() && + { + if(this->type_ != value_t::offset_datetime) + { + detail::throw_bad_cast( + "toml::value::as_offset_datetime(): ", this->type_, *this); + } + return std::move(this->offset_datetime_); + } + local_datetime && as_local_datetime() && + { + if(this->type_ != value_t::local_datetime) + { + detail::throw_bad_cast( + "toml::value::as_local_datetime(): ", this->type_, *this); + } + return std::move(this->local_datetime_); + } + local_date && as_local_date() && + { + if(this->type_ != value_t::local_date) + { + detail::throw_bad_cast( + "toml::value::as_local_date(): ", this->type_, *this); + } + return std::move(this->local_date_); + } + local_time && as_local_time() && + { + if(this->type_ != value_t::local_time) + { + detail::throw_bad_cast( + "toml::value::as_local_time(): ", this->type_, *this); + } + return std::move(this->local_time_); + } + array_type && as_array() && + { + if(this->type_ != value_t::array) + { + detail::throw_bad_cast( + "toml::value::as_array(): ", this->type_, *this); + } + return std::move(this->array_.value()); + } + table_type && as_table() && + { + if(this->type_ != value_t::table) + { + detail::throw_bad_cast( + "toml::value::as_table(): ", this->type_, *this); + } + return std::move(this->table_.value()); + } + // }}} + + // accessors ============================================================= + // + // may throw type_error or out_of_range + // + value_type& at(const key& k) + { + if(!this->is_table()) + { + detail::throw_bad_cast( + "toml::value::at(key): ", this->type_, *this); + } + if(this->as_table(std::nothrow).count(k) == 0) + { + detail::throw_key_not_found_error(*this, k); + } + return this->as_table(std::nothrow).at(k); + } + value_type const& at(const key& k) const + { + if(!this->is_table()) + { + detail::throw_bad_cast( + "toml::value::at(key): ", this->type_, *this); + } + if(this->as_table(std::nothrow).count(k) == 0) + { + detail::throw_key_not_found_error(*this, k); + } + return this->as_table(std::nothrow).at(k); + } + value_type& operator[](const key& k) + { + if(this->is_uninitialized()) + { + *this = table_type{}; + } + else if(!this->is_table()) // initialized, but not a table + { + detail::throw_bad_cast( + "toml::value::operator[](key): ", this->type_, *this); + } + return this->as_table(std::nothrow)[k]; + } + + value_type& at(const std::size_t idx) + { + if(!this->is_array()) + { + detail::throw_bad_cast( + "toml::value::at(idx): ", this->type_, *this); + } + if(this->as_array(std::nothrow).size() <= idx) + { + throw std::out_of_range(detail::format_underline( + "toml::value::at(idx): no element corresponding to the index", { + {this->location(), concat_to_string("the length is ", + this->as_array(std::nothrow).size(), + ", and the specified index is ", idx)} + })); + } + return this->as_array().at(idx); + } + value_type const& at(const std::size_t idx) const + { + if(!this->is_array()) + { + detail::throw_bad_cast( + "toml::value::at(idx): ", this->type_, *this); + } + if(this->as_array(std::nothrow).size() <= idx) + { + throw std::out_of_range(detail::format_underline( + "toml::value::at(idx): no element corresponding to the index", { + {this->location(), concat_to_string("the length is ", + this->as_array(std::nothrow).size(), + ", and the specified index is ", idx)} + })); + } + return this->as_array(std::nothrow).at(idx); + } + + value_type& operator[](const std::size_t idx) noexcept + { + // no check... + return this->as_array(std::nothrow)[idx]; + } + value_type const& operator[](const std::size_t idx) const noexcept + { + // no check... + return this->as_array(std::nothrow)[idx]; + } + + void push_back(const value_type& x) + { + if(!this->is_array()) + { + detail::throw_bad_cast( + "toml::value::push_back(value): ", this->type_, *this); + } + this->as_array(std::nothrow).push_back(x); + return; + } + void push_back(value_type&& x) + { + if(!this->is_array()) + { + detail::throw_bad_cast( + "toml::value::push_back(value): ", this->type_, *this); + } + this->as_array(std::nothrow).push_back(std::move(x)); + return; + } + + template + value_type& emplace_back(Ts&& ... args) + { + if(!this->is_array()) + { + detail::throw_bad_cast( + "toml::value::emplace_back(...): ", this->type_, *this); + } + this->as_array(std::nothrow).emplace_back(std::forward(args) ...); + return this->as_array(std::nothrow).back(); + } + + std::size_t size() const + { + switch(this->type_) + { + case value_t::array: + { + return this->as_array(std::nothrow).size(); + } + case value_t::table: + { + return this->as_table(std::nothrow).size(); + } + case value_t::string: + { + return this->as_string(std::nothrow).str.size(); + } + default: + { + throw type_error(detail::format_underline( + "toml::value::size(): bad_cast to container types", { + {this->location(), + concat_to_string("the actual type is ", this->type_)} + }), this->location()); + } + } + } + + std::size_t count(const key_type& k) const + { + if(!this->is_table()) + { + detail::throw_bad_cast( + "toml::value::count(key): ", this->type_, *this); + } + return this->as_table(std::nothrow).count(k); + } + + bool contains(const key_type& k) const + { + if(!this->is_table()) + { + detail::throw_bad_cast( + "toml::value::contains(key): ", this->type_, *this); + } + return (this->as_table(std::nothrow).count(k) != 0); + } + + source_location location() const + { + return source_location(this->region_info_.get()); + } + + comment_type const& comments() const noexcept {return this->comments_;} + comment_type& comments() noexcept {return this->comments_;} + + private: + + void cleanup() noexcept + { + switch(this->type_) + { + case value_t::string : {string_.~string(); return;} + case value_t::array : {array_.~array_storage(); return;} + case value_t::table : {table_.~table_storage(); return;} + default : return; + } + } + + // for error messages + template + friend region_base const* detail::get_region(const Value& v); + + template + friend void detail::change_region(Value& v, detail::region reg); + + private: + + using array_storage = detail::storage; + using table_storage = detail::storage; + + value_t type_; + union + { + boolean boolean_; + integer integer_; + floating floating_; + string string_; + offset_datetime offset_datetime_; + local_datetime local_datetime_; + local_date local_date_; + local_time local_time_; + array_storage array_; + table_storage table_; + }; + std::shared_ptr region_info_; + comment_type comments_; +}; + +// default toml::value and default array/table. +// TOML11_DEFAULT_COMMENT_STRATEGY is defined in comments.hpp +using value = basic_value; +using array = typename value::array_type; +using table = typename value::table_type; + +template class T, template class A> +inline bool +operator==(const basic_value& lhs, const basic_value& rhs) +{ + if(lhs.type() != rhs.type()) {return false;} + if(lhs.comments() != rhs.comments()) {return false;} + + switch(lhs.type()) + { + case value_t::boolean : + { + return lhs.as_boolean() == rhs.as_boolean(); + } + case value_t::integer : + { + return lhs.as_integer() == rhs.as_integer(); + } + case value_t::floating : + { + return lhs.as_floating() == rhs.as_floating(); + } + case value_t::string : + { + return lhs.as_string() == rhs.as_string(); + } + case value_t::offset_datetime: + { + return lhs.as_offset_datetime() == rhs.as_offset_datetime(); + } + case value_t::local_datetime: + { + return lhs.as_local_datetime() == rhs.as_local_datetime(); + } + case value_t::local_date: + { + return lhs.as_local_date() == rhs.as_local_date(); + } + case value_t::local_time: + { + return lhs.as_local_time() == rhs.as_local_time(); + } + case value_t::array : + { + return lhs.as_array() == rhs.as_array(); + } + case value_t::table : + { + return lhs.as_table() == rhs.as_table(); + } + case value_t::empty : {return true; } + default: {return false;} + } +} + +template class T, template class A> +inline bool operator!=(const basic_value& lhs, const basic_value& rhs) +{ + return !(lhs == rhs); +} + +template class T, template class A> +typename std::enable_if::array_type>, + detail::is_comparable::table_type> + >::value, bool>::type +operator<(const basic_value& lhs, const basic_value& rhs) +{ + if(lhs.type() != rhs.type()){return (lhs.type() < rhs.type());} + switch(lhs.type()) + { + case value_t::boolean : + { + return lhs.as_boolean() < rhs.as_boolean() || + (lhs.as_boolean() == rhs.as_boolean() && + lhs.comments() < rhs.comments()); + } + case value_t::integer : + { + return lhs.as_integer() < rhs.as_integer() || + (lhs.as_integer() == rhs.as_integer() && + lhs.comments() < rhs.comments()); + } + case value_t::floating : + { + return lhs.as_floating() < rhs.as_floating() || + (lhs.as_floating() == rhs.as_floating() && + lhs.comments() < rhs.comments()); + } + case value_t::string : + { + return lhs.as_string() < rhs.as_string() || + (lhs.as_string() == rhs.as_string() && + lhs.comments() < rhs.comments()); + } + case value_t::offset_datetime: + { + return lhs.as_offset_datetime() < rhs.as_offset_datetime() || + (lhs.as_offset_datetime() == rhs.as_offset_datetime() && + lhs.comments() < rhs.comments()); + } + case value_t::local_datetime: + { + return lhs.as_local_datetime() < rhs.as_local_datetime() || + (lhs.as_local_datetime() == rhs.as_local_datetime() && + lhs.comments() < rhs.comments()); + } + case value_t::local_date: + { + return lhs.as_local_date() < rhs.as_local_date() || + (lhs.as_local_date() == rhs.as_local_date() && + lhs.comments() < rhs.comments()); + } + case value_t::local_time: + { + return lhs.as_local_time() < rhs.as_local_time() || + (lhs.as_local_time() == rhs.as_local_time() && + lhs.comments() < rhs.comments()); + } + case value_t::array : + { + return lhs.as_array() < rhs.as_array() || + (lhs.as_array() == rhs.as_array() && + lhs.comments() < rhs.comments()); + } + case value_t::table : + { + return lhs.as_table() < rhs.as_table() || + (lhs.as_table() == rhs.as_table() && + lhs.comments() < rhs.comments()); + } + case value_t::empty : + { + return lhs.comments() < rhs.comments(); + } + default: + { + return lhs.comments() < rhs.comments(); + } + } +} + +template class T, template class A> +typename std::enable_if::array_type>, + detail::is_comparable::table_type> + >::value, bool>::type +operator<=(const basic_value& lhs, const basic_value& rhs) +{ + return (lhs < rhs) || (lhs == rhs); +} +template class T, template class A> +typename std::enable_if::array_type>, + detail::is_comparable::table_type> + >::value, bool>::type +operator>(const basic_value& lhs, const basic_value& rhs) +{ + return !(lhs <= rhs); +} +template class T, template class A> +typename std::enable_if::array_type>, + detail::is_comparable::table_type> + >::value, bool>::type +operator>=(const basic_value& lhs, const basic_value& rhs) +{ + return !(lhs < rhs); +} + +template class T, template class A> +inline std::string format_error(const std::string& err_msg, + const basic_value& v, const std::string& comment, + std::vector hints = {}, + const bool colorize = TOML11_ERROR_MESSAGE_COLORIZED) +{ + return detail::format_underline(err_msg, {{v.location(), comment}}, + std::move(hints), colorize); +} + +template class T, template class A> +inline std::string format_error(const std::string& err_msg, + const toml::basic_value& v1, const std::string& comment1, + const toml::basic_value& v2, const std::string& comment2, + std::vector hints = {}, + const bool colorize = TOML11_ERROR_MESSAGE_COLORIZED) +{ + return detail::format_underline(err_msg, { + {v1.location(), comment1}, {v2.location(), comment2} + }, std::move(hints), colorize); +} + +template class T, template class A> +inline std::string format_error(const std::string& err_msg, + const toml::basic_value& v1, const std::string& comment1, + const toml::basic_value& v2, const std::string& comment2, + const toml::basic_value& v3, const std::string& comment3, + std::vector hints = {}, + const bool colorize = TOML11_ERROR_MESSAGE_COLORIZED) +{ + return detail::format_underline(err_msg, {{v1.location(), comment1}, + {v2.location(), comment2}, {v3.location(), comment3} + }, std::move(hints), colorize); +} + +template class T, template class A> +detail::return_type_of_t +visit(Visitor&& visitor, const toml::basic_value& v) +{ + switch(v.type()) + { + case value_t::boolean : {return visitor(v.as_boolean ());} + case value_t::integer : {return visitor(v.as_integer ());} + case value_t::floating : {return visitor(v.as_floating ());} + case value_t::string : {return visitor(v.as_string ());} + case value_t::offset_datetime: {return visitor(v.as_offset_datetime());} + case value_t::local_datetime : {return visitor(v.as_local_datetime ());} + case value_t::local_date : {return visitor(v.as_local_date ());} + case value_t::local_time : {return visitor(v.as_local_time ());} + case value_t::array : {return visitor(v.as_array ());} + case value_t::table : {return visitor(v.as_table ());} + case value_t::empty : break; + default: break; + } + throw std::runtime_error(format_error("[error] toml::visit: toml::basic_value " + "does not have any valid basic_value.", v, "here")); +} + +template class T, template class A> +detail::return_type_of_t +visit(Visitor&& visitor, toml::basic_value& v) +{ + switch(v.type()) + { + case value_t::boolean : {return visitor(v.as_boolean ());} + case value_t::integer : {return visitor(v.as_integer ());} + case value_t::floating : {return visitor(v.as_floating ());} + case value_t::string : {return visitor(v.as_string ());} + case value_t::offset_datetime: {return visitor(v.as_offset_datetime());} + case value_t::local_datetime : {return visitor(v.as_local_datetime ());} + case value_t::local_date : {return visitor(v.as_local_date ());} + case value_t::local_time : {return visitor(v.as_local_time ());} + case value_t::array : {return visitor(v.as_array ());} + case value_t::table : {return visitor(v.as_table ());} + case value_t::empty : break; + default: break; + } + throw std::runtime_error(format_error("[error] toml::visit: toml::basic_value " + "does not have any valid basic_value.", v, "here")); +} + +template class T, template class A> +detail::return_type_of_t +visit(Visitor&& visitor, toml::basic_value&& v) +{ + switch(v.type()) + { + case value_t::boolean : {return visitor(std::move(v.as_boolean ()));} + case value_t::integer : {return visitor(std::move(v.as_integer ()));} + case value_t::floating : {return visitor(std::move(v.as_floating ()));} + case value_t::string : {return visitor(std::move(v.as_string ()));} + case value_t::offset_datetime: {return visitor(std::move(v.as_offset_datetime()));} + case value_t::local_datetime : {return visitor(std::move(v.as_local_datetime ()));} + case value_t::local_date : {return visitor(std::move(v.as_local_date ()));} + case value_t::local_time : {return visitor(std::move(v.as_local_time ()));} + case value_t::array : {return visitor(std::move(v.as_array ()));} + case value_t::table : {return visitor(std::move(v.as_table ()));} + case value_t::empty : break; + default: break; + } + throw std::runtime_error(format_error("[error] toml::visit: toml::basic_value " + "does not have any valid basic_value.", v, "here")); +} + +}// toml +#endif// TOML11_VALUE From 10a8b5d3ae65cbf842faa83a91e7805276cb799e Mon Sep 17 00:00:00 2001 From: Nicolas Mattia Date: Fri, 17 Dec 2021 22:03:48 +0100 Subject: [PATCH 094/117] Update primops/fromTOML.cc to use toml11 --- src/libexpr/primops/fromTOML.cc | 113 +++++++++++++++----------------- 1 file changed, 53 insertions(+), 60 deletions(-) diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 4c6682dfd..8fba0bf04 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -1,86 +1,79 @@ #include "primops.hh" #include "eval-inline.hh" -#include "../../cpptoml/cpptoml.h" +#include "../../toml11/toml.hpp" namespace nix { -static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Value & v) +static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Value & val) { - using namespace cpptoml; - auto toml = state.forceStringNoCtx(*args[0], pos); std::istringstream tomlStream(toml); - std::function)> visit; + std::function visit; - visit = [&](Value & v, std::shared_ptr t) { + visit = [&](Value & v, toml::value t) { - if (auto t2 = t->as_table()) { + switch(t.type()) + { + case toml::value_t::table: + { + auto table = toml::get(t); - size_t size = 0; - for (auto & i : *t2) { (void) i; size++; } + size_t size = 0; + for (auto & i : table) { (void) i; size++; } - state.mkAttrs(v, size); + state.mkAttrs(v, size); - for (auto & i : *t2) { - auto & v2 = *state.allocAttr(v, state.symbols.create(i.first)); + for(auto & elem: table) { - if (auto i2 = i.second->as_table_array()) { - size_t size2 = i2->get().size(); - state.mkList(v2, size2); - for (size_t j = 0; j < size2; ++j) - visit(*(v2.listElems()[j] = state.allocValue()), i2->get()[j]); + auto & v2 = *state.allocAttr(v, state.symbols.create(elem.first)); + + // TODO: note about creating children here in old code + visit(v2, elem.second); + } } - else - visit(v2, i.second); - } + break;; + case toml::value_t::array: + { + auto array = toml::get>(t); + + size_t size = array.size(); + state.mkList(v, size); + for (size_t i = 0; i < size; ++i) + visit(*(v.listElems()[i] = state.allocValue()), array[i]); + } + break;; + case toml::value_t::boolean: + mkBool(v, toml::get(t)); + break;; + case toml::value_t::integer: + mkInt(v, toml::get(t)); + break;; + case toml::value_t::floating: + mkFloat(v, toml::get(t)); + break;; + case toml::value_t::string: + mkString(v, toml::get(t)); + break;; + case toml::value_t::local_datetime: + case toml::value_t::offset_datetime: + case toml::value_t::local_date: + case toml::value_t::local_time: + // TODO: convert to string? + throw std::runtime_error("Dates and times are not supported"); + break;; + case toml::value_t::empty: + mkNull(v); + break;; - v.attrs->sort(); } - - else if (auto t2 = t->as_array()) { - size_t size = t2->get().size(); - - state.mkList(v, size); - - for (size_t i = 0; i < size; ++i) - visit(*(v.listElems()[i] = state.allocValue()), t2->get()[i]); - } - - // Handle cases like 'a = [[{ a = true }]]', which IMHO should be - // parsed as a array containing an array containing a table, - // but instead are parsed as an array containing a table array - // containing a table. - else if (auto t2 = t->as_table_array()) { - size_t size = t2->get().size(); - - state.mkList(v, size); - - for (size_t j = 0; j < size; ++j) - visit(*(v.listElems()[j] = state.allocValue()), t2->get()[j]); - } - - else if (t->is_value()) { - if (auto val = t->as()) - mkInt(v, val->get()); - else if (auto val = t->as()) - mkFloat(v, val->get()); - else if (auto val = t->as()) - mkBool(v, val->get()); - else if (auto val = t->as()) - mkString(v, val->get()); - else - throw EvalError("unsupported value type in TOML"); - } - - else abort(); }; try { - visit(v, parser(tomlStream).parse()); - } catch (std::runtime_error & e) { + visit(val, toml::parse(tomlStream)); // TODO give filename + } catch (std::exception & e) { // TODO: toml::syntax_error throw EvalError({ .msg = hintfmt("while parsing a TOML string: %s", e.what()), .errPos = pos From 5d06836b9f8b675bdbaa1ce2497d704cfe5e2456 Mon Sep 17 00:00:00 2001 From: Nicolas Mattia Date: Fri, 17 Dec 2021 23:15:26 +0100 Subject: [PATCH 095/117] Use long in fromTOML --- src/libexpr/primops/fromTOML.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 8fba0bf04..4ec64daf2 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -49,7 +49,7 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va mkBool(v, toml::get(t)); break;; case toml::value_t::integer: - mkInt(v, toml::get(t)); + mkInt(v, toml::get(t)); break;; case toml::value_t::floating: mkFloat(v, toml::get(t)); From 1827cf5a9b8fa0f8f1dff1ee775f67f7f326606c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Mon, 20 Dec 2021 19:57:48 +0100 Subject: [PATCH 096/117] Add shell completion for why-depends Fixes #5806 --- src/nix/why-depends.cc | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 2f6b361bb..e27f56ef5 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -34,8 +34,21 @@ struct CmdWhyDepends : SourceExprCommand CmdWhyDepends() { - expectArg("package", &_package); - expectArg("dependency", &_dependency); + expectArgs({ + .label = "package", + .handler = {&_package}, + .completer = {[&](size_t, std::string_view prefix) { + completeInstallable(prefix); + }} + }); + + expectArgs({ + .label = "dependency", + .handler = {&_dependency}, + .completer = {[&](size_t, std::string_view prefix) { + completeInstallable(prefix); + }} + }); addFlag({ .longName = "all", From 9dea5768ef0e3d11725926a96eeadac81c0e2543 Mon Sep 17 00:00:00 2001 From: Nicolas Mattia Date: Mon, 20 Dec 2021 21:39:37 +0100 Subject: [PATCH 097/117] Clean up toml parsing code --- src/libexpr/primops/fromTOML.cc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 4ec64daf2..bfea1ce34 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -29,8 +29,6 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va for(auto & elem: table) { auto & v2 = *state.allocAttr(v, state.symbols.create(elem.first)); - - // TODO: note about creating children here in old code visit(v2, elem.second); } } @@ -61,7 +59,7 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va case toml::value_t::offset_datetime: case toml::value_t::local_date: case toml::value_t::local_time: - // TODO: convert to string? + // We fail since Nix doesn't have date and time types throw std::runtime_error("Dates and times are not supported"); break;; case toml::value_t::empty: @@ -72,7 +70,7 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va }; try { - visit(val, toml::parse(tomlStream)); // TODO give filename + visit(val, toml::parse(tomlStream, "fromTOML" /* the "filename" */)); } catch (std::exception & e) { // TODO: toml::syntax_error throw EvalError({ .msg = hintfmt("while parsing a TOML string: %s", e.what()), From 09b245690a9ab40847e3faa96646e4bd6ce033e0 Mon Sep 17 00:00:00 2001 From: pennae Date: Mon, 20 Dec 2021 13:28:54 +0100 Subject: [PATCH 098/117] bulk-allocate Value instances in the evaluator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit calling GC_malloc for each value is significantly more expensive than allocating a bunch of values at once with GC_malloc_many. "a bunch" here is a GC block size, ie 16KiB or less. this gives a 1.5% performance boost when evaluating our nixos system. tested with nix eval --raw --impure --expr 'with import {}; system' # on master Time (mean ± σ): 3.335 s ± 0.007 s [User: 2.774 s, System: 0.293 s] Range (min … max): 3.315 s … 3.347 s 50 runs # with this change Time (mean ± σ): 3.288 s ± 0.006 s [User: 2.728 s, System: 0.292 s] Range (min … max): 3.274 s … 3.307 s 50 runs --- src/libexpr/eval.cc | 17 ++++++++++++++++- src/libexpr/eval.hh | 3 +++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 8cf4d9549..a95726f5f 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -821,8 +821,23 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) Value * EvalState::allocValue() { + /* We use the boehm batch allocator to speed up allocations of Values (of which there are many). + GC_malloc_many returns a linked list of objects of the given size, where the first word + of each object is also the pointer to the next object in the list. This also means that we + have to explicitly clear the first word of every object we take. */ + if (!valueAllocCache) { + valueAllocCache = GC_malloc_many(sizeof(Value)); + if (!valueAllocCache) throw std::bad_alloc(); + } + + /* GC_NEXT is a convenience macro for accessing the first word of an object. + Take the first list item, advance the list to the next item, and clear the next pointer. */ + void * p = valueAllocCache; + GC_PTR_STORE_AND_DIRTY(&valueAllocCache, GC_NEXT(p)); + GC_NEXT(p) = nullptr; + nrValues++; - auto v = (Value *) allocBytes(sizeof(Value)); + auto v = (Value *) p; return v; } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 1aab8e166..cc63294c6 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -133,6 +133,9 @@ private: /* Cache used by prim_match(). */ std::shared_ptr regexCache; + /* Allocation cache for GC'd Value objects. */ + void * valueAllocCache = nullptr; + public: EvalState( From cbbd21ec073781f68029daff153dac2516dafc23 Mon Sep 17 00:00:00 2001 From: regnat Date: Tue, 21 Dec 2021 08:42:19 +0100 Subject: [PATCH 099/117] Factor out the path realisation bit of IFD --- src/libexpr/primops.cc | 75 +++++++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 66af373d7..5f00fad1c 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -76,6 +76,19 @@ void EvalState::realiseContext(const PathSet & context) } } +static Path realisePath(EvalState & state, const Pos & pos, Value & v, bool requireAbsolutePath = true) +{ + PathSet context; + + Path path = requireAbsolutePath + ? state.coerceToPath(pos, v, context) + : state.coerceToString(pos, v, context, false, false); + + state.realiseContext(context); + + return state.checkSourcePath(state.toRealPath(path, context)); +} + /* Add and attribute to the given attribute map from the output name to the output path, or a placeholder. @@ -109,11 +122,9 @@ static void mkOutputString(EvalState & state, Value & v, argument. */ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vScope, Value & v) { - PathSet context; - Path path = state.coerceToPath(pos, vPath, context); - + Path path; try { - state.realiseContext(context); + path = realisePath(state, pos, vPath); } catch (InvalidPathError & e) { throw EvalError({ .msg = hintfmt("cannot import '%1%', since path '%2%' is not valid", path, e.path), @@ -124,8 +135,6 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS throw; } - Path realPath = state.checkSourcePath(state.toRealPath(path, context)); - // FIXME auto isValidDerivationInStore = [&]() -> std::optional { if (!state.store->isStorePath(path)) @@ -177,7 +186,7 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS else { if (!vScope) - state.evalFile(realPath, v); + state.evalFile(path, v); else { state.forceAttrs(*vScope); @@ -195,8 +204,8 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS // No need to call staticEnv.sort(), because // args[0]->attrs is already sorted. - printTalkative("evaluating file '%1%'", realPath); - Expr * e = state.parseExprFromFile(resolveExprPath(realPath), staticEnv); + printTalkative("evaluating file '%1%'", path); + Expr * e = state.parseExprFromFile(resolveExprPath(path), staticEnv); e->eval(state, *env, v); } @@ -281,22 +290,19 @@ extern "C" typedef void (*ValueInitializer)(EvalState & state, Value & v); /* Load a ValueInitializer from a DSO and return whatever it initializes */ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v) { - PathSet context; - Path path = state.coerceToPath(pos, *args[0], context); - + Path path; try { - state.realiseContext(context); + path = realisePath(state, pos, *args[0]); } catch (InvalidPathError & e) { throw EvalError({ - .msg = hintfmt( - "cannot import '%1%', since path '%2%' is not valid", - path, e.path), + .msg = hintfmt("cannot import '%1%', since path '%2%' is not valid", path, e.path), .errPos = pos }); + } catch (Error & e) { + e.addTrace(pos, "while importing '%s'", path); + throw; } - path = state.checkSourcePath(path); - string sym = state.forceStringNoCtx(*args[1], pos); void *handle = dlopen(path.c_str(), RTLD_LAZY | RTLD_LOCAL); @@ -1349,10 +1355,9 @@ static RegisterPrimOp primop_storePath({ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args, Value & v) { - PathSet context; - Path path = state.coerceToPath(pos, *args[0], context); + Path path; try { - state.realiseContext(context); + path = realisePath(state, pos, *args[0]); } catch (InvalidPathError & e) { throw EvalError({ .msg = hintfmt( @@ -1363,7 +1368,7 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args, } try { - mkBool(v, pathExists(state.checkSourcePath(path))); + mkBool(v, pathExists(path)); } catch (SysError & e) { /* Don't give away info from errors while canonicalising ‘path’ in restricted mode. */ @@ -1426,17 +1431,16 @@ static RegisterPrimOp primop_dirOf({ /* Return the contents of a file as a string. */ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Value & v) { - PathSet context; - Path path = state.coerceToPath(pos, *args[0], context); + Path path; try { - state.realiseContext(context); + path = realisePath(state, pos, *args[0]); } catch (InvalidPathError & e) { throw EvalError({ .msg = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path), .errPos = pos }); } - string s = readFile(state.checkSourcePath(state.toRealPath(path, context))); + string s = readFile(path); if (s.find((char) 0) != string::npos) throw Error("the contents of the file '%1%' cannot be represented as a Nix string", path); mkString(v, s.c_str()); @@ -1475,11 +1479,10 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va pos ); - PathSet context; - string path = state.coerceToString(pos, *i->value, context, false, false); + Path path; try { - state.realiseContext(context); + path = realisePath(state, pos, *i->value, false); } catch (InvalidPathError & e) { throw EvalError({ .msg = hintfmt("cannot find '%1%', since path '%2%' is not valid", path, e.path), @@ -1512,15 +1515,14 @@ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Va .errPos = pos }); - PathSet context; - Path path = state.coerceToPath(pos, *args[1], context); + Path path; try { - state.realiseContext(context); + path = realisePath(state, pos, *args[1]); } catch (InvalidPathError & e) { throw EvalError("cannot read '%s' since path '%s' is not valid, at %s", path, e.path, pos); } - mkString(v, hashFile(*ht, state.checkSourcePath(state.toRealPath(path, context))).to_string(Base16, false)); + mkString(v, hashFile(*ht, path).to_string(Base16, false)); } static RegisterPrimOp primop_hashFile({ @@ -1537,10 +1539,9 @@ static RegisterPrimOp primop_hashFile({ /* Read a directory (without . or ..) */ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Value & v) { - PathSet ctx; - Path path = state.coerceToPath(pos, *args[0], ctx); + Path path; try { - state.realiseContext(ctx); + path = realisePath(state, pos, *args[0]); } catch (InvalidPathError & e) { throw EvalError({ .msg = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path), @@ -1548,7 +1549,7 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val }); } - DirEntries entries = readDirectory(state.checkSourcePath(path)); + DirEntries entries = readDirectory(path); state.mkAttrs(v, entries.size()); for (auto & ent : entries) { From d90f9d4b9994dc1f15b9d664ae313f06261d6058 Mon Sep 17 00:00:00 2001 From: regnat Date: Mon, 20 Dec 2021 19:46:55 +0100 Subject: [PATCH 100/117] Fix IFD with CA derivations Rewrite the string taken by the IFD-like primops to contain the actual output paths of the derivations rather than the placeholders Fix #5805 --- src/libexpr/eval.hh | 5 +++- src/libexpr/primops.cc | 43 +++++++++++++++++++++++------------ tests/ca/import-derivation.sh | 6 +++++ tests/local.mk | 2 +- 4 files changed, 40 insertions(+), 16 deletions(-) create mode 100644 tests/ca/import-derivation.sh diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 1aab8e166..0ba570434 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -350,7 +350,10 @@ public: /* Print statistics. */ void printStats(); - void realiseContext(const PathSet & context); + /* Realise the given context, and return a mapping from the placeholders + * used to construct the associated value to their final store path + */ + [[nodiscard]] StringMap realiseContext(const PathSet & context); private: diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 5f00fad1c..aff8f951e 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -35,9 +35,10 @@ namespace nix { InvalidPathError::InvalidPathError(const Path & path) : EvalError("path '%s' is not valid", path), path(path) {} -void EvalState::realiseContext(const PathSet & context) +StringMap EvalState::realiseContext(const PathSet & context) { std::vector drvs; + StringMap res; for (auto & i : context) { auto [ctxS, outputName] = decodeContext(i); @@ -46,10 +47,12 @@ void EvalState::realiseContext(const PathSet & context) throw InvalidPathError(store->printStorePath(ctx)); if (!outputName.empty() && ctx.isDerivation()) { drvs.push_back({ctx, {outputName}}); + } else { + res.insert_or_assign(ctxS, ctxS); } } - if (drvs.empty()) return; + if (drvs.empty()) return {}; if (!evalSettings.enableImportFromDerivation) throw Error( @@ -61,19 +64,29 @@ void EvalState::realiseContext(const PathSet & context) for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d }); store->buildPaths(buildReqs); + /* Get all the output paths corresponding to the placeholders we had */ + for (auto & [drvPath, outputs] : drvs) { + auto outputPaths = store->queryDerivationOutputMap(drvPath); + for (auto & outputName : outputs) { + if (outputPaths.count(outputName) == 0) + throw Error("derivation '%s' does not have an output named '%s'", + store->printStorePath(drvPath), outputName); + res.insert_or_assign( + downstreamPlaceholder(*store, drvPath, outputName), + store->printStorePath(outputPaths.at(outputName)) + ); + } + } + /* Add the output of this derivations to the allowed paths. */ if (allowedPaths) { - for (auto & [drvPath, outputs] : drvs) { - auto outputPaths = store->queryDerivationOutputMap(drvPath); - for (auto & outputName : outputs) { - if (outputPaths.count(outputName) == 0) - throw Error("derivation '%s' does not have an output named '%s'", - store->printStorePath(drvPath), outputName); - allowPath(outputPaths.at(outputName)); - } + for (auto & [_placeholder, outputPath] : res) { + allowPath(outputPath); } } + + return res; } static Path realisePath(EvalState & state, const Pos & pos, Value & v, bool requireAbsolutePath = true) @@ -84,9 +97,10 @@ static Path realisePath(EvalState & state, const Pos & pos, Value & v, bool requ ? state.coerceToPath(pos, v, context) : state.coerceToString(pos, v, context, false, false); - state.realiseContext(context); + StringMap rewrites = state.realiseContext(context); - return state.checkSourcePath(state.toRealPath(path, context)); + return state.checkSourcePath( + state.toRealPath(rewriteStrings(path, rewrites), context)); } /* Add and attribute to the given attribute map from the output name to @@ -344,7 +358,7 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v) for (unsigned int i = 1; i < args[0]->listSize(); ++i) commandArgs.emplace_back(state.coerceToString(pos, *elems[i], context, false, false)); try { - state.realiseContext(context); + auto _ = state.realiseContext(context); // FIXME: Handle CA derivations } catch (InvalidPathError & e) { throw EvalError({ .msg = hintfmt("cannot execute '%1%', since path '%2%' is not valid", @@ -1876,7 +1890,8 @@ static void addPath( try { // FIXME: handle CA derivation outputs (where path needs to // be rewritten to the actual output). - state.realiseContext(context); + auto rewrites = state.realiseContext(context); + path = state.toRealPath(rewriteStrings(path, rewrites), context); StorePathSet refs; diff --git a/tests/ca/import-derivation.sh b/tests/ca/import-derivation.sh new file mode 100644 index 000000000..e98e0fbd0 --- /dev/null +++ b/tests/ca/import-derivation.sh @@ -0,0 +1,6 @@ +source common.sh + +export NIX_TESTS_CA_BY_DEFAULT=1 + +cd .. && source import-derivation.sh + diff --git a/tests/local.mk b/tests/local.mk index 936b72c2a..c22e779a6 100644 --- a/tests/local.mk +++ b/tests/local.mk @@ -11,7 +11,7 @@ nix_tests = \ local-store.sh remote-store.sh export.sh export-graph.sh \ db-migration.sh \ timeout.sh secure-drv-outputs.sh nix-channel.sh \ - multiple-outputs.sh import-derivation.sh fetchurl.sh optimise-store.sh \ + multiple-outputs.sh import-derivation.sh ca/import-derivation.sh fetchurl.sh optimise-store.sh \ binary-cache.sh \ substitute-with-invalid-ca.sh \ binary-cache-build-remote.sh \ From 881b610266f553dd45d398cc7c5cd0d3d1491ee5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 21 Dec 2021 14:19:21 +0100 Subject: [PATCH 101/117] Update release notes --- doc/manual/src/release-notes/rl-next.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index c869b5e2f..ac662c132 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -1 +1,4 @@ # Release X.Y (202?-??-??) + +* The TOML parser used by `builtins.fromTOML` has been replaced by [a + more compliant one](https://github.com/ToruNiina/toml11). From 534bc5a3d1fba33f25f9921e8cea7ac8e98288c5 Mon Sep 17 00:00:00 2001 From: Rok Garbas Date: Tue, 21 Dec 2021 21:17:29 +0100 Subject: [PATCH 102/117] Add cacert to /etc/ssl/certs in the docker image Fixes #5797 --- docker.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker.nix b/docker.nix index ffb553f78..25930bb87 100644 --- a/docker.nix +++ b/docker.nix @@ -180,6 +180,9 @@ let set -x mkdir -p $out/etc + mkdir -p $out/etc/ssl/certs + ln -s /nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt $out/etc/ssl/certs + cat $passwdContentsPath > $out/etc/passwd echo "" >> $out/etc/passwd From 0a7c1da9f3182e685a08eadd6185803a83a16b59 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Dec 2021 13:21:45 +0100 Subject: [PATCH 103/117] Make bash completion for 'nix' work in a devshell --- flake.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flake.nix b/flake.nix index 806ed45de..f4fae2a00 100644 --- a/flake.nix +++ b/flake.nix @@ -667,6 +667,9 @@ PATH=$prefix/bin:$PATH unset PYTHONPATH export MANPATH=$out/share/man:$MANPATH + + # Make bash completion work. + XDG_DATA_DIRS+=:$out/share ''; }); From 5b5f68f1dd4395f0ef7231218295d1c633d4288b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Dec 2021 13:26:35 +0100 Subject: [PATCH 104/117] Make bash completion work when nix is invoked as ~/.../nix --- misc/bash/completion.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/bash/completion.sh b/misc/bash/completion.sh index bea2a40bc..0af90d8c1 100644 --- a/misc/bash/completion.sh +++ b/misc/bash/completion.sh @@ -13,7 +13,7 @@ function _complete_nix { else COMPREPLY+=("$completion") fi - done < <(NIX_GET_COMPLETIONS=$cword "${words[@]}") + done < <(NIX_GET_COMPLETIONS=$cword "${words[@]/#\~/$HOME}") __ltrim_colon_completions "$cur" } From 9747ea84b43c75f719d719673b9d0a7cb50d34e5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 22 Dec 2021 15:49:51 +0100 Subject: [PATCH 105/117] Remove CPU locking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This was already accidentally disabled in ba87b08. It also no longer appears to be beneficial, and in fact slow things down, e.g. when evaluating a NixOS system configuration: elapsed time: median = 3.8170 mean = 3.8202 stddev = 0.0195 min = 3.7894 max = 3.8600 [rejected, p=0.00000, Δ=0.36929±0.02513] --- src/libstore/daemon.cc | 5 ++- src/libstore/remote-store.cc | 8 ++--- src/libutil/affinity.cc | 70 ------------------------------------ src/libutil/affinity.hh | 9 ----- src/libutil/thread-pool.cc | 3 -- src/libutil/util.cc | 4 --- src/libutil/util.hh | 2 +- src/nix-build/nix-build.cc | 1 - src/nix/develop.cc | 1 - src/nix/repl.cc | 1 - src/nix/run.cc | 1 - 11 files changed, 5 insertions(+), 100 deletions(-) delete mode 100644 src/libutil/affinity.cc delete mode 100644 src/libutil/affinity.hh diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index bafab6fd5..59325da79 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -4,7 +4,6 @@ #include "store-api.hh" #include "path-with-outputs.hh" #include "finally.hh" -#include "affinity.hh" #include "archive.hh" #include "derivations.hh" #include "args.hh" @@ -960,8 +959,8 @@ void processConnection( }); if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) { - auto affinity = readInt(from); - setAffinityTo(affinity); + // Obsolete CPU affinity. + readInt(from); } readInt(from); // obsolete reserveSpace diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 7f7e973e9..57cc260b0 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -5,7 +5,6 @@ #include "remote-store.hh" #include "worker-protocol.hh" #include "archive.hh" -#include "affinity.hh" #include "globals.hh" #include "derivations.hh" #include "pool.hh" @@ -184,11 +183,8 @@ void RemoteStore::initConnection(Connection & conn) conn.to << PROTOCOL_VERSION; if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 14) { - int cpu = sameMachine() && settings.lockCPU ? lockToCurrentCPU() : -1; - if (cpu != -1) - conn.to << 1 << cpu; - else - conn.to << 0; + // Obsolete CPU affinity. + conn.to << 0; } if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11) diff --git a/src/libutil/affinity.cc b/src/libutil/affinity.cc deleted file mode 100644 index ac2295e4a..000000000 --- a/src/libutil/affinity.cc +++ /dev/null @@ -1,70 +0,0 @@ -#include "types.hh" -#include "util.hh" -#include "affinity.hh" - -#if __linux__ -#include -#endif - -namespace nix { - - -#if __linux__ -static bool didSaveAffinity = false; -static cpu_set_t savedAffinity; - -std::ostream& operator<<(std::ostream &os, const cpu_set_t &cset) -{ - auto count = CPU_COUNT(&cset); - for (int i=0; i < count; ++i) - { - os << (CPU_ISSET(i,&cset) ? "1" : "0"); - } - - return os; -} -#endif - - -void setAffinityTo(int cpu) -{ -#if __linux__ - if (sched_getaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) return; - didSaveAffinity = true; - debug(format("locking this thread to CPU %1%") % cpu); - cpu_set_t newAffinity; - CPU_ZERO(&newAffinity); - CPU_SET(cpu, &newAffinity); - if (sched_setaffinity(0, sizeof(cpu_set_t), &newAffinity) == -1) - printError("failed to lock thread to CPU %1%", cpu); -#endif -} - - -int lockToCurrentCPU() -{ -#if __linux__ - int cpu = sched_getcpu(); - if (cpu != -1) setAffinityTo(cpu); - return cpu; -#else - return -1; -#endif -} - - -void restoreAffinity() -{ -#if __linux__ - if (!didSaveAffinity) return; - if (sched_setaffinity(0, sizeof(cpu_set_t), &savedAffinity) == -1) - { - std::ostringstream oss; - oss << savedAffinity; - printError("failed to restore CPU affinity %1%", oss.str()); - } -#endif -} - - -} diff --git a/src/libutil/affinity.hh b/src/libutil/affinity.hh deleted file mode 100644 index c1bd28e13..000000000 --- a/src/libutil/affinity.hh +++ /dev/null @@ -1,9 +0,0 @@ -#pragma once - -namespace nix { - -void setAffinityTo(int cpu); -int lockToCurrentCPU(); -void restoreAffinity(); - -} diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc index 857ee91f8..dc4067f1b 100644 --- a/src/libutil/thread-pool.cc +++ b/src/libutil/thread-pool.cc @@ -1,13 +1,10 @@ #include "thread-pool.hh" -#include "affinity.hh" namespace nix { ThreadPool::ThreadPool(size_t _maxThreads) : maxThreads(_maxThreads) { - restoreAffinity(); // FIXME - if (!maxThreads) { maxThreads = std::thread::hardware_concurrency(); if (!maxThreads) maxThreads = 1; diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 43fea1b1e..e18648557 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1,5 +1,4 @@ #include "util.hh" -#include "affinity.hh" #include "sync.hh" #include "finally.hh" #include "serialise.hh" @@ -1004,7 +1003,6 @@ pid_t startProcess(std::function fun, const ProcessOptions & options) if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) throw SysError("setting death signal"); #endif - restoreAffinity(); fun(); } catch (std::exception & e) { try { @@ -1675,8 +1673,6 @@ void restoreProcessContext(bool restoreMounts) restoreMountNamespace(); } - restoreAffinity(); - #if __linux__ if (savedStackSize) { struct rlimit limit; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 4cc043a84..d08f42826 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -300,7 +300,7 @@ void setStackSize(size_t stackSize); /* Restore the original inherited Unix process context (such as signal - masks, stack size, CPU affinity). */ + masks, stack size). */ void restoreProcessContext(bool restoreMounts = true); /* Save the current mount namespace. Ignored if called more than diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index e2325c91f..d3d6ce1ce 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -14,7 +14,6 @@ #include "local-fs-store.hh" #include "globals.hh" #include "derivations.hh" -#include "affinity.hh" #include "util.hh" #include "shared.hh" #include "path-with-outputs.hh" diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 5aad53919..a8ca1cac2 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -5,7 +5,6 @@ #include "store-api.hh" #include "path-with-outputs.hh" #include "derivations.hh" -#include "affinity.hh" #include "progress-bar.hh" #include "run.hh" diff --git a/src/nix/repl.cc b/src/nix/repl.cc index f453343f3..63ccbfda3 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -28,7 +28,6 @@ extern "C" { #include "common-eval-args.hh" #include "get-drvs.hh" #include "derivations.hh" -#include "affinity.hh" #include "globals.hh" #include "command.hh" #include "finally.hh" diff --git a/src/nix/run.cc b/src/nix/run.cc index b01fdebaa..bae64ed39 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -8,7 +8,6 @@ #include "finally.hh" #include "fs-accessor.hh" #include "progress-bar.hh" -#include "affinity.hh" #include "eval.hh" #if __linux__ From 1da1b2b345ccf32220a2628622ee8b170d9d521a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Wed, 22 Dec 2021 12:37:59 +0100 Subject: [PATCH 106/117] Don't insert spaces when completing attribute paths --- misc/bash/completion.sh | 4 +++- misc/fish/completion.fish | 1 - src/libcmd/installables.cc | 10 +++++++--- src/libutil/args.cc | 4 ++-- src/libutil/args.hh | 8 +++++++- src/nix/main.cc | 9 ++++++++- 6 files changed, 27 insertions(+), 9 deletions(-) diff --git a/misc/bash/completion.sh b/misc/bash/completion.sh index bea2a40bc..3eb44dfa6 100644 --- a/misc/bash/completion.sh +++ b/misc/bash/completion.sh @@ -7,8 +7,10 @@ function _complete_nix { local completion=${line%% *} if [[ -z $have_type ]]; then have_type=1 - if [[ $completion = filenames ]]; then + if [[ $completion == filenames ]]; then compopt -o filenames + elif [[ $completion == attrs ]]; then + compopt -o nospace fi else COMPREPLY+=("$completion") diff --git a/misc/fish/completion.fish b/misc/fish/completion.fish index bedbefaf8..c6b8ef16a 100644 --- a/misc/fish/completion.fish +++ b/misc/fish/completion.fish @@ -19,7 +19,6 @@ end function _nix_accepts_files set -l response (_nix_complete) - # First line is either filenames or no-filenames. test $response[1] = 'filenames' end diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index ef200b1d2..5f118b8d0 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -203,6 +203,8 @@ void SourceExprCommand::completeInstallable(std::string_view prefix) Value v2; state->autoCallFunction(*autoArgs, v1, v2); + completionType = ctAttrs; + if (v2.type() == nAttrs) { for (auto & i : *v2.attrs) { std::string name = i.name; @@ -232,7 +234,9 @@ void completeFlakeRefWithFragment( prefix. */ try { auto hash = prefix.find('#'); - if (hash != std::string::npos) { + if (hash == std::string::npos) { + completeFlakeRef(evalState->store, prefix); + } else { auto fragment = prefix.substr(hash + 1); auto flakeRefS = std::string(prefix.substr(0, hash)); // FIXME: do tilde expansion. @@ -248,6 +252,8 @@ void completeFlakeRefWithFragment( flake. */ attrPathPrefixes.push_back(""); + completionType = ctAttrs; + for (auto & attrPathPrefixS : attrPathPrefixes) { auto attrPathPrefix = parseAttrPath(*evalState, attrPathPrefixS); auto attrPathS = attrPathPrefixS + std::string(fragment); @@ -285,8 +291,6 @@ void completeFlakeRefWithFragment( } catch (Error & e) { warn(e.msg()); } - - completeFlakeRef(evalState->store, prefix); } void completeFlakeRef(ref store, std::string_view prefix) diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 9df279faf..a739d6b4e 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -39,7 +39,7 @@ void Completions::add(std::string completion, std::string description) bool Completion::operator<(const Completion & other) const { return completion < other.completion || (completion == other.completion && description < other.description); } -bool pathCompletions = false; +CompletionType completionType = ctNormal; std::shared_ptr completions; std::string completionMarker = "___COMPLETE___"; @@ -277,7 +277,7 @@ Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional< static void _completePath(std::string_view prefix, bool onlyDirs) { - pathCompletions = true; + completionType = ctFilenames; glob_t globbuf; int flags = GLOB_NOESCAPE | GLOB_TILDE; #ifdef GLOB_ONLYDIR diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 7521b3065..76b1cfe92 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -237,7 +237,13 @@ public: void add(std::string completion, std::string description = ""); }; extern std::shared_ptr completions; -extern bool pathCompletions; + +enum CompletionType { + ctNormal, + ctFilenames, + ctAttrs +}; +extern CompletionType completionType; std::optional needsCompletion(std::string_view s); diff --git a/src/nix/main.cc b/src/nix/main.cc index 60b0aa410..759118fd5 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -310,7 +310,14 @@ void mainWrapped(int argc, char * * argv) Finally printCompletions([&]() { if (completions) { - std::cout << (pathCompletions ? "filenames\n" : "no-filenames\n"); + switch (completionType) { + case ctNormal: + std::cout << "normal\n"; break; + case ctFilenames: + std::cout << "filenames\n"; break; + case ctAttrs: + std::cout << "attrs\n"; break; + } for (auto & s : *completions) std::cout << s.completion << "\t" << s.description << "\n"; } From c4a03bc4ae7ac89dfca3ada27d1a7946d568d4d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Na=C3=AFm=20Favier?= Date: Wed, 22 Dec 2021 16:37:58 +0100 Subject: [PATCH 107/117] Fix attr path completion after a dot --- src/libcmd/installables.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 5f118b8d0..6c20bb7b1 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -191,7 +191,7 @@ void SourceExprCommand::completeInstallable(std::string_view prefix) auto sep = prefix_.rfind('.'); std::string searchWord; if (sep != std::string::npos) { - searchWord = prefix_.substr(sep, std::string::npos); + searchWord = prefix_.substr(sep + 1, std::string::npos); prefix_ = prefix_.substr(0, sep); } else { searchWord = prefix_; From dc89dfa7b3f8fa7368cd4c5e20891e60e418722b Mon Sep 17 00:00:00 2001 From: regnat Date: Thu, 23 Dec 2021 10:35:09 +0100 Subject: [PATCH 108/117] Properly return false on `builtins.pathExists /someNonAllowedPath` Follow-up from https://github.com/NixOS/nix/pull/5807 to fix https://github.com/NixOS/nix/pull/5807#issuecomment-1000135394 --- src/libexpr/primops.cc | 29 ++++++++++++++++++++++------- tests/pure-eval.sh | 3 +++ 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index aff8f951e..62c21c7c5 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -89,18 +89,28 @@ StringMap EvalState::realiseContext(const PathSet & context) return res; } -static Path realisePath(EvalState & state, const Pos & pos, Value & v, bool requireAbsolutePath = true) +struct RealisePathFlags { + // Whether to check whether the path is a valid absolute path + bool requireAbsolutePath = true; + // Whether to check that the path is allowed in pure eval mode + bool checkForPureEval = true; +}; + +static Path realisePath(EvalState & state, const Pos & pos, Value & v, const RealisePathFlags flags = {}) { PathSet context; - Path path = requireAbsolutePath + Path path = flags.requireAbsolutePath ? state.coerceToPath(pos, v, context) : state.coerceToString(pos, v, context, false, false); StringMap rewrites = state.realiseContext(context); - return state.checkSourcePath( - state.toRealPath(rewriteStrings(path, rewrites), context)); + auto realPath = state.toRealPath(rewriteStrings(path, rewrites), context); + + return flags.checkForPureEval + ? state.checkSourcePath(realPath) + : realPath; } /* Add and attribute to the given attribute map from the output name to @@ -1371,7 +1381,12 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args, { Path path; try { - path = realisePath(state, pos, *args[0]); + // We don’t check the path right now, because we don’t want to throw if + // the path isn’t allowed, but just return false + // (and we can’t just catch the exception here because we still want to + // throw if something in the evaluation of `*args[0]` tries to access an + // unauthorized path) + path = realisePath(state, pos, *args[0], { .checkForPureEval = false }); } catch (InvalidPathError & e) { throw EvalError({ .msg = hintfmt( @@ -1382,7 +1397,7 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args, } try { - mkBool(v, pathExists(path)); + mkBool(v, pathExists(state.checkSourcePath(path))); } catch (SysError & e) { /* Don't give away info from errors while canonicalising ‘path’ in restricted mode. */ @@ -1496,7 +1511,7 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va Path path; try { - path = realisePath(state, pos, *i->value, false); + path = realisePath(state, pos, *i->value, { .requireAbsolutePath = false }); } catch (InvalidPathError & e) { throw EvalError({ .msg = hintfmt("cannot find '%1%', since path '%2%' is not valid", path, e.path), diff --git a/tests/pure-eval.sh b/tests/pure-eval.sh index cb4b5c5fc..1a4568ea6 100644 --- a/tests/pure-eval.sh +++ b/tests/pure-eval.sh @@ -11,6 +11,9 @@ missingImpureErrorMsg=$(! nix eval --expr 'builtins.readFile ./pure-eval.sh' 2>& echo "$missingImpureErrorMsg" | grep -q -- --impure || \ fail "The error message should mention the “--impure” flag to unblock users" +[[ $(nix eval --expr 'builtins.pathExists ./pure-eval.sh') == false ]] || \ + fail "Calling 'pathExists' on a non-authorised path should return false" + (! nix eval --expr builtins.currentTime) (! nix eval --expr builtins.currentSystem) From b16643b6fc9abb5c243a1d83cfe0da93c441bb2c Mon Sep 17 00:00:00 2001 From: Nicolas Mattia Date: Thu, 23 Dec 2021 14:12:48 +0100 Subject: [PATCH 109/117] Use int64_t and NixFloat in fromTOML types This makes sure that values parsed from TOML have a proper size. Using e.g. `double` caused issues on i686 where the size of `double` (32bit) was too small to accommodate some values. --- src/libexpr/primops/fromTOML.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index bfea1ce34..221c40db9 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -47,10 +47,10 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va mkBool(v, toml::get(t)); break;; case toml::value_t::integer: - mkInt(v, toml::get(t)); + mkInt(v, toml::get(t)); break;; case toml::value_t::floating: - mkFloat(v, toml::get(t)); + mkFloat(v, toml::get(t)); break;; case toml::value_t::string: mkString(v, toml::get(t)); From a4ab0a74d97c7c31df69f04870fa56cde89701a3 Mon Sep 17 00:00:00 2001 From: Yorick van Pelt Date: Mon, 27 Dec 2021 13:18:55 +0100 Subject: [PATCH 110/117] Fix accidental O(n^2 * log n) performance in NixRepl::addAttrsToScope Only sort once, after adding all of the attrs first. This reduces my `nix repl ''` loading time from 1.07s to 103ms. Fixes #5823 --- src/libexpr/nixexpr.hh | 7 +++++++ src/nix/repl.cc | 12 ++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index c013f5deb..b328b3941 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -368,6 +368,13 @@ struct StaticEnv [](const Vars::value_type & a, const Vars::value_type & b) { return a.first < b.first; }); } + void deduplicate() + { + const auto last = std::unique(vars.begin(), vars.end(), + [] (const Vars::value_type & a, const Vars::value_type & b) { return a.first == b.first; }); + vars.erase(last, vars.end()); + } + Vars::const_iterator find(const Symbol & name) const { Vars::value_type key(name, 0); diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 63ccbfda3..39a5a31de 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -661,8 +661,16 @@ void NixRepl::reloadFiles() void NixRepl::addAttrsToScope(Value & attrs) { state->forceAttrs(attrs); - for (auto & i : *attrs.attrs) - addVarToScope(i.name, *i.value); + if (displ + attrs.attrs->size() >= envSize) + throw Error("environment full; cannot add more variables"); + + for (auto & i : *attrs.attrs) { + staticEnv.vars.emplace_back(i.name, displ); + env->values[displ++] = i.value; + varNames.insert((string) i.name); + } + staticEnv.sort(); + staticEnv.deduplicate(); notice("Added %1% variables.", attrs.attrs->size()); } From 2664a216e57169ec57d7f51be1b8383c1be83fd5 Mon Sep 17 00:00:00 2001 From: Alexander Bantyev Date: Tue, 28 Dec 2021 15:30:06 +0300 Subject: [PATCH 111/117] flake.cc: computeLocks: Only verify overrides when they could change When we check for disappeared overrides, we can get "false positives" for follows and overrides which are defined in the dependencies of the flake we are locking, since they are not parsed by parseFlakeInputs. However, at that point we already know that the overrides couldn't have possible been changed if the input itself hasn't changed (since we check that oldLock->originalRef == *input.ref for the input's parent). So, to prevent this, only perform this check when it was possible that the flake changed (e.g. the flake we're locking, or a new input, or the input has changed and mustRefetch == true). --- src/libexpr/flake/flake.cc | 36 ++++++++++++++++++++++-------------- tests/flakes.sh | 10 +++------- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index b15878d5c..c549c5971 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -344,7 +344,8 @@ LockedFlake lockFlake( const InputPath & inputPathPrefix, std::shared_ptr oldNode, const LockParent & parent, - const Path & parentPath)> + const Path & parentPath, + bool trustLock)> computeLocks; computeLocks = [&]( @@ -353,7 +354,8 @@ LockedFlake lockFlake( const InputPath & inputPathPrefix, std::shared_ptr oldNode, const LockParent & parent, - const Path & parentPath) + const Path & parentPath, + bool trustLock) { debug("computing lock file node '%s'", printInputPath(inputPathPrefix)); @@ -464,14 +466,20 @@ LockedFlake lockFlake( .isFlake = (*lockedNode)->isFlake, }); } else if (auto follows = std::get_if<1>(&i.second)) { - auto o = input.overrides.find(i.first); - // If the override disappeared, we have to refetch the flake, - // since some of the inputs may not be present in the lockfile. - if (o == input.overrides.end()) { - mustRefetch = true; - // There's no point populating the rest of the fake inputs, - // since we'll refetch the flake anyways. - break; + if (! trustLock) { + // It is possible that the flake has changed, + // so we must confirm all the follows that are in the lockfile are also in the flake. + auto overridePath(inputPath); + overridePath.push_back(i.first); + auto o = overrides.find(overridePath); + // If the override disappeared, we have to refetch the flake, + // since some of the inputs may not be present in the lockfile. + if (o == overrides.end()) { + mustRefetch = true; + // There's no point populating the rest of the fake inputs, + // since we'll refetch the flake anyways. + break; + } } fakeInputs.emplace(i.first, FlakeInput { .follows = *follows, @@ -482,14 +490,14 @@ LockedFlake lockFlake( LockParent newParent { .path = inputPath, - .absolute = false + .absolute = true }; computeLocks( mustRefetch ? getFlake(state, oldLock->lockedRef, false, flakeCache).inputs : fakeInputs, - childNode, inputPath, oldLock, newParent, parentPath); + childNode, inputPath, oldLock, newParent, parentPath, !mustRefetch); } else { /* We need to create a new lock file entry. So fetch @@ -546,7 +554,7 @@ LockedFlake lockFlake( ? std::dynamic_pointer_cast(oldLock) : LockFile::read( inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root, - newParent, localPath); + newParent, localPath, false); } else { @@ -574,7 +582,7 @@ LockedFlake lockFlake( computeLocks( flake.inputs, newLockFile.root, {}, - lockFlags.recreateLockFile ? nullptr : oldLockFile.root, parent, parentPath); + lockFlags.recreateLockFile ? nullptr : oldLockFile.root, parent, parentPath, false); for (auto & i : lockFlags.inputOverrides) if (!overridesUsed.count(i.first)) diff --git a/tests/flakes.sh b/tests/flakes.sh index 942d07010..20e6d09c1 100644 --- a/tests/flakes.sh +++ b/tests/flakes.sh @@ -716,11 +716,10 @@ cat > $flakeFollowsA/flake.nix < $flakeFollowsB/flake.nix < $flakeFollowsC/flake.nix < Date: Tue, 28 Dec 2021 15:53:21 +0300 Subject: [PATCH 112/117] BaseError::calcWhat: take loggerSettings.showTrace into account Text representation for errors should include the trace if --show-trace is passed. --- src/libutil/error.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 203d79087..dd9678ee7 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -25,7 +25,7 @@ const string & BaseError::calcWhat() const err.name = sname(); std::ostringstream oss; - showErrorInfo(oss, err, false); + showErrorInfo(oss, err, loggerSettings.showTrace); what_ = oss.str(); return *what_; From 2dc29e0d932822baa50dbc3e81ae108382c39062 Mon Sep 17 00:00:00 2001 From: Alexander Bantyev Date: Tue, 28 Dec 2021 15:54:46 +0300 Subject: [PATCH 113/117] Add a test that nix repl --show-trace actually shows the trace --- tests/repl.sh | 10 +++++++++- tests/undefined-variable.nix | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 tests/undefined-variable.nix diff --git a/tests/repl.sh b/tests/repl.sh index 995db869c..0e23a98db 100644 --- a/tests/repl.sh +++ b/tests/repl.sh @@ -13,6 +13,10 @@ failing = import ./simple-failing.nix :log failing " +replUndefinedVariable=" +import ./undefined-variable.nix +" + testRepl () { local nixArgs=("$@") local replOutput="$(nix repl "${nixArgs[@]}" <<< "$replCmds")" @@ -22,10 +26,14 @@ testRepl () { nix path-info "${nixArgs[@]}" "$outPath" # simple.nix prints a PATH during build echo "$replOutput" | grep -qs 'PATH=' || fail "nix repl :log doesn't output logs" - local replOutput="$(nix repl "${nixArgs[@]}" <<< "$replFailingCmds")" + local replOutput="$(nix repl "${nixArgs[@]}" <<< "$replFailingCmds" 2>&1)" echo "$replOutput" echo "$replOutput" | grep -qs 'This should fail' \ || fail "nix repl :log doesn't output logs for a failed derivation" + local replOutput="$(nix repl --show-trace "${nixArgs[@]}" <<< "$replUndefinedVariable" 2>&1)" + echo "$replOutput" + echo "$replOutput" | grep -qs "while evaluating the file" \ + || fail "nix repl --show-trace doesn't show the trace" } # Simple test, try building a drv diff --git a/tests/undefined-variable.nix b/tests/undefined-variable.nix new file mode 100644 index 000000000..579985497 --- /dev/null +++ b/tests/undefined-variable.nix @@ -0,0 +1 @@ +let f = builtins.toFile "test-file.nix" "asd"; in import f From 92c7d33ee32b9597fedee5f7371de54c0a10a5a5 Mon Sep 17 00:00:00 2001 From: Lily Foster Date: Tue, 28 Dec 2021 10:10:58 -0500 Subject: [PATCH 114/117] Sort attrs from tables in fromTOML This was dropped in 10a8b5d for the migration from cpptoml to toml11 but seems to be necessary for the attrsets to work correctly. Fixes #5833 --- src/libexpr/primops/fromTOML.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 221c40db9..30466fc5b 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -31,6 +31,8 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va auto & v2 = *state.allocAttr(v, state.symbols.create(elem.first)); visit(v2, elem.second); } + + v.attrs->sort(); } break;; case toml::value_t::array: From a26351da02689d1e3ee562a6d89d93c12816e476 Mon Sep 17 00:00:00 2001 From: Alexander Bantyev Date: Tue, 28 Dec 2021 16:13:07 +0300 Subject: [PATCH 115/117] Add ability to toggle show-trace from within the repl --- doc/manual/src/release-notes/rl-next.md | 2 ++ src/nix/repl.cc | 15 ++++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md index ac662c132..2a67a39b1 100644 --- a/doc/manual/src/release-notes/rl-next.md +++ b/doc/manual/src/release-notes/rl-next.md @@ -2,3 +2,5 @@ * The TOML parser used by `builtins.fromTOML` has been replaced by [a more compliant one](https://github.com/ToruNiina/toml11). +* Added `:st`/`:show-trace` commands to nix repl, which are used to + set or toggle display of error traces. diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 39a5a31de..a62ad66c2 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -430,7 +430,8 @@ bool NixRepl::processLine(string line) << " :t Describe result of evaluation\n" << " :u Build derivation, then start nix-shell\n" << " :doc Show documentation of a builtin function\n" - << " :log Show logs for a derivation\n"; + << " :log Show logs for a derivation\n" + << " :st [bool] Enable, disable or toggle showing traces for errors\n"; } else if (command == ":a" || command == ":add") { @@ -572,6 +573,18 @@ bool NixRepl::processLine(string line) throw Error("value does not have documentation"); } + else if (command == ":st" || command == ":show-trace") { + if (arg == "false" || (arg == "" && loggerSettings.showTrace)) { + std::cout << "not showing error traces\n"; + loggerSettings.showTrace = false; + } else if (arg == "true" || (arg == "" && !loggerSettings.showTrace)) { + std::cout << "showing error traces\n"; + loggerSettings.showTrace = true; + } else { + throw Error("unexpected argument '%s' to %s", arg, command); + }; + } + else if (command != "") throw Error("unknown command '%1%'", command); From d0c8e9254e8a8271a3f044ee69774096e57a74bb Mon Sep 17 00:00:00 2001 From: Sebastian Ullrich Date: Wed, 29 Dec 2021 18:42:02 +0100 Subject: [PATCH 116/117] Fix IFD with chroot store --- src/libexpr/primops.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 62c21c7c5..7c964bd0d 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -82,7 +82,7 @@ StringMap EvalState::realiseContext(const PathSet & context) paths. */ if (allowedPaths) { for (auto & [_placeholder, outputPath] : res) { - allowPath(outputPath); + allowPath(store->toRealPath(outputPath)); } } From d9aabb7acb4d697d4ba9527326602cf3abe77590 Mon Sep 17 00:00:00 2001 From: Wolfgang Thaller Date: Sat, 1 Jan 2022 16:47:05 +0100 Subject: [PATCH 117/117] docker.nix: set correct permissions on /tmp --- docker.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker.nix b/docker.nix index 25930bb87..bfccdb8f5 100644 --- a/docker.nix +++ b/docker.nix @@ -232,6 +232,9 @@ pkgs.dockerTools.buildLayeredImageWithNixDb { rm -rf nix-support ln -s /nix/var/nix/profiles nix/var/nix/gcroots/profiles ''; + fakeRootCommands = '' + chmod 1777 tmp + ''; config = { Cmd = [ "/root/.nix-profile/bin/bash" ];