Merge remote-tracking branch 'nixos/master'

.github/workflows/ci.yml

@@ -100,7 +100,7 @@ jobs:
     - run: docker tag nix:$NIX_VERSION nixos/nix:$NIX_VERSION
     - run: docker tag nix:$NIX_VERSION nixos/nix:master
     - name: Login to Docker Hub
-      uses: docker/login-action@v1
+      uses: docker/login-action@v2
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_TOKEN }}

doc/manual/src/contributing/hacking.md

@@ -71,18 +71,6 @@ To install it in `$(pwd)/outputs` and test it:
 nix (Nix) 3.0
 ```
 
-To run a functional test:
-
-```console
-make tests/test-name-should-auto-complete.sh.test
-```
-
-To run the unit-tests for C++ code:
-
-```
-make check
-```
-
 If you have a flakes-enabled Nix you can replace:
 
 ```console
@@ -94,3 +82,29 @@ by:
 ```console
 $ nix develop
 ```
+
+## Testing
+
+Nix comes with three different flavors of tests: unit, functional and integration.
+
+### Unit-tests
+
+The unit-tests for each Nix library (`libexpr`, `libstore`, etc..) are defined
+under `src/{library_name}/tests` using the
+[googletest](https://google.github.io/googletest/) framework.
+
+You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option.
+
+### Functional tests
+
+The functional tests reside under the `tests` directory and are listed in `tests/local.mk`.
+The whole testsuite can be run with `make install && make installcheck`.
+Individual tests can be run with `make tests/{testName}.sh.test`.
+
+### Integration tests
+
+The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute.
+These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup.
+Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on <https://hydra.nixos.org/jobset/nix/master>).
+
+You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}`

src/libcmd/installables.cc

@@ -686,11 +686,15 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
     auto drvPath = attr->forceDerivation();
 
     std::set<std::string> outputsToInstall;
+    std::optional<NixInt> priority;
 
-    if (auto aMeta = attr->maybeGetAttr(state->sMeta))
+    if (auto aMeta = attr->maybeGetAttr(state->sMeta)) {
         if (auto aOutputsToInstall = aMeta->maybeGetAttr("outputsToInstall"))
             for (auto & s : aOutputsToInstall->getListOfStrings())
                 outputsToInstall.insert(s);
+        if (auto aPriority = aMeta->maybeGetAttr("priority"))
+            priority = aPriority->getInt();
+    }
 
     if (outputsToInstall.empty() || std::get_if<AllOutputs>(&outputsSpec)) {
         outputsToInstall.clear();
@@ -708,6 +712,7 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
     auto drvInfo = DerivationInfo {
         .drvPath = std::move(drvPath),
         .outputsToInstall = std::move(outputsToInstall),
+        .priority = priority,
     };
 
     return {attrPath, getLockedFlake()->flake.lockedRef, std::move(drvInfo)};

src/libcmd/installables.hh

@@ -142,6 +142,7 @@ struct InstallableValue : Installable
     {
         StorePath drvPath;
         std::set<std::string> outputsToInstall;
+        std::optional<NixInt> priority;
     };
 
     virtual std::vector<DerivationInfo> toDerivations() = 0;

src/libexpr/eval-cache.cc

@@ -47,7 +47,7 @@ struct AttrDb
     {
         auto state(_state->lock());
 
-        Path cacheDir = getCacheDir() + "/nix/eval-cache-v3";
+        Path cacheDir = getCacheDir() + "/nix/eval-cache-v4";
         createDirs(cacheDir);
 
         Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
@@ -175,6 +175,24 @@ struct AttrDb
         });
     }
 
+    AttrId setInt(
+        AttrKey key,
+        int n)
+    {
+        return doSQLite([&]()
+        {
+            auto state(_state->lock());
+
+            state->insertAttribute.use()
+                (key.first)
+                (symbols[key.second])
+                (AttrType::Int)
+                (n).exec();
+
+            return state->db.getLastInsertedRowId();
+        });
+    }
+
     AttrId setListOfStrings(
         AttrKey key,
         const std::vector<std::string> & l)
@@ -287,6 +305,8 @@ struct AttrDb
             }
             case AttrType::Bool:
                 return {{rowId, queryAttribute.getInt(2) != 0}};
+            case AttrType::Int:
+                return {{rowId, int_t{queryAttribute.getInt(2)}}};
             case AttrType::ListOfStrings:
                 return {{rowId, tokenizeString<std::vector<std::string>>(queryAttribute.getStr(2), "\t")}};
             case AttrType::Missing:
@@ -426,6 +446,8 @@ Value & AttrCursor::forceValue()
             cachedValue = {root->db->setString(getKey(), v.path), string_t{v.path, {}}};
         else if (v.type() == nBool)
             cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean};
+        else if (v.type() == nInt)
+            cachedValue = {root->db->setInt(getKey(), v.integer), int_t{v.integer}};
         else if (v.type() == nAttrs)
             ; // FIXME: do something?
         else
@@ -621,6 +643,28 @@ bool AttrCursor::getBool()
     return v.boolean;
 }
 
+NixInt AttrCursor::getInt()
+{
+    if (root->db) {
+        if (!cachedValue)
+            cachedValue = root->db->getAttr(getKey());
+        if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
+            if (auto i = std::get_if<int_t>(&cachedValue->second)) {
+                debug("using cached Integer attribute '%s'", getAttrPathStr());
+                return i->x;
+            } else
+                throw TypeError("'%s' is not an Integer", getAttrPathStr());
+        }
+    }
+
+    auto & v = forceValue();
+
+    if (v.type() != nInt)
+        throw TypeError("'%s' is not an Integer", getAttrPathStr());
+
+    return v.integer;
+}
+
 std::vector<std::string> AttrCursor::getListOfStrings()
 {
     if (root->db) {

src/libexpr/eval-cache.hh

@@ -45,12 +45,14 @@ enum AttrType {
     Failed = 5,
     Bool = 6,
     ListOfStrings = 7,
+    Int = 8,
 };
 
 struct placeholder_t {};
 struct missing_t {};
 struct misc_t {};
 struct failed_t {};
+struct int_t { NixInt x; };
 typedef uint64_t AttrId;
 typedef std::pair<AttrId, Symbol> AttrKey;
 typedef std::pair<std::string, NixStringContext> string_t;
@@ -63,6 +65,7 @@ typedef std::variant<
     misc_t,
     failed_t,
     bool,
+    int_t,
     std::vector<std::string>
     > AttrValue;
 
@@ -116,6 +119,8 @@ public:
 
     bool getBool();
 
+    NixInt getInt();
+
     std::vector<std::string> getListOfStrings();
 
     std::vector<Symbol> getAttrs();

src/libexpr/eval.cc

@@ -1590,7 +1590,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
 Nix attempted to evaluate a function as a top level expression; in
 this case it must have its arguments supplied either by default
 values, or passed explicitly with '--arg' or '--argstr'. See
-https://nixos.org/manual/nix/stable/#ss-functions.)", symbols[i.name]);
+https://nixos.org/manual/nix/stable/expressions/language-constructs.html#functions.)", symbols[i.name]);
 
             }
         }

src/libexpr/flake/config.cc

@@ -31,7 +31,7 @@ static void writeTrustedList(const TrustedList & trustedList)
 
 void ConfigFile::apply()
 {
-    std::set<std::string> whitelist{"bash-prompt", "bash-prompt-suffix", "flake-registry"};
+    std::set<std::string> whitelist{"bash-prompt", "bash-prompt-prefix", "bash-prompt-suffix", "flake-registry"};
 
     for (auto & [name, value] : settings) {
 

src/libexpr/tests/local.mk

@@ -10,6 +10,6 @@ libexpr-tests_SOURCES := $(wildcard $(d)/*.cc)
 
 libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests
 
-libexpr-tests_LIBS = libexpr libutil libstore
+libexpr-tests_LIBS = libexpr libutil libstore libfetchers
 
 libexpr-tests_LDFLAGS := $(GTEST_LIBS) -lgmock

src/libfetchers/github.cc

@@ -243,7 +243,10 @@ struct GitHubInputScheme : GitArchiveInputScheme
     Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
     {
         auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
-        auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check
+        auto url = fmt(
+            host == "github.com"
+            ? "https://api.%s/repos/%s/%s/commits/%s"
+            : "https://%s/api/v3/repos/%s/%s/commits/%s",
             host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
 
         Headers headers = makeHeadersWithAuthTokens(host);
@@ -262,7 +265,10 @@ struct GitHubInputScheme : GitArchiveInputScheme
         // FIXME: use regular /archive URLs instead? api.github.com
         // might have stricter rate limits.
         auto host = maybeGetStrAttr(input.attrs, "host").value_or("github.com");
-        auto url = fmt("https://api.%s/repos/%s/%s/tarball/%s", // FIXME: check if this is correct for self hosted instances
+        auto url = fmt(
+            host == "github.com"
+            ? "https://api.%s/repos/%s/%s/tarball/%s"
+            : "https://%s/api/v3/repos/%s/%s/tarball/%s",
             host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
             input.getRev()->to_string(Base16, false));
 

src/libstore/builtins/buildenv.cc

@@ -93,8 +93,9 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
                     auto prevPriority = state.priorities[dstFile];
                     if (prevPriority == priority)
                         throw Error(
-                                "packages '%1%' and '%2%' have the same priority %3%; "
+                                "files '%1%' and '%2%' have the same priority %3%; "
                                 "use 'nix-env --set-flag priority NUMBER INSTALLED_PKGNAME' "
+                                "or type 'nix profile install --help' if using 'nix profile' to find out how"
                                 "to change the priority of one of the conflicting packages"
                                 " (0 being the highest priority)",
                                 srcFile, readLink(dstFile), priority);

src/nix-build/nix-build.cc

@@ -543,8 +543,6 @@ static void main_nix_build(int argc, char * * argv)
 
         restoreProcessContext();
 
-        logger->stop();
-
         execvp(shell->c_str(), argPtrs.data());
 
         throw SysError("executing shell '%s'", *shell);
@@ -603,8 +601,6 @@ static void main_nix_build(int argc, char * * argv)
             outPaths.push_back(outputPath);
         }
 
-        logger->stop();
-
         for (auto & path : outPaths)
             std::cout << store->printStorePath(path) << '\n';
     }

src/nix-env/nix-env.cc

@@ -1489,8 +1489,6 @@ static int main_nix_env(int argc, char * * argv)
 
         globals.state->printStats();
 
-        logger->stop();
-
         return 0;
     }
 }

src/nix-store/nix-store.cc

@@ -1095,8 +1095,6 @@ static int main_nix_store(int argc, char * * argv)
 
         op(opFlags, opArgs);
 
-        logger->stop();
-
         return 0;
     }
 }

src/nix/develop.cc

@@ -18,6 +18,9 @@ struct DevelopSettings : Config
     Setting<std::string> bashPrompt{this, "", "bash-prompt",
         "The bash prompt (`PS1`) in `nix develop` shells."};
 
+    Setting<std::string> bashPromptPrefix{this, "", "bash-prompt-prefix",
+        "Prefix prepended to the `PS1` environment variable in `nix develop` shells."};
+
     Setting<std::string> bashPromptSuffix{this, "", "bash-prompt-suffix",
         "Suffix appended to the `PS1` environment variable in `nix develop` shells."};
 };
@@ -482,6 +485,9 @@ struct CmdDevelop : Common, MixEnvironment
             if (developSettings.bashPrompt != "")
                 script += fmt("[ -n \"$PS1\" ] && PS1=%s;\n",
                     shellEscape(developSettings.bashPrompt.get()));
+            if (developSettings.bashPromptPrefix != "")
+                script += fmt("[ -n \"$PS1\" ] && PS1=%s\"$PS1\";\n",
+                    shellEscape(developSettings.bashPromptPrefix.get()));
             if (developSettings.bashPromptSuffix != "")
                 script += fmt("[ -n \"$PS1\" ] && PS1+=%s;\n",
                     shellEscape(developSettings.bashPromptSuffix.get()));
@@ -512,9 +518,20 @@ struct CmdDevelop : Common, MixEnvironment
                 Strings{"legacyPackages." + settings.thisSystem.get() + "."},
                 nixpkgsLockFlags);
 
-            shell = store->printStorePath(
+            bool found = false;
-                Installable::toStorePath(getEvalStore(), store, Realise::Outputs, OperateOn::Output, bashInstallable))
+
-                + "/bin/bash";
+            for (auto & path : Installable::toStorePaths(getEvalStore(), store, Realise::Outputs, OperateOn::Output, {bashInstallable})) {
+                auto s = store->printStorePath(path) + "/bin/bash";
+                if (pathExists(s)) {
+                    shell = s;
+                    found = true;
+                    break;
+                }
+            }
+
+            if (!found)
+                throw Error("package 'nixpkgs#bashInteractive' does not provide a 'bin/bash'");
+
         } catch (Error &) {
             ignoreException();
         }

src/nix/develop.md

@@ -80,8 +80,8 @@ initialised by `stdenv` and exits. This build environment can be
 recorded into a profile using `--profile`.
 
 The prompt used by the `bash` shell can be customised by setting the
-`bash-prompt` and `bash-prompt-suffix` settings in `nix.conf` or in
+`bash-prompt`, `bash-prompt-prefix`, and `bash-prompt-suffix` settings in
-the flake's `nixConfig` attribute.
+`nix.conf` or in the flake's `nixConfig` attribute.
 
 # Flake output attributes
 

src/nix/flake.md

@@ -331,9 +331,10 @@ The following attributes are supported in `flake.nix`:
 
 * `nixConfig`: a set of `nix.conf` options to be set when evaluating any
   part of a flake. In the interests of security, only a small set of
-  whitelisted options (currently `bash-prompt`, `bash-prompt-suffix`,
+  whitelisted options (currently `bash-prompt`, `bash-prompt-prefix`,
-  and `flake-registry`) are allowed to be set without confirmation so long as
+  `bash-prompt-suffix`, and `flake-registry`) are allowed to be set without
-  `accept-flake-config` is not set in the global configuration.
+  confirmation so long as `accept-flake-config` is not set in the global
+  configuration.
 
 ## Flake inputs
 

src/nix/main.cc

@@ -261,6 +261,8 @@ void mainWrapped(int argc, char * * argv)
     }
     #endif
 
+    Finally f([] { logger->stop(); });
+
     programPath = argv[0];
     auto programName = std::string(baseNameOf(programPath));
 
@@ -279,8 +281,6 @@ void mainWrapped(int argc, char * * argv)
         verbosity = lvlInfo;
     }
 
-    Finally f([] { logger->stop(); });
-
     NixArgs args;
 
     if (argc == 2 && std::string(argv[1]) == "__dump-args") {

src/nix/profile.cc

@@ -37,7 +37,7 @@ struct ProfileElement
     StorePathSet storePaths;
     std::optional<ProfileElementSource> source;
     bool active = true;
-    // FIXME: priority
+    int priority = 5;
 
     std::string describe() const
     {
@@ -116,6 +116,9 @@ struct ProfileManifest
                 for (auto & p : e["storePaths"])
                     element.storePaths.insert(state.store->parseStorePath((std::string) p));
                 element.active = e["active"];
+                if(e.contains("priority")) {
+                    element.priority = e["priority"];
+                }
                 if (e.value(sUrl, "") != "") {
                     element.source = ProfileElementSource {
                         parseFlakeRef(e[sOriginalUrl]),
@@ -153,6 +156,7 @@ struct ProfileManifest
             nlohmann::json obj;
             obj["storePaths"] = paths;
             obj["active"] = element.active;
+            obj["priority"] = element.priority;
             if (element.source) {
                 obj["originalUrl"] = element.source->originalRef.to_string();
                 obj["url"] = element.source->resolvedRef.to_string();
@@ -177,7 +181,7 @@ struct ProfileManifest
         for (auto & element : elements) {
             for (auto & path : element.storePaths) {
                 if (element.active)
-                    pkgs.emplace_back(store->printStorePath(path), true, 5);
+                    pkgs.emplace_back(store->printStorePath(path), true, element.priority);
                 references.insert(path);
             }
         }
@@ -259,6 +263,16 @@ builtPathsPerInstallable(
 
 struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
 {
+    std::optional<int> priority;
+    CmdProfileInstall() {
+        addFlag({
+            .longName = "priority",
+            .description = "The priority of the package to install.",
+            .labels = {"priority"},
+            .handler = {&priority},
+        });
+    };
+
     std::string description() override
     {
         return "install a package into a profile";
@@ -282,6 +296,8 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
         for (auto & installable : installables) {
             ProfileElement element;
 
+
+
             if (auto installable2 = std::dynamic_pointer_cast<InstallableFlake>(installable)) {
                 // FIXME: make build() return this?
                 auto [attrPath, resolvedRef, drv] = installable2->toDerivation();
@@ -291,8 +307,16 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
                     attrPath,
                     installable2->outputsSpec
                 };
+
+                if(drv.priority) {
+                    element.priority = *drv.priority;
+                }
             }
 
+            if(priority) { // if --priority was specified we want to override the priority of the installable
+                element.priority = *priority;
+            };
+
             element.updateStorePaths(getEvalStore(), store, builtPaths[installable.get()]);
 
             manifest.elements.push_back(std::move(element));

tests/nix-profile.sh

@@ -120,3 +120,21 @@ nix profile install "$flake1Dir^man"
 (! [ -e $TEST_HOME/.nix-profile/bin/hello ])
 [ -e $TEST_HOME/.nix-profile/share/man ]
 (! [ -e $TEST_HOME/.nix-profile/include ])
+
+# test priority
+nix profile remove 0
+
+# Make another flake.
+flake2Dir=$TEST_ROOT/flake2
+printf World > $flake1Dir/who
+cp -r $flake1Dir $flake2Dir
+printf World2 > $flake2Dir/who
+
+nix profile install $flake1Dir
+[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]]
+nix profile install $flake2Dir --priority 100
+[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]]
+nix profile install $flake2Dir --priority 0
+[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World2" ]]
+# nix profile install $flake1Dir --priority 100
+# [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]]