Update to use access-tokens configuration for github/gitlab access.

This change provides support for using access tokens with other
instances of GitHub and GitLab beyond just github.com and
gitlab.com (especially company-specific or foundation-specific
instances).

This change also provides the ability to specify the type of access
token being used, where different types may have different handling,
based on the forge type.

src/libfetchers/github.cc

@@ -5,6 +5,7 @@
 #include "store-api.hh"
 #include "types.hh"
 
+#include <optional>
 #include <nlohmann/json.hpp>
 
 namespace nix::fetchers {
@@ -12,13 +13,10 @@ namespace nix::fetchers {
 struct DownloadUrl
 {
     std::string url;
-    std::optional<std::pair<std::string, std::string>> access_token_header;
+    Headers headers;
 
-    DownloadUrl(const std::string & url)
+    DownloadUrl(const std::string & url, const Headers & headers)
-        : url(url) { }
+        : url(url), headers(headers) { }
-
-    DownloadUrl(const std::string & url, const std::pair<std::string, std::string> & access_token_header)
-        : url(url), access_token_header(access_token_header) { }
 };
 
 // A github or gitlab url
@@ -29,7 +27,7 @@ struct GitArchiveInputScheme : InputScheme
 {
     virtual std::string type() = 0;
 
-    virtual std::pair<std::string, std::string> accessHeaderFromToken(const std::string & token) const = 0;
+    virtual std::optional<std::pair<std::string, std::string> > accessHeaderFromToken(const std::string & token) const = 0;
 
     std::optional<Input> inputFromURL(const ParsedURL & url) override
     {
@@ -144,6 +142,27 @@ struct GitArchiveInputScheme : InputScheme
         return input;
     }
 
+    std::optional<std::string> getAccessToken(const std::string &host) const {
+        auto tokens = settings.accessTokens.get();
+        auto pat = tokens.find(host);
+        if (pat == tokens.end())
+            return std::nullopt;
+        return pat->second;
+    }
+
+    Headers makeHeadersWithAuthTokens(const std::string & host) const {
+        Headers headers;
+        auto accessToken = getAccessToken(host);
+        if (accessToken) {
+            auto hdr = accessHeaderFromToken(*accessToken);
+            if (hdr)
+                headers.push_back(*hdr);
+            else
+                warn("Unrecognized access token for host '%s'", host);
+        }
+        return headers;
+    }
+
     virtual Hash getRevFromRef(nix::ref<Store> store, const Input & input) const = 0;
 
     virtual DownloadUrl getDownloadUrl(const Input & input) const = 0;
@@ -175,12 +194,7 @@ struct GitArchiveInputScheme : InputScheme
 
         auto url = getDownloadUrl(input);
 
-        Headers headers;
+        auto [tree, lastModified] = downloadTarball(store, url.url, url.headers, "source", true);
-        if (url.access_token_header) {
-            headers.push_back(*url.access_token_header);
-        }
-
-        auto [tree, lastModified] = downloadTarball(store, url.url, headers, "source", true);
 
         input.attrs.insert_or_assign("lastModified", lastModified);
 
@@ -202,7 +216,13 @@ struct GitHubInputScheme : GitArchiveInputScheme
 {
     std::string type() override { return "github"; }
 
-    std::pair<std::string, std::string> accessHeaderFromToken(const std::string & token) const {
+    std::optional<std::pair<std::string, std::string> > accessHeaderFromToken(const std::string & token) const {
+        // Github supports PAT/OAuth2 tokens and HTTP Basic
+        // Authentication.  The former simply specifies the token, the
+        // latter can use the token as the password.  Only the first
+        // is used here. See
+        // https://developer.github.com/v3/#authentication and
+        // https://docs.github.com/en/developers/apps/authorizing-oath-apps
         return std::pair<std::string, std::string>("Authorization", fmt("token %s", token));
     }
 
@@ -212,10 +232,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
         auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check
             host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
 
-        Headers headers;
+        Headers headers = makeHeadersWithAuthTokens(host);
-        std::string accessToken = settings.githubAccessToken.get();
-        if (accessToken != "")
-            headers.push_back(accessHeaderFromToken(accessToken));
 
         auto json = nlohmann::json::parse(
             readFile(
@@ -235,13 +252,8 @@ struct GitHubInputScheme : GitArchiveInputScheme
             host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
             input.getRev()->to_string(Base16, false));
 
-        std::string accessToken = settings.githubAccessToken.get();
+        Headers headers = makeHeadersWithAuthTokens(host);
-        if (accessToken != "") {
+        return DownloadUrl(url, headers);
-            auto auth_header = accessHeaderFromToken(accessToken);
-            return DownloadUrl(url, auth_header);
-        } else {
-            return DownloadUrl(url);
-        }
     }
 
     void clone(const Input & input, const Path & destDir) override
@@ -258,20 +270,32 @@ struct GitLabInputScheme : GitArchiveInputScheme
 {
     std::string type() override { return "gitlab"; }
 
-    std::pair<std::string, std::string> accessHeaderFromToken(const std::string & token) const {
+    std::optional<std::pair<std::string, std::string> > accessHeaderFromToken(const std::string & token) const {
-        return std::pair<std::string, std::string>("Authorization", fmt("Bearer %s", token));
+        // Gitlab supports 4 kinds of authorization, two of which are
+        // relevant here: OAuth2 and PAT (Private Access Token).  The
+        // user can indicate which token is used by specifying the
+        // token as <TYPE>:<VALUE>, where type is "OAuth2" or "PAT".
+        // If the <TYPE> is unrecognized, this will fall back to
+        // treating this simply has <HDRNAME>:<HDRVAL>.  See
+        // https://docs.gitlab.com/12.10/ee/api/README.html#authentication
+        auto fldsplit = token.find_first_of(':');
+        // n.b. C++20 would allow: if (token.starts_with("OAuth2:")) ...
+        if ("OAuth2" == token.substr(0, fldsplit))
+            return std::make_pair("Authorization", fmt("Bearer %s", token.substr(fldsplit+1)));
+        if ("PAT" == token.substr(0, fldsplit))
+            return std::make_pair("Private-token", token.substr(fldsplit+1));
+        warn("Unrecognized GitLab token type %s",  token.substr(0, fldsplit));
+        return std::nullopt;
     }
 
     Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
     {
-        auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
+        auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com");
+        // See rate limiting note below
         auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s",
-            host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
+            host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
 
-        Headers headers;
+        Headers headers = makeHeadersWithAuthTokens(host);
-        std::string accessToken = settings.gitlabAccessToken.get();
-        if (accessToken != "")
-            headers.push_back(accessHeaderFromToken(accessToken));
 
         auto json = nlohmann::json::parse(
             readFile(
@@ -294,14 +318,8 @@ struct GitLabInputScheme : GitArchiveInputScheme
             host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
             input.getRev()->to_string(Base16, false));
 
-        std::string accessToken = settings.gitlabAccessToken.get();
+        Headers headers = makeHeadersWithAuthTokens(host);
-        if (accessToken != "") {
+        return DownloadUrl(url, headers);
-            auto auth_header = accessHeaderFromToken(accessToken);
-            return DownloadUrl(url, auth_header);
-        } else {
-            return DownloadUrl(url);
-        }
-
     }
 
     void clone(const Input & input, const Path & destDir) override

src/libstore/globals.hh

@@ -863,8 +863,54 @@ public:
     Setting<std::string> githubAccessToken{this, "", "github-access-token",
         "GitHub access token to get access to GitHub data through the GitHub API for `github:<..>` flakes."};
 
-    Setting<std::string> gitlabAccessToken{this, "", "gitlab-access-token",
+    Setting<StringMap> accessTokens{this, {}, "access-tokens",
-        "GitLab access token to get access to GitLab data through the GitLab API for gitlab:<..> flakes."};
+        R"(
+          Access tokens used to access protected GitHub, GitLab, or
+          other locations requiring token-based authentication.
+
+          Access tokens are specified as a string made up of
+          space-separated `host=token` values.  The specific token
+          used is selected by matching the `host` portion against the
+          "host" specification of the input. The actual use of the
+          `token` value is determined by the type of resource being
+          accessed:
+
+          * Github: the token value is the OAUTH-TOKEN string obtained
+            as the Personal Access Token from the Github server (see
+            https://docs.github.com/en/developers/apps/authorizing-oath-apps).
+
+          * Gitlab: the token value is either the OAuth2 token or the
+            Personal Access Token (these are different types tokens
+            for gitlab, see
+            https://docs.gitlab.com/12.10/ee/api/README.html#authentication).
+            The `token` value should be `type:tokenstring` where
+            `type` is either `OAuth2` or `PAT` to indicate which type
+            of token is being specified.
+
+          Example `~/.config/nix/nix.conf`:
+
+          ```
+          personal-access-tokens = "github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk"
+          ```
+
+          Example `~/code/flake.nix`:
+
+          ```nix
+          input.foo = {
+            type="gitlab";
+            host="gitlab.mycompany.com";
+            owner="mycompany";
+            repo="pro";
+          };
+          ```
+
+          This example specifies three tokens, one each for accessing
+          github.com, gitlab.mycompany.com, and sourceforge.net.
+
+          The `input.foo` uses the "gitlab" fetcher, which might
+          requires specifying the token type along with the token
+          value.
+          )"};
 
     Setting<Strings> experimentalFeatures{this, {}, "experimental-features",
         "Experimental Nix features to enable."};