privatevoid.net/depot
1
1
Fork 0
Code Issues 23 Pull requests 2 Projects 3 Releases Packages Wiki Activity Actions

Merge branch 'platform-22.11' into 'master'

Browse source 
Platform 22.11

See merge request private-void/infrastructure!43
This commit is contained in:
Max Headroom 2022-12-18 15:21:11 +00:00
commit a1081f80c4
24 changed files with 1322 additions and 1129 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

77
flake.lock
View file

@ -100,6 +100,9 @@
"devshell": [ "devshell": [
"blank" "blank"
], ],
"flake-parts": [
"flake-parts"
],
"flake-utils-pre-commit": [ "flake-utils-pre-commit": [
"blank" "blank"
], ],
@ -112,6 +115,9 @@
"mach-nix": [ "mach-nix": [
"blank" "blank"
], ],
"nix-pypi-fetcher": [
"blank"
],
"nixpkgs": [ "nixpkgs": [
"nixpkgs" "nixpkgs"
], ],
@ -123,11 +129,11 @@
] ]
}, },
"locked": { "locked": {
"lastModified": 1668343672, "lastModified": 1671279390,
"narHash": "sha256-IDtrs0q6Eq5l+qSmpngnkFd3VSV3NoBc+Pxv/jDcsXg=", "narHash": "sha256-odJdJi84jDdZNxgDZK629gnyEXsTVTP/yEFn86hbjHA=",
"owner": "nix-community", "owner": "nix-community",
"repo": "dream2nix", "repo": "dream2nix",
"rev": "d158e88bc2dc49b1f295bae24a793936afab823f", "rev": "59de99b3be966328e16ff06085c513691c301261",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -139,15 +145,15 @@
"flake-parts": { "flake-parts": {
"inputs": { "inputs": {
"nixpkgs-lib": [ "nixpkgs-lib": [
"nixpkgs-lib" "nixpkgs"
] ]
}, },
"locked": { "locked": {
"lastModified": 1668338998, "lastModified": 1670441596,
"narHash": "sha256-rHDdgtymiPI4yEc2IUxNgq83cWsW2XiLUD0JCb0ASqI=", "narHash": "sha256-+T487QnluBT5F9tVk0chG/zzv+9zzTrx3o7rlOBK7ps=",
"owner": "hercules-ci", "owner": "hercules-ci",
"repo": "flake-parts", "repo": "flake-parts",
"rev": "60c4f3b26eac705948c658a133f4e282ea9d015f", "rev": "8d0e2444ab05f79df93b70e5e497f8c708eb6b9b",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -185,11 +191,11 @@
"pre-commit-hooks-nix": "pre-commit-hooks-nix" "pre-commit-hooks-nix": "pre-commit-hooks-nix"
}, },
"locked": { "locked": {
"lastModified": 1668726610, "lastModified": 1671131253,
"narHash": "sha256-W69usEQnEC8ZNwjBCBGVFv23iWt6gdOFFMad+tPZbHQ=", "narHash": "sha256-LKVuohzil/Hbwqy7D+3d5Liv3CbljSDdOQ4+E7QyusU=",
"owner": "hercules-ci", "owner": "hercules-ci",
"repo": "hercules-ci-agent", "repo": "hercules-ci-agent",
"rev": "0ccf2e9add1cff4c0a4459c622bc955b2c1e1e4d", "rev": "c8ade27f903c8ddb0309ff810f612d3e557146c2",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -211,11 +217,11 @@
] ]
}, },
"locked": { "locked": {
"lastModified": 1668147980, "lastModified": 1669995554,
"narHash": "sha256-dHu5VIGQvZoHtqr5tmndpiVVk5JnJ/6wZUNRwEfvIxk=", "narHash": "sha256-sF7KYqJbtEIeyK1Z/UBLwCIjGOtOZhEOs3xlsxo68Zs=",
"owner": "hercules-ci", "owner": "hercules-ci",
"repo": "hercules-ci-effects", "repo": "hercules-ci-effects",
"rev": "244595b8dcfea8c6ac52a99157653162d26ea6d3", "rev": "392a0777dd3e7d10a0a37edff196fea623b42a9d",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -256,11 +262,11 @@
] ]
}, },
"locked": { "locked": {
"lastModified": 1656185874, "lastModified": 1669478601,
"narHash": "sha256-sv7lsuARTT+LgMq0mFdFP73CHeqTeIvz2ZHzceQAK+0=", "narHash": "sha256-IQcS8IPeXvSoIrQRPgtnLxAs0Pkh8wmglDAtqE4ivNo=",
"owner": "mkaito", "owner": "mkaito",
"repo": "nixos-modded-minecraft-servers", "repo": "nixos-modded-minecraft-servers",
"rev": "74477f0041e76097a6800cda892744b4c4ab26d2", "rev": "68f2066499c035fd81c9dacfea2f512d6b0b62e5",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -320,11 +326,11 @@
}, },
"locked": { "locked": {
"host": "git.privatevoid.net", "host": "git.privatevoid.net",
"lastModified": 1666194069, "lastModified": 1671314458,
"narHash": "sha256-p3vx4NG4ZgY8j0p0n3yOy1wENPOeQj60XG+x1wJLiMY=", "narHash": "sha256-q9OOPWSA4/Q3jOuV/fE3cMQNmbHMklP+wOPWDUXjQKU=",
"owner": "max", "owner": "max",
"repo": "nix-super", "repo": "nix-super",
"rev": "ad8422ed3f56448a7b05a25ed764c242b3d0dd64", "rev": "8615547a51a083d360883ea896dc765e5c66adff",
"type": "gitlab" "type": "gitlab"
}, },
"original": { "original": {
@ -362,34 +368,16 @@
}, },
"nixpkgs": { "nixpkgs": {
"locked": { "locked": {
"lastModified": 1668307144, "lastModified": 1671257711,
"narHash": "sha256-uY2StvGJvTfgtLaiz3uvX+EQeWZDkiLFiz2vekgJ9ZE=", "narHash": "sha256-HjacTCbXyhwH5vlrldx3H1MwtAOClAaT6A1dSjx6bfM=",
"owner": "NixOS", "owner": "NixOS",
"repo": "nixpkgs", "repo": "nixpkgs",
"rev": "eac99848dfd869e486573d8272b0c10729675ca2", "rev": "ed64de008fc095d05bbd6498f711464a80e72ad3",
"type": "github" "type": "github"
}, },
"original": { "original": {
"owner": "NixOS", "owner": "NixOS",
"ref": "nixos-22.05-small", "ref": "nixos-22.11-small",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs-lib": {
"locked": {
"dir": "lib",
"lastModified": 1668307144,
"narHash": "sha256-uY2StvGJvTfgtLaiz3uvX+EQeWZDkiLFiz2vekgJ9ZE=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "eac99848dfd869e486573d8272b0c10729675ca2",
"type": "github"
},
"original": {
"dir": "lib",
"owner": "NixOS",
"ref": "nixos-22.05-small",
"repo": "nixpkgs", "repo": "nixpkgs",
"type": "github" "type": "github"
} }
@ -404,11 +392,11 @@
] ]
}, },
"locked": { "locked": {
"lastModified": 1668166239, "lastModified": 1671247072,
"narHash": "sha256-V3ak48Ei4wJ6PF45s6zukRahBG6NARwLyVmbppKysgk=", "narHash": "sha256-uxudJf+DWS5zHejxv29fhuMycf+K4DlWsh8+Gstoe+g=",
"owner": "nix-community", "owner": "nix-community",
"repo": "poetry2nix", "repo": "poetry2nix",
"rev": "185c10c78a3943057c3eeb18511c968f205a26ac", "rev": "f1cd4ad3937df5c9378feb7ad5cc3841e4b3cc5e",
"type": "github" "type": "github"
}, },
"original": { "original": {
@ -470,7 +458,6 @@
"nix-super": "nix-super", "nix-super": "nix-super",
"nixos-command": "nixos-command", "nixos-command": "nixos-command",
"nixpkgs": "nixpkgs", "nixpkgs": "nixpkgs",
"nixpkgs-lib": "nixpkgs-lib",
"poetry2nix": "poetry2nix", "poetry2nix": "poetry2nix",
"repin-flake-utils": "repin-flake-utils" "repin-flake-utils": "repin-flake-utils"
} }

7
flake.nix
View file

@ -92,8 +92,7 @@
}; };
inputs = { inputs = {
nixpkgs.url = "github:NixOS/nixpkgs/nixos-22.05-small"; nixpkgs.url = "github:NixOS/nixpkgs/nixos-22.11-small";
nixpkgs-lib.url = "github:NixOS/nixpkgs/nixos-22.05-small?dir=lib";
nix-super = { nix-super = {
url = "gitlab:max/nix-super?host=git.privatevoid.net"; url = "gitlab:max/nix-super?host=git.privatevoid.net";
@ -134,9 +133,11 @@
crane.follows = "blank"; crane.follows = "blank";
devshell.follows = "blank"; devshell.follows = "blank";
flake-utils-pre-commit.follows = "blank"; flake-utils-pre-commit.follows = "blank";
flake-parts.follows = "flake-parts";
ghc-utils.follows = "blank"; ghc-utils.follows = "blank";
gomod2nix.follows = "blank"; gomod2nix.follows = "blank";
mach-nix.follows = "blank"; mach-nix.follows = "blank";
nix-pypi-fetcher.follows = "blank";
poetry2nix.follows = "poetry2nix"; poetry2nix.follows = "poetry2nix";
pre-commit-hooks.follows = "blank"; pre-commit-hooks.follows = "blank";
}; };
@ -179,7 +180,7 @@
flake-parts = { flake-parts = {
url = "github:hercules-ci/flake-parts"; url = "github:hercules-ci/flake-parts";
inputs.nixpkgs-lib.follows = "nixpkgs-lib"; inputs.nixpkgs-lib.follows = "nixpkgs";
}; };
nix-filter.url = "github:numtide/nix-filter"; nix-filter.url = "github:numtide/nix-filter";

1
hosts/VEGAS/modules/virtualisation/default.nix
View file

@ -5,6 +5,7 @@
enable = true; enable = true;
qemu.package = pkgs.qemu_kvm; qemu.package = pkgs.qemu_kvm;
}; };
security.polkit.enable = true;
# TODO: maybe be more strict # TODO: maybe be more strict
networking.firewall.trustedInterfaces = [ networking.firewall.trustedInterfaces = [
"vmcore" "vmcore"

2
hosts/VEGAS/services/matrix/default.nix
View file

@ -154,7 +154,7 @@ in {
})) }))
{ {
matrix-synapse.preStart = '' matrix-synapse.preStart = ''
${pkgs.jq}/bin/jq -c --slurp '.[0] * .[1]' ${dbConfigJSON} '${dbPasswordFile}' | install -Dm400 -o matrix-synapse -g matrix-synapse /dev/stdin '${dbConfigOut}' ${pkgs.jq}/bin/jq -c --slurp '.[0] * .[1]' ${dbConfigJSON} '${dbPasswordFile}' | install -Dm400 /dev/stdin '${dbConfigOut}'
''; '';
} }
]; ];

40
hosts/VEGAS/services/monitoring/default.nix
View file

@ -8,22 +8,7 @@ let
cfg = { inherit (config.services) loki; }; cfg = { inherit (config.services) loki; };
toString' = v: iniList = lib.concatStringsSep " ";
if v == true then "true" else
if v == false then "false" else
toString v;
mapPaths = lib.mapAttrsRecursive (
path: value: lib.nameValuePair
(lib.toUpper (lib.concatStringsSep "_" path))
(toString' value)
);
translateConfig = config: lib.listToAttrs (
lib.collect
(x: x ? name && x ? value)
(mapPaths config)
);
login = x: "https://login.${domain}/auth/realms/master/protocol/openid-connect/${x}"; login = x: "https://login.${domain}/auth/realms/master/protocol/openid-connect/${x}";
in in
@ -45,19 +30,21 @@ in
services.grafana = { services.grafana = {
enable = true; enable = true;
package = inputs.self.packages.${pkgs.system}.grafana; package = inputs.self.packages.${pkgs.system}.grafana;
inherit (links.grafana) port;
rootUrl = "https://monitoring.${domain}/";
dataDir = "/srv/storage/private/grafana"; dataDir = "/srv/storage/private/grafana";
analytics.reporting.enable = false; settings = {
extraOptions = translateConfig { server = {
auth.generic_oauth = { root_url = "https://monitoring.${domain}/";
http_port = links.grafana.port;
};
analytics.reporting_enabled = false;
"auth.generic_oauth" = {
enabled = true; enabled = true;
allow_sign_up = true; allow_sign_up = true;
client_id = "net.privatevoid.monitoring1"; client_id = "net.privatevoid.monitoring1";
auth_url = login "auth"; auth_url = login "auth";
token_url = login "token"; token_url = login "token";
api_url = login "userinfo"; api_url = login "userinfo";
scopes = [ "openid" "profile" "email" "roles" ]; scopes = iniList [ "openid" "profile" "email" "roles" ];
role_attribute_strict = true; role_attribute_strict = true;
role_attribute_path = "resource_access.monitoring.roles[0]"; role_attribute_path = "resource_access.monitoring.roles[0]";
}; };
@ -65,7 +52,7 @@ in
cookie_secure = true; cookie_secure = true;
disable_gravatar = true; disable_gravatar = true;
}; };
feature_toggles.enable = [ feature_toggles.enable = iniList [
"tempoSearch" "tempoSearch"
"tempoBackendSearch" "tempoBackendSearch"
"tempoServiceGraph" "tempoServiceGraph"
@ -73,18 +60,17 @@ in
}; };
provision = { provision = {
enable = true; enable = true;
datasources = [ datasources.settings.datasources = [
{ {
name = "Prometheus"; name = "Prometheus";
# wait for https://github.com/NixOS/nixpkgs/pull/175330 uid = "PBFA97CFB590B2093";
# uid = "PBFA97CFB590B2093";
inherit (links.prometheus) url; inherit (links.prometheus) url;
type = "prometheus"; type = "prometheus";
isDefault = true; isDefault = true;
} }
{ {
name = "Loki"; name = "Loki";
# uid = "P8E80F9AEF21F6940"; uid = "P8E80F9AEF21F6940";
inherit (loki-ingest) url; inherit (loki-ingest) url;
type = "loki"; type = "loki";
} }

5
hosts/VEGAS/services/monitoring/tracing.nix
View file

@ -91,11 +91,10 @@ in {
PrivateTmp = true; PrivateTmp = true;
}; };
}; };
services.grafana.provision.datasources = [ services.grafana.provision.datasources.settings.datasources = [
{ {
name = "Tempo"; name = "Tempo";
# wait for https://github.com/NixOS/nixpkgs/pull/175330 uid = "P214B5B846CF3925F";
# uid = "P214B5B846CF3925F";
inherit (links.tempo) url; inherit (links.tempo) url;
type = "tempo"; type = "tempo";
jsonData = { jsonData = {

3
hosts/VEGAS/services/nextcloud/default.nix
View file

@ -18,7 +18,8 @@ in
}; };
}; };
services.nextcloud = { services.nextcloud = {
package = pkgs.nextcloud24; package = pkgs.nextcloud25;
enableBrokenCiphersForSSE = false;
enable = true; enable = true;
https = true; https = true;
hostName = "storage.${tools.meta.domain}"; hostName = "storage.${tools.meta.domain}";

2
modules/deploy-rs-receiver/default.nix
View file

@ -9,7 +9,7 @@
runAs = "root"; runAs = "root";
} }
]; ];
nix.trustedUsers = [ "deploy" ]; nix.settings.trusted-users = [ "deploy" ];
users.users.deploy = { users.users.deploy = {
isNormalUser = true; isNormalUser = true;
uid = 1999; uid = 1999;

2
modules/ipfs/default.nix
View file

@ -46,7 +46,7 @@ let
in in
{ {
disabledModules = [ disabledModules = [
"services/network-filesystems/ipfs.nix" "services/network-filesystems/kubo.nix"
]; ];
###### interface ###### interface

2
modules/nix-builder/default.nix
View file

@ -1,5 +1,5 @@
{ {
nix.trustedUsers = [ "nix" ]; nix.settings.trusted-users = [ "nix" ];
users.users.nix = { users.users.nix = {
isSystemUser = true; isSystemUser = true;
description = "Nix Remote Build"; description = "Nix Remote Build";

18
modules/nix-config/server.nix
View file

@ -1,15 +1,16 @@
{ pkgs, lib, config, inputs, tools, ... }: { pkgs, lib, config, inputs, tools, ... }:
let
fixPriority = x: if config.services.hydra.enable {
then lib.mkForce x
else x;
in {
nix = { nix = {
package = inputs.nix-super.packages.${pkgs.system}.default; package = inputs.nix-super.packages.${pkgs.system}.default;
trustedUsers = [ "root" "@wheel" "@admins" ]; settings = {
trusted-users = [ "root" "@wheel" "@admins" ];
binary-caches = [ "https://cache.${tools.meta.domain}" ];
trusted-public-keys = [ "cache.privatevoid.net:SErQ8bvNWANeAvtsOESUwVYr2VJynfuc9JRwlzTTkVg=" ];
};
extraOptions = fixPriority '' extraOptions = ''
experimental-features = nix-command flakes experimental-features = nix-command flakes
builders-use-substitutes = true builders-use-substitutes = true
flake-registry = https://git.${tools.meta.domain}/private-void/registry/-/raw/master/registry.json flake-registry = https://git.${tools.meta.domain}/private-void/registry/-/raw/master/registry.json
@ -18,9 +19,6 @@ in {
narinfo-cache-negative-ttl = 0 narinfo-cache-negative-ttl = 0
''; '';
binaryCaches = [ "https://cache.${tools.meta.domain}" ];
binaryCachePublicKeys = [ "cache.privatevoid.net:SErQ8bvNWANeAvtsOESUwVYr2VJynfuc9JRwlzTTkVg=" ];
gc = { gc = {
automatic = true; automatic = true;
dates = "weekly"; dates = "weekly";

4
modules/patroni/default.nix
View file

@ -17,6 +17,10 @@ let
configFile = format.generate "patroni.yml" cfg.settings; configFile = format.generate "patroni.yml" cfg.settings;
in in
{ {
disabledModules = [
"services/cluster/patroni/default.nix"
];
options.services.patroni = { options.services.patroni = {
enable = mkEnableOption "Whether to enable Patroni"; enable = mkEnableOption "Whether to enable Patroni";

2
packages/checks/keycloak-custom-jre.nix
View file

@ -13,7 +13,7 @@ nixosTest {
}; };
testScript = '' testScript = ''
machine.wait_for_unit("keycloak.service") machine.wait_for_unit("keycloak.service")
machine.wait_for_open_port("80") machine.wait_for_open_port(80)
machine.succeed("curl --fail http://127.0.0.1:80") machine.succeed("curl --fail http://127.0.0.1:80")
''; '';
} }

2
packages/checks/tempo.nix
View file

@ -72,7 +72,7 @@ nixosTest {
}; };
testScript = '' testScript = ''
machine.wait_for_unit("tempo.service") machine.wait_for_unit("tempo.service")
machine.wait_for_open_port("8888") machine.wait_for_open_port(8888)
machine.succeed("curl --fail http://127.0.0.1:8888/status/version") machine.succeed("curl --fail http://127.0.0.1:8888/status/version")
''; '';
} }

18
packages/patched-derivations.nix
View file

@ -3,8 +3,16 @@ let
pins = import ./sources; pins = import ./sources;
in with tools; in with tools;
super: rec { super: rec {
dvc = patch (super.dvc.overrideAttrs (old: { dvc = patch (super.dvc.overrideAttrs (old: let
propagatedBuildInputs = with super.python3Packages; old.propagatedBuildInputs ++ [ filteredBaseDeps = super.lib.subtractLists [
super.python3Packages.dvc-data
] old.propagatedBuildInputs;
baseDeps = filteredBaseDeps ++ [
dvc-data
];
in {
propagatedBuildInputs = with super.python3Packages; baseDeps ++ [
aiobotocore aiobotocore
boto3 boto3
(s3fs.overrideAttrs (_: { postPatch = '' (s3fs.overrideAttrs (_: { postPatch = ''
@ -16,6 +24,12 @@ super: rec {
]; ];
})) "patches/base/dvc"; })) "patches/base/dvc";
dvc-data = patch (super.python3Packages.dvc-data.override {
inherit dvc-objects;
}) "patches/base/dvc-data";
dvc-objects = patch super.python3Packages.dvc-objects "patches/base/dvc-objects";
sssd = (super.sssd.override { withSudo = true; }).overrideAttrs (old: { sssd = (super.sssd.override { withSudo = true; }).overrideAttrs (old: {
postFixup = (old.postFixup or "") + '' postFixup = (old.postFixup or "") + ''
${super.removeReferencesTo}/bin/remove-references-to -t ${super.stdenv.cc.cc} $out/modules/ldb/memberof.so ${super.removeReferencesTo}/bin/remove-references-to -t ${super.stdenv.cc.cc} $out/modules/ldb/memberof.so

4
packages/projects.nix
View file

@ -36,7 +36,7 @@
cinny = pkgs.callPackage ./web-apps/cinny { inherit pins; }; cinny = pkgs.callPackage ./web-apps/cinny { inherit pins; };
excalidraw = let excalidraw = let
dream = dream2nix.makeOutputs { dream = dream2nix.dream2nix-interface.makeOutputs {
source = pins.excalidraw; source = pins.excalidraw;
}; };
inherit (dream.packages) excalidraw; inherit (dream.packages) excalidraw;
@ -44,7 +44,7 @@
excalidraw // { webroot = "${excalidraw}/${excalidraw.webPath}"; }; excalidraw // { webroot = "${excalidraw}/${excalidraw.webPath}"; };
uptime-kuma = let uptime-kuma = let
dream = dream2nix.makeOutputs { dream = dream2nix.dream2nix-interface.makeOutputs {
source = pins.uptime-kuma; source = pins.uptime-kuma;
}; };
inherit (dream.packages) uptime-kuma; inherit (dream.packages) uptime-kuma;

20
packages/sources/sources.json
View file

@ -22,9 +22,9 @@
"repo": "excalidraw" "repo": "excalidraw"
}, },
"branch": "master", "branch": "master",
"revision": "640affe7c06fc20cf93787b67cc8dea7aa50c51b", "revision": "73a45e19889de08be64faf2cdddae2b9ecaa1a20",
"url": "https://github.com/excalidraw/excalidraw/archive/640affe7c06fc20cf93787b67cc8dea7aa50c51b.tar.gz", "url": "https://github.com/excalidraw/excalidraw/archive/73a45e19889de08be64faf2cdddae2b9ecaa1a20.tar.gz",
"hash": "0yfx3s6hb0mqci6agr6cm8g7dc4yzc4yn0gcwn8fpl6931k26y49" "hash": "1mdyf8f0ayiy9dary648y6zdqg5qpr6vgidifqf3jl1r7l44ysg5"
}, },
"searxng": { "searxng": {
"type": "Git", "type": "Git",
@ -34,9 +34,9 @@
"repo": "searxng" "repo": "searxng"
}, },
"branch": "master", "branch": "master",
"revision": "3a765113eefc7ac0b1ee94e9a0ac0db8069ec336", "revision": "b92748219504e67b18771b9831ffcebe76cca62f",
"url": "https://github.com/searxng/searxng/archive/3a765113eefc7ac0b1ee94e9a0ac0db8069ec336.tar.gz", "url": "https://github.com/searxng/searxng/archive/b92748219504e67b18771b9831ffcebe76cca62f.tar.gz",
"hash": "0i22np3kvwwmzxg1h5xw5vslgq3d0x69a6qy93xy4n4b708drwf0" "hash": "0zjzv8xr95rf2xakav91xnnh360614lx0658x979nirisrb35mh2"
}, },
"stevenblack-hosts": { "stevenblack-hosts": {
"type": "GitRelease", "type": "GitRelease",
@ -47,10 +47,10 @@
}, },
"pre_releases": false, "pre_releases": false,
"version_upper_bound": null, "version_upper_bound": null,
"version": "3.11.29", "version": "3.11.42",
"revision": "cfddf556709f28efc4a958200d3fb4e6e6b4119a", "revision": "5a7df99b1125e789c28a03fe5b87de32c112ffa0",
"url": "https://api.github.com/repos/StevenBlack/hosts/tarball/3.11.29", "url": "https://api.github.com/repos/StevenBlack/hosts/tarball/3.11.42",
"hash": "1whv0qrcq2i59i6lhb72alaapcsm3pkw065v7nlg2vi9g5y7jziq" "hash": "02qbmpwkfyb0xaf9f85v4rwhmwhdwb6iswk28jydvmfr3yhzkz83"
}, },
"tempo": { "tempo": {
"type": "GitRelease", "type": "GitRelease",

612
patches/base/dvc-data/md5-to-sha256.patch Normal file
View file

@ -0,0 +1,612 @@
commit d7d093fcb91b0d21faf36dbf62924f23b45abb9b
Author: Max <max@privatevoid.net>
Date: Sat Dec 17 14:23:59 2022 +0100
md5 to sha256 for 2.17.0
diff --git a/src/dvc_data/build.py b/src/dvc_data/build.py
index 3656ca5..3837763 100644
--- a/src/dvc_data/build.py
+++ b/src/dvc_data/build.py
@@ -63,7 +63,7 @@ def _build_file(path, fs, name, odb=None, upload_odb=None, dry_run=False):
state = odb.state if odb else None
meta, hash_info = hash_file(path, fs, name, state=state)
if upload_odb and not dry_run:
- assert odb and name == "md5"
+ assert odb and name == "sha256"
return _upload_file(path, fs, odb, upload_odb)
oid = hash_info.value
@@ -195,9 +195,9 @@ def _get_staging(odb: "HashFileDB") -> "ReferenceHashFileDB":
def _build_external_tree_info(odb, tree, name):
# NOTE: used only for external outputs. Initial reasoning was to be
# able to validate .dir files right in the workspace (e.g. check s3
- # etag), but could be dropped for manual validation with regular md5,
+ # etag), but could be dropped for manual validation with regular sha256,
# that would be universal for all clouds.
- assert odb and name != "md5"
+ assert odb and name != "sha256"
oid = tree.hash_info.value
odb.add(tree.path, tree.fs, oid)
@@ -253,7 +253,7 @@ def build(
**kwargs,
)
logger.debug("built tree '%s'", obj)
- if name != "md5":
+ if name != "sha256":
obj = _build_external_tree_info(odb, obj, name)
else:
meta, obj = _build_file(
diff --git a/src/dvc_data/cli.py b/src/dvc_data/cli.py
index 2348875..ece639a 100644
--- a/src/dvc_data/cli.py
+++ b/src/dvc_data/cli.py
@@ -29,8 +29,8 @@ from dvc_data.diff import ROOT
from dvc_data.diff import diff as _diff
from dvc_data.hashfile.db import HashFileDB
from dvc_data.hashfile.hash import algorithms_available
-from dvc_data.hashfile.hash import file_md5 as _file_md5
-from dvc_data.hashfile.hash import fobj_md5 as _fobj_md5
+from dvc_data.hashfile.hash import file_sha256 as _file_sha256
+from dvc_data.hashfile.hash import fobj_sha256 as _fobj_sha256
from dvc_data.hashfile.hash_info import HashInfo
from dvc_data.hashfile.obj import HashFile
from dvc_data.hashfile.state import State
@@ -93,7 +93,7 @@ app = Application(
@app.command(name="hash", help="Compute checksum of the file")
def hash_file(
file: Path = file_type,
- name: HashEnum = typer.Option("md5", "-n", "--name"),
+ name: HashEnum = typer.Option("sha256", "-n", "--name"),
progress: bool = typer.Option(False, "--progress", "-p"),
text: Optional[bool] = typer.Option(None, "--text/--binary", "-t/-b"),
):
@@ -108,9 +108,9 @@ def hash_file(
with callback:
if path == "-":
fobj = callback.wrap_attr(sys.stdin.buffer)
- hash_value = _fobj_md5(fobj, text=text, name=hash_name)
+ hash_value = _fobj_sha256(fobj, text=text, name=hash_name)
else:
- hash_value = _file_md5(
+ hash_value = _file_sha256(
path, name=hash_name, callback=callback, text=text
)
print(hash_name, hash_value, sep=": ")
@@ -262,7 +262,7 @@ def build(
fs = MemoryFileSystem()
fs.put_file(sys.stdin.buffer, fs_path)
- object_store, _, obj = _build(odb, fs_path, fs, name="md5")
+ object_store, _, obj = _build(odb, fs_path, fs, name="sha256")
if write:
_transfer(
object_store,
@@ -285,7 +285,7 @@ def ls(oid: str = typer.Argument(..., allow_dash=True)):
odb = get_odb()
oid = from_shortoid(odb, oid)
try:
- tree = Tree.load(odb, HashInfo("md5", oid))
+ tree = Tree.load(odb, HashInfo("sha256", oid))
except ObjectFormatError as exc:
typer.echo(exc, err=True)
raise typer.Exit(1) from exc
@@ -454,7 +454,7 @@ def apply_op(odb, obj, application):
)
fs = LocalFileSystem()
- _, meta, new_obj = _build(odb, path, fs, "md5")
+ _, meta, new_obj = _build(odb, path, fs, "sha256")
odb.add(path, fs, new_obj.hash_info.value, hardlink=False)
return obj.add(new, meta, new_obj.hash_info)
diff --git a/src/dvc_data/fs.py b/src/dvc_data/fs.py
index c972981..ac45ad3 100644
--- a/src/dvc_data/fs.py
+++ b/src/dvc_data/fs.py
@@ -47,7 +47,7 @@ class DataFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
if info["type"] == "directory":
raise IsADirectoryError
- value = info.get("md5")
+ value = info.get("sha256")
if not value:
raise FileNotFoundError
@@ -142,7 +142,7 @@ class DataFileSystem(AbstractFileSystem): # pylint:disable=abstract-method
def checksum(self, path):
info = self.info(path)
- md5 = info.get("md5")
- if md5:
- return md5
+ sha256 = info.get("sha256")
+ if sha256:
+ return sha256
raise NotImplementedError
diff --git a/src/dvc_data/hashfile/hash.py b/src/dvc_data/hashfile/hash.py
index 9bef01d..03f731c 100644
--- a/src/dvc_data/hashfile/hash.py
+++ b/src/dvc_data/hashfile/hash.py
@@ -42,7 +42,7 @@ class HashStreamFile(io.IOBase):
def __init__(
self,
fobj: BinaryIO,
- hash_name: str = "md5",
+ hash_name: str = "sha256",
text: Optional[bool] = None,
) -> None:
self.fobj = fobj
@@ -77,11 +77,11 @@ class HashStreamFile(io.IOBase):
return self.hasher.name
-def fobj_md5(
+def fobj_sha256(
fobj: BinaryIO,
chunk_size: int = 2**20,
text: Optional[bool] = None,
- name="md5",
+ name="sha256",
) -> str:
# ideally, we want the heuristics to be applied in a similar way,
# regardless of the size of the first chunk,
@@ -95,17 +95,17 @@ def fobj_md5(
return stream.hash_value
-def file_md5(
+def file_sha256(
fname: "AnyFSPath",
fs: "FileSystem" = localfs,
callback: "Callback" = DEFAULT_CALLBACK,
text: Optional[bool] = None,
- name: str = "md5",
+ name: str = "sha256",
) -> str:
size = fs.size(fname) or 0
callback.set_size(size)
with fs.open(fname, "rb") as fobj:
- return fobj_md5(callback.wrap_attr(fobj), text=text, name=name)
+ return fobj_sha256(callback.wrap_attr(fobj), text=text, name=name)
def _adapt_info(info: Dict[str, Any], scheme: str) -> Dict[str, Any]:
@@ -139,8 +139,8 @@ def _hash_file(
func = getattr(fs, name)
return str(func(path)), info
- if name == "md5":
- return file_md5(path, fs, callback=callback), info
+ if name == "sha256":
+ return file_sha256(path, fs, callback=callback), info
raise NotImplementedError
@@ -162,7 +162,7 @@ class LargeFileHashingCallback(TqdmCallback):
if self.size and self.size > self.LARGE_FILE_SIZE:
if not self._logged:
logger.info(
- f"Computing md5 for a large file '{self.fname}'. "
+ f"Computing sha256 for a large file '{self.fname}'. "
"This is only done once."
)
self._logged = True
diff --git a/src/dvc_data/hashfile/utils.py b/src/dvc_data/hashfile/utils.py
index ea2da9c..b1e7726 100644
--- a/src/dvc_data/hashfile/utils.py
+++ b/src/dvc_data/hashfile/utils.py
@@ -38,7 +38,7 @@ def get_mtime_and_size(
# We track file changes and moves, which cannot be detected with simply
# max(mtime(f) for f in non_ignored_files)
- hasher = hashlib.md5()
+ hasher = hashlib.sha256()
hasher.update(json.dumps(files_mtimes, sort_keys=True).encode("utf-8"))
mtime = hasher.hexdigest()
return mtime, size
diff --git a/src/dvc_data/objects/tree.py b/src/dvc_data/objects/tree.py
index 4f11fa4..7c8b417 100644
--- a/src/dvc_data/objects/tree.py
+++ b/src/dvc_data/objects/tree.py
@@ -81,7 +81,7 @@ class Tree(HashFile):
memfs.pipe_file(path, self.as_bytes())
self.fs = memfs
self.path = path
- _, self.hash_info = hash_file(path, memfs, "md5")
+ _, self.hash_info = hash_file(path, memfs, "sha256")
assert self.hash_info.value
self.hash_info.value += ".dir"
self.oid = self.hash_info.value
diff --git a/tests/hashfile/test_hash.py b/tests/hashfile/test_hash.py
index ca920d8..59bf765 100644
--- a/tests/hashfile/test_hash.py
+++ b/tests/hashfile/test_hash.py
@@ -2,21 +2,21 @@ from os import fspath
from dvc_objects.fs import LocalFileSystem
-from dvc_data.hashfile.hash import file_md5
+from dvc_data.hashfile.hash import file_sha256
-def test_file_md5(tmp_path):
+def test_file_sha256(tmp_path):
foo = tmp_path / "foo"
foo.write_text("foo content", encoding="utf8")
fs = LocalFileSystem()
- assert file_md5(fspath(foo), fs) == file_md5(fspath(foo), fs)
+ assert file_sha256(fspath(foo), fs) == file_sha256(fspath(foo), fs)
-def test_file_md5_crlf(tmp_path):
+def test_file_sha256_crlf(tmp_path):
fs = LocalFileSystem()
cr = tmp_path / "cr"
crlf = tmp_path / "crlf"
cr.write_bytes(b"a\nb\nc")
crlf.write_bytes(b"a\r\nb\r\nc")
- assert file_md5(fspath(cr), fs) == file_md5(fspath(crlf), fs)
+ assert file_sha256(fspath(cr), fs) == file_sha256(fspath(crlf), fs)
diff --git a/tests/hashfile/test_hash_stream.py b/tests/hashfile/test_hash_stream.py
index a003a29..e67b7c1 100644
--- a/tests/hashfile/test_hash_stream.py
+++ b/tests/hashfile/test_hash_stream.py
@@ -3,7 +3,7 @@ from os import fspath
import pytest
from dvc_objects.fs import LocalFileSystem
-from dvc_data.hashfile.hash import HashStreamFile, file_md5
+from dvc_data.hashfile.hash import HashStreamFile, file_sha256
from dvc_data.hashfile.istextfile import DEFAULT_CHUNK_SIZE, istextfile
@@ -23,7 +23,7 @@ def test_hashed_stream_reader(tmp_path):
assert stream_reader.read(1) == b"o"
assert stream_reader.tell() == 3
- hex_digest = file_md5(fspath(foo), LocalFileSystem())
+ hex_digest = file_sha256(fspath(foo), LocalFileSystem())
assert stream_reader.is_text
assert hex_digest == stream_reader.hash_value
@@ -46,7 +46,7 @@ def test_hashed_stream_reader_as_chunks(tmp_path):
assert stream_reader.tell() == actual_size == total_read
- hex_digest = file_md5(fspath(foo), LocalFileSystem())
+ hex_digest = file_sha256(fspath(foo), LocalFileSystem())
assert not stream_reader.is_text
assert hex_digest == stream_reader.hash_value
@@ -68,7 +68,7 @@ def test_hashed_stream_reader_compatibility(tmp_path, contents):
stream_reader.read(chunk_size)
local_fs = LocalFileSystem()
- hex_digest = file_md5(fspath(data), local_fs)
+ hex_digest = file_sha256(fspath(data), local_fs)
assert stream_reader.is_text is istextfile(fspath(data), local_fs)
assert stream_reader.hash_value == hex_digest
diff --git a/tests/hashfile/test_obj.py b/tests/hashfile/test_obj.py
index 01e9fc2..6c47b3c 100644
--- a/tests/hashfile/test_obj.py
+++ b/tests/hashfile/test_obj.py
@@ -3,7 +3,7 @@ from dvc_data.hashfile.obj import HashFile
def test_obj(tmp_upath):
- hash_info = HashInfo("md5", "123456")
+ hash_info = HashInfo("sha256", "123456")
obj = HashFile(tmp_upath, tmp_upath.fs, hash_info)
assert obj.path == tmp_upath
assert obj.fs == tmp_upath.fs
diff --git a/tests/objects/test_tree.py b/tests/objects/test_tree.py
index 6c514ba..611a72f 100644
--- a/tests/objects/test_tree.py
+++ b/tests/objects/test_tree.py
@@ -13,57 +13,57 @@ from dvc_data.objects.tree import Tree, _merge
([], {}),
(
[
- {"md5": "def", "relpath": "zzz"},
- {"md5": "123", "relpath": "foo"},
- {"md5": "abc", "relpath": "aaa"},
- {"md5": "456", "relpath": "bar"},
+ {"sha256": "def", "relpath": "zzz"},
+ {"sha256": "123", "relpath": "foo"},
+ {"sha256": "abc", "relpath": "aaa"},
+ {"sha256": "456", "relpath": "bar"},
],
{
- ("zzz",): (None, HashInfo("md5", "def")),
- ("foo",): (None, HashInfo("md5", "123")),
- ("bar",): (None, HashInfo("md5", "456")),
- ("aaa",): (None, HashInfo("md5", "abc")),
+ ("zzz",): (None, HashInfo("sha256", "def")),
+ ("foo",): (None, HashInfo("sha256", "123")),
+ ("bar",): (None, HashInfo("sha256", "456")),
+ ("aaa",): (None, HashInfo("sha256", "abc")),
},
),
(
[
- {"md5": "123", "relpath": "dir/b"},
- {"md5": "456", "relpath": "dir/z"},
- {"md5": "789", "relpath": "dir/a"},
- {"md5": "abc", "relpath": "b"},
- {"md5": "def", "relpath": "a"},
- {"md5": "ghi", "relpath": "z"},
- {"md5": "jkl", "relpath": "dir/subdir/b"},
- {"md5": "mno", "relpath": "dir/subdir/z"},
- {"md5": "pqr", "relpath": "dir/subdir/a"},
+ {"sha256": "123", "relpath": "dir/b"},
+ {"sha256": "456", "relpath": "dir/z"},
+ {"sha256": "789", "relpath": "dir/a"},
+ {"sha256": "abc", "relpath": "b"},
+ {"sha256": "def", "relpath": "a"},
+ {"sha256": "ghi", "relpath": "z"},
+ {"sha256": "jkl", "relpath": "dir/subdir/b"},
+ {"sha256": "mno", "relpath": "dir/subdir/z"},
+ {"sha256": "pqr", "relpath": "dir/subdir/a"},
],
{
("dir", "b"): (
None,
- HashInfo("md5", "123"),
+ HashInfo("sha256", "123"),
),
("dir", "z"): (
None,
- HashInfo("md5", "456"),
+ HashInfo("sha256", "456"),
),
("dir", "a"): (
None,
- HashInfo("md5", "789"),
+ HashInfo("sha256", "789"),
),
- ("b",): (None, HashInfo("md5", "abc")),
- ("a",): (None, HashInfo("md5", "def")),
- ("z",): (None, HashInfo("md5", "ghi")),
+ ("b",): (None, HashInfo("sha256", "abc")),
+ ("a",): (None, HashInfo("sha256", "def")),
+ ("z",): (None, HashInfo("sha256", "ghi")),
("dir", "subdir", "b"): (
None,
- HashInfo("md5", "jkl"),
+ HashInfo("sha256", "jkl"),
),
("dir", "subdir", "z"): (
None,
- HashInfo("md5", "mno"),
+ HashInfo("sha256", "mno"),
),
("dir", "subdir", "a"): (
None,
- HashInfo("md5", "pqr"),
+ HashInfo("sha256", "pqr"),
),
},
),
@@ -81,19 +81,19 @@ def test_list(lst, trie_dict):
({}, 0),
(
{
- ("a",): (Meta(size=1), HashInfo("md5", "abc")),
- ("b",): (Meta(size=2), HashInfo("md5", "def")),
- ("c",): (Meta(size=3), HashInfo("md5", "ghi")),
- ("dir", "foo"): (Meta(size=4), HashInfo("md5", "jkl")),
- ("dir", "bar"): (Meta(size=5), HashInfo("md5", "mno")),
- ("dir", "baz"): (Meta(size=6), HashInfo("md5", "pqr")),
+ ("a",): (Meta(size=1), HashInfo("sha256", "abc")),
+ ("b",): (Meta(size=2), HashInfo("sha256", "def")),
+ ("c",): (Meta(size=3), HashInfo("sha256", "ghi")),
+ ("dir", "foo"): (Meta(size=4), HashInfo("sha256", "jkl")),
+ ("dir", "bar"): (Meta(size=5), HashInfo("sha256", "mno")),
+ ("dir", "baz"): (Meta(size=6), HashInfo("sha256", "pqr")),
},
6,
),
(
{
- ("a",): (Meta(size=1), HashInfo("md5", "abc")),
- ("b",): (Meta(), HashInfo("md5", "def")),
+ ("a",): (Meta(size=1), HashInfo("sha256", "abc")),
+ ("b",): (Meta(), HashInfo("sha256", "def")),
},
2,
),
@@ -110,15 +110,15 @@ def test_nfiles(trie_dict, nfiles):
[
{},
{
- ("a",): (None, HashInfo("md5", "abc")),
- ("b",): (None, HashInfo("md5", "def")),
- ("c",): (None, HashInfo("md5", "ghi")),
- ("dir", "foo"): (None, HashInfo("md5", "jkl")),
- ("dir", "bar"): (None, HashInfo("md5", "mno")),
- ("dir", "baz"): (None, HashInfo("md5", "pqr")),
- ("dir", "subdir", "1"): (None, HashInfo("md5", "stu")),
- ("dir", "subdir", "2"): (None, HashInfo("md5", "vwx")),
- ("dir", "subdir", "3"): (None, HashInfo("md5", "yz")),
+ ("a",): (None, HashInfo("sha256", "abc")),
+ ("b",): (None, HashInfo("sha256", "def")),
+ ("c",): (None, HashInfo("sha256", "ghi")),
+ ("dir", "foo"): (None, HashInfo("sha256", "jkl")),
+ ("dir", "bar"): (None, HashInfo("sha256", "mno")),
+ ("dir", "baz"): (None, HashInfo("sha256", "pqr")),
+ ("dir", "subdir", "1"): (None, HashInfo("sha256", "stu")),
+ ("dir", "subdir", "2"): (None, HashInfo("sha256", "vwx")),
+ ("dir", "subdir", "3"): (None, HashInfo("sha256", "yz")),
},
],
)
@@ -135,63 +135,63 @@ def test_items(trie_dict):
[
({}, {}, {}, {}),
(
- {("foo",): HashInfo("md5", "123")},
+ {("foo",): HashInfo("sha256", "123")},
{
- ("foo",): HashInfo("md5", "123"),
- ("bar",): HashInfo("md5", "345"),
+ ("foo",): HashInfo("sha256", "123"),
+ ("bar",): HashInfo("sha256", "345"),
},
{
- ("foo",): HashInfo("md5", "123"),
- ("baz",): HashInfo("md5", "678"),
+ ("foo",): HashInfo("sha256", "123"),
+ ("baz",): HashInfo("sha256", "678"),
},
{
- ("foo",): HashInfo("md5", "123"),
- ("bar",): HashInfo("md5", "345"),
- ("baz",): HashInfo("md5", "678"),
+ ("foo",): HashInfo("sha256", "123"),
+ ("bar",): HashInfo("sha256", "345"),
+ ("baz",): HashInfo("sha256", "678"),
},
),
(
{
- ("common",): HashInfo("md5", "123"),
- ("subdir", "foo"): HashInfo("md5", "345"),
+ ("common",): HashInfo("sha256", "123"),
+ ("subdir", "foo"): HashInfo("sha256", "345"),
},
{
- ("common",): HashInfo("md5", "123"),
- ("subdir", "foo"): HashInfo("md5", "345"),
- ("subdir", "bar"): HashInfo("md5", "678"),
+ ("common",): HashInfo("sha256", "123"),
+ ("subdir", "foo"): HashInfo("sha256", "345"),
+ ("subdir", "bar"): HashInfo("sha256", "678"),
},
{
- ("common",): HashInfo("md5", "123"),
- ("subdir", "foo"): HashInfo("md5", "345"),
- ("subdir", "baz"): HashInfo("md5", "91011"),
+ ("common",): HashInfo("sha256", "123"),
+ ("subdir", "foo"): HashInfo("sha256", "345"),
+ ("subdir", "baz"): HashInfo("sha256", "91011"),
},
{
- ("common",): HashInfo("md5", "123"),
- ("subdir", "foo"): HashInfo("md5", "345"),
- ("subdir", "bar"): HashInfo("md5", "678"),
- ("subdir", "baz"): HashInfo("md5", "91011"),
+ ("common",): HashInfo("sha256", "123"),
+ ("subdir", "foo"): HashInfo("sha256", "345"),
+ ("subdir", "bar"): HashInfo("sha256", "678"),
+ ("subdir", "baz"): HashInfo("sha256", "91011"),
},
),
(
{},
- {("foo",): HashInfo("md5", "123")},
- {("bar",): HashInfo("md5", "456")},
+ {("foo",): HashInfo("sha256", "123")},
+ {("bar",): HashInfo("sha256", "456")},
{
- ("foo",): HashInfo("md5", "123"),
- ("bar",): HashInfo("md5", "456"),
+ ("foo",): HashInfo("sha256", "123"),
+ ("bar",): HashInfo("sha256", "456"),
},
),
(
{},
{},
- {("bar",): HashInfo("md5", "123")},
- {("bar",): HashInfo("md5", "123")},
+ {("bar",): HashInfo("sha256", "123")},
+ {("bar",): HashInfo("sha256", "123")},
),
(
{},
- {("bar",): HashInfo("md5", "123")},
+ {("bar",): HashInfo("sha256", "123")},
{},
- {("bar",): HashInfo("md5", "123")},
+ {("bar",): HashInfo("sha256", "123")},
),
],
)
diff --git a/tests/test_index.py b/tests/test_index.py
index c6404fa..635bf66 100644
--- a/tests/test_index.py
+++ b/tests/test_index.py
@@ -17,8 +17,8 @@ def odb(tmp_upath_factory, as_filesystem):
data = tmp_upath_factory.mktemp() / "data.dir"
data.write_bytes(
- b'[{"md5": "c157a79031e1c40f85931829bc5fc552", "relpath": "bar"}, '
- b'{"md5": "258622b1688250cb619f3c9ccaefb7eb", "relpath": "baz"}]'
+ b'[{"sha256": "c157a79031e1c40f85931829bc5fc552", "relpath": "bar"}, '
+ b'{"sha256": "258622b1688250cb619f3c9ccaefb7eb", "relpath": "baz"}]'
)
bar = tmp_upath_factory.mktemp() / "bar"
@@ -46,13 +46,13 @@ def test_fs(tmp_upath, odb, as_filesystem):
("foo",): DataIndexEntry(
odb=odb,
hash_info=HashInfo(
- name="md5", value="d3b07384d113edec49eaa6238ad5ff00"
+ name="sha256", value="d3b07384d113edec49eaa6238ad5ff00"
),
),
("data",): DataIndexEntry(
odb=odb,
hash_info=HashInfo(
- name="md5",
+ name="sha256",
value="1f69c66028c35037e8bf67e5bc4ceb6a.dir",
),
),
@@ -80,22 +80,22 @@ def test_build(tmp_upath, odb, as_filesystem):
},
)
build(index, tmp_upath, as_filesystem(tmp_upath.fs))
- assert index[("foo",)].hash_info.name == "md5"
+ assert index[("foo",)].hash_info.name == "sha256"
assert (
index[("foo",)].hash_info.value == "d3b07384d113edec49eaa6238ad5ff00"
)
assert index[("foo",)].odb == odb
- assert index[("data",)].hash_info.name == "md5"
+ assert index[("data",)].hash_info.name == "sha256"
assert (
index[("data",)].hash_info.value
== "1f69c66028c35037e8bf67e5bc4ceb6a.dir"
)
- assert index[("data", "bar")].hash_info.name == "md5"
+ assert index[("data", "bar")].hash_info.name == "sha256"
assert (
index[("data", "bar")].hash_info.value
== "c157a79031e1c40f85931829bc5fc552"
)
- assert index[("data", "baz")].hash_info.name == "md5"
+ assert index[("data", "baz")].hash_info.name == "sha256"
assert (
index[("data", "baz")].hash_info.value
== "258622b1688250cb619f3c9ccaefb7eb"
@@ -108,13 +108,13 @@ def test_checkout(tmp_upath, odb, as_filesystem):
("foo",): DataIndexEntry(
odb=odb,
hash_info=HashInfo(
- name="md5", value="d3b07384d113edec49eaa6238ad5ff00"
+ name="sha256", value="d3b07384d113edec49eaa6238ad5ff00"
),
),
("data",): DataIndexEntry(
odb=odb,
hash_info=HashInfo(
- name="md5",
+ name="sha256",
value="1f69c66028c35037e8bf67e5bc4ceb6a.dir",
),
),

71
patches/base/dvc-objects/md5-to-sha256.patch Normal file
View file

@ -0,0 +1,71 @@
commit 2065fc148ce77be68c95a81a05391e1bb35da79d
Author: Max <max@privatevoid.net>
Date: Sat Dec 17 14:35:20 2022 +0100
md5 to sha256 for 2.17.0
diff --git a/src/dvc_objects/db.py b/src/dvc_objects/db.py
index 0f0ab16..3b87fdb 100644
--- a/src/dvc_objects/db.py
+++ b/src/dvc_objects/db.py
@@ -229,7 +229,7 @@ class ObjectDB:
returned.
NOTE: For large remotes the list of oids will be very
- big(e.g. 100M entries, md5 for each is 32 bytes, so ~3200Mb list)
+ big(e.g. 100M entries, sha256 for each is 32 bytes, so ~3200Mb list)
and we don't really need all of it at the same time, so it makes
sense to use a generator to gradually iterate over it, without
keeping all of it in memory.
diff --git a/src/dvc_objects/fs/__init__.py b/src/dvc_objects/fs/__init__.py
index d236fdc..74db3fe 100644
--- a/src/dvc_objects/fs/__init__.py
+++ b/src/dvc_objects/fs/__init__.py
@@ -62,7 +62,7 @@ def get_fs_cls(remote_conf, cls=None, scheme=None):
def as_filesystem(
fs: "AbstractFileSystem",
- checksum: str = "md5",
+ checksum: str = "sha256",
object_based: bool = False,
**fs_args,
) -> "FileSystem":
diff --git a/src/dvc_objects/fs/implementations/local.py b/src/dvc_objects/fs/implementations/local.py
index 7f888ec..3e1a61a 100644
--- a/src/dvc_objects/fs/implementations/local.py
+++ b/src/dvc_objects/fs/implementations/local.py
@@ -167,7 +167,7 @@ class LocalFileSystem(FileSystem):
sep = os.sep
protocol = "local"
- PARAM_CHECKSUM = "md5"
+ PARAM_CHECKSUM = "sha256"
PARAM_PATH = "path"
TRAVERSE_PREFIX_LEN = 2
diff --git a/src/dvc_objects/fs/implementations/memory.py b/src/dvc_objects/fs/implementations/memory.py
index 97702cb..c5b5ad7 100644
--- a/src/dvc_objects/fs/implementations/memory.py
+++ b/src/dvc_objects/fs/implementations/memory.py
@@ -3,7 +3,7 @@ from ..base import FileSystem
class MemoryFileSystem(FileSystem): # pylint:disable=abstract-method
protocol = "memory"
- PARAM_CHECKSUM = "md5"
+ PARAM_CHECKSUM = "sha256"
def __init__(self, global_store=True, trie_based=False, fs=None, **kwargs):
super().__init__(fs=fs, **kwargs)
diff --git a/src/dvc_objects/fs/implementations/ssh.py b/src/dvc_objects/fs/implementations/ssh.py
index 8b93faf..8aed5e4 100644
--- a/src/dvc_objects/fs/implementations/ssh.py
+++ b/src/dvc_objects/fs/implementations/ssh.py
@@ -24,7 +24,7 @@ def ask_password(host, user, port):
class SSHFileSystem(FileSystem):
protocol = "ssh"
REQUIRES = {"sshfs": "sshfs"}
- PARAM_CHECKSUM = "md5"
+ PARAM_CHECKSUM = "sha256"
@classmethod
def _strip_protocol(cls, path: str) -> str:

36
patches/base/dvc/GHPR_8663-ignore-solve-re-error-on-group-name-redefinition-in-pathspec-0-10-x.patch Normal file
View file

@ -0,0 +1,36 @@
From 18eab2ef9a6e5fa1d9d7e93ea96710ad3949ccec Mon Sep 17 00:00:00 2001
From: hiroto7 <32275337+hiroto7@users.noreply.github.com>
Date: Tue, 6 Dec 2022 10:43:20 +0000
Subject: [PATCH] ignore: solve re.error on group name redefinition in pathspec
0.10.x
Remove regex concatenation that causes re.error
Fixes #8217
---
dvc/ignore.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/dvc/ignore.py b/dvc/ignore.py
index 2177768c29..2696e2678b 100644
--- a/dvc/ignore.py
+++ b/dvc/ignore.py
@@ -40,7 +40,7 @@ def __init__(self, pattern_list, dirname, sep):
]
self.ignore_spec = [
- (ignore, re.compile("|".join(item[0] for item in group)))
+ (ignore, [re.compile(item[0]) for item in group])
for ignore, group in groupby(
self.regex_pattern_list, lambda x: x[1]
)
@@ -107,8 +107,8 @@ def matches(pattern, path, is_dir) -> bool:
result = False
- for ignore, pattern in self.ignore_spec[::-1]:
- if matches(pattern, path, is_dir):
+ for ignore, patterns in self.ignore_spec[::-1]:
+ if any(matches(pattern, path, is_dir) for pattern in patterns):
result = ignore
break
return result

1486
patches/base/dvc/md5-to-sha256.patch
View file

File diff suppressed because it is too large Load diff

27
patches/base/dvc/no-analytics.patch
View file

@ -1,5 +1,11 @@
commit de4f3a29629628c24ca9b69533c83b571c92c73f
Author: Max <max@privatevoid.net>
Date: Sat Dec 17 13:47:49 2022 +0100
no analytics for 2.17.0
diff --git a/dvc/analytics.py b/dvc/analytics.py diff --git a/dvc/analytics.py b/dvc/analytics.py
index 8b9c6310..e6586005 100644 index af4823ea..7effc0b8 100644
--- a/dvc/analytics.py --- a/dvc/analytics.py
+++ b/dvc/analytics.py +++ b/dvc/analytics.py
@@ -1,12 +1,3 @@ @@ -1,12 +1,3 @@
@ -51,14 +57,14 @@ index 8b9c6310..e6586005 100644
- Config(validate=False).get("core", {}).get("analytics", "true") - Config(validate=False).get("core", {}).get("analytics", "true")
- ) - )
- -
- logger.debug("Analytics is {}abled.".format("en" if enabled else "dis")) - logger.debug("Analytics is %sabled.", "en" if enabled else "dis")
- -
- return enabled - return enabled
+ return False + return False
def send(path): def send(path):
@@ -63,88 +25,21 @@ def send(path): @@ -63,88 +25,22 @@ def send(path):
`collect_and_send_report`. Sending happens on another process, `collect_and_send_report`. Sending happens on another process,
thus, the need of removing such file afterwards. thus, the need of removing such file afterwards.
""" """
@ -147,10 +153,11 @@ index 8b9c6310..e6586005 100644
- } - }
- -
- # We don't collect data for any other system. - # We don't collect data for any other system.
+ # We don't collect data :)
raise NotImplementedError raise NotImplementedError
@@ -158,33 +53,4 @@ def _find_or_create_user_id(): @@ -158,33 +54,4 @@ def _find_or_create_user_id():
IDs are generated randomly with UUID. IDs are generated randomly with UUID.
""" """
@ -183,14 +190,14 @@ index 8b9c6310..e6586005 100644
- return user_id - return user_id
- -
- except LockError: - except LockError:
- logger.debug(f"Failed to acquire '{lockfile}'") - logger.debug("Failed to acquire '%s'", lockfile)
+ raise NotImplementedError + raise NotImplementedError
diff --git a/dvc/cli/__init__.py b/dvc/cli/__init__.py diff --git a/dvc/cli/__init__.py b/dvc/cli/__init__.py
index 3002654e..ee61bee4 100644 index e206befd..590b0790 100644
--- a/dvc/cli/__init__.py --- a/dvc/cli/__init__.py
+++ b/dvc/cli/__init__.py +++ b/dvc/cli/__init__.py
@@ -115,11 +115,6 @@ def main(argv=None): # noqa: C901 @@ -211,11 +211,6 @@ def main(argv=None): # noqa: C901
ret = 255 ret = _log_exceptions(exc) or 255
try: try:
- from dvc import analytics - from dvc import analytics
@ -202,7 +209,7 @@ index 3002654e..ee61bee4 100644
finally: finally:
logger.setLevel(outerLogLevel) logger.setLevel(outerLogLevel)
diff --git a/dvc/commands/daemon.py b/dvc/commands/daemon.py diff --git a/dvc/commands/daemon.py b/dvc/commands/daemon.py
index 3e212899..09abaccf 100644 index 2a22de3d..d64a6404 100644
--- a/dvc/commands/daemon.py --- a/dvc/commands/daemon.py
+++ b/dvc/commands/daemon.py +++ b/dvc/commands/daemon.py
@@ -26,15 +26,6 @@ class CmdDaemonUpdater(CmdDaemonBase): @@ -26,15 +26,6 @@ class CmdDaemonUpdater(CmdDaemonBase):
@ -267,7 +274,7 @@ index 3d0c774b..19b86b50 100644
"{yellow}What's next?{nc}\n" "{yellow}What's next?{nc}\n"
"{yellow}------------{nc}\n" "{yellow}------------{nc}\n"
diff --git a/dvc/config_schema.py b/dvc/config_schema.py diff --git a/dvc/config_schema.py b/dvc/config_schema.py
index aa495a89..54ae30b1 100644 index bd514c61..e358b949 100644
--- a/dvc/config_schema.py --- a/dvc/config_schema.py
+++ b/dvc/config_schema.py +++ b/dvc/config_schema.py
@@ -114,7 +114,6 @@ SCHEMA = { @@ -114,7 +114,6 @@ SCHEMA = {

10
patches/base/dvc/yaml-to-json.patch
View file

@ -1,5 +1,11 @@
commit eceb8d19ba9da3c7d07fc5a12636027d499a3a06
Author: Max <max@privatevoid.net>
Date: Sat Dec 17 13:57:49 2022 +0100
yaml to json for 2.17.0
diff --git a/dvc/dvcfile.py b/dvc/dvcfile.py diff --git a/dvc/dvcfile.py b/dvc/dvcfile.py
index e542ba98..40d13a40 100644 index 04db6d5f..4eb40e90 100644
--- a/dvc/dvcfile.py --- a/dvc/dvcfile.py
+++ b/dvc/dvcfile.py +++ b/dvc/dvcfile.py
@@ -14,7 +14,7 @@ from dvc.stage.exceptions import ( @@ -14,7 +14,7 @@ from dvc.stage.exceptions import (
@ -41,7 +47,7 @@ index e542ba98..40d13a40 100644
@@ -198,7 +198,7 @@ class SingleStageFile(FileMixin): @@ -198,7 +198,7 @@ class SingleStageFile(FileMixin):
if self.verify: if self.verify:
check_dvcfile_path(self.repo, self.path) check_dvcfile_path(self.repo, self.path)
logger.debug(f"Saving information to '{relpath(self.path)}'.") logger.debug("Saving information to '%s'.", relpath(self.path))
- dump_yaml(self.path, serialize.to_single_stage_file(stage)) - dump_yaml(self.path, serialize.to_single_stage_file(stage))
+ dump_json(self.path, serialize.to_single_stage_file(stage)) + dump_json(self.path, serialize.to_single_stage_file(stage))
self.repo.scm_context.track_file(self.relpath) self.repo.scm_context.track_file(self.relpath)

0
patches/dvc/yaml-to-json.patch
View file