2024-01-03 22:02:20 +02:00
# include "signature/local-keys.hh"
2023-11-01 18:09:28 +02:00
# include "source-accessor.hh"
2006-11-30 19:43:04 +02:00
# include "globals.hh"
2022-03-18 17:35:45 +02:00
# include "derived-path.hh"
# include "realisation.hh"
2022-03-18 00:29:15 +02:00
# include "derivations.hh"
2016-03-24 12:41:00 +02:00
# include "store-api.hh"
2006-11-30 20:35:36 +02:00
# include "util.hh"
2016-04-20 15:12:38 +03:00
# include "nar-info-disk-cache.hh"
2016-07-19 01:50:27 +03:00
# include "thread-pool.hh"
2020-10-07 16:52:20 +03:00
# include "references.hh"
2020-07-10 14:21:37 +03:00
# include "archive.hh"
2020-09-21 19:40:11 +03:00
# include "callback.hh"
2023-09-04 16:51:23 +03:00
# include "git.hh"
# include "posix-source-accessor.hh"
2022-03-09 01:09:26 +02:00
// FIXME this should not be here, see TODO below on
// `addMultipleToStore`.
# include "worker-protocol.hh"
2023-10-25 07:43:36 +03:00
# include "signals.hh"
# include "users.hh"
2019-07-10 20:46:15 +03:00
2024-06-10 09:00:39 +03:00
# include <filesystem>
2022-11-16 17:49:49 +02:00
# include <nlohmann/json.hpp>
2020-12-05 16:33:16 +02:00
2022-11-16 17:49:49 +02:00
using json = nlohmann : : json ;
2006-11-30 19:43:04 +02:00
namespace nix {
2023-01-07 01:06:03 +02:00
bool StoreDirConfig : : isInStore ( PathView path ) const
2006-11-30 19:43:04 +02:00
{
2016-06-01 15:49:12 +03:00
return isInDir ( path , storeDir ) ;
2006-11-30 19:43:04 +02:00
}
2023-01-07 01:06:03 +02:00
std : : pair < StorePath , Path > StoreDirConfig : : toStorePath ( PathView path ) const
2006-11-30 19:43:04 +02:00
{
if ( ! isInStore ( path ) )
2020-04-22 02:07:07 +03:00
throw Error ( " path '%1%' is not in the Nix store " , path ) ;
2022-12-07 13:58:58 +02:00
auto slash = path . find ( ' / ' , storeDir . size ( ) + 1 ) ;
2006-11-30 19:43:04 +02:00
if ( slash = = Path : : npos )
2020-07-13 17:19:37 +03:00
return { parseStorePath ( path ) , " " } ;
2006-11-30 19:43:04 +02:00
else
2022-12-07 13:58:58 +02:00
return { parseStorePath ( path . substr ( 0 , slash ) ) , ( Path ) path . substr ( slash ) } ;
2006-11-30 19:43:04 +02:00
}
2019-12-16 20:11:47 +02:00
Path Store : : followLinksToStore ( std : : string_view _path ) const
2007-11-29 18:18:24 +02:00
{
2019-12-16 20:11:47 +02:00
Path path = absPath ( std : : string ( _path ) ) ;
2007-11-29 18:18:24 +02:00
while ( ! isInStore ( path ) ) {
2024-05-08 17:29:37 +03:00
if ( ! std : : filesystem : : is_symlink ( path ) ) break ;
2022-02-25 17:00:00 +02:00
auto target = readLink ( path ) ;
2007-11-29 18:18:24 +02:00
path = absPath ( target , dirOf ( path ) ) ;
}
if ( ! isInStore ( path ) )
2020-07-13 17:19:37 +03:00
throw BadStorePath ( " path '%1%' is not in the Nix store " , path ) ;
2007-11-29 18:18:24 +02:00
return path ;
}
2019-12-16 20:11:47 +02:00
StorePath Store : : followLinksToStorePath ( std : : string_view path ) const
2007-11-29 18:18:24 +02:00
{
2020-07-13 17:19:37 +03:00
return toStorePath ( followLinksToStore ( path ) ) . first ;
2016-02-15 13:49:01 +02:00
}
2023-11-05 02:10:55 +02:00
/*
The exact specification of store paths is in ` protocols / store - path . md `
in the Nix manual . These few functions implement that specification .
2024-02-12 19:01:54 +02:00
If changes to these functions go beyond mere implementation changes i . e .
2023-11-05 02:10:55 +02:00
also update the user - visible behavior , please update the specification
to match .
2008-12-03 17:06:30 +02:00
*/
2023-01-07 01:06:03 +02:00
StorePath StoreDirConfig : : makeStorePath ( std : : string_view type ,
2020-08-07 22:09:26 +03:00
std : : string_view hash , std : : string_view name ) const
2006-11-30 19:43:04 +02:00
{
/* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */
2022-02-25 17:00:00 +02:00
auto s = std : : string ( type ) + " : " + std : : string ( hash )
+ " : " + storeDir + " : " + std : : string ( name ) ;
2023-11-28 15:20:27 +02:00
auto h = compressHash ( hashString ( HashAlgorithm : : SHA256 , s ) , 20 ) ;
2020-06-16 15:16:39 +03:00
return StorePath ( h , name ) ;
2006-11-30 19:43:04 +02:00
}
2023-01-07 01:06:03 +02:00
StorePath StoreDirConfig : : makeStorePath ( std : : string_view type ,
2020-08-07 22:09:26 +03:00
const Hash & hash , std : : string_view name ) const
{
2023-10-13 04:48:15 +03:00
return makeStorePath ( type , hash . to_string ( HashFormat : : Base16 , true ) , name ) ;
2020-08-07 22:09:26 +03:00
}
2023-01-07 01:06:03 +02:00
StorePath StoreDirConfig : : makeOutputPath ( std : : string_view id ,
2019-12-05 20:11:09 +02:00
const Hash & hash , std : : string_view name ) const
2011-07-20 21:10:47 +03:00
{
2020-08-07 22:09:26 +03:00
return makeStorePath ( " output: " + std : : string { id } , hash , outputPathName ( name , id ) ) ;
2011-07-20 21:10:47 +03:00
}
2020-10-07 16:52:20 +03:00
/* Stuff the references (if any) into the type. This is a bit
2023-02-28 18:49:13 +02:00
hacky , but we can ' t put them in , say , < s2 > ( per the grammar above )
since that would be ambiguous . */
2019-12-05 20:11:09 +02:00
static std : : string makeType (
2023-01-07 01:06:03 +02:00
const StoreDirConfig & store ,
2022-02-25 17:00:00 +02:00
std : : string & & type ,
2023-01-06 22:36:05 +02:00
const StoreReferences & references )
2018-03-30 01:56:13 +03:00
{
2023-01-06 22:36:05 +02:00
for ( auto & i : references . others ) {
2018-03-30 01:56:13 +03:00
type + = " : " ;
2019-12-05 20:11:09 +02:00
type + = store . printStorePath ( i ) ;
2018-03-30 01:56:13 +03:00
}
2023-01-06 22:36:05 +02:00
if ( references . self ) type + = " :self " ;
2019-11-26 22:07:44 +02:00
return std : : move ( type ) ;
2018-03-30 01:56:13 +03:00
}
2023-01-07 01:06:03 +02:00
StorePath StoreDirConfig : : makeFixedOutputPath ( std : : string_view name , const FixedOutputInfo & info ) const
2006-11-30 19:43:04 +02:00
{
2023-09-04 16:51:23 +03:00
if ( info . method = = FileIngestionMethod : : Git & & info . hash . algo ! = HashAlgorithm : : SHA1 )
throw Error ( " Git file ingestion must use SHA-1 hash " ) ;
2024-05-17 00:46:43 +03:00
if ( info . hash . algo = = HashAlgorithm : : SHA256 & & info . method = = FileIngestionMethod : : NixArchive ) {
2023-07-06 01:53:44 +03:00
return makeStorePath ( makeType ( * this , " source " , info . references ) , info . hash , name ) ;
2018-03-30 01:56:13 +03:00
} else {
2023-12-24 13:44:56 +02:00
if ( ! info . references . empty ( ) ) {
throw Error ( " fixed output derivation '%s' is not allowed to refer to other store paths. \n You may need to use the 'unsafeDiscardReferences' derivation attribute, see the manual for more details. " ,
name ) ;
}
2024-04-12 18:43:35 +03:00
// make a unique digest based on the parameters for creating this store object
auto payload = " fixed:out: "
2023-07-06 01:53:44 +03:00
+ makeFileIngestionPrefix ( info . method )
2024-04-12 18:43:35 +03:00
+ info . hash . to_string ( HashFormat : : Base16 , true ) + " : " ;
auto digest = hashString ( HashAlgorithm : : SHA256 , payload ) ;
return makeStorePath ( " output:out " , digest , name ) ;
2018-03-30 01:56:13 +03:00
}
2006-11-30 19:43:04 +02:00
}
2020-10-07 16:52:20 +03:00
2023-01-07 01:06:03 +02:00
StorePath StoreDirConfig : : makeFixedOutputPathFromCA ( std : : string_view name , const ContentAddressWithReferences & ca ) const
2020-06-13 00:36:35 +03:00
{
2020-06-22 20:08:11 +03:00
// New template
return std : : visit ( overloaded {
2021-10-01 20:12:54 +03:00
[ & ] ( const TextInfo & ti ) {
2023-11-09 04:11:48 +02:00
assert ( ti . hash . algo = = HashAlgorithm : : SHA256 ) ;
return makeStorePath (
makeType ( * this , " text " , StoreReferences {
. others = ti . references ,
. self = false ,
} ) ,
ti . hash ,
name ) ;
2020-06-22 20:08:11 +03:00
} ,
2021-10-01 20:12:54 +03:00
[ & ] ( const FixedOutputInfo & foi ) {
2023-01-23 19:58:11 +02:00
return makeFixedOutputPath ( name , foi ) ;
2020-06-22 20:08:11 +03:00
}
2023-03-31 00:12:49 +03:00
} , ca . raw ) ;
2016-08-03 14:17:11 +03:00
}
2023-11-04 22:25:41 +02:00
std : : pair < StorePath , Hash > StoreDirConfig : : computeStorePath (
std : : string_view name ,
2024-05-06 20:05:42 +03:00
const SourcePath & path ,
2023-11-04 22:25:41 +02:00
ContentAddressMethod method ,
HashAlgorithm hashAlgo ,
const StorePathSet & references ,
PathFilter & filter ) const
2006-12-01 20:00:01 +02:00
{
2024-05-07 20:25:44 +03:00
auto [ h , size ] = hashPath ( path , method . getFileIngestionMethod ( ) , hashAlgo , filter ) ;
2024-06-03 16:49:15 +03:00
if ( size & & * size > = settings . warnLargePathThreshold )
2024-05-10 17:58:19 +03:00
warn ( " hashed large path '%s' (%s) " , path , renderSize ( * size ) ) ;
2023-11-04 22:25:41 +02:00
return {
makeFixedOutputPathFromCA (
name ,
ContentAddressWithReferences : : fromParts (
method ,
h ,
{
. others = references ,
. self = false ,
} ) ) ,
h ,
2020-10-07 16:52:20 +03:00
} ;
2006-12-01 20:00:01 +02:00
}
2022-02-25 17:00:00 +02:00
StorePath Store : : addToStore (
std : : string_view name ,
2024-05-06 20:05:42 +03:00
const SourcePath & path ,
2023-11-04 22:25:41 +02:00
ContentAddressMethod method ,
HashAlgorithm hashAlgo ,
const StorePathSet & references ,
PathFilter & filter ,
RepairFlag repair )
2006-12-01 20:00:01 +02:00
{
2024-01-19 06:57:26 +02:00
FileSerialisationMethod fsm ;
switch ( method . getFileIngestionMethod ( ) ) {
case FileIngestionMethod : : Flat :
fsm = FileSerialisationMethod : : Flat ;
break ;
2024-05-17 00:46:43 +03:00
case FileIngestionMethod : : NixArchive :
fsm = FileSerialisationMethod : : NixArchive ;
2024-01-19 06:57:26 +02:00
break ;
case FileIngestionMethod : : Git :
// Use NAR; Git is not a serialization method
2024-05-17 00:46:43 +03:00
fsm = FileSerialisationMethod : : NixArchive ;
2024-01-19 06:57:26 +02:00
break ;
}
2020-08-03 07:13:45 +03:00
auto source = sinkToSource ( [ & ] ( Sink & sink ) {
2024-05-06 20:05:42 +03:00
dumpPath ( path , sink , fsm , filter ) ;
2020-08-03 07:13:45 +03:00
} ) ;
2024-05-07 20:25:44 +03:00
LengthSource lengthSource ( * source ) ;
auto storePath = addToStoreFromDump ( lengthSource , name , fsm , method , hashAlgo , references , repair ) ;
2024-06-03 16:49:15 +03:00
if ( lengthSource . total > = settings . warnLargePathThreshold )
2024-05-10 17:58:19 +03:00
warn ( " copied large path '%s' to the store (%s) " , path , renderSize ( lengthSource . total ) ) ;
2024-05-07 20:25:44 +03:00
return storePath ;
2020-08-03 07:13:45 +03:00
}
2022-06-03 18:01:16 +03:00
void Store : : addMultipleToStore (
2022-06-08 15:03:46 +03:00
PathsSource & pathsToCopy ,
2022-06-03 18:01:16 +03:00
Activity & act ,
RepairFlag repair ,
CheckSigsFlag checkSigs )
{
std : : atomic < size_t > nrDone { 0 } ;
std : : atomic < size_t > nrFailed { 0 } ;
std : : atomic < uint64_t > bytesExpected { 0 } ;
std : : atomic < uint64_t > nrRunning { 0 } ;
using PathWithInfo = std : : pair < ValidPathInfo , std : : unique_ptr < Source > > ;
2022-08-23 15:14:47 +03:00
std : : map < StorePath , PathWithInfo * > infosMap ;
2022-06-03 18:01:16 +03:00
StorePathSet storePathsToAdd ;
for ( auto & thingToAdd : pathsToCopy ) {
infosMap . insert_or_assign ( thingToAdd . first . path , & thingToAdd ) ;
storePathsToAdd . insert ( thingToAdd . first . path ) ;
}
auto showProgress = [ & ] ( ) {
act . progress ( nrDone , pathsToCopy . size ( ) , nrRunning , nrFailed ) ;
} ;
ThreadPool pool ;
processGraph < StorePath > ( pool ,
storePathsToAdd ,
[ & ] ( const StorePath & path ) {
2022-08-23 15:14:47 +03:00
auto & [ info , _ ] = * infosMap . at ( path ) ;
2022-06-03 18:01:16 +03:00
if ( isValidPath ( info . path ) ) {
nrDone + + ;
showProgress ( ) ;
return StorePathSet ( ) ;
}
bytesExpected + = info . narSize ;
act . setExpected ( actCopyPath , bytesExpected ) ;
return info . references ;
} ,
[ & ] ( const StorePath & path ) {
checkInterrupt ( ) ;
2022-08-24 15:11:03 +03:00
auto & [ info_ , source_ ] = * infosMap . at ( path ) ;
2022-07-19 20:46:00 +03:00
auto info = info_ ;
info . ultimate = false ;
2022-06-03 18:01:16 +03:00
2022-08-23 15:14:47 +03:00
/* Make sure that the Source object is destroyed when
we ' re done . In particular , a SinkToSource object must
be destroyed to ensure that the destructors on its
stack frame are run ; this includes
LegacySSHStore : : narFromPath ( ) ' s connection lock . */
2022-08-24 15:49:58 +03:00
auto source = std : : move ( source_ ) ;
2022-08-23 15:14:47 +03:00
2022-06-03 18:01:16 +03:00
if ( ! isValidPath ( info . path ) ) {
MaintainCount < decltype ( nrRunning ) > mc ( nrRunning ) ;
showProgress ( ) ;
try {
addToStore ( info , * source , repair , checkSigs ) ;
2022-08-23 15:14:47 +03:00
} catch ( Error & e ) {
2022-06-03 18:01:16 +03:00
nrFailed + + ;
if ( ! settings . keepGoing )
throw e ;
printMsg ( lvlError , " could not copy %s: %s " , printStorePath ( path ) , e . what ( ) ) ;
showProgress ( ) ;
return ;
}
}
nrDone + + ;
showProgress ( ) ;
} ) ;
}
2020-08-03 07:13:45 +03:00
2021-07-26 14:31:09 +03:00
void Store : : addMultipleToStore (
Source & source ,
RepairFlag repair ,
CheckSigsFlag checkSigs )
{
auto expected = readNum < uint64_t > ( source ) ;
for ( uint64_t i = 0 ; i < expected ; + + i ) {
2022-03-09 01:09:26 +02:00
// FIXME we should not be using the worker protocol here, let
// alone the worker protocol with a hard-coded version!
auto info = WorkerProto : : Serialise < ValidPathInfo > : : read ( * this ,
WorkerProto : : ReadConn {
. from = source ,
. version = 16 ,
} ) ;
2021-07-26 14:31:09 +03:00
info . ultimate = false ;
addToStore ( info , source , repair , checkSigs ) ;
}
}
2020-07-21 03:18:12 +03:00
/*
The aim of this function is to compute in one pass the correct ValidPathInfo for
the files that we are trying to add to the store . To accomplish that in one
pass , given the different kind of inputs that we can take ( normal nar archives ,
nar archives with non SHA - 256 hashes , and flat files ) , we set up a net of sinks
and aliases . Also , since the dataflow is obfuscated by this , we include here a
graphviz diagram :
digraph graphname {
node [ shape = box ]
fileSource - > narSink
narSink [ style = dashed ]
narSink - > unsualHashTee [ style = dashed , label = " Recursive && !SHA-256 " ]
narSink - > narHashSink [ style = dashed , label = " else " ]
unsualHashTee - > narHashSink
unsualHashTee - > caHashSink
fileSource - > parseSink
parseSink [ style = dashed ]
parseSink - > fileSink [ style = dashed , label = " Flat " ]
parseSink - > blank [ style = dashed , label = " Recursive " ]
fileSink - > caHashSink
}
*/
2023-11-04 22:25:41 +02:00
ValidPathInfo Store : : addToStoreSlow (
std : : string_view name ,
2024-05-06 20:05:42 +03:00
const SourcePath & srcPath ,
2023-11-04 22:25:41 +02:00
ContentAddressMethod method , HashAlgorithm hashAlgo ,
const StorePathSet & references ,
std : : optional < Hash > expectedCAHash )
2020-07-10 14:21:37 +03:00
{
2023-11-28 15:20:27 +02:00
HashSink narHashSink { HashAlgorithm : : SHA256 } ;
2020-07-16 08:09:41 +03:00
HashSink caHashSink { hashAlgo } ;
2020-07-10 14:21:37 +03:00
2020-07-21 03:18:12 +03:00
/* Note that fileSink and unusualHashTee must be mutually exclusive, since
they both write to caHashSink . Note that that requisite is currently true
because the former is only used in the flat case . */
2023-11-01 02:39:39 +02:00
RegularFileSink fileSink { caHashSink } ;
2020-07-21 03:18:12 +03:00
TeeSink unusualHashTee { narHashSink , caHashSink } ;
2020-07-10 16:56:24 +03:00
2024-05-17 02:08:28 +03:00
auto & narSink = method = = ContentAddressMethod : : Raw : : NixArchive & & hashAlgo ! = HashAlgorithm : : SHA256
2020-07-21 03:18:12 +03:00
? static_cast < Sink & > ( unusualHashTee )
2020-07-16 08:09:41 +03:00
: narHashSink ;
2020-07-21 03:18:12 +03:00
/* Functionally, this means that fileSource will yield the content of
srcPath . The fact that we use scratchpadSink as a temporary buffer here
is an implementation detail . */
auto fileSource = sinkToSource ( [ & ] ( Sink & scratchpadSink ) {
2024-05-06 20:05:42 +03:00
srcPath . dumpPath ( scratchpadSink ) ;
2020-07-16 08:09:41 +03:00
} ) ;
2020-07-10 14:21:37 +03:00
2020-07-21 03:18:12 +03:00
/* tapped provides the same data as fileSource, but we also write all the
information to narSink . */
TeeSource tapped { * fileSource , narSink } ;
2020-07-10 16:56:24 +03:00
2024-01-23 00:59:34 +02:00
NullFileSystemObjectSink blank ;
2023-11-04 22:25:41 +02:00
auto & parseSink = method . getFileIngestionMethod ( ) = = FileIngestionMethod : : Flat
2024-01-23 00:59:34 +02:00
? ( FileSystemObjectSink & ) fileSink
2023-09-04 16:51:23 +03:00
: ( FileSystemObjectSink & ) blank ; // for recursive or git we do recursive
2020-07-16 08:09:41 +03:00
2020-07-21 03:18:12 +03:00
/* The information that flows from tapped (besides being replicated in
narSink ) , is now put in parseSink . */
parseDump ( parseSink , tapped ) ;
2020-07-16 08:09:41 +03:00
2020-07-21 03:18:12 +03:00
/* We extract the result of the computation from the sink by calling
finish . */
2020-07-16 08:09:41 +03:00
auto [ narHash , narSize ] = narHashSink . finish ( ) ;
2024-05-17 02:08:28 +03:00
auto hash = method = = ContentAddressMethod : : Raw : : NixArchive & & hashAlgo = = HashAlgorithm : : SHA256
2020-07-16 08:09:41 +03:00
? narHash
2024-05-17 02:08:28 +03:00
: method = = ContentAddressMethod : : Raw : : Git
2024-05-06 20:05:42 +03:00
? git : : dumpHash ( hashAlgo , srcPath ) . hash
2020-07-16 08:09:41 +03:00
: caHashSink . finish ( ) . first ;
2020-07-10 14:21:37 +03:00
if ( expectedCAHash & & expectedCAHash ! = hash )
throw Error ( " hash mismatch for '%s' " , srcPath ) ;
2020-08-06 21:31:48 +03:00
ValidPathInfo info {
2020-10-07 16:52:20 +03:00
* this ,
2023-01-23 19:58:11 +02:00
name ,
2023-11-04 22:25:41 +02:00
ContentAddressWithReferences : : fromParts (
method ,
hash ,
{
. others = references ,
. self = false ,
} ) ,
2020-08-06 21:31:48 +03:00
narHash ,
} ;
2020-07-10 14:21:37 +03:00
info . narSize = narSize ;
if ( ! isValidPath ( info . path ) ) {
2020-07-21 03:18:12 +03:00
auto source = sinkToSource ( [ & ] ( Sink & scratchpadSink ) {
2024-05-06 20:05:42 +03:00
srcPath . dumpPath ( scratchpadSink ) ;
2020-07-10 14:21:37 +03:00
} ) ;
addToStore ( info , * source ) ;
}
return info ;
}
2021-06-11 09:37:59 +03:00
StringSet StoreConfig : : getDefaultSystemFeatures ( )
{
auto res = settings . systemFeatures . get ( ) ;
2021-11-19 04:28:20 +02:00
2023-03-17 16:33:48 +02:00
if ( experimentalFeatureSettings . isEnabled ( Xp : : CaDerivations ) )
2021-06-11 09:37:59 +03:00
res . insert ( " ca-derivations " ) ;
2021-11-19 04:28:20 +02:00
2023-03-17 16:33:48 +02:00
if ( experimentalFeatureSettings . isEnabled ( Xp : : RecursiveNix ) )
2021-11-19 04:28:20 +02:00
res . insert ( " recursive-nix " ) ;
2021-06-11 09:37:59 +03:00
return res ;
}
2020-07-10 14:21:37 +03:00
2016-06-01 15:49:12 +03:00
Store : : Store ( const Params & params )
2020-09-10 11:55:51 +03:00
: StoreConfig ( params )
2017-04-13 16:55:38 +03:00
, state ( { ( size_t ) pathInfoCacheSize } )
2016-06-01 15:49:12 +03:00
{
2022-12-19 15:06:07 +02:00
assertLibStoreInitialized ( ) ;
2016-06-01 15:49:12 +03:00
}
2016-04-20 15:12:38 +03:00
std : : string Store : : getUri ( )
{
return " " ;
}
2020-03-11 21:04:47 +02:00
bool Store : : PathInfoCacheValue : : isKnownNow ( )
{
std : : chrono : : duration ttl = didExist ( )
? std : : chrono : : seconds ( settings . ttlPositiveNarInfoCache )
: std : : chrono : : seconds ( settings . ttlNegativeNarInfoCache ) ;
return std : : chrono : : steady_clock : : now ( ) < time_point + ttl ;
}
2016-04-20 15:12:38 +03:00
2023-07-19 21:52:35 +03:00
std : : map < std : : string , std : : optional < StorePath > > Store : : queryStaticPartialDerivationOutputMap ( const StorePath & path )
2020-12-17 12:35:24 +02:00
{
std : : map < std : : string , std : : optional < StorePath > > outputs ;
auto drv = readInvalidDerivation ( path ) ;
2023-07-19 21:52:35 +03:00
for ( auto & [ outputName , output ] : drv . outputsAndOptPaths ( * this ) ) {
2020-12-17 12:35:24 +02:00
outputs . emplace ( outputName , output . second ) ;
}
return outputs ;
}
2023-07-19 21:52:35 +03:00
std : : map < std : : string , std : : optional < StorePath > > Store : : queryPartialDerivationOutputMap (
const StorePath & path ,
Store * evalStore_ )
{
auto & evalStore = evalStore_ ? * evalStore_ : * this ;
auto outputs = evalStore . queryStaticPartialDerivationOutputMap ( path ) ;
if ( ! experimentalFeatureSettings . isEnabled ( Xp : : CaDerivations ) )
return outputs ;
auto drv = evalStore . readInvalidDerivation ( path ) ;
auto drvHashes = staticOutputHashes ( * this , drv ) ;
for ( auto & [ outputName , hash ] : drvHashes ) {
auto realisation = queryRealisation ( DrvOutput { hash , outputName } ) ;
if ( realisation ) {
outputs . insert_or_assign ( outputName , realisation - > outPath ) ;
} else {
// queryStaticPartialDerivationOutputMap is not guaranteed
// to return std::nullopt for outputs which are not
// statically known.
outputs . insert ( { outputName , std : : nullopt } ) ;
}
}
return outputs ;
}
2023-12-11 01:51:23 +02:00
OutputPathMap Store : : queryDerivationOutputMap ( const StorePath & path , Store * evalStore ) {
auto resp = queryPartialDerivationOutputMap ( path , evalStore ) ;
2020-08-05 01:28:10 +03:00
OutputPathMap result ;
for ( auto & [ outName , optOutPath ] : resp ) {
if ( ! optOutPath )
2023-07-19 21:52:35 +03:00
throw MissingRealisation ( printStorePath ( path ) , outName ) ;
2020-08-05 01:28:10 +03:00
result . insert_or_assign ( outName , * optOutPath ) ;
}
return result ;
2020-07-25 00:02:51 +03:00
}
2020-06-10 12:20:52 +03:00
StorePathSet Store : : queryDerivationOutputs ( const StorePath & path )
{
auto outputMap = this - > queryDerivationOutputMap ( path ) ;
StorePathSet outputPaths ;
for ( auto & i : outputMap ) {
outputPaths . emplace ( std : : move ( i . second ) ) ;
}
return outputPaths ;
}
2023-04-03 03:28:10 +03:00
void Store : : querySubstitutablePathInfos ( const StorePathCAMap & paths , SubstitutablePathInfos & infos )
{
if ( ! settings . useSubstitutes ) return ;
for ( auto & sub : getDefaultSubstituters ( ) ) {
for ( auto & path : paths ) {
if ( infos . count ( path . first ) )
// Choose first succeeding substituter.
continue ;
auto subPath ( path . first ) ;
// Recompute store path so that we can use a different store root.
if ( path . second ) {
2023-04-08 03:39:04 +03:00
subPath = makeFixedOutputPathFromCA (
path . first . name ( ) ,
ContentAddressWithReferences : : withoutRefs ( * path . second ) ) ;
2023-04-03 03:28:10 +03:00
if ( sub - > storeDir = = storeDir )
assert ( subPath = = path . first ) ;
if ( subPath ! = path . first )
debug ( " replaced path '%s' with '%s' for substituter '%s' " , printStorePath ( path . first ) , sub - > printStorePath ( subPath ) , sub - > getUri ( ) ) ;
} else if ( sub - > storeDir ! = storeDir ) continue ;
debug ( " checking substituter '%s' for path '%s' " , sub - > getUri ( ) , sub - > printStorePath ( subPath ) ) ;
try {
auto info = sub - > queryPathInfo ( subPath ) ;
if ( sub - > storeDir ! = storeDir & & ! ( info - > isContentAddressed ( * sub ) & & info - > references . empty ( ) ) )
continue ;
auto narInfo = std : : dynamic_pointer_cast < const NarInfo > (
std : : shared_ptr < const ValidPathInfo > ( info ) ) ;
infos . insert_or_assign ( path . first , SubstitutablePathInfo {
2023-04-08 03:39:04 +03:00
. deriver = info - > deriver ,
. references = info - > references ,
. downloadSize = narInfo ? narInfo - > fileSize : 0 ,
. narSize = info - > narSize ,
} ) ;
2023-04-03 03:28:10 +03:00
} catch ( InvalidPath & ) {
} catch ( SubstituterDisabled & ) {
} catch ( Error & e ) {
if ( settings . tryFallback )
logError ( e . info ( ) ) ;
else
throw ;
}
}
}
}
2019-12-05 20:11:09 +02:00
bool Store : : isValidPath ( const StorePath & storePath )
2016-02-15 15:48:38 +02:00
{
2016-04-19 19:50:15 +03:00
{
auto state_ ( state . lock ( ) ) ;
2021-10-14 14:28:22 +03:00
auto res = state_ - > pathInfoCache . get ( std : : string ( storePath . to_string ( ) ) ) ;
2020-03-11 21:04:47 +02:00
if ( res & & res - > isKnownNow ( ) ) {
2016-04-19 19:50:15 +03:00
stats . narInfoReadAverted + + ;
2020-03-11 21:04:47 +02:00
return res - > didExist ( ) ;
2016-04-19 19:50:15 +03:00
}
}
2016-04-20 15:12:38 +03:00
if ( diskCache ) {
2021-10-14 14:28:22 +03:00
auto res = diskCache - > lookupNarInfo ( getUri ( ) , std : : string ( storePath . hashPart ( ) ) ) ;
2016-04-20 15:12:38 +03:00
if ( res . first ! = NarInfoDiskCache : : oUnknown ) {
2016-04-21 18:53:47 +03:00
stats . narInfoReadAverted + + ;
2016-04-20 15:12:38 +03:00
auto state_ ( state . lock ( ) ) ;
2021-10-14 14:28:22 +03:00
state_ - > pathInfoCache . upsert ( std : : string ( storePath . to_string ( ) ) ,
2020-03-11 21:04:47 +02:00
res . first = = NarInfoDiskCache : : oInvalid ? PathInfoCacheValue { } : PathInfoCacheValue { . value = res . second } ) ;
2016-04-20 15:12:38 +03:00
return res . first = = NarInfoDiskCache : : oValid ;
}
}
2016-06-20 18:39:05 +03:00
bool valid = isValidPathUncached ( storePath ) ;
2016-04-20 15:12:38 +03:00
2016-06-20 18:39:05 +03:00
if ( diskCache & & ! valid )
// FIXME: handle valid = true case.
2021-10-14 14:28:22 +03:00
diskCache - > upsertNarInfo ( getUri ( ) , std : : string ( storePath . hashPart ( ) ) , 0 ) ;
2016-06-20 18:39:05 +03:00
return valid ;
2016-04-19 19:50:15 +03:00
}
2017-02-07 20:22:48 +02:00
/* Default implementation for stores that only implement
queryPathInfoUncached ( ) . */
2019-12-05 20:11:09 +02:00
bool Store : : isValidPathUncached ( const StorePath & path )
2017-02-07 20:22:48 +02:00
{
try {
queryPathInfo ( path ) ;
return true ;
} catch ( InvalidPath & ) {
return false ;
}
}
2019-12-05 20:11:09 +02:00
ref < const ValidPathInfo > Store : : queryPathInfo ( const StorePath & storePath )
2016-09-16 19:54:14 +03:00
{
2018-09-25 19:54:16 +03:00
std : : promise < ref < const ValidPathInfo > > promise ;
2016-09-16 19:54:14 +03:00
queryPathInfo ( storePath ,
2018-09-25 19:54:16 +03:00
{ [ & ] ( std : : future < ref < const ValidPathInfo > > result ) {
2018-03-27 23:16:01 +03:00
try {
promise . set_value ( result . get ( ) ) ;
} catch ( . . . ) {
promise . set_exception ( std : : current_exception ( ) ) ;
}
} } ) ;
2016-09-16 19:54:14 +03:00
return promise . get_future ( ) . get ( ) ;
}
2020-07-13 21:17:00 +03:00
static bool goodStorePath ( const StorePath & expected , const StorePath & actual )
{
return
expected . hashPart ( ) = = actual . hashPart ( )
& & ( expected . name ( ) = = Store : : MissingName | | expected . name ( ) = = actual . name ( ) ) ;
}
2024-01-19 18:00:39 +02:00
std : : optional < std : : shared_ptr < const ValidPathInfo > > Store : : queryPathInfoFromClientCache ( const StorePath & storePath )
2016-04-19 19:50:15 +03:00
{
2021-10-14 14:28:22 +03:00
auto hashPart = std : : string ( storePath . hashPart ( ) ) ;
2016-04-21 18:53:47 +03:00
2024-01-17 17:54:45 +02:00
{
auto res = state . lock ( ) - > pathInfoCache . get ( std : : string ( storePath . to_string ( ) ) ) ;
if ( res & & res - > isKnownNow ( ) ) {
stats . narInfoReadAverted + + ;
2024-01-19 18:00:39 +02:00
if ( res - > didExist ( ) )
return std : : make_optional ( res - > value ) ;
else
return std : : make_optional ( nullptr ) ;
2016-04-19 19:50:15 +03:00
}
2024-01-17 17:54:45 +02:00
}
2016-04-19 19:50:15 +03:00
2024-01-17 17:54:45 +02:00
if ( diskCache ) {
auto res = diskCache - > lookupNarInfo ( getUri ( ) , hashPart ) ;
if ( res . first ! = NarInfoDiskCache : : oUnknown ) {
stats . narInfoReadAverted + + ;
{
auto state_ ( state . lock ( ) ) ;
state_ - > pathInfoCache . upsert ( std : : string ( storePath . to_string ( ) ) ,
res . first = = NarInfoDiskCache : : oInvalid ? PathInfoCacheValue { } : PathInfoCacheValue { . value = res . second } ) ;
if ( res . first = = NarInfoDiskCache : : oInvalid | |
! goodStorePath ( storePath , res . second - > path ) )
2024-01-19 18:00:39 +02:00
return std : : make_optional ( nullptr ) ;
2016-09-16 19:54:14 +03:00
}
2024-01-19 18:00:39 +02:00
assert ( res . second ) ;
return std : : make_optional ( res . second ) ;
2016-04-20 15:12:38 +03:00
}
2024-01-17 17:54:45 +02:00
}
2016-09-16 19:54:14 +03:00
2024-01-17 18:54:03 +02:00
return std : : nullopt ;
2024-01-17 17:54:45 +02:00
}
2016-09-16 19:54:14 +03:00
2024-01-17 17:54:45 +02:00
void Store : : queryPathInfo ( const StorePath & storePath ,
Callback < ref < const ValidPathInfo > > callback ) noexcept
{
auto hashPart = std : : string ( storePath . hashPart ( ) ) ;
2016-09-16 19:54:14 +03:00
2024-01-17 17:54:45 +02:00
try {
2024-01-17 18:54:03 +02:00
auto r = queryPathInfoFromClientCache ( storePath ) ;
if ( r . has_value ( ) ) {
2024-01-19 18:00:39 +02:00
std : : shared_ptr < const ValidPathInfo > & info = * r ;
if ( info )
return callback ( ref ( info ) ) ;
else
throw InvalidPath ( " path '%s' is not valid " , printStorePath ( storePath ) ) ;
2024-01-17 18:54:03 +02:00
}
2018-03-27 23:16:01 +03:00
} catch ( . . . ) { return callback . rethrow ( ) ; }
2016-04-20 15:12:38 +03:00
2019-09-03 13:51:35 +03:00
auto callbackPtr = std : : make_shared < decltype ( callback ) > ( std : : move ( callback ) ) ;
2016-09-16 19:54:14 +03:00
queryPathInfoUncached ( storePath ,
2021-10-14 14:28:22 +03:00
{ [ this , storePath , hashPart , callbackPtr ] ( std : : future < std : : shared_ptr < const ValidPathInfo > > fut ) {
2016-04-19 19:50:15 +03:00
2018-03-27 23:16:01 +03:00
try {
auto info = fut . get ( ) ;
2016-04-20 15:12:38 +03:00
2018-03-27 23:16:01 +03:00
if ( diskCache )
diskCache - > upsertNarInfo ( getUri ( ) , hashPart , info ) ;
2016-04-19 19:50:15 +03:00
2018-03-27 23:16:01 +03:00
{
auto state_ ( state . lock ( ) ) ;
2021-10-14 14:28:22 +03:00
state_ - > pathInfoCache . upsert ( std : : string ( storePath . to_string ( ) ) , PathInfoCacheValue { . value = info } ) ;
2018-03-27 23:16:01 +03:00
}
2016-09-16 19:54:14 +03:00
2020-07-14 12:55:54 +03:00
if ( ! info | | ! goodStorePath ( storePath , info - > path ) ) {
2018-03-27 23:16:01 +03:00
stats . narInfoMissing + + ;
2021-10-14 14:28:22 +03:00
throw InvalidPath ( " path '%s' is not valid " , printStorePath ( storePath ) ) ;
2018-03-27 23:16:01 +03:00
}
2016-04-19 19:50:15 +03:00
2018-09-25 19:54:16 +03:00
( * callbackPtr ) ( ref < const ValidPathInfo > ( info ) ) ;
2019-09-03 13:51:35 +03:00
} catch ( . . . ) { callbackPtr - > rethrow ( ) ; }
2018-03-27 23:16:01 +03:00
} } ) ;
2016-02-15 15:48:38 +02:00
}
2021-10-27 12:36:51 +03:00
void Store : : queryRealisation ( const DrvOutput & id ,
Callback < std : : shared_ptr < const Realisation > > callback ) noexcept
{
try {
if ( diskCache ) {
auto [ cacheOutcome , maybeCachedRealisation ]
= diskCache - > lookupRealisation ( getUri ( ) , id ) ;
switch ( cacheOutcome ) {
case NarInfoDiskCache : : oValid :
debug ( " Returning a cached realisation for %s " , id . to_string ( ) ) ;
callback ( maybeCachedRealisation ) ;
return ;
case NarInfoDiskCache : : oInvalid :
debug (
" Returning a cached missing realisation for %s " ,
id . to_string ( ) ) ;
callback ( nullptr ) ;
return ;
case NarInfoDiskCache : : oUnknown :
break ;
}
}
} catch ( . . . ) {
return callback . rethrow ( ) ;
}
auto callbackPtr
= std : : make_shared < decltype ( callback ) > ( std : : move ( callback ) ) ;
queryRealisationUncached (
id ,
{ [ this , id , callbackPtr ] (
std : : future < std : : shared_ptr < const Realisation > > fut ) {
try {
auto info = fut . get ( ) ;
if ( diskCache ) {
if ( info )
diskCache - > upsertRealisation ( getUri ( ) , * info ) ;
else
diskCache - > upsertAbsentRealisation ( getUri ( ) , id ) ;
}
( * callbackPtr ) ( std : : shared_ptr < const Realisation > ( info ) ) ;
} catch ( . . . ) {
callbackPtr - > rethrow ( ) ;
}
} } ) ;
}
std : : shared_ptr < const Realisation > Store : : queryRealisation ( const DrvOutput & id )
{
using RealPtr = std : : shared_ptr < const Realisation > ;
std : : promise < RealPtr > promise ;
queryRealisation ( id ,
{ [ & ] ( std : : future < RealPtr > result ) {
try {
promise . set_value ( result . get ( ) ) ;
} catch ( . . . ) {
promise . set_exception ( std : : current_exception ( ) ) ;
}
} } ) ;
return promise . get_future ( ) . get ( ) ;
}
2016-02-15 15:48:38 +02:00
2020-10-21 22:31:19 +03:00
void Store : : substitutePaths ( const StorePathSet & paths )
{
2021-04-05 16:48:18 +03:00
std : : vector < DerivedPath > paths2 ;
2020-10-21 22:31:19 +03:00
for ( auto & path : paths )
if ( ! path . isDerivation ( ) )
2023-11-03 12:58:47 +02:00
paths2 . emplace_back ( DerivedPath : : Opaque { path } ) ;
2020-10-21 22:31:19 +03:00
uint64_t downloadSize , narSize ;
StorePathSet willBuild , willSubstitute , unknown ;
queryMissing ( paths2 ,
willBuild , willSubstitute , unknown , downloadSize , narSize ) ;
if ( ! willSubstitute . empty ( ) )
try {
2021-04-05 16:48:18 +03:00
std : : vector < DerivedPath > subs ;
2024-02-12 16:29:48 +02:00
for ( auto & p : willSubstitute ) subs . emplace_back ( DerivedPath : : Opaque { p } ) ;
2020-10-21 22:31:19 +03:00
buildPaths ( subs ) ;
} catch ( Error & e ) {
logWarning ( e . info ( ) ) ;
}
}
2019-12-05 20:11:09 +02:00
StorePathSet Store : : queryValidPaths ( const StorePathSet & paths , SubstituteFlag maybeSubstitute )
2016-10-07 20:20:47 +03:00
{
2016-10-07 20:43:36 +03:00
struct State
{
size_t left ;
2019-12-05 20:11:09 +02:00
StorePathSet valid ;
2016-10-07 20:43:36 +03:00
std : : exception_ptr exc ;
} ;
2016-10-07 20:20:47 +03:00
2019-12-05 20:11:09 +02:00
Sync < State > state_ ( State { paths . size ( ) , StorePathSet ( ) } ) ;
2016-10-07 20:20:47 +03:00
2016-10-07 20:43:36 +03:00
std : : condition_variable wakeup ;
2017-10-25 18:13:49 +03:00
ThreadPool pool ;
2016-10-07 20:43:36 +03:00
2023-01-30 16:35:25 +02:00
auto doQuery = [ & ] ( const StorePath & path ) {
2017-10-25 18:51:45 +03:00
checkInterrupt ( ) ;
2023-02-01 21:27:35 +02:00
queryPathInfo ( path , { [ path , & state_ , & wakeup ] ( std : : future < ref < const ValidPathInfo > > fut ) {
2018-03-27 23:16:01 +03:00
auto state ( state_ . lock ( ) ) ;
try {
auto info = fut . get ( ) ;
2023-01-30 16:35:25 +02:00
state - > valid . insert ( path ) ;
2018-03-27 23:16:01 +03:00
} catch ( InvalidPath & ) {
} catch ( . . . ) {
state - > exc = std : : current_exception ( ) ;
}
assert ( state - > left ) ;
if ( ! - - state - > left )
wakeup . notify_one ( ) ;
} } ) ;
2017-10-25 18:13:49 +03:00
} ;
for ( auto & path : paths )
2023-01-30 16:35:25 +02:00
pool . enqueue ( std : : bind ( doQuery , path ) ) ;
2017-10-25 18:13:49 +03:00
pool . process ( ) ;
2016-10-07 20:43:36 +03:00
while ( true ) {
auto state ( state_ . lock ( ) ) ;
if ( ! state - > left ) {
if ( state - > exc ) std : : rethrow_exception ( state - > exc ) ;
2019-12-05 20:11:09 +02:00
return std : : move ( state - > valid ) ;
2016-10-07 20:43:36 +03:00
}
state . wait ( wakeup ) ;
}
2016-10-07 20:20:47 +03:00
}
2008-01-29 20:17:36 +02:00
/* Return a string accepted by decodeValidPathInfo() that
registers the specified paths as valid . Note : it ' s the
responsibility of the caller to provide a closure . */
2022-02-25 17:00:00 +02:00
std : : string Store : : makeValidityRegistration ( const StorePathSet & paths ,
2008-01-29 20:17:36 +02:00
bool showDerivers , bool showHash )
{
2022-02-25 17:00:00 +02:00
std : : string s = " " ;
2008-01-29 20:17:36 +02:00
2015-07-17 20:24:28 +03:00
for ( auto & i : paths ) {
2019-12-05 20:11:09 +02:00
s + = printStorePath ( i ) + " \n " ;
2015-07-17 20:24:28 +03:00
2016-04-19 19:50:15 +03:00
auto info = queryPathInfo ( i ) ;
2008-01-29 20:17:36 +02:00
2010-11-16 19:11:46 +02:00
if ( showHash ) {
2023-10-13 04:48:15 +03:00
s + = info - > narHash . to_string ( HashFormat : : Base16 , false ) + " \n " ;
2023-03-02 16:44:19 +02:00
s + = fmt ( " %1% \n " , info - > narSize ) ;
2010-11-16 19:11:46 +02:00
}
2019-12-05 20:11:09 +02:00
auto deriver = showDerivers & & info - > deriver ? printStorePath ( * info - > deriver ) : " " ;
2008-01-29 20:17:36 +02:00
s + = deriver + " \n " ;
2023-03-02 16:44:19 +02:00
s + = fmt ( " %1% \n " , info - > references . size ( ) ) ;
2008-01-29 20:17:36 +02:00
2016-04-19 19:50:15 +03:00
for ( auto & j : info - > references )
2019-12-05 20:11:09 +02:00
s + = printStorePath ( j ) + " \n " ;
2008-01-29 20:17:36 +02:00
}
return s ;
}
2021-05-02 18:24:14 +03:00
StorePathSet Store : : exportReferences ( const StorePathSet & storePaths , const StorePathSet & inputPaths )
{
StorePathSet paths ;
for ( auto & storePath : storePaths ) {
if ( ! inputPaths . count ( storePath ) )
throw BuildError ( " cannot export references of path '%s' because it is not in the input closure of the derivation " , printStorePath ( storePath ) ) ;
computeFSClosure ( { storePath } , paths ) ;
}
/* If there are derivations in the graph, then include their
outputs as well . This is useful if you want to do things
like passing all build - time dependencies of some path to a
derivation that builds a NixOS DVD image . */
auto paths2 = paths ;
for ( auto & j : paths2 ) {
if ( j . isDerivation ( ) ) {
Derivation drv = derivationFromPath ( j ) ;
for ( auto & k : drv . outputsAndOptPaths ( * this ) ) {
if ( ! k . second . second )
/* FIXME: I am confused why we are calling
` computeFSClosure ` on the output path , rather than
derivation itself . That doesn ' t seem right to me , so I
won ' t try to implemented this for CA derivations . */
throw UnimplementedError ( " exportReferences on CA derivations is not yet implemented " ) ;
computeFSClosure ( * k . second . second , paths ) ;
}
}
}
return paths ;
}
exportReferencesGraph: Export more complete info in JSON format
This writes info about every path in the closure in the same format as
‘nix path-info --json’. Thus it also includes NAR hashes and sizes.
Example:
[
{
"path": "/nix/store/10h6li26i7g6z3mdpvra09yyf10mmzdr-hello-2.10",
"narHash": "sha256:0ckdc4z20kkmpqdilx0wl6cricxv90lh85xpv2qljppcmz6vzcxl",
"narSize": 197648,
"references": [
"/nix/store/10h6li26i7g6z3mdpvra09yyf10mmzdr-hello-2.10",
"/nix/store/27binbdy296qvjycdgr1535v8872vz3z-glibc-2.24"
],
"closureSize": 20939776
},
{
"path": "/nix/store/27binbdy296qvjycdgr1535v8872vz3z-glibc-2.24",
"narHash": "sha256:1nfn3m3p98y1c0kd0brp80dn9n5mycwgrk183j17rajya0h7gax3",
"narSize": 20742128,
"references": [
"/nix/store/27binbdy296qvjycdgr1535v8872vz3z-glibc-2.24"
],
"closureSize": 20742128
}
]
Fixes #1134.
2017-01-26 21:36:20 +02:00
2016-04-19 19:50:15 +03:00
const Store : : Stats & Store : : getStats ( )
{
2016-04-20 15:12:38 +03:00
{
auto state_ ( state . lock ( ) ) ;
stats . pathInfoCacheSize = state_ - > pathInfoCache . size ( ) ;
}
2016-04-19 19:50:15 +03:00
return stats ;
}
2021-07-26 14:31:09 +03:00
static std : : string makeCopyPathMessage (
std : : string_view srcUri ,
std : : string_view dstUri ,
std : : string_view storePath )
{
return srcUri = = " local " | | srcUri = = " daemon "
? fmt ( " copying path '%s' to '%s' " , storePath , dstUri )
: dstUri = = " local " | | dstUri = = " daemon "
? fmt ( " copying path '%s' from '%s' " , storePath , srcUri )
: fmt ( " copying path '%s' from '%s' to '%s' " , storePath , srcUri , dstUri ) ;
}
2021-07-19 13:01:06 +03:00
void copyStorePath (
Store & srcStore ,
Store & dstStore ,
const StorePath & storePath ,
RepairFlag repair ,
CheckSigsFlag checkSigs )
2016-05-03 15:45:50 +03:00
{
2024-01-18 18:16:34 +02:00
/* Bail out early (before starting a download from srcStore) if
dstStore already has this path . */
if ( ! repair & & dstStore . isValidPath ( storePath ) )
return ;
2021-07-19 13:01:06 +03:00
auto srcUri = srcStore . getUri ( ) ;
auto dstUri = dstStore . getUri ( ) ;
2021-07-26 14:31:09 +03:00
auto storePathS = srcStore . printStorePath ( storePath ) ;
2019-07-10 20:46:15 +03:00
Activity act ( * logger , lvlInfo , actCopyPath ,
2021-07-26 14:31:09 +03:00
makeCopyPathMessage ( srcUri , dstUri , storePathS ) ,
{ storePathS , srcUri , dstUri } ) ;
2019-07-10 20:46:15 +03:00
PushActivity pact ( act . id ) ;
2021-07-19 13:01:06 +03:00
auto info = srcStore . queryPathInfo ( storePath ) ;
2019-07-10 20:46:15 +03:00
uint64_t total = 0 ;
2020-06-12 17:49:09 +03:00
// recompute store path on the chance dstStore does it differently
2020-06-22 20:08:11 +03:00
if ( info - > ca & & info - > references . empty ( ) ) {
2019-07-10 20:46:15 +03:00
auto info2 = make_ref < ValidPathInfo > ( * info ) ;
2021-10-01 01:36:50 +03:00
info2 - > path = dstStore . makeFixedOutputPathFromCA (
2023-01-23 19:58:11 +02:00
info - > path . name ( ) ,
2023-02-28 18:34:18 +02:00
info - > contentAddressWithReferences ( ) . value ( ) ) ;
2021-07-19 13:01:06 +03:00
if ( dstStore . storeDir = = srcStore . storeDir )
2020-06-17 21:04:46 +03:00
assert ( info - > path = = info2 - > path ) ;
2019-07-10 20:46:15 +03:00
info = info2 ;
}
2017-02-07 20:23:16 +02:00
2019-07-10 20:46:15 +03:00
if ( info - > ultimate ) {
auto info2 = make_ref < ValidPathInfo > ( * info ) ;
info2 - > ultimate = false ;
info = info2 ;
}
2017-05-01 21:03:25 +03:00
2019-07-10 20:46:15 +03:00
auto source = sinkToSource ( [ & ] ( Sink & sink ) {
2020-12-02 15:00:43 +02:00
LambdaSink progressSink ( [ & ] ( std : : string_view data ) {
total + = data . size ( ) ;
2019-07-10 20:46:15 +03:00
act . progress ( total , info - > narSize ) ;
2018-03-16 21:22:34 +02:00
} ) ;
2020-08-13 17:47:53 +03:00
TeeSink tee { sink , progressSink } ;
2021-07-19 13:01:06 +03:00
srcStore . narFromPath ( storePath , tee ) ;
2019-07-10 20:46:15 +03:00
} , [ & ] ( ) {
2021-07-19 13:01:06 +03:00
throw EndOfFile ( " NAR for '%s' fetched from '%s' is incomplete " , srcStore . printStorePath ( storePath ) , srcStore . getUri ( ) ) ;
2019-06-24 22:48:52 +03:00
} ) ;
2019-07-10 20:46:15 +03:00
2021-07-19 13:01:06 +03:00
dstStore . addToStore ( * info , * source , repair , checkSigs ) ;
2016-05-03 15:45:50 +03:00
}
2021-07-19 13:01:06 +03:00
std : : map < StorePath , StorePath > copyPaths (
Store & srcStore ,
Store & dstStore ,
const RealisedPath : : Set & paths ,
RepairFlag repair ,
CheckSigsFlag checkSigs ,
SubstituteFlag substitute )
2020-12-14 20:43:53 +02:00
{
StorePathSet storePaths ;
2021-05-19 11:36:48 +03:00
std : : set < Realisation > toplevelRealisations ;
2021-02-25 18:10:45 +02:00
for ( auto & path : paths ) {
2020-12-14 20:43:53 +02:00
storePaths . insert ( path . path ( ) ) ;
2021-02-25 17:58:27 +02:00
if ( auto realisation = std : : get_if < Realisation > ( & path . raw ) ) {
2023-03-17 16:33:48 +02:00
experimentalFeatureSettings . require ( Xp : : CaDerivations ) ;
2021-05-19 11:36:48 +03:00
toplevelRealisations . insert ( * realisation ) ;
2021-02-25 17:58:27 +02:00
}
2020-12-14 20:43:53 +02:00
}
auto pathsMap = copyPaths ( srcStore , dstStore , storePaths , repair , checkSigs , substitute ) ;
2021-05-19 11:36:48 +03:00
ThreadPool pool ;
2021-02-19 18:58:28 +02:00
try {
2021-05-19 11:36:48 +03:00
// Copy the realisation closure
processGraph < Realisation > (
2021-07-19 13:01:06 +03:00
pool , Realisation : : closure ( srcStore , toplevelRealisations ) ,
2021-06-25 16:35:14 +03:00
[ & ] ( const Realisation & current ) - > std : : set < Realisation > {
2021-05-19 11:36:48 +03:00
std : : set < Realisation > children ;
2021-06-25 16:35:14 +03:00
for ( const auto & [ drvOutput , _ ] : current . dependentRealisations ) {
2021-07-19 13:01:06 +03:00
auto currentChild = srcStore . queryRealisation ( drvOutput ) ;
2021-05-19 11:36:48 +03:00
if ( ! currentChild )
throw Error (
2021-06-25 16:35:14 +03:00
" incomplete realisation closure: '%s' is a "
" dependency of '%s' but isn't registered " ,
2021-05-19 11:36:48 +03:00
drvOutput . to_string ( ) , current . id . to_string ( ) ) ;
children . insert ( * currentChild ) ;
}
return children ;
} ,
[ & ] ( const Realisation & current ) - > void {
2021-07-19 13:01:06 +03:00
dstStore . registerDrvOutput ( current , checkSigs ) ;
2021-05-19 11:36:48 +03:00
} ) ;
2021-07-19 13:01:06 +03:00
} catch ( MissingExperimentalFeature & e ) {
2021-02-19 18:58:28 +02:00
// Don't fail if the remote doesn't support CA derivations is it might
2021-02-25 18:10:45 +02:00
// not be within our control to change that, and we might still want
2021-02-19 18:58:28 +02:00
// to at least copy the output paths.
2021-10-25 16:53:01 +03:00
if ( e . missingFeature = = Xp : : CaDerivations )
2021-02-19 18:58:28 +02:00
ignoreException ( ) ;
else
throw ;
2020-12-14 20:43:53 +02:00
}
return pathsMap ;
}
2021-07-19 13:01:06 +03:00
std : : map < StorePath , StorePath > copyPaths (
Store & srcStore ,
Store & dstStore ,
const StorePathSet & storePaths ,
RepairFlag repair ,
CheckSigsFlag checkSigs ,
SubstituteFlag substitute )
2016-10-07 20:15:25 +03:00
{
2021-07-19 13:01:06 +03:00
auto valid = dstStore . queryValidPaths ( storePaths , substitute ) ;
2017-06-28 19:11:01 +03:00
2020-07-29 01:24:55 +03:00
StorePathSet missing ;
2016-10-07 20:15:25 +03:00
for ( auto & path : storePaths )
2020-07-29 01:24:55 +03:00
if ( ! valid . count ( path ) ) missing . insert ( path ) ;
2016-10-07 20:15:25 +03:00
2022-06-08 15:03:46 +03:00
Activity act ( * logger , lvlInfo , actCopyPaths , fmt ( " copying %d paths " , missing . size ( ) ) ) ;
// In the general case, `addMultipleToStore` requires a sorted list of
// store paths to add, so sort them right now
auto sortedMissing = srcStore . topoSortPaths ( missing ) ;
std : : reverse ( sortedMissing . begin ( ) , sortedMissing . end ( ) ) ;
2020-06-19 21:48:57 +03:00
std : : map < StorePath , StorePath > pathsMap ;
2016-10-07 20:15:25 +03:00
for ( auto & path : storePaths )
2020-06-19 21:48:57 +03:00
pathsMap . insert_or_assign ( path , path ) ;
2016-10-07 20:15:25 +03:00
2022-06-08 15:03:46 +03:00
Store : : PathsSource pathsToCopy ;
2021-07-26 14:31:09 +03:00
2022-06-08 16:13:11 +03:00
auto computeStorePathForDst = [ & ] ( const ValidPathInfo & currentPathInfo ) - > StorePath {
auto storePathForSrc = currentPathInfo . path ;
auto storePathForDst = storePathForSrc ;
if ( currentPathInfo . ca & & currentPathInfo . references . empty ( ) ) {
2023-01-06 17:35:20 +02:00
storePathForDst = dstStore . makeFixedOutputPathFromCA (
2023-01-23 19:58:11 +02:00
currentPathInfo . path . name ( ) ,
2023-02-28 18:34:18 +02:00
currentPathInfo . contentAddressWithReferences ( ) . value ( ) ) ;
2022-06-08 16:13:11 +03:00
if ( dstStore . storeDir = = srcStore . storeDir )
assert ( storePathForDst = = storePathForSrc ) ;
if ( storePathForDst ! = storePathForSrc )
debug ( " replaced path '%s' to '%s' for substituter '%s' " ,
srcStore . printStorePath ( storePathForSrc ) ,
dstStore . printStorePath ( storePathForDst ) ,
dstStore . getUri ( ) ) ;
}
return storePathForDst ;
} ;
2023-03-31 02:58:07 +03:00
// total is accessed by each copy, which are each handled in separate threads
2023-04-01 03:44:13 +03:00
std : : atomic < uint64_t > total = 0 ;
2023-03-09 14:03:48 +02:00
2022-06-08 15:03:46 +03:00
for ( auto & missingPath : sortedMissing ) {
2022-06-03 18:01:16 +03:00
auto info = srcStore . queryPathInfo ( missingPath ) ;
2022-06-08 15:03:46 +03:00
2022-06-08 16:13:11 +03:00
auto storePathForDst = computeStorePathForDst ( * info ) ;
pathsMap . insert_or_assign ( missingPath , storePathForDst ) ;
ValidPathInfo infoForDst = * info ;
infoForDst . path = storePathForDst ;
auto source = sinkToSource ( [ & ] ( Sink & sink ) {
2022-06-08 15:03:46 +03:00
// We can reasonably assume that the copy will happen whenever we
// read the path, so log something about that at that point
auto srcUri = srcStore . getUri ( ) ;
auto dstUri = dstStore . getUri ( ) ;
auto storePathS = srcStore . printStorePath ( missingPath ) ;
Activity act ( * logger , lvlInfo , actCopyPath ,
makeCopyPathMessage ( srcUri , dstUri , storePathS ) ,
{ storePathS , srcUri , dstUri } ) ;
PushActivity pact ( act . id ) ;
2023-03-09 14:03:48 +02:00
LambdaSink progressSink ( [ & ] ( std : : string_view data ) {
2023-04-01 03:44:13 +03:00
total + = data . size ( ) ;
act . progress ( total , info - > narSize ) ;
2023-03-09 14:03:48 +02:00
} ) ;
TeeSink tee { sink , progressSink } ;
srcStore . narFromPath ( missingPath , tee ) ;
2022-06-03 18:01:16 +03:00
} ) ;
2022-06-08 16:13:11 +03:00
pathsToCopy . push_back ( std : : pair { infoForDst , std : : move ( source ) } ) ;
2022-06-03 18:01:16 +03:00
}
2017-10-18 16:02:58 +03:00
2022-06-03 18:01:16 +03:00
dstStore . addMultipleToStore ( pathsToCopy , act , repair , checkSigs ) ;
2017-08-14 16:28:16 +03:00
2020-06-19 21:48:57 +03:00
return pathsMap ;
2017-06-28 19:11:01 +03:00
}
2016-10-07 20:15:25 +03:00
2021-07-16 10:37:33 +03:00
void copyClosure (
2021-07-19 13:01:06 +03:00
Store & srcStore ,
Store & dstStore ,
2021-07-16 10:37:33 +03:00
const RealisedPath : : Set & paths ,
RepairFlag repair ,
CheckSigsFlag checkSigs ,
SubstituteFlag substitute )
{
2021-07-19 13:01:06 +03:00
if ( & srcStore = = & dstStore ) return ;
2021-07-16 10:37:33 +03:00
RealisedPath : : Set closure ;
2021-07-19 13:01:06 +03:00
RealisedPath : : closure ( srcStore , paths , closure ) ;
2021-07-16 10:37:33 +03:00
copyPaths ( srcStore , dstStore , closure , repair , checkSigs , substitute ) ;
}
2022-01-20 22:45:34 +02:00
void copyClosure (
Store & srcStore ,
Store & dstStore ,
const StorePathSet & storePaths ,
RepairFlag repair ,
CheckSigsFlag checkSigs ,
SubstituteFlag substitute )
{
if ( & srcStore = = & dstStore ) return ;
StorePathSet closure ;
srcStore . computeFSClosure ( storePaths , closure ) ;
copyPaths ( srcStore , dstStore , closure , repair , checkSigs , substitute ) ;
}
2020-08-06 21:31:48 +03:00
std : : optional < ValidPathInfo > decodeValidPathInfo ( const Store & store , std : : istream & str , std : : optional < HashResult > hashGiven )
2007-08-12 03:29:28 +03:00
{
2019-12-05 20:11:09 +02:00
std : : string path ;
getline ( str , path ) ;
if ( str . eof ( ) ) { return { } ; }
2020-08-06 21:31:48 +03:00
if ( ! hashGiven ) {
2022-02-25 17:00:00 +02:00
std : : string s ;
2008-01-29 20:17:36 +02:00
getline ( str , s ) ;
2023-11-28 15:20:27 +02:00
auto narHash = Hash : : parseAny ( s , HashAlgorithm : : SHA256 ) ;
2010-11-16 19:11:46 +02:00
getline ( str , s ) ;
2021-01-08 13:22:21 +02:00
auto narSize = string2Int < uint64_t > ( s ) ;
if ( ! narSize ) throw Error ( " number expected " ) ;
hashGiven = { narHash , * narSize } ;
2008-01-29 20:17:36 +02:00
}
2020-08-06 21:31:48 +03:00
ValidPathInfo info ( store . parseStorePath ( path ) , hashGiven - > first ) ;
info . narSize = hashGiven - > second ;
2019-12-05 20:11:09 +02:00
std : : string deriver ;
getline ( str , deriver ) ;
if ( deriver ! = " " ) info . deriver = store . parseStorePath ( deriver ) ;
2022-02-25 17:00:00 +02:00
std : : string s ;
2007-08-12 03:29:28 +03:00
getline ( str , s ) ;
2021-01-08 13:22:21 +02:00
auto n = string2Int < int > ( s ) ;
if ( ! n ) throw Error ( " number expected " ) ;
while ( ( * n ) - - ) {
2007-08-12 03:29:28 +03:00
getline ( str , s ) ;
2019-12-05 20:11:09 +02:00
info . references . insert ( store . parseStorePath ( s ) ) ;
2007-08-12 03:29:28 +03:00
}
if ( ! str | | str . eof ( ) ) throw Error ( " missing input " ) ;
2019-12-05 20:11:09 +02:00
return std : : optional < ValidPathInfo > ( std : : move ( info ) ) ;
}
2023-01-07 01:06:03 +02:00
std : : string StoreDirConfig : : showPaths ( const StorePathSet & paths )
2019-12-05 20:11:09 +02:00
{
std : : string s ;
for ( auto & i : paths ) {
if ( s . size ( ) ! = 0 ) s + = " , " ;
s + = " ' " + printStorePath ( i ) + " ' " ;
}
return s ;
2007-08-12 03:29:28 +03:00
}
2022-02-25 17:00:00 +02:00
std : : string showPaths ( const PathSet & paths )
2008-06-09 16:52:45 +03:00
{
2024-07-12 23:09:27 +03:00
return dropEmptyInitThenConcatStringsSep ( " , " , quoteStrings ( paths ) ) ;
2008-06-09 16:52:45 +03:00
}
2020-06-17 06:56:48 +03:00
Derivation Store : : derivationFromPath ( const StorePath & drvPath )
{
ensurePath ( drvPath ) ;
return readDerivation ( drvPath ) ;
}
2023-11-01 18:09:28 +02:00
static Derivation readDerivationCommon ( Store & store , const StorePath & drvPath , bool requireValidPath )
2020-06-17 06:56:48 +03:00
{
2023-11-01 18:09:28 +02:00
auto accessor = store . getFSAccessor ( requireValidPath ) ;
2020-06-17 06:56:48 +03:00
try {
2020-12-15 11:54:24 +02:00
return parseDerivation ( store ,
2023-11-01 18:09:28 +02:00
accessor - > readFile ( CanonPath ( store . printStorePath ( drvPath ) ) ) ,
2020-08-01 22:38:35 +03:00
Derivation : : nameFromPath ( drvPath ) ) ;
2020-06-17 06:56:48 +03:00
} catch ( FormatError & e ) {
2020-12-15 11:54:24 +02:00
throw Error ( " error parsing derivation '%s': %s " , store . printStorePath ( drvPath ) , e . msg ( ) ) ;
2020-06-17 06:56:48 +03:00
}
}
2022-03-08 07:02:25 +02:00
std : : optional < StorePath > Store : : getBuildDerivationPath ( const StorePath & path )
{
if ( ! path . isDerivation ( ) ) {
try {
auto info = queryPathInfo ( path ) ;
if ( ! info - > deriver ) return std : : nullopt ;
return * info - > deriver ;
} catch ( InvalidPath & ) {
return std : : nullopt ;
}
}
2023-03-17 16:33:48 +02:00
if ( ! experimentalFeatureSettings . isEnabled ( Xp : : CaDerivations ) | | ! isValidPath ( path ) )
2022-03-08 07:02:25 +02:00
return path ;
auto drv = readDerivation ( path ) ;
2022-12-08 23:59:21 +02:00
if ( ! drv . type ( ) . hasKnownOutputPaths ( ) ) {
2022-03-08 07:02:25 +02:00
// The build log is actually attached to the corresponding
// resolved derivation, so we need to get it first
auto resolvedDrv = drv . tryResolve ( * this ) ;
if ( resolvedDrv )
return writeDerivation ( * this , * resolvedDrv , NoRepair , true ) ;
}
return path ;
}
2020-12-15 11:54:24 +02:00
Derivation Store : : readDerivation ( const StorePath & drvPath )
{ return readDerivationCommon ( * this , drvPath , true ) ; }
2020-11-17 14:58:55 +02:00
Derivation Store : : readInvalidDerivation ( const StorePath & drvPath )
2020-12-15 11:54:24 +02:00
{ return readDerivationCommon ( * this , drvPath , false ) ; }
2020-11-17 14:58:55 +02:00
2006-11-30 19:43:04 +02:00
}
2024-04-19 00:49:17 +03:00
# include "local-store.hh"
# include "uds-remote-store.hh"
2006-11-30 19:43:04 +02:00
namespace nix {
2024-01-23 21:23:03 +02:00
ref < Store > openStore ( const std : : string & uri ,
const Store : : Params & extraParams )
2006-11-30 19:43:04 +02:00
{
2024-01-23 21:23:03 +02:00
return openStore ( StoreReference : : parse ( uri , extraParams ) ) ;
2019-02-21 12:44:25 +02:00
}
2024-01-23 21:23:03 +02:00
ref < Store > openStore ( StoreReference & & storeURI )
2022-06-23 17:29:50 +03:00
{
2024-01-23 21:23:03 +02:00
auto & params = storeURI . params ;
auto store = std : : visit ( overloaded {
[ & ] ( const StoreReference : : Auto & ) - > std : : shared_ptr < Store > {
auto stateDir = getOr ( params , " state " , settings . nixStateDir ) ;
if ( access ( stateDir . c_str ( ) , R_OK | W_OK ) = = 0 )
return std : : make_shared < LocalStore > ( params ) ;
else if ( pathExists ( settings . nixDaemonSocketFile ) )
return std : : make_shared < UDSRemoteStore > ( params ) ;
# if __linux__
else if ( ! pathExists ( stateDir )
& & params . empty ( )
& & ! isRootUser ( )
& & ! getEnv ( " NIX_STORE_DIR " ) . has_value ( )
& & ! getEnv ( " NIX_STATE_DIR " ) . has_value ( ) )
{
/* If /nix doesn't exist, there is no daemon socket, and
we ' re not root , then automatically set up a chroot
store in ~ / . local / share / nix / root . */
auto chrootStore = getDataDir ( ) + " /nix/root " ;
if ( ! pathExists ( chrootStore ) ) {
try {
createDirs ( chrootStore ) ;
2024-06-20 17:23:25 +03:00
} catch ( SystemError & e ) {
2024-01-23 21:23:03 +02:00
return std : : make_shared < LocalStore > ( params ) ;
}
warn ( " '%s' does not exist, so Nix will use '%s' as a chroot store " , stateDir , chrootStore ) ;
} else
debug ( " '%s' does not exist, so Nix will use '%s' as a chroot store " , stateDir , chrootStore ) ;
return std : : make_shared < LocalStore > ( " local " , chrootStore , params ) ;
2020-12-05 16:33:16 +02:00
}
2024-01-23 21:23:03 +02:00
# endif
else
return std : : make_shared < LocalStore > ( params ) ;
} ,
[ & ] ( const StoreReference : : Specified & g ) {
for ( auto implem : * Implementations : : registered )
if ( implem . uriSchemes . count ( g . scheme ) )
return implem . create ( g . scheme , g . authority , params ) ;
2020-12-05 16:33:16 +02:00
2024-01-23 21:23:03 +02:00
throw Error ( " don't know how to open Nix store with scheme '%s' " , g . scheme ) ;
} ,
} , storeURI . variant ) ;
2020-12-05 16:33:16 +02:00
2024-01-23 21:23:03 +02:00
experimentalFeatureSettings . require ( store - > experimentalFeature ( ) ) ;
store - > warnUnknownSettings ( ) ;
store - > init ( ) ;
2016-02-24 15:48:16 +02:00
2024-01-23 21:23:03 +02:00
return ref < Store > { store } ;
2016-02-29 17:11:11 +02:00
}
2016-04-29 14:57:08 +03:00
std : : list < ref < Store > > getDefaultSubstituters ( )
{
2017-07-04 17:26:48 +03:00
static auto stores ( [ ] ( ) {
2016-04-29 14:57:08 +03:00
std : : list < ref < Store > > stores ;
2017-07-04 17:26:48 +03:00
StringSet done ;
2016-04-29 14:57:08 +03:00
2017-07-04 17:26:48 +03:00
auto addStore = [ & ] ( const std : : string & uri ) {
2019-10-09 16:51:52 +03:00
if ( ! done . insert ( uri ) . second ) return ;
2018-02-09 15:36:38 +02:00
try {
stores . push_back ( openStore ( uri ) ) ;
} catch ( Error & e ) {
2020-05-03 17:01:25 +03:00
logWarning ( e . info ( ) ) ;
2018-02-09 15:36:38 +02:00
}
2017-07-04 17:26:48 +03:00
} ;
2016-04-29 14:57:08 +03:00
2017-07-04 17:26:48 +03:00
for ( auto uri : settings . substituters . get ( ) )
addStore ( uri ) ;
2017-03-21 18:59:18 +02:00
2017-07-04 17:34:53 +03:00
stores . sort ( [ ] ( ref < Store > & a , ref < Store > & b ) {
2019-12-17 18:17:53 +02:00
return a - > priority < b - > priority ;
2017-07-04 17:34:53 +03:00
} ) ;
2017-07-04 17:26:48 +03:00
return stores ;
} ( ) ) ;
2016-04-29 14:57:08 +03:00
2017-07-04 17:26:48 +03:00
return stores ;
2016-04-29 14:57:08 +03:00
}
2020-09-09 12:18:12 +03:00
std : : vector < StoreFactory > * Implementations : : registered = 0 ;
2016-04-29 14:57:08 +03:00
2006-11-30 19:43:04 +02:00
}