Merge pull request #62 from theuni/cache

Add common cache and per-build cache

bin/gbuild

@@ -80,6 +80,14 @@ def build_one_configuration(suite, arch, build_desc, reference_datetime)
     system! "copy-to-target #{@quiet_flag} inputs/#{filename} build/"
   end
 
+  if Dir.exists?("cache/#{build_desc["name"]}")
+    system! "copy-to-target #{@quiet_flag} cache/#{build_desc["name"]}/ cache/"
+  end
+
+  if Dir.exists?("cache/common")
+    system! "copy-to-target #{@quiet_flag} cache/common/ cache/"
+  end
+
   info "Updating apt-get repository (log in var/install.log)"
   system! "on-target -u root apt-get update > var/install.log 2>&1"
 
@@ -164,10 +172,12 @@ in_sums = []
 
 build_dir = 'build'
 result_dir = 'result'
+cache_dir = 'cache'
 
 FileUtils.rm_rf(build_dir)
 FileUtils.mkdir(build_dir)
 FileUtils.mkdir_p(result_dir)
+FileUtils.mkdir_p(cache_dir)
 
 package_name = build_desc["name"] or raise "must supply name"
 package_name = sanitize(package_name, "package name")
@@ -237,12 +247,18 @@ suites.each do |suite|
     info "Grabbing results"
     system! "copy-from-target #{@quiet_flag} out #{build_dir}"
 
+    info "Grabbing cache"
+    system! "copy-from-target #{@quiet_flag} cache/#{package_name}/ #{cache_dir}"
+    system! "copy-from-target #{@quiet_flag} cache/common/ #{cache_dir}"
+
     base_manifest = File.read("var/base-#{suite}-#{arch}.manifest")
     base_manifests["#{suite}-#{arch}"] = base_manifest
   end
 end
 
 out_dir = File.join(build_dir, "out")
+cache_common_dir = File.join(cache_dir, "common")
+cache_package_dir = File.join(cache_dir, "#{package_name}")
 out_sums = {}
 
 info "Generating report"
@@ -255,6 +271,24 @@ Dir.glob(File.join(out_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_i
   puts out_sums[file] unless @options[:quiet]
 end
 
+Dir.glob(File.join(cache_common_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_in_out|
+  next if File.directory?(file_in_out)
+  file = file_in_out.sub(cache_common_dir + File::SEPARATOR, '')
+  file = sanitize_path(file, file_in_out)
+  out_sums[file] = `cd #{cache_common_dir} && sha256sum #{file}`
+  raise "failed to sum #{file}" unless $? == 0
+  puts out_sums[file] unless @options[:quiet]
+end
+
+Dir.glob(File.join(cache_package_dir, '**', '*'), File::FNM_DOTMATCH).sort.each do |file_in_out|
+  next if File.directory?(file_in_out)
+  file = file_in_out.sub(cache_package_dir + File::SEPARATOR, '')
+  file = sanitize_path(file, file_in_out)
+  out_sums[file] = `cd #{cache_package_dir} && sha256sum #{file}`
+  raise "failed to sum #{file}" unless $? == 0
+  puts out_sums[file] unless @options[:quiet]
+end
+
 out_manifest = out_sums.keys.sort.map { |key| out_sums[key] }.join('')
 
 in_manifest = in_sums.join('')

doc/CACHE

@@ -0,0 +1,23 @@
+Gitian includes two caches where build-assets may be saved for use with
+subsequent builds.
+
+Common cache:
+All descriptors share this cache. It can be useful for storing fetched sources,
+sharing build assets between descriptors, etc.
+To add or update files, copy them to ~/cache/common from the build script.
+
+Per-descriptor cache
+Files installed to this cache can only be seen by this descriptor. Use this to
+store assets created as part of the build process, to avoid having to rebuild
+them in future builds.
+To add or update files, copy them to ~/cache/$NAME from the build script, where
+$NAME is the value of the descriptor's "name" key.
+
+Before each build, all files and folders in the cache directories will be
+transferred to the VM. After each successful build, the caches will be
+transferred back to the builder (overwriting existing files as necessary).
+
+It is entirely up to the user and the descriptors to verify that cached files
+are actually suitable for re-use. Overwriting existing cached files is _highly_
+discouraged, as that would likely lead to non-deterministic results. Unique
+filenames should always be used.

target-bin/init-build.sh

@@ -1,6 +1,7 @@
 #!/bin/sh
 
-rm -rf install out build
+rm -rf install out build cache
 mkdir build
 mkdir out
 mkdir install
+mkdir -p cache/common