Finally, the new stdenv-linux.

* The statically linked bootstrap tools are now automatically
  reproducable, just do:

  $ nix-build ./make-bootstrap-tools.nix

  The resulting binaries in result/in-nixpkgs go to
  stdenv/linux/bootstrap/<platform>/, and the tarballs in
  result/on-server go to
  https://svn.cs.uu.nl:12443/repos/trace/tarballs/trunk/stdenv-linux/<platform>/<revision>/.
  These are checked out on nix.cs.uu.nl under http://.../dist/tarballs.

* The statically linked libraries all use dietlibc now (except
  patchelf and glibc), so they are much smaller.  This is especially
  nice for the tools in the Nixpkgs tree, since it makes Nixpkgs
  tarballs smaller.

* Use Binutils 2.17 and GCC 4.1.1 for the bootstrap.

* The stdenv is now based on Glibc 2.5.  I hope it works ;-)

svn path=/nixpkgs/trunk/; revision=6803

18 files changed, 338 insertions, 27 deletions

diff --git a/pkgs/development/libraries/glibc-2.5/default.nix b/pkgs/development/libraries/glibc-2.5/default.nix
deleted file mode 100644
index 20f5cba82b15d..0000000000000
--- a/pkgs/development/libraries/glibc-2.5/default.nix
+++ /dev/null
@@ -1,21 +0,0 @@
-{stdenv, fetchurl, kernelHeaders, installLocales ? true}:
-
-stdenv.mkDerivation {
-  name = "glibc-2.5";
-  builder = ./builder.sh;
-  substitute = ../../../build-support/substitute/substitute.sh;
-
-  src = fetchurl {
-    url = http://ftp.gnu.org/gnu/glibc/glibc-2.5.tar.bz2;
-    md5 = "1fb29764a6a650a4d5b409dda227ac9f";
-  };
-
-  linuxthreadsSrc = fetchurl {
-    url = http://ftp.gnu.org/gnu/glibc/glibc-linuxthreads-2.5.tar.bz2;
-    md5 = "870d76d46dcaba37c13d01dca47d1774";
-  };
-
-  patches = [ ./glibc-pwd.patch ];
-
-  inherit kernelHeaders installLocales;
-}
diff --git a/pkgs/development/libraries/glibc-2.5/builder.sh b/pkgs/development/libraries/glibc-new/builder.sh
index 0219e30c4ba98..2007235ab7660 100644
--- a/pkgs/development/libraries/glibc-2.5/builder.sh
+++ b/pkgs/development/libraries/glibc-new/builder.sh
@@ -37,12 +37,6 @@ preConfigure() {
     cd ../build
     
     configureScript=../$sourceRoot/configure
-    # `--with-tls --without-__thread' enables support for TLS but
-    # causes it not to be used.  Required if we don't want to barf on
-    # 2.4 kernels.  Or something.
-    configureFlags="--enable-add-ons \
-      --with-headers=$kernelHeaders/include
-      --with-tls --without-__thread --disable-sanity-checks"
 }
 
 
diff --git a/pkgs/development/libraries/glibc-new/default.nix b/pkgs/development/libraries/glibc-new/default.nix
new file mode 100644
index 0000000000000..e87d14165533b
--- /dev/null
+++ b/pkgs/development/libraries/glibc-new/default.nix
@@ -0,0 +1,44 @@
+{ stdenv, fetchurl, kernelHeaders
+, installLocales ? true
+, profilingLibraries ? false
+}:
+
+stdenv.mkDerivation {
+  name = "glibc-2.5";
+  builder = ./builder.sh;
+  substitute = ../../../build-support/substitute/substitute.sh;
+
+  /*
+  src = fetchurl {
+    url = http://nix.cs.uu.nl/dist/tarballs/glibc-2.3.6.tar.bz2;
+    md5 = "bfdce99f82d6dbcb64b7f11c05d6bc96";
+  };
+
+  linuxthreadsSrc = fetchurl {
+    url = http://nix.cs.uu.nl/dist/tarballs/glibc-linuxthreads-2.3.6.tar.bz2;
+    md5 = "d4eeda37472666a15cc1f407e9c987a9";
+  };
+  */
+
+  src = fetchurl {
+    url = http://ftp.gnu.org/gnu/glibc/glibc-2.5.tar.bz2;
+    md5 = "1fb29764a6a650a4d5b409dda227ac9f";
+  };
+
+  linuxthreadsSrc = fetchurl {
+    url = http://ftp.gnu.org/gnu/glibc/glibc-linuxthreads-2.5.tar.bz2;
+    md5 = "870d76d46dcaba37c13d01dca47d1774";
+  };
+
+  patches = [ ./glibc-pwd.patch ];
+
+  inherit kernelHeaders installLocales;
+
+  # `--with-tls --without-__thread' enables support for TLS but causes
+  # it not to be used.  Required if we don't want to barf on 2.4
+  # kernels.  Or something.
+  configureFlags="--enable-add-ons
+    --with-headers=${kernelHeaders}/include
+    --with-tls --without-__thread --disable-sanity-checks
+    ${if profilingLibraries then "--enable-profile" else "--disable-profile"}";
+}
diff --git a/pkgs/development/libraries/glibc-2.5/glibc-pwd.patch b/pkgs/development/libraries/glibc-new/glibc-pwd.patch
index 07039718477fe..07039718477fe 100644
--- a/pkgs/development/libraries/glibc-2.5/glibc-pwd.patch
+++ b/pkgs/development/libraries/glibc-new/glibc-pwd.patch
diff --git a/pkgs/os-specific/linux/kernel-headers/kernel-headers-2.6.18.1.nix b/pkgs/os-specific/linux/kernel-headers/kernel-headers-2.6.18.1.nix
new file mode 100644
index 0000000000000..73311a4092087
--- /dev/null
+++ b/pkgs/os-specific/linux/kernel-headers/kernel-headers-2.6.18.1.nix
@@ -0,0 +1,12 @@
+{stdenv, fetchurl}:
+
+assert stdenv.isLinux;
+
+stdenv.mkDerivation {
+  name = "linux-headers-2.6.18.1";
+  builder = ./builder.sh;
+  src = fetchurl {
+    url = ftp://ftp.nl.kernel.org/pub/linux/kernel/v2.6/linux-2.6.18.1.tar.bz2;
+    md5 = "38f00633b02f07819d17bcd87d03eb3a";
+  };
+}
diff --git a/pkgs/stdenv/linux/bootstrap/i686/bash b/pkgs/stdenv/linux/bootstrap/i686/bash
new file mode 100755
index 0000000000000..259e3ffe163a5
--- /dev/null
+++ b/pkgs/stdenv/linux/bootstrap/i686/bash
Binary files differdiff --git a/pkgs/stdenv/linux/bootstrap/i686/bunzip2 b/pkgs/stdenv/linux/bootstrap/i686/bunzip2
new file mode 100755
index 0000000000000..3f3f810e77ee2
--- /dev/null
+++ b/pkgs/stdenv/linux/bootstrap/i686/bunzip2
Binary files differdiff --git a/pkgs/stdenv/linux/bootstrap/i686/cp b/pkgs/stdenv/linux/bootstrap/i686/cp
new file mode 100755
index 0000000000000..06de68ad9a857
--- /dev/null
+++ b/pkgs/stdenv/linux/bootstrap/i686/cp
Binary files differdiff --git a/pkgs/stdenv/linux/bootstrap/i686/curl.bz2 b/pkgs/stdenv/linux/bootstrap/i686/curl.bz2
new file mode 100755
index 0000000000000..f1d4e4216932a
--- /dev/null
+++ b/pkgs/stdenv/linux/bootstrap/i686/curl.bz2
Binary files differdiff --git a/pkgs/stdenv/linux/bootstrap/i686/default.nix b/pkgs/stdenv/linux/bootstrap/i686/default.nix
new file mode 100644
index 0000000000000..695039692f5e1
--- /dev/null
+++ b/pkgs/stdenv/linux/bootstrap/i686/default.nix
@@ -0,0 +1,27 @@
+{
+  bash = ./bash;
+  bunzip2 = ./bunzip2;
+  cp = ./cp;
+  curl = ./curl.bz2;
+  tar = ./tar.bz2;
+
+  staticToolsURL = {
+    url = http://nix.cs.uu.nl/dist/tarballs/stdenv-linux/i686/r6800/static-tools.tar.bz2;
+    sha1 = "18c5e93a23a16282a12e9af05f4dc28254dc9013";
+  };
+
+  binutilsURL = {
+    url = http://nix.cs.uu.nl/dist/tarballs/stdenv-linux/i686/r6800/binutils.tar.bz2;
+    sha1 = "4bf2859aa705acdcc08d333200f0e55754fab4a9";
+  };
+
+  gccURL = {
+    url = http://nix.cs.uu.nl/dist/tarballs/stdenv-linux/i686/r6800/gcc.tar.bz2;
+    sha1 = "bd69a67b779014a683fa93706497eef0afede2b2";
+  };
+
+  glibcURL = {
+    url = http://nix.cs.uu.nl/dist/tarballs/stdenv-linux/i686/r6800/glibc.tar.bz2;
+    sha1 = "9f3f3f1248d672d5a845326ed36d8ca470de0094";
+  };
+}
diff --git a/pkgs/stdenv/linux/bootstrap/i686/tar.bz2 b/pkgs/stdenv/linux/bootstrap/i686/tar.bz2
new file mode 100755
index 0000000000000..26ea95b44404c
--- /dev/null
+++ b/pkgs/stdenv/linux/bootstrap/i686/tar.bz2
Binary files differdiff --git a/pkgs/stdenv/linux/new-scripts/builder-stdenv-initial.sh b/pkgs/stdenv/linux/new-scripts/builder-stdenv-initial.sh
new file mode 100644
index 0000000000000..f753667d260ad
--- /dev/null
+++ b/pkgs/stdenv/linux/new-scripts/builder-stdenv-initial.sh
@@ -0,0 +1,10 @@
+set -e
+
+PATH=$staticTools/bin
+
+mkdir $out
+
+cat > $out/setup <<EOF
+PATH=$staticTools/bin
+EOF
+
diff --git a/pkgs/stdenv/linux/new-scripts/download.sh b/pkgs/stdenv/linux/new-scripts/download.sh
new file mode 100644
index 0000000000000..1d010f88d1e4e
--- /dev/null
+++ b/pkgs/stdenv/linux/new-scripts/download.sh
@@ -0,0 +1,3 @@
+set -e
+echo "downloading $out from $url"
+$curl/bin/curl --fail --location --max-redirs 20 "$url" > "$out"
diff --git a/pkgs/stdenv/linux/new-scripts/prehook.sh b/pkgs/stdenv/linux/new-scripts/prehook.sh
new file mode 100644
index 0000000000000..4ba81f1eef74c
--- /dev/null
+++ b/pkgs/stdenv/linux/new-scripts/prehook.sh
@@ -0,0 +1,8 @@
+export NIX_ENFORCE_PURITY=1
+
+if test "$param1" = "static"; then
+    export NIX_CFLAGS_LINK="-static"
+    export NIX_LDFLAGS_BEFORE="-static"
+fi
+
+havePatchELF=1
diff --git a/pkgs/stdenv/linux/new-scripts/unpack-curl.sh b/pkgs/stdenv/linux/new-scripts/unpack-curl.sh
new file mode 100644
index 0000000000000..147edcebe04a1
--- /dev/null
+++ b/pkgs/stdenv/linux/new-scripts/unpack-curl.sh
@@ -0,0 +1,11 @@
+set -x
+set -e
+
+# Tricky: need to make $out/bin without mkdir ;-).  So use cp to copy
+# the current (empty) directory.
+$cp -prvd . $out
+$cp -prvd . $out/bin
+
+$cp $curl curl.bz2
+$bunzip2 -d curl.bz2
+$cp curl $out/bin
diff --git a/pkgs/stdenv/linux/new-scripts/unpack.sh b/pkgs/stdenv/linux/new-scripts/unpack.sh
new file mode 100644
index 0000000000000..ef3eaf6349c29
--- /dev/null
+++ b/pkgs/stdenv/linux/new-scripts/unpack.sh
@@ -0,0 +1,8 @@
+set -e
+
+$cp $tar .tar.bz2
+$bunzip2 .tar.bz2
+
+$bunzip2 -d < $tarball | ./.tar xvf -
+
+$cp -prd * $out
diff --git a/pkgs/stdenv/linux/new.nix b/pkgs/stdenv/linux/new.nix
new file mode 100644
index 0000000000000..31a78ec20579c
--- /dev/null
+++ b/pkgs/stdenv/linux/new.nix
@@ -0,0 +1,206 @@
+# This file constructs the standard build environment for the
+# Linux/i686 platform.  It's completely pure; that is, it relies on no
+# external (non-Nix) tools, such as /usr/bin/gcc, and it contains a C
+# compiler and linker that do not search in default locations,
+# ensuring purity of components produced by it.
+
+{system, allPackages}:
+
+rec {
+
+  bootstrapTools = import ./bootstrap/i686;
+
+
+  # The bootstrap process proceeds in several steps.
+
+  
+  # 1) Create a standard environment by downloading pre-built
+  # statically linked binaries of coreutils, gcc, etc.
+
+  # To fetch the pre-built binaries, we use a statically linked `curl'
+  # binary which is unpacked here.
+  curl = derivation {
+    inherit system;
+    name = "curl";
+    builder = bootstrapTools.bash;
+    inherit (bootstrapTools) bunzip2 cp curl;
+    args = [ ./new-scripts/unpack-curl.sh ];
+  };
+
+  # This function downloads a file.
+  download = {url, sha1, pkgname}: derivation {
+    name = baseNameOf (toString url);
+    builder = bootstrapTools.bash;
+    inherit system curl url;
+    args = [ ./new-scripts/download.sh ];
+    outputHashAlgo = "sha1";
+    outputHash = sha1;
+  };
+
+  # This function downloads and unpacks a file.
+  downloadAndUnpack = pkgname: {url, sha1}: derivation {
+    name = pkgname;
+    builder = bootstrapTools.bash;
+    inherit (bootstrapTools) bunzip2 tar cp;
+    args = [ ./new-scripts/unpack.sh ];
+    tarball = download {inherit url sha1 pkgname;};
+    inherit system;
+    allowedReferences = [];
+  };
+
+  # The various statically linked components that make up the standard
+  # environment.
+  staticTools = downloadAndUnpack "static-tools" bootstrapTools.staticToolsURL;
+  staticBinutils = downloadAndUnpack "static-binutils" bootstrapTools.binutilsURL;
+  staticGCC = (downloadAndUnpack "static-gcc" bootstrapTools.gccURL)
+    // { langC = true; langCC = false; langF77 = false; };
+  staticGlibc = downloadAndUnpack "static-glibc" bootstrapTools.glibcURL;
+
+
+  # A helper function to call gcc-wrapper.
+  wrapGCC =
+    {gcc ? staticGCC, glibc, binutils, shell ? ""}:
+    (import ../../build-support/gcc-wrapper) {
+      nativeTools = false;
+      nativeGlibc = false;
+      inherit gcc binutils glibc shell;
+      stdenv = stdenvInitial;
+    };
+
+
+  # The "fake" standard environment used to build "real" standard
+  # environments.  It consists of just the basic statically linked
+  # tools.
+  stdenvInitial = let {
+    body = derivation {
+      name = "stdenv-linux-initial";
+      builder = bootstrapTools.bash;
+      args = [ ./new-scripts/builder-stdenv-initial.sh ];
+      inherit system staticTools curl;
+    } // {
+      # !!! too much duplication with stdenv/generic/default.nix
+      mkDerivation = attrs: (derivation ((removeAttrs attrs ["meta"]) // {
+        builder = bootstrapTools.bash;
+        args = ["-e" attrs.builder];
+        stdenv = body;
+        system = body.system;
+      })) // { meta = if attrs ? meta then attrs.meta else {}; };
+      shell = bootstrapTools.bash;
+    };
+  };
+
+
+  # This function builds the various standard environments used during
+  # the bootstrap.
+  stdenvBootFun =
+    {gcc, staticGlibc, extraAttrs ? {}}:
+    
+    import ../generic {
+      name = "stdenv-linux-boot";
+      param1 = if staticGlibc then "static" else "dynamic";
+      preHook = ./new-scripts/prehook.sh;
+      stdenv = stdenvInitial;
+      shell = bootstrapTools.bash;
+      initialPath = [
+        staticTools
+      ];
+      inherit gcc extraAttrs;
+    };
+
+
+  # Create the first "real" standard environment.  This one consists
+  # of statically linked components only, and a minimal glibc to keep
+  # the gcc configure script happy.
+  stdenvLinuxBoot1 = stdenvBootFun {
+    # Use the statically linked, downloaded glibc/gcc/binutils.
+    gcc = wrapGCC {glibc = staticGlibc; binutils = staticBinutils;};
+    staticGlibc = true;
+    extraAttrs = {inherit curl;};
+  };
+  
+
+  # 2) These are the packages that we can build with the first
+  #    stdenv.  We only need Glibc (in step 3).
+  stdenvLinuxBoot1Pkgs = allPackages {
+    inherit system;
+    bootStdenv = stdenvLinuxBoot1;
+  };
+
+  
+  # 3) Build Glibc with the statically linked tools.  The result is the
+  #    full, dynamically linked, final Glibc.
+  stdenvLinuxGlibc = stdenvLinuxBoot1Pkgs.glibcNew;
+
+  
+  # 4) Construct a second stdenv identical to the first, except that
+  #    this one uses the Glibc built in step 3.  It still uses
+  #    statically linked tools.
+  stdenvLinuxBoot2 = removeAttrs (stdenvBootFun {
+    staticGlibc = false;
+    gcc = wrapGCC {binutils = staticBinutils; glibc = stdenvLinuxGlibc;};
+    extraAttrs = {inherit curl; glibc = stdenvLinuxGlibc;};
+  }) ["gcc" "binutils"];
+
+  
+  # 5) The packages that can be built using the second stdenv.
+  stdenvLinuxBoot2Pkgs = allPackages {
+    inherit system;
+    bootStdenv = stdenvLinuxBoot2;
+  };
+
+  
+  # 6) Construct a third stdenv identical to the second, except that
+  #    this one uses the dynamically linked GCC and Binutils from step
+  #    5.  The other tools (e.g. coreutils) are still static.
+  stdenvLinuxBoot3 = stdenvBootFun {
+    staticGlibc = false;
+    gcc = wrapGCC {
+#      inherit (stdenvLinuxBoot2Pkgs) binutils;
+      binutils = stdenvLinuxBoot2Pkgs.binutils217;
+      glibc = stdenvLinuxGlibc;
+      gcc = stdenvLinuxBoot2Pkgs.gcc.gcc;
+    };
+    extraAttrs = {inherit curl;};
+  };
+
+  
+  # 7) The packages that can be built using the third stdenv.
+  stdenvLinuxBoot3Pkgs = allPackages {
+    inherit system;
+    bootStdenv = stdenvLinuxBoot3;
+  };
+
+  
+  # 8) Construct the final stdenv.  It uses the Glibc, GCC and
+  #    Binutils built above, and adds in dynamically linked versions
+  #    of all other tools.
+  stdenvLinux = (import ../generic) {
+    name = "stdenv-linux";
+    preHook = ./new-scripts/prehook.sh;
+    initialPath = [
+      ((import ../common-path.nix) {pkgs = stdenvLinuxBoot3Pkgs;})
+      stdenvLinuxBoot3Pkgs.patchelf
+    ];
+
+    stdenv = stdenvInitial;
+
+    gcc = wrapGCC {
+#      inherit (stdenvLinuxBoot2Pkgs) binutils;
+      binutils = stdenvLinuxBoot2Pkgs.binutils217;
+      glibc = stdenvLinuxGlibc;
+      gcc = stdenvLinuxBoot2Pkgs.gcc.gcc;
+      shell = stdenvLinuxBoot3Pkgs.bash + "/bin/sh";
+    };
+
+    shell = stdenvLinuxBoot3Pkgs.bash + "/bin/sh";
+    
+    extraAttrs = {
+      curl = stdenvLinuxBoot3Pkgs.realCurl;
+      inherit (stdenvLinuxBoot2Pkgs) binutils /* gcc */ glibc;
+      inherit (stdenvLinuxBoot3Pkgs)
+        gzip bzip2 bash coreutils diffutils findutils gawk
+        gnumake gnused gnutar gnugrep patch patchelf;
+    };
+  };
+
+}
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index ee45b6c1fa458..f0c282e3dca3e 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -1253,6 +1253,11 @@ rec {
       installLocales = true;
     });
 
+  glibcNew = import ../development/libraries/glibc-new {
+      inherit fetchurl stdenv;
+      kernelHeaders = kernelHeadersNew;
+  };
+
   glibmm = import ../development/libraries/gtk-libs-2.6/glibmm {
     inherit fetchurl stdenv pkgconfig libsigcxx;
     inherit (gtkLibs26) glib;
@@ -2083,6 +2088,10 @@ rec {
     inherit fetchurl stdenv;
   };
 
+  kernelHeadersNew = import ../os-specific/linux/kernel-headers/kernel-headers-2.6.18.1.nix {
+    inherit fetchurl stdenv;
+  };
+
   kernelHeadersArm = import ../os-specific/linux/kernel-headers-cross {
     inherit fetchurl stdenv;
     cross = "arm-linux";