Add overlays mechanism to Nixpkgs.

This patch add a new argument to Nixpkgs default expression named "overlays".

By default, the value of the argument is either taken from the environment variable `NIXPKGS_OVERLAYS`,
or from the directory `~/.nixpkgs/overlays/`.  If the environment variable does not name a valid directory
then this mechanism would fallback on the home directory.  If the home directory does not exists it will
fallback on an empty list of overlays.

The overlays directory should contain the list of extra Nixpkgs stages which would be used to extend the
content of Nixpkgs, with additional set of packages.  The overlays, i-e directory, files, symbolic links
are used in alphabetical order.

The simplest overlay which extends Nixpkgs with nothing looks like:

```nix
self: super: {
}
```

More refined overlays can use `super` as the basis for building new packages, and `self` as a way to query
the final result of the fix-point.

An example of overlay which extends Nixpkgs with a small set of packages can be found at:
  https://github.com/nbp/nixpkgs-mozilla/blob/nixpkgs-overlay/moz-overlay.nix

To use this file, checkout the repository and add a symbolic link to
the `moz-overlay.nix` file in `~/.nixpkgs/overlays` directory.

15 files changed, 161 insertions, 104 deletions

diff --git a/doc/functions.xml b/doc/functions.xml
index f6a0a4352f63c..6374c15ddf2b5 100644
--- a/doc/functions.xml
+++ b/doc/functions.xml
@@ -17,66 +17,6 @@
     derivations or even the whole package set.
   </para>
 
-  <section xml:id="sec-pkgs-overridePackages">
-    <title>pkgs.overridePackages</title>
-
-    <para>
-      This function inside the nixpkgs expression (<varname>pkgs</varname>)
-      can be used to override the set of packages itself.
-    </para>
-    <para>
-      Warning: this function is expensive and must not be used from within
-      the nixpkgs repository.
-    </para>
-    <para>
-      Example usage:
-
-      <programlisting>let
-    pkgs = import &lt;nixpkgs&gt; {};
-    newpkgs = pkgs.overridePackages (self: super: {
-      foo = super.foo.override { ... };
-    });
-  in ...</programlisting>
-    </para>
-
-    <para>
-      The resulting <varname>newpkgs</varname> will have the new <varname>foo</varname>
-      expression, and all other expressions depending on <varname>foo</varname> will also
-      use the new <varname>foo</varname> expression.
-    </para>
-
-    <para>
-      The behavior of this function is similar to <link 
-      linkend="sec-modify-via-packageOverrides">config.packageOverrides</link>.
-    </para>
-
-    <para>
-      The <varname>self</varname> parameter refers to the final package set with the
-      applied overrides. Using this parameter may lead to infinite recursion if not
-      used consciously.
-    </para>
-
-    <para>
-      The <varname>super</varname> parameter refers to the old package set.
-      It's equivalent to <varname>pkgs</varname> in the above example.
-    </para>
-
-    <para>
-      Note that in previous versions of nixpkgs, this method replaced any changes from <link 
-      linkend="sec-modify-via-packageOverrides">config.packageOverrides</link>,
-      along with that from previous calls if this function was called repeatedly.
-      Now those previous changes will be preserved so this function can be "chained" meaningfully.
-      To recover the old behavior, make sure <varname>config.packageOverrides</varname> is unset,
-      and call this only once off a "freshly" imported nixpkgs:
-
-      <programlisting>let
-    pkgs = import &lt;nixpkgs&gt; { config: {}; };
-    newpkgs = pkgs.overridePackages ...;
-  in ...</programlisting>
-    </para>
-
-  </section>
-
   <section xml:id="sec-pkg-override">
     <title>&lt;pkg&gt;.override</title>
 
@@ -91,9 +31,9 @@
       Example usages:
 
       <programlisting>pkgs.foo.override { arg1 = val1; arg2 = val2; ... }</programlisting>
-      <programlisting>pkgs.overridePackages (self: super: {
+      <programlisting>import pkgs.path { overlays = [ (self: super: {
     foo = super.foo.override { barSupport = true ; };
-  })</programlisting>
+  })]};</programlisting>
       <programlisting>mypkg = pkgs.callPackage ./mypkg.nix {
     mydep = pkgs.mydep.override { ... };
   }</programlisting>
diff --git a/doc/languages-frameworks/python.md b/doc/languages-frameworks/python.md
index 82aeb112c93ed..48d9e0a0952f9 100644
--- a/doc/languages-frameworks/python.md
+++ b/doc/languages-frameworks/python.md
@@ -737,18 +737,18 @@ in (pkgs.python35.override {inherit packageOverrides;}).withPackages (ps: [ps.bl
 ```
 The requested package `blaze` depends on `pandas` which itself depends on `scipy`.
 
-If you want the whole of Nixpkgs to use your modifications, then you can use `pkgs.overridePackages`
+If you want the whole of Nixpkgs to use your modifications, then you can use `overlays`
 as explained in this manual. In the following example we build a `inkscape` using a different version of `numpy`.
 ```
 let
   pkgs = import <nixpkgs> {};
-  newpkgs = pkgs.overridePackages ( pkgsself: pkgssuper: {
+  newpkgs = import pkgs.path { overlays = [ (pkgsself: pkgssuper: {
     python27 = let
       packageOverrides = self: super: {
         numpy = super.numpy_1_10;
       };
     in pkgssuper.python27.override {inherit packageOverrides;};
-  } );
+  } ) ]; };
 in newpkgs.inkscape
 ```
 
diff --git a/doc/manual.xml b/doc/manual.xml
index 6ad66d486525d..1c0dac6e4df77 100644
--- a/doc/manual.xml
+++ b/doc/manual.xml
@@ -18,6 +18,7 @@
   <xi:include href="meta.xml" />
   <xi:include href="languages-frameworks/index.xml" />
   <xi:include href="package-notes.xml" />
+  <xi:include href="overlays.xml" />
   <xi:include href="coding-conventions.xml" />
   <xi:include href="submitting-changes.xml" />
   <xi:include href="reviewing-contributions.xml" />
diff --git a/doc/overlays.xml b/doc/overlays.xml
new file mode 100644
index 0000000000000..d194c6bfc8920
--- /dev/null
+++ b/doc/overlays.xml
@@ -0,0 +1,99 @@
+<chapter xmlns="http://docbook.org/ns/docbook"
+         xmlns:xlink="http://www.w3.org/1999/xlink"
+         xml:id="chap-overlays">
+
+<title>Overlays</title>
+
+<para>This chapter describes how to extend and change Nixpkgs content using
+overlays.  Overlays are used to add phases in the fix-point used by Nixpkgs
+to bind the dependencies of all packages.</para>
+
+<!--============================================================-->
+
+<section xml:id="sec-overlays-install">
+<title>Installing Overlays</title>
+
+<para>Overlays are looked for in the following order, the first valid one is
+considered, and all the rest are ignored:
+
+<orderedlist>
+
+  <listitem>
+
+    <para>As argument of the imported attribute set. When importing Nixpkgs,
+    the <varname>overlays</varname> attribute argument can be set to a list of
+    functions, which would be describe in <xref linkend="sec-overlays-layout"/>.</para>
+
+  </listitem>
+
+  <listitem>
+
+    <para>As a directory pointed by the environment variable named
+<varname>NIXPKGS_OVERLAYS</varname>. This directory can contain symbolic
+links to Nix expressions.
+
+  </listitem>
+
+  <listitem>
+
+    <para>As the directory located at
+<filename>~/.nixpkgs/overlays/</filename>. This directory can contain
+symbolic links to Nix expressions.
+
+  </listitem>
+
+</orderedlist>
+</para>
+
+<para>For the second and third option, the directory contains either
+directories providing a default.nix expression, or files, or symbolic links
+to the entry Nix expression of the overlay.  These Nix expressions are
+following the syntax described in <xref
+linkend="sec-overlays-layout"/>.</para>
+
+<para>To install an overlay, using the last option.  Clone the repository of
+the overlay, and add a symbolic link to it in the
+<filename>~/.nixpkgs/overlays/</filename> directory.</para>
+
+</section>
+
+<!--============================================================-->
+
+<section xml:id="sec-overlays-layout">
+<title>Overlays Layout</title>
+
+<para>An overlay is a Nix expression, which is a function which accepts 2
+arguments.</para>
+
+<programlisting>
+self: super:
+
+{
+  foo = super.foo.override { ... };
+  bar = import ./pkgs/bar {
+    inherit (self) stdenv fetchurl;
+    inherit (self) ninja crawl dwarf-fortress;
+  };
+}
+</programlisting>
+
+<para>The first argument, usualy named <varname>self</varname>, corresponds
+to the final package set. You should use this set to inherit all the
+dependencies needed by your package expression.</para>
+
+<para>The second argument, usualy named <varname>super</varname>,
+corresponds to the result of the evaluation of the previous stages of
+Nixpkgs, it does not contain any of the packages added by the current
+overlay nor any of the following overlays.  This set is used in to override
+existing packages, either by changing their dependencies or their
+recipes.</para>
+
+<para>The value returned by this function should be a set similar to
+<filename>pkgs/top-level/all-packages.nix</filename>, which contains either
+extra packages defined by the overlay, or which overwrite Nixpkgs packages
+with other custom defaults. This is similar to <xref
+linkend="sec-modify-via-packageOverrides"/>.</para>
+
+</section>
+
+</chapter>
diff --git a/nixos/doc/manual/release-notes/rl-1703.xml b/nixos/doc/manual/release-notes/rl-1703.xml
index 6997155d94d05..bbb9c53eaa363 100644
--- a/nixos/doc/manual/release-notes/rl-1703.xml
+++ b/nixos/doc/manual/release-notes/rl-1703.xml
@@ -11,7 +11,9 @@ has the following highlights: </para>
 
 <itemizedlist>
   <listitem>
-    <para></para>
+    <para>Nixpkgs is now extensible through <link
+    xlink:href="https://github.com/NixOS/nixpkgs/pull/21243">overlays</link>.
+    See the Nixpkgs manual for more information.</para>
   </listitem>
 </itemizedlist>
 
@@ -96,6 +98,11 @@ following incompatible changes:</para>
      <literal>networking.timeServers</literal>.
    </para>
   </listitem>
+
+  <listitem>
+    <para><literal>pkgs.overridePackages</literal> function no longer exists.     Instead import Nixpkgs a second time using <literal>import pkgs.path {
+    overlays = [ <replaceable>...</replaceable> ]; }</literal>.</para>
+  </listitem>
 </itemizedlist>
 
 
diff --git a/pkgs/stdenv/cross/default.nix b/pkgs/stdenv/cross/default.nix
index 359d45b78b967..16f41671b768f 100644
--- a/pkgs/stdenv/cross/default.nix
+++ b/pkgs/stdenv/cross/default.nix
@@ -1,10 +1,10 @@
 { lib
-, system, platform, crossSystem, config
+, system, platform, crossSystem, config, overlays
 }:
 
 let
   bootStages = import ../. {
-    inherit lib system platform;
+    inherit lib system platform overlays;
     crossSystem = null;
     # Ignore custom stdenvs when cross compiling for compatability
     config = builtins.removeAttrs config [ "replaceStdenv" ];
@@ -18,7 +18,7 @@ in bootStages ++ [
   # should be used to build compilers and other such tools targeting the cross
   # platform. Then, `forceNativeDrv` can be removed.
   (vanillaPackages: {
-    inherit system platform crossSystem config;
+    inherit system platform crossSystem config overlays;
     # It's OK to change the built-time dependencies
     allowCustomOverrides = true;
     stdenv = vanillaPackages.stdenv // {
@@ -30,7 +30,7 @@ in bootStages ++ [
 
   # Run packages
   (buildPackages: {
-    inherit system platform crossSystem config;
+    inherit system platform crossSystem config overlays;
     stdenv = if crossSystem.useiOSCross or false
       then let
           inherit (buildPackages.darwin.ios-cross {
diff --git a/pkgs/stdenv/custom/default.nix b/pkgs/stdenv/custom/default.nix
index 59a6e871cfdec..d7e9bf53bed1b 100644
--- a/pkgs/stdenv/custom/default.nix
+++ b/pkgs/stdenv/custom/default.nix
@@ -1,12 +1,12 @@
 { lib
-, system, platform, crossSystem, config
+, system, platform, crossSystem, config, overlays
 }:
 
 assert crossSystem == null;
 
 let
   bootStages = import ../. {
-    inherit lib system platform crossSystem;
+    inherit lib system platform crossSystem overlays;
     # Remove config.replaceStdenv to ensure termination.
     config = builtins.removeAttrs config [ "replaceStdenv" ];
   };
@@ -15,7 +15,7 @@ in bootStages ++ [
 
   # Additional stage, built using custom stdenv
   (vanillaPackages: {
-    inherit system platform crossSystem config;
+    inherit system platform crossSystem config overlays;
     stdenv = config.replaceStdenv { pkgs = vanillaPackages; };
   })
 
diff --git a/pkgs/stdenv/darwin/default.nix b/pkgs/stdenv/darwin/default.nix
index 9c860bfb0f927..e3a87ea078fca 100644
--- a/pkgs/stdenv/darwin/default.nix
+++ b/pkgs/stdenv/darwin/default.nix
@@ -1,5 +1,5 @@
 { lib
-, system, platform, crossSystem, config
+, system, platform, crossSystem, config, overlays
 
 # Allow passing in bootstrap files directly so we can test the stdenv bootstrap process when changing the bootstrap tools
 , bootstrapFiles ? let
@@ -98,7 +98,7 @@ in rec {
       };
 
     in {
-      inherit system platform crossSystem config;
+      inherit system platform crossSystem config overlays;
       stdenv = thisStdenv;
     };
 
@@ -316,7 +316,7 @@ in rec {
     stage3
     stage4
     (prevStage: {
-      inherit system crossSystem platform config;
+      inherit system crossSystem platform config overlays;
       stdenv = stdenvDarwin prevStage;
     })
   ];
diff --git a/pkgs/stdenv/default.nix b/pkgs/stdenv/default.nix
index 0ca03ecdde189..f60ffec4b564e 100644
--- a/pkgs/stdenv/default.nix
+++ b/pkgs/stdenv/default.nix
@@ -7,7 +7,7 @@
 { # Args just for stdenvs' usage
   lib
   # Args to pass on to the pkgset builder, too
-, system, platform, crossSystem, config
+, system, platform, crossSystem, config, overlays
 } @ args:
 
 let
diff --git a/pkgs/stdenv/freebsd/default.nix b/pkgs/stdenv/freebsd/default.nix
index e0256e26f5fb5..2cb059deb34ba 100644
--- a/pkgs/stdenv/freebsd/default.nix
+++ b/pkgs/stdenv/freebsd/default.nix
@@ -1,5 +1,5 @@
 { lib
-, system, platform, crossSystem, config
+, system, platform, crossSystem, config, overlays
 }:
 
 assert crossSystem == null;
@@ -58,7 +58,7 @@ assert crossSystem == null;
   })
 
   (prevStage: {
-    inherit system crossSystem platform config;
+    inherit system crossSystem platform config overlays;
     stdenv = import ../generic {
       name = "stdenv-freebsd-boot-3";
       inherit system config;
diff --git a/pkgs/stdenv/linux/default.nix b/pkgs/stdenv/linux/default.nix
index 430122c36f7b7..bc8a03a485a03 100644
--- a/pkgs/stdenv/linux/default.nix
+++ b/pkgs/stdenv/linux/default.nix
@@ -4,7 +4,7 @@
 # compiler and linker that do not search in default locations,
 # ensuring purity of components produced by it.
 { lib
-, system, platform, crossSystem, config
+, system, platform, crossSystem, config, overlays
 
 , bootstrapFiles ?
     if system == "i686-linux" then import ./bootstrap-files/i686.nix
@@ -91,7 +91,7 @@ let
       };
 
     in {
-      inherit system platform crossSystem config;
+      inherit system platform crossSystem config overlays;
       stdenv = thisStdenv;
     };
 
diff --git a/pkgs/stdenv/native/default.nix b/pkgs/stdenv/native/default.nix
index 57182fae5236e..4028638009e1e 100644
--- a/pkgs/stdenv/native/default.nix
+++ b/pkgs/stdenv/native/default.nix
@@ -1,5 +1,5 @@
 { lib
-, system, platform, crossSystem, config
+, system, platform, crossSystem, config, overlays
 }:
 
 assert crossSystem == null;
@@ -134,7 +134,7 @@ in
 
   # First build a stdenv based only on tools outside the store.
   (prevStage: {
-    inherit system crossSystem platform config;
+    inherit system crossSystem platform config overlays;
     stdenv = makeStdenv {
       inherit (prevStage) cc fetchurl;
     } // { inherit (prevStage) fetchurl; };
@@ -143,7 +143,7 @@ in
   # Using that, build a stdenv that adds the ‘xz’ command (which most systems
   # don't have, so we mustn't rely on the native environment providing it).
   (prevStage: {
-    inherit system crossSystem platform config;
+    inherit system crossSystem platform config overlays;
     stdenv = makeStdenv {
       inherit (prevStage.stdenv) cc fetchurl;
       extraPath = [ prevStage.xz ];
diff --git a/pkgs/top-level/default.nix b/pkgs/top-level/default.nix
index 04daf9778ffa1..a146dad63bc89 100644
--- a/pkgs/top-level/default.nix
+++ b/pkgs/top-level/default.nix
@@ -23,6 +23,9 @@
 , # Allow a configuration attribute set to be passed in as an argument.
   config ? {}
 
+, # List of overlays layers used to extend Nixpkgs.
+  overlays ? []
+
 , # A function booting the final package set for a specific standard
   # environment. See below for the arguments given to that function,
   # the type of list it returns.
@@ -80,7 +83,7 @@ in let
   boot = import ../stdenv/booter.nix { inherit lib allPackages; };
 
   stages = stdenvStages {
-    inherit lib system platform crossSystem config;
+    inherit lib system platform crossSystem config overlays;
   };
 
   pkgs = boot stages;
diff --git a/pkgs/top-level/impure.nix b/pkgs/top-level/impure.nix
index e90668159279b..1aa55f7293ba4 100644
--- a/pkgs/top-level/impure.nix
+++ b/pkgs/top-level/impure.nix
@@ -18,7 +18,24 @@
       else if homeDir != "" && pathExists configFile2 then import configFile2
       else {}
 
+, # Overlays are used to extend Nixpkgs collection with additional
+  # collections of packages.  These collection of packages are part of the
+  # fix-point made by Nixpkgs.
+  overlays ? let
+      inherit (builtins) getEnv pathExists readDir attrNames map sort
+        lessThan;
+      dirEnv = getEnv "NIXPKGS_OVERLAYS";
+      dirHome = (getEnv "HOME") + "/.nixpkgs/overlays";
+      dirCheck = dir: dir != "" && pathExists (dir + "/.");
+      overlays = dir:
+        let content = readDir dir; in
+        map (n: import "${dir}/${n}") (sort lessThan (attrNames content));
+    in
+      if dirCheck dirEnv then overlays dirEnv
+      else if dirCheck dirHome then overlays dirHome
+      else []
+
 , ...
 } @ args:
 
-import ./. (args // { inherit system config; })
+import ./. (args // { inherit system config overlays; })
diff --git a/pkgs/top-level/stage.nix b/pkgs/top-level/stage.nix
index ebc6473e42548..cbf65870eb7e8 100644
--- a/pkgs/top-level/stage.nix
+++ b/pkgs/top-level/stage.nix
@@ -30,6 +30,8 @@
 , # The configuration attribute set
   config
 
+, overlays # List of overlays to use in the fix-point.
+
 , crossSystem
 , platform
 , lib
@@ -79,7 +81,7 @@ let
       ((config.packageOverrides or (super: {})) super);
 
   # The complete chain of package set builders, applied from top to bottom
-  toFix = lib.foldl' (lib.flip lib.extends) (self: {}) [
+  toFix = lib.foldl' (lib.flip lib.extends) (self: {}) ([
     stdenvBootstappingAndPlatforms
     stdenvAdapters
     trivialBuilders
@@ -87,20 +89,8 @@ let
     aliases
     stdenvOverrides
     configOverrides
-  ];
-
-  # Use `overridePackages` to easily override this package set.
-  # Warning: this function is very expensive and must not be used
-  # from within the nixpkgs repository.
-  #
-  # Example:
-  #  pkgs.overridePackages (self: super: {
-  #    foo = super.foo.override { ... };
-  #  }
-  #
-  # The result is `pkgs' where all the derivations depending on `foo'
-  # will use the new version.
-
-  # Return the complete set of packages. Warning: this function is very
-  # expensive!
-in lib.makeExtensibleWithCustomName "overridePackages" toFix
+  ] ++ overlays);
+
+in
+  # Return the complete set of packages.
+  lib.fix toFix