gerbil-support: various improvements

Add an indirection between gerbil package parameters and nix packages
computed from those parameters, so overlays can easily override the former.

Run unit-tests of gerbil packages (not integration tests).

Fix binaries for gerbil packages.

Fix support for gerbil packages including binaries.

9 files changed, 145 insertions, 69 deletions

diff --git a/pkgs/development/compilers/gerbil/build.nix b/pkgs/development/compilers/gerbil/build.nix
index 110378244fd00..227f11b7c5843 100644
--- a/pkgs/development/compilers/gerbil/build.nix
+++ b/pkgs/development/compilers/gerbil/build.nix
@@ -29,7 +29,7 @@ stdenv.mkDerivation rec {
     grep -Fl '#!/usr/bin/env' `find . -type f -executable` | while read f ; do
       substituteInPlace "$f" --replace '#!/usr/bin/env' '#!${coreutils}/bin/env' ;
     done ;
-'';
+  '';
 
 ## TODO: make static compilation work.
 ## For that, get all the packages below to somehow expose static libraries,
@@ -92,8 +92,8 @@ stdenv.mkDerivation rec {
   meta = {
     description = "Gerbil Scheme";
     homepage    = "https://github.com/vyzo/gerbil";
-    license     = lib.licenses.lgpl21; # also asl20, like Gambit
-    # NB regarding platforms: regularly tested on Linux, only occasionally on macOS.
+    license     = lib.licenses.lgpl21Only; # dual, also asl20, like Gambit
+    # NB regarding platforms: regularly tested on Linux and on macOS.
     # Please report success and/or failure to fare.
     platforms   = lib.platforms.unix;
     maintainers = with lib.maintainers; [ fare ];
diff --git a/pkgs/development/compilers/gerbil/gerbil-crypto.nix b/pkgs/development/compilers/gerbil/gerbil-crypto.nix
index 38943471a376d..ff211cc610dcf 100644
--- a/pkgs/development/compilers/gerbil/gerbil-crypto.nix
+++ b/pkgs/development/compilers/gerbil/gerbil-crypto.nix
@@ -1,6 +1,5 @@
 { pkgs, lib, fetchFromGitHub, gerbil-unstable, gerbil-support, gambit-support }:
-
-gerbil-support.gerbilPackage {
+{
   pname = "gerbil-crypto";
   version = "unstable-2020-08-01";
   git-version = "0.0-6-ga228862";
@@ -12,7 +11,8 @@ gerbil-support.gerbilPackage {
   gambit-params = gambit-support.unstable-params;
   version-path = "version";
   softwareName = "Gerbil-crypto";
-  src = fetchFromGitHub {
+  pre-src = {
+    fun = fetchFromGitHub;
     owner = "fare";
     repo = "gerbil-crypto";
     rev = "a22886260849ec92c3a34bfeedc1574e41e49e33";
diff --git a/pkgs/development/compilers/gerbil/gerbil-ethereum.nix b/pkgs/development/compilers/gerbil/gerbil-ethereum.nix
index e498818d2e611..231162ede913f 100644
--- a/pkgs/development/compilers/gerbil/gerbil-ethereum.nix
+++ b/pkgs/development/compilers/gerbil/gerbil-ethereum.nix
@@ -1,6 +1,5 @@
 { pkgs, lib, fetchFromGitHub, gerbil-unstable, gerbil-support, gambit-support }:
-
-gerbil-support.gerbilPackage {
+{
   pname = "gerbil-ethereum";
   version = "unstable-2020-10-18";
   git-version = "0.0-26-gf27ada8";
@@ -12,7 +11,8 @@ gerbil-support.gerbilPackage {
   gambit-params = gambit-support.unstable-params;
   version-path = "version";
   softwareName = "Gerbil-ethereum";
-  src = fetchFromGitHub {
+  pre-src = {
+    fun = fetchFromGitHub;
     owner = "fare";
     repo = "gerbil-ethereum";
     rev = "f27ada8e7f4de4f8fbdfede9fe055914b254d8e7";
diff --git a/pkgs/development/compilers/gerbil/gerbil-libp2p.nix b/pkgs/development/compilers/gerbil/gerbil-libp2p.nix
index 336f5af89c3bb..78ca2a113a817 100644
--- a/pkgs/development/compilers/gerbil/gerbil-libp2p.nix
+++ b/pkgs/development/compilers/gerbil/gerbil-libp2p.nix
@@ -1,6 +1,5 @@
 { pkgs, lib, fetchFromGitHub, gerbil-unstable, gerbil-support, gambit-support }:
-
-gerbil-support.gerbilPackage {
+{
   pname = "gerbil-libp2p";
   version = "unstable-2018-12-27";
   git-version = "2376b3f";
@@ -11,7 +10,8 @@ gerbil-support.gerbilPackage {
   gambit-params = gambit-support.unstable-params;
   version-path = "version";
   softwareName = "Gerbil-libp2p";
-  src = fetchFromGitHub {
+  pre-src = {
+    fun = fetchFromGitHub;
     owner = "vyzo";
     repo = "gerbil-libp2p";
     rev = "2376b3f39cee04dd4ec455c8ea4e5faa93c2bf88";
diff --git a/pkgs/development/compilers/gerbil/gerbil-persist.nix b/pkgs/development/compilers/gerbil/gerbil-persist.nix
index 74485c19c2d0a..b9d19dc3711d0 100644
--- a/pkgs/development/compilers/gerbil/gerbil-persist.nix
+++ b/pkgs/development/compilers/gerbil/gerbil-persist.nix
@@ -1,6 +1,5 @@
 { pkgs, lib, fetchFromGitHub, gerbil-unstable, gerbil-support, gambit-support }:
-
-gerbil-support.gerbilPackage {
+{
   pname = "gerbil-persist";
   version = "unstable-2020-08-31";
   git-version = "0.0-8-gd211390";
@@ -11,7 +10,8 @@ gerbil-support.gerbilPackage {
   gambit-params = gambit-support.unstable-params;
   version-path = "version";
   softwareName = "Gerbil-persist";
-  src = fetchFromGitHub {
+  pre-src = {
+    fun = fetchFromGitHub;
     owner = "fare";
     repo = "gerbil-persist";
     rev = "d211390c8a199cf2b8c7400cd98977524e960015";
diff --git a/pkgs/development/compilers/gerbil/gerbil-poo.nix b/pkgs/development/compilers/gerbil/gerbil-poo.nix
index c6580ea0c77a6..fd98780155096 100644
--- a/pkgs/development/compilers/gerbil/gerbil-poo.nix
+++ b/pkgs/development/compilers/gerbil/gerbil-poo.nix
@@ -1,17 +1,17 @@
 { pkgs, lib, fetchFromGitHub, gerbil-unstable, gerbil-support, gambit-support }:
-
-gerbil-support.gerbilPackage {
-  pname = "gerbil-ethereum";
+{
+  pname = "gerbil-poo";
+  gerbil-package = "clan/poo";
   version = "unstable-2020-10-17";
   git-version = "0.0-35-g44d490d";
-  gerbil-package = "clan/poo";
   gerbil = gerbil-unstable;
   gerbilInputs = with gerbil-support.gerbilPackages-unstable; [gerbil-utils gerbil-crypto];
   buildInputs = [];
   gambit-params = gambit-support.unstable-params;
   version-path = "version";
   softwareName = "Gerbil-POO";
-  src = fetchFromGitHub {
+  pre-src = {
+    fun = fetchFromGitHub;
     owner = "fare";
     repo = "gerbil-poo";
     rev = "44d490d95b9d1b5d54eaedf2602419af8e086837";
diff --git a/pkgs/development/compilers/gerbil/gerbil-support.nix b/pkgs/development/compilers/gerbil/gerbil-support.nix
index f51076fd5d4d1..b628c6a48f34b 100644
--- a/pkgs/development/compilers/gerbil/gerbil-support.nix
+++ b/pkgs/development/compilers/gerbil/gerbil-support.nix
@@ -1,54 +1,112 @@
-{ pkgs, lib, gccStdenv, callPackage, fetchFromGitHub }:
-# See ../gambit/build.nix regarding gccStdenv
-
-rec {
-  # Gerbil libraries
-  gerbilPackages-unstable = {
-    gerbil-libp2p = callPackage ./gerbil-libp2p.nix { };
-    gerbil-utils = callPackage ./gerbil-utils.nix { };
-    gerbil-crypto = callPackage ./gerbil-crypto.nix { };
-    gerbil-poo = callPackage ./gerbil-poo.nix { };
-    gerbil-persist = callPackage ./gerbil-persist.nix { };
-    gerbil-ethereum = callPackage ./gerbil-ethereum.nix { };
-    smug-gerbil = callPackage ./smug-gerbil.nix { };
+{ pkgs, lib, callPackage, ... }:
+
+  with pkgs.gerbil-support; {
+
+  prePackages-unstable =
+    let pks = [ ./gerbil-libp2p.nix ./smug-gerbil.nix
+                ./gerbil-utils.nix ./gerbil-crypto.nix ./gerbil-poo.nix
+                ./gerbil-persist.nix ./gerbil-ethereum.nix ];
+        call = pkg: callPackage pkg prePackage-defaults;
+        pkgName = pkg: lib.removeSuffix ".nix" (baseNameOf pkg);
+        f = pkg: { name = pkgName pkg; value = call pkg; }; in
+    builtins.listToAttrs (map f pks);
+
+  prePackage-defaults = {
+    gerbil = pkgs.gerbil-unstable;
+    gambit-params = pkgs.gambit-support.unstable-params;
+    gerbilPackages = gerbilPackages-unstable;
+    git-version = "";
+    version-path = "";
+    gerbilInputs = [];
+    nativeBuildInputs = [];
+    buildInputs = [];
+    buildScript = "./build.ss";
+    postInstall = "";
+    softwareName = "";
   };
 
+  gerbilPackages-unstable =
+    builtins.mapAttrs (_: gerbilPackage) prePackages-unstable;
+
+  resolve-pre-src = pre-src: pre-src.fun (removeAttrs pre-src ["fun"]);
+
+  gerbilVersionFromGit = pkg:
+    let version-path = "${pkg.passthru.pre-pkg.version-path}.ss"; in
+    if builtins.pathExists version-path then
+      let m =
+        builtins.match "\\(import :clan/versioning.*\\)\n\\(register-software \"([-_.A-Za-z0-9]+)\" \"([-_.A-Za-z0-9]+)\"\\) ;; ([-0-9]+)\n"
+          (builtins.readFile version-path); in
+          { version = builtins.elemAt m 2; git-version = builtins.elemAt m 1; }
+     else { version = "0.0";
+            git-version = let gitpath = "${toString pkg.src}/.git"; in
+              if builtins.pathExists gitpath then lib.commitIdFromGitRepo gitpath else "0"; };
+
+  gerbilSkippableFiles = [".git" ".build" ".build_outputs" "run" "result" "dep" "BLAH"
+                          "version.ss" "tmp.nix"];
+
+  gerbilSourceFilter = path: type:
+    let baseName = baseNameOf path; in
+      ! (builtins.elem baseName gerbilSkippableFiles || lib.hasSuffix "~" baseName);
+
+  gerbilFilterSource = builtins.filterSource gerbilSourceFilter;
+
   # Use this function in any package that uses Gerbil libraries, to define the GERBIL_LOADPATH.
   gerbilLoadPath =
-    gerbilInputs : builtins.concatStringsSep ":" (map (x : x + "/gerbil/lib") gerbilInputs);
+    gerbilInputs: builtins.concatStringsSep ":" (map (x: x + "/gerbil/lib") gerbilInputs);
+
+  path-src = path: { fun = _: path; };
+
+  view = lib.debug.traceSeqN 4;
+
+  sha256-of-pre-src = pre-src: if pre-src ? sha256 then pre-src.sha256 else "none";
+
+  overrideSrcIfShaDiff = name: new-pre-src: super:
+    let old-sha256 = sha256-of-pre-src super.${name}.pre-src;
+        new-sha256 = sha256-of-pre-src new-pre-src; in
+    if old-sha256 == new-sha256 then {} else
+    view "Overriding ${name} old-sha256: ${old-sha256} new-sha256: ${new-sha256}"
+    { ${name} = super.${name} // {
+        pre-src = new-pre-src;
+        version = "override";
+        git-version = if new-pre-src ? rev then lib.substring 0 7 new-pre-src.rev else "unknown";};};
+
+  pkgsOverrideGerbilPackageSrc = name: pre-src: pkgs: super: {
+    gerbil-support = (super-support:
+      { prePackages-unstable =
+          (super-ppu: super-ppu // (overrideSrcIfShaDiff name pre-src super-ppu))
+          super-support.prePackages-unstable;}) super.gerbil-support;};
 
   # Use this function to create a Gerbil library. See gerbil-utils as an example.
-  gerbilPackage = {
-    pname, version, src, meta, gerbil-package,
-    git-version ? "", version-path ? "",
-    gerbil ? pkgs.gerbil-unstable,
-    gambit-params ? pkgs.gambit-support.stable-params,
-    gerbilInputs ? [],
-    nativeBuildInputs ? [],
-    buildInputs ? [],
-    buildScript ? "./build.ss",
-    softwareName ? ""} :
-    let buildInputs_ = buildInputs; in
-    gccStdenv.mkDerivation rec {
-      inherit src meta pname version nativeBuildInputs;
-      passthru = { inherit gerbil-package version-path ;};
+  gerbilPackage = prePackage:
+    let pre-pkg = prePackage-defaults // prePackage;
+        inherit (pre-pkg) pname version pre-src git-version meta
+          softwareName gerbil-package version-path gerbil gambit-params
+          gerbilInputs nativeBuildInputs buildInputs buildScript postInstall;
+        buildInputs_ = buildInputs; in
+    pkgs.gccStdenv.mkDerivation rec { # See ../gambit/build.nix regarding why we use gccStdenv
+      inherit meta pname version nativeBuildInputs postInstall;
+      passthru = {
+        inherit pre-pkg;
+      };
+      src = resolve-pre-src pre-src;
       buildInputs = [ gerbil ] ++ gerbilInputs ++ buildInputs_;
+
       postPatch = ''
         set -e ;
-        if [ -n "${version-path}.ss" ] ; then
-          echo -e '(import :clan/versioning${builtins.concatStringsSep ""
-                     (map (x : lib.optionalString (x.passthru.version-path != "")
-                               " :${x.passthru.gerbil-package}/${x.passthru.version-path}")
+        ${lib.optionalString (version-path != "")
+          ''echo -e '(import :clan/versioning${builtins.concatStringsSep ""
+                     (map (x: let px = x.passthru.pre-pkg; in
+                              lib.optionalString (px.version-path != "")
+                                " :${px.gerbil-package}/${px.version-path}")
                           gerbilInputs)
-                     })\n(register-software "${softwareName}" "v${git-version}")\n' > "${passthru.version-path}.ss"
-        fi
+                     })\n(register-software "${softwareName}" "v${git-version}")\n' > "${version-path}.ss"''}
         patchShebangs . ;
       '';
 
       postConfigure = ''
         export GERBIL_BUILD_CORES=$NIX_BUILD_CORES
         export GERBIL_PATH=$PWD/.build
-        export GERBIL_LOADPATH=${gerbilLoadPath gerbilInputs}
+        export GERBIL_LOADPATH=${gerbilLoadPath (["$out"] ++ gerbilInputs)}
         ${pkgs.gambit-support.export-gambopt gambit-params}
       '';
 
@@ -60,18 +118,36 @@ rec {
 
       installPhase = ''
         runHook preInstall
-        mkdir -p $out/gerbil/lib
-        cp -fa .build/lib $out/gerbil/
-        bins=(.build/bin/*)
-        if [ 0 -lt ''${#bins} ] ; then
-          cp -fa .build/bin $out/gerbil/
-          mkdir $out/bin
-          cd $out/bin
-          ln -s ../gerbil/bin/* .
+        mkdir -p $out/gerbil
+        cp -fa .build/* $out/gerbil/
+        if [[ -d $out/gerbil/bin ]] ; then
+          ( cd $out/gerbil
+            bins=$(find ../gerbil/bin -type f)
+            if [[ -n $bins ]] ; then
+              ( mkdir -p ../bin
+                cd ..
+                ln -s $bins bin
+              )
+            fi
+          )
         fi
         runHook postInstall
       '';
 
       dontFixup = true;
+
+      checkPhase = ''
+        runHook preCheck
+        if [[ -f unit-tests.ss ]] ; then
+          export GERBIL_APPLICATION_HOME=$PWD
+          ./unit-tests.ss version
+          ./unit-tests.ss
+        else
+          echo "No gerbil-utils style unit-tests.ss detected for ${pname} ${version}.";
+        fi
+        runHook postCheck
+      '';
+
+      doCheck = true;
     };
 }
diff --git a/pkgs/development/compilers/gerbil/gerbil-utils.nix b/pkgs/development/compilers/gerbil/gerbil-utils.nix
index 6d89e1181f2d3..9a31b6a96c74d 100644
--- a/pkgs/development/compilers/gerbil/gerbil-utils.nix
+++ b/pkgs/development/compilers/gerbil/gerbil-utils.nix
@@ -1,6 +1,5 @@
 { lib, fetchFromGitHub, gerbil-unstable, gerbil-support, gambit-support }:
-
-gerbil-support.gerbilPackage {
+{
   pname = "gerbil-utils";
   version = "unstable-2020-10-18";
   git-version = "0.2-36-g8b481b7";
@@ -9,7 +8,8 @@ gerbil-support.gerbilPackage {
   gambit-params = gambit-support.unstable-params;
   version-path = "version";
   softwareName = "Gerbil-utils";
-  src = fetchFromGitHub {
+  pre-src = {
+    fun = fetchFromGitHub;
     owner = "fare";
     repo = "gerbil-utils";
     rev = "8b481b787e13e07e14d0718d670aab016131a090";
diff --git a/pkgs/development/compilers/gerbil/smug-gerbil.nix b/pkgs/development/compilers/gerbil/smug-gerbil.nix
index 732cdb500b9b9..1b6923451e681 100644
--- a/pkgs/development/compilers/gerbil/smug-gerbil.nix
+++ b/pkgs/development/compilers/gerbil/smug-gerbil.nix
@@ -1,6 +1,5 @@
 { pkgs, lib, fetchFromGitHub, gerbil-unstable, gerbil-support, gambit-support }:
-
-gerbil-support.gerbilPackage {
+{
   pname = "smug-gerbil";
   version = "unstable-2019-12-24";
   git-version = "95d60d4";
@@ -11,7 +10,8 @@ gerbil-support.gerbilPackage {
   gambit-params = gambit-support.unstable-params;
   version-path = ""; #"version";
   softwareName = "Smug-Gerbil";
-  src = fetchFromGitHub {
+  pre-src = {
+    fun = fetchFromGitHub;
     owner = "drewc";
     repo = "smug-gerbil";
     rev = "95d60d486c1603743c6d3c525e6d5f5761b984e5";