Merge pull request #244835 from zimbatm/data-writers

writers: add data-centric writers

4 files changed, 585 insertions, 441 deletions

diff --git a/pkgs/build-support/writers/data.nix b/pkgs/build-support/writers/data.nix
new file mode 100644
index 0000000000000..a080f8c29d0e0
--- /dev/null
+++ b/pkgs/build-support/writers/data.nix
@@ -0,0 +1,80 @@
+{ lib, runCommandNoCC, dasel }:
+let
+  daselBin = lib.getExe dasel;
+
+  inherit (lib)
+    last
+    optionalString
+    types
+    ;
+in
+rec {
+  # Creates a transformer function that writes input data to disk, transformed
+  # by both the `input` and `output` arguments.
+  #
+  # Type: makeDataWriter :: input -> output -> nameOrPath -> data -> (any -> string) -> string -> string -> any -> derivation
+  #
+  #   input :: T -> string: function that takes the nix data and returns a string
+  #   output :: string: script that takes the $inputFile and write the result into $out
+  #   nameOrPath :: string: if the name contains a / the files gets written to a sub-folder of $out. The derivation name is the basename of this argument.
+  #   data :: T: the data that will be converted.
+  #
+  # Example:
+  #   writeJSON = makeDataWriter { input = builtins.toJSON; output = "cp $inputPath $out"; };
+  #   myConfig = writeJSON "config.json" { hello = "world"; }
+  #
+  makeDataWriter = { input ? lib.id, output ? "cp $inputPath $out" }: nameOrPath: data:
+    assert lib.or (types.path.check nameOrPath) (builtins.match "([0-9A-Za-z._])[0-9A-Za-z._-]*" nameOrPath != null);
+    let
+      name = last (builtins.split "/" nameOrPath);
+    in
+    runCommandNoCC name
+      {
+        input = input data;
+        passAsFile = [ "input" ];
+      } ''
+      ${output}
+
+      ${optionalString (types.path.check nameOrPath) ''
+        mv $out tmp
+        mkdir -p $out/$(dirname "${nameOrPath}")
+        mv tmp $out/${nameOrPath}
+      ''}
+    '';
+
+  # Writes the content to text.
+  #
+  # Example:
+  #   writeText "filename.txt" "file content"
+  writeText = makeDataWriter {
+    input = toString;
+    output = "cp $inputPath $out";
+  };
+
+  # Writes the content to a JSON file.
+  #
+  # Example:
+  #   writeJSON "data.json" { hello = "world"; }
+  writeJSON = makeDataWriter {
+    input = builtins.toJSON;
+    output = "${daselBin} -f $inputPath -r json -w json > $out";
+  };
+
+  # Writes the content to a TOML file.
+  #
+  # Example:
+  #   writeTOML "data.toml" { hello = "world"; }
+  writeTOML = makeDataWriter {
+    input = builtins.toJSON;
+    output = "${daselBin} -f $inputPath -r json -w toml > $out";
+  };
+
+  # Writes the content to a YAML file.
+  #
+  # Example:
+  #   writeYAML "data.yaml" { hello = "world"; }
+  writeYAML = makeDataWriter {
+    input = builtins.toJSON;
+    output = "${daselBin} -f $inputPath -r json -w yaml > $out";
+  };
+}
diff --git a/pkgs/build-support/writers/default.nix b/pkgs/build-support/writers/default.nix
index 3016a45b8a0f1..5ef5794221953 100644
--- a/pkgs/build-support/writers/default.nix
+++ b/pkgs/build-support/writers/default.nix
@@ -1,367 +1,18 @@
-{ pkgs, config, buildPackages, lib, stdenv, libiconv, mkNugetDeps, mkNugetSource, gixy }:
+{ pkgs, config, lib }:
 
 let
   aliases = if config.allowAliases then (import ./aliases.nix lib) else prev: {};
 
-  writers = with lib; rec {
-  # Base implementation for non-compiled executables.
-  # Takes an interpreter, for example `${pkgs.bash}/bin/bash`
-  #
-  # Examples:
-  #   writeBash = makeScriptWriter { interpreter = "${pkgs.bash}/bin/bash"; }
-  #   makeScriptWriter { interpreter = "${pkgs.dash}/bin/dash"; } "hello" "echo hello world"
-  makeScriptWriter = { interpreter, check ? "" }: nameOrPath: content:
-    assert lib.or (types.path.check nameOrPath) (builtins.match "([0-9A-Za-z._])[0-9A-Za-z._-]*" nameOrPath != null);
-    assert lib.or (types.path.check content) (types.str.check content);
-    let
-      name = last (builtins.split "/" nameOrPath);
-    in
-
-    pkgs.runCommandLocal name (if (types.str.check content) then {
-      inherit content interpreter;
-      passAsFile = [ "content" ];
-    } else {
-      inherit interpreter;
-      contentPath = content;
-    }) ''
-      # On darwin a script cannot be used as an interpreter in a shebang but
-      # there doesn't seem to be a limit to the size of shebang and multiple
-      # arguments to the interpreter are allowed.
-      if [[ -n "${toString pkgs.stdenvNoCC.isDarwin}" ]] && isScript $interpreter
-      then
-        wrapperInterpreterLine=$(head -1 "$interpreter" | tail -c+3)
-        # Get first word from the line (note: xargs echo remove leading spaces)
-        wrapperInterpreter=$(echo "$wrapperInterpreterLine" | xargs echo | cut -d " " -f1)
-
-        if isScript $wrapperInterpreter
-        then
-          echo "error: passed interpreter ($interpreter) is a script which has another script ($wrapperInterpreter) as an interpreter, which is not supported."
-          exit 1
-        fi
-
-        # This should work as long as wrapperInterpreter is a shell, which is
-        # the case for programs wrapped with makeWrapper, like
-        # python3.withPackages etc.
-        interpreterLine="$wrapperInterpreterLine $interpreter"
-      else
-        interpreterLine=$interpreter
-      fi
-
-      echo "#! $interpreterLine" > $out
-      cat "$contentPath" >> $out
-      ${optionalString (check != "") ''
-        ${check} $out
-      ''}
-      chmod +x $out
-      ${optionalString (types.path.check nameOrPath) ''
-        mv $out tmp
-        mkdir -p $out/$(dirname "${nameOrPath}")
-        mv tmp $out/${nameOrPath}
-      ''}
-    '';
-
-  # Base implementation for compiled executables.
-  # Takes a compile script, which in turn takes the name as an argument.
-  #
-  # Examples:
-  #   writeSimpleC = makeBinWriter { compileScript = name: "gcc -o $out $contentPath"; }
-  makeBinWriter = { compileScript, strip ? true }: nameOrPath: content:
-    assert lib.or (types.path.check nameOrPath) (builtins.match "([0-9A-Za-z._])[0-9A-Za-z._-]*" nameOrPath != null);
-    assert lib.or (types.path.check content) (types.str.check content);
-    let
-      name = last (builtins.split "/" nameOrPath);
-    in
-    pkgs.runCommand name ((if (types.str.check content) then {
-      inherit content;
-      passAsFile = [ "content" ];
-    } else {
-      contentPath = content;
-    }) // lib.optionalAttrs (stdenv.hostPlatform.isDarwin && stdenv.hostPlatform.isAarch64) {
-      # post-link-hook expects codesign_allocate to be in PATH
-      # https://github.com/NixOS/nixpkgs/issues/154203
-      # https://github.com/NixOS/nixpkgs/issues/148189
-      nativeBuildInputs = [ stdenv.cc.bintools ];
-    }) ''
-      ${compileScript}
-      ${lib.optionalString strip
-          "${lib.getBin buildPackages.bintools-unwrapped}/bin/${buildPackages.bintools-unwrapped.targetPrefix}strip -S $out"}
-      # Sometimes binaries produced for darwin (e. g. by GHC) won't be valid
-      # mach-o executables from the get-go, but need to be corrected somehow
-      # which is done by fixupPhase.
-      ${lib.optionalString pkgs.stdenvNoCC.hostPlatform.isDarwin "fixupPhase"}
-      ${optionalString (types.path.check nameOrPath) ''
-        mv $out tmp
-        mkdir -p $out/$(dirname "${nameOrPath}")
-        mv tmp $out/${nameOrPath}
-      ''}
-    '';
-
-  # Like writeScript but the first line is a shebang to bash
-  #
-  # Example:
-  #   writeBash "example" ''
-  #     echo hello world
-  #   ''
-  writeBash = makeScriptWriter {
-    interpreter = "${pkgs.bash}/bin/bash";
-  };
-
-  # Like writeScriptBin but the first line is a shebang to bash
-  writeBashBin = name:
-    writeBash "/bin/${name}";
-
-  # Like writeScript but the first line is a shebang to dash
-  #
-  # Example:
-  #   writeDash "example" ''
-  #     echo hello world
-  #   ''
-  writeDash = makeScriptWriter {
-    interpreter = "${pkgs.dash}/bin/dash";
+  # Writers for JSON-like data structures
+  dataWriters = import ./data.nix {
+    inherit lib; inherit (pkgs) runCommandNoCC dasel;
   };
 
-  # Like writeScriptBin but the first line is a shebang to dash
-  writeDashBin = name:
-    writeDash "/bin/${name}";
-
-  # Like writeScript but the first line is a shebang to fish
-  #
-  # Example:
-  #   writeFish "example" ''
-  #     echo hello world
-  #   ''
-  writeFish = makeScriptWriter {
-    interpreter = "${pkgs.fish}/bin/fish --no-config";
-    check = "${pkgs.fish}/bin/fish --no-config --no-execute";  # syntax check only
+  # Writers for scripts
+  scriptWriters = import ./scripts.nix {
+    inherit lib pkgs;
   };
 
-  # Like writeScriptBin but the first line is a shebang to fish
-  writeFishBin = name:
-    writeFish "/bin/${name}";
-
-  # writeHaskell takes a name, an attrset with libraries and haskell version (both optional)
-  # and some haskell source code and returns an executable.
-  #
-  # Example:
-  #   writeHaskell "missiles" { libraries = [ pkgs.haskellPackages.acme-missiles ]; } ''
-  #     import Acme.Missiles
-  #
-  #     main = launchMissiles
-  #   '';
-  writeHaskell = name: {
-    libraries ? [],
-    ghc ? pkgs.ghc,
-    ghcArgs ? [],
-    threadedRuntime ? true,
-    strip ? true
-  }:
-    let
-      appendIfNotSet = el: list: if elem el list then list else list ++ [ el ];
-      ghcArgs' = if threadedRuntime then appendIfNotSet "-threaded" ghcArgs else ghcArgs;
-
-    in makeBinWriter {
-      compileScript = ''
-        cp $contentPath tmp.hs
-        ${ghc.withPackages (_: libraries )}/bin/ghc ${lib.escapeShellArgs ghcArgs'} tmp.hs
-        mv tmp $out
-      '';
-      inherit strip;
-    } name;
-
-  # writeHaskellBin takes the same arguments as writeHaskell but outputs a directory (like writeScriptBin)
-  writeHaskellBin = name:
-    writeHaskell "/bin/${name}";
-
-  writeRust = name: {
-      rustc ? pkgs.rustc,
-      rustcArgs ? [],
-      strip ? true
-  }:
-  let
-    darwinArgs = lib.optionals stdenv.isDarwin [ "-L${lib.getLib libiconv}/lib" ];
-  in
-    makeBinWriter {
-      compileScript = ''
-        cp "$contentPath" tmp.rs
-        PATH=${makeBinPath [pkgs.gcc]} ${lib.getBin rustc}/bin/rustc ${lib.escapeShellArgs rustcArgs} ${lib.escapeShellArgs darwinArgs} -o "$out" tmp.rs
-      '';
-      inherit strip;
-    } name;
-
-  writeRustBin = name:
-    writeRust "/bin/${name}";
-
-  # writeJS takes a name an attributeset with libraries and some JavaScript sourcecode and
-  # returns an executable
-  #
-  # Example:
-  #   writeJS "example" { libraries = [ pkgs.nodePackages.uglify-js ]; } ''
-  #     var UglifyJS = require("uglify-js");
-  #     var code = "function add(first, second) { return first + second; }";
-  #     var result = UglifyJS.minify(code);
-  #     console.log(result.code);
-  #   ''
-  writeJS = name: { libraries ? [] }: content:
-  let
-    node-env = pkgs.buildEnv {
-      name = "node";
-      paths = libraries;
-      pathsToLink = [
-        "/lib/node_modules"
-      ];
-    };
-  in writeDash name ''
-    export NODE_PATH=${node-env}/lib/node_modules
-    exec ${pkgs.nodejs}/bin/node ${pkgs.writeText "js" content} "$@"
-  '';
-
-  # writeJSBin takes the same arguments as writeJS but outputs a directory (like writeScriptBin)
-  writeJSBin = name:
-    writeJS "/bin/${name}";
-
-  awkFormatNginx = builtins.toFile "awkFormat-nginx.awk" ''
-    awk -f
-    {sub(/^[ \t]+/,"");idx=0}
-    /\{/{ctx++;idx=1}
-    /\}/{ctx--}
-    {id="";for(i=idx;i<ctx;i++)id=sprintf("%s%s", id, "\t");printf "%s%s\n", id, $0}
-   '';
-
-  writeNginxConfig = name: text: pkgs.runCommandLocal name {
-    inherit text;
-    passAsFile = [ "text" ];
-    nativeBuildInputs = [ gixy ];
-  } /* sh */ ''
-    # nginx-config-formatter has an error - https://github.com/1connect/nginx-config-formatter/issues/16
-    awk -f ${awkFormatNginx} "$textPath" | sed '/^\s*$/d' > $out
-    gixy $out
-  '';
-
-  # writePerl takes a name an attributeset with libraries and some perl sourcecode and
-  # returns an executable
-  #
-  # Example:
-  #   writePerl "example" { libraries = [ pkgs.perlPackages.boolean ]; } ''
-  #     use boolean;
-  #     print "Howdy!\n" if true;
-  #   ''
-  writePerl = name: { libraries ? [] }:
-    makeScriptWriter {
-      interpreter = "${pkgs.perl.withPackages (p: libraries)}/bin/perl";
-    } name;
-
-  # writePerlBin takes the same arguments as writePerl but outputs a directory (like writeScriptBin)
-  writePerlBin = name:
-    writePerl "/bin/${name}";
-
-  # makePythonWriter takes python and compatible pythonPackages and produces python script writer,
-  # which validates the script with flake8 at build time. If any libraries are specified,
-  # python.withPackages is used as interpreter, otherwise the "bare" python is used.
-  makePythonWriter = python: pythonPackages: buildPythonPackages: name: { libraries ? [], flakeIgnore ? [] }:
-  let
-    ignoreAttribute = optionalString (flakeIgnore != []) "--ignore ${concatMapStringsSep "," escapeShellArg flakeIgnore}";
-  in
-  makeScriptWriter {
-    interpreter =
-      if libraries == []
-      then python.interpreter
-      else (python.withPackages (ps: libraries)).interpreter
-    ;
-    check = optionalString python.isPy3k (writeDash "pythoncheck.sh" ''
-      exec ${buildPythonPackages.flake8}/bin/flake8 --show-source ${ignoreAttribute} "$1"
-    '');
-  } name;
-
-  # writePyPy2 takes a name an attributeset with libraries and some pypy2 sourcecode and
-  # returns an executable
-  #
-  # Example:
-  # writePyPy2 "test_pypy2" { libraries = [ pkgs.pypy2Packages.enum ]; } ''
-  #   from enum import Enum
-  #
-  #   class Test(Enum):
-  #       a = "success"
-  #
-  #   print Test.a
-  # ''
-  writePyPy2 = makePythonWriter pkgs.pypy2 pkgs.pypy2Packages buildPackages.pypy2Packages;
-
-  # writePyPy2Bin takes the same arguments as writePyPy2 but outputs a directory (like writeScriptBin)
-  writePyPy2Bin = name:
-    writePyPy2 "/bin/${name}";
-
-  # writePython3 takes a name an attributeset with libraries and some python3 sourcecode and
-  # returns an executable
-  #
-  # Example:
-  # writePython3 "test_python3" { libraries = [ pkgs.python3Packages.pyyaml ]; } ''
-  #   import yaml
-  #
-  #   y = yaml.load("""
-  #     - test: success
-  #   """)
-  #   print(y[0]['test'])
-  # ''
-  writePython3 = makePythonWriter pkgs.python3 pkgs.python3Packages buildPackages.python3Packages;
-
-  # writePython3Bin takes the same arguments as writePython3 but outputs a directory (like writeScriptBin)
-  writePython3Bin = name:
-    writePython3 "/bin/${name}";
-
-  # writePyPy3 takes a name an attributeset with libraries and some pypy3 sourcecode and
-  # returns an executable
-  #
-  # Example:
-  # writePyPy3 "test_pypy3" { libraries = [ pkgs.pypy3Packages.pyyaml ]; } ''
-  #   import yaml
-  #
-  #   y = yaml.load("""
-  #     - test: success
-  #   """)
-  #   print(y[0]['test'])
-  # ''
-  writePyPy3 = makePythonWriter pkgs.pypy3 pkgs.pypy3Packages buildPackages.pypy3Packages;
-
-  # writePyPy3Bin takes the same arguments as writePyPy3 but outputs a directory (like writeScriptBin)
-  writePyPy3Bin = name:
-    writePyPy3 "/bin/${name}";
-
-
-  makeFSharpWriter = { dotnet-sdk ? pkgs.dotnet-sdk, fsi-flags ? "", libraries ? _: [] }: nameOrPath:
-  let
-    fname = last (builtins.split "/" nameOrPath);
-    path = if strings.hasSuffix ".fsx" nameOrPath then nameOrPath else "${nameOrPath}.fsx";
-    _nugetDeps = mkNugetDeps { name = "${fname}-nuget-deps"; nugetDeps = libraries; };
-
-    nuget-source = mkNugetSource {
-      name = "${fname}-nuget-source";
-      description = "A Nuget source with the dependencies for ${fname}";
-      deps = [ _nugetDeps ];
-    };
-
-    fsi = writeBash "fsi" ''
-      export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
-      export DOTNET_CLI_TELEMETRY_OPTOUT=1
-      export DOTNET_NOLOGO=1
-      script="$1"; shift
-      ${dotnet-sdk}/bin/dotnet fsi --quiet --nologo --readline- ${fsi-flags} "$@" < "$script"
-    '';
-
-  in content: writers.makeScriptWriter {
-    interpreter = fsi;
-  } path
-  ''
-    #i "nuget: ${nuget-source}/lib"
-    ${ content }
-    exit 0
-  '';
-
-  writeFSharp =
-    makeFSharpWriter {};
-
-  writeFSharpBin = name:
-    writeFSharp "/bin/${name}";
-
-};
+  writers = scriptWriters // dataWriters;
 in
 writers // (aliases writers)
diff --git a/pkgs/build-support/writers/scripts.nix b/pkgs/build-support/writers/scripts.nix
new file mode 100644
index 0000000000000..b8c5964c3fa33
--- /dev/null
+++ b/pkgs/build-support/writers/scripts.nix
@@ -0,0 +1,383 @@
+{ pkgs, lib }:
+let
+  inherit (lib)
+    concatMapStringsSep
+    elem
+    escapeShellArg
+    last
+    optionalString
+    stringLength
+    strings
+    types
+    ;
+
+  inherit (pkgs)
+    buildPackages
+    gixy
+    libiconv
+    mkNugetDeps
+    mkNugetSource
+    stdenv
+    ;
+in
+rec {
+  # Base implementation for non-compiled executables.
+  # Takes an interpreter, for example `${pkgs.bash}/bin/bash`
+  #
+  # Examples:
+  #   writeBash = makeScriptWriter { interpreter = "${pkgs.bash}/bin/bash"; }
+  #   makeScriptWriter { interpreter = "${pkgs.dash}/bin/dash"; } "hello" "echo hello world"
+  makeScriptWriter = { interpreter, check ? "" }: nameOrPath: content:
+    assert lib.or (types.path.check nameOrPath) (builtins.match "([0-9A-Za-z._])[0-9A-Za-z._-]*" nameOrPath != null);
+    assert lib.or (types.path.check content) (types.str.check content);
+    let
+      name = last (builtins.split "/" nameOrPath);
+    in
+
+    pkgs.runCommandLocal name (if (types.str.check content) then {
+      inherit content interpreter;
+      passAsFile = [ "content" ];
+    } else {
+      inherit interpreter;
+      contentPath = content;
+    }) ''
+      # On darwin a script cannot be used as an interpreter in a shebang but
+      # there doesn't seem to be a limit to the size of shebang and multiple
+      # arguments to the interpreter are allowed.
+      if [[ -n "${toString pkgs.stdenvNoCC.isDarwin}" ]] && isScript $interpreter
+      then
+        wrapperInterpreterLine=$(head -1 "$interpreter" | tail -c+3)
+        # Get first word from the line (note: xargs echo remove leading spaces)
+        wrapperInterpreter=$(echo "$wrapperInterpreterLine" | xargs echo | cut -d " " -f1)
+
+        if isScript $wrapperInterpreter
+        then
+          echo "error: passed interpreter ($interpreter) is a script which has another script ($wrapperInterpreter) as an interpreter, which is not supported."
+          exit 1
+        fi
+
+        # This should work as long as wrapperInterpreter is a shell, which is
+        # the case for programs wrapped with makeWrapper, like
+        # python3.withPackages etc.
+        interpreterLine="$wrapperInterpreterLine $interpreter"
+      else
+        interpreterLine=$interpreter
+      fi
+
+      echo "#! $interpreterLine" > $out
+      cat "$contentPath" >> $out
+      ${optionalString (check != "") ''
+        ${check} $out
+      ''}
+      chmod +x $out
+      ${optionalString (types.path.check nameOrPath) ''
+        mv $out tmp
+        mkdir -p $out/$(dirname "${nameOrPath}")
+        mv tmp $out/${nameOrPath}
+      ''}
+    '';
+
+  # Base implementation for compiled executables.
+  # Takes a compile script, which in turn takes the name as an argument.
+  #
+  # Examples:
+  #   writeSimpleC = makeBinWriter { compileScript = name: "gcc -o $out $contentPath"; }
+  makeBinWriter = { compileScript, strip ? true }: nameOrPath: content:
+    assert lib.or (types.path.check nameOrPath) (builtins.match "([0-9A-Za-z._])[0-9A-Za-z._-]*" nameOrPath != null);
+    assert lib.or (types.path.check content) (types.str.check content);
+    let
+      name = last (builtins.split "/" nameOrPath);
+    in
+    pkgs.runCommand name ((if (types.str.check content) then {
+      inherit content;
+      passAsFile = [ "content" ];
+    } else {
+      contentPath = content;
+    }) // lib.optionalAttrs (stdenv.hostPlatform.isDarwin && stdenv.hostPlatform.isAarch64) {
+      # post-link-hook expects codesign_allocate to be in PATH
+      # https://github.com/NixOS/nixpkgs/issues/154203
+      # https://github.com/NixOS/nixpkgs/issues/148189
+      nativeBuildInputs = [ stdenv.cc.bintools ];
+    }) ''
+      ${compileScript}
+      ${lib.optionalString strip
+          "${lib.getBin buildPackages.bintools-unwrapped}/bin/${buildPackages.bintools-unwrapped.targetPrefix}strip -S $out"}
+      # Sometimes binaries produced for darwin (e. g. by GHC) won't be valid
+      # mach-o executables from the get-go, but need to be corrected somehow
+      # which is done by fixupPhase.
+      ${lib.optionalString pkgs.stdenvNoCC.hostPlatform.isDarwin "fixupPhase"}
+      ${optionalString (types.path.check nameOrPath) ''
+        mv $out tmp
+        mkdir -p $out/$(dirname "${nameOrPath}")
+        mv tmp $out/${nameOrPath}
+      ''}
+    '';
+
+  # Like writeScript but the first line is a shebang to bash
+  #
+  # Example:
+  #   writeBash "example" ''
+  #     echo hello world
+  #   ''
+  writeBash = makeScriptWriter {
+    interpreter = "${pkgs.bash}/bin/bash";
+  };
+
+  # Like writeScriptBin but the first line is a shebang to bash
+  writeBashBin = name:
+    writeBash "/bin/${name}";
+
+  # Like writeScript but the first line is a shebang to dash
+  #
+  # Example:
+  #   writeDash "example" ''
+  #     echo hello world
+  #   ''
+  writeDash = makeScriptWriter {
+    interpreter = "${pkgs.dash}/bin/dash";
+  };
+
+  # Like writeScriptBin but the first line is a shebang to dash
+  writeDashBin = name:
+    writeDash "/bin/${name}";
+
+  # Like writeScript but the first line is a shebang to fish
+  #
+  # Example:
+  #   writeFish "example" ''
+  #     echo hello world
+  #   ''
+  writeFish = makeScriptWriter {
+    interpreter = "${pkgs.fish}/bin/fish --no-config";
+    check = "${pkgs.fish}/bin/fish --no-config --no-execute";  # syntax check only
+  };
+
+  # Like writeScriptBin but the first line is a shebang to fish
+  writeFishBin = name:
+    writeFish "/bin/${name}";
+
+  # writeHaskell takes a name, an attrset with libraries and haskell version (both optional)
+  # and some haskell source code and returns an executable.
+  #
+  # Example:
+  #   writeHaskell "missiles" { libraries = [ pkgs.haskellPackages.acme-missiles ]; } ''
+  #     import Acme.Missiles
+  #
+  #     main = launchMissiles
+  #   '';
+  writeHaskell = name: {
+    libraries ? [],
+    ghc ? pkgs.ghc,
+    ghcArgs ? [],
+    threadedRuntime ? true,
+    strip ? true
+  }:
+    let
+      appendIfNotSet = el: list: if elem el list then list else list ++ [ el ];
+      ghcArgs' = if threadedRuntime then appendIfNotSet "-threaded" ghcArgs else ghcArgs;
+
+    in makeBinWriter {
+      compileScript = ''
+        cp $contentPath tmp.hs
+        ${ghc.withPackages (_: libraries )}/bin/ghc ${lib.escapeShellArgs ghcArgs'} tmp.hs
+        mv tmp $out
+      '';
+      inherit strip;
+    } name;
+
+  # writeHaskellBin takes the same arguments as writeHaskell but outputs a directory (like writeScriptBin)
+  writeHaskellBin = name:
+    writeHaskell "/bin/${name}";
+
+  writeRust = name: {
+      rustc ? pkgs.rustc,
+      rustcArgs ? [],
+      strip ? true
+  }:
+  let
+    darwinArgs = lib.optionals stdenv.isDarwin [ "-L${lib.getLib libiconv}/lib" ];
+  in
+    makeBinWriter {
+      compileScript = ''
+        cp "$contentPath" tmp.rs
+        PATH=${lib.makeBinPath [pkgs.gcc]} ${lib.getBin rustc}/bin/rustc ${lib.escapeShellArgs rustcArgs} ${lib.escapeShellArgs darwinArgs} -o "$out" tmp.rs
+      '';
+      inherit strip;
+    } name;
+
+  writeRustBin = name:
+    writeRust "/bin/${name}";
+
+  # writeJS takes a name an attributeset with libraries and some JavaScript sourcecode and
+  # returns an executable
+  #
+  # Example:
+  #   writeJS "example" { libraries = [ pkgs.nodePackages.uglify-js ]; } ''
+  #     var UglifyJS = require("uglify-js");
+  #     var code = "function add(first, second) { return first + second; }";
+  #     var result = UglifyJS.minify(code);
+  #     console.log(result.code);
+  #   ''
+  writeJS = name: { libraries ? [] }: content:
+  let
+    node-env = pkgs.buildEnv {
+      name = "node";
+      paths = libraries;
+      pathsToLink = [
+        "/lib/node_modules"
+      ];
+    };
+  in writeDash name ''
+    export NODE_PATH=${node-env}/lib/node_modules
+    exec ${pkgs.nodejs}/bin/node ${pkgs.writeText "js" content} "$@"
+  '';
+
+  # writeJSBin takes the same arguments as writeJS but outputs a directory (like writeScriptBin)
+  writeJSBin = name:
+    writeJS "/bin/${name}";
+
+  awkFormatNginx = builtins.toFile "awkFormat-nginx.awk" ''
+    awk -f
+    {sub(/^[ \t]+/,"");idx=0}
+    /\{/{ctx++;idx=1}
+    /\}/{ctx--}
+    {id="";for(i=idx;i<ctx;i++)id=sprintf("%s%s", id, "\t");printf "%s%s\n", id, $0}
+   '';
+
+  writeNginxConfig = name: text: pkgs.runCommandLocal name {
+    inherit text;
+    passAsFile = [ "text" ];
+    nativeBuildInputs = [ gixy ];
+  } /* sh */ ''
+    # nginx-config-formatter has an error - https://github.com/1connect/nginx-config-formatter/issues/16
+    awk -f ${awkFormatNginx} "$textPath" | sed '/^\s*$/d' > $out
+    gixy $out
+  '';
+
+  # writePerl takes a name an attributeset with libraries and some perl sourcecode and
+  # returns an executable
+  #
+  # Example:
+  #   writePerl "example" { libraries = [ pkgs.perlPackages.boolean ]; } ''
+  #     use boolean;
+  #     print "Howdy!\n" if true;
+  #   ''
+  writePerl = name: { libraries ? [] }:
+    makeScriptWriter {
+      interpreter = "${pkgs.perl.withPackages (p: libraries)}/bin/perl";
+    } name;
+
+  # writePerlBin takes the same arguments as writePerl but outputs a directory (like writeScriptBin)
+  writePerlBin = name:
+    writePerl "/bin/${name}";
+
+  # makePythonWriter takes python and compatible pythonPackages and produces python script writer,
+  # which validates the script with flake8 at build time. If any libraries are specified,
+  # python.withPackages is used as interpreter, otherwise the "bare" python is used.
+  makePythonWriter = python: pythonPackages: buildPythonPackages: name: { libraries ? [], flakeIgnore ? [] }:
+  let
+    ignoreAttribute = optionalString (flakeIgnore != []) "--ignore ${concatMapStringsSep "," escapeShellArg flakeIgnore}";
+  in
+  makeScriptWriter {
+    interpreter =
+      if libraries == []
+      then python.interpreter
+      else (python.withPackages (ps: libraries)).interpreter
+    ;
+    check = optionalString python.isPy3k (writeDash "pythoncheck.sh" ''
+      exec ${buildPythonPackages.flake8}/bin/flake8 --show-source ${ignoreAttribute} "$1"
+    '');
+  } name;
+
+  # writePyPy2 takes a name an attributeset with libraries and some pypy2 sourcecode and
+  # returns an executable
+  #
+  # Example:
+  # writePyPy2 "test_pypy2" { libraries = [ pkgs.pypy2Packages.enum ]; } ''
+  #   from enum import Enum
+  #
+  #   class Test(Enum):
+  #       a = "success"
+  #
+  #   print Test.a
+  # ''
+  writePyPy2 = makePythonWriter pkgs.pypy2 pkgs.pypy2Packages buildPackages.pypy2Packages;
+
+  # writePyPy2Bin takes the same arguments as writePyPy2 but outputs a directory (like writeScriptBin)
+  writePyPy2Bin = name:
+    writePyPy2 "/bin/${name}";
+
+  # writePython3 takes a name an attributeset with libraries and some python3 sourcecode and
+  # returns an executable
+  #
+  # Example:
+  # writePython3 "test_python3" { libraries = [ pkgs.python3Packages.pyyaml ]; } ''
+  #   import yaml
+  #
+  #   y = yaml.load("""
+  #     - test: success
+  #   """)
+  #   print(y[0]['test'])
+  # ''
+  writePython3 = makePythonWriter pkgs.python3 pkgs.python3Packages buildPackages.python3Packages;
+
+  # writePython3Bin takes the same arguments as writePython3 but outputs a directory (like writeScriptBin)
+  writePython3Bin = name:
+    writePython3 "/bin/${name}";
+
+  # writePyPy3 takes a name an attributeset with libraries and some pypy3 sourcecode and
+  # returns an executable
+  #
+  # Example:
+  # writePyPy3 "test_pypy3" { libraries = [ pkgs.pypy3Packages.pyyaml ]; } ''
+  #   import yaml
+  #
+  #   y = yaml.load("""
+  #     - test: success
+  #   """)
+  #   print(y[0]['test'])
+  # ''
+  writePyPy3 = makePythonWriter pkgs.pypy3 pkgs.pypy3Packages buildPackages.pypy3Packages;
+
+  # writePyPy3Bin takes the same arguments as writePyPy3 but outputs a directory (like writeScriptBin)
+  writePyPy3Bin = name:
+    writePyPy3 "/bin/${name}";
+
+
+  makeFSharpWriter = { dotnet-sdk ? pkgs.dotnet-sdk, fsi-flags ? "", libraries ? _: [] }: nameOrPath:
+  let
+    fname = last (builtins.split "/" nameOrPath);
+    path = if strings.hasSuffix ".fsx" nameOrPath then nameOrPath else "${nameOrPath}.fsx";
+    _nugetDeps = mkNugetDeps { name = "${fname}-nuget-deps"; nugetDeps = libraries; };
+
+    nuget-source = mkNugetSource {
+      name = "${fname}-nuget-source";
+      description = "A Nuget source with the dependencies for ${fname}";
+      deps = [ _nugetDeps ];
+    };
+
+    fsi = writeBash "fsi" ''
+      export HOME=$NIX_BUILD_TOP/.home
+      export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
+      export DOTNET_CLI_TELEMETRY_OPTOUT=1
+      export DOTNET_NOLOGO=1
+      script="$1"; shift
+      ${dotnet-sdk}/bin/dotnet fsi --quiet --nologo --readline- ${fsi-flags} "$@" < "$script"
+    '';
+
+  in content: makeScriptWriter {
+    interpreter = fsi;
+  } path
+  ''
+    #i "nuget: ${nuget-source}/lib"
+    ${ content }
+    exit 0
+  '';
+
+  writeFSharp =
+    makeFSharpWriter {};
+
+  writeFSharpBin = name:
+    writeFSharp "/bin/${name}";
+
+}
diff --git a/pkgs/build-support/writers/test.nix b/pkgs/build-support/writers/test.nix
index 561c3e4ab0021..2411f8c03a70b 100644
--- a/pkgs/build-support/writers/test.nix
+++ b/pkgs/build-support/writers/test.nix
@@ -12,29 +12,62 @@
 }:
 with writers;
 let
+  expectSuccess = test:
+    runCommand "run-${test.name}" {} ''
+      if [[ "$(${test})" != success ]]; then
+        echo 'test ${test.name} failed'
+        exit 1
+      fi
 
-  bin = {
-    bash = writeBashBin "test-writers-bash-bin" ''
-     if [[ "test" == "test" ]]; then echo "success"; fi
+      touch $out
     '';
 
-    dash = writeDashBin "test-writers-dash-bin" ''
-     test '~' = '~' && echo 'success'
+  expectSuccessBin = test:
+    runCommand "run-${test.name}" {} ''
+      if [[ "$(${lib.getExe test})" != success ]]; then
+        echo 'test ${test.name} failed'
+        exit 1
+      fi
+
+      touch $out
     '';
 
-    fish = writeFishBin "test-writers-fish-bin" ''
+  expectDataEqual = { file, expected }:
+    let
+      expectedFile = writeText "${file.name}-expected" expected;
+    in
+    runCommand "run-${file.name}" {} ''
+      if ! diff -u ${file} ${expectedFile}; then
+        echo 'test ${file.name} failed'
+        exit 1
+      fi
+
+      touch $out
+    '';
+in
+lib.recurseIntoAttrs {
+  bin = lib.recurseIntoAttrs {
+    bash = expectSuccessBin (writeBashBin "test-writers-bash-bin" ''
+     if [[ "test" == "test" ]]; then echo "success"; fi
+    '');
+
+    dash = expectSuccessBin (writeDashBin "test-writers-dash-bin" ''
+     test '~' = '~' && echo 'success'
+    '');
+
+    fish = expectSuccessBin (writeFishBin "test-writers-fish-bin" ''
       if test "test" = "test"
         echo "success"
       end
-    '';
+    '');
 
-    rust = writeRustBin "test-writers-rust-bin" {} ''
+    rust = expectSuccessBin (writeRustBin "test-writers-rust-bin" {} ''
       fn main(){
         println!("success")
       }
-    '';
+    '');
 
-    haskell = writeHaskellBin "test-writers-haskell-bin" { libraries = [ haskellPackages.acme-default ]; } ''
+    haskell = expectSuccessBin (writeHaskellBin "test-writers-haskell-bin" { libraries = [ haskellPackages.acme-default ]; } ''
       import Data.Default
 
       int :: Int
@@ -44,9 +77,9 @@ let
       main = case int of
         18871 -> putStrLn $ id "success"
         _ -> print "fail"
-    '';
+    '');
 
-    js = writeJSBin "test-writers-js-bin" { libraries = [ nodePackages.semver ]; } ''
+    js = expectSuccessBin (writeJSBin "test-writers-js-bin" { libraries = [ nodePackages.semver ]; } ''
       var semver = require('semver');
 
       if (semver.valid('1.2.3')) {
@@ -54,59 +87,57 @@ let
       } else {
         console.log('fail')
       }
-    '';
+    '');
 
-    perl = writePerlBin "test-writers-perl-bin" { libraries = [ perlPackages.boolean ]; } ''
+    perl = expectSuccessBin (writePerlBin "test-writers-perl-bin" { libraries = [ perlPackages.boolean ]; } ''
       use boolean;
       print "success\n" if true;
-    '';
+    '');
 
-    pypy2 = writePyPy2Bin "test-writers-pypy2-bin" { libraries = [ pypy2Packages.enum ]; } ''
+    pypy2 = expectSuccessBin (writePyPy2Bin "test-writers-pypy2-bin" { libraries = [ pypy2Packages.enum ]; } ''
       from enum import Enum
 
-
       class Test(Enum):
           a = "success"
 
-
       print Test.a
-    '';
+    '');
 
-    python3 = writePython3Bin "test-writers-python3-bin" { libraries = [ python3Packages.pyyaml ]; } ''
+    python3 = expectSuccessBin (writePython3Bin "test-writers-python3-bin" { libraries = [ python3Packages.pyyaml ]; } ''
       import yaml
 
-      y = yaml.load("""
+      y = yaml.safe_load("""
         - test: success
       """)
       print(y[0]['test'])
-    '';
+    '');
 
-    pypy3 = writePyPy3Bin "test-writers-pypy3-bin" { libraries = [ pypy3Packages.pyyaml ]; } ''
+    pypy3 = expectSuccessBin (writePyPy3Bin "test-writers-pypy3-bin" { libraries = [ pypy3Packages.pyyaml ]; } ''
       import yaml
 
-      y = yaml.load("""
+      y = yaml.safe_load("""
         - test: success
       """)
       print(y[0]['test'])
-    '';
+    '');
   };
 
-  simple = {
-    bash = writeBash "test-writers-bash" ''
+  simple = lib.recurseIntoAttrs {
+    bash = expectSuccess (writeBash "test-writers-bash" ''
      if [[ "test" == "test" ]]; then echo "success"; fi
-    '';
+    '');
 
-    dash = writeDash "test-writers-dash" ''
+    dash = expectSuccess (writeDash "test-writers-dash" ''
      test '~' = '~' && echo 'success'
-    '';
+    '');
 
-    fish = writeFish "test-writers-fish" ''
+    fish = expectSuccess (writeFish "test-writers-fish" ''
       if test "test" = "test"
         echo "success"
       end
-    '';
+    '');
 
-    haskell = writeHaskell "test-writers-haskell" { libraries = [ haskellPackages.acme-default ]; } ''
+    haskell = expectSuccess (writeHaskell "test-writers-haskell" { libraries = [ haskellPackages.acme-default ]; } ''
       import Data.Default
 
       int :: Int
@@ -116,9 +147,9 @@ let
       main = case int of
         18871 -> putStrLn $ id "success"
         _ -> print "fail"
-    '';
+    '');
 
-    js = writeJS "test-writers-js" { libraries = [ nodePackages.semver ]; } ''
+    js = expectSuccess (writeJS "test-writers-js" { libraries = [ nodePackages.semver ]; } ''
       var semver = require('semver');
 
       if (semver.valid('1.2.3')) {
@@ -126,43 +157,41 @@ let
       } else {
         console.log('fail')
       }
-    '';
+    '');
 
-    perl = writePerl "test-writers-perl" { libraries = [ perlPackages.boolean ]; } ''
+    perl = expectSuccess (writePerl "test-writers-perl" { libraries = [ perlPackages.boolean ]; } ''
       use boolean;
       print "success\n" if true;
-    '';
+    '');
 
-    pypy2 = writePyPy2 "test-writers-pypy2" { libraries = [ pypy2Packages.enum ]; } ''
+    pypy2 = expectSuccess (writePyPy2 "test-writers-pypy2" { libraries = [ pypy2Packages.enum ]; } ''
       from enum import Enum
 
-
       class Test(Enum):
           a = "success"
 
-
       print Test.a
-    '';
+    '');
 
-    python3 = writePython3 "test-writers-python3" { libraries = [ python3Packages.pyyaml ]; } ''
+    python3 = expectSuccess (writePython3 "test-writers-python3" { libraries = [ python3Packages.pyyaml ]; } ''
       import yaml
 
-      y = yaml.load("""
+      y = yaml.safe_load("""
         - test: success
       """)
       print(y[0]['test'])
-    '';
+    '');
 
-    pypy3 = writePyPy3 "test-writers-pypy3" { libraries = [ pypy3Packages.pyyaml ]; } ''
+    pypy3 = expectSuccess (writePyPy3 "test-writers-pypy3" { libraries = [ pypy3Packages.pyyaml ]; } ''
       import yaml
 
-      y = yaml.load("""
+      y = yaml.safe_load("""
         - test: success
       """)
       print(y[0]['test'])
-    '';
+    '');
 
-    fsharp = makeFSharpWriter {
+    fsharp = expectSuccess (makeFSharpWriter {
       libraries = { fetchNuGet }: [
         (fetchNuGet { pname = "FSharp.SystemTextJson"; version = "0.17.4"; sha256 = "1bplzc9ybdqspii4q28l8gmfvzpkmgq5l1hlsiyg2h46w881lwg2"; })
       ];
@@ -183,31 +212,31 @@ let
       then "success"
       else "failed"
       |> printfn "%s"
-    '';
+    '');
 
-    pypy2NoLibs = writePyPy2 "test-writers-pypy2-no-libs" {} ''
+    pypy2NoLibs = expectSuccess (writePyPy2 "test-writers-pypy2-no-libs" {} ''
       print("success")
-    '';
+    '');
 
-    python3NoLibs = writePython3 "test-writers-python3-no-libs" {} ''
+    python3NoLibs = expectSuccess (writePython3 "test-writers-python3-no-libs" {} ''
       print("success")
-    '';
+    '');
 
-    pypy3NoLibs = writePyPy3 "test-writers-pypy3-no-libs" {} ''
+    pypy3NoLibs = expectSuccess (writePyPy3 "test-writers-pypy3-no-libs" {} ''
       print("success")
-    '';
+    '');
 
-    fsharpNoNugetDeps = writeFSharp "test-writers-fsharp-no-nuget-deps" ''
+    fsharpNoNugetDeps = expectSuccess (writeFSharp "test-writers-fsharp-no-nuget-deps" ''
       printfn "success"
-    '';
+    '');
   };
 
-
-  path = {
-    bash = writeBash "test-writers-bash-path" (writeText "test" ''
+  path = lib.recurseIntoAttrs {
+    bash = expectSuccess (writeBash "test-writers-bash-path" (writeText "test" ''
       if [[ "test" == "test" ]]; then echo "success"; fi
-    '');
-    haskell = writeHaskell "test-writers-haskell-path" { libraries = [ haskellPackages.acme-default ]; } (writeText "test" ''
+    ''));
+
+    haskell = expectSuccess (writeHaskell "test-writers-haskell-path" { libraries = [ haskellPackages.acme-default ]; } (writeText "test" ''
       import Data.Default
 
       int :: Int
@@ -217,26 +246,27 @@ let
       main = case int of
         18871 -> putStrLn $ id "success"
         _ -> print "fail"
-    '');
+    ''));
   };
 
-  writeTest = expectedValue: name: test:
-    writeDash "run-${name}" ''
-      if test "$(${test})" != "${expectedValue}"; then
-        echo 'test ${test} failed'
-        exit 1
-      fi
-    '';
-
-in runCommand "test-writers" {
-  passthru = { inherit writeTest bin simple path; };
-  meta.platforms = lib.platforms.all;
-} ''
-  ${lib.concatMapStringsSep "\n" (test: writeTest "success" test.name "${test}/bin/${test.name}") (lib.attrValues bin)}
-  ${lib.concatMapStringsSep "\n" (test: writeTest "success" test.name test) (lib.attrValues simple)}
-  ${lib.concatMapStringsSep "\n" (test: writeTest "success" test.name test) (lib.attrValues path)}
-
-  echo 'nix-writers successfully tested' >&2
-  touch $out
-''
-
+  data = {
+    json = expectDataEqual {
+      file = writeJSON "data.json" { hello = "world"; };
+      expected = ''
+        {
+          "hello": "world"
+        }
+      '';
+    };
+
+    toml = expectDataEqual {
+      file = writeTOML "data.toml" { hello = "world"; };
+      expected = "hello = 'world'\n";
+    };
+
+    yaml = expectDataEqual {
+      file = writeYAML "data.yaml" { hello = "world"; };
+      expected = "hello: world\n";
+    };
+  };
+}