buildbot: move out of python3.pkgs

By moving it out we can start apply overrides to python dependendencies
starting with sqlalchemy.

This fixes the build with the current version.

7 files changed, 457 insertions, 0 deletions

diff --git a/pkgs/development/tools/continuous-integration/buildbot/default.nix b/pkgs/development/tools/continuous-integration/buildbot/default.nix
new file mode 100644
index 0000000000000..ec3a4a81f60f0
--- /dev/null
+++ b/pkgs/development/tools/continuous-integration/buildbot/default.nix
@@ -0,0 +1,42 @@
+{ python3
+, recurseIntoAttrs
+, callPackage
+}:
+let
+  python = python3.override {
+    packageOverrides = self: super: {
+      sqlalchemy = super.sqlalchemy.overridePythonAttrs (oldAttrs: rec {
+        version = "1.4.40";
+        src = super.fetchPypi {
+          pname = "SQLAlchemy";
+          inherit version;
+          hash = "sha256-RKZgUGCAzJdeHfpXdv5fYxXdxiane1C/Du4YsDieomU=";
+        };
+      });
+      moto = super.moto.overridePythonAttrs (oldAttrs: rec {
+        # a lot of tests -> very slow, we already build them when building python packages
+        doCheck = false;
+      });
+    };
+  };
+
+  buildbot-pkg = python.pkgs.callPackage ./pkg.nix {
+    inherit buildbot;
+  };
+  buildbot-worker = python3.pkgs.callPackage ./worker.nix {
+    inherit buildbot;
+  };
+  buildbot = python.pkgs.callPackage ./master.nix {
+    inherit buildbot-pkg buildbot-worker buildbot-plugins;
+  };
+  buildbot-plugins = recurseIntoAttrs (callPackage ./plugins.nix {
+    inherit buildbot-pkg;
+  });
+in
+{
+  inherit buildbot buildbot-plugins buildbot-worker;
+  buildbot-ui = buildbot.withPlugins (with buildbot-plugins; [ www ]);
+  buildbot-full = buildbot.withPlugins (with buildbot-plugins; [
+    www console-view waterfall-view grid-view wsgi-dashboards badges
+  ]);
+}
diff --git a/pkgs/development/tools/continuous-integration/buildbot/master.nix b/pkgs/development/tools/continuous-integration/buildbot/master.nix
new file mode 100644
index 0000000000000..6e59e31867b62
--- /dev/null
+++ b/pkgs/development/tools/continuous-integration/buildbot/master.nix
@@ -0,0 +1,154 @@
+{ lib
+, stdenv
+, buildPythonPackage
+, buildPythonApplication
+, fetchPypi
+, makeWrapper
+, pythonOlder
+, python
+, twisted
+, jinja2
+, msgpack
+, zope_interface
+, sqlalchemy
+, alembic
+, python-dateutil
+, txaio
+, autobahn
+, pyjwt
+, pyyaml
+, treq
+, txrequests
+, pypugjs
+, boto3
+, moto
+, mock
+, lz4
+, setuptoolsTrial
+, buildbot-worker
+, buildbot-plugins
+, buildbot-pkg
+, parameterized
+, git
+, openssh
+, glibcLocales
+, nixosTests
+, callPackage
+}:
+
+let
+  withPlugins = plugins: buildPythonApplication {
+    pname = "${package.pname}-with-plugins";
+    inherit (package) version;
+    format = "other";
+
+    dontUnpack = true;
+    dontBuild = true;
+    doCheck = false;
+
+    nativeBuildInputs = [
+      makeWrapper
+    ];
+
+    propagatedBuildInputs = plugins ++ package.propagatedBuildInputs;
+
+    installPhase = ''
+      makeWrapper ${package}/bin/buildbot $out/bin/buildbot \
+        --prefix PYTHONPATH : "${package}/${python.sitePackages}:$PYTHONPATH"
+      ln -sfv ${package}/lib $out/lib
+    '';
+
+    passthru = package.passthru // {
+      withPlugins = morePlugins: withPlugins (morePlugins ++ plugins);
+    };
+  };
+
+  package = buildPythonApplication rec {
+    pname = "buildbot";
+    version = "3.7.0";
+    format = "setuptools";
+
+    disabled = pythonOlder "3.7";
+
+    src = fetchPypi {
+      inherit pname version;
+      hash = "sha256-YMLT1SP6NenJIUVTvr58GVrtNXHw+bhfgMpZu3revG4=";
+    };
+
+    propagatedBuildInputs = [
+      # core
+      twisted
+      jinja2
+      msgpack
+      zope_interface
+      sqlalchemy
+      alembic
+      python-dateutil
+      txaio
+      autobahn
+      pyjwt
+      pyyaml
+    ]
+      # tls
+      ++ twisted.optional-dependencies.tls;
+
+    nativeCheckInputs = [
+      treq
+      txrequests
+      pypugjs
+      boto3
+      moto
+      mock
+      lz4
+      setuptoolsTrial
+      buildbot-worker
+      buildbot-pkg
+      buildbot-plugins.www
+      parameterized
+      git
+      openssh
+      glibcLocales
+    ];
+
+    patches = [
+      # This patch disables the test that tries to read /etc/os-release which
+      # is not accessible in sandboxed builds.
+      ./skip_test_linux_distro.patch
+    ];
+
+    postPatch = ''
+      substituteInPlace buildbot/scripts/logwatcher.py --replace '/usr/bin/tail' "$(type -P tail)"
+    '';
+
+    # Silence the depreciation warning from SqlAlchemy
+    SQLALCHEMY_SILENCE_UBER_WARNING = 1;
+
+    # TimeoutErrors on slow machines -> aarch64
+    doCheck = !stdenv.isAarch64;
+
+    preCheck = ''
+      export LC_ALL="en_US.UTF-8"
+      export PATH="$out/bin:$PATH"
+
+      # remove testfile which is missing configuration file from sdist
+      rm buildbot/test/integration/test_graphql.py
+      # tests in this file are flaky, see https://github.com/buildbot/buildbot/issues/6776
+      rm buildbot/test/integration/test_try_client.py
+    '';
+
+    passthru = {
+      inherit withPlugins;
+      tests.buildbot = nixosTests.buildbot;
+      updateScript = ./update.sh;
+    };
+
+    meta = with lib; {
+      description = "An open-source continuous integration framework for automating software build, test, and release processes";
+      homepage = "https://buildbot.net/";
+      changelog = "https://github.com/buildbot/buildbot/releases/tag/v${version}";
+      maintainers = with maintainers; [ ryansydnor lopsided98 ];
+      license = licenses.gpl2Only;
+      broken = stdenv.isDarwin;
+    };
+  };
+in package
diff --git a/pkgs/development/tools/continuous-integration/buildbot/pkg.nix b/pkgs/development/tools/continuous-integration/buildbot/pkg.nix
new file mode 100644
index 0000000000000..13c0ec8694854
--- /dev/null
+++ b/pkgs/development/tools/continuous-integration/buildbot/pkg.nix
@@ -0,0 +1,31 @@
+{ lib, buildPythonPackage, fetchPypi, isPy3k, buildbot }:
+
+buildPythonPackage rec {
+  pname = "buildbot-pkg";
+  inherit (buildbot) version;
+
+  src = fetchPypi {
+    inherit pname version;
+    hash = "sha256-4BXCOLW5e7RuZEzyD+oRmS2I4oT2W3oTcH4ZPxwKKvU=";
+  };
+
+  postPatch = ''
+    # Their listdir function filters out `node_modules` folders.
+    # Do we have to care about that with Nix...?
+    substituteInPlace buildbot_pkg.py --replace "os.listdir = listdir" ""
+  '';
+
+  # No tests
+  doCheck = false;
+
+  pythonImportsCheck = [ "buildbot_pkg" ];
+
+  disabled = !isPy3k;
+
+  meta = with lib; {
+    homepage = "https://buildbot.net/";
+    description = "Buildbot Packaging Helper";
+    maintainers = with maintainers; [ ryansydnor lopsided98 ];
+    license = licenses.gpl2;
+  };
+}
diff --git a/pkgs/development/tools/continuous-integration/buildbot/plugins.nix b/pkgs/development/tools/continuous-integration/buildbot/plugins.nix
new file mode 100644
index 0000000000000..42a77fa0f4bc5
--- /dev/null
+++ b/pkgs/development/tools/continuous-integration/buildbot/plugins.nix
@@ -0,0 +1,141 @@
+{ lib, buildPythonPackage, fetchPypi, callPackage, mock, cairosvg, klein, jinja2, buildbot-pkg }:
+{
+  www = buildPythonPackage rec {
+    pname = "buildbot-www";
+    inherit (buildbot-pkg) version;
+
+    src = fetchPypi {
+      inherit pname version;
+      hash = "sha256-t4xHfox6h5PY4+phdCQbClHd77+WfpUWErMZCEcMxu0=";
+    };
+
+    # Remove unnecessary circular dependency on buildbot
+    postPatch = ''
+      sed -i "s/'buildbot'//" setup.py
+    '';
+
+    buildInputs = [ buildbot-pkg mock ];
+
+    # No tests
+    doCheck = false;
+
+    meta = with lib; {
+      homepage = "https://buildbot.net/";
+      description = "Buildbot UI";
+      maintainers = with maintainers; [ ryansydnor lopsided98 ];
+      license = licenses.gpl2;
+    };
+  };
+
+  console-view = buildPythonPackage rec {
+    pname = "buildbot-console-view";
+    inherit (buildbot-pkg) version;
+
+    src = fetchPypi {
+      inherit pname version;
+      hash = "sha256-JySn7QO+SgoXjVeV4wYwc9twr0Q2c/wsEspqeq038+k=";
+    };
+
+    buildInputs = [ buildbot-pkg ];
+
+    # No tests
+    doCheck = false;
+
+    meta = with lib; {
+      homepage = "https://buildbot.net/";
+      description = "Buildbot Console View Plugin";
+      maintainers = with maintainers; [ ryansydnor lopsided98 ];
+      license = licenses.gpl2;
+    };
+  };
+
+  waterfall-view = buildPythonPackage rec {
+    pname = "buildbot-waterfall-view";
+    inherit (buildbot-pkg) version;
+
+    src = fetchPypi {
+      inherit pname version;
+      hash = "sha256-935eeF2kpT68lK/UMg8MZQOYEj7D8FaT9iSs/lNahYA=";
+    };
+
+    buildInputs = [ buildbot-pkg ];
+
+    # No tests
+    doCheck = false;
+
+    meta = with lib; {
+      homepage = "https://buildbot.net/";
+      description = "Buildbot Waterfall View Plugin";
+      maintainers = with maintainers; [ ryansydnor lopsided98 ];
+      license = licenses.gpl2;
+    };
+  };
+
+  grid-view = buildPythonPackage rec {
+    pname = "buildbot-grid-view";
+    inherit (buildbot-pkg) version;
+
+    src = fetchPypi {
+      inherit pname version;
+      hash = "sha256-3pHSiVoOZj2iCGfiz+tMWWMPHSBH5Ggp6e3+a8topsg=";
+    };
+
+    buildInputs = [ buildbot-pkg ];
+
+    # No tests
+    doCheck = false;
+
+    meta = with lib; {
+      homepage = "https://buildbot.net/";
+      description = "Buildbot Grid View Plugin";
+      maintainers = with maintainers; [ lopsided98 ];
+      license = licenses.gpl2;
+    };
+  };
+
+  wsgi-dashboards = buildPythonPackage rec {
+    pname = "buildbot-wsgi-dashboards";
+    inherit (buildbot-pkg) version;
+
+    src = fetchPypi {
+      inherit pname version;
+      hash = "sha256-5Qr1FeYIJG/qaFaTB7ScFN9uca+joHKE6FlfKwhubfo=";
+    };
+
+    buildInputs = [ buildbot-pkg ];
+
+    # No tests
+    doCheck = false;
+
+    meta = with lib; {
+      homepage = "https://buildbot.net/";
+      description = "Buildbot WSGI dashboards Plugin";
+      maintainers = with maintainers; [ lopsided98 ];
+      license = licenses.gpl2;
+    };
+  };
+
+  badges = buildPythonPackage rec {
+    pname = "buildbot-badges";
+    inherit (buildbot-pkg) version;
+
+    src = fetchPypi {
+      inherit pname version;
+      hash = "sha256-H0Dn+uTtFyZgyqbk3QQEc5t7CJovyzU+XuCoTe4Ajug=";
+    };
+
+    buildInputs = [ buildbot-pkg ];
+    propagatedBuildInputs = [ cairosvg klein jinja2 ];
+
+    # No tests
+    doCheck = false;
+
+    meta = with lib; {
+      homepage = "https://buildbot.net/";
+      description = "Buildbot Badges Plugin";
+      maintainers = with maintainers; [ julienmalka ];
+      license = licenses.gpl2;
+    };
+  };
+
+}
diff --git a/pkgs/development/tools/continuous-integration/buildbot/skip_test_linux_distro.patch b/pkgs/development/tools/continuous-integration/buildbot/skip_test_linux_distro.patch
new file mode 100644
index 0000000000000..8fe5c7b56b4f2
--- /dev/null
+++ b/pkgs/development/tools/continuous-integration/buildbot/skip_test_linux_distro.patch
@@ -0,0 +1,11 @@
+diff -Nur buildbot-0.9.6/buildbot/test/unit/test_buildbot_net_usage_data.py buildbot-0.9.6.patched/buildbot/test/unit/test_buildbot_net_usage_data.py
+--- buildbot-0.9.6/buildbot/test/unit/test_buildbot_net_usage_data.py	2017-04-19 16:57:02.000000000 +0200
++++ buildbot-0.9.6.patched/buildbot/test/unit/test_buildbot_net_usage_data.py	2017-05-04 12:22:54.575762551 +0200
+@@ -147,6 +147,7 @@
+         _sendBuildbotNetUsageData({'foo': 'bar'})
+ 
+     def test_linux_distro(self):
++        raise SkipTest("NixOS sandboxed builds hides /etc/os-release")
+         system = platform.system()
+         if system != "Linux":
+             raise SkipTest("test is only for linux")
diff --git a/pkgs/development/tools/continuous-integration/buildbot/update.sh b/pkgs/development/tools/continuous-integration/buildbot/update.sh
new file mode 100755
index 0000000000000..3406f05db3e80
--- /dev/null
+++ b/pkgs/development/tools/continuous-integration/buildbot/update.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env nix-shell
+#!nix-shell -i bash -p nix-update
+set -eu -o pipefail
+
+nix-update python3Packages.buildbot
+nix-update --version=skip python3Packages.buildbot-worker
+nix-update --version=skip python3Packages.buildbot-pkg
+nix-update --version=skip python3Packages.buildbot-plugins.www
+nix-update --version=skip python3Packages.buildbot-plugins.console-view
+nix-update --version=skip python3Packages.buildbot-plugins.waterfall-view
+nix-update --version=skip python3Packages.buildbot-plugins.grid-view
+nix-update --version=skip python3Packages.buildbot-plugins.wsgi-dashboards
diff --git a/pkgs/development/tools/continuous-integration/buildbot/worker.nix b/pkgs/development/tools/continuous-integration/buildbot/worker.nix
new file mode 100644
index 0000000000000..3ea0feb293ec5
--- /dev/null
+++ b/pkgs/development/tools/continuous-integration/buildbot/worker.nix
@@ -0,0 +1,66 @@
+{ lib
+, buildPythonPackage
+, fetchPypi
+, buildbot
+
+# patch
+, coreutils
+
+# propagates
+, autobahn
+, future
+, msgpack
+, twisted
+
+# tests
+, mock
+, parameterized
+, psutil
+, setuptoolsTrial
+
+# passthru
+, nixosTests
+}:
+
+buildPythonPackage (rec {
+  pname = "buildbot-worker";
+  inherit (buildbot) version;
+
+  src = fetchPypi {
+    inherit pname version;
+    hash = "sha256-Lc+FNrfXLfeEnDqNBs9R96jtoFEOCG2vLRWGKip/+VM=";
+  };
+
+  postPatch = ''
+    substituteInPlace buildbot_worker/scripts/logwatcher.py \
+      --replace /usr/bin/tail "${coreutils}/bin/tail"
+  '';
+
+  nativeBuildInputs = [
+    setuptoolsTrial
+  ];
+
+  propagatedBuildInputs = [
+    autobahn
+    future
+    msgpack
+    twisted
+  ];
+
+  nativeCheckInputs = [
+    mock
+    parameterized
+    psutil
+  ];
+
+  passthru.tests = {
+    smoke-test = nixosTests.buildbot;
+  };
+
+  meta = with lib; {
+    homepage = "https://buildbot.net/";
+    description = "Buildbot Worker Daemon";
+    maintainers = with maintainers; [ ryansydnor lopsided98 ];
+    license = licenses.gpl2;
+  };
+})