From 90b750456ea62408dc4c0bbe68adabdd96e381a7 Mon Sep 17 00:00:00 2001 From: Jörg Thalheim Date: Fri, 7 Apr 2023 18:34:08 +0200 Subject: buildbot: move out of python3.pkgs By moving it out we can start apply overrides to python dependendencies starting with sqlalchemy. This fixes the build with the current version. --- .../continuous-integration/buildbot/master.nix | 11 +- .../continuous-integration/buildbot/worker.nix | 7 +- nixos/tests/buildbot.nix | 4 +- .../python-modules/buildbot/default.nix | 152 -------------------- pkgs/development/python-modules/buildbot/pkg.nix | 31 ----- .../python-modules/buildbot/plugins.nix | 142 ------------------- .../buildbot/skip_test_linux_distro.patch | 11 -- pkgs/development/python-modules/buildbot/update.sh | 12 -- .../development/python-modules/buildbot/worker.nix | 66 --------- .../continuous-integration/buildbot/default.nix | 42 ++++++ .../continuous-integration/buildbot/master.nix | 154 +++++++++++++++++++++ .../tools/continuous-integration/buildbot/pkg.nix | 31 +++++ .../continuous-integration/buildbot/plugins.nix | 141 +++++++++++++++++++ .../buildbot/skip_test_linux_distro.patch | 11 ++ .../continuous-integration/buildbot/update.sh | 12 ++ .../continuous-integration/buildbot/worker.nix | 66 +++++++++ pkgs/top-level/all-packages.nix | 6 +- pkgs/top-level/python-aliases.nix | 6 + pkgs/top-level/python-packages.nix | 12 -- 19 files changed, 477 insertions(+), 440 deletions(-) delete mode 100644 pkgs/development/python-modules/buildbot/default.nix delete mode 100644 pkgs/development/python-modules/buildbot/pkg.nix delete mode 100644 pkgs/development/python-modules/buildbot/plugins.nix delete mode 100644 pkgs/development/python-modules/buildbot/skip_test_linux_distro.patch delete mode 100755 pkgs/development/python-modules/buildbot/update.sh delete mode 100644 pkgs/development/python-modules/buildbot/worker.nix create mode 100644 pkgs/development/tools/continuous-integration/buildbot/default.nix create mode 100644 pkgs/development/tools/continuous-integration/buildbot/master.nix create mode 100644 pkgs/development/tools/continuous-integration/buildbot/pkg.nix create mode 100644 pkgs/development/tools/continuous-integration/buildbot/plugins.nix create mode 100644 pkgs/development/tools/continuous-integration/buildbot/skip_test_linux_distro.patch create mode 100755 pkgs/development/tools/continuous-integration/buildbot/update.sh create mode 100644 pkgs/development/tools/continuous-integration/buildbot/worker.nix diff --git a/nixos/modules/services/continuous-integration/buildbot/master.nix b/nixos/modules/services/continuous-integration/buildbot/master.nix index 5666199c48453..595374ea1e5b3 100644 --- a/nixos/modules/services/continuous-integration/buildbot/master.nix +++ b/nixos/modules/services/continuous-integration/buildbot/master.nix @@ -8,7 +8,8 @@ let cfg = config.services.buildbot-master; opt = options.services.buildbot-master; - python = cfg.package.pythonModule; + package = pkgs.python3.pkgs.toPythonModule cfg.package; + python = package.pythonModule; escapeStr = escape [ "'" ]; @@ -212,10 +213,10 @@ in { package = mkOption { type = types.package; - default = pkgs.python3Packages.buildbot-full; - defaultText = literalExpression "pkgs.python3Packages.buildbot-full"; + default = pkgs.buildbot-full; + defaultText = literalExpression "pkgs.buildbot-full"; description = lib.mdDoc "Package to use for buildbot."; - example = literalExpression "pkgs.python3Packages.buildbot"; + example = literalExpression "pkgs.buildbot"; }; packages = mkOption { @@ -255,7 +256,7 @@ in { after = [ "network-online.target" ]; wantedBy = [ "multi-user.target" ]; path = cfg.packages ++ cfg.pythonPackages python.pkgs; - environment.PYTHONPATH = "${python.withPackages (self: cfg.pythonPackages self ++ [ cfg.package ])}/${python.sitePackages}"; + environment.PYTHONPATH = "${python.withPackages (self: cfg.pythonPackages self ++ [ package ])}/${python.sitePackages}"; preStart = '' mkdir -vp "${cfg.buildbotDir}" diff --git a/nixos/modules/services/continuous-integration/buildbot/worker.nix b/nixos/modules/services/continuous-integration/buildbot/worker.nix index 52c41c4a7584c..7e78b8935f81c 100644 --- a/nixos/modules/services/continuous-integration/buildbot/worker.nix +++ b/nixos/modules/services/continuous-integration/buildbot/worker.nix @@ -8,7 +8,8 @@ let cfg = config.services.buildbot-worker; opt = options.services.buildbot-worker; - python = cfg.package.pythonModule; + package = pkgs.python3.pkgs.toPythonModule cfg.package; + python = package.pythonModule; tacFile = pkgs.writeText "aur-buildbot-worker.tac" '' import os @@ -129,7 +130,7 @@ in { package = mkOption { type = types.package; - default = pkgs.python3Packages.buildbot-worker; + default = pkgs.buildbot-worker; defaultText = literalExpression "pkgs.python3Packages.buildbot-worker"; description = lib.mdDoc "Package to use for buildbot worker."; example = literalExpression "pkgs.python2Packages.buildbot-worker"; @@ -168,7 +169,7 @@ in { after = [ "network.target" "buildbot-master.service" ]; wantedBy = [ "multi-user.target" ]; path = cfg.packages; - environment.PYTHONPATH = "${python.withPackages (p: [ cfg.package ])}/${python.sitePackages}"; + environment.PYTHONPATH = "${python.withPackages (p: [ package ])}/${python.sitePackages}"; preStart = '' mkdir -vp "${cfg.buildbotDir}/info" diff --git a/nixos/tests/buildbot.nix b/nixos/tests/buildbot.nix index 977c728835f03..467c8d8baff40 100644 --- a/nixos/tests/buildbot.nix +++ b/nixos/tests/buildbot.nix @@ -23,7 +23,7 @@ import ./make-test-python.nix { ]; }; networking.firewall.allowedTCPPorts = [ 8010 8011 9989 ]; - environment.systemPackages = with pkgs; [ git python3Packages.buildbot-full ]; + environment.systemPackages = with pkgs; [ git buildbot-full ]; }; bbworker = { pkgs, ... }: { @@ -31,7 +31,7 @@ import ./make-test-python.nix { enable = true; masterUrl = "bbmaster:9989"; }; - environment.systemPackages = with pkgs; [ git python3Packages.buildbot-worker ]; + environment.systemPackages = with pkgs; [ git buildbot-worker ]; }; gitrepo = { pkgs, ... }: { diff --git a/pkgs/development/python-modules/buildbot/default.nix b/pkgs/development/python-modules/buildbot/default.nix deleted file mode 100644 index c1a0de7182006..0000000000000 --- a/pkgs/development/python-modules/buildbot/default.nix +++ /dev/null @@ -1,152 +0,0 @@ -{ lib -, stdenv -, buildPythonPackage -, fetchPypi -, makeWrapper -, pythonOlder -, python -, twisted -, jinja2 -, msgpack -, zope_interface -, sqlalchemy -, alembic -, python-dateutil -, txaio -, autobahn -, pyjwt -, pyyaml -, treq -, txrequests -, pypugjs -, boto3 -, moto -, mock -, lz4 -, setuptoolsTrial -, buildbot-worker -, buildbot-pkg -, buildbot-plugins -, parameterized -, git -, openssh -, glibcLocales -, nixosTests -}: - -let - withPlugins = plugins: buildPythonPackage { - pname = "${package.pname}-with-plugins"; - inherit (package) version; - format = "other"; - - dontUnpack = true; - dontBuild = true; - doCheck = false; - - nativeBuildInputs = [ - makeWrapper - ]; - - propagatedBuildInputs = plugins ++ package.propagatedBuildInputs; - - installPhase = '' - makeWrapper ${package}/bin/buildbot $out/bin/buildbot \ - --prefix PYTHONPATH : "${package}/${python.sitePackages}:$PYTHONPATH" - ln -sfv ${package}/lib $out/lib - ''; - - passthru = package.passthru // { - withPlugins = morePlugins: withPlugins (morePlugins ++ plugins); - }; - }; - - package = buildPythonPackage rec { - pname = "buildbot"; - version = "3.7.0"; - format = "setuptools"; - - disabled = pythonOlder "3.7"; - - src = fetchPypi { - inherit pname version; - hash = "sha256-YMLT1SP6NenJIUVTvr58GVrtNXHw+bhfgMpZu3revG4="; - }; - - propagatedBuildInputs = [ - # core - twisted - jinja2 - msgpack - zope_interface - sqlalchemy - alembic - python-dateutil - txaio - autobahn - pyjwt - pyyaml - ] - # tls - ++ twisted.optional-dependencies.tls; - - nativeCheckInputs = [ - treq - txrequests - pypugjs - boto3 - moto - mock - lz4 - setuptoolsTrial - buildbot-worker - buildbot-pkg - buildbot-plugins.www - parameterized - git - openssh - glibcLocales - ]; - - patches = [ - # This patch disables the test that tries to read /etc/os-release which - # is not accessible in sandboxed builds. - ./skip_test_linux_distro.patch - ]; - - postPatch = '' - substituteInPlace buildbot/scripts/logwatcher.py --replace '/usr/bin/tail' "$(type -P tail)" - ''; - - # Silence the depreciation warning from SqlAlchemy - SQLALCHEMY_SILENCE_UBER_WARNING = 1; - - # TimeoutErrors on slow machines -> aarch64 - doCheck = !stdenv.isAarch64; - - preCheck = '' - export LC_ALL="en_US.UTF-8" - export PATH="$out/bin:$PATH" - - # remove testfile which is missing configuration file from sdist - rm buildbot/test/integration/test_graphql.py - # tests in this file are flaky, see https://github.com/buildbot/buildbot/issues/6776 - rm buildbot/test/integration/test_try_client.py - ''; - - passthru = { - inherit withPlugins; - tests.buildbot = nixosTests.buildbot; - updateScript = ./update.sh; - }; - - meta = with lib; { - description = "An open-source continuous integration framework for automating software build, test, and release processes"; - homepage = "https://buildbot.net/"; - changelog = "https://github.com/buildbot/buildbot/releases/tag/v${version}"; - maintainers = with maintainers; [ ryansydnor lopsided98 ]; - license = licenses.gpl2Only; - broken = stdenv.isDarwin; - }; - }; -in package diff --git a/pkgs/development/python-modules/buildbot/pkg.nix b/pkgs/development/python-modules/buildbot/pkg.nix deleted file mode 100644 index 13c0ec8694854..0000000000000 --- a/pkgs/development/python-modules/buildbot/pkg.nix +++ /dev/null @@ -1,31 +0,0 @@ -{ lib, buildPythonPackage, fetchPypi, isPy3k, buildbot }: - -buildPythonPackage rec { - pname = "buildbot-pkg"; - inherit (buildbot) version; - - src = fetchPypi { - inherit pname version; - hash = "sha256-4BXCOLW5e7RuZEzyD+oRmS2I4oT2W3oTcH4ZPxwKKvU="; - }; - - postPatch = '' - # Their listdir function filters out `node_modules` folders. - # Do we have to care about that with Nix...? - substituteInPlace buildbot_pkg.py --replace "os.listdir = listdir" "" - ''; - - # No tests - doCheck = false; - - pythonImportsCheck = [ "buildbot_pkg" ]; - - disabled = !isPy3k; - - meta = with lib; { - homepage = "https://buildbot.net/"; - description = "Buildbot Packaging Helper"; - maintainers = with maintainers; [ ryansydnor lopsided98 ]; - license = licenses.gpl2; - }; -} diff --git a/pkgs/development/python-modules/buildbot/plugins.nix b/pkgs/development/python-modules/buildbot/plugins.nix deleted file mode 100644 index df9efd66efe86..0000000000000 --- a/pkgs/development/python-modules/buildbot/plugins.nix +++ /dev/null @@ -1,142 +0,0 @@ -{ lib, buildPythonPackage, fetchPypi, buildbot-pkg, mock, cairosvg, klein, jinja2 }: - -{ - www = buildPythonPackage rec { - pname = "buildbot-www"; - inherit (buildbot-pkg) version; - - src = fetchPypi { - inherit pname version; - hash = "sha256-t4xHfox6h5PY4+phdCQbClHd77+WfpUWErMZCEcMxu0="; - }; - - # Remove unnecessary circular dependency on buildbot - postPatch = '' - sed -i "s/'buildbot'//" setup.py - ''; - - buildInputs = [ buildbot-pkg mock ]; - - # No tests - doCheck = false; - - meta = with lib; { - homepage = "https://buildbot.net/"; - description = "Buildbot UI"; - maintainers = with maintainers; [ ryansydnor lopsided98 ]; - license = licenses.gpl2; - }; - }; - - console-view = buildPythonPackage rec { - pname = "buildbot-console-view"; - inherit (buildbot-pkg) version; - - src = fetchPypi { - inherit pname version; - hash = "sha256-JySn7QO+SgoXjVeV4wYwc9twr0Q2c/wsEspqeq038+k="; - }; - - buildInputs = [ buildbot-pkg ]; - - # No tests - doCheck = false; - - meta = with lib; { - homepage = "https://buildbot.net/"; - description = "Buildbot Console View Plugin"; - maintainers = with maintainers; [ ryansydnor lopsided98 ]; - license = licenses.gpl2; - }; - }; - - waterfall-view = buildPythonPackage rec { - pname = "buildbot-waterfall-view"; - inherit (buildbot-pkg) version; - - src = fetchPypi { - inherit pname version; - hash = "sha256-935eeF2kpT68lK/UMg8MZQOYEj7D8FaT9iSs/lNahYA="; - }; - - buildInputs = [ buildbot-pkg ]; - - # No tests - doCheck = false; - - meta = with lib; { - homepage = "https://buildbot.net/"; - description = "Buildbot Waterfall View Plugin"; - maintainers = with maintainers; [ ryansydnor lopsided98 ]; - license = licenses.gpl2; - }; - }; - - grid-view = buildPythonPackage rec { - pname = "buildbot-grid-view"; - inherit (buildbot-pkg) version; - - src = fetchPypi { - inherit pname version; - hash = "sha256-3pHSiVoOZj2iCGfiz+tMWWMPHSBH5Ggp6e3+a8topsg="; - }; - - buildInputs = [ buildbot-pkg ]; - - # No tests - doCheck = false; - - meta = with lib; { - homepage = "https://buildbot.net/"; - description = "Buildbot Grid View Plugin"; - maintainers = with maintainers; [ lopsided98 ]; - license = licenses.gpl2; - }; - }; - - wsgi-dashboards = buildPythonPackage rec { - pname = "buildbot-wsgi-dashboards"; - inherit (buildbot-pkg) version; - - src = fetchPypi { - inherit pname version; - hash = "sha256-5Qr1FeYIJG/qaFaTB7ScFN9uca+joHKE6FlfKwhubfo="; - }; - - buildInputs = [ buildbot-pkg ]; - - # No tests - doCheck = false; - - meta = with lib; { - homepage = "https://buildbot.net/"; - description = "Buildbot WSGI dashboards Plugin"; - maintainers = with maintainers; [ lopsided98 ]; - license = licenses.gpl2; - }; - }; - - badges = buildPythonPackage rec { - pname = "buildbot-badges"; - inherit (buildbot-pkg) version; - - src = fetchPypi { - inherit pname version; - hash = "sha256-H0Dn+uTtFyZgyqbk3QQEc5t7CJovyzU+XuCoTe4Ajug="; - }; - - buildInputs = [ buildbot-pkg ]; - propagatedBuildInputs = [ cairosvg klein jinja2 ]; - - # No tests - doCheck = false; - - meta = with lib; { - homepage = "https://buildbot.net/"; - description = "Buildbot Badges Plugin"; - maintainers = with maintainers; [ julienmalka ]; - license = licenses.gpl2; - }; - }; - -} diff --git a/pkgs/development/python-modules/buildbot/skip_test_linux_distro.patch b/pkgs/development/python-modules/buildbot/skip_test_linux_distro.patch deleted file mode 100644 index 8fe5c7b56b4f2..0000000000000 --- a/pkgs/development/python-modules/buildbot/skip_test_linux_distro.patch +++ /dev/null @@ -1,11 +0,0 @@ -diff -Nur buildbot-0.9.6/buildbot/test/unit/test_buildbot_net_usage_data.py buildbot-0.9.6.patched/buildbot/test/unit/test_buildbot_net_usage_data.py ---- buildbot-0.9.6/buildbot/test/unit/test_buildbot_net_usage_data.py 2017-04-19 16:57:02.000000000 +0200 -+++ buildbot-0.9.6.patched/buildbot/test/unit/test_buildbot_net_usage_data.py 2017-05-04 12:22:54.575762551 +0200 -@@ -147,6 +147,7 @@ - _sendBuildbotNetUsageData({'foo': 'bar'}) - - def test_linux_distro(self): -+ raise SkipTest("NixOS sandboxed builds hides /etc/os-release") - system = platform.system() - if system != "Linux": - raise SkipTest("test is only for linux") diff --git a/pkgs/development/python-modules/buildbot/update.sh b/pkgs/development/python-modules/buildbot/update.sh deleted file mode 100755 index 3406f05db3e80..0000000000000 --- a/pkgs/development/python-modules/buildbot/update.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env nix-shell -#!nix-shell -i bash -p nix-update -set -eu -o pipefail - -nix-update python3Packages.buildbot -nix-update --version=skip python3Packages.buildbot-worker -nix-update --version=skip python3Packages.buildbot-pkg -nix-update --version=skip python3Packages.buildbot-plugins.www -nix-update --version=skip python3Packages.buildbot-plugins.console-view -nix-update --version=skip python3Packages.buildbot-plugins.waterfall-view -nix-update --version=skip python3Packages.buildbot-plugins.grid-view -nix-update --version=skip python3Packages.buildbot-plugins.wsgi-dashboards diff --git a/pkgs/development/python-modules/buildbot/worker.nix b/pkgs/development/python-modules/buildbot/worker.nix deleted file mode 100644 index 3ea0feb293ec5..0000000000000 --- a/pkgs/development/python-modules/buildbot/worker.nix +++ /dev/null @@ -1,66 +0,0 @@ -{ lib -, buildPythonPackage -, fetchPypi -, buildbot - -# patch -, coreutils - -# propagates -, autobahn -, future -, msgpack -, twisted - -# tests -, mock -, parameterized -, psutil -, setuptoolsTrial - -# passthru -, nixosTests -}: - -buildPythonPackage (rec { - pname = "buildbot-worker"; - inherit (buildbot) version; - - src = fetchPypi { - inherit pname version; - hash = "sha256-Lc+FNrfXLfeEnDqNBs9R96jtoFEOCG2vLRWGKip/+VM="; - }; - - postPatch = '' - substituteInPlace buildbot_worker/scripts/logwatcher.py \ - --replace /usr/bin/tail "${coreutils}/bin/tail" - ''; - - nativeBuildInputs = [ - setuptoolsTrial - ]; - - propagatedBuildInputs = [ - autobahn - future - msgpack - twisted - ]; - - nativeCheckInputs = [ - mock - parameterized - psutil - ]; - - passthru.tests = { - smoke-test = nixosTests.buildbot; - }; - - meta = with lib; { - homepage = "https://buildbot.net/"; - description = "Buildbot Worker Daemon"; - maintainers = with maintainers; [ ryansydnor lopsided98 ]; - license = licenses.gpl2; - }; -}) diff --git a/pkgs/development/tools/continuous-integration/buildbot/default.nix b/pkgs/development/tools/continuous-integration/buildbot/default.nix new file mode 100644 index 0000000000000..ec3a4a81f60f0 --- /dev/null +++ b/pkgs/development/tools/continuous-integration/buildbot/default.nix @@ -0,0 +1,42 @@ +{ python3 +, recurseIntoAttrs +, callPackage +}: +let + python = python3.override { + packageOverrides = self: super: { + sqlalchemy = super.sqlalchemy.overridePythonAttrs (oldAttrs: rec { + version = "1.4.40"; + src = super.fetchPypi { + pname = "SQLAlchemy"; + inherit version; + hash = "sha256-RKZgUGCAzJdeHfpXdv5fYxXdxiane1C/Du4YsDieomU="; + }; + }); + moto = super.moto.overridePythonAttrs (oldAttrs: rec { + # a lot of tests -> very slow, we already build them when building python packages + doCheck = false; + }); + }; + }; + + buildbot-pkg = python.pkgs.callPackage ./pkg.nix { + inherit buildbot; + }; + buildbot-worker = python3.pkgs.callPackage ./worker.nix { + inherit buildbot; + }; + buildbot = python.pkgs.callPackage ./master.nix { + inherit buildbot-pkg buildbot-worker buildbot-plugins; + }; + buildbot-plugins = recurseIntoAttrs (callPackage ./plugins.nix { + inherit buildbot-pkg; + }); +in +{ + inherit buildbot buildbot-plugins buildbot-worker; + buildbot-ui = buildbot.withPlugins (with buildbot-plugins; [ www ]); + buildbot-full = buildbot.withPlugins (with buildbot-plugins; [ + www console-view waterfall-view grid-view wsgi-dashboards badges + ]); +} diff --git a/pkgs/development/tools/continuous-integration/buildbot/master.nix b/pkgs/development/tools/continuous-integration/buildbot/master.nix new file mode 100644 index 0000000000000..6e59e31867b62 --- /dev/null +++ b/pkgs/development/tools/continuous-integration/buildbot/master.nix @@ -0,0 +1,154 @@ +{ lib +, stdenv +, buildPythonPackage +, buildPythonApplication +, fetchPypi +, makeWrapper +, pythonOlder +, python +, twisted +, jinja2 +, msgpack +, zope_interface +, sqlalchemy +, alembic +, python-dateutil +, txaio +, autobahn +, pyjwt +, pyyaml +, treq +, txrequests +, pypugjs +, boto3 +, moto +, mock +, lz4 +, setuptoolsTrial +, buildbot-worker +, buildbot-plugins +, buildbot-pkg +, parameterized +, git +, openssh +, glibcLocales +, nixosTests +, callPackage +}: + +let + withPlugins = plugins: buildPythonApplication { + pname = "${package.pname}-with-plugins"; + inherit (package) version; + format = "other"; + + dontUnpack = true; + dontBuild = true; + doCheck = false; + + nativeBuildInputs = [ + makeWrapper + ]; + + propagatedBuildInputs = plugins ++ package.propagatedBuildInputs; + + installPhase = '' + makeWrapper ${package}/bin/buildbot $out/bin/buildbot \ + --prefix PYTHONPATH : "${package}/${python.sitePackages}:$PYTHONPATH" + ln -sfv ${package}/lib $out/lib + ''; + + passthru = package.passthru // { + withPlugins = morePlugins: withPlugins (morePlugins ++ plugins); + }; + }; + + package = buildPythonApplication rec { + pname = "buildbot"; + version = "3.7.0"; + format = "setuptools"; + + disabled = pythonOlder "3.7"; + + src = fetchPypi { + inherit pname version; + hash = "sha256-YMLT1SP6NenJIUVTvr58GVrtNXHw+bhfgMpZu3revG4="; + }; + + propagatedBuildInputs = [ + # core + twisted + jinja2 + msgpack + zope_interface + sqlalchemy + alembic + python-dateutil + txaio + autobahn + pyjwt + pyyaml + ] + # tls + ++ twisted.optional-dependencies.tls; + + nativeCheckInputs = [ + treq + txrequests + pypugjs + boto3 + moto + mock + lz4 + setuptoolsTrial + buildbot-worker + buildbot-pkg + buildbot-plugins.www + parameterized + git + openssh + glibcLocales + ]; + + patches = [ + # This patch disables the test that tries to read /etc/os-release which + # is not accessible in sandboxed builds. + ./skip_test_linux_distro.patch + ]; + + postPatch = '' + substituteInPlace buildbot/scripts/logwatcher.py --replace '/usr/bin/tail' "$(type -P tail)" + ''; + + # Silence the depreciation warning from SqlAlchemy + SQLALCHEMY_SILENCE_UBER_WARNING = 1; + + # TimeoutErrors on slow machines -> aarch64 + doCheck = !stdenv.isAarch64; + + preCheck = '' + export LC_ALL="en_US.UTF-8" + export PATH="$out/bin:$PATH" + + # remove testfile which is missing configuration file from sdist + rm buildbot/test/integration/test_graphql.py + # tests in this file are flaky, see https://github.com/buildbot/buildbot/issues/6776 + rm buildbot/test/integration/test_try_client.py + ''; + + passthru = { + inherit withPlugins; + tests.buildbot = nixosTests.buildbot; + updateScript = ./update.sh; + }; + + meta = with lib; { + description = "An open-source continuous integration framework for automating software build, test, and release processes"; + homepage = "https://buildbot.net/"; + changelog = "https://github.com/buildbot/buildbot/releases/tag/v${version}"; + maintainers = with maintainers; [ ryansydnor lopsided98 ]; + license = licenses.gpl2Only; + broken = stdenv.isDarwin; + }; + }; +in package diff --git a/pkgs/development/tools/continuous-integration/buildbot/pkg.nix b/pkgs/development/tools/continuous-integration/buildbot/pkg.nix new file mode 100644 index 0000000000000..13c0ec8694854 --- /dev/null +++ b/pkgs/development/tools/continuous-integration/buildbot/pkg.nix @@ -0,0 +1,31 @@ +{ lib, buildPythonPackage, fetchPypi, isPy3k, buildbot }: + +buildPythonPackage rec { + pname = "buildbot-pkg"; + inherit (buildbot) version; + + src = fetchPypi { + inherit pname version; + hash = "sha256-4BXCOLW5e7RuZEzyD+oRmS2I4oT2W3oTcH4ZPxwKKvU="; + }; + + postPatch = '' + # Their listdir function filters out `node_modules` folders. + # Do we have to care about that with Nix...? + substituteInPlace buildbot_pkg.py --replace "os.listdir = listdir" "" + ''; + + # No tests + doCheck = false; + + pythonImportsCheck = [ "buildbot_pkg" ]; + + disabled = !isPy3k; + + meta = with lib; { + homepage = "https://buildbot.net/"; + description = "Buildbot Packaging Helper"; + maintainers = with maintainers; [ ryansydnor lopsided98 ]; + license = licenses.gpl2; + }; +} diff --git a/pkgs/development/tools/continuous-integration/buildbot/plugins.nix b/pkgs/development/tools/continuous-integration/buildbot/plugins.nix new file mode 100644 index 0000000000000..42a77fa0f4bc5 --- /dev/null +++ b/pkgs/development/tools/continuous-integration/buildbot/plugins.nix @@ -0,0 +1,141 @@ +{ lib, buildPythonPackage, fetchPypi, callPackage, mock, cairosvg, klein, jinja2, buildbot-pkg }: +{ + www = buildPythonPackage rec { + pname = "buildbot-www"; + inherit (buildbot-pkg) version; + + src = fetchPypi { + inherit pname version; + hash = "sha256-t4xHfox6h5PY4+phdCQbClHd77+WfpUWErMZCEcMxu0="; + }; + + # Remove unnecessary circular dependency on buildbot + postPatch = '' + sed -i "s/'buildbot'//" setup.py + ''; + + buildInputs = [ buildbot-pkg mock ]; + + # No tests + doCheck = false; + + meta = with lib; { + homepage = "https://buildbot.net/"; + description = "Buildbot UI"; + maintainers = with maintainers; [ ryansydnor lopsided98 ]; + license = licenses.gpl2; + }; + }; + + console-view = buildPythonPackage rec { + pname = "buildbot-console-view"; + inherit (buildbot-pkg) version; + + src = fetchPypi { + inherit pname version; + hash = "sha256-JySn7QO+SgoXjVeV4wYwc9twr0Q2c/wsEspqeq038+k="; + }; + + buildInputs = [ buildbot-pkg ]; + + # No tests + doCheck = false; + + meta = with lib; { + homepage = "https://buildbot.net/"; + description = "Buildbot Console View Plugin"; + maintainers = with maintainers; [ ryansydnor lopsided98 ]; + license = licenses.gpl2; + }; + }; + + waterfall-view = buildPythonPackage rec { + pname = "buildbot-waterfall-view"; + inherit (buildbot-pkg) version; + + src = fetchPypi { + inherit pname version; + hash = "sha256-935eeF2kpT68lK/UMg8MZQOYEj7D8FaT9iSs/lNahYA="; + }; + + buildInputs = [ buildbot-pkg ]; + + # No tests + doCheck = false; + + meta = with lib; { + homepage = "https://buildbot.net/"; + description = "Buildbot Waterfall View Plugin"; + maintainers = with maintainers; [ ryansydnor lopsided98 ]; + license = licenses.gpl2; + }; + }; + + grid-view = buildPythonPackage rec { + pname = "buildbot-grid-view"; + inherit (buildbot-pkg) version; + + src = fetchPypi { + inherit pname version; + hash = "sha256-3pHSiVoOZj2iCGfiz+tMWWMPHSBH5Ggp6e3+a8topsg="; + }; + + buildInputs = [ buildbot-pkg ]; + + # No tests + doCheck = false; + + meta = with lib; { + homepage = "https://buildbot.net/"; + description = "Buildbot Grid View Plugin"; + maintainers = with maintainers; [ lopsided98 ]; + license = licenses.gpl2; + }; + }; + + wsgi-dashboards = buildPythonPackage rec { + pname = "buildbot-wsgi-dashboards"; + inherit (buildbot-pkg) version; + + src = fetchPypi { + inherit pname version; + hash = "sha256-5Qr1FeYIJG/qaFaTB7ScFN9uca+joHKE6FlfKwhubfo="; + }; + + buildInputs = [ buildbot-pkg ]; + + # No tests + doCheck = false; + + meta = with lib; { + homepage = "https://buildbot.net/"; + description = "Buildbot WSGI dashboards Plugin"; + maintainers = with maintainers; [ lopsided98 ]; + license = licenses.gpl2; + }; + }; + + badges = buildPythonPackage rec { + pname = "buildbot-badges"; + inherit (buildbot-pkg) version; + + src = fetchPypi { + inherit pname version; + hash = "sha256-H0Dn+uTtFyZgyqbk3QQEc5t7CJovyzU+XuCoTe4Ajug="; + }; + + buildInputs = [ buildbot-pkg ]; + propagatedBuildInputs = [ cairosvg klein jinja2 ]; + + # No tests + doCheck = false; + + meta = with lib; { + homepage = "https://buildbot.net/"; + description = "Buildbot Badges Plugin"; + maintainers = with maintainers; [ julienmalka ]; + license = licenses.gpl2; + }; + }; + +} diff --git a/pkgs/development/tools/continuous-integration/buildbot/skip_test_linux_distro.patch b/pkgs/development/tools/continuous-integration/buildbot/skip_test_linux_distro.patch new file mode 100644 index 0000000000000..8fe5c7b56b4f2 --- /dev/null +++ b/pkgs/development/tools/continuous-integration/buildbot/skip_test_linux_distro.patch @@ -0,0 +1,11 @@ +diff -Nur buildbot-0.9.6/buildbot/test/unit/test_buildbot_net_usage_data.py buildbot-0.9.6.patched/buildbot/test/unit/test_buildbot_net_usage_data.py +--- buildbot-0.9.6/buildbot/test/unit/test_buildbot_net_usage_data.py 2017-04-19 16:57:02.000000000 +0200 ++++ buildbot-0.9.6.patched/buildbot/test/unit/test_buildbot_net_usage_data.py 2017-05-04 12:22:54.575762551 +0200 +@@ -147,6 +147,7 @@ + _sendBuildbotNetUsageData({'foo': 'bar'}) + + def test_linux_distro(self): ++ raise SkipTest("NixOS sandboxed builds hides /etc/os-release") + system = platform.system() + if system != "Linux": + raise SkipTest("test is only for linux") diff --git a/pkgs/development/tools/continuous-integration/buildbot/update.sh b/pkgs/development/tools/continuous-integration/buildbot/update.sh new file mode 100755 index 0000000000000..3406f05db3e80 --- /dev/null +++ b/pkgs/development/tools/continuous-integration/buildbot/update.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p nix-update +set -eu -o pipefail + +nix-update python3Packages.buildbot +nix-update --version=skip python3Packages.buildbot-worker +nix-update --version=skip python3Packages.buildbot-pkg +nix-update --version=skip python3Packages.buildbot-plugins.www +nix-update --version=skip python3Packages.buildbot-plugins.console-view +nix-update --version=skip python3Packages.buildbot-plugins.waterfall-view +nix-update --version=skip python3Packages.buildbot-plugins.grid-view +nix-update --version=skip python3Packages.buildbot-plugins.wsgi-dashboards diff --git a/pkgs/development/tools/continuous-integration/buildbot/worker.nix b/pkgs/development/tools/continuous-integration/buildbot/worker.nix new file mode 100644 index 0000000000000..3ea0feb293ec5 --- /dev/null +++ b/pkgs/development/tools/continuous-integration/buildbot/worker.nix @@ -0,0 +1,66 @@ +{ lib +, buildPythonPackage +, fetchPypi +, buildbot + +# patch +, coreutils + +# propagates +, autobahn +, future +, msgpack +, twisted + +# tests +, mock +, parameterized +, psutil +, setuptoolsTrial + +# passthru +, nixosTests +}: + +buildPythonPackage (rec { + pname = "buildbot-worker"; + inherit (buildbot) version; + + src = fetchPypi { + inherit pname version; + hash = "sha256-Lc+FNrfXLfeEnDqNBs9R96jtoFEOCG2vLRWGKip/+VM="; + }; + + postPatch = '' + substituteInPlace buildbot_worker/scripts/logwatcher.py \ + --replace /usr/bin/tail "${coreutils}/bin/tail" + ''; + + nativeBuildInputs = [ + setuptoolsTrial + ]; + + propagatedBuildInputs = [ + autobahn + future + msgpack + twisted + ]; + + nativeCheckInputs = [ + mock + parameterized + psutil + ]; + + passthru.tests = { + smoke-test = nixosTests.buildbot; + }; + + meta = with lib; { + homepage = "https://buildbot.net/"; + description = "Buildbot Worker Daemon"; + maintainers = with maintainers; [ ryansydnor lopsided98 ]; + license = licenses.gpl2; + }; +}) diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 01b088004bbf1..82929439254fb 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -3079,10 +3079,8 @@ with pkgs; bucklespring-libinput = callPackage ../applications/audio/bucklespring { }; bucklespring-x11 = callPackage ../applications/audio/bucklespring { legacy = true; }; - buildbot = with python3Packages; toPythonApplication buildbot; - buildbot-ui = with python3Packages; toPythonApplication buildbot-ui; - buildbot-full = with python3Packages; toPythonApplication buildbot-full; - buildbot-worker = with python3Packages; toPythonApplication buildbot-worker; + inherit (python3.pkgs.callPackage ../development/tools/continuous-integration/buildbot {}) + buildbot buildbot-ui buildbot-full buildbot-plugins buildbot-worker; bunyan-rs = callPackage ../development/tools/bunyan-rs { }; diff --git a/pkgs/top-level/python-aliases.nix b/pkgs/top-level/python-aliases.nix index 719fb88362ab1..e39a0259b2aee 100644 --- a/pkgs/top-level/python-aliases.nix +++ b/pkgs/top-level/python-aliases.nix @@ -48,6 +48,12 @@ mapAliases ({ blockdiagcontrib-cisco = throw "blockdiagcontrib-cisco is not compatible with blockdiag 2.0.0 and has been removed."; # added 2020-11-29 bsblan = python-bsblan; # added 2022-11-04 btchip = btchip-python; # added 2023-03-03 + buildbot = throw "use pkgs.buildbot instead"; # added 2022-04-07 + buildbot-ui = throw "use pkgs.buildbot-ui instead"; # added 2022-04-07 + buildbot-full = throw "use pkgs.buildbot-full instead"; # added 2022-04-07 + buildbot-plugins = throw "use pkgs.buildbot-plugins instead"; # added 2022-04-07 + buildbot-worker = throw "use pkgs.buildbot-worker instead"; # added 2022-04-07 + buildbot-pkg = throw "buildbot-pkg has been removed, it's only internally used in buildbot"; # added 2022-04-07 bt_proximity = bt-proximity; # added 2021-07-02 BTrees = btrees; # added 2023-02-19 carrot = throw "carrot has been removed, as its development was discontinued in 2012"; # added 2022-01-18 diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 627b11b5ad995..d7c214555f222 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -1513,18 +1513,6 @@ self: super: with self; { buienradar = callPackage ../development/python-modules/buienradar { }; - buildbot = callPackage ../development/python-modules/buildbot { }; - - buildbot-ui = self.buildbot.withPlugins (with self.buildbot-plugins; [ www ]); - - buildbot-full = self.buildbot.withPlugins (with self.buildbot-plugins; [ www console-view waterfall-view grid-view wsgi-dashboards badges ]); - - buildbot-pkg = callPackage ../development/python-modules/buildbot/pkg.nix { }; - - buildbot-plugins = pkgs.recurseIntoAttrs (callPackage ../development/python-modules/buildbot/plugins.nix { }); - - buildbot-worker = callPackage ../development/python-modules/buildbot/worker.nix { }; - build = callPackage ../development/python-modules/build { }; buildcatrust = callPackage ../development/python-modules/buildcatrust { }; -- cgit 1.4.1